Make pip installable.

.circleci/config.yml

@@ -15,15 +15,14 @@ commands:
     description: "Test ndingest"
     steps:
       - run: ln -s /home/circleci/project /home/circleci/ndingest
-      - run: cp settings/settings.ini.test settings/settings.ini
+      - run: cp ndingest/settings/settings.ini.test ndingest/settings/settings.ini
       - run: python -m pytest -c test_apl.cfg
 jobs:
   test_py3_8:
     docker:
       - image: circleci/python:3.8
       - image: circleci/dynamodb:latest
     environment:
-      NDINGEST_TEST: 1
       AWS_ACCESS_KEY_ID: testing
       AWS_SECRET_ACCESS_KEY: testing
       AWS_SECURITY_TOKEN: testing

settings/boss_build_settings.py → ndingest/settings/boss_build_settings.py

@@ -1,4 +1,4 @@
-# Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
+# Copyright 2021 The Johns Hopkins University Applied Physics Laboratory
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,12 +13,14 @@
 # limitations under the License.
 
 """
-Generate the Boss settings.ini for ndingest for continuous integration.
+Generate the Boss settings.ini for ndingest for a Boss endpoint.
 """
 
 import configparser
-from spdb.c_lib.ndtype import CUBOIDSIZE
+import json
 import os
+from spdb.c_lib.ndtype import CUBOIDSIZE
+import urllib.request
 
 # Location of settings files for ndingest.
 NDINGEST_SETTINGS_FOLDER = os.path.dirname(os.path.realpath(__file__))
@@ -29,8 +31,23 @@
 BOSS_CONFIG = "/etc/boss/boss.config"
 
 
+def get_region():
+    """
+    Get the AWS region from the EC2 instance.
+
+    Returns:
+        (str): Region name.
+    """
+    url = 'http://169.254.169.254/latest/dynamic/instance-identity/document'
+    doc = urllib.request.urlopen(url).read().decode('utf-8')
+    doc = json.loads(doc)
+    return doc['region']
+
+
 def create_settings(tmpl_fp, boss_fp):
     """
+    Create the ndingest settings file.
+
     Args:
         tmpl_fp (file-like object): ndingest settings.ini template.
         boss_fp (file-like object): Boss config.
@@ -43,20 +60,24 @@ def create_settings(tmpl_fp, boss_fp):
 
     nd_config["boss"]["domain"] = boss_config["system"]["fqdn"].split(".", 1)[1]
 
-    nd_config["aws"]["region"] = "us-east-1"  # Hardcoded for testing
+    nd_config["aws"]["region"] = get_region()
     nd_config["aws"]["tile_bucket"] = boss_config["aws"]["tile_bucket"]
     nd_config["aws"]["cuboid_bucket"] = boss_config["aws"]["cuboid_bucket"]
     nd_config["aws"]["tile_index_table"] = boss_config["aws"]["tile-index-table"]
     nd_config["aws"]["cuboid_index_table"] = boss_config["aws"]["s3-index-table"]
-    nd_config["aws"]["max_task_id_suffix"] = "100"
+    nd_config['aws']['max_task_id_suffix'] = boss_config['aws']['max_task_id_suffix']
 
     nd_config["spdb"]["SUPER_CUBOID_SIZE"] = ", ".join(str(x) for x in CUBOIDSIZE[0])
 
     with open(NDINGEST_SETTINGS_FOLDER + "/settings.ini", "w") as out:
         nd_config.write(out)
 
 
-if __name__ == "__main__":
+def run():
     with open(NDINGEST_SETTINGS_TEMPLATE) as nd_tmpl:
         with open(BOSS_CONFIG) as boss_cfg:
             create_settings(nd_tmpl, boss_cfg)
+
+
+if __name__ == "__main__":
+    run()

setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='ndingest',
+    version='1.0.0',
+    url='https://github.com/jhuapl-boss/ndingest',
+    #package_dir={'': 'ndingest'},
+    packages=find_packages(),
+    package_data={
+        'ndingest.settings': ['settings.ini.apl', 'settings.ini.test'],
+        'ndingest.nddynamo.schemas': ['boss_tile_index.json'],
+        'ndingest.nddynamo': ['README.md'],
+    },
+    data_files=[('ndingest/nddynamo/schemas', ['ndingest/nddynamo/schemas/boss_tile_index.json'])],
+    python_requires='>=3.6, <4',
+    install_requires=['boto3'],
+    extras_require={
+        'test': ['moto', 'pytest'],
+    },
+)

test/README.md

@@ -11,10 +11,9 @@ pytest
 Only the following tests should be run when working with the Boss.
 `test_apl.cfg` configures pytest to run only the tests that matter to the Boss.
 
-```shell
-# Use randomized queue names.
-export NDINGEST_TEST=1
+Note that `test_bosstileindexdb.py` requires a local DynamoDB to test against.
 
+```shell
 # From the ndingest root folder:
 pytest -c test_apl.cfg
 ```

test/cleanuplambda.py

@@ -1 +1 @@
-../ndlambda/functions/cleanup/cleanuplambda.py
+../ndingest/ndlambda/functions/cleanup/cleanuplambda.py

test/conftest.py

@@ -25,6 +25,10 @@
 
 from ndingest.settings.settings import Settings
 
+# Enable test mode by setting the expected env variable that
+# settings/bosssettings.py looks for.
+os.environ['NDINGEST_TEST'] = '1'
+
 settings = Settings.load()
 from io import BytesIO
 from ndingest.ndingestproj.ingestproj import IngestProj

test/ingestlambda.py

@@ -1 +1 @@
-../ndlambda/functions/ingest/ingestlambda.py
+../ndingest/ndlambda/functions/ingest/ingestlambda.py

test/test_bosstileindexdb.py

@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+Note that these tests require a local DynamoDB to run against.
+"""
+
 from __future__ import print_function
 from __future__ import absolute_import
 import sys
@@ -44,7 +48,7 @@ def setUp(self):
         # Suppress ResourceWarning messages about unclosed connections.
         warnings.simplefilter("ignore")
 
-        with open("nddynamo/schemas/boss_tile_index.json") as fp:
+        with open("ndingest/nddynamo/schemas/boss_tile_index.json") as fp:
             schema = json.load(fp)
 
         BossTileIndexDB.createTable(schema, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)

test/uploadlambda.py

@@ -1 +1 @@
-../ndlambda/functions/upload/uploadlambda.py
+../ndingest/ndlambda/functions/upload/uploadlambda.py