Tech Debt: Use modules, safety enums, docs updates

src/handlers/dashboard/get_chart_data.py

@@ -7,10 +7,10 @@
 import boto3
 import pandas
 
-from src.handlers.dashboard.filter_config import get_filter_string
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import get_latest_data_package_version, http_response
+from ..dashboard.filter_config import get_filter_string
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import BucketPath
+from ..shared.functions import get_latest_data_package_version, http_response
 
 
 def _get_table_cols(table_name: str, version: str = None) -> list:

src/handlers/dashboard/get_data_packages.py

@@ -3,9 +3,9 @@
 
 import os
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath, JsonFilename
-from src.handlers.shared.functions import get_s3_json_as_dict, http_response
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import BucketPath, JsonFilename
+from ..shared.functions import get_s3_json_as_dict, http_response
 
 
 @generic_error_handler(msg="Error retrieving data packages")

src/handlers/dashboard/get_metadata.py

@@ -4,8 +4,8 @@
 
 import boto3
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.functions import http_response, read_metadata
+from ..shared.decorators import generic_error_handler
+from ..shared.functions import http_response, read_metadata
 
 
 @generic_error_handler(msg="Error retrieving metadata")

src/handlers/dashboard/get_study_periods.py

@@ -4,9 +4,9 @@
 
 import boto3
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import JsonFilename
-from src.handlers.shared.functions import http_response, read_metadata
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import JsonFilename
+from ..shared.functions import http_response, read_metadata
 
 
 @generic_error_handler(msg="Error retrieving study period")

src/handlers/shared/awswrangler_functions.py

@@ -1,7 +1,7 @@
 """ functions specifically requiring AWSWranger, which requires a lambda layer"""
 import awswrangler
 
-from src.handlers.shared.enums import BucketPath
+from .enums import BucketPath
 
 
 def get_s3_data_package_list(

src/handlers/shared/decorators.py

@@ -3,7 +3,7 @@
 import functools
 import logging
 
-from src.handlers.shared.functions import http_response
+from .functions import http_response
 
 
 def generic_error_handler(msg="Internal server error"):

src/handlers/shared/enums.py

@@ -24,3 +24,23 @@ class JsonFilename(Enum):
     TRANSACTIONS = "transactions"
     DATA_PACKAGES = "data_packages"
     STUDY_PERIODS = "study_periods"
+
+
+class TransactionKeys(Enum):
+    """stores names of expected keys in the transaction dictionary"""
+
+    TRANSACTION_FORMAT_VERSION = "transaction_format_version"
+    LAST_UPLOAD = "last_upload"
+    LAST_DATA_UPDATE = "last_data_update"
+    LAST_AGGREGATION = "last_aggregation"
+    LAST_ERROR = "last_error"
+    DELETED = "deleted"
+
+
+class StudyPeriodMetadataKeys(Enum):
+    """stores names of expected keys in the study period metadata dictionary"""
+
+    STUDY_PERIOD_FORMAT_VERSION = "study_period_format_version"
+    EARLIEST_DATE = "earliest_date"
+    LATEST_DATE = "latest_date"
+    LAST_DATA_UPDATE = "last_data_update"

src/handlers/shared/functions.py

@@ -7,21 +7,22 @@
 
 import boto3
 
-from src.handlers.shared.enums import BucketPath, JsonFilename
+from .enums import BucketPath, JsonFilename, StudyPeriodMetadataKeys, TransactionKeys
 
 TRANSACTION_METADATA_TEMPLATE = {
-    "transaction_format_version": "2",
-    "last_upload": None,
-    "last_data_update": None,
-    "last_aggregation": None,
-    "last_error": None,
-    "deleted": None,
+    TransactionKeys.TRANSACTION_FORMAT_VERSION.value: "2",
+    TransactionKeys.LAST_UPLOAD.value: None,
+    TransactionKeys.LAST_DATA_UPDATE.value: None,
+    TransactionKeys.LAST_AGGREGATION.value: None,
+    TransactionKeys.LAST_ERROR.value: None,
+    TransactionKeys.DELETED.value: None,
 }
+
 STUDY_PERIOD_METADATA_TEMPLATE = {
-    "study_period_format_version": "2",
-    "earliest_date": None,
-    "latest_date": None,
-    "last_data_update": None,
+    StudyPeriodMetadataKeys.STUDY_PERIOD_FORMAT_VERSION.value: "2",
+    StudyPeriodMetadataKeys.EARLIEST_DATE.value: None,
+    StudyPeriodMetadataKeys.LATEST_DATE.value: None,
+    StudyPeriodMetadataKeys.LAST_DATA_UPDATE.value: None,
 }
 
 

src/handlers/site_upload/api_gateway_authorizer.py

@@ -8,8 +8,8 @@
 import os
 import re
 
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import get_s3_json_as_dict
+from ..shared.enums import BucketPath
+from ..shared.functions import get_s3_json_as_dict
 
 
 class AuthError(Exception):

src/handlers/site_upload/cache_api.py

@@ -6,9 +6,9 @@
 import awswrangler
 import boto3
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath, JsonFilename
-from src.handlers.shared.functions import http_response
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import BucketPath, JsonFilename
+from ..shared.functions import http_response
 
 
 def cache_api_data(s3_client, s3_bucket_name: str, db: str, target: str) -> None:

src/handlers/site_upload/fetch_upload_url.py

@@ -6,9 +6,9 @@
 import boto3
 import botocore.exceptions
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import get_s3_json_as_dict, http_response
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import BucketPath
+from ..shared.functions import get_s3_json_as_dict, http_response
 
 
 def create_presigned_post(

src/handlers/site_upload/powerset_merge.py

@@ -11,10 +11,10 @@
 from numpy import nan
 from pandas.core.indexes.range import RangeIndex
 
-from src.handlers.shared.awswrangler_functions import get_s3_data_package_list
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import (
+from ..shared.awswrangler_functions import get_s3_data_package_list
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import BucketPath, TransactionKeys
+from ..shared.functions import (
     get_s3_site_filename_suffix,
     http_response,
     move_s3_file,
@@ -130,7 +130,7 @@ def merge_error_handler(
             s3_path.replace(f"s3://{self.s3_bucket_name}/", ""),
             f"{BucketPath.ERROR.value}/{subbucket_path}",
         )
-        self.update_local_metadata("last_error")
+        self.update_local_metadata(TransactionKeys.LAST_ERROR.value)
 
 
 def get_static_string_series(static_str: str, index: RangeIndex) -> pandas.Series:
@@ -215,7 +215,9 @@ def merge_powersets(manager: S3Manager) -> None:
         try:
             if not any(x.endswith(site_specific_name) for x in latest_file_list):
                 df = expand_and_concat_sets(df, last_valid_path, last_valid_site)
-                manager.update_local_metadata("last_aggregation", site=last_valid_site)
+                manager.update_local_metadata(
+                    TransactionKeys.LAST_AGGREGATION.value, site=last_valid_site
+                )
         except MergeError as e:
             # This is expected to trigger if there's an issue in expand_and_concat_sets;
             # this usually means there's a data problem.
@@ -257,8 +259,12 @@ def merge_powersets(manager: S3Manager) -> None:
                 f"{BucketPath.LAST_VALID.value}/{subbucket_path}",
             )
             latest_site = site_specific_name.split("/", maxsplit=1)[0]
-            manager.update_local_metadata("last_data_update", site=latest_site)
-            manager.update_local_metadata("last_aggregation", site=latest_site)
+            manager.update_local_metadata(
+                TransactionKeys.LAST_DATA_UPDATE.value, site=latest_site
+            )
+            manager.update_local_metadata(
+                TransactionKeys.LAST_AGGREGATION.value, site=latest_site
+            )
         except Exception as e:  # pylint: disable=broad-except
             manager.merge_error_handler(
                 latest_path,
@@ -274,7 +280,7 @@ def merge_powersets(manager: S3Manager) -> None:
                     f"/{subbucket_path}",
                     manager.site,
                 )
-                manager.update_local_metadata("last_aggregation")
+                manager.update_local_metadata(TransactionKeys.LAST_AGGREGATION.value)
     manager.write_local_metadata()
 
     # In this section, we are trying to accomplish two things:

src/handlers/site_upload/process_upload.py

@@ -3,9 +3,9 @@
 
 import boto3
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import (
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import BucketPath, TransactionKeys
+from ..shared.functions import (
     http_response,
     move_s3_file,
     read_metadata,
@@ -51,7 +51,7 @@ def process_upload(s3_client, sns_client, s3_bucket_name: str, s3_key: str) -> N
             study,
             data_package,
             version,
-            "last_upload",
+            TransactionKeys.LAST_UPLOAD.value,
             last_uploaded_date,
         )
         sns_client.publish(TopicArn=topic_sns_arn, Message=new_key, Subject=sns_subject)
@@ -65,7 +65,7 @@ def process_upload(s3_client, sns_client, s3_bucket_name: str, s3_key: str) -> N
             study,
             data_package,
             version,
-            "last_upload",
+            TransactionKeys.LAST_UPLOAD.value,
             last_uploaded_date,
         )
         metadata = update_metadata(
@@ -74,7 +74,7 @@ def process_upload(s3_client, sns_client, s3_bucket_name: str, s3_key: str) -> N
             study,
             data_package,
             version,
-            "last_error",
+            TransactionKeys.LAST_ERROR.value,
             last_uploaded_date,
         )
         write_metadata(s3_client, s3_bucket_name, metadata)

src/handlers/site_upload/study_period.py

@@ -6,10 +6,10 @@
 import awswrangler
 import boto3
 
-from src.handlers.shared.awswrangler_functions import get_s3_study_meta_list
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import JsonFilename
-from src.handlers.shared.functions import (
+from ..shared.awswrangler_functions import get_s3_study_meta_list
+from ..shared.decorators import generic_error_handler
+from ..shared.enums import JsonFilename, StudyPeriodMetadataKeys
+from ..shared.functions import (
     http_response,
     read_metadata,
     update_metadata,
@@ -33,7 +33,7 @@ def update_study_period(s3_client, s3_bucket, site, study, data_package, version
         study,
         data_package,
         version,
-        "earliest_date",
+        StudyPeriodMetadataKeys.EARLIEST_DATE.value,
         df["min_date"][0],
         meta_type=JsonFilename.STUDY_PERIODS.value,
     )
@@ -43,7 +43,7 @@ def update_study_period(s3_client, s3_bucket, site, study, data_package, version
         study,
         data_package,
         version,
-        "latest_date",
+        StudyPeriodMetadataKeys.LATEST_DATE.value,
         df["max_date"][0],
         meta_type=JsonFilename.STUDY_PERIODS.value,
     )
@@ -53,7 +53,7 @@ def update_study_period(s3_client, s3_bucket, site, study, data_package, version
         study,
         data_package,
         version,
-        "last_data_update",
+        StudyPeriodMetadataKeys.LAST_DATA_UPDATE.value,
         datetime.now(timezone.utc),
         meta_type=JsonFilename.STUDY_PERIODS.value,
     )

template.hostedzone.yaml

@@ -34,33 +34,7 @@ Resources:
       ValidationMethod: DNS
 
 
-# Dashboard
-  AcmDashCertificate:
-    Type: AWS::CertificateManager::Certificate
-    Properties:
-      DomainName: !Sub "dashboard.${Domain}"
-      DomainValidationOptions:
-        - DomainName: !Sub "dashboard.${Domain}"
-          HostedZoneId: !Ref CumulusHostedZone
-      ValidationMethod: DNS
-
-  AcmStagingDashCertificate:
-    Type: AWS::CertificateManager::Certificate
-    Properties:
-      DomainName: !Sub "staging.dashboard.${Domain}"
-      DomainValidationOptions:
-        - DomainName: !Sub "staging.dashboard.${Domain}"
-          HostedZoneId: !Ref CumulusHostedZone
-      ValidationMethod: DNS
-
-  AcmDevDashCertificate:
-    Type: AWS::CertificateManager::Certificate
-    Properties:
-      DomainName: !Sub "dev.dashboard.${Domain}"
-      DomainValidationOptions:
-        - DomainName: !Sub "dev.dashboard.${Domain}"
-          HostedZoneId: !Ref CumulusHostedZone
-      ValidationMethod: DNS
+# Aggregator
 
   AcmAggCertificate:
     Type: AWS::CertificateManager::Certificate
@@ -89,6 +63,9 @@ Resources:
           HostedZoneId: !Ref CumulusHostedZone
       ValidationMethod: DNS
 
+
+#Upload API
+
   AcmApiCertificate:
     Type: AWS::CertificateManager::Certificate
     Properties:
@@ -125,18 +102,6 @@ Outputs:
     Description: "ACM Www Certificate ARN"
     Value: !Ref AcmWwwCertificate
 
-  DashCertificateArn:
-    Description: "ACM Dashboard Certificate ARN"
-    Value: !Ref AcmDashCertificate
-
-  DashStagingCertificateArn:
-    Description: "ACM Staging Dashboard Certificate ARN"
-    Value: !Ref AcmStagingDashCertificate
-
-  DashDevCertificateArn:
-    Description: "ACM Dev Dashboard Certificate ARN"
-    Value: !Ref AcmDevDashCertificate
-
   AggCertificateArn:
     Description: "ACM Aggregator Certificate ARN"
     Value: !Ref AcmAggCertificate