diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/events/event-fetch-upload-url.json b/examples/aws_events/event-fetch-upload-url.json similarity index 100% rename from events/event-fetch-upload-url.json rename to examples/aws_events/event-fetch-upload-url.json diff --git a/events/event-get-chart-data.json b/examples/aws_events/event-get-chart-data.json similarity index 100% rename from events/event-get-chart-data.json rename to examples/aws_events/event-get-chart-data.json diff --git a/events/event-powerset-merge.json b/examples/aws_events/event-powerset-merge.json similarity index 100% rename from events/event-powerset-merge.json rename to examples/aws_events/event-powerset-merge.json diff --git a/src/handlers/dashboard/get_chart_data.py b/src/handlers/dashboard/get_chart_data.py index 485b138..b1ae477 100644 --- a/src/handlers/dashboard/get_chart_data.py +++ b/src/handlers/dashboard/get_chart_data.py @@ -7,10 +7,10 @@ import boto3 import pandas -from src.handlers.dashboard.filter_config import get_filter_string -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import get_latest_data_package_version, http_response +from ..dashboard.filter_config import get_filter_string +from ..shared.decorators import generic_error_handler +from ..shared.enums import BucketPath +from ..shared.functions import get_latest_data_package_version, http_response def _get_table_cols(table_name: str, version: str = None) -> list: diff --git a/src/handlers/dashboard/get_data_packages.py b/src/handlers/dashboard/get_data_packages.py index 423425b..c3c349b 100644 --- a/src/handlers/dashboard/get_data_packages.py +++ b/src/handlers/dashboard/get_data_packages.py @@ -3,9 +3,9 @@ import os -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath, JsonFilename -from src.handlers.shared.functions import get_s3_json_as_dict, http_response +from ..shared.decorators import generic_error_handler +from ..shared.enums import BucketPath, JsonFilename +from ..shared.functions import get_s3_json_as_dict, http_response @generic_error_handler(msg="Error retrieving data packages") diff --git a/src/handlers/dashboard/get_metadata.py b/src/handlers/dashboard/get_metadata.py index 1d5660e..fa6c2d0 100644 --- a/src/handlers/dashboard/get_metadata.py +++ b/src/handlers/dashboard/get_metadata.py @@ -4,8 +4,8 @@ import boto3 -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.functions import http_response, read_metadata +from ..shared.decorators import generic_error_handler +from ..shared.functions import http_response, read_metadata @generic_error_handler(msg="Error retrieving metadata") diff --git a/src/handlers/dashboard/get_study_periods.py b/src/handlers/dashboard/get_study_periods.py index 89b2100..c275891 100644 --- a/src/handlers/dashboard/get_study_periods.py +++ b/src/handlers/dashboard/get_study_periods.py @@ -4,9 +4,9 @@ import boto3 -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import JsonFilename -from src.handlers.shared.functions import http_response, read_metadata +from ..shared.decorators import generic_error_handler +from ..shared.enums import JsonFilename +from ..shared.functions import http_response, read_metadata @generic_error_handler(msg="Error retrieving study period") diff --git a/src/handlers/shared/awswrangler_functions.py b/src/handlers/shared/awswrangler_functions.py index 52ec1f2..e64ca54 100644 --- a/src/handlers/shared/awswrangler_functions.py +++ b/src/handlers/shared/awswrangler_functions.py @@ -1,7 +1,7 @@ """ functions specifically requiring AWSWranger, which requires a lambda layer""" import awswrangler -from src.handlers.shared.enums import BucketPath +from .enums import BucketPath def get_s3_data_package_list( diff --git a/src/handlers/shared/decorators.py b/src/handlers/shared/decorators.py index 22016e7..5a88b41 100644 --- a/src/handlers/shared/decorators.py +++ b/src/handlers/shared/decorators.py @@ -3,7 +3,7 @@ import functools import logging -from src.handlers.shared.functions import http_response +from .functions import http_response def generic_error_handler(msg="Internal server error"): diff --git a/src/handlers/shared/enums.py b/src/handlers/shared/enums.py index 81d8fa9..1084d51 100644 --- a/src/handlers/shared/enums.py +++ b/src/handlers/shared/enums.py @@ -24,3 +24,23 @@ class JsonFilename(Enum): TRANSACTIONS = "transactions" DATA_PACKAGES = "data_packages" STUDY_PERIODS = "study_periods" + + +class TransactionKeys(Enum): + """stores names of expected keys in the transaction dictionary""" + + TRANSACTION_FORMAT_VERSION = "transaction_format_version" + LAST_UPLOAD = "last_upload" + LAST_DATA_UPDATE = "last_data_update" + LAST_AGGREGATION = "last_aggregation" + LAST_ERROR = "last_error" + DELETED = "deleted" + + +class StudyPeriodMetadataKeys(Enum): + """stores names of expected keys in the study period metadata dictionary""" + + STUDY_PERIOD_FORMAT_VERSION = "study_period_format_version" + EARLIEST_DATE = "earliest_date" + LATEST_DATE = "latest_date" + LAST_DATA_UPDATE = "last_data_update" diff --git a/src/handlers/shared/functions.py b/src/handlers/shared/functions.py index a85dbfb..2309ef8 100644 --- a/src/handlers/shared/functions.py +++ b/src/handlers/shared/functions.py @@ -7,21 +7,22 @@ import boto3 -from src.handlers.shared.enums import BucketPath, JsonFilename +from .enums import BucketPath, JsonFilename, StudyPeriodMetadataKeys, TransactionKeys TRANSACTION_METADATA_TEMPLATE = { - "transaction_format_version": "2", - "last_upload": None, - "last_data_update": None, - "last_aggregation": None, - "last_error": None, - "deleted": None, + TransactionKeys.TRANSACTION_FORMAT_VERSION.value: "2", + TransactionKeys.LAST_UPLOAD.value: None, + TransactionKeys.LAST_DATA_UPDATE.value: None, + TransactionKeys.LAST_AGGREGATION.value: None, + TransactionKeys.LAST_ERROR.value: None, + TransactionKeys.DELETED.value: None, } + STUDY_PERIOD_METADATA_TEMPLATE = { - "study_period_format_version": "2", - "earliest_date": None, - "latest_date": None, - "last_data_update": None, + StudyPeriodMetadataKeys.STUDY_PERIOD_FORMAT_VERSION.value: "2", + StudyPeriodMetadataKeys.EARLIEST_DATE.value: None, + StudyPeriodMetadataKeys.LATEST_DATE.value: None, + StudyPeriodMetadataKeys.LAST_DATA_UPDATE.value: None, } diff --git a/src/handlers/site_upload/api_gateway_authorizer.py b/src/handlers/site_upload/api_gateway_authorizer.py index 809dccd..aee9a4c 100644 --- a/src/handlers/site_upload/api_gateway_authorizer.py +++ b/src/handlers/site_upload/api_gateway_authorizer.py @@ -8,8 +8,8 @@ import os import re -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import get_s3_json_as_dict +from ..shared.enums import BucketPath +from ..shared.functions import get_s3_json_as_dict class AuthError(Exception): diff --git a/src/handlers/site_upload/cache_api.py b/src/handlers/site_upload/cache_api.py index 362c4fe..f42a27f 100644 --- a/src/handlers/site_upload/cache_api.py +++ b/src/handlers/site_upload/cache_api.py @@ -6,9 +6,9 @@ import awswrangler import boto3 -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath, JsonFilename -from src.handlers.shared.functions import http_response +from ..shared.decorators import generic_error_handler +from ..shared.enums import BucketPath, JsonFilename +from ..shared.functions import http_response def cache_api_data(s3_client, s3_bucket_name: str, db: str, target: str) -> None: diff --git a/src/handlers/site_upload/fetch_upload_url.py b/src/handlers/site_upload/fetch_upload_url.py index a1d95d6..d532cde 100644 --- a/src/handlers/site_upload/fetch_upload_url.py +++ b/src/handlers/site_upload/fetch_upload_url.py @@ -6,9 +6,9 @@ import boto3 import botocore.exceptions -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import get_s3_json_as_dict, http_response +from ..shared.decorators import generic_error_handler +from ..shared.enums import BucketPath +from ..shared.functions import get_s3_json_as_dict, http_response def create_presigned_post( diff --git a/src/handlers/site_upload/powerset_merge.py b/src/handlers/site_upload/powerset_merge.py index 716183b..4cfc25c 100644 --- a/src/handlers/site_upload/powerset_merge.py +++ b/src/handlers/site_upload/powerset_merge.py @@ -11,10 +11,10 @@ from numpy import nan from pandas.core.indexes.range import RangeIndex -from src.handlers.shared.awswrangler_functions import get_s3_data_package_list -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import ( +from ..shared.awswrangler_functions import get_s3_data_package_list +from ..shared.decorators import generic_error_handler +from ..shared.enums import BucketPath, TransactionKeys +from ..shared.functions import ( get_s3_site_filename_suffix, http_response, move_s3_file, @@ -130,7 +130,7 @@ def merge_error_handler( s3_path.replace(f"s3://{self.s3_bucket_name}/", ""), f"{BucketPath.ERROR.value}/{subbucket_path}", ) - self.update_local_metadata("last_error") + self.update_local_metadata(TransactionKeys.LAST_ERROR.value) def get_static_string_series(static_str: str, index: RangeIndex) -> pandas.Series: @@ -215,7 +215,9 @@ def merge_powersets(manager: S3Manager) -> None: try: if not any(x.endswith(site_specific_name) for x in latest_file_list): df = expand_and_concat_sets(df, last_valid_path, last_valid_site) - manager.update_local_metadata("last_aggregation", site=last_valid_site) + manager.update_local_metadata( + TransactionKeys.LAST_AGGREGATION.value, site=last_valid_site + ) except MergeError as e: # This is expected to trigger if there's an issue in expand_and_concat_sets; # this usually means there's a data problem. @@ -257,8 +259,12 @@ def merge_powersets(manager: S3Manager) -> None: f"{BucketPath.LAST_VALID.value}/{subbucket_path}", ) latest_site = site_specific_name.split("/", maxsplit=1)[0] - manager.update_local_metadata("last_data_update", site=latest_site) - manager.update_local_metadata("last_aggregation", site=latest_site) + manager.update_local_metadata( + TransactionKeys.LAST_DATA_UPDATE.value, site=latest_site + ) + manager.update_local_metadata( + TransactionKeys.LAST_AGGREGATION.value, site=latest_site + ) except Exception as e: # pylint: disable=broad-except manager.merge_error_handler( latest_path, @@ -274,7 +280,7 @@ def merge_powersets(manager: S3Manager) -> None: f"/{subbucket_path}", manager.site, ) - manager.update_local_metadata("last_aggregation") + manager.update_local_metadata(TransactionKeys.LAST_AGGREGATION.value) manager.write_local_metadata() # In this section, we are trying to accomplish two things: diff --git a/src/handlers/site_upload/process_upload.py b/src/handlers/site_upload/process_upload.py index af1bc91..9671f94 100644 --- a/src/handlers/site_upload/process_upload.py +++ b/src/handlers/site_upload/process_upload.py @@ -3,9 +3,9 @@ import boto3 -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import ( +from ..shared.decorators import generic_error_handler +from ..shared.enums import BucketPath, TransactionKeys +from ..shared.functions import ( http_response, move_s3_file, read_metadata, @@ -51,7 +51,7 @@ def process_upload(s3_client, sns_client, s3_bucket_name: str, s3_key: str) -> N study, data_package, version, - "last_upload", + TransactionKeys.LAST_UPLOAD.value, last_uploaded_date, ) sns_client.publish(TopicArn=topic_sns_arn, Message=new_key, Subject=sns_subject) @@ -65,7 +65,7 @@ def process_upload(s3_client, sns_client, s3_bucket_name: str, s3_key: str) -> N study, data_package, version, - "last_upload", + TransactionKeys.LAST_UPLOAD.value, last_uploaded_date, ) metadata = update_metadata( @@ -74,7 +74,7 @@ def process_upload(s3_client, sns_client, s3_bucket_name: str, s3_key: str) -> N study, data_package, version, - "last_error", + TransactionKeys.LAST_ERROR.value, last_uploaded_date, ) write_metadata(s3_client, s3_bucket_name, metadata) diff --git a/src/handlers/site_upload/study_period.py b/src/handlers/site_upload/study_period.py index fac7bd9..93e5edc 100644 --- a/src/handlers/site_upload/study_period.py +++ b/src/handlers/site_upload/study_period.py @@ -6,10 +6,10 @@ import awswrangler import boto3 -from src.handlers.shared.awswrangler_functions import get_s3_study_meta_list -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import JsonFilename -from src.handlers.shared.functions import ( +from ..shared.awswrangler_functions import get_s3_study_meta_list +from ..shared.decorators import generic_error_handler +from ..shared.enums import JsonFilename, StudyPeriodMetadataKeys +from ..shared.functions import ( http_response, read_metadata, update_metadata, @@ -33,7 +33,7 @@ def update_study_period(s3_client, s3_bucket, site, study, data_package, version study, data_package, version, - "earliest_date", + StudyPeriodMetadataKeys.EARLIEST_DATE.value, df["min_date"][0], meta_type=JsonFilename.STUDY_PERIODS.value, ) @@ -43,7 +43,7 @@ def update_study_period(s3_client, s3_bucket, site, study, data_package, version study, data_package, version, - "latest_date", + StudyPeriodMetadataKeys.LATEST_DATE.value, df["max_date"][0], meta_type=JsonFilename.STUDY_PERIODS.value, ) @@ -53,7 +53,7 @@ def update_study_period(s3_client, s3_bucket, site, study, data_package, version study, data_package, version, - "last_data_update", + StudyPeriodMetadataKeys.LAST_DATA_UPDATE.value, datetime.now(timezone.utc), meta_type=JsonFilename.STUDY_PERIODS.value, ) diff --git a/template.hostedzone.yaml b/template.hostedzone.yaml index 5998558..1c5e0b3 100644 --- a/template.hostedzone.yaml +++ b/template.hostedzone.yaml @@ -34,33 +34,7 @@ Resources: ValidationMethod: DNS -# Dashboard - AcmDashCertificate: - Type: AWS::CertificateManager::Certificate - Properties: - DomainName: !Sub "dashboard.${Domain}" - DomainValidationOptions: - - DomainName: !Sub "dashboard.${Domain}" - HostedZoneId: !Ref CumulusHostedZone - ValidationMethod: DNS - - AcmStagingDashCertificate: - Type: AWS::CertificateManager::Certificate - Properties: - DomainName: !Sub "staging.dashboard.${Domain}" - DomainValidationOptions: - - DomainName: !Sub "staging.dashboard.${Domain}" - HostedZoneId: !Ref CumulusHostedZone - ValidationMethod: DNS - - AcmDevDashCertificate: - Type: AWS::CertificateManager::Certificate - Properties: - DomainName: !Sub "dev.dashboard.${Domain}" - DomainValidationOptions: - - DomainName: !Sub "dev.dashboard.${Domain}" - HostedZoneId: !Ref CumulusHostedZone - ValidationMethod: DNS +# Aggregator AcmAggCertificate: Type: AWS::CertificateManager::Certificate @@ -89,6 +63,9 @@ Resources: HostedZoneId: !Ref CumulusHostedZone ValidationMethod: DNS + +#Upload API + AcmApiCertificate: Type: AWS::CertificateManager::Certificate Properties: @@ -125,18 +102,6 @@ Outputs: Description: "ACM Www Certificate ARN" Value: !Ref AcmWwwCertificate - DashCertificateArn: - Description: "ACM Dashboard Certificate ARN" - Value: !Ref AcmDashCertificate - - DashStagingCertificateArn: - Description: "ACM Staging Dashboard Certificate ARN" - Value: !Ref AcmStagingDashCertificate - - DashDevCertificateArn: - Description: "ACM Dev Dashboard Certificate ARN" - Value: !Ref AcmDevDashCertificate - AggCertificateArn: Description: "ACM Aggregator Certificate ARN" Value: !Ref AcmAggCertificate