diff --git a/CHANGELOG.md b/CHANGELOG.md index aed866e..0bec4e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added + - Issue 211 - Query track ingest table for granules with "to_ingest" status ### Changed ### Deprecated ### Removed diff --git a/docs/examples.md b/docs/examples.md index 5090424..cf87fa5 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -266,193 +266,6 @@ Will return GeoJSON: } ] } -} -``` - -** geometry simplified for example - -## Get time series GeoJSON for river node - -Search for a single river node by ID. - -[https://soto.podaac.earthdatacloud.nasa.gov/hydrocron/v1/timeseries?feature=Node&feature_id=12228200110861&start_time=2024-01-25T00:00:00Z&end_time=2024-03-30T00:00:00Z&output=geojson&fields=reach_id,node_id,time_str,wse](https://soto.podaac.earthdatacloud.nasa.gov/hydrocron/v1/timeseries?feature=Node&feature_id=12228200110861&start_time=2024-01-25T00:00:00Z&end_time=2024-03-30T00:00:00Z&output=geojson&fields=reach_id,node_id,time_str,wse) - -Will return GeoJSON: - -```json -{ -"status": "200 OK", -"time": 604.705, -"hits": 9, -"results": { - "csv": "", - "geojson": { - "type": "FeatureCollection", - "features": [ - { - "id": "0", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "2024-01-30T21:19:19Z", - "wse": "677.9232", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "1", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "2024-02-06T08:37:09Z", - "wse": "673.46918", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "2", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "no_data", - "wse": "-999999999999.0", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "3", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "2024-02-20T18:04:24Z", - "wse": "673.69799", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "4", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "2024-02-27T05:22:15Z", - "wse": "674.66235", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "5", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "no_data", - "wse": "-999999999999.0", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "6", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "2024-03-12T14:49:26Z", - "wse": "673.47788", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "7", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "2024-03-19T02:07:17Z", - "wse": "675.23219", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - }, - { - "id": "8", - "type": "Feature", - "properties": { - "reach_id": "12228200111", - "node_id": "12228200110861", - "time_str": "no_data", - "wse": "-999999999999.0", - "wse_units": "m" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 35.149314, - -10.256285 - ] - } - } - ] - } } } ``` diff --git a/hydrocron/api/data_access/db.py b/hydrocron/api/data_access/db.py index 1093869..138a7c5 100644 --- a/hydrocron/api/data_access/db.py +++ b/hydrocron/api/data_access/db.py @@ -76,6 +76,44 @@ def get_prior_lake_series_by_feature_id(self, feature_id, start_time, end_time): ) return items + def get_series_granule_ur(self, table_name, feature_name, granule_ur): + """ + + @param table_name: str - Hydrocron table to query + @param granule_ur: str - Granule UR + @return: dictionary of items + """ + + hydrocron_table = self._dynamo_instance.Table(table_name) + hydrocron_table.load() + + items = hydrocron_table.query( + ProjectionExpression=feature_name, + IndexName="GranuleURIndex", + KeyConditionExpression=( + Key("granuleUR").eq(granule_ur) + ) + ) + last_key_evaluated = "" + if "LastEvaluatedKey" in items.keys(): + last_key_evaluated = items["LastEvaluatedKey"] + + while last_key_evaluated: + next_items = hydrocron_table.query( + ExclusiveStartKey=last_key_evaluated, + ProjectionExpression=feature_name, + IndexName="GranuleURIndex", + KeyConditionExpression=( + Key("granuleUR").eq(granule_ur) + ) + ) + items["Items"].extend(next_items["Items"]) + last_key_evaluated = "" + if "LastEvaluatedKey" in next_items.keys(): + last_key_evaluated = next_items["LastEvaluatedKey"] + + return items["Items"] + def get_granule_ur(self, table_name, granule_ur): """ @@ -96,3 +134,32 @@ def get_granule_ur(self, table_name, granule_ur): ) ) return items + + def get_status(self, table_name, status): + """ + + @param table_name: str - Hydrocron table to query + @param status: str - Status to query for + """ + + hydrocron_table = self._dynamo_instance.Table(table_name) + items = hydrocron_table.query( + IndexName="statusIndex", + KeyConditionExpression=(Key("status").eq(status)) + ) + last_key_evaluated = "" + if "LastEvaluatedKey" in items.keys(): + last_key_evaluated = items["LastEvaluatedKey"] + + while last_key_evaluated: + next_items = hydrocron_table.query( + ExclusiveStartKey=last_key_evaluated, + IndexName="statusIndex", + KeyConditionExpression=(Key("status").eq(status)) + ) + items["Items"].extend(next_items["Items"]) + last_key_evaluated = "" + if "LastEvaluatedKey" in next_items.keys(): + last_key_evaluated = next_items["LastEvaluatedKey"] + + return items["Items"] diff --git a/hydrocron/db/track_ingest.py b/hydrocron/db/track_ingest.py index faeff87..b5b4c71 100644 --- a/hydrocron/db/track_ingest.py +++ b/hydrocron/db/track_ingest.py @@ -30,42 +30,77 @@ class Track: CMR_API = "https://cmr.earthdata.nasa.gov/search/granules.umm_json" PAGE_SIZE = 2000 + FEATURE_ID = { + "SWOT_L2_HR_RiverSP_reach_2.0": "reach_id", + "SWOT_L2_HR_RiverSP_node_2.0": "node_id", + "SWOT_L2_HR_LakeSP_prior_2.0": "lake_id" + } - def __init__(self, collection_shortname, collection_start_date, hydrocron_table): + def __init__(self, collection_shortname, collection_start_date=None, query_start=None, query_end=None): + """ + :param collection_shortname: Collection shortname to query CMR for + :type collection_shortname: string + :param collection_start_date: Date to begin revision_date query in CMR + :type collection_start_date: datetime + :param query_start: Start date to query for granules on + :type query_start: datetime + :param query_end: End date to query for granules on + :type query_end: datetime + """ self.collection_shortname = collection_shortname - self.cmr_granules = {} - self.hydrocron_granules = {} - self.hydrocron_table = hydrocron_table - self.revision_start = self._get_revision_start(collection_start_date) - self.revision_end = datetime.datetime.now(timezone.utc) # TODO - Decide on latency and subtract from current datetime - - def _get_revision_start(self, collection_start_date): + self.data_repository = DynamoDataRepository(connection.dynamodb_resource) + self.ingested = [] + self.to_ingest = [] + self.ssm_client = connection.ssm_client + if collection_start_date: + self.query_start = self._get_query_start(collection_start_date) + self.query_end = datetime.datetime.now(timezone.utc) # TODO - Decide on latency and subtract from current datetime + else: + self.query_start = query_start + self.query_end = query_end + + def _get_query_start(self, collection_start_date): """Locate the last most recent date that was queried in order to only - query on granules that have not seen before.""" + query on granules that have not seen before. + + :param collection_start_date: Date to begin revision_date query in CMR + :type collection_start_date: datetime + """ - # TODO - Query track-ingest table to determine max revision date - revision_start = None + last_run = self.ssm_client.get_parameter(Name=f"/service/hydrocron/track-ingest-runtime/{self.collection_shortname}")["Parameter"]["Value"] + if last_run != "no_data": + query_start = datetime.datetime.strptime(last_run, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) + else: + query_start = None - if not revision_start: - revision_start = collection_start_date + if not query_start: + query_start = collection_start_date - return revision_start + return query_start - def query_cmr(self): + def query_cmr(self, temporal): """Query CMR to locate all granules for a specific time range. Note: The use of "revision_date" should capture any granules that have been reprocessed in the time range. + + :param temporal: Indicates if temporal search should be conducted + :type temporal: boolean """ query = GranuleQuery() - logging.info("Querying revision_date range: %s to %s.", self.revision_start, self.revision_end) - granules = query.short_name(self.collection_shortname).revision_date(self.revision_start, self.revision_end).format("umm_json").get(query.hits()) + if temporal: + logging.info("Querying CMR temporal range: %s to %s.", self.query_start, self.query_end) + granules = query.short_name(self.collection_shortname).temporal(self.query_start, self.query_end).format("umm_json").get(query.hits()) + else: + logging.info("Querying CMR revision_date range: %s to %s.", self.query_start, self.query_end) + granules = query.short_name(self.collection_shortname).revision_date(self.query_start, self.query_end).format("umm_json").get(query.hits()) + cmr_granules = {} for granule in granules: granule_json = json.loads(granule) - self.cmr_granules.update(self._get_granule_ur_list(granule_json)) - - logging.info("Located %s granules in CMR.", len(self.cmr_granules.keys())) + cmr_granules.update(self._get_granule_ur_list(granule_json)) + logging.info("Located %s granules in CMR.", len(cmr_granules.keys())) + return cmr_granules @staticmethod def _get_granule_ur_list(granules): @@ -79,10 +114,10 @@ def _get_granule_ur_list(granules): granule_dict = {} for item in granules["items"]: - granule_ur = item["umm"]["GranuleUR"].replace("_swot", "") + granule_ur = item["umm"]["GranuleUR"].replace("_swot", ".zip") checksum = 0 for file in item["umm"]["DataGranule"]["ArchiveAndDistributionInformation"]: - if f"{granule_ur}.zip" == file["Name"]: + if granule_ur == file["Name"]: checksum = file["Checksum"]["Value"] granule_dict[granule_ur] = { "revision_date": item["meta"]["revision-date"], @@ -90,32 +125,90 @@ def _get_granule_ur_list(granules): } return granule_dict - def query_hydrocron(self): - """Query Hydrocron for time range and gather GranuleURs.""" + def query_hydrocron(self, hydrocron_table, cmr_granules): + """Query Hydrocron for time range and gather GranuleURs that do NOT exist in the Hydrocron table. - data_repository = DynamoDataRepository(connection.dynamodb_resource) - for granule_ur, data in self.cmr_granules.items(): - items = data_repository.get_granule_ur(self.hydrocron_table, f"{granule_ur}.zip") - if len(items["Items"]) > 0: - self.hydrocron_granules[granule_ur] = data + :param hydrocron_table: Name of hydrocron table to query + :type hydrocron_table: str + :param cmr_granules: List of CMR granules to query for + :type cmr_granules: list + """ - logging.info("Located %s granules in Hydrocron.", len(self.hydrocron_granules.keys())) + for granule_ur, data in cmr_granules.items(): + items = self.data_repository.get_granule_ur(hydrocron_table, granule_ur) + if len(items["Items"]) == 0: + self.to_ingest.append({ + "granuleUR": granule_ur, + "revision_date": data["revision_date"], + "checksum": data["checksum"], + "expected_feature_count": -1, + "actual_feature_count": 0, + "status": "to_ingest" + }) + logging.info("Located %s granules NOT in Hydrocron.", len(self.to_ingest)) + + def query_track_ingest(self, hydrocron_track_table, hydrocron_table): + """Query track status table for granules with "to_ingest" status. + + :param hydrocron_track_table: Name of hydrocron track table to query + :type hydrocron_track_table: str + :param hydrocron_table: Name of hydrocron table to query + :type hydrocron_table: str + """ - def query_track_ingest(self): - """Query track status table for granules with "to_ingest" status.""" + items = self.data_repository.get_status(hydrocron_track_table, "to_ingest") + logging.info("Located %s granules with 'to_ingest' status.", len(items)) + + for item in items: + granule_ur = item["granuleUR"] + features = self.data_repository.get_series_granule_ur( + hydrocron_table, + self.FEATURE_ID[self.collection_shortname], + granule_ur + ) + number_features = len(features) + ingest_item = { + "granuleUR": granule_ur, + "revision_date": item["revision_date"], + "checksum": item["checksum"], + "expected_feature_count": int(item["expected_feature_count"]), + "actual_feature_count": number_features + } + if number_features == item["expected_feature_count"]: + self.ingested.append(ingest_item) + else: + ingest_item["status"] = "to_ingest" + self.to_ingest.append(ingest_item) - def count_features(self): - """Count granule features to determine if all features have been ingested.""" + logging.info("Located %s granules that require ingestion.", len(self.to_ingest)) + logging.info("Located %s granules that are already ingested.", len(self.ingested)) def publish_cnm_ingest(self): """Publish CNM message to trigger granule ingestion.""" - def update_track_ingest(self): - """Update track status table with new granules and statuses.""" + def update_track_ingest(self, hydrocron_track_table): + """Update track status table with new granules and statuses. + + :param hydrocron_track_table: Name of hydrocron track table to query + :type hydrocron_track_table: str + """ + + def update_runtime(self): + """Update SSM parameter runtime for next execution.""" + + self.ssm_client.put_parameter(Name=f"/service/hydrocron/track-ingest-runtime/{self.collection_shortname}", + Value=self.query_end.strftime("%Y-%m-%dT%H:%M:%S"), + Overwrite=True) def track_ingest_handler(event, context): - """Lambda handler to track status of ingested granules to Hydrocron.""" + """Lambda handler to track status of ingested granules to Hydrocron. + + :param event: Lambda handler Event object + :type event: dict + :param context: Lambda handler Context object + :type context: dict + """ start = datetime.datetime.now() @@ -123,16 +216,37 @@ def track_ingest_handler(event, context): logging.info("Event: %s", event) collection_shortname = event["collection_shortname"] - collection_start_date = datetime.datetime.strptime(event["collection_start_date"], "%Y%m%d").replace(tzinfo=timezone.utc) hydrocron_table = event["hydrocron_table"] - - track = Track(collection_shortname, collection_start_date, hydrocron_table) - track.query_cmr() - track.query_hydrocron() - track.query_track_ingest() - track.count_features() + hydrocron_track_table = event["hydrocron_track_table"] + temporal = "temporal" in event.keys() + if temporal: + query_start = datetime.datetime.strptime(event["query_start"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) + query_end = datetime.datetime.strptime(event["query_end"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) + track = Track(collection_shortname, query_start=query_start, query_end=query_end) + else: + collection_start_date = datetime.datetime.strptime(event["collection_start_date"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) + track = Track(collection_shortname, collection_start_date=collection_start_date) + + logging.info("Collection shortname: %s", collection_shortname) + logging.info("Hydrocron table: %s", hydrocron_table) + logging.info("Hydrocron track ingest table: %s", hydrocron_track_table) + logging.info("Temporal indicator for revision dates: %s", temporal) + if temporal: + logging.info("Temporal start date: %s", query_start) + logging.info("Temporal end date: %s", query_end) + else: + logging.info("Collection start date: %s", collection_start_date) + + cmr_granules = track.query_cmr(temporal) + track.query_hydrocron(hydrocron_table, cmr_granules) + track.query_track_ingest(hydrocron_track_table, hydrocron_table) track.publish_cnm_ingest() - track.update_track_ingest() + track.update_track_ingest(hydrocron_track_table) + if not temporal: + track.update_runtime() + + logging.info("To Ingest: %s", track.to_ingest) + logging.info("Ingested: %s", track.ingested) end = datetime.datetime.now() logging.info("Elapsed: %s", (end - start)) diff --git a/terraform/hydrocron-apigw.tf b/terraform/hydrocron-apigw.tf index a5826f8..9f3a8ca 100644 --- a/terraform/hydrocron-apigw.tf +++ b/terraform/hydrocron-apigw.tf @@ -94,13 +94,6 @@ output "url" { } -resource "aws_ssm_parameter" "hydrocron-api-url" { - name = "/service/${var.app_name}/api-url" - type = "String" - value = aws_api_gateway_deployment.hydrocron-api-gateway-deployment.invoke_url -} - - # API Keys resource "aws_api_gateway_api_key" "default-user-key" { name = "${local.aws_resource_prefix}-api-key-default" @@ -112,26 +105,6 @@ resource "aws_api_gateway_api_key" "confluence-user-key" { } -resource "aws_ssm_parameter" "default-user-parameter" { - name = "/service/${var.app_name}/api-key-default" - description = "Hydrocron default user API key" - type = "SecureString" - value = aws_api_gateway_api_key.default-user-key.value -} - - -resource "aws_ssm_parameter" "trusted-user-parameter" { - name = "/service/${var.app_name}/api-key-trusted" - description = "Hydrocron trusted user API key" - type = "SecureString" - value = jsonencode( - [ - "${aws_api_gateway_api_key.confluence-user-key.value}" - ] - ) -} - - # Usage Plans resource "aws_api_gateway_usage_plan" "default-user-usage-plan" { name = "${local.aws_resource_prefix}-usage-plan-default" diff --git a/terraform/hydrocron-iam.tf b/terraform/hydrocron-iam.tf index bbee64e..ae3ba3a 100644 --- a/terraform/hydrocron-iam.tf +++ b/terraform/hydrocron-iam.tf @@ -68,8 +68,11 @@ data "aws_iam_policy_document" "dynamo-read-policy-track-ingest" { resources = [ aws_dynamodb_table.hydrocron-reach-track-ingest-table.arn, + "${aws_dynamodb_table.hydrocron-reach-track-ingest-table.arn}/index/*", aws_dynamodb_table.hydrocron-node-track-ingest-table.arn, + "${aws_dynamodb_table.hydrocron-node-track-ingest-table.arn}/index/*", aws_dynamodb_table.hydrocron-priorlake-track-ingest-table.arn, + "${aws_dynamodb_table.hydrocron-priorlake-track-ingest-table.arn}/index/*", ] } @@ -182,6 +185,19 @@ data "aws_iam_policy_document" "ssm-read-policy" { } +data "aws_iam_policy_document" "ssm-put-policy-track-ingest" { + + statement { + effect = "Allow" + actions = [ + "ssm:PutParameter" + ] + resources = ["arn:aws:ssm:${data.aws_region.current.id}:${local.account_id}:parameter/service/${var.app_name}/track-ingest-runtime/*"] + } + +} + + data "aws_iam_policy_document" "s3-read-policy" { statement { effect = "Allow" @@ -468,4 +484,8 @@ resource "aws_iam_role" "hydrocron_lambda_track_ingest_role" { name = "HydrocronSSMRead" policy = data.aws_iam_policy_document.ssm-read-policy.json } + inline_policy { + name = "HydrocronSSMPutTrack" + policy = data.aws_iam_policy_document.ssm-put-policy-track-ingest.json + } } \ No newline at end of file diff --git a/terraform/hydrocron-lambda.tf b/terraform/hydrocron-lambda.tf index 9a61509..6916502 100644 --- a/terraform/hydrocron-lambda.tf +++ b/terraform/hydrocron-lambda.tf @@ -220,6 +220,7 @@ resource "aws_lambda_function" "hydrocron_lambda_track_ingest" { environment { variables = { GRANULE_LAMBDA_FUNCTION_NAME = aws_lambda_function.hydrocron_lambda_load_granule.function_name + HYDROCRON_ENV = local.environment } } } diff --git a/terraform/hydrocron-ssm.tf b/terraform/hydrocron-ssm.tf new file mode 100644 index 0000000..b65bbcd --- /dev/null +++ b/terraform/hydrocron-ssm.tf @@ -0,0 +1,49 @@ +resource "aws_ssm_parameter" "hydrocron-api-url" { + name = "/service/${var.app_name}/api-url" + type = "String" + value = aws_api_gateway_deployment.hydrocron-api-gateway-deployment.invoke_url +} + + +resource "aws_ssm_parameter" "default-user-parameter" { + name = "/service/${var.app_name}/api-key-default" + description = "Hydrocron default user API key" + type = "SecureString" + value = aws_api_gateway_api_key.default-user-key.value +} + + +resource "aws_ssm_parameter" "trusted-user-parameter" { + name = "/service/${var.app_name}/api-key-trusted" + description = "Hydrocron trusted user API key" + type = "SecureString" + value = jsonencode( + [ + "${aws_api_gateway_api_key.confluence-user-key.value}" + ] + ) +} + + +resource "aws_ssm_parameter" "hydrocron-reach-track-ingest-runtime" { + name = "/service/${var.app_name}/track-ingest-runtime/SWOT_L2_HR_RiverSP_reach_2.0" + description = "Hydrocron track ingest last time executed on reaches" + type = "String" + value = "no_data" +} + + +resource "aws_ssm_parameter" "hydrocron-node-track-ingest-runtime" { + name = "/service/${var.app_name}/track-ingest-runtime/SWOT_L2_HR_RiverSP_node_2.0" + description = "Hydrocron track ingest last time executed on nodes" + type = "String" + value = "no_data" +} + + +resource "aws_ssm_parameter" "hydrocron-priorlake-track-ingest-runtime" { + name = "/service/${var.app_name}/track-ingest-runtime/SWOT_L2_HR_LakeSP_prior_2.0" + description = "Hydrocron track ingest last time executed on lakes" + type = "String" + value = "no_data" +} diff --git a/tests/conftest.py b/tests/conftest.py index 26073af..6974e82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,11 @@ """ conftest file to set up local dynamodb connection """ +import datetime import os.path import boto3 +import moto import pytest from pytest_dynamodb import factories @@ -19,6 +21,12 @@ 'SWOT_L2_HR_RiverSP_Reach_548_011_NA_20230610T193337_20230610T193344_PIA1_01.zip' # noqa ) +TEST_SHAPEFILE_PATH_REACH_TRACK = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'data', + "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip" +) + TEST_SHAPEFILE_PATH_LAKE = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'data', @@ -180,7 +188,39 @@ def hydrocron_dynamo_table(dynamo_db_resource): @pytest.fixture() -def hydrocron_api(hydrocron_dynamo_instance, dynamo_test_proc): +def mock_s3(): + + mock_aws = moto.mock_aws() + mock_aws.start() + + s3 = boto3.resource("s3") + s3.Bucket("podaac-swot-ops-cumulus-protected").create(CreateBucketConfiguration={"LocationConstraint": "us-west-2"}) + key = f"SWOT_L2_HR_RiverSP_2.0/{os.path.basename(TEST_SHAPEFILE_PATH_REACH_TRACK)}" + s3.Bucket("podaac-swot-ops-cumulus-protected").upload_file(Filename=TEST_SHAPEFILE_PATH_REACH_TRACK, Key=key) + + yield s3 + + mock_aws.stop() + + +@pytest.fixture() +def mock_ssm(): + + mock_aws = moto.mock_aws() + mock_aws.start() + + os.environ["AWS_DEFAULT_REGION"] = "us-west-2" + + ssm = boto3.client("ssm") + runtime = (datetime.datetime.now() - datetime.timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%S") + ssm.put_parameter(Name="/service/hydrocron/track-ingest-runtime/SWOT_L2_HR_RiverSP_reach_2.0", Value=runtime, Type="String") + + yield mock_aws + + mock_aws.stop() + +@pytest.fixture() +def hydrocron_api(hydrocron_dynamo_instance, dynamo_test_proc, mock_ssm): os.environ['HYDROCRON_ENV'] = 'test' os.environ['HYDROCRON_dynamodb_endpoint_url'] = f"http://{dynamo_test_proc.host}:{dynamo_test_proc.port}" import hydrocron.utils.connection # noqa: E501 # pylint: disable=import-outside-toplevel @@ -189,10 +229,109 @@ def hydrocron_api(hydrocron_dynamo_instance, dynamo_test_proc): @pytest.fixture() def s3_connection(): - import hydrocron.utils.connection # noqa: E501 # pylint: disable=import-outside-toplevel hydrocron.utils.connection.retrieve_credentials = lambda: { "accessKeyId": "testkey", "secretAccessKey": "testsecret", "sessionToken": "testtoken" } + + +@pytest.fixture() +def track_ingest_dynamo_instance(request, dynamo_test_proc): + dynamo_db = boto3.resource( + "dynamodb", + endpoint_url=f"http://{dynamo_test_proc.host}:{dynamo_test_proc.port}", + aws_access_key_id='fakeMyKeyId', + aws_secret_access_key='fakeSecretAccessKey', + region_name='us-west-2', + ) + + # reach table + create_tables( + dynamo_db, + constants.SWOT_REACH_TABLE_NAME, + 'reach_id', + ['reach_id', 'collection_shortname', 'collection_version', 'crid', 'cycle_id', 'pass_id', 'continent_id', 'ingest_time'] + ) + reach_hydro_table = HydrocronTable(dynamo_db, constants.SWOT_REACH_TABLE_NAME) + reach_items = swot_shp.read_shapefile( + TEST_SHAPEFILE_PATH_REACH_TRACK, + obscure_data=False, + columns=constants.REACH_DATA_COLUMNS) + for item_attrs in reach_items: + reach_hydro_table.add_data(**item_attrs) + + # track table + dynamo_db.create_table( + TableName=constants.SWOT_REACH_TRACK_INGEST_TABLE_NAME, + AttributeDefinitions=[ + {'AttributeName': 'granuleUR', 'AttributeType': 'S'}, + {'AttributeName': 'revision_date', 'AttributeType': 'S'}, + {'AttributeName': 'status', 'AttributeType': 'S'} + ], + KeySchema=[ + { + 'AttributeName': 'granuleUR', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'revision_date', + 'KeyType': 'RANGE' + } + ], + BillingMode='PROVISIONED', + ProvisionedThroughput={ + 'ReadCapacityUnits': 10, + 'WriteCapacityUnits': 10 + }, + GlobalSecondaryIndexes=[ + { + "IndexName": "statusIndex", + "KeySchema": [ + { + "AttributeName": "status", + "KeyType": "HASH" + } + ], + "Projection": { + "ProjectionType": "ALL", + }, + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5 + } + } + ] + ) + track_reach_table = HydrocronTable(dynamo_db, constants.SWOT_REACH_TRACK_INGEST_TABLE_NAME) + track_items = swot_shp.read_shapefile( + TEST_SHAPEFILE_PATH_REACH_TRACK, + obscure_data=False, + columns=constants.REACH_DATA_COLUMNS) + track_ingest_record = [{ + "granuleUR": os.path.basename(TEST_SHAPEFILE_PATH_REACH_TRACK), + "revision_date": "2024-05-22T19:15:44.572Z", + "expected_feature_count": len(track_items), + "actual_feature_count": 0, + "checksum": "0823db619be0044e809a5f992e067d03", + "status": "to_ingest" + }] + track_reach_table.batch_fill_table(track_ingest_record) + + try: + request.cls.dynamo_db = dynamo_db + except AttributeError: + pass + + yield dynamo_db + for table in dynamo_db.tables.all(): # pylint:disable=no-member + table.delete() + +@pytest.fixture() +def track_ingest_fixture(track_ingest_dynamo_instance, dynamo_test_proc, mock_ssm, mock_s3): + os.environ['HYDROCRON_ENV'] = 'test' + os.environ['HYDROCRON_dynamodb_endpoint_url'] = f"http://{dynamo_test_proc.host}:{dynamo_test_proc.port}" + import hydrocron.utils.connection # noqa: E501 # pylint: disable=import-outside-toplevel + hydrocron.utils.connection._dynamodb_resource = track_ingest_dynamo_instance + hydrocron.utils.connection._s3_resource = mock_s3 \ No newline at end of file diff --git a/tests/data/SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip b/tests/data/SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip new file mode 100644 index 0000000..791d919 Binary files /dev/null and b/tests/data/SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip differ diff --git a/tests/test_data/query_cmr_granule_results.json b/tests/test_data/query_cmr_granule_results.json index bdc137f..6900c76 100644 --- a/tests/test_data/query_cmr_granule_results.json +++ b/tests/test_data/query_cmr_granule_results.json @@ -1,441 +1,441 @@ { - "SWOT_L2_HR_RiverSP_Reach_017_117_SA_20240623T051259_20240623T051310_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_117_SA_20240623T051259_20240623T051310_PIC0_01.zip": { "revision_date": "2024-06-30T01:33:13.037Z", "checksum": "edda7230d20f1a85bae82f9917c86aa1" }, - "SWOT_L2_HR_RiverSP_Reach_017_117_GR_20240623T054620_20240623T054631_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_117_GR_20240623T054620_20240623T054631_PIC0_01.zip": { "revision_date": "2024-06-30T01:33:17.466Z", "checksum": "3c0967bda14e6cd82fa28742cd44a21e" }, - "SWOT_L2_HR_RiverSP_Reach_017_124_GR_20240623T110500_20240623T110505_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_124_GR_20240623T110500_20240623T110505_PIC0_01.zip": { "revision_date": "2024-06-30T01:34:00.170Z", "checksum": "94eeafb0da0583437b52f3358d6b2390" }, - "SWOT_L2_HR_RiverSP_Reach_017_124_EU_20240623T110959_20240623T111005_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_124_EU_20240623T110959_20240623T111005_PIC0_01.zip": { "revision_date": "2024-06-30T01:34:01.340Z", "checksum": "1f91ad708e027c59d61bade498f8d60e" }, - "SWOT_L2_HR_RiverSP_Reach_017_124_AF_20240623T112429_20240623T112435_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_124_AF_20240623T112429_20240623T112435_PIC0_01.zip": { "revision_date": "2024-06-30T01:34:05.374Z", "checksum": "8933a6c98f7c244dc8ac528b81d77e06" }, - "SWOT_L2_HR_RiverSP_Reach_017_132_AR_20240623T175719_20240623T175729_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_132_AR_20240623T175719_20240623T175729_PIC0_01.zip": { "revision_date": "2024-06-30T01:34:40.642Z", "checksum": "9878ca07afac703ec3ec7f2612583172" }, - "SWOT_L2_HR_RiverSP_Reach_017_132_NA_20240623T180323_20240623T180329_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_132_NA_20240623T180323_20240623T180329_PIC0_01.zip": { "revision_date": "2024-06-30T01:34:43.508Z", "checksum": "dc4f428acdfeb2916fbc11d19ae3e81f" }, - "SWOT_L2_HR_RiverSP_Reach_017_132_SA_20240623T181748_20240623T181750_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_132_SA_20240623T181748_20240623T181750_PIC0_01.zip": { "revision_date": "2024-06-30T01:34:46.518Z", "checksum": "8b2330a6d84e3cf2d7f24b51c609a0ad" }, - "SWOT_L2_HR_RiverSP_Reach_017_137_AF_20240623T223217_20240623T223228_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_137_AF_20240623T223217_20240623T223228_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:07.001Z", "checksum": "c06d7348ec65dd81cc57791eabfad7ba" }, - "SWOT_L2_HR_RiverSP_Reach_017_137_EU_20240623T224638_20240623T224647_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_137_EU_20240623T224638_20240623T224647_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:09.635Z", "checksum": "013e14741ef815e6000af79515248909" }, - "SWOT_L2_HR_RiverSP_Reach_017_137_SI_20240623T225526_20240623T225537_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_137_SI_20240623T225526_20240623T225537_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:13.105Z", "checksum": "46a58709b4136192bde2e404ff5c0755" }, - "SWOT_L2_HR_RiverSP_Reach_017_139_AF_20240624T001830_20240624T001831_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_139_AF_20240624T001830_20240624T001831_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:18.808Z", "checksum": "6472608fe5510b248c21fb3c9fbd3492" }, - "SWOT_L2_HR_RiverSP_Reach_017_139_EU_20240624T003149_20240624T003151_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_139_EU_20240624T003149_20240624T003151_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:21.812Z", "checksum": "394dcb72ad5d1413ada4bf518eea8cae" }, - "SWOT_L2_HR_RiverSP_Reach_017_145_SA_20240624T051420_20240624T051431_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_145_SA_20240624T051420_20240624T051431_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:48.514Z", "checksum": "d20a744c82f961dfe834f8be11c6b770" }, - "SWOT_L2_HR_RiverSP_Reach_017_145_NA_20240624T054257_20240624T054302_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_145_NA_20240624T054257_20240624T054302_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:51.468Z", "checksum": "298021087492c2014910fc9829340465" }, - "SWOT_L2_HR_RiverSP_Reach_017_145_GR_20240624T054704_20240624T054712_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_145_GR_20240624T054704_20240624T054712_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:54.191Z", "checksum": "477b93aa3accb94ef68bdbf9329e16a2" }, - "SWOT_L2_HR_RiverSP_Reach_017_146_EU_20240624T055730_20240624T055735_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_146_EU_20240624T055730_20240624T055735_PIC0_01.zip": { "revision_date": "2024-06-30T01:35:57.042Z", "checksum": "e8ffa4ef462168a5ff2816a93d38284f" }, - "SWOT_L2_HR_RiverSP_Reach_017_146_SI_20240624T055852_20240624T055855_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_146_SI_20240624T055852_20240624T055855_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:01.305Z", "checksum": "30371a8eb0f89645c4e5e4f82e0e8724" }, - "SWOT_L2_HR_RiverSP_Reach_017_146_AS_20240624T060644_20240624T060655_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_146_AS_20240624T060644_20240624T060655_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:03.916Z", "checksum": "9f8cd5ac3ae38e4d653e83d8704d76ce" }, - "SWOT_L2_HR_RiverSP_Reach_017_146_AU_20240624T062004_20240624T062015_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_146_AU_20240624T062004_20240624T062015_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:06.386Z", "checksum": "e439a5b8095a92f9595827473ec6ffa2" }, - "SWOT_L2_HR_RiverSP_Reach_017_148_EU_20240624T074303_20240624T074309_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_148_EU_20240624T074303_20240624T074309_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:07.868Z", "checksum": "bdf7457f69f1ecc9ffbc9365204cd144" }, - "SWOT_L2_HR_RiverSP_Reach_017_148_SI_20240624T074508_20240624T074519_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_148_SI_20240624T074508_20240624T074519_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:10.534Z", "checksum": "e967ccf3730f6d03b2c539447f15f5df" }, - "SWOT_L2_HR_RiverSP_Reach_017_148_AS_20240624T074808_20240624T074819_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_148_AS_20240624T074808_20240624T074819_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:13.183Z", "checksum": "8577783c8b87cd80f8d2f445854dc893" }, - "SWOT_L2_HR_RiverSP_Reach_017_149_NA_20240624T090207_20240624T090209_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_149_NA_20240624T090207_20240624T090209_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:17.102Z", "checksum": "09ab02065330c3cd2575f2dae21ab738" }, - "SWOT_L2_HR_RiverSP_Reach_017_149_AR_20240624T091218_20240624T091229_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_149_AR_20240624T091218_20240624T091229_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:19.542Z", "checksum": "344195c24886a093e7111c0dee63eb55" }, - "SWOT_L2_HR_RiverSP_Reach_017_149_GR_20240624T091842_20240624T091850_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_149_GR_20240624T091842_20240624T091850_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:20.896Z", "checksum": "60e6aece04610ec4d423c94e2bad755f" }, - "SWOT_L2_HR_RiverSP_Reach_017_150_GR_20240624T092044_20240624T092052_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_150_GR_20240624T092044_20240624T092052_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:23.601Z", "checksum": "5d1b9a13de4354d80f8cd2cc438ab9e2" }, - "SWOT_L2_HR_RiverSP_Reach_017_150_EU_20240624T092516_20240624T092522_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_150_EU_20240624T092516_20240624T092522_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:26.362Z", "checksum": "e59eef9799523c701bc9b4c100cd1fd9" }, - "SWOT_L2_HR_RiverSP_Reach_017_152_GR_20240624T110552_20240624T110556_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_152_GR_20240624T110552_20240624T110556_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:37.023Z", "checksum": "3f2175d955eb0255753d4aeeb7a5baff" }, - "SWOT_L2_HR_RiverSP_Reach_017_152_EU_20240624T111046_20240624T111056_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_152_EU_20240624T111046_20240624T111056_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:40.073Z", "checksum": "0feebb2dc94a637e5b8b24a0e2c5e405" }, - "SWOT_L2_HR_RiverSP_Reach_017_152_AF_20240624T112457_20240624T112506_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_152_AF_20240624T112457_20240624T112506_PIC0_01.zip": { "revision_date": "2024-06-30T01:36:42.796Z", "checksum": "1cec60eb886f1eba0c911aa78befcd28" }, - "SWOT_L2_HR_RiverSP_Reach_017_158_AR_20240624T161520_20240624T161526_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_158_AR_20240624T161520_20240624T161526_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:08.652Z", "checksum": "a62a99b85b414543141d104cf05bfc62" }, - "SWOT_L2_HR_RiverSP_Reach_017_158_NA_20240624T161947_20240624T161957_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_158_NA_20240624T161947_20240624T161957_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:10.046Z", "checksum": "a8fd07dd14a2e16fb3279adc99b65296" }, - "SWOT_L2_HR_RiverSP_Reach_017_158_SA_20240624T163620_20240624T163627_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_158_SA_20240624T163620_20240624T163627_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:12.720Z", "checksum": "432bdd428510ce811faa954ac9a2a799" }, - "SWOT_L2_HR_RiverSP_Reach_017_159_AU_20240624T172707_20240624T172713_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_159_AU_20240624T172707_20240624T172713_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:15.619Z", "checksum": "b4eac61e405e4e3ddd8ae1bc69746abc" }, - "SWOT_L2_HR_RiverSP_Reach_017_159_AS_20240624T173641_20240624T173647_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_159_AS_20240624T173641_20240624T173647_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:18.402Z", "checksum": "c2fe43fb0ff7136ce534a0aee8fedd9c" }, - "SWOT_L2_HR_RiverSP_Reach_017_159_SI_20240624T174546_20240624T174557_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_159_SI_20240624T174546_20240624T174557_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:21.478Z", "checksum": "582c5c3fcf99a3f12748a35911ca2957" }, - "SWOT_L2_HR_RiverSP_Reach_017_160_AR_20240624T175753_20240624T175800_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_160_AR_20240624T175753_20240624T175800_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:24.224Z", "checksum": "f6696681bdc85e06b3c1ace8d2662ecd" }, - "SWOT_L2_HR_RiverSP_Reach_017_160_NA_20240624T180249_20240624T180300_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_160_NA_20240624T180249_20240624T180300_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:28.243Z", "checksum": "8608bea271b35a2a256157a333846e3b" }, - "SWOT_L2_HR_RiverSP_Reach_017_160_SA_20240624T181801_20240624T181810_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_160_SA_20240624T181801_20240624T181810_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:29.767Z", "checksum": "0108f47ff5f09d8ba2d7ea005e852bf7" }, - "SWOT_L2_HR_RiverSP_Reach_017_161_AS_20240624T191826_20240624T191831_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_161_AS_20240624T191826_20240624T191831_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:32.740Z", "checksum": "0113ec8d72d04ec789881f8a4bddb0e3" }, - "SWOT_L2_HR_RiverSP_Reach_017_161_SI_20240624T192550_20240624T192601_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_161_SI_20240624T192550_20240624T192601_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:36.476Z", "checksum": "68bd5074a448d9d2a7776785e0665b9a" }, - "SWOT_L2_HR_RiverSP_Reach_017_162_AR_20240624T194233_20240624T194244_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_162_AR_20240624T194233_20240624T194244_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:39.364Z", "checksum": "1e813e6850b9907da0ff195deae2052f" }, - "SWOT_L2_HR_RiverSP_Reach_017_162_NA_20240624T194803_20240624T194814_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_162_NA_20240624T194803_20240624T194814_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:42.085Z", "checksum": "cfa61d84ee42eddd8bc45927cc5d145a" }, - "SWOT_L2_HR_RiverSP_Reach_017_163_AS_20240624T210224_20240624T210234_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_163_AS_20240624T210224_20240624T210234_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:44.877Z", "checksum": "14c8fca412fb1e9d2ae4cdf7d8f10f90" }, - "SWOT_L2_HR_RiverSP_Reach_017_163_EU_20240624T210403_20240624T210414_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_163_EU_20240624T210403_20240624T210414_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:47.664Z", "checksum": "9f004a97eee17c4e7244ed26689c3069" }, - "SWOT_L2_HR_RiverSP_Reach_017_163_SI_20240624T210923_20240624T210934_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_163_SI_20240624T210923_20240624T210934_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:50.261Z", "checksum": "af61940cad0de6b8c8912137b751d823" }, - "SWOT_L2_HR_RiverSP_Reach_017_165_AF_20240624T222947_20240624T222948_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_165_AF_20240624T222947_20240624T222948_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:55.508Z", "checksum": "11e09d9673f175b95d3ef6ee23c66e05" }, - "SWOT_L2_HR_RiverSP_Reach_017_165_EU_20240624T224741_20240624T224748_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_165_EU_20240624T224741_20240624T224748_PIC0_01.zip": { "revision_date": "2024-06-30T01:37:58.390Z", "checksum": "e7d165b53d0e0909c259ab9d8a679467" }, - "SWOT_L2_HR_RiverSP_Reach_017_165_SI_20240624T225647_20240624T225658_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_165_SI_20240624T225647_20240624T225658_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:01.232Z", "checksum": "0bfe09363e365aa1f8312b9145378b28" }, - "SWOT_L2_HR_RiverSP_Reach_017_167_AF_20240625T001959_20240625T002002_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_167_AF_20240625T001959_20240625T002002_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:06.220Z", "checksum": "6ce0d761af62011bb9c6b3770c49c41a" }, - "SWOT_L2_HR_RiverSP_Reach_017_167_EU_20240625T003315_20240625T003322_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_167_EU_20240625T003315_20240625T003322_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:08.989Z", "checksum": "e773e4dbd8e70bdf8a9272c31cc2c426" }, - "SWOT_L2_HR_RiverSP_Reach_017_168_SI_20240625T004713_20240625T004714_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_168_SI_20240625T004713_20240625T004714_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:13.092Z", "checksum": "760ed362642546e6cc5ad081e7720afa" }, - "SWOT_L2_HR_RiverSP_Reach_017_169_AF_20240625T020631_20240625T020636_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_169_AF_20240625T020631_20240625T020636_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:14.441Z", "checksum": "e3133380abf3c4bc15e5a641f2363d77" }, - "SWOT_L2_HR_RiverSP_Reach_017_169_EU_20240625T021443_20240625T021445_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_169_EU_20240625T021443_20240625T021445_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:17.361Z", "checksum": "98d0e2f8674834b50659d348261cfaef" }, - "SWOT_L2_HR_RiverSP_Reach_017_170_SI_20240625T023132_20240625T023138_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_170_SI_20240625T023132_20240625T023138_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:20.151Z", "checksum": "2e41df807623b5b8dff53279ca091c01" }, - "SWOT_L2_HR_RiverSP_Reach_017_170_AS_20240625T023917_20240625T023919_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_170_AS_20240625T023917_20240625T023919_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:23.795Z", "checksum": "0929ee48342206e92213697f6b72a089" }, - "SWOT_L2_HR_RiverSP_Reach_017_171_SA_20240625T034001_20240625T034009_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_171_SA_20240625T034001_20240625T034009_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:26.649Z", "checksum": "7d65b41d9d23a16fb8302319011ba06e" }, - "SWOT_L2_HR_RiverSP_Reach_017_171_EU_20240625T040542_20240625T040549_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_171_EU_20240625T040542_20240625T040549_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:29.379Z", "checksum": "e6df4579ba3a1644d205fec0b09d91bc" }, - "SWOT_L2_HR_RiverSP_Reach_017_172_EU_20240625T041400_20240625T041402_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_172_EU_20240625T041400_20240625T041402_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:33.453Z", "checksum": "fb250364f9d5fca9a3b64c5a5c84236e" }, - "SWOT_L2_HR_RiverSP_Reach_017_172_SI_20240625T041535_20240625T041542_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_172_SI_20240625T041535_20240625T041542_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:34.811Z", "checksum": "4fd6e581555fb235e4192ec00b65a00b" }, - "SWOT_L2_HR_RiverSP_Reach_017_172_AS_20240625T042231_20240625T042242_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_172_AS_20240625T042231_20240625T042242_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:38.867Z", "checksum": "09b3361e120550461b2eac345d257349" }, - "SWOT_L2_HR_RiverSP_Reach_017_172_AU_20240625T043432_20240625T043442_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_172_AU_20240625T043432_20240625T043442_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:40.266Z", "checksum": "f5464e06f875a17bfd86da5c68e9e8b6" }, - "SWOT_L2_HR_RiverSP_Reach_017_173_SA_20240625T051827_20240625T051832_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_173_SA_20240625T051827_20240625T051832_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:43.111Z", "checksum": "aaa5e6efd6613964e411172ef7141752" }, - "SWOT_L2_HR_RiverSP_Reach_017_173_NA_20240625T054338_20240625T054343_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_173_NA_20240625T054338_20240625T054343_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:45.897Z", "checksum": "9e9f4ffbd92d71d3110ade1095e06115" }, - "SWOT_L2_HR_RiverSP_Reach_017_173_GR_20240625T054742_20240625T054753_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_173_GR_20240625T054742_20240625T054753_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:48.971Z", "checksum": "a87b6733d8c4523a9261fdb33f0e13e2" }, - "SWOT_L2_HR_RiverSP_Reach_017_174_EU_20240625T055813_20240625T055815_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_174_EU_20240625T055813_20240625T055815_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:51.919Z", "checksum": "3f83979719c4f0c4ba13649286303963" }, - "SWOT_L2_HR_RiverSP_Reach_017_174_SI_20240625T055934_20240625T055936_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_174_SI_20240625T055934_20240625T055936_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:54.557Z", "checksum": "7c53100b6f1b8cf24ce27798785972b2" }, - "SWOT_L2_HR_RiverSP_Reach_017_174_AS_20240625T060725_20240625T060736_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_174_AS_20240625T060725_20240625T060736_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:58.622Z", "checksum": "072832f4ed1ea68256723bc7a979ce08" }, - "SWOT_L2_HR_RiverSP_Reach_017_174_AU_20240625T062007_20240625T062016_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_174_AU_20240625T062007_20240625T062016_PIC0_01.zip": { "revision_date": "2024-06-30T01:38:59.949Z", "checksum": "c21323e521b56b3e6ed43d258cd4d043" }, - "SWOT_L2_HR_RiverSP_Reach_017_175_NA_20240625T071636_20240625T071646_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_175_NA_20240625T071636_20240625T071646_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:03.021Z", "checksum": "2dd869e639beabccd63480411c5f9ee7" }, - "SWOT_L2_HR_RiverSP_Reach_017_175_AR_20240625T073123_20240625T073127_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_175_AR_20240625T073123_20240625T073127_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:07.553Z", "checksum": "79631b13fe68c87b2be87214cb498b35" }, - "SWOT_L2_HR_RiverSP_Reach_017_175_GR_20240625T073429_20240625T073437_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_175_GR_20240625T073429_20240625T073437_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:08.984Z", "checksum": "426d70919e59d9fcf76f7078e6920f32" }, - "SWOT_L2_HR_RiverSP_Reach_017_176_EU_20240625T074342_20240625T074349_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_176_EU_20240625T074342_20240625T074349_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:11.804Z", "checksum": "fa7e82deed9b2a16712d3f1e7ebb141e" }, - "SWOT_L2_HR_RiverSP_Reach_017_176_SI_20240625T074629_20240625T074640_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_176_SI_20240625T074629_20240625T074640_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:16.097Z", "checksum": "70ec26bb9cfc7db60f62358af0ba8aba" }, - "SWOT_L2_HR_RiverSP_Reach_017_176_AS_20240625T074839_20240625T074850_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_176_AS_20240625T074839_20240625T074850_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:17.647Z", "checksum": "49233438e0e9c6d74ba580790deed74e" }, - "SWOT_L2_HR_RiverSP_Reach_017_177_NA_20240625T090320_20240625T090330_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_177_NA_20240625T090320_20240625T090330_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:21.343Z", "checksum": "319e0c28a8c57e321c8d30b1f7769daa" }, - "SWOT_L2_HR_RiverSP_Reach_017_177_AR_20240625T091249_20240625T091300_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_177_AR_20240625T091249_20240625T091300_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:24.219Z", "checksum": "a9f932b540a2d645082c0006832453c9" }, - "SWOT_L2_HR_RiverSP_Reach_017_177_GR_20240625T091920_20240625T091931_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_177_GR_20240625T091920_20240625T091931_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:28.383Z", "checksum": "686700db4b3d9cad76653d9c7855ff65" }, - "SWOT_L2_HR_RiverSP_Reach_017_178_GR_20240625T092124_20240625T092133_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_178_GR_20240625T092124_20240625T092133_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:31.408Z", "checksum": "4ddadb6bad04019fb5fcf09ec2bf1c53" }, - "SWOT_L2_HR_RiverSP_Reach_017_178_EU_20240625T092552_20240625T092553_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_178_EU_20240625T092552_20240625T092553_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:32.759Z", "checksum": "21ca23ce2001cf89ca489f57fab3a6e5" }, - "SWOT_L2_HR_RiverSP_Reach_017_179_NA_20240625T105347_20240625T105351_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_179_NA_20240625T105347_20240625T105351_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:36.809Z", "checksum": "1ad54c4d26be12ee92a185f0c96c206b" }, - "SWOT_L2_HR_RiverSP_Reach_017_179_AR_20240625T105513_20240625T105524_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_179_AR_20240625T105513_20240625T105524_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:39.339Z", "checksum": "25d4fd73cdb29c58a58c9e37d3f50e9d" }, - "SWOT_L2_HR_RiverSP_Reach_017_179_GR_20240625T110320_20240625T110325_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_179_GR_20240625T110320_20240625T110325_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:41.752Z", "checksum": "3d09f54a185efa48341f9b2cc2aefb6f" }, - "SWOT_L2_HR_RiverSP_Reach_017_180_GR_20240625T110630_20240625T110637_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_180_GR_20240625T110630_20240625T110637_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:42.938Z", "checksum": "a917f8162613a6f869662fb542c3d172" }, - "SWOT_L2_HR_RiverSP_Reach_017_180_EU_20240625T111204_20240625T111207_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_180_EU_20240625T111204_20240625T111207_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:45.365Z", "checksum": "8cff648d67141f2ea78214987e2b3098" }, - "SWOT_L2_HR_RiverSP_Reach_017_180_AF_20240625T112039_20240625T112046_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_180_AF_20240625T112039_20240625T112046_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:47.885Z", "checksum": "215f3661db344da7e53251bb819f102f" }, - "SWOT_L2_HR_RiverSP_Reach_017_182_GR_20240625T124757_20240625T124800_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_182_GR_20240625T124757_20240625T124800_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:52.794Z", "checksum": "60786a7b90b65d9b2c9aeef46eb83374" }, - "SWOT_L2_HR_RiverSP_Reach_017_182_AF_20240625T130253_20240625T130259_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_182_AF_20240625T130253_20240625T130259_PIC0_01.zip": { "revision_date": "2024-06-30T01:39:55.385Z", "checksum": "0c55f91f2abfdd987d1595ab9b2179ba" }, - "SWOT_L2_HR_RiverSP_Reach_017_185_AU_20240625T153858_20240625T153905_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_185_AU_20240625T153858_20240625T153905_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:02.594Z", "checksum": "6ea6a176d46af9ca0f9ed3180ebf75dd" }, - "SWOT_L2_HR_RiverSP_Reach_017_185_SI_20240625T160525_20240625T160535_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_185_SI_20240625T160525_20240625T160535_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:05.336Z", "checksum": "9b669ca997d64795f31da428470b50d7" }, - "SWOT_L2_HR_RiverSP_Reach_017_186_AR_20240625T161541_20240625T161547_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_186_AR_20240625T161541_20240625T161547_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:09.110Z", "checksum": "0effa78c9d78a62180fca7c2cc2edece" }, - "SWOT_L2_HR_RiverSP_Reach_017_186_NA_20240625T161956_20240625T161958_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_186_NA_20240625T161956_20240625T161958_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:10.709Z", "checksum": "950c1f67d3c344b4d14856a360fdb83a" }, - "SWOT_L2_HR_RiverSP_Reach_017_186_SA_20240625T163642_20240625T163648_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_186_SA_20240625T163642_20240625T163648_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:13.068Z", "checksum": "f6053ab6d6176e820646126d9c30ad53" }, - "SWOT_L2_HR_RiverSP_Reach_017_187_AU_20240625T172738_20240625T172748_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_187_AU_20240625T172738_20240625T172748_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:17.221Z", "checksum": "fe1179e031bfe307afe293f5e46b907d" }, - "SWOT_L2_HR_RiverSP_Reach_017_187_AS_20240625T173636_20240625T173638_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_187_AS_20240625T173636_20240625T173638_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:18.498Z", "checksum": "b0019c174abf6ac03931f097d9b0f019" }, - "SWOT_L2_HR_RiverSP_Reach_017_187_SI_20240625T174617_20240625T174628_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_187_SI_20240625T174617_20240625T174628_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:21.181Z", "checksum": "16e55408dd104ec0e2472206e10a443d" }, - "SWOT_L2_HR_RiverSP_Reach_017_189_AS_20240625T191901_20240625T191912_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_189_AS_20240625T191901_20240625T191912_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:23.704Z", "checksum": "2b8c99a7a9fb7238535e93838c1dda2a" }, - "SWOT_L2_HR_RiverSP_Reach_017_189_SI_20240625T192625_20240625T192632_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_189_SI_20240625T192625_20240625T192632_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:27.516Z", "checksum": "0de636a95c1f8ef27dbdaaa13a84b5c7" }, - "SWOT_L2_HR_RiverSP_Reach_017_190_AR_20240625T194315_20240625T194325_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_190_AR_20240625T194315_20240625T194325_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:30.219Z", "checksum": "1d90e8badcbcffb4529a40fa1baf25dd" }, - "SWOT_L2_HR_RiverSP_Reach_017_190_NA_20240625T194804_20240625T194815_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_190_NA_20240625T194804_20240625T194815_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:33.998Z", "checksum": "1ee374432d19aafdd4d151f0d86f5c03" }, - "SWOT_L2_HR_RiverSP_Reach_017_191_EU_20240625T210307_20240625T210315_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_191_EU_20240625T210307_20240625T210315_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:35.224Z", "checksum": "61371e63d483585772a183c18e9e7d3f" }, - "SWOT_L2_HR_RiverSP_Reach_017_191_AS_20240625T210544_20240625T210555_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_191_AS_20240625T210544_20240625T210555_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:38.137Z", "checksum": "0d378ec7801d89bc3bc651208bc0ff2e" }, - "SWOT_L2_HR_RiverSP_Reach_017_191_SI_20240625T211004_20240625T211015_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_191_SI_20240625T211004_20240625T211015_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:40.877Z", "checksum": "8e1c37afdbba6bffde67b05aca0ca4e2" }, - "SWOT_L2_HR_RiverSP_Reach_017_197_AF_20240626T020741_20240626T020746_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_197_AF_20240626T020741_20240626T020746_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:48.242Z", "checksum": "9d34268fe5ce775684486e5c22409639" }, - "SWOT_L2_HR_RiverSP_Reach_017_197_EU_20240626T021639_20240626T021646_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_197_EU_20240626T021639_20240626T021646_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:50.742Z", "checksum": "33a83518cd9624fe2b61f6c346c7a809" }, - "SWOT_L2_HR_RiverSP_Reach_017_209_AR_20240626T123834_20240626T123838_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_209_AR_20240626T123834_20240626T123838_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:53.271Z", "checksum": "ec90497ccc7ff0b40f094fd8e2b7ff88" }, - "SWOT_L2_HR_RiverSP_Reach_017_211_SI_20240626T142402_20240626T142412_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_211_SI_20240626T142402_20240626T142412_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:55.674Z", "checksum": "bfa56d41d66e81f1dda4c38fea4ae969" }, - "SWOT_L2_HR_RiverSP_Reach_017_220_AR_20240626T212653_20240626T212659_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_220_AR_20240626T212653_20240626T212659_PIC0_01.zip": { "revision_date": "2024-06-30T01:40:58.117Z", "checksum": "aa477f1f86311e0bbc528db4b04fc42f" }, - "SWOT_L2_HR_RiverSP_Reach_017_222_SI_20240626T230926_20240626T230933_PIC0_01": { + "SWOT_L2_HR_RiverSP_Reach_017_222_SI_20240626T230926_20240626T230933_PIC0_01.zip": { "revision_date": "2024-06-30T01:41:00.538Z", "checksum": "d127979e6759367e79ad3a9e260fc3e7" } diff --git a/tests/test_track_ingest.py b/tests/test_track_ingest.py index 94d85fa..930fe27 100644 --- a/tests/test_track_ingest.py +++ b/tests/test_track_ingest.py @@ -6,12 +6,13 @@ import json import os import pathlib +from unittest.mock import MagicMock import vcr from hydrocron.utils import constants -def test_query_cmr(): +def test_query_cmr(mock_ssm): """Test the query_cmr function. Uses vcrpy to record CMR API response. @@ -20,25 +21,24 @@ def test_query_cmr(): collection_shortname = "SWOT_L2_HR_RiverSP_reach_2.0" collection_start_date = datetime.datetime.strptime("20240630", "%Y%m%d").replace(tzinfo=datetime.timezone.utc) - hydrocron_table = "hydrocron-swot-reach-table" - track = Track(collection_shortname, collection_start_date, hydrocron_table) - track.revision_end = datetime.datetime(2024, 6, 30, 12, 0, 0, tzinfo=datetime.timezone.utc) + track = Track(collection_shortname, collection_start_date) + track.query_start = datetime.datetime(2024, 6, 30, 0, 0, 0, tzinfo=datetime.timezone.utc) + track.query_end = datetime.datetime(2024, 6, 30, 12, 0, 0, tzinfo=datetime.timezone.utc) vcr_cassette = pathlib.Path(os.path.dirname(os.path.realpath(__file__))) \ .joinpath('vcr_cassettes').joinpath('cmr_query.yaml') with vcr.use_cassette(vcr_cassette): - track.query_cmr() - actual_data = track.cmr_granules + actual_data = track.query_cmr(False) expected_file = (pathlib.Path(os.path.dirname(os.path.realpath(__file__))) - .joinpath('test_data').joinpath('query_cmr_granule_results.json')) + .joinpath('test_data').joinpath('query_cmr_granule_results.json')) with open(expected_file) as jf: expected_data = json.load(jf) assert sorted(actual_data.items()) == sorted(expected_data.items()) -def test_get_granule_ur(hydrocron_api): +def test_get_granule_ur(track_ingest_fixture): """ Test query granuleUR item. @@ -51,34 +51,161 @@ def test_get_granule_ur(hydrocron_api): data_repository = DynamoDataRepository(connection.dynamodb_resource) - table_name = constants.API_TEST_REACH_TABLE_NAME - granule_ur = "SWOT_L2_HR_RiverSP_Reach_548_011_NA_20230610T193337_20230610T193344_PIA1_01.zip" + table_name = constants.SWOT_REACH_TABLE_NAME + granule_ur = "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip" actual_data = data_repository.get_granule_ur(table_name, granule_ur) assert actual_data["Items"][0]["granuleUR"] == granule_ur - -def test_query_hydrocron(hydrocron_api): + + +def test_query_hydrocron(track_ingest_fixture): """ Test query_hydrocron function. Parameters ---------- - hydrocron_api: Fixture ensuring the database is configured for the api + track_ingest_fixture: Fixture ensuring the database is configured for track ingest operations """ from hydrocron.db.track_ingest import Track collection_shortname = "SWOT_L2_HR_RiverSP_reach_2.0" collection_start_date = datetime.datetime.strptime("20240630", "%Y%m%d").replace(tzinfo=datetime.timezone.utc) - hydrocron_table = "hydrocron-swot-reach-table" - track = Track(collection_shortname, collection_start_date, hydrocron_table) - track.cmr_granules = { - "SWOT_L2_HR_RiverSP_Reach_548_011_NA_20230610T193337_20230610T193344_PIA1_01": { + track = Track(collection_shortname, collection_start_date) + cmr_granules = { + "SWOT_L2_HR_RiverSP_Reach_584_024_NA_20230610T193337_20230610T193344_PIA1_01": { "revision_date": "2024-06-30T01:33:13.037Z", "checksum": "edda7230d20f1a85bae82f9917c86aa1" } } - track.query_hydrocron() - actual_data = track.hydrocron_granules - print(actual_data) + hydrocron_table = "hydrocron-swot-reach-table" + track.query_hydrocron(hydrocron_table, cmr_granules) + actual_data = track.to_ingest + + expected = [{ + "granuleUR": "SWOT_L2_HR_RiverSP_Reach_584_024_NA_20230610T193337_20230610T193344_PIA1_01", + "revision_date": "2024-06-30T01:33:13.037Z", + "checksum": "edda7230d20f1a85bae82f9917c86aa1", + "expected_feature_count": -1, + "actual_feature_count": 0, + "status": "to_ingest" + }] + assert actual_data == expected + + +def test_get_status(track_ingest_fixture): + """Test get_status function. + + Parameters + ---------- + track_ingest_fixture: Fixture ensuring the database is configured for track ingest operations + """ + import hydrocron.utils.connection + from hydrocron.api.data_access.db import DynamoDataRepository + + hydrocron_table = DynamoDataRepository(hydrocron.utils.connection._dynamodb_resource) + items = hydrocron_table.get_status(constants.SWOT_REACH_TRACK_INGEST_TABLE_NAME, "to_ingest") + expected = [{ + "granuleUR": "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip", + "revision_date": "2024-05-22T19:15:44.572Z", + "checksum": "0823db619be0044e809a5f992e067d03", + "expected_feature_count": 664, + "actual_feature_count": 0, + "status": "to_ingest" + }] + assert items == expected + + +def test_get_series_granule_ur(track_ingest_fixture): + """Test get_series_granule_ur. + + Parameters + ---------- + track_ingest_fixture: Fixture ensuring the database is configured for track ingest operations + """ + + import hydrocron.utils.connection + from hydrocron.api.data_access.db import DynamoDataRepository + + hydrocron_table = DynamoDataRepository(hydrocron.utils.connection._dynamodb_resource) + table_name = constants.SWOT_REACH_TABLE_NAME + feature_name = "reach_id" + granule_ur = "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip" + items = hydrocron_table.get_series_granule_ur(table_name, feature_name, granule_ur) + assert len(items) == 664 + +def test_query_ingest(track_ingest_fixture): + """Test query_ingest function. + + Parameters + ---------- + track_ingest_fixture: Fixture ensuring the database is configured for track ingest operations + """ + from hydrocron.db.track_ingest import Track + + collection_shortname = "SWOT_L2_HR_RiverSP_reach_2.0" + collection_start_date = datetime.datetime.strptime("20240630", "%Y%m%d").replace(tzinfo=datetime.timezone.utc) + track = Track(collection_shortname, collection_start_date) + track._query_for_granule_ur = MagicMock(name="_query_for_granule_ur") + track._query_for_granule_ur.return_value = "s3://podaac-swot-ops-cumulus-protected/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip" + + hydrocron_track_table = constants.SWOT_REACH_TRACK_INGEST_TABLE_NAME + hydrocron_table = constants.SWOT_REACH_TABLE_NAME + track.query_track_ingest(hydrocron_track_table, hydrocron_table) + + expected = [{ + "granuleUR": "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip", + "revision_date": "2024-05-22T19:15:44.572Z", + "checksum": "0823db619be0044e809a5f992e067d03", + "expected_feature_count": 664, + "actual_feature_count": 664, + }] + assert track.ingested == expected + + +def test_query_ingest_to_ingest(track_ingest_fixture): + """Test query_ingest function for require ingest. + + Parameters + ---------- + track_ingest_fixture: Fixture ensuring the database is configured for track ingest operations + """ + from hydrocron.db.track_ingest import Track + from hydrocron.db import HydrocronTable + import hydrocron.utils.connection + + TEST_SHAPEFILE_PATH_REACH_TRACK = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'data', + "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip" + ) + + track_reach_table = HydrocronTable(hydrocron.utils.connection._dynamodb_resource, constants.SWOT_REACH_TRACK_INGEST_TABLE_NAME) + track_ingest_record = [{ + "granuleUR": os.path.basename(TEST_SHAPEFILE_PATH_REACH_TRACK), + "revision_date": "2024-05-22T19:15:44.572Z", + "expected_feature_count": 665, + "actual_feature_count": 0, + "checksum": "0823db619be0044e809a5f992e067d03", + "status": "to_ingest" + }] + track_reach_table.batch_fill_table(track_ingest_record) + + collection_shortname = "SWOT_L2_HR_RiverSP_reach_2.0" + collection_start_date = datetime.datetime.strptime("20240630", "%Y%m%d").replace(tzinfo=datetime.timezone.utc) + track = Track(collection_shortname, collection_start_date) + track._query_for_granule_ur = MagicMock(name="_query_for_granule_ur") + track._query_for_granule_ur.return_value = "s3://podaac-swot-ops-cumulus-protected/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip" + + hydrocron_track_table = constants.SWOT_REACH_TRACK_INGEST_TABLE_NAME + hydrocron_table = constants.SWOT_REACH_TABLE_NAME + track.query_track_ingest(hydrocron_track_table, hydrocron_table) - assert sorted(actual_data.items()) == sorted(track.cmr_granules.items()) \ No newline at end of file + expected = [{ + "granuleUR": "SWOT_L2_HR_RiverSP_Reach_020_149_NA_20240825T231711_20240825T231722_PIC0_01.zip", + "revision_date": "2024-05-22T19:15:44.572Z", + "checksum": "0823db619be0044e809a5f992e067d03", + "expected_feature_count": 665, + "actual_feature_count": 664, + "status": "to_ingest" + }] + assert track.to_ingest == expected