diff --git a/tests/data/harmony/message.json b/tests/data/harmony/message.json new file mode 100644 index 0000000..21eb885 --- /dev/null +++ b/tests/data/harmony/message.json @@ -0,0 +1,16 @@ +{ + "sources": [{ + "collection": "C1234088182-EEDTEST" + }], + "format": { + "mime": "application/x-netcdf4" + }, + "subset": {}, + "requestId": "00001111-2222-3333-4444-555566667777", + "user": "jdoe", + "client": "harmony-example", + "isSynchronous": false, + "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/", + "callback": "http://localhost/some-path", + "version": "0.10.0" +} diff --git a/tests/data/harmony/source/catalog.json b/tests/data/harmony/source/catalog.json new file mode 100644 index 0000000..fd99944 --- /dev/null +++ b/tests/data/harmony/source/catalog.json @@ -0,0 +1,48 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59", + "links": [ + { + "rel": "harmony_source", + "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST" + }, + { + "rel": "item", + "href": "./granule_S012G01.json", + "type": "application/json", + "title": "granule_S012G01" + }, + { + "rel": "item", + "href": "./granule_S012G02.json", + "type": "application/json", + "title": "granule_S012G02" + }, + { + "rel": "item", + "href": "./granule_S013G01.json", + "type": "application/json", + "title": "granule_S013G01" + }, + { + "rel": "item", + "href": "./granule_S013G02.json", + "type": "application/json", + "title": "granule_S013G02" + }, + { + "rel": "item", + "href": "./granule_S014G01.json", + "type": "application/json", + "title": "granule_S014G01" + }, + { + "rel": "item", + "href": "./granule_S014G02.json", + "type": "application/json", + "title": "granule_S014G02" + } + ], + "description": "CMR Granules for C1234088182-EEDTEST batch 1" +} diff --git a/tests/data/harmony/source/catalog0.json b/tests/data/harmony/source/catalog0.json new file mode 100644 index 0000000..3512b68 --- /dev/null +++ b/tests/data/harmony/source/catalog0.json @@ -0,0 +1,36 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59", + "links": [ + { + "rel": "harmony_source", + "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST" + }, + { + "rel": "item", + "href": "./granule_S012G01.json", + "type": "application/json", + "title": "granule_S012G01" + }, + { + "rel": "item", + "href": "./granule_S012G02.json", + "type": "application/json", + "title": "granule_S012G02" + }, + { + "rel": "item", + "href": "./granule_S013G01.json", + "type": "application/json", + "title": "granule_S013G01" + }, + { + "rel": "next", + "href": "data/harmony/source/catalog1.json", + "type": "application/json", + "title": "Next page" + } + ], + "description": "CMR Granules for C1234088182-EEDTEST batch 1" +} diff --git a/tests/data/harmony/source/catalog1.json b/tests/data/harmony/source/catalog1.json new file mode 100644 index 0000000..25f1540 --- /dev/null +++ b/tests/data/harmony/source/catalog1.json @@ -0,0 +1,36 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59", + "links": [ + { + "rel": "harmony_source", + "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST" + }, + { + "rel": "item", + "href": "./granule_S013G02.json", + "type": "application/json", + "title": "granule_S013G02" + }, + { + "rel": "item", + "href": "./granule_S014G01.json", + "type": "application/json", + "title": "granule_S014G01" + }, + { + "rel": "item", + "href": "./granule_S014G02.json", + "type": "application/json", + "title": "granule_S014G02" + }, + { + "rel": "prev", + "href": "data/harmony/source/catalog0.json", + "type": "application/json", + "title": "Previous page" + } + ], + "description": "CMR Granules for C1234088182-EEDTEST batch 2" +} diff --git a/tests/data/harmony/source/granule_S012G01.json b/tests/data/harmony/source/granule_S012G01.json new file mode 100644 index 0000000..f468f54 --- /dev/null +++ b/tests/data/harmony/source/granule_S012G01.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "51d02b24-d00e-4640-9887-05f98f6b96d8", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-02T00:00:00.000Z", + "end_datetime": "2020-01-02T23:59:59.000Z" + }, + "bbox": [1, 3, 1, 3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120101Z_S012G01.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120101Z_S012G01.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S012G02.json b/tests/data/harmony/source/granule_S012G02.json new file mode 100644 index 0000000..1d25a15 --- /dev/null +++ b/tests/data/harmony/source/granule_S012G02.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "21eb7dc5-7a9d-4374-9a88-12451ba653ef", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-03T00:00:00.000Z", + "end_datetime": "2020-01-03T23:59:59.000Z" + }, + "bbox": [-1, -3, -1, -3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120107Z_S012G02.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120107Z_S012G02.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S013G01.json b/tests/data/harmony/source/granule_S013G01.json new file mode 100644 index 0000000..c22cbc2 --- /dev/null +++ b/tests/data/harmony/source/granule_S013G01.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "dc07a25a-54bd-4389-97b6-c1d68cf0d586", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-05T00:00:00.000Z", + "end_datetime": "2020-01-05T23:59:59.000Z" + }, + "bbox": [-4, -2, -4, 2], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120202Z_S013G01.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120202Z_S013G01.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S013G02.json b/tests/data/harmony/source/granule_S013G02.json new file mode 100644 index 0000000..98d22f9 --- /dev/null +++ b/tests/data/harmony/source/granule_S013G02.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "53cdf09b-4004-4405-9363-f93455a11c2f", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-04T00:00:00.000Z", + "end_datetime": "2020-01-04T23:59:59.000Z" + }, + "bbox": [4, 2, 4, 2], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120209Z_S013G02.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120209Z_S013G02.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S014G01.json b/tests/data/harmony/source/granule_S014G01.json new file mode 100644 index 0000000..8800f8a --- /dev/null +++ b/tests/data/harmony/source/granule_S014G01.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "349b5f4a-34ab-4b0d-ae00-b55fe3d166fb", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-05T00:00:00.000Z", + "end_datetime": "2020-01-05T23:59:59.000Z" + }, + "bbox": [-5, -3, -5, 3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120303Z_S014G01.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120303Z_S014G01.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S014G02.json b/tests/data/harmony/source/granule_S014G02.json new file mode 100644 index 0000000..6f6dffc --- /dev/null +++ b/tests/data/harmony/source/granule_S014G02.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cb0271ad-05a9-4138-8bfb-fa6b83fbe6fa", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-05T00:00:00.000Z", + "end_datetime": "2020-01-05T23:59:59.000Z" + }, + "bbox": [5, 3, 5, 3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120310Z_S014G02.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120310Z_S014G02.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/test_harmony_adapter.py b/tests/test_harmony_adapter.py new file mode 100644 index 0000000..2726a6e --- /dev/null +++ b/tests/test_harmony_adapter.py @@ -0,0 +1,101 @@ +import json +import sys +from os import environ +from pathlib import Path +from unittest.mock import patch +from urllib.parse import urlsplit + +import pytest + +import batcher.harmony.cli + + +@pytest.mark.usefixtures("pass_options") +class TestBatching: + __test_path = Path(__file__).parent.resolve() + __data_path = __test_path.joinpath("data") + __harmony_path = __data_path.joinpath("harmony") + + def test_service_invoke(self, temp_output_dir): + in_message_path = self.__harmony_path.joinpath("message.json") + in_message_data = in_message_path.read_text() + + # test with both paged catalogs and un-paged catalogs + for in_catalog_name in ["catalog.json", "catalog0.json"]: + + in_catalog_path = self.__harmony_path.joinpath("source", in_catalog_name) + + test_args = [ + batcher.harmony.cli.__file__, + "--harmony-action", + "invoke", + "--harmony-input", + in_message_data, + "--harmony-source", + str(in_catalog_path), + "--harmony-metadata-dir", + str(temp_output_dir), + "--harmony-data-location", + temp_output_dir.as_uri(), + ] + + test_env = { + "ENV": "dev", + "OAUTH_CLIENT_ID": "", + "OAUTH_UID": "", + "OAUTH_PASSWORD": "", + "OAUTH_REDIRECT_URI": "", + "STAGING_PATH": "", + "STAGING_BUCKET": "", + } + + with patch.object(sys, "argv", test_args), patch.dict(environ, test_env): + batcher.harmony.cli.main() + + # Open the outputs + out_batch_catalog_path = temp_output_dir.joinpath("batch-catalogs.json") + out_batch_catalogs = json.loads(out_batch_catalog_path.read_text()) + + # Go through each batched catalog + batched_files = {0: [], 1: [], 2: []} + for batch_index, catalog in enumerate(out_batch_catalogs): + out_catalog_path = temp_output_dir.joinpath(catalog) + out_catalog = json.loads(out_catalog_path.read_text()) + + for item_meta in out_catalog["links"]: + if item_meta["rel"] == "item": + item_path = temp_output_dir.joinpath(item_meta["href"]).resolve() + + # -- Item Verification -- + item = json.loads(item_path.read_text()) + properties = item["properties"] + assert item["bbox"] + assert properties["start_datetime"] + assert properties["end_datetime"] + + # -- Asset Verification -- + data = item["assets"]["data"] + + # Sanity checks on metadata + assert data["type"] == "application/x-netcdf4" + assert data["roles"] == ["data"] + + batched_files[batch_index].append(Path(urlsplit(data["href"]).path).stem) + + # -- batch file list verification -- + files_dict = { + 0: [ + "TEMPO_NO2_L2_V03_20240601T120101Z_S012G01", + "TEMPO_NO2_L2_V03_20240601T120107Z_S012G02", + ], + 1: [ + "TEMPO_NO2_L2_V03_20240601T120202Z_S013G01", + "TEMPO_NO2_L2_V03_20240601T120209Z_S013G02", + ], + 2: [ + "TEMPO_NO2_L2_V03_20240601T120303Z_S014G01", + "TEMPO_NO2_L2_V03_20240601T120310Z_S014G02", + ], + } + + assert batched_files == files_dict