Add regression test for net2cog (#94)

* Add regression test for net2cog

* Add regression test for net2cog

* updates after review

---------

Co-authored-by: Frank Greguska <[email protected]>

.github/workflows/build-all-images.yml

@@ -45,6 +45,9 @@ jobs:
           - image: "geoloco"
             notebook: "Geoloco_Regression.ipynb"
 
+          - image: "net2cog"
+            notebook: "net2cog_Regression.ipynb"
+
     uses: ./.github/workflows/build-target-image.yml
     with:
       image-short-name: ${{ matrix.targets.image }}

CHANGELOG.md

@@ -5,6 +5,9 @@ versioning. Rather than a static releases, this repository contains of a number
 of regression tests that are each semi-independent.  This CHANGELOG file should be used
 to document pull requests to this repository.
 
+## 2024-08-30
+
+Add regression test for net2cog
 
 ## 2024-08-05 ([#86](https://github.com/nasa/harmony-regression-tests/pull/86))
 

script/test-in-bamboo.sh

@@ -42,7 +42,7 @@ echo "harmony host url: ${harmony_host_url}"
 ## e.g. if REGRESSION_TESTS_N2Z_IMAGE environment was set, the value would be used instead of the default.
 
 image_names=()
-all_tests=(harmony harmony-regression hoss hga n2z swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco)
+all_tests=(harmony harmony-regression hoss hga n2z swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog)
 for image in "${all_tests[@]}"; do
     image_names+=($(image_name "$image" true))
 done

test/Makefile

@@ -31,4 +31,7 @@ variable-subsetter-image: Dockerfile variable-subsetter/environment.yaml
 geoloco-image: Dockerfile geoloco/environment.yaml
 	docker build -t ghcr.io/nasa/regression-tests-geoloco:latest -f ./Dockerfile --build-arg notebook=Geoloco_Regression.ipynb --build-arg sub_dir=geoloco .
 
-images: harmony-image harmony-regression-image hga-image hoss-image hybig-image n2z-image regridder-image swath-projector-image trajectory-subsetter-image variable-subsetter-image geoloco-image
+net2cog-image: Dockerfile net2cog/environment.yaml
+	docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile --build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog .
+
+images: harmony-image harmony-regression-image hga-image hoss-image hybig-image n2z-image regridder-image swath-projector-image trajectory-subsetter-image variable-subsetter-image geoloco-image net2cog-image

test/net2cog/environment.yaml

@@ -0,0 +1,14 @@
+name: papermill-net2cog
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - python=3.10
+  - notebook=6.5.4
+  - papermill=2.3.4
+  - rasterio=1.3.7
+  - rio-cogeo=5.3.4
+  - numpy=1.24.3
+  - pip=23.1.2
+  - pip:
+    - harmony-py==0.4.8

test/net2cog/net2cog_Regression.ipynb

@@ -0,0 +1,288 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "32af63de",
+   "metadata": {},
+   "source": [
+    "# net2cog regression tests\n",
+    "\n",
+    "This Jupyter notebook runs a suite of regression tests against the net2cog Harmony Service.\n",
+    "\n",
+    "These tests use [SMAP_RSS_L3_SSS_SMI_8DAY-RUNNINGMEAN_V4](https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234410736-POCLOUD) as netcdf input data to test the net2cog service for the smap_sss variable.\n",
+    "\n",
+    "## Set the Harmony environment:\n",
+    "\n",
+    "The cell below sets the `harmony_host_url` to one of the following valid values:\n",
+    "\n",
+    "* Production: <https://harmony.earthdata.nasa.gov>\n",
+    "* UAT: <https://harmony.uat.earthdata.nasa.gov>\n",
+    "* SIT: <https://harmony.sit.earthdata.nasa.gov>\n",
+    "* Local: <http://localhost:3000>\n",
+    "\n",
+    "The default value is for the UAT environment. When using this notebook there are two ways to use the non-default environment:\n",
+    "\n",
+    "* Run this notebook in a local Jupyter notebook server and change the value of `harmony_host_url` in the cell below to the value for the environment you require from the above list.\n",
+    "\n",
+    "* Use the `run_notebooks.sh` script, which requires you to declare an environment variable `HARMONY_HOST_URL`. Set that environment variable to the value above that corresponds to the environment you want to test. That environment variable will take precedence over the default value in the cell below."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dec3bc66",
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e969d81",
+   "metadata": {},
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-net2cog`\n",
+    "\n",
+    "A `.netrc` file must also be located in the `test` directory of this repository."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "802241b5",
+   "metadata": {},
+   "source": [
+    "### Import required packages:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "295b8341",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "from tempfile import TemporaryDirectory\n",
+    "\n",
+    "from harmony import BBox, Collection, Environment, Client, Request\n",
+    "from harmony.harmony import ProcessingFailedException\n",
+    "from numpy.testing import assert_array_almost_equal\n",
+    "import rasterio\n",
+    "from rasterio.transform import Affine\n",
+    "from rasterio.crs import CRS\n",
+    "\n",
+    "import utility\n",
+    "\n",
+    "\n",
+    "reference_dir = Path('./reference_data')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9b644811",
+   "metadata": {},
+   "source": [
+    "### Set up environment dependent variables:\n",
+    "\n",
+    "This includes the Harmony `Client` object and `Collection` objects for each of the collections for which there are regression tests. The local, SIT and UAT Harmony instances all utilise resources from CMR UAT, meaning any non-production environment will use the same resources.\n",
+    "\n",
+    "When adding a production entry to the dictionary below, the collection instances can be included directly in the production dictionary entry, as they do not need to be shared."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "437af5f8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "non_production_collection = {\n",
+    "    'smap_collection': Collection(id='C1234410736-POCLOUD'),\n",
+    "}\n",
+    "\n",
+    "non_prod_granule_data = {\n",
+    "    'smap_granules': ['G1234601650-POCLOUD'],\n",
+    "}\n",
+    "\n",
+    "collection_data = {\n",
+    "    'https://harmony.uat.earthdata.nasa.gov': {\n",
+    "        **non_production_collection,\n",
+    "        **non_prod_granule_data,\n",
+    "        'env': Environment.UAT,\n",
+    "    },\n",
+    "    'https://harmony.sit.earthdata.nasa.gov': {\n",
+    "        **non_production_collection,\n",
+    "        **non_prod_granule_data,\n",
+    "        'env': Environment.SIT,\n",
+    "    },\n",
+    "    'http://localhost:3000': {\n",
+    "        **non_production_collection,\n",
+    "        **non_prod_granule_data,\n",
+    "        'env': Environment.LOCAL,\n",
+    "    },\n",
+    "}\n",
+    "\n",
+    "environment_information = collection_data.get(harmony_host_url)\n",
+    "\n",
+    "if environment_information is not None:\n",
+    "    harmony_client = Client(env=environment_information['env'])\n",
+    "    endpoint_url = environment_information.get('endpoint_url', None)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd8d6cb3",
+   "metadata": {},
+   "source": [
+    "## Test conversion of sss_smap variable\n",
+    "\n",
+    "Use SMAP data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cc7d75c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if environment_information is not None:\n",
+    "\n",
+    "    smap_request = Request(\n",
+    "        collection=environment_information['smap_collection'],\n",
+    "        granule_id=environment_information['smap_granules'][0],\n",
+    "        variables=['sss_smap'],\n",
+    "        max_results=1,\n",
+    "        format='image/tiff',\n",
+    "    )\n",
+    "    print(harmony_client.request_as_curl(smap_request))\n",
+    "\n",
+    "    smap_job_id = harmony_client.submit(smap_request)\n",
+    "    harmony_client.wait_for_processing(smap_job_id, show_progress=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22348be6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with TemporaryDirectory() as temp_dir:\n",
+    "    downloaded_cog_outputs = [\n",
+    "        file_future.result()\n",
+    "        for file_future in harmony_client.download_all(\n",
+    "            smap_job_id, overwrite=True, directory=temp_dir\n",
+    "        )\n",
+    "    ]\n",
+    "\n",
+    "    for cog_file in downloaded_cog_outputs:\n",
+    "        utility.validate_cog(cog_file)\n",
+    "\n",
+    "        expected_metadata = {\n",
+    "            'driver': 'GTiff',\n",
+    "            'dtype': 'float32',\n",
+    "            'nodata': -9999.0,\n",
+    "            'width': 1440,\n",
+    "            'height': 720,\n",
+    "            'count': 1,\n",
+    "            'crs': CRS.from_epsg(4326),\n",
+    "            'transform': Affine(0.25, 0.0, 0.0, 0.0, 0.25, -90.0),\n",
+    "        }\n",
+    "        reference_file = Path(\n",
+    "            './reference_data',\n",
+    "            'RSS_smap_SSS_L3_8day_running_2020_005_FNL_v04.0_converted_sss_smap.tiff',\n",
+    "        )\n",
+    "\n",
+    "        utility.assert_dataset_produced_correct_results(\n",
+    "            cog_file, expected_metadata, reference_file\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1edc6d53",
+   "metadata": {},
+   "source": [
+    "## Test conversion of ALL variables FAILS\n",
+    "\n",
+    "net2cog only supports conversion of a single variable within a netcdf file. This tests that an appropriate error message is shown if more than one variable is specified as input."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa3ec02c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if environment_information is not None:\n",
+    "\n",
+    "    smap_request = Request(\n",
+    "        collection=environment_information['smap_collection'],\n",
+    "        granule_id=environment_information['smap_granules'][0],\n",
+    "        variables=['all'],\n",
+    "        max_results=1,\n",
+    "        format='image/tiff',\n",
+    "    )\n",
+    "    print(harmony_client.request_as_curl(smap_request))\n",
+    "\n",
+    "    smap_job_id = harmony_client.submit(smap_request)\n",
+    "    raised_expected_error = False\n",
+    "    try:\n",
+    "        harmony_client.wait_for_processing(smap_job_id, show_progress=True)\n",
+    "    except ProcessingFailedException as error:\n",
+    "        assert (\n",
+    "            \"net2cog harmony adapter currently only supports processing one variable at a time\"\n",
+    "            in str(error)\n",
+    "        )\n",
+    "        raised_expected_error = True\n",
+    "        print(error)\n",
+    "\n",
+    "    assert (\n",
+    "        raised_expected_error\n",
+    "    ), 'Expected request to raise an exception but it did not.'\n",
+    "    utility.print_success('All variables raised expected error')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0caf714",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Tags",
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/net2cog/utility.py

@@ -0,0 +1,47 @@
+"""Simple utility functions used in the net2cog test notebook."""
+
+from pathlib import Path
+import rasterio
+import subprocess
+from numpy.testing import assert_array_almost_equal
+
+
+def print_error(error_string: str) -> str:
+    """Print an error, with formatting for red text."""
+    print(f'\033[91m{error_string}\033[0m')
+
+
+def print_success(success_string: str) -> str:
+    """Print a success message, with formatting for green text."""
+    print(f'\033[92mSuccess: {success_string}\033[0m')
+
+
+def assert_dataset_produced_correct_results(
+    generated_file: Path, expected_metadata: dict, reference_file: Path
+) -> None:
+    """Check that the generated data matches the expected data."""
+    with rasterio.open(generated_file) as test_dataset:
+        assert (
+            test_dataset.meta == expected_metadata
+        ), f'output has incorrect metadata: {test_dataset.meta}'
+        print_success('Generated image has correct metadata.')
+
+        with rasterio.open(reference_file) as reference_dataset:
+            ref_image = reference_dataset.read()
+            test_image = test_dataset.read()
+            assert_array_almost_equal(ref_image, test_image)
+
+    print_success('Generated image contains correct data.')
+
+
+def validate_cog(path: Path) -> None:
+    cogtif_val = ['rio', 'cogeo', 'validate', f'{path}']
+
+    process = subprocess.run(
+        cogtif_val, check=True, stdout=subprocess.PIPE, universal_newlines=True
+    )
+    cog_test = process.stdout
+    cog_test = cog_test.replace("\n", "")
+
+    valid_cog = f"{path} is a valid cloud optimized GeoTIFF"
+    assert cog_test == valid_cog

test/net2cog/version.txt

@@ -0,0 +1 @@
+0.1.0