Merge pull request #211 from nasa/feature/issue-208-increase-test-cov…

…erage

Feature/issue 208 increase test coverage

.github/workflows/run_tests.yml

@@ -36,7 +36,7 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
 
       - name: Install package
-        run: poetry install
+        run: poetry install --with=harmony --without integration
 
       - name: Run linting
         run: |

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [Issue #194](https://github.com/nasa/stitchee/issues/194): Add page about the SAMBAH service chain to the Readthedocs documentation
 ### Changed
 - [Issue #206](https://github.com/nasa/stitchee/issues/206): Group dependabot updates into one PR
+- [issue #208](https://github.com/nasa/stitchee/issues/208): Increase continuous integration/unit test coverage
 ### Deprecated
 ### Removed
 ### Fixed

concatenator/__init__.py

@@ -24,24 +24,9 @@ def __getattr__(name):  # type: ignore
     """
     global _options
 
-    if name == "__options__":
-        return _options
     if name == "group_delim":
         return _options.group_delim
     if name == "coord_delim":
         return _options.coord_delim
     else:
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
-def __setattr__(name, value):  # type: ignore
-    """Module-level setattr to handle setting of `concatenator.options`.
-
-    Other unhandled attributes raise as `AttributeError` as expected.
-    """
-    if name == "group_delim":
-        _options.group_delim = value
-    elif name == "coord_delim":
-        _options.coord_delim = value
-    else:
-        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

concatenator/harmony/cli.py

@@ -1,4 +1,5 @@
 """A Harmony CLI wrapper around the concatenate-batcher"""
+
 from argparse import ArgumentParser
 
 import harmony

concatenator/harmony/download_worker.py

@@ -113,6 +113,6 @@ def _download_worker(
             dest_path = path.parent.joinpath(filename)
             path = path.rename(dest_path)
         else:
-            logger.warning("Origin filename could not be assertained - %s", url)
+            logger.warning("Origin filename could not be ascertained - %s", url)
 
         path_list.append(str(path))

pyproject.toml

@@ -19,28 +19,29 @@ stitchee = 'concatenator.run_stitchee:main'
 
 [tool.poetry.dependencies]
 python = "^3.10"
-netcdf4 = "^1.6.5"
-xarray = "^2024.3.0"
-dask = "^2024.4.1"
+netcdf4 = ">=1.6.5"
+xarray = ">=2024.3.0"
+dask = ">=2024.4.1"
+pystac = ">=0.5.6"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^8.1.1"
-mypy = "^1.9.0"
-black = "^24.4.0"
+pytest = ">=8.1.1"
+mypy = ">=1.9.0"
+black = ">=24.4.0"
 ruff = ">=0.3.7,<0.5.0"
-pytest-cov = "^5.0.0"
-mkdocs = "^1.6.0"
-markdown-callouts = "^0.4.0"
-markdown-include = "^0.8.1"
-mkdocstrings = "^0.25.1"
-mkdocs-jupyter = "^0.24.7"
-mkdocs-material = "^9.5.22"
+pytest-cov = ">=5.0.0"
+mkdocs = ">=1.6.0"
+markdown-callouts = ">=0.4.0"
+markdown-include = ">=0.8.1"
+mkdocstrings = ">=0.25.1"
+mkdocs-jupyter = ">=0.24.7"
+mkdocs-material = ">=9.5.22"
 
 [tool.poetry.group.harmony.dependencies]
-harmony-service-lib = "^1.0.26"
+harmony-service-lib = ">=1.0.26"
 
 [tool.poetry.group.integration.dependencies]
-harmony-py = "^0.4.14"
+harmony-py = ">=0.4.14"
 
 [build-system]
 requires = ["poetry-core"]
@@ -78,3 +79,21 @@ select = [
   "I", # isort
   "UP", # Pyupgrade
 ]
+
+[tool.coverage.run]
+concurrency = ["multiprocessing"]
+
+[tool.coverage.report]
+# Regexes for lines to exclude from consideration
+exclude_lines = [
+    # Have to re-enable the standard pragma
+    "pragma: no cover",
+
+    # Don't complain if tests don't hit defensive assertion code:
+    "raise AssertionError",
+    "raise NotImplementedError",
+
+    # Don't complain if non-runnable code isn't run:
+    "if 0:",
+    "if __name__ == .__main__.:"
+]

tests/conftest.py

@@ -32,7 +32,7 @@ def prep_input_files(input_dir: Path, output_dir: Path) -> list[str]:
     """Prepare input by copying from the original test data directory."""
     input_files = []
     for filepath in input_dir.iterdir():
-        if Path(filepath).suffix.lower() in (".nc", ".h5", ".hdf"):
+        if Path(filepath).suffix.lower() in (".nc", ".nc4", ".h5", ".hdf"):
             copied_input_new_path = output_dir / Path(filepath).name  # type: ignore
             shutil.copyfile(filepath, copied_input_new_path)
             input_files.append(str(copied_input_new_path))

tests/data/harmony/message.json

@@ -0,0 +1,16 @@
+{
+    "sources": [{
+        "collection": "C1234088182-EEDTEST"
+    }],
+    "format": {
+        "mime": "application/x-netcdf4"
+    },
+    "subset": {},
+    "requestId": "00001111-2222-3333-4444-555566667777",
+    "user": "jdoe",
+    "client": "harmony-example",
+    "isSynchronous": false,
+    "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/",
+    "callback": "http://localhost/some-path",
+    "version": "0.10.0"
+}

tests/data/harmony/source/catalog.json

@@ -0,0 +1,30 @@
+{
+  "stac_version": "1.0.0-beta.2",
+  "stac_extensions": [],
+  "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59",
+  "links": [
+    {
+      "rel": "harmony_source",
+      "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST"
+    },
+    {
+      "rel": "item",
+      "href": "./granule_S012G01.json",
+      "type": "application/json",
+      "title": "granule_S012G01"
+    },
+    {
+      "rel": "item",
+      "href": "./granule_S012G02.json",
+      "type": "application/json",
+      "title": "granule_S012G02"
+    },
+    {
+      "rel": "item",
+      "href": "./granule_S012G03.json",
+      "type": "application/json",
+      "title": "granule_S012G03"
+    }
+  ],
+  "description": "CMR Granules for C1234088182-EEDTEST batch 1"
+}

tests/data/harmony/source/catalog0.json

@@ -0,0 +1,30 @@
+{
+  "stac_version": "1.0.0-beta.2",
+  "stac_extensions": [],
+  "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59",
+  "links": [
+    {
+      "rel": "harmony_source",
+      "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST"
+    },
+    {
+      "rel": "item",
+      "href": "./granule_S012G01.json",
+      "type": "application/json",
+      "title": "granule_S012G01"
+    },
+    {
+      "rel": "item",
+      "href": "./granule_S012G02.json",
+      "type": "application/json",
+      "title": "granule_S012G02"
+    },
+    {
+      "rel": "next",
+      "href": "tests/data/harmony/source/catalog1.json",
+      "type": "application/json",
+      "title": "Next page"
+    }
+  ],
+  "description": "CMR Granules for C1234088182-EEDTEST batch 1"
+}

tests/data/harmony/source/catalog1.json

@@ -0,0 +1,24 @@
+{
+  "stac_version": "1.0.0-beta.2",
+  "stac_extensions": [],
+  "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59",
+  "links": [
+    {
+      "rel": "harmony_source",
+      "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST"
+    },
+    {
+      "rel": "item",
+      "href": "./granule_S012G03.json",
+      "type": "application/json",
+      "title": "granule_S012G03"
+    },
+    {
+      "rel": "prev",
+      "href": "tests/data/harmony/source/catalog0.json",
+      "type": "application/json",
+      "title": "Previous page"
+    }
+  ],
+  "description": "CMR Granules for C1234088182-EEDTEST batch 2"
+}

tests/data/harmony/source/granule_S012G01.json

@@ -0,0 +1,49 @@
+{
+  "stac_version": "1.0.0-beta.2",
+  "stac_extensions": [],
+  "id": "51d02b24-d00e-4640-9887-05f98f6b96d8",
+  "type": "Feature",
+  "links": [],
+  "properties": {
+    "start_datetime": "2020-01-02T00:00:00.000Z",
+    "end_datetime": "2020-01-02T23:59:59.000Z"
+  },
+  "bbox": [1, 3, 1, 3],
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [
+          -179.95,
+          -89.95
+        ],
+        [
+          -179.95,
+          89.95
+        ],
+        [
+          179.95,
+          89.95
+        ],
+        [
+          179.95,
+          -89.95
+        ],
+        [
+          -179.95,
+          -89.95
+        ]
+      ]
+    ]
+  },
+  "assets": {
+    "data": {
+      "href": "file://tests/data/harmony/granule_copies_dir/TEMPO_NO2_L2_V03_20240601T210934Z_S012G01_subsetted.nc4",
+      "title": "TEMPO_NO2_L2_V03_20240601T210934Z_S012G01_subsetted.nc4",
+      "type": "application/x-netcdf4",
+      "roles": [
+        "data"
+      ]
+    }
+  }
+}

tests/data/harmony/source/granule_S012G02.json

@@ -0,0 +1,49 @@
+{
+  "stac_version": "1.0.0-beta.2",
+  "stac_extensions": [],
+  "id": "21eb7dc5-7a9d-4374-9a88-12451ba653ef",
+  "type": "Feature",
+  "links": [],
+  "properties": {
+    "start_datetime": "2020-01-03T00:00:00.000Z",
+    "end_datetime": "2020-01-03T23:59:59.000Z"
+  },
+  "bbox": [-1, -3, -1, -3],
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [
+          -179.95,
+          -89.95
+        ],
+        [
+          -179.95,
+          89.95
+        ],
+        [
+          179.95,
+          89.95
+        ],
+        [
+          179.95,
+          -89.95
+        ],
+        [
+          -179.95,
+          -89.95
+        ]
+      ]
+    ]
+  },
+  "assets": {
+    "data": {
+      "href": "file://tests/data/harmony/granule_copies_dir/TEMPO_NO2_L2_V03_20240601T211614Z_S012G02_subsetted.nc4",
+      "title": "TEMPO_NO2_L2_V03_20240601T211614Z_S012G02_subsetted.nc4",
+      "type": "application/x-netcdf4",
+      "roles": [
+        "data"
+      ]
+    }
+  }
+}

tests/data/harmony/source/granule_S012G03.json

@@ -0,0 +1,49 @@
+{
+  "stac_version": "1.0.0-beta.2",
+  "stac_extensions": [],
+  "id": "dc07a25a-54bd-4389-97b6-c1d68cf0d586",
+  "type": "Feature",
+  "links": [],
+  "properties": {
+    "start_datetime": "2020-01-05T00:00:00.000Z",
+    "end_datetime": "2020-01-05T23:59:59.000Z"
+  },
+  "bbox": [-4, -2, -4, 2],
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [
+          -179.95,
+          -89.95
+        ],
+        [
+          -179.95,
+          89.95
+        ],
+        [
+          179.95,
+          89.95
+        ],
+        [
+          179.95,
+          -89.95
+        ],
+        [
+          -179.95,
+          -89.95
+        ]
+      ]
+    ]
+  },
+  "assets": {
+    "data": {
+      "href": "file://tests/data/harmony/granule_copies_dir/TEMPO_NO2_L2_V03_20240601T212254Z_S012G03_subsetted.nc4",
+      "title": "TEMPO_NO2_L2_V03_20240601T212254Z_S012G03_subsetted.nc4",
+      "type": "application/x-netcdf4",
+      "roles": [
+        "data"
+      ]
+    }
+  }
+}