DAS-2214 Add NSIDC IceSAT2 Regression Tests (#92)

Co-authored-by: Amy Steiker <[email protected]>

.gitattributes

@@ -0,0 +1 @@
+test/nsidc-icesat2/reference_files/*.h5 filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-all-images.yml

@@ -29,6 +29,11 @@ jobs:
           -
             image: "n2z"
             notebook: "N2Z_Regression.ipynb"
+          -
+            image: "nsidc-icesat2"
+            notebook: "NSIDC-ICESAT2_Regression.ipynb"
+            shared-utils: "true"
+            lfs: "true"
           -
             image: "regridder"
             notebook: "Regridder_Regression.ipynb"
@@ -41,14 +46,16 @@ jobs:
           -
             image: "variable-subsetter"
             notebook: "VariableSubsetter_Regression.ipynb"
-
-          - image: "geoloco"
+          -
+            image: "geoloco"
             notebook: "Geoloco_Regression.ipynb"
-
-          - image: "net2cog"
+          -
+            image: "net2cog"
             notebook: "net2cog_Regression.ipynb"
 
     uses: ./.github/workflows/build-target-image.yml
     with:
       image-short-name: ${{ matrix.targets.image }}
       notebook-name: ${{ matrix.targets.notebook }}
+      shared-utils: ${{ matrix.targets.shared-utils || 'false' }}
+      lfs: ${{ matrix.targets.lfs || '' }}

.github/workflows/build-target-image.yml

@@ -13,6 +13,12 @@ on:
       notebook-name:
         required: true
         type: string
+      shared-utils:
+        required: true
+        type: string
+      lfs:
+        required: true
+        type: string
 
 env:
   REGISTRY: ghcr.io
@@ -22,10 +28,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout regression test repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          lfs: ${{inputs.lfs}}
 
       - name: Did this image's version change?
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           filters: |
@@ -54,13 +62,12 @@ jobs:
           tags: |
             type=semver,pattern={{version}},value=${{ env.semantic_version }}
 
-      # https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381
       - name: Set up Docker Buildx
         if: steps.changes.outputs.src == 'true'
         uses: docker/setup-buildx-action@v2
         with:
           driver-opts: |
-            image=moby/buildkit:v0.10.6
+            image=moby/buildkit:latest
 
       - name: Build and Push Docker image
         uses: docker/build-push-action@v3
@@ -69,6 +76,8 @@ jobs:
           build-args: |
               notebook=${{inputs.notebook-name}}
               sub_dir=${{inputs.image-short-name}}
+              shared_utils=${{inputs.shared-utils}}
+
           file: ./test/Dockerfile
           context: ./test
           push: true

CHANGELOG.md

@@ -5,14 +5,23 @@ versioning. Rather than a static releases, this repository contains of a number
 of regression tests that are each semi-independent.  This CHANGELOG file should be used
 to document pull requests to this repository.
 
-## 2024-08-30
+## 2024-09-24 ([#92](https://github.com/nasa/harmony-regression-tests/pull/92))
+
+- Adds NSIDC ICESat2 Regression test suite.
+
+- Adds `shared_utils` functionality. This directory contains routines that are commonly used in regression tests and limits code duplication. To include the `shared_utils` directory in your docker container, update the `Makefile` to add a shared_utils build arg. E.g. `--build-arg shared_utils=true` and update the `.github/workflows/build-all-images.yml` to add a `shared-utils` key of "true" (see the nsidc-icesat2-image target in each file)
+
+- Adds Git LFS functionality. Large files can be configured to use [Git LFS](https://git-lfs.com/). This PR configures the NSIDC reference files  `test/nsidc-icesat2/reference_files/*.h5`.
+
+
+## 2024-08-30 ([#94](https://github.com/nasa/harmony-regression-tests/pull/94))
 
 Add regression test for net2cog
 
 ## 2024-08-05 ([#86](https://github.com/nasa/harmony-regression-tests/pull/86))
 
-Adds this file to capture changes to the repository.
+- Adds this file to capture changes to the repository.
 
-Adds pre-commit.ci behavior to the repository. This setup ensures consistent code style, catches common errors, and maintains file hygiene across the project.
+- Adds pre-commit.ci behavior to the repository. This setup ensures consistent code style, catches common errors, and maintains file hygiene across the project.
 
-Updates the base image for all regression tests to `mambaorg/micromamba:1.5.8-jammy`
+- Updates the base image for all regression tests to `mambaorg/micromamba:1.5.8-jammy`

README.md

@@ -90,12 +90,20 @@ environment before installing from the environment.yml.
    is simplest to use the same string for the subdirectory name and the suite
    name.
 1. Update the `test/Makefile` to be able to build a Docker image for the new
-   test suite:
+   test suite optionally including the shared utility directory:
 
    ```
    <new-suite-name>-image
-       docker build -t ghcr.io/nasa/regression-tests-<new-suite-name>:latest -f ./Dockerfile --build-arg notebook=<new-test-notebook-name> --build-arg sub_dir=<new-suite-subdirectory> .
+       docker build -t ghcr.io/nasa/regression-tests-<new-suite-name>:latest -f ./Dockerfile --build-arg notebook=<new-test-notebook-name> --build-arg sub_dir=<new-suite-subdirectory> [--build-arg shared_utils=true] .
    ```
+
+1. If you would like to use shared utilities to help ease the coding you can
+   add the shared_util build-arg to your docker build command in the Makefile
+   (as well as adding it as a key in the `workflow/build-all-images.yml` file).
+   When enabed, this argument will include the `tests/shared_utils` directory
+   as a sibling directory to your tests.  See the
+   `tests/shared_utils/README.md` file for more information.
+
 1. Update the `make images` rule to include building the new image.
 
    ```
@@ -123,12 +131,24 @@ to create a new version of the test image any time the related `version.txt`
 file is updated. To do so, simply add a new target to the
 [build-all-images.yml](https://github.com/nasa/harmony-regression-tests/blob/main/.github/workflows/build-all-images.yml) workflow in the `.github/workflows` directory:
 
-```
+```yaml
 -
   image: <new-suite-name>
   notebook: <new-notebook-name>
 ```
 
+The above is the basic structure for adding a new image to the CI/CD.  Two additional options `shared-utils` and `lfs` default to off, but can be over-ridden as they are for the nsidc-icesat2 image. `shared-utils` controls the addition of the `tests/shared_utils` directory into your image. `lfs` enables git LFS for your image and should be enabled only if you have added reference files with git LFS.
+
+``` yaml
+    -
+      image: "nsidc-icesat2"
+      notebook: "NSIDC-ICESAT2_Regression.ipynb"
+      shared-utils: "true"
+      lfs: "true"
+
+```
+
+
 ## Test suite contents:
 
 This section of the README describes the files that are expected in every test
@@ -145,7 +165,8 @@ For example, in the `swath-projector` directory we have
 ```
 
 * `reference_files` contains golden template files for expected outputs of
-  `tests.
+   `tests`.  When you add new binary files to your test, you should configure
+   them to to use Git LFS as well as keep them as small as possible.
 * `SwathProjector_Regression.ipynb` is the regression test Jupyter notebook
   itself, running tests in cells. A test suite fails when a Jupyter notebook
   cell returns an error from the execution. Each regression test is designed to

script/test-in-bamboo.sh

@@ -42,7 +42,7 @@ echo "harmony host url: ${harmony_host_url}"
 ## e.g. if REGRESSION_TESTS_N2Z_IMAGE environment was set, the value would be used instead of the default.
 
 image_names=()
-all_tests=(harmony harmony-regression hoss hga n2z swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog)
+all_tests=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog)
 for image in "${all_tests[@]}"; do
     image_names+=($(image_name "$image" true))
 done

test/Dockerfile

@@ -3,9 +3,11 @@ USER root
 
 ARG sub_dir
 ARG notebook
+ARG shared_utils=false
 ENV env_sub_dir=$sub_dir
 ENV env_notebook=$notebook
 
+
 ENV AWS_SECRET_ACCESS_KEY=no-access
 ENV AWS_ACCESS_KEY_ID=no-access
 ENV AWS_SESSION_TOKEN=no-access
@@ -23,6 +25,14 @@ COPY ${sub_dir}/environment.yaml ./${sub_dir}
 RUN micromamba create -y -f ${sub_dir}/environment.yaml \
     && micromamba clean --all --force-pkgs-dirs --yes
 
+# Include shared utility functions if requested.  This is a bit awkward, it
+# always copies the shared utils directory to the image, but then deletes it if
+# you didn't want it.
+COPY shared_utils ./shared_utils
+RUN if [ "$shared_utils" = "false" ]; then \
+        rm -rf ./shared_utils; \
+    fi
+
 COPY ${sub_dir} ./${sub_dir}
 
 ENTRYPOINT /bin/bash ./notebook-entrypoint.sh

test/Makefile

@@ -1,37 +1,65 @@
 harmony-image: Dockerfile harmony/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-harmony:latest -f ./Dockerfile --build-arg notebook=Harmony.ipynb --build-arg sub_dir=harmony .
+	docker build -t ghcr.io/nasa/regression-tests-harmony:latest -f ./Dockerfile \
+	--build-arg notebook=Harmony.ipynb --build-arg sub_dir=harmony .
 
 harmony-regression-image: Dockerfile harmony-regression/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-harmony-regression:latest -f ./Dockerfile --build-arg notebook=HarmonyRegression.ipynb --build-arg sub_dir=harmony-regression .
+	docker build -t ghcr.io/nasa/regression-tests-harmony-regression:latest -f ./Dockerfile \
+	--build-arg notebook=HarmonyRegression.ipynb --build-arg sub_dir=harmony-regression .
 
 hga-image: Dockerfile hga/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-hga:latest -f ./Dockerfile --build-arg notebook=HGA_Regression.ipynb --build-arg sub_dir=hga .
+	docker build -t ghcr.io/nasa/regression-tests-hga:latest -f ./Dockerfile \
+	--build-arg notebook=HGA_Regression.ipynb --build-arg sub_dir=hga .
 
 hoss-image: Dockerfile hoss/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-hoss:latest -f ./Dockerfile --build-arg notebook=HOSS_Regression.ipynb --build-arg sub_dir=hoss .
+	docker build -t ghcr.io/nasa/regression-tests-hoss:latest -f ./Dockerfile \
+	--build-arg notebook=HOSS_Regression.ipynb --build-arg sub_dir=hoss .
 
 hybig-image: Dockerfile hybig/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-hybig:latest -f ./Dockerfile --build-arg notebook=HyBIG_Regression.ipynb --build-arg sub_dir=hybig .
+	docker build -t ghcr.io/nasa/regression-tests-hybig:latest -f ./Dockerfile \
+	--build-arg notebook=HyBIG_Regression.ipynb --build-arg sub_dir=hybig .
 
 n2z-image: Dockerfile n2z/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-n2z:latest -f ./Dockerfile --build-arg notebook=N2Z_Regression.ipynb --build-arg sub_dir=n2z .
+	docker build -t ghcr.io/nasa/regression-tests-n2z:latest -f ./Dockerfile \
+	--build-arg notebook=N2Z_Regression.ipynb --build-arg sub_dir=n2z .
+
+nsidc-icesat2-image: Dockerfile nsidc-icesat2/environment.yaml
+	docker build -t ghcr.io/nasa/regression-tests-nsidc-icesat2:latest -f ./Dockerfile \
+	--build-arg notebook=NSIDC-ICESAT2_Regression.ipynb --build-arg sub_dir=nsidc-icesat2 --build-arg shared_utils=true .
 
 regridder-image: Dockerfile regridder/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-regridder:latest -f ./Dockerfile --build-arg notebook=Regridder_Regression.ipynb --build-arg sub_dir=regridder .
+	docker build -t ghcr.io/nasa/regression-tests-regridder:latest -f ./Dockerfile \
+	--build-arg notebook=Regridder_Regression.ipynb --build-arg sub_dir=regridder .
 
 swath-projector-image: Dockerfile swath-projector/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-swath-projector:latest -f ./Dockerfile --build-arg notebook=SwathProjector_Regression.ipynb --build-arg sub_dir=swath-projector .
+	docker build -t ghcr.io/nasa/regression-tests-swath-projector:latest -f ./Dockerfile \
+	--build-arg notebook=SwathProjector_Regression.ipynb --build-arg sub_dir=swath-projector .
 
 trajectory-subsetter-image: Dockerfile trajectory-subsetter/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-trajectory-subsetter:latest -f ./Dockerfile --build-arg notebook=TrajectorySubsetter_Regression.ipynb --build-arg sub_dir=trajectory-subsetter .
+	docker build -t ghcr.io/nasa/regression-tests-trajectory-subsetter:latest -f ./Dockerfile \
+	--build-arg notebook=TrajectorySubsetter_Regression.ipynb --build-arg sub_dir=trajectory-subsetter .
 
 variable-subsetter-image: Dockerfile variable-subsetter/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-variable-subsetter:latest -f ./Dockerfile --build-arg notebook=VariableSubsetter_Regression.ipynb --build-arg sub_dir=variable-subsetter .
+	docker build -t ghcr.io/nasa/regression-tests-variable-subsetter:latest -f ./Dockerfile \
+	--build-arg notebook=VariableSubsetter_Regression.ipynb --build-arg sub_dir=variable-subsetter .
 
 geoloco-image: Dockerfile geoloco/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-geoloco:latest -f ./Dockerfile --build-arg notebook=Geoloco_Regression.ipynb --build-arg sub_dir=geoloco .
+	docker build -t ghcr.io/nasa/regression-tests-geoloco:latest -f ./Dockerfile \
+	--build-arg notebook=Geoloco_Regression.ipynb --build-arg sub_dir=geoloco .
 
 net2cog-image: Dockerfile net2cog/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile --build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog .
+	docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile \
+	--build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog .
 
-images: harmony-image harmony-regression-image hga-image hoss-image hybig-image n2z-image regridder-image swath-projector-image trajectory-subsetter-image variable-subsetter-image geoloco-image net2cog-image
+images: harmony-image \
+	harmony-regression-image \
+	hga-image \
+	hoss-image \
+	hybig-image \
+	n2z-image \
+	nsidc-icesat2-image \
+	regridder-image \
+	swath-projector-image \
+	trajectory-subsetter-image \
+	variable-subsetter-image \
+	geoloco-image \
+	net2cog-image