CI and testing infrastructure improvements. (#3896)

* Update micromamba docker image and setup-micromamba action

* Re-lock dependencies and update pre-commit hooks

* Update to pre-commit 4.0.0

* Only try to remove pudl-dev conda env if it exists.

* Use standard 4-core GHA runner

* Re-lock conda dependencies

.github/workflows/pytest.yml

@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 2
 
       - name: Install conda-lock environment with micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev
@@ -78,7 +78,7 @@ jobs:
           fetch-depth: 2
 
       - name: Install conda-lock environment with micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev
@@ -120,9 +120,7 @@ jobs:
           path: .coverage
 
   ci-integration:
-    runs-on:
-      group: large-runner-group
-      labels: ubuntu-latest-4core
+    runs-on: ubuntu-latest
     if: github.event_name == 'workflow_dispatch' || github.event.merge_group
     permissions:
       contents: read
@@ -139,7 +137,7 @@ jobs:
           fetch-depth: 2
 
       - name: Install Conda environment using mamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev
@@ -224,7 +222,7 @@ jobs:
         run: |
           find coverage -type f
       - name: Install Micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           init-shell: bash
           environment-name: coverage

.github/workflows/update-conda-lockfile.yml

@@ -25,12 +25,12 @@ jobs:
         with:
           token: ${{ secrets.PUDL_BOT_PAT }}
       - name: Install Micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-name: conda-lock
           create-args: >-
             python=3.12
-            conda-lock>=2.5.2
+            conda-lock>=2.5.7
             prettier
       - name: Run conda-lock to recreate lockfile from scratch
         run: |

.github/workflows/zenodo-cache-sync.yml

@@ -32,7 +32,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Conda environment using mamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
 
   # Other file formatting, plus common Git mistakes & text file standardization:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files # Don't accidentally commit giant files.
       - id: check-merge-conflict # Watch for lingering merge markers.
@@ -29,7 +29,7 @@ repos:
   # Formatters: hooks that re-write Python & documentation files
   ####################################################################################
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.8
+    rev: v0.6.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -60,7 +60,7 @@ repos:
 
   # Check Github Actions
   - repo: https://github.com/rhysd/actionlint
-    rev: v1.7.2
+    rev: v1.7.3
     hooks:
       - id: actionlint
 
@@ -71,7 +71,7 @@ repos:
     hooks:
       - id: nb-output-clear
         name: nb-output-clear
-        stages: [commit]
+        stages: [pre-commit]
         language: system
         verbose: false
         files: ".*\\.ipynb"
@@ -81,7 +81,7 @@ repos:
 
       - id: unit-tests
         name: unit-tests
-        stages: [commit]
+        stages: [pre-commit]
         language: system
         verbose: false
         pass_filenames: false
@@ -90,7 +90,7 @@ repos:
 
       - id: conda-lock
         name: conda-lock
-        stages: [commit]
+        stages: [pre-commit]
         language: system
         verbose: false
         files: "pyproject.toml"

Makefile

@@ -52,7 +52,10 @@ conda-lock.yml: pyproject.toml
 .PHONY: pudl-dev
 pudl-dev:
 	${mamba} run --name base ${mamba} install --quiet --yes "conda-lock>=2.5.7"
-	${mamba} run --name base ${mamba} env remove --yes --name pudl-dev
+# Only attempt to remove the pudl-dev environment if it already exists.
+	if ${mamba} env list | grep -q pudl-dev; then \
+		${mamba} env remove --quiet --yes --name pudl-dev; \
+	fi
 	${mamba} run --name base conda-lock install \
 		--name pudl-dev \
 		--${mamba} \

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM mambaorg/micromamba:1.5.9
+FROM mambaorg/micromamba:1.5.10
 
 ENV PGDATA=${CONTAINER_HOME}/pgdata