Update pre-commit and github workflows (#111)

Signed-off-by: Fabrice Normandin <[email protected]>
mila-iqia · Feb 13, 2025 · e9114ec · e9114ec
1 parent 69ed612
commit e9114ec
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 85 deletions.
diff --git a/.github/actions-runner-job.sh b/.github/actions-runner-job.sh
@@ -3,7 +3,7 @@
 #SBATCH --ntasks=1
 #SBATCH --cpus-per-task=4
 #SBATCH --mem=32G
-#SBATCH --gpus=rtx8000:1
+#SBATCH --gpus=1
 #SBATCH --time=00:30:00
 #SBATCH --dependency=singleton
 #SBATCH --output=logs/runner_%j.out

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -25,27 +25,27 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: "3.10"
-      - run: pip install 'pre-commit<4.0.0'
-      - run: pre-commit --version
-      - run: pre-commit install
-      - run: pre-commit run --all-files --show-diff-on-failure
+          version: "latest"
+          # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - run: uvx pre-commit run --all-files --show-diff-on-failure
 
+  # todo: turn off in the projects created from the template.
   check_docs:
     needs: [linting]
     runs-on: ubuntu-latest
+    continue-on-error: true
     steps:
       - uses: actions/checkout@v4
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
-          enable-cache: true
           # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-suffix: "3.10"
       - name: Install dependencies
         run: uv sync --frozen --extra docs
       - name: Build the documentation (strict mode)
@@ -62,63 +62,24 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
           version: "latest"
-          enable-cache: true
+          python-version: ${{ matrix.python-version }}
           # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
           github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-suffix: ${{ matrix.python-version }}
-      - name: Pin python-version ${{ matrix.python-version }}
-        run: uv python pin ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: uv sync --frozen
       - name: Test with pytest
         env:
           JAX_PLATFORMS: cpu
-        run: uv run pytest -v --cov=project --cov-report=xml --cov-append --skip-if-files-missing
+        run: uv run --frozen pytest -v --cov=project --cov-report=xml --cov-append --gen-missing
 
       - name: Store coverage report as an artifact
         uses: actions/upload-artifact@v4
         with:
           name: coverage-reports-unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
           path: ./coverage.xml
 
-  # local_integration_tests:
-  #   needs: [unit_tests, check_docs]
-  #   runs-on: self-hosted
-  #   timeout-minutes: 30
-  #   strategy:
-  #     max-parallel: 1
-  #     matrix:
-  #       python-version: ["3.10"]
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install the latest version of uv
-  #       uses: astral-sh/setup-uv@v3
-  #       with:
-  #         version: "latest"
-  #         enable-cache: false  # no need, uses the local uv cache.
-  #         # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
-  #         github-token: ${{ secrets.GITHUB_TOKEN }}
-
-  #     - name: Pin python-version ${{ matrix.python-version }}
-  #       run: uv python pin ${{ matrix.python-version }}
-
-  #     - name: Install dependencies
-  #       run: uv sync --all-extras --frozen
-
-  #     - name: Test with pytest
-  #       run: uv run pytest -v --cov=project --cov-report=xml --cov-append
-
-  #     - name: Test with pytest (only slow tests)
-  #       run: uv run pytest -v -m slow --slow --cov=project --cov-report=xml --cov-append
-
-  #     - name: Store coverage report as an artifact
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: coverage-reports-integration-tests-${{ matrix.python-version }}
-  #         path: ./coverage.xml
 
   launch-slurm-actions-runner:
     needs: [unit_tests, check_docs]
@@ -129,19 +90,27 @@ jobs:
       matrix:
         cluster: ["mila"] #, 'narval', 'beluga']
     outputs:
+      # note: doesn't really work. Not that big a deal. Would have been nice to show the jobid in
+      # the name of the next job.
       job_id: ${{ steps.sbatch.outputs.stdout }}
     steps:
       - uses: actions/checkout@v4
       - name: Copy job script to the cluster
+        # note: The script will be overwritten by different CI runs, but it shouldn't really
+        # change, so not a big deal.
+        # todo: there are some assumptions about the GPU type to use in that script.
         run: "scp .github/actions-runner-job.sh ${{ matrix.cluster }}:actions-runner-job.sh"
 
       - name: Launch Slurm Actions Runner
         id: sbatch
-        # TODO: for DRAC clusters, the account needs to be set somehow (and obviously not be hard-coded here).
+        # TODO: for DRAC clusters, the account would need to be passed somehow (and obviously not be hard-coded here).
+        # Maybe the account name on DRAC could be stored as a GitHub secret? But this is fishy.
         # Output the job ID to a file so that the next step can use it.
         # NOTE: Could also use the --wait flag to wait for the job to finish (and have this run at the same time as the other step).
+        # TODO: Hard-coded mila-specific GPU to use for the tests.
+        # This isn't great, but currently necessary for reproducibility tests.
         run: |
-          job_id=`ssh ${{ matrix.cluster }} 'cd $SCRATCH && sbatch --parsable $HOME/actions-runner-job.sh'`
+          job_id=`ssh ${{ matrix.cluster }} 'cd $SCRATCH && sbatch --gpus=rtx8000:1 --parsable $HOME/actions-runner-job.sh'`
           echo "Submitted job $job_id on the ${{ matrix.cluster }} cluster!"
           echo "job_id=$job_id" >> "$GITHUB_OUTPUT"
 
@@ -160,6 +129,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up the repo using the setup script
+        # note: this script is idempotent (running twice is fine).
         run: scripts/mila_setup.sh
       - name: Install dependencies
         run: uv sync --all-extras --frozen
@@ -170,7 +140,7 @@ jobs:
 
       # TODO: Disabling full regression tests on the cluster for now, because the worker is often
       # interrupted and we want to avoid using the unkillable partition to not interrupt other's
-      # work. The full regression tests are still run on the local_integration_tests job.
+      # work. The full regression tests can be run manually with the `--slow` flag.
       # - name: Test with pytest (only slow tests)
       #   run: rye run pytest -v -m slow --slow --cov=project --cov-report=xml --cov-append
 

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -15,7 +15,6 @@ jobs:
         uses: astral-sh/setup-uv@v3
         with:
           version: "latest"
-          enable-cache: true  # no need, uses the local uv cache.
           # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
           github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-suffix: "3.10"
@@ -28,8 +27,3 @@ jobs:
 
       - name: Deploy docs
         run: uv run mkdocs gh-deploy --force
-        # note: Checking if we really need the one below:
-        # uses: mhausenblas/[email protected]
-        # # Or use mhausenblas/mkdocs-deploy-gh-pages@nomaterial to build without the mkdocs-material theme
-        # env:
-        #   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -40,10 +40,12 @@ repos:
 
   # python docstring formatting
   - repo: https://github.com/myint/docformatter
-    rev: v1.7.5
+    # Don't autoupdate until https://github.com/PyCQA/docformatter/issues/293 is fixed
+    rev: eb1df347edd128b30cd3368dddc3aa65edcfac38
     hooks:
       - id: docformatter
         language: python
+        additional_dependencies: [tomli]
         args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99]
         require_serial: true
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -48,7 +48,7 @@ requires-python = ">= 3.10"
 [dependency-groups]
 dev = [
     "mktestdocs>=0.2.2",
-    "pre-commit<4.0.0",
+    "pre-commit>=4.1.0",
     "pytest-benchmark>=4.0.0",
     "pytest-cov>=5.0.0",
     "pytest-env>=1.1.3",

diff --git a/uv.lock b/uv.lock