Merge pull request #693 from conda-forge/rerend-dispatch-jobs

feat: rerender in one spot using admin token and VMs for isolation
conda-forge · Sep 30, 2024 · 5c8b91c · 5c8b91c
2 parents b198942 + b9a4154
commit 5c8b91c
Show file tree

Hide file tree

Showing 22 changed files with 2,976 additions and 136 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -52,75 +52,77 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  # FIXME: uncomment this when dispatch work is here
-  # build-and-push-dispatch-container:
-  #   name: build and push dispatch container
-  #   runs-on: "ubuntu-latest"
-  #   needs: tag-and-release
-  #   defaults:
-  #     run:
-  #       shell: bash -leo pipefail {0}
-  #   env:
-  #     IMAGE_NAME: condaforge/conda-forge-webservices-gha-dispatch
-
-  #   steps:
-  #     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
-  #       with:
-  #         fetch-depth: 0
-
-  #     - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1
-  #       with:
-  #         environment-file: conda-lock.yml
-  #         environment-name: webservices
-  #         condarc: |
-  #           show_channel_urls: true
-  #           channel_priority: strict
-  #           channels:
-  #             - conda-forge
-
-  #     - name: set up docker buildx
-  #       uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3
-
-  #     - name: login to docker hub
-  #       uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
-  #       with:
-  #         username: condaforgebot
-  #         password: ${{ secrets.CF_BOT_DH_PASSWORD }}
-
-  #     - name: build docker metadata
-  #       id: meta
-  #       uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5
-  #       with:
-  #         images: ${{ env.IMAGE_NAME }}
-  #         flavor: |
-  #           latest=false
-  #         tags: |
-  #           type=raw,value=${{ needs.tag-and-release.outputs.new_version }}
-  #           type=raw,value=latest
-
-  #     - name: build and push image
-  #       uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6
-  #       with:
-  #         context: .
-  #         push: true
-  #         tags: ${{ steps.meta.outputs.tags }}
-  #         labels: ${{ steps.meta.outputs.labels }}
-
-  #     - name: push README to docker hub
-  #       uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1
-  #       env:
-  #         DOCKER_USER: condaforgebot
-  #         DOCKER_PASS: ${{ secrets.CF_BOT_DH_PASSWORD }}
-  #       with:
-  #         destination_container_repo: ${{ env.IMAGE_NAME }}:${{ needs.tag-and-release.outputs.new_version }}
-  #         provider: dockerhub
-  #         short_description: "conda-forge-webservices image used to power the admin webservices GitHub Actions integrations"
-  #         readme_file: "Dockerfile_README.md"
+  build-and-push-dispatch-container:
+    name: build and push dispatch container
+    runs-on: "ubuntu-latest"
+    needs: tag-and-release
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+    env:
+      IMAGE_NAME: condaforge/conda-forge-webservices-gha-dispatch
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 0
+
+      - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1
+        with:
+          environment-file: conda-lock.yml
+          environment-name: webservices
+          condarc: |
+            show_channel_urls: true
+            channel_priority: strict
+            channels:
+              - conda-forge
+
+      - name: set up docker buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3
+
+      - name: login to docker hub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
+        with:
+          username: condaforgebot
+          password: ${{ secrets.CF_BOT_DH_PASSWORD }}
+
+      - name: build docker metadata
+        id: meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5
+        with:
+          images: condaforge/webservices-dispatch-action
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ needs.tag-and-release.outputs.new_version }}
+            type=raw,value=latest
+
+      - name: build and push image
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6
+        with:
+          context: .
+          file: Dockerfile_wda
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: push README to docker hub
+        uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1
+        env:
+          DOCKER_USER: condaforgebot
+          DOCKER_PASS: ${{ secrets.CF_BOT_DH_PASSWORD }}
+        with:
+          destination_container_repo: condaforge/webservices-dispatch-action:${{ needs.tag-and-release.outputs.new_version }}
+          provider: dockerhub
+          short_description: "conda-forge-webservices image used to power the admin webservices GitHub Actions integrations"
+          readme_file: "Dockerfile_wda_README.md"
 
   build-and-push-to-heroku:
     name: build and push to heroku
     runs-on: "ubuntu-latest"
-    needs: tag-and-release  # FIXME: change to build-and-push-dispatch-container when we do that
+    needs:
+      - tag-and-release
+      - build-and-push-dispatch-container
 
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -20,6 +20,8 @@ jobs:
       cancel-in-progress: true
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 0
 
       - name: setup conda
         uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1
@@ -42,13 +44,17 @@ jobs:
           owner: ${{ github.repository_owner }}
 
       - name: install code
+        id: install-code
         shell: bash -l {0}
         run: |
           git config --global user.email "79913779+conda-forge-curator[bot]@users.noreply.github.com"
           git config --global user.name "conda-forge-curator[bot]"
           git config --global pull.rebase false
           mkdir -p ~/.conda-smithy/ && echo $GH_TOKEN > ~/.conda-smithy/github.token
           pip install --no-deps -e .
+
+          version=$(python -c "import conda_forge_webservices; print(conda_forge_webservices.__version__.replace('+', '.'))")
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
         env:
           GH_TOKEN: ${{ steps.generate_token.outputs.token }}
 
@@ -77,15 +83,37 @@ jobs:
           CF_WEBSERVICES_FEEDSTOCK_PRIVATE_KEY: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
           ACTION_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
 
-  live-tests:
-    name: live-tests
+      - name: set up docker buildx
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'conda-forge/conda-forge-webservices'
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3
+
+      - name: login to docker hub
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'conda-forge/conda-forge-webservices'
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
+        with:
+          username: condaforgebot
+          password: ${{ secrets.CF_BOT_DH_PASSWORD }}
+
+      - name: build and push docker image
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'conda-forge/conda-forge-webservices'
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v5
+        with:
+          context: .
+          file: Dockerfile_wda
+          push: true
+          tags: condaforge/webservices-dispatch-action:${{ steps.install-code.outputs.version }}
+
+  live-tests-upload:
+    name: live-tests-upload
     runs-on: "ubuntu-latest"
     needs: tests
     concurrency:
-      group: ${{ github.event.pull_request.head.repo.fork != 'true' && 'live-tests' || format('{0}-{1}', github.workflow, github.ref) }}
+      group: ${{ github.event.pull_request.head.repo.fork != 'true' && 'live-tests-upload' || format('{0}-{1}', github.workflow, github.ref) }}
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
         if: ${{ !github.event.pull_request.head.repo.fork }}
+        with:
+          fetch-depth: 0
 
       - name: setup conda
         if: ${{ !github.event.pull_request.head.repo.fork }}
@@ -134,3 +162,67 @@ jobs:
           CF_WEBSERVICES_APP_ID: ${{ secrets.CF_CURATOR_APP_ID }}
           CF_WEBSERVICES_PRIVATE_KEY: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
           ACTION_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
+
+  live-tests-rerender:
+    name: live-tests-rerender
+    runs-on: "ubuntu-latest"
+    needs: tests
+    concurrency:
+      group: ${{ github.event.pull_request.head.repo.fork != 'true' && 'live-tests-rerender' || format('{0}-{1}', github.workflow, github.ref) }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        with:
+          fetch-depth: 0
+
+      - name: setup conda
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1
+        with:
+          environment-file: conda-lock.yml
+          environment-name: webservices
+          condarc: |
+            show_channel_urls: true
+            channel_priority: strict
+            channels:
+              - conda-forge
+
+      - name: generate token
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        id: generate_token
+        uses: actions/create-github-app-token@31c86eb3b33c9b601a1f60f98dcbfd1d70f379b4 # v1
+        with:
+          app-id: ${{ secrets.CF_CURATOR_APP_ID }}
+          private-key: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: install code
+        shell: bash -l {0}
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        run: |
+          git config --global user.email "79913779+conda-forge-curator[bot]@users.noreply.github.com"
+          git config --global user.name "conda-forge-curator[bot]"
+          git config --global pull.rebase false
+          mkdir -p ~/.conda-smithy/ && echo $GH_TOKEN > ~/.conda-smithy/github.token
+          pip install --no-deps -e .
+        env:
+          GH_TOKEN: ${{ steps.generate_token.outputs.token }}
+
+      - name: run rerender tests
+        shell: bash -l {0}
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        run: |
+          if [[ "${GITHUB_HEAD_REF}" != "" ]]; then
+            branch="${GITHUB_HEAD_REF}"
+          else
+            branch="${GITHUB_REF_NAME}"
+          fi
+
+          version=$(python -c "import conda_forge_webservices; print(conda_forge_webservices.__version__.replace('+', '.'))")
+          export CF_FEEDSTOCK_OPS_CONTAINER_NAME=condaforge/webservices-dispatch-action
+          export CF_FEEDSTOCK_OPS_CONTAINER_TAG="${version}"
+
+          cd tests
+          pytest -vvs --branch=${branch} test_live_rerender.py
+        env:
+          GH_TOKEN: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
diff --git a/.github/workflows/webservices-workflow-dispatch.yml b/.github/workflows/webservices-workflow-dispatch.yml
@@ -15,6 +15,10 @@ on:
         description: 'the pull request number'
         required: true
         type: string
+      container_tag:
+        description: 'the container tag to use'
+        required: true
+        type: string
 
 env:
   PY_COLORS: 1
@@ -33,6 +37,7 @@ jobs:
       - name: checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
+          fetch-depth: 0
           ref: ${{ github.ref }}
 
       - name: setup conda
@@ -48,10 +53,16 @@ jobs:
 
       - name: install code
         run: |
-          pip install --no-deps --no-build-backend -e .
+          pip install --no-deps --no-build-isolation -e .
 
       - name: run task
         run: |
+          git config --global user.name "conda-forge-webservices[bot]"
+          git config --global user.email "91080706+conda-forge-webservices[bot]@users.noreply.github.com"
+
+          export CF_FEEDSTOCK_OPS_CONTAINER_NAME=condaforge/webservices-dispatch-action
+          export CF_FEEDSTOCK_OPS_CONTAINER_TAG="${{ inputs.container_tag }}"
+
           conda-forge-webservices-run-task \
             --task=${{ inputs.task }} \
             --repo=${{ inputs.repo }} \
@@ -76,6 +87,7 @@ jobs:
       - name: checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
+          fetch-depth: 0
           ref: ${{ github.ref }}
 
       - name: setup conda
@@ -91,7 +103,7 @@ jobs:
 
       - name: install code
         run: |
-          pip install --no-deps --no-build-backend -e .
+          pip install --no-deps --no-build-isolation -e .
 
       - name: download task data
         uses: actions/download-artifact@v4
@@ -101,6 +113,12 @@ jobs:
 
       - name: finalize task
         run: |
+          git config --global user.name "conda-forge-webservices[bot]"
+          git config --global user.email "91080706+conda-forge-webservices[bot]@users.noreply.github.com"
+
+          export CF_FEEDSTOCK_OPS_CONTAINER_NAME=condaforge/webservices-dispatch-action
+          export CF_FEEDSTOCK_OPS_CONTAINER_TAG="${{ inputs.container_tag }}"
+
           conda-forge-webservices-finalize-task \
             --task-data-dir=${{ github.workspace }}/task-data
         env:

diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@ recipe/*
 .ruff_cache/
 built_dists/
 conda_forge_webservices/_version.py
+recipe/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@ repos:
   rev: v0.6.3
   hooks:
     - id: ruff
-      args: [ --fix ]    
+      args: [ --fix ]
     - id: ruff-format
 - repo: https://github.com/pre-commit/mirrors-mypy
   rev: v1.11.2
@@ -15,6 +15,7 @@ repos:
     - types-python-dateutil
     - types-cachetools
     - types-mock
+    - types-pyyaml
 ci:
     autofix_commit_msg: |
         [pre-commit.ci] auto fixes from pre-commit.com hooks