chore(test, release): nightly snapshot release and upgrade testing (#…

…1141)
defenseunicorns · Sep 27, 2024 · b43c30c · b43c30c
1 parent 75cf696
commit b43c30c
Show file tree

Hide file tree

Showing 5 changed files with 385 additions and 142 deletions.
diff --git a/.github/actions/release/action.yaml b/.github/actions/release/action.yaml
@@ -0,0 +1,174 @@
+name: release
+description: "Cut a release of all LeapfrogAI artifacts"
+
+inputs:
+  releaseTag:
+    description: The release tag to be published, cannot be left empty
+    required: true
+  subRepository:
+    description: The sub-repository to publish the artifacts to
+    required: false
+    default: /uds/
+  registry1Username:
+    description: Registry1 Username
+  registry1Password:
+    description: Registry1 Password
+  ghToken:
+    description: GitHub Token
+  chainguardIdentity:
+    description: Chainguard login identity
+
+runs:
+  using: composite
+
+  steps:
+    - name: Setup UDS Environment
+      uses: defenseunicorns/uds-common/.github/actions/setup@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1
+      with:
+        registry1Username: ${{ inputs.registry1Username }}
+        registry1Password: ${{ inputs.registry1Password }}
+        ghToken: ${{ inputs.ghToken }}
+        chainguardIdentity: ${{ inputs.chainguardIdentity }}
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
+
+    - name: Setup Python
+      uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+      with:
+        python-version-file: "pyproject.toml"
+
+    - name: Install Dev Dependencies
+      shell: bash
+      run: |
+        python -m pip install ".[dev]" ".[dev-vllm]" ".[dev-whisper]"
+
+    - name: Build and Publish K3d GPU
+      shell: bash
+      run: |
+        cd packages/k3d-gpu
+        docker build \
+          --platform linux/amd64 \
+          -t ghcr.io/defenseunicorns/leapfrogai/k3d-gpu:${{ inputs.releaseTag }} .
+        docker push ghcr.io/defenseunicorns/leapfrogai/k3d-gpu:${{ inputs.releaseTag }}
+        cd ../..
+
+    - name: Download Python Wheels and Publish Builder Image
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${{ inputs.releaseTag }} --push -f src/leapfrogai_sdk/Dockerfile .
+
+    - name: Install Zarf
+      uses: defenseunicorns/setup-zarf@10e539efed02f75ec39eb8823e22a5c795f492ae #v1.0.1
+
+    - name: Build and Publish API
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 --build-arg LOCAL_VERSION=${{ inputs.releaseTag }} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ inputs.releaseTag }} --push -f packages/api/Dockerfile .
+        docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ inputs.releaseTag }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" .
+
+        zarf package create packages/api --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/api --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-leapfrogai-api-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-leapfrogai-api-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-leapfrogai-api-*.tar.zst
+
+    - name: Build and Publish UI
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${{ inputs.releaseTag }} --push src/leapfrogai_ui
+        docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${{ inputs.releaseTag }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" .
+
+        zarf package create packages/ui --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/ui --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-leapfrogai-ui-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-leapfrogai-ui-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-leapfrogai-ui-*.tar.zst
+
+    - name: Build and Publish Supabase
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${{ inputs.releaseTag }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" .
+
+        zarf package create packages/supabase --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/supabase --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-supabase-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-supabase-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+    - name: Build and Publish Repeater
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 --build-arg LOCAL_VERSION=${{ inputs.releaseTag }} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ inputs.releaseTag }} --push -f packages/repeater/Dockerfile .
+
+        zarf package create packages/repeater --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/repeater --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-repeater-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-repeater-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-repeater-*.tar.zst
+
+    - name: Build and Publish LLaMA-CPP-Python
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 --build-arg LOCAL_VERSION=${{ inputs.releaseTag }} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ inputs.releaseTag }} --push -f packages/llama-cpp-python/Dockerfile .
+
+        zarf package create packages/llama-cpp-python --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/llama-cpp-python --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-llama-cpp-python-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-llama-cpp-python-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-llama-*.tar.zst
+
+    - name: Build and Publish vLLM
+      shell: bash
+      run: |
+        docker buildx build --build-arg LOCAL_VERSION=${{ inputs.releaseTag }} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${{ inputs.releaseTag }} --push -f packages/vllm/Dockerfile .
+
+        zarf package create packages/vllm --set=IMAGE_VERSION=${{ inputs.releaseTag }} --flavor upstream --confirm
+
+        zarf package publish zarf-package-vllm-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-vllm-*.tar.zst
+
+    - name: Build and Publish Text-Embeddings
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 --build-arg LOCAL_VERSION=${{ inputs.releaseTag }} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${{ inputs.releaseTag }} --push -f packages/text-embeddings/Dockerfile .
+
+        zarf package create packages/text-embeddings --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/text-embeddings --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-text-embeddings-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-text-embeddings-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-text-embeddings-*.tar.zst
+
+    - name: Build and Publish Whisper
+      shell: bash
+      run: |
+        docker buildx build --platform amd64,arm64 --build-arg LOCAL_VERSION=${{ inputs.releaseTag }} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${{ inputs.releaseTag }} --push -f packages/whisper/Dockerfile .
+
+        zarf package create packages/whisper --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture amd64 --flavor upstream --confirm
+        zarf package create packages/whisper --set=IMAGE_VERSION=${{ inputs.releaseTag }} --architecture arm64 --flavor upstream --confirm
+
+        zarf package publish zarf-package-whisper-amd64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+        zarf package publish zarf-package-whisper-arm64-${{ inputs.releaseTag }}.tar.zst oci://ghcr.io/defenseunicorns/packages${{ inputs.subRepository }}leapfrogai
+
+        docker image prune -af
+        rm zarf-package-whisper-*.tar.zst
diff --git a/.github/actions/uds-cluster/action.yaml b/.github/actions/uds-cluster/action.yaml
@@ -8,6 +8,8 @@ inputs:
     description: Registry1 Password
   ghToken:
     description: GitHub Token
+  chainguardIdentity:
+    description: Chainguard login identity
 
 runs:
   using: composite
@@ -18,7 +20,7 @@ runs:
         registry1Username: ${{ inputs.registry1Username }}
         registry1Password: ${{ inputs.registry1Password }}
         ghToken: ${{ inputs.ghToken }}
-        udsCliVersion: 0.14.0
+        chainguardIdentity: ${{ inputs.chainguardIdentity }}
 
     - name: Checkout Repo
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

diff --git a/.github/release-please-config.json b/.github/release-please-config.json
@@ -11,6 +11,7 @@
       "versioning": "default",
       "extra-files": [
         "pyproject.toml",
+        ".github/workflows/nightly-snapshot-release.yaml",
         {
           "type": "generic",
           "path": "**/Chart.yaml",

diff --git a/.github/workflows/nightly-snapshot-release.yaml b/.github/workflows/nightly-snapshot-release.yaml
@@ -0,0 +1,197 @@
+name: nightly-snapshot-release
+
+on:
+  schedule:
+    - cron: "0 8 * * *" # Runs daily at 12 AM PST
+  workflow_dispatch: # trigger manually as needed
+  pull_request:
+    types:
+      - opened # default trigger
+      - reopened # default trigger
+      - synchronize # default trigger
+      - ready_for_review # don't run on draft PRs
+      - milestoned # allows us to trigger on bot PRs
+    paths:
+      - .github/workflows/nightly-snapshot-release.yaml
+
+concurrency:
+  group: nightly-snapshot-release-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  # x-release-please-start-version
+  LEAPFROGAI_VERSION: 0.13.0
+  # x-release-please-end
+  SNAPSHOT_VERSION: snapshot-latest
+
+permissions:
+  contents: read
+  packages: write
+  id-token: write # This is needed for OIDC federation.
+
+jobs:
+  snapshot-release:
+    runs-on: ai-ubuntu-big-boy-8-core
+    name: nightly_snapshot_release
+    if: ${{ !github.event.pull_request.draft }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Release LeapfrogAI ${{ env.SNAPSHOT_VERSION }}
+        uses: ./.github/actions/release
+        with:
+          releaseTag: ${{ env.SNAPSHOT_VERSION }}
+          subRepository: /uds/snapshots/
+          registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }}
+          registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }}
+          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }}
+
+  test-snapshot:
+    runs-on: ai-ubuntu-big-boy-8-core
+    name: nightly_test_snapshot
+    if: ${{ !github.event.pull_request.draft }}
+    needs: snapshot-release
+
+    permissions:
+      contents: read
+      packages: write
+      id-token: write # This is needed for OIDC federation.
+
+    steps:
+      - name: Checkout Repo (v${{ env.LEAPFROGAI_VERSION }})
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: v${{ env.LEAPFROGAI_VERSION }}
+
+      - name: Setup Python (v${{ env.LEAPFROGAI_VERSION }})
+        uses: ./.github/actions/python
+
+      - name: Install Dev Dependencies
+        run: |
+          python -m pip install ".[dev]" ".[dev-vllm]" ".[dev-whisper]"
+
+      - name: Setup UDS Cluster
+        uses: ./.github/actions/uds-cluster
+        with:
+          registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }}
+          registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }}
+          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }}
+
+      # This is needed due to delay in tagged release versus
+      # package publishing and the latest versions of each package in the UDS bundle
+      - name: Mutation of the UDS Bundle
+        run: |
+          uds zarf tools yq -i '.metadata.version = "v${{ env.LEAPFROGAI_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml
+
+          uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ env.LEAPFROGAI_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml
+
+      - name: Create and Deploy UDS Bundle (v${{ env.LEAPFROGAI_VERSION }})
+        run: |
+          cd bundles/latest/cpu
+          uds create . --confirm && \
+            uds deploy uds-bundle-leapfrogai-amd64-v${{ env.LEAPFROGAI_VERSION }}.tar.zst --confirm --no-progress && \
+            rm -rf uds-bundle-leapfrogai-amd64-v${{ env.LEAPFROGAI_VERSION }}.tar.zst && \
+            docker system prune -af
+
+      - name: Checkout Repo (main)
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: main
+
+      - name: Print the Commit SHA
+        run: |
+          COMMIT_SHA=$(git rev-parse HEAD)
+          echo "The latest commit on the main branch is: $COMMIT_SHA"
+
+      - name: Install Dev Dependencies (main)
+        run: |
+          python -m pip install ".[dev]" ".[dev-vllm]" ".[dev-whisper]" --force-reinstall --no-cache-dir
+
+      # Set UDS CPU bundle refs and repositories to snapshot-latest
+      - name: Mutation of the UDS Bundle
+        run: |
+          uds zarf tools yq -i '.metadata.version = "${{ env.SNAPSHOT_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml
+
+          uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ env.SNAPSHOT_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml
+
+          uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml
+
+      - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }})
+        run: |
+          cd bundles/latest/cpu
+          uds create . --confirm && \
+            uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \
+            rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \
+            docker system prune -af
+
+      - name: Generate Secrets
+        id: generate_secrets
+        run: |
+          ANON_KEY=$(uds zarf tools kubectl get secret supabase-bootstrap-jwt -n leapfrogai -o jsonpath='{.data.anon-key}' | base64 -d)
+          echo "::add-mask::$ANON_KEY"
+          echo "ANON_KEY=$ANON_KEY" >> $GITHUB_OUTPUT
+          FAKE_PASSWORD=$(cat <(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9!@#$%^&*()_+-=[]{}|;:,.<>?' | head -c 20) <(echo '!@1Aa') | fold -w1 | shuf | tr -d '\n')
+          echo "::add-mask::$FAKE_PASSWORD"
+          echo "FAKE_PASSWORD=$FAKE_PASSWORD" >> $GITHUB_OUTPUT
+          SERVICE_KEY=$(uds zarf tools kubectl get secret -n leapfrogai supabase-bootstrap-jwt -o jsonpath={.data.service-key} | base64 -d)
+          echo "::add-mask::$SERVICE_KEY"
+          echo "SERVICE_KEY=$SERVICE_KEY" >> $GITHUB_OUTPUT
+
+      - name: Verify Secrets
+        run: |
+          echo "FAKE_PASSWORD is set: ${{ steps.generate_secrets.outputs.FAKE_PASSWORD != '' }}"
+          echo "ANON_KEY is set: ${{ steps.generate_secrets.outputs.ANON_KEY != '' }}"
+          echo "SERVICE_KEY is set: ${{ steps.generate_secrets.outputs.SERVICE_KEY != '' }}"
+
+      # Backends
+      - name: Run Backend E2E Tests
+        env:
+          ANON_KEY: ${{ steps.generate_secrets.outputs.ANON_KEY }}
+          SERVICE_KEY: ${{ steps.generate_secrets.outputs.SERVICE_KEY }}
+        run: |
+          python -m pytest -vvv -s ./tests/e2e
+
+      - name: Setup Playwright
+        run: |
+          npm --prefix src/leapfrogai_ui ci
+          npx --prefix src/leapfrogai_ui playwright install
+
+      - name: Run Playwright E2E Tests
+        env:
+          SERVICE_ROLE_KEY: ${{ steps.generate_secrets.outputs.SERVICE_KEY }}
+          FAKE_E2E_USER_PASSWORD: ${{ steps.generate_secrets.outputs.FAKE_PASSWORD }}
+          ANON_KEY: ${{ steps.generate_secrets.outputs.ANON_KEY }}
+        run: |
+          chmod +x ./.github/scripts/createUser.sh
+          ./.github/scripts/createUser.sh
+
+          cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env
+          mkdir -p playwright/auth
+          touch playwright/auth.user.json
+
+          SERVICE_ROLE_KEY=$SERVICE_ROLE_KEY TEST_ENV=CI USERNAME=doug PASSWORD=$FAKE_E2E_USER_PASSWORD PUBLIC_SUPABASE_ANON_KEY=$ANON_KEY DEFAULT_MODEL=llama-cpp-python npm --prefix src/leapfrogai_ui run test:integration:ci
+
+      - name: Archive Playwright Report
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: src/leapfrogai_ui/e2e-report/
+          retention-days: 30
+
+      - name: Get Cluster Debug Information
+        id: debug
+        if: ${{ !cancelled() }}
+        uses: defenseunicorns/uds-common/.github/actions/debug-output@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1
+
+      - name: Get Cluster Debug Information
+        if: ${{ !cancelled() && steps.debug.conclusion == 'success' }}
+        uses: defenseunicorns/uds-common/.github/actions/save-logs@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1