diff --git a/.github/scripts/uds_verification_report.py b/.github/scripts/uds_verification_report.py new file mode 100755 index 000000000..0e4d4e8fe --- /dev/null +++ b/.github/scripts/uds_verification_report.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +import os +import re + + +def remove_ansi_escape_sequences(text): + ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]") + return ansi_escape.sub("", text) + + +# Capabilities that affect the entire capability, not just a single package +def uds_capability_wide_errors(text: str) -> bool: + if "Not all pods have the istio sidecar" in text: + return True + return False + + +# CI environment variable enables GitHub annotations +def print_package_info( + package_name, + failures_count, + errors_count, + warnings_count, + failure_descriptions, + error_descriptions, + warning_descriptions, + uds_capability_wide_errors_count, +): + if uds_capability_wide_errors_count >= 1: + errors_count -= uds_capability_wide_errors_count + if package_name: + print("-----------------------------") + if os.getenv("CI") == "true": + print(f"::group::{package_name}") + print(f"Package: {package_name}\n") + if failures_count > 0: + if os.getenv("CI") == "true": + print("::error::", end="") + print(f"⛔ Failures: {failures_count}") + else: + if errors_count > 0: + if os.getenv("CI") == "true": + print("::error::", end="") + print(f"❌ Errors: {errors_count}") + if warnings_count > 0: + if os.getenv("CI") == "true": + print("::warning::", end="") + print(f"⚠️ Warnings: {warnings_count}") + if failures_count > 0: + print("\n⛔ Failure Descriptions:") + for desc in failure_descriptions: + print(f" - {desc}") + else: + if errors_count > 0: + print("\n❌ Error Descriptions:") + for desc in error_descriptions: + print(f" - {desc}") + if warnings_count > 0: + print("\n⚠️ Warning Descriptions:") + for desc in warning_descriptions: + print(f" - {desc}") + if os.getenv("CI") == "true": + print("::endgroup::") + + +def main(): + # Read data from the specified file instead of stdin + file_path = os.path.join( + os.getenv("GITHUB_WORKSPACE", ""), "reports/intermediate-report.txt" + ) + with open(file_path, mode="r", encoding="utf-8", errors="ignore") as file: + data = file.read() + # Remove ANSI escape sequences + clean_data = remove_ansi_escape_sequences(data) + # Initialize variables + package_name = "" + failures_count = 0 + errors_count = 0 + warnings_count = 0 + uds_capability_wide_errors_count = 0 + failure_descriptions = [] + error_descriptions = [] + warning_descriptions = [] + uds_capability_wide_error_descriptions = [] + previous_package_name = None + + # Process each line + for line in clean_data.splitlines(): + # Remove leading and trailing whitespace + line = line.strip() + + # Match and extract the package name + match = re.match(r"^ℹ️\s+Package\s+Name:\s+(.*)$", line) + if match: + # Print the previous package's info before starting a new one + if previous_package_name is not None: + print_package_info( + previous_package_name, + failures_count, + errors_count, + warnings_count, + failure_descriptions, + error_descriptions, + warning_descriptions, + uds_capability_wide_errors_count, + ) + # Reset variables for the new package + package_name = match.group(1) + failures_count = 0 + errors_count = 0 + warnings_count = 0 + failure_descriptions = [] + error_descriptions = [] + warning_descriptions = [] + previous_package_name = package_name + continue + + if uds_capability_wide_errors(line): + uds_capability_wide_errors_count = 1 + uds_capability_wide_error_descriptions = [ + "Not all pods have the istio sidecar" + ] + continue + else: + # Match and extract counts for failures, errors, and warnings + match = re.match(r"^(❌|⚠️|⛔)\s+(\d+)\s+([a-z]+)\s+found$", line) + if match: + count = int(match.group(2)) + type_ = match.group(3) + if type_ == "errors": + errors_count = count + elif type_ == "warnings": + warnings_count = count + elif type_ == "failures": + failures_count = count + continue + + # Match and collect issue descriptions + match = re.match(r"^(❌|⚠️|⛔)\s+(.*)$", line) + if match: + emoji = match.group(1) + description = match.group(2) + if emoji == "❌": + error_descriptions.append(description) + elif emoji == "⚠️": + warning_descriptions.append(description) + elif emoji == "⛔": + failure_descriptions.append(description) + continue + + # Print the last package's information + if previous_package_name is not None: + print_package_info( + previous_package_name, + failures_count, + errors_count, + warnings_count, + failure_descriptions, + error_descriptions, + warning_descriptions, + uds_capability_wide_errors_count, + ) + if uds_capability_wide_errors_count >= 1: + print("-----------------------------") + if os.getenv("CI") == "true": + print("::group::UDS Capability-Wide Issues") + print("::error::", end="") + print("UDS Capability Issues") + print("\n❌ Error Descriptions:") + for desc in uds_capability_wide_error_descriptions: + print(f" - {desc}") + if os.getenv("CI") == "true": + print("::endgroup::") + + +if __name__ == "__main__": + main() + # Print the final ending separator + print("-----------------------------") diff --git a/.github/workflows/e2e-llama-cpp-python.yaml b/.github/workflows/e2e-llama-cpp-python.yaml index e3d573bba..416146b78 100644 --- a/.github/workflows/e2e-llama-cpp-python.yaml +++ b/.github/workflows/e2e-llama-cpp-python.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-playwright.yaml b/.github/workflows/e2e-playwright.yaml index 7200155fe..cf2db1d47 100644 --- a/.github/workflows/e2e-playwright.yaml +++ b/.github/workflows/e2e-playwright.yaml @@ -34,6 +34,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" @@ -120,7 +121,7 @@ jobs: - name: UI/API/Supabase E2E Playwright Tests run: | cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env - rm src/leapfrogai_ui/tests/global.teardown.ts + rm src/leapfrogai_ui/tests/global.teardown.ts mkdir -p src/leapfrogai_ui/playwright/.auth SERVICE_ROLE_KEY=$(uds zarf tools kubectl get secret -n leapfrogai supabase-bootstrap-jwt -o jsonpath={.data.service-key} | base64 -d) echo "::add-mask::$SERVICE_ROLE_KEY" diff --git a/.github/workflows/e2e-text-backend-full-cpu.yaml b/.github/workflows/e2e-text-backend-full-cpu.yaml index 6e8507ae3..7cb282d47 100644 --- a/.github/workflows/e2e-text-backend-full-cpu.yaml +++ b/.github/workflows/e2e-text-backend-full-cpu.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-text-embeddings.yaml b/.github/workflows/e2e-text-embeddings.yaml index 20f7eb97a..f60919724 100644 --- a/.github/workflows/e2e-text-embeddings.yaml +++ b/.github/workflows/e2e-text-embeddings.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-vllm.yaml b/.github/workflows/e2e-vllm.yaml index 07e9f046f..e808f15ea 100644 --- a/.github/workflows/e2e-vllm.yaml +++ b/.github/workflows/e2e-vllm.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-whisper.yaml b/.github/workflows/e2e-whisper.yaml index dee2cf45a..a7cd17e14 100644 --- a/.github/workflows/e2e-whisper.yaml +++ b/.github/workflows/e2e-whisper.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml new file mode 100644 index 000000000..6be419ebb --- /dev/null +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -0,0 +1,94 @@ +name: nightly-uds-badge-verification + +on: + schedule: + - cron: "0 11 * * *" # Runs daily at 3 AM PST + workflow_dispatch: # trigger manually as needed + pull_request: + paths: + - .github/workflows/nightly-uds-badge-verification.yaml + - tasks.yaml + +concurrency: + group: nightly-uds-badge-verification-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + SNAPSHOT_VERSION: snapshot-latest + +permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + +jobs: + uds-badge-verification: + runs-on: ai-ubuntu-big-boy-8-core + name: nightly_uds_badge_verification + + steps: + - name: Checkout Repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: main + + - name: Setup UDS Cluster + uses: ./.github/actions/uds-cluster + with: + registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + - name: Print the Commit SHA + run: | + COMMIT_SHA=$(git rev-parse HEAD) + echo "The latest commit on the main branch is: $COMMIT_SHA" + + # Set UDS CPU bundle refs and repositories to snapshot-latest + - name: Mutation of the UDS Bundle + run: | + uds zarf tools yq -i '.metadata.version = "${{ env.SNAPSHOT_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml + + uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ env.SNAPSHOT_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml + + uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml + + - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) + run: | + cd bundles/latest/cpu + uds create . --confirm && \ + uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ + rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ + docker system prune -af + + # Workaround for handling emojis in the upstream badge verification UDS task + - name: Set Locale to UTF-8 + run: | + sudo apt-get update + sudo apt-get install -y locales + sudo locale-gen en_US.UTF-8 + export LANG=en_US.UTF-8 + export LANGUAGE=en_US:en + export LC_ALL=en_US.UTF-8 + + # Setup Python for the report cleaning script in the next step + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version-file: "pyproject.toml" + + - name: Run UDS Badge Verification Task + run: | + uds run nightly-uds-badge-verification --no-progress + + - name: Archive UDS Badge Verification Report + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + with: + name: uds-badge-verification-report + path: reports + retention-days: 7 diff --git a/.github/workflows/uds-lint.yaml b/.github/workflows/uds-lint.yaml index 8f2e6834c..cf7050cb8 100644 --- a/.github/workflows/uds-lint.yaml +++ b/.github/workflows/uds-lint.yaml @@ -46,3 +46,11 @@ jobs: run: | check-jsonschema bundles/latest/gpu/uds-bundle.yaml --schemafile uds.schema.json check-jsonschema bundles/latest/cpu/uds-bundle.yaml --schemafile uds.schema.json + + - name: Download UDS Tasks Schema + run: curl -o tasks.schema.json https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json + + - name: Validate tasks.yaml + if: always() + run: | + check-jsonschema tasks.yaml --schemafile tasks.schema.json diff --git a/.gitignore b/.gitignore index 645bd6ff5..d0c8a20f3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ node_modules package.json package-lock.json **/*.schema.json +reports # local model and tokenizer files *.bin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6caadd6c8..401bcba03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -137,3 +137,26 @@ repos: files: "uds-bundle.yaml" types: [yaml] args: ["--schemafile", "uds-v0.14.0.schema.json"] + + # UDS TASKS CHECK + - repo: local + hooks: + - id: download-schema + name: "Download UDS Tasks Schema" + entry: | + bash -c 'FILE="tasks-v0.14.0.schema.json" + if [ -f "$(git rev-parse --show-toplevel)/$FILE" ]; then + echo "$FILE already exists in the root of the git project, skipping download." + else + curl -o "$FILE" https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json + fi' + language: system + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.14.0 + hooks: + - id: check-jsonschema + name: "Validate UDS Bundles Against Schema" + files: "tasks.yaml" + types: [yaml] + args: ["--schemafile", "tasks-v0.14.0.schema.json"] diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 897bfaf5d..7aa8d5c02 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -62,6 +62,52 @@ Many of the directories and sub-directories within this project contain Make tar Please refer to each Makefile for more arguments and details on what each target does and is dependent on. +## UDS Tasks + +UDS tasks use the UDS CLI runner, and are defined in the root `tasks.yaml` file. + +Currently, the only tasks within the file are for checking the progress of the LeapfrogAI towards the `Made for UDS` packaging standards. To run the task verification task you must have a [UDS Kubernetes cluster](../packages/k3d-gpu/README.md) and LeapfrogAI (GPU or CPU) deployed. After deploying both major capabilities, you can execute the following: + +```bash +uds run nightly-uds-badge-verification --no-progress +``` + +You should get an output similar to this, depending on how many components of LeapfrogAI are actually deployed: + +```bash + • Running "Create Reports Directory" + + ✔ Completed "Create Reports Directory" + + • Running "Run UDS Badge Verification Task" + + ✔ Completed "Run UDS Badge Verification Task" + + • Running "Clean Up Final Report" +----------------------------- +Package: leapfrogai-api + +❌ Errors: 4 +⚠️ Warnings: 3 + +❌ Error Descriptions: + - Endpoint leapfrogai-api.uds.dev is returning 404 + - Not all applicable network policies are using selectors + - Not all applicable network policies are using ports + - No monitors defined + +⚠️ Warning Descriptions: + - Version is not consistent across flavors and package + - Network policies with 'remoteGenerated: Anywhere' are present, review needed + - No SSO configuration found, review needed +----------------------------- +UDS Capability Issues + +❌ Error Descriptions: + - Not all pods have the istio sidecar +----------------------------- +``` + ## Environment Variables Be wary of `*config*.yaml` or `.env*` files that are in individual components of the stack. The component's README will usually tell the developer when to fill them out or supply environment variables to a script. diff --git a/packages/ui/chart/templates/ui/service.yaml b/packages/ui/chart/templates/ui/service.yaml index 15243e806..2cb919567 100644 --- a/packages/ui/chart/templates/ui/service.yaml +++ b/packages/ui/chart/templates/ui/service.yaml @@ -18,11 +18,3 @@ spec: protocol: TCP port: {{ .Values.service.port }} targetPort: {{ .Values.service.port }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "chart.serviceAccountName" . }} - namespace: {{ .Release.Namespace | default "leapfrogai" }} - labels: - {{- include "chart.labels" . | nindent 4 }} diff --git a/tasks.yaml b/tasks.yaml new file mode 100644 index 000000000..2298757ba --- /dev/null +++ b/tasks.yaml @@ -0,0 +1,133 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json + +includes: + - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/82e63be82766a2e550a847af904b2d738c9d3478/tasks/badge.yaml + +tasks: + - name: nightly-uds-badge-verification + description: "Runs in a pipeline and produces a report for archiving" + actions: + - description: "Create Reports Directory" + cmd: | + mkdir -p reports + - description: "Run UDS Badge Verification Task" + cmd: | + uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt + - description: "Clean Up Final Report" + cmd: | + python3 .github/scripts/uds_verification_report.py | tee ./reports/final-report.txt + + ############# + # BADGE TASKS + ############# + - name: verify-uds-badge-cpu + description: "Runs through all CPU UDS bundle packages with the UDS badge verification test" + actions: + - task: verify-uds-badge-api + - task: verify-uds-badge-ui + - task: verify-uds-badge-llama-cpp-python + - task: verify-uds-badge-text-embeddings + - task: verify-uds-badge-whisper + - task: verify-uds-badge-supabase + + - name: verify-uds-badge-gpu + description: "Runs through all GPU UDS bundle packages with the UDS badge verification test" + actions: + - task: verify-uds-badge-api + - task: verify-uds-badge-ui + - task: verify-uds-badge-vllm + - task: verify-uds-badge-text-embeddings + - task: verify-uds-badge-whisper + - task: verify-uds-badge-supabase + + ####################### + # RE-USABLE BADGE TASKS + ####################### + + - name: verify-uds-badge-api + actions: + - description: "Verify API" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="true" \ + --set PACKAGE_DIR="packages/api" \ + --no-progress + + - name: verify-uds-badge-ui + actions: + - description: "Verify UI" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/ui" \ + --no-progress + + - name: verify-uds-badge-llama-cpp-python + actions: + - description: "Verify LLaMA-CPP-Python" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/llama-cpp-python" \ + --no-progress + + - name: verify-uds-badge-vllm + actions: + - description: "Verify vLLM" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/vllm" \ + --no-progress + + - name: verify-uds-badge-text-embeddings + actions: + - description: "Verify Text-Embeddings" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/text-embeddings" \ + --no-progress + + - name: verify-uds-badge-whisper + actions: + - description: "Verify Whisper" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/whisper" \ + --no-progress + + - name: verify-uds-badge-repeater + actions: + - description: "Verify Repeater" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/repeater" \ + --no-progress + + - name: verify-uds-badge-supabase + actions: + - description: "Verify Supabase" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/supabase" \ + --no-progress