From 0f0f1d2b710bc4d87ffb8fe87efe0c86a9b3616b Mon Sep 17 00:00:00 2001 From: Marlon Saglia Date: Tue, 13 Aug 2024 09:46:31 +0200 Subject: [PATCH] feat: migrate several "verify-guides" workflow to Github actions --- .github/actions/free-space/action.yml | 36 +++++++++++++ .github/actions/install-deps/action.yml | 26 ++++++++++ .../verify-guides-billion-vector.yml | 20 ++++++++ .github/workflows/verify-guides-main.yml | 24 +++++++++ .github/workflows/verify-guides.yml | 51 +++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 .github/actions/free-space/action.yml create mode 100644 .github/actions/install-deps/action.yml create mode 100644 .github/workflows/verify-guides-billion-vector.yml create mode 100644 .github/workflows/verify-guides-main.yml create mode 100644 .github/workflows/verify-guides.yml diff --git a/.github/actions/free-space/action.yml b/.github/actions/free-space/action.yml new file mode 100644 index 000000000..5c1f946a4 --- /dev/null +++ b/.github/actions/free-space/action.yml @@ -0,0 +1,36 @@ +name: Free disk space +description: Frees some disk space on the runner. + +runs: + using: "composite" + steps: + - name: Remove Android library (~14GiB) + shell: bash + run: | + sudo rm -rf /usr/local/lib/android|| true + + - name: Remove .Net runtime (~2.7GiB) + shell: bash + run: | + sudo rm -rf /usr/share/dotnet|| true + + - name: Remove misc large packages (~5.3GiB) + shell: bash + run: | + sudo apt-get remove -yq --fix-missing --autoremove '^aspnetcore-.*' || true + sudo apt-get remove -yq --fix-missing --autoremove '^dotnet-.*' || true + sudo apt-get remove -yq --fix-missing --autoremove '^llvm-.*' || true + sudo apt-get remove -yq --fix-missing --autoremove '^mongodb-.*' || true + sudo apt-get remove -yq --fix-missing --autoremove '^google-cloud-*' || true + sudo apt-get remove -yq --fix-missing --autoremove php-common php-pear|| true + sudo apt-get remove -yq --fix-missing --autoremove mysql-common || true + sudo apt-get remove -yq --fix-missing --autoremove azure-cli powershell mono-devel libgl1-mesa-dri|| true + sudo apt-get remove -yq --fix-missing --autoremove google-chrome-stable firefox microsoft-edge-stable|| true + sudo apt-get remove -yq --fix-missing --autoremove snapd|| true + sudo apt-get autoremove -yq|| true + sudo apt-get clean|| true + + - name: Remove Tool Cache (~5.9GiB) + shell: bash + run: | + sudo rm -rf "$AGENT_TOOLSDIRECTORY"|| true diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yml new file mode 100644 index 000000000..011dd3e40 --- /dev/null +++ b/.github/actions/install-deps/action.yml @@ -0,0 +1,26 @@ +name: "Setup Vespa CLI and dependencies" +description: "Install Vespa CLI and dependencies" + +runs: + using: "composite" + steps: + # - name: Setup Ruby + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: 3.1 + # bundler-cache: true + + - uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Install python dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -qqq -r test/requirements.txt --user + python3 -m pip install -qqq pytest nbmake --user + + - name: Install Vespa CLI + uses: vespa-engine/setup-vespa-cli-action@v1 diff --git a/.github/workflows/verify-guides-billion-vector.yml b/.github/workflows/verify-guides-billion-vector.yml new file mode 100644 index 000000000..9dbd99600 --- /dev/null +++ b/.github/workflows/verify-guides-billion-vector.yml @@ -0,0 +1,20 @@ +name: Verify Giudes (billion scale vector search) + +on: + workflow_dispatch: # Allow manual triggering of this workflow. + + push: + branches: ["master"] + paths: + - "billion-scale-vector-search/" + + pull_request: + branches: ["master"] + paths: + - "billion-scale-vector-search/" + +jobs: + main: + uses: ./.github/workflows/verify-guides.yml + with: + test-file: "billion-scale-vector-search/README.md" diff --git a/.github/workflows/verify-guides-main.yml b/.github/workflows/verify-guides-main.yml new file mode 100644 index 000000000..217444f95 --- /dev/null +++ b/.github/workflows/verify-guides-main.yml @@ -0,0 +1,24 @@ +name: Verify Giudes (Main) + +on: + workflow_dispatch: # Allow manual triggering of this workflow. + + push: + branches: ["master"] + paths-ignore: + - "billion-scale-vector-search/" + - "billion-scale-image-search/" + - "examples/model-deployment/" + + pull_request: + branches: ["master"] + paths-ignore: + - "billion-scale-vector-search/" + - "billion-scale-image-search/" + - "examples/model-deployment/" + +jobs: + main: + uses: ./.github/workflows/verify-guides.yml + with: + test-config-path: "test/_test_config.yml" diff --git a/.github/workflows/verify-guides.yml b/.github/workflows/verify-guides.yml new file mode 100644 index 000000000..3e6e8817c --- /dev/null +++ b/.github/workflows/verify-guides.yml @@ -0,0 +1,51 @@ +name: Verify Giudes + +on: + workflow_call: + inputs: + test-config-path: + description: | + The path to the test configuration file, relative to the repository root. + + ::warning:: This is mutually exclusive with `test-file`. + + Example: `test/_test_config.yml` + type: string + required: false + test-file: + description: | + The path to the test file, relative to the repository root. This is used to run tests for a specific guide. + + ::warning:: This is mutually exclusive with `test-config-path` and will take precedence over it. + + Example: `billion-scale-vector-search/README.md` + type: string + required: false + +defaults: + run: + # Specify to ensure "pipefail and errexit" are set. + # Ref: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#defaultsrunshell + shell: bash + +jobs: + test: + runs-on: ubuntu-latest + env: + LANG: "C.UTF-8" + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/free-space + + - uses: ./.github/actions/install-deps + + - name: run-tests (config) + if: ${{ inputs.test-config-path && !inputs.test-file }} + run: | + ./test/test.py -w $GITHUB_WORKSPACE -c ${{ inputs.test-config-path }} + + - name: run-tests (file) + if: ${{ inputs.test-file && !inputs.test-config-path }} + run: | + ./test/test.py -w $GITHUB_WORKSPACE ${{ inputs.test-file }}