manual-perf-test.yml

# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: Run performance test on a Kubernetes cluster

on:
  workflow_dispatch:
    inputs:
      runner:
        default: "aise-perf"
        description: "Runner label 'aise-perf'"
        required: true
        type: string
      cleanup:
        default: true
        description: "Whether to clean up the pods and the labels"
        required: true
        type: boolean
      node_nums:
        default: "1"
        description: "List of test node numbers, e.g., 1,2,4,8"
        required: true
        type: string
      registry:
        default: ""
        description: "Registry to store images, empty string means docker.io, default is empty"
        required: false
        type: string
      tag:
        default: "1.1"
        description: "Tag to apply to images"
        required: true
        type: string
      example_branch:
        default: "tmp/v1.1"
        description: "GenAIExamples branch to test manifests"
        required: true
        type: string
      infra_branch:
        default: "v1.1"
        description: "GenAIInfra branch to test manifests"
        required: true
        type: string
      eval_branch:
        default: "v1.1"
        description: "GenAIEval branch to test manifests"
        required: true
        type: string
      mode:
        default: "with_rerank:oob"
        description: "The mode of the test, e.g., with_rerank:tuned"
        required: true
        type: string
      test_cases:
        default: "e2e"
        description: "The test cases of chatqna, e.g., e2e, llmserve, embedserve, rerankserve"
        required: false
        type: string
      # user_queries:
      #   default: ''
      #   description: "The user query list, e.g., '4, 8, 16, 640', empty input means the default setting."
      #   required: false
      #   type: string
      #load_config:
      #  default: "constant:5"
      #  description: "configuration for load test in format load_type:value, constant:5 or poisson:1.0"
      #  required: false
      #  type: string

jobs:
  get-build-matrix:
    runs-on: ubuntu-latest
    outputs:
      node_nums: ${{ steps.get-services.outputs.node_nums }}
    steps:
      - name: Get test Services
        id: get-services
        run: |
          set -x
          node_num_list=($(echo ${{ github.event.inputs.node_nums }} | tr ',' ' '))
          node_nums=$(printf '%s\n' "${node_num_list[@]}" | sort | jq -R '.' | jq -sc '.')
          echo "node_nums=$node_nums" >> $GITHUB_OUTPUT

  run-benchmark:
    needs: [get-build-matrix]
    runs-on: "${{ inputs.runner }}"
    env:
      conda_env_name: "OPEA_perf"
    strategy:
      matrix:
        node_num: ${{ fromJSON(needs.get-build-matrix.outputs.node_nums) }}
    steps:
      - name: Clean Up Working Directory
        run: |
          sudo rm -rf ${{github.workspace}}/*
          export PATH=${HOME}/miniforge3/bin/:$PATH
          if conda info --envs | grep -q "$conda_env_name"; then
            echo "$conda_env_name exist!"
          else
            conda create -n ${conda_env_name} python=3.12 -y
          fi

      - name: Checkout out Validation
        uses: actions/checkout@v4
        with:
          path: Validation

      - name: Checkout out GenAIExamples
        uses: actions/checkout@v4
        with:
          repository: opea-project/GenAIExamples
          ref: ${{ inputs.example_branch }}
          path: GenAIExamples

      - name: Checkout out GenAIEval
        uses: actions/checkout@v4
        with:
          repository: opea-project/GenAIEval
          ref: ${{ inputs.eval_branch }}
          path: GenAIEval

      - name: Checkout out GenAIInfra
        uses: actions/checkout@v4
        with:
          repository: opea-project/GenAIInfra
          ref: ${{ inputs.infra_branch }}
          path: GenAIInfra

      - name: Set up stress tool
        run: |
          export PATH=${HOME}/miniforge3/bin/:$PATH
          source activate ${conda_env_name}
          if ! command -v yq &> /dev/null; then
            sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
            sudo chmod +x /usr/bin/yq
          fi
          pip install pandas # install python lib pandas for csv processing
          cp Validation/.github/scripts/process_csv.py GenAIEval/evals/benchmark # copy process_csv.py to GenAIEval folder
          cd GenAIEval
          pip install -r requirements.txt

          # config env values
          LOAD_SHAPE=$(echo "${{ inputs.load_config }}" | cut -d':' -f1 | xargs)
          second_part=$(echo "${{ inputs.load_config }}" | cut -d':' -f2 | xargs)
          if [ "$LOAD_SHAPE" == "constant" ]; then
              echo "LOAD_SHAPE=$LOAD_SHAPE" >> $GITHUB_ENV
              echo "CONCURRENT_LEVEL=$second_part" >> $GITHUB_ENV
          elif [ "$LOAD_SHAPE" == "poisson" ]; then
              echo "LOAD_SHAPE=$LOAD_SHAPE" >> $GITHUB_ENV
              echo "ARRIVAL_RATE=$second_part" >> $GITHUB_ENV
          else
              echo "Unknown LOAD_SHAPE: $LOAD_SHAPE"
          fi

      - name: K8s Label Nodes
        working-directory: ./Validation
        env:
          NODE_NUM: ${{ matrix.node_num }}
        run: |
          echo "uncordon=true" >> $GITHUB_ENV
          .github/scripts/perf_test.sh --label

      - name: Prepare benchmark configuration
        id: prepare_benchmark
        working-directory: ./Validation
        env:
          TEST_OUTPUT_DIR: /home/sdp/benchmark_output/node_${{ matrix.node_num }}
          TEST_CASES: ${{ inputs.test_cases }}
          USER_QUERIES: ${{ inputs.user_queries }}
          NODE_NUM: ${{ matrix.node_num }}
        run: |
          rm -rf $TEST_OUTPUT_DIR
          .github/scripts/perf_test.sh --generate_config ${{ matrix.node_num }}
          echo "uninstall=false" >> $GITHUB_ENV
          echo "randomstr=$(echo $RANDOM)" >> $GITHUB_OUTPUT
          echo "TEST_OUTPUT_DIR=$TEST_OUTPUT_DIR" >> $GITHUB_ENV

      - name: Install Workload
        working-directory: ./Validation
        env:
          IMAGE_REPO: ${{ inputs.registry }}
          IMAGE_TAG: ${{ inputs.tag }}
          HF_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
          MODE: ${{ inputs.mode }}
          NODE_NUM: ${{ matrix.node_num }}
        run: |
          export PATH=${HOME}/miniforge3/bin/:$PATH
          source activate ${conda_env_name}
          echo "uninstall=true" >> $GITHUB_ENV
          .github/scripts/perf_test.sh --installChatQnA

      - name: Stress Test
        working-directory: ./GenAIEval
        env:
          TEST_OUTPUT_DIR: /home/sdp/benchmark_output/node_${{ matrix.node_num }}
          TEST_CASES: ${{ inputs.test_cases }}
          NODE_NUM: ${{ matrix.node_num }}
        run: |
          export PATH=${HOME}/miniforge3/bin/:$PATH
          source activate ${conda_env_name}
          cd evals/benchmark
          python benchmark.py
          
          ${{ github.workspace }}/Validation/.github/scripts/perf_test.sh --process_result_data
          cp $TEST_OUTPUT_DIR/*_result.csv ${{ github.workspace }}
          cp $TEST_OUTPUT_DIR/*_testspec.yaml ${{ github.workspace }}
          cd $TEST_OUTPUT_DIR
          tar -cvf ${{ github.workspace }}/node_${{ matrix.node_num }}.tar *_result.csv *_testspec.yaml

      - name: Print Test Result
        run: |
          for file in "${{ github.workspace }}"/*_result.csv; do
              if [[ -f "$file" ]]; then
                  echo "Printing contents of: $file"
                  cat "$file" | cut -d, -f2,5,6,7,8
                  echo "-----------------------------------"
              fi
          done

      - name: Dump test specification
        run: |
          spec_file=$(find "${{ github.workspace }}" -maxdepth 1 -name 'e2e_testspec.yaml' | head -n 1)
          if [[ -f "$spec_file" ]]; then
              echo "Dump test specification: $spec_file"
              cat "$spec_file"
          fi

      - uses: actions/upload-artifact@v4.3.4
        with:
          name: ${{ matrix.node_num }}node_raw_data_tar_${{ steps.prepare_benchmark.outputs.randomstr }}
          path: node_${{ matrix.node_num }}.tar
          overwrite: true

      - name: Uninstall Workload
        if: always()
        working-directory: ./Validation
        env:
          MODE: ${{ inputs.mode }}
          NODE_NUM: ${{ matrix.node_num }}
        run: |
          if [[ "$uninstall" == true && "${{ inputs.cleanup }}" == true ]]; then
            .github/scripts/perf_test.sh --uninstallChatQnA
            sleep 200s
          fi

      - name: K8s Unlabel Nodes
        if: always()
        working-directory: ./Validation
        run: |
          if [[ "$uncordon" == true && "${{ inputs.cleanup }}" == true ]]; then
            .github/scripts/perf_test.sh --unlabel
            sleep 10s
          fi