From 29cdbf43c7ec828b97c74ffe05b9476c6f2b0849 Mon Sep 17 00:00:00 2001 From: Andreas Fritzler Date: Tue, 5 Dec 2023 12:59:53 +0100 Subject: [PATCH] Add ci test workflow --- .github/workflows/publish-docker-tester.yml | 89 --------------- .github/workflows/test.yml | 14 ++- Dockerfile | 1 - Makefile | 26 +++++ hack/ci-requirements.txt | 5 + hack/ci-test.sh | 114 ++++++++++++++++++++ 6 files changed, 156 insertions(+), 93 deletions(-) delete mode 100644 .github/workflows/publish-docker-tester.yml create mode 100644 Makefile create mode 100644 hack/ci-requirements.txt create mode 100755 hack/ci-test.sh diff --git a/.github/workflows/publish-docker-tester.yml b/.github/workflows/publish-docker-tester.yml deleted file mode 100644 index 78148a863..000000000 --- a/.github/workflows/publish-docker-tester.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Build and Publish Docker Tester Image - -env: - platforms: linux/amd64 - -on: - push: - branches: - - main - tags: - - v* - paths-ignore: - - 'docs/**' - - '**/*.md' - pull_request: - paths-ignore: - - 'docs/**' - - '**/*.md' - -jobs: - buildAndPushTester: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'release' && github.ref || github.event_name == 'push' && github.ref || github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.event_name == 'pull_request' && github.event.pull_request.head.ref }} - fetch-depth: 0 - - name: Determine SHA for Docker Image - id: get_sha - run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "DOCKER_IMAGE_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - else - echo "DOCKER_IMAGE_SHA=${{ github.sha }}" >> $GITHUB_ENV - fi - - uses: docker/metadata-action@v5 - id: meta - with: - images: | - ghcr.io/${{ github.repository_owner }}/dpservice-tester - tags: | - type=semver,pattern={{version}} - type=schedule - type=raw,${{ env.DOCKER_IMAGE_SHA }} - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - type=sha - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:latest - platforms: ${{env.platforms}} - # workaround for self-hosted runner - # https://github.com/mumoshu/actions-runner-controller-ci/commit/e91c8c0f6ca82aa7618010c6d2f417aa46c4a4bf - - name: Set up Docker Context for Buildx - id: buildx-context - run: | - CONTEXT_NAME="builders-tester-${GITHUB_RUN_ID}" - docker context create $CONTEXT_NAME - echo "CONTEXT_NAME=$CONTEXT_NAME" >> $GITHUB_ENV - - name: Set up Docker Buildx - timeout-minutes: 5 - uses: docker/setup-buildx-action@v3 - with: - version: latest - endpoint: ${{ env.CONTEXT_NAME }} # self-hosted - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push tester - timeout-minutes: 60 - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{env.platforms}} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - target: tester - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 584526744..d7063ea49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-python@v4 with: - go-version-file: 'go.mod' - - run: make test + python-version: '3.11' + cache: 'pip' + cache-dependency-path: '**/ci-requirements.txt' + - name: Install dependencies + run: | + sudo pip install -r hack/ci-requirements.txt + echo "$HOME/.local/bin" >> $GITHUB_PATH + - run: make ci-test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 8641f6f95..ee58e668c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,7 +164,6 @@ WORKDIR / COPY --from=builder /workspace/build/src/dpservice-bin \ /workspace/build/tools/dump/dpservice-dump \ /workspace/client/* \ - /workspace/exporter/* \ /workspace/hack/prepare.sh \ /usr/local/bin COPY --from=builder /usr/local/lib /usr/local/lib diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..bfba346c3 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +.PHONY: ci-test +ci-test: ## Run tests in a CI environment. + ./hack/ci-test.sh diff --git a/hack/ci-requirements.txt b/hack/ci-requirements.txt new file mode 100644 index 000000000..6db6f9dfe --- /dev/null +++ b/hack/ci-requirements.txt @@ -0,0 +1,5 @@ +meson==1.3.0 +pyelftools==0.30 +pytest==7.4.3 +scapy==2.5.0 +ninja==1.11.1.1 diff --git a/hack/ci-test.sh b/hack/ci-test.sh new file mode 100755 index 000000000..d4af995f7 --- /dev/null +++ b/hack/ci-test.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +REPO_ROOT="$SCRIPT_DIR/.." + +DPDK_VER=22.11 +DPSERVICE_FEATURES="" + +# create working directory +mkdir -p "$REPO_ROOT"/workspace +WORKSPACE_DIR="$REPO_ROOT"/workspace + +# Install prerequisite packages +sudo apt-get update && sudo sudo apt-get install -y --no-install-recommends ON \ +libibverbs-dev \ +libmnl-dev \ +libnuma-dev \ +numactl \ +libnuma1 \ +unzip \ +wget \ +make \ +gcc \ +g++ \ +clang \ +git \ +ethtool \ +pciutils \ +procps \ +iproute2 \ +libuuid1 \ +uuid-dev \ +net-tools \ +xz-utils \ +tar \ +findutils \ +jq \ +curl \ +build-essential \ +pkg-config \ +protobuf-compiler-grpc \ +libgrpc++-dev \ +libpcap0.8-dev \ +linux-headers-$(uname -r) \ +udev \ +gawk + +wget http://de.archive.ubuntu.com/ubuntu/pool/universe/g/grpc/libgrpc++1.51_1.51.1-3build3_amd64.deb -P "$WORKSPACE_DIR" > /dev/null 2>&1 +sudo dpkg -i "$WORKSPACE_DIR"/libgrpc++1.51_1.51.1-3build3_amd64.deb + +# Download DPDK +wget http://git.dpdk.org/dpdk/snapshot/dpdk-${DPDK_VER}.zip -P "$WORKSPACE_DIR" > /dev/null 2>&1 +cd "$WORKSPACE_DIR" && unzip dpdk-${DPDK_VER}.zip > /dev/null 2>&1 + +DPDK_DIR="$WORKSPACE_DIR"/dpdk-${DPDK_VER} + +# Copy DPDK patches +cd $DPDK_DIR && patch -p1 < "$REPO_ROOT"/hack/dpdk_22_11_gcc12.patch +cd $DPDK_DIR && patch -p1 < "$REPO_ROOT"/hack/dpdk_22_11_log.patch +cd $DPDK_DIR && patch -p1 < "$REPO_ROOT"/hack/dpdk_22_11_telemetry_key.patch +cd $DPDK_DIR && patch -p1 < "$REPO_ROOT"/hack/dpdk_22_11_ethdev_conversion.patch + +# Compile DPDK +cd $DPDK_DIR && meson setup -Dmax_ethports=132 -Dplatform=generic -Ddisable_drivers=common/dpaax,\ +common/cpt,common/iavf,\ +common/octeontx,common/octeontx2,common/cnxk,common/qat,regex/octeontx2,net/cnxk,dma/cnxk,\ +common/sfc_efx,common/auxiliary,common/dpaa,common/fslmc,common/ifpga,common/vdev,common/vmbus,\ +mempool/octeontx,mempool/octeontx2,baseband/*,event/*,net/ark,net/atlantic,net/avp,net/axgbe,\ +net/bnxt,net/bond,net/cxgbe,net/dpaa,net/dpaa2,net/e1000,net/ena,net/enetc,net/enetfec,net/enic,\ +net/failsafe,net/fm10k,net/hinic,net/hns3,net/i40e,net/iavf,net/ice,net/igc,net/ionic,net/ipn3ke,\ +net/ixgbe,net/liquidio,net/memif,net/netsvs,net/nfp,net/ngbe,net/null,net/octeontx,net/octeontx2,\ +net/octeontx_ep,net/pcap,net/pfe,net/qede,net/sfc,net/softnic,net/thunderx,net/txgbe,\ +net/vdev_ntsvc,net/vhost,net/virtio,net/vmxnet3,net/bnx2x,net/netsvc,net/vdev_netsvc,\ +crypto/dpaa_sec,crypto/bcmfs,crypto/caam_jr,crypto/cnxk,dpaa_sec,crypto/dpaa2_sec,crypto/nitrox,\ +crypto/null,crypto/octeontx,crypto/octeontx2,crypto/scheduler,crypto/virtio -Ddisable_libs=power,\ +vhost,gpudev build -Ddisable_apps="*" -Dtests=false > /dev/null 2>&1 +cd $DPDK_DIR/build && ninja > /dev/null 2>&1 +cd $DPDK_DIR/build && sudo ninja install > /dev/null 2>&1 + +sudo ldconfig + +# Build dpservice +cd "$REPO_ROOT" + +# Compile dpservice-bin itself +meson setup build $DPSERVICE_FEATURES && ninja -C build + +meson setup release_build $DPSERVICE_FEATURES --buildtype=release && ninja -C release_build +CC=clang CXX=clang++ meson setup clang_build $DPSERVICE_FEATURES && ninja -C clang_build +meson setup xtratest_build $DPSERVICE_FEATURES -Denable_tests=true && ninja -C xtratest_build + +sudo ldconfig + +"$REPO_ROOT"/hack/rel_download.sh -dir=client -owner=ironcore-dev -repo=dpservice-cli -strip=2 -pat=$GITHUB_TOKEN + +ls -la "$REPO_ROOT" + +cd "$REPO_ROOT" + +echo "finding dpservice-bin" +find . -name "dpservice-bin" +ls -al "$REPO_ROOT"/build/ + +#cp "$WORKSPACE_DIR"/build/src/dpservice-bin "$REPO_ROOT"/build/src/dpservice-bin +#cp "$WORKSPACE_DIR"/client/* "$REPO_ROOT"/build/ +#cp "$WORKSPACE_DIR"/xtratest_build/src/dpservice-bin "$REPO_ROOT"/xtratest_build/src/dpservice-bin +#cp "$WORKSPACE_DIR"/client/* "$REPO_ROOT"/xtratest_build + +export PYTHONUNBUFFERED=1 +sudo "$REPO_ROOT"/test/runtest.py "$REPO_ROOT"/build "$REPO_ROOT"/xtratest_build