Cleanup on bpflsm #52
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci-coverage | |
| on: | |
| push: | |
| branches: [main, non-k8s-hsp-test] | |
| paths: | |
| - "KubeArmor/**" | |
| - ".github/workflows/ci-coverage.yaml" | |
| - "!STABLE-RELEASE" | |
| - "tests/**" | |
| jobs: | |
| calculate-k8s-coverage: | |
| name: Auto-testing Framework / ${{ matrix.os }} / ${{ matrix.runtime }} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| RUNTIME: ${{ matrix.runtime }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-20.04, bpflsm] | |
| runtime: ["containerd", "crio"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'KubeArmor/go.mod' | |
| - name: Install the latest LLVM toolchain | |
| run: ./.github/workflows/install-llvm.sh | |
| - name: Compile libbpf | |
| run: ./.github/workflows/install-libbpf.sh | |
| - name: Setup a Kubernetes environment | |
| run: ./.github/workflows/install-k3s.sh | |
| - name: Generate KubeArmor artifacts | |
| run: | | |
| # set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation | |
| export IS_COVERAGE=true | |
| GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh | |
| - name: Build Kubearmor-Operator | |
| working-directory: pkg/KubeArmorOperator | |
| run: | | |
| make docker-build | |
| - name: deploy pre existing pod | |
| run: | | |
| kubectl apply -f ./tests/k8s_env/ksp/pre-run-pod.yaml | |
| sleep 60 | |
| kubectl get pods -A | |
| - name: Run KubeArmor | |
| run: | | |
| if [ ${{ matrix.runtime }} == "containerd" ]; then | |
| docker save kubearmor/kubearmor-test-init:latest | sudo k3s ctr images import - | |
| docker save kubearmor/kubearmor-test:latest | sudo k3s ctr images import - | |
| docker save kubearmor/kubearmor-operator:latest | sudo k3s ctr images import - | |
| docker save kubearmor/kubearmor-snitch:latest | sudo k3s ctr images import - | |
| else | |
| if [ ${{ matrix.runtime }} == "crio" ]; then | |
| docker save kubearmor/kubearmor-test-init:latest | sudo podman load | |
| sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test-init:latest | |
| docker save kubearmor/kubearmor-test:latest | sudo podman load | |
| sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test:latest | |
| docker save kubearmor/kubearmor-operator:latest | sudo podman load | |
| sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-operator:latest | |
| docker save kubearmor/kubearmor-snitch:latest | sudo podman load | |
| sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-snitch:latest | |
| fi | |
| fi | |
| helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest | |
| kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator | |
| kubectl get pods -A | |
| kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml | |
| # kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test | |
| # kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor | |
| # kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor | |
| sleep 60 | |
| kubectl get pods -A | |
| DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}') | |
| echo "DaemonSet: $DAEMONSET_NAME" | |
| kubectl patch daemonset $DAEMONSET_NAME -n kubearmor --type='json' -p='[ | |
| { | |
| "op": "add", | |
| "path": "/spec/template/spec/volumes/-", | |
| "value": { | |
| "name": "coverage-storage", | |
| "hostPath": { | |
| "path": "/coverage", | |
| "type": "DirectoryOrCreate" | |
| } | |
| } | |
| }, | |
| { | |
| "op": "add", | |
| "path": "/spec/template/spec/containers/0/volumeMounts/-", | |
| "value": { | |
| "mountPath": "/coverage", | |
| "name": "coverage-storage" | |
| } | |
| }, | |
| { | |
| "op": "add", | |
| "path": "/spec/template/spec/containers/0/args/-", | |
| "value": "-test.coverprofile=/coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out" | |
| } | |
| ]' | |
| sleep 15 | |
| - name: Add KubeArmor host visibility | |
| run: | | |
| ./.github/workflows/host-visibility.sh | |
| DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}') | |
| # normal approach didn't worked for cri-o pod | |
| LABEL_SELECTOR=$(kubectl get daemonset $DAEMONSET_NAME -n kubearmor -o jsonpath='{.spec.selector.matchLabels}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' | paste -sd, -) | |
| POD_NAME=$(kubectl get pods -n kubearmor -l "$LABEL_SELECTOR" -o jsonpath='{.items[*].metadata.name}') | |
| echo "Pod: $POD_NAME" | |
| echo "POD_NAME=$POD_NAME" >> $GITHUB_ENV | |
| - name: Test KubeArmor using Ginkgo | |
| run: | | |
| kubectl logs $POD_NAME -n kubearmor | |
| go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo | |
| make | |
| working-directory: ./tests/k8s_env | |
| timeout-minutes: 30 | |
| env: | |
| POD_NAME: ${{ env.POD_NAME }} | |
| - name: Kill KubeArmor prcoess in the pod | |
| run: | | |
| KUBEARMOR_PID=$(kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "ps aux | grep '[K]ubeArmor/kubearmor-test' | awk '{print \$1}'") | |
| kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "kill -s SIGINT $KUBEARMOR_PID" | |
| sleep 10 | |
| env: | |
| POD_NAME: ${{ env.POD_NAME }} | |
| - name: Extract coverage file | |
| run: | | |
| for i in {1..24}; do | |
| if [ -f /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out ]; then | |
| cp /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out | |
| break | |
| fi | |
| sleep 5 | |
| done | |
| ls -l | |
| working-directory: KubeArmor | |
| env: | |
| POD_NAME: ${{ env.POD_NAME }} | |
| - name: Get karmor sysdump | |
| if: ${{ failure() }} | |
| run: | | |
| kubectl describe pod -n kubearmor -l kubearmor-app=kubearmor | |
| curl -sfL http://get.kubearmor.io/ | sudo sh -s -- -b /usr/local/bin | |
| mkdir -p /tmp/kubearmor/ && cd /tmp/kubearmor && karmor sysdump | |
| - name: Archive log artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: kubearmor.logs | |
| path: | | |
| /tmp/kubearmor/ | |
| /tmp/kubearmor.* | |
| - name: Measure code coverage | |
| if: ${{ always() }} | |
| run: | | |
| ls -l | |
| go tool cover -func coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out | |
| # go install github.com/modocache/gover@latest | |
| # gover | |
| working-directory: KubeArmor | |
| env: | |
| GOPATH: ${{ matrix.os == 'bpflsm' && '/home/vagrant/go' || '/home/runner/go' }} | |
| - name: Upload coverage file | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-k8s-${{ matrix.os }}-${{ matrix.runtime }} | |
| path: KubeArmor/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out | |
| - name: Run cleanup | |
| if: ${{ always() && matrix.os == 'bpflsm' }} | |
| run: ./.github/workflows/cleanup.sh | |
| docker-compose-coverage: | |
| name: Calculate coverage in docker mode / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'KubeArmor/go.mod' | |
| - name: Install the latest LLVM toolchain | |
| run: ./.github/workflows/install-llvm.sh | |
| - name: Compile libbpf | |
| run: ./.github/workflows/install-libbpf.sh | |
| - name: Generate KubeArmor artifacts | |
| run: | | |
| # set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation | |
| export IS_COVERAGE=true | |
| GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh | |
| - name: Run KubeArmor init container | |
| run: | | |
| docker run --name kubearmor-test-init -v /tmp/:/opt/kubearmor/BPF kubearmor/kubearmor-test-init:latest | |
| - name: Run KubeArmor test container | |
| run: | | |
| sudo mkdir -p /coverage | |
| docker run -d --name kubearmor-test --privileged --pid host -p 32767:32767 \ | |
| -v /tmp/:/opt/kubearmor/BPF \ | |
| -v /sys/fs/bpf:/sys/fs/bpf \ | |
| -v /sys/kernel/security:/sys/kernel/security \ | |
| -v /sys/kernel/debug:/sys/kernel/debug \ | |
| -v /var/run/docker.sock:/var/run/docker.sock \ | |
| -v /var/lib/docker:/var/lib/docker \ | |
| -v /etc/apparmor.d:/etc/apparmor.d \ | |
| -v /coverage/:/coverage/ \ | |
| kubearmor/kubearmor-test:latest -k8s=false -enableKubeArmorHostPolicy -coverageTest=false -test.coverprofile=/coverage/coverage_docker.out | |
| - name: Test KubeArmor using Ginkgo | |
| run: | | |
| go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo | |
| make | |
| working-directory: ./tests/nonk8s_env | |
| timeout-minutes: 30 | |
| - name: Copy coverage report | |
| run: | | |
| CONTAINER_ID=$(docker ps -qf "name=kubearmor") | |
| sleep 2 | |
| KUBEARMOR_PID=$(docker exec $CONTAINER_ID sh -c "pgrep -o -f 'kubearmor'") | |
| sleep 2 | |
| docker exec $CONTAINER_ID sh -c "kill -s SIGINT $KUBEARMOR_PID" | |
| sleep 15 | |
| for i in {1..24}; do | |
| if [ -f /coverage/coverage_docker.out ]; then | |
| cp /coverage/coverage_docker.out coverage_docker_${{ matrix.os }}.out | |
| break | |
| fi | |
| sleep 5 | |
| done | |
| sleep 2 | |
| working-directory: KubeArmor | |
| - name: Archive log artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: kubearmor.logs | |
| path: | | |
| /tmp/kubearmor/ | |
| /tmp/kubearmor.* | |
| - name: Measure code coverage | |
| if: ${{ always() }} | |
| run: | | |
| ls -l | |
| go tool cover -func coverage_docker_${{ matrix.os }}.out | |
| working-directory: KubeArmor | |
| env: | |
| GOPATH: ${{ matrix.os == 'bpflsm' && '/home/vagrant/go' || '/home/runner/go' }} | |
| - name: Save coverage file | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-docker-${{ matrix.os }} | |
| path: KubeArmor/coverage_docker_${{ matrix.os }}.out | |
| - name: Run cleanup | |
| if: ${{ always() && matrix.os == 'bpflsm' }} | |
| run: ./.github/workflows/cleanup.sh | |
| calculate-systemd-coverage: | |
| name: Calculate coverage in systemd mode / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'KubeArmor/go.mod' | |
| - name: Install the latest LLVM toolchain | |
| run: ./.github/workflows/install-llvm.sh | |
| - name: Compile libbpf | |
| run: ./.github/workflows/install-libbpf.sh | |
| - name: Install GoReleaser | |
| uses: goreleaser/goreleaser-action@v2 | |
| with: | |
| install-only: true | |
| version: v1.25.0 | |
| - name: Build Systemd Release | |
| run: make local-release | |
| working-directory: KubeArmor | |
| - name: Install KubeArmor | |
| run: sudo apt install -y ./dist/kubearmor*amd64.deb | |
| working-directory: KubeArmor | |
| - name: Compile test binary | |
| run: go test -covermode=atomic -coverpkg=./... -c . -o kubearmor-test | |
| working-directory: KubeArmor | |
| - name: Replace with test binary | |
| run: | | |
| sudo mkdir -p /coverage | |
| sudo rm /opt/kubearmor/kubearmor | |
| sudo cp kubearmor-test /opt/kubearmor/ | |
| ls -l /opt/kubearmor/ | |
| sudo sed -i 's|ExecStart=/opt/kubearmor/kubearmor|ExecStart=/opt/kubearmor/kubearmor-test -test.coverprofile=/coverage/coverage_systemd.out|' /lib/systemd/system/kubearmor.service | |
| sudo systemctl daemon-reload | |
| sudo systemctl restart kubearmor.service | |
| working-directory: KubeArmor | |
| - name: Check journalctl | |
| run: sudo journalctl -u kubearmor --no-pager || true | |
| - name: Test kubearmor using ginkgo | |
| run: | | |
| go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo | |
| make | |
| working-directory: ./tests/nonk8s_env | |
| timeout-minutes: 30 | |
| - name: Kill kubearmor process and copy coverage file | |
| run: | | |
| sudo systemctl stop kubearmor | |
| sleep 15 | |
| for i in {1..24}; do | |
| if [ -f /coverage/coverage_systemd.out ]; then | |
| sudo cp /coverage/coverage_systemd.out coverage_systemd_${{ matrix.os }}.out | |
| break | |
| fi | |
| sleep 5 | |
| done | |
| working-directory: KubeArmor | |
| - name: Measure code coverage | |
| if: ${{ always() }} | |
| run: | | |
| ls -l | |
| go tool cover -func coverage_systemd_${{ matrix.os }}.out | |
| working-directory: KubeArmor | |
| env: | |
| GOPATH: ${{ matrix.os == 'bpflsm' && '/home/vagrant/go' || '/home/runner/go' }} | |
| - name: Save coverage file | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-systemd-${{ matrix.os }} | |
| path: KubeArmor/coverage_systemd_${{ matrix.os }}.out | |
| - name: Run cleanup | |
| if: ${{ always() && matrix.os == 'bpflsm' }} | |
| run: | | |
| sudo systemctl stop kubearmor | |
| sudo systemctl disable kubearmor | |
| sudo apt remove -y kubearmor | |
| merge-and-upload-coverage: | |
| name: Merge and Upload Coverage | |
| runs-on: ubuntu-latest | |
| needs: [calculate-k8s-coverage, docker-compose-coverage, calculate-systemd-coverage] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'KubeArmor/go.mod' | |
| - name: Download all coverage files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: KubeArmor/ | |
| pattern: coverage* | |
| merge-multiple: true | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |