From c6cedb5e051e2c656d6dbdcdb3959e7febafbf0f Mon Sep 17 00:00:00 2001 From: timflannagan Date: Mon, 19 Aug 2024 16:49:46 +0000 Subject: [PATCH] WIP Signed-off-by: timflannagan --- .../action.yaml | 14 ++++- .github/workflows/conformance-tests.yaml | 53 +++++++++++++++++++ .github/workflows/nightly-tests.yaml | 4 +- .github/workflows/regression-tests.yaml | 4 +- Makefile | 25 +++++++-- 5 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/conformance-tests.yaml diff --git a/.github/workflows/composite-actions/kube-gateway-api-conformance-tests/action.yaml b/.github/workflows/composite-actions/kube-gateway-api-conformance-tests/action.yaml index ca71ef0b1f5..5d26951082c 100644 --- a/.github/workflows/composite-actions/kube-gateway-api-conformance-tests/action.yaml +++ b/.github/workflows/composite-actions/kube-gateway-api-conformance-tests/action.yaml @@ -32,8 +32,20 @@ runs: --set global.image.variant=${{ matrix.image-variant }} - name: Run kubernetes gateway api conformance tests shell: bash - run: make conformance + run: | + if [[ ${{ matrix.profile }} == "experimental" ]]; then + make conformance-experimental + else + make conformance + fi - name: print events if: ${{ failure() }} shell: bash run: kubectl -n gloo-system get events --sort-by='{.lastTimestamp}' + - name: Upload report artifacts + if: "{{ success && matrix.profile == 'experimental' }}" + uses: actions/upload-artifact@v4 + with: + name: _test/conformance/report.yaml + path: _test/conformance/report.yaml + if-no-files-found: ignore diff --git a/.github/workflows/conformance-tests.yaml b/.github/workflows/conformance-tests.yaml new file mode 100644 index 00000000000..07912716f7b --- /dev/null +++ b/.github/workflows/conformance-tests.yaml @@ -0,0 +1,53 @@ +name: Run Conformance Tests + +on: + workflow_dispatch: + inputs: + # kube-version: + # description: 'Kubernetes version to test' + # required: true + # type: choice + # options: + # - v1.26.0 + # - v1.25.0 + # - v1.24.0 + image-variant: + description: 'Image variant to use' + required: true + type: choice + options: + - standard + - distroless + profile: + description: 'Conformance profile to run' + required: true + type: choice + options: + - standard + - experimental + +jobs: + run-conformance-tests: + runs-on: ubuntu-latest + strategy: + matrix: + # kube-version: + # - kind: ${{ inputs.kube-version }} + # kubectl: ${{ inputs.kube-version }} + # helm: v3.8.0 + # node: v16.13.0 + kube-version: + - node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245' + kubectl: 'v1.29.2' + kind: 'v0.20.0' + helm: 'v3.14.4' + image-variant: + - ${{ inputs.image-variant }} + profile: + - ${{ inputs.profile }} + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Run Conformance Tests + uses: ./.github/workflows/composite-actions/conformance-tests diff --git a/.github/workflows/nightly-tests.yaml b/.github/workflows/nightly-tests.yaml index bee207fcb87..a827bcf973a 100644 --- a/.github/workflows/nightly-tests.yaml +++ b/.github/workflows/nightly-tests.yaml @@ -404,13 +404,15 @@ jobs: { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' }] image-variant: - standard + profile: + - standard + - experimental steps: - uses: actions/checkout@v4 with: ref: main - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests - kube_gateway_api_conformance_tests_17: name: Conformance (branch=v1.17.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} ) if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }} diff --git a/.github/workflows/regression-tests.yaml b/.github/workflows/regression-tests.yaml index 4482d1433b6..f053515809a 100644 --- a/.github/workflows/regression-tests.yaml +++ b/.github/workflows/regression-tests.yaml @@ -82,6 +82,8 @@ jobs: # (see https://github.com/solo-io/solo-projects/issues/6094) image-variant: - standard + profiles: + - standard steps: - uses: actions/checkout@v4 - id: auto-succeed-tests @@ -90,4 +92,4 @@ jobs: echo "Kubernetes Gateway API conformance tests auto-succeeded" - id: run-tests if: needs.prepare_env.outputs.should-auto-succeed-regression-tests != 'true' - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests \ No newline at end of file + uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests diff --git a/Makefile b/Makefile index b4c5a4767e7..d7d352ab725 100644 --- a/Makefile +++ b/Makefile @@ -1182,17 +1182,26 @@ build-test-chart: ## Build the Helm chart and place it in the _test directory # Targets for running Kubernetes Gateway API conformance tests #---------------------------------------------------------------------------------- +# FIXME(tim): Re-evaluate the --version & --contact flags. I think both are fine, but need to validate. +# TODO(tim): We should have a single Makefile target to drive all of this... +# Note(tim): Looks like adding the "TLS" profile breaks everything. We obviously don't support the "MESH" profile either. +# Note(tim): All of the profiles and suites were refactored in the release-1.1 branch. The profiles became more granular +# among other things like the experimental_conformance_test.go file being removed. + # Pull the conformance test suite from the k8s gateway api repo and copy it into the test dir. $(TEST_ASSET_DIR)/conformance/conformance_test.go: mkdir -p $(TEST_ASSET_DIR)/conformance - echo "//go:build conformance" > $@ cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/conformance_test.go >> $@ go fmt $@ -CONFORMANCE_ARGS:=-gateway-class=gloo-gateway -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror +$(TEST_ASSET_DIR)/conformance/experimental_conformance_test.go: + mkdir -p $(TEST_ASSET_DIR)/conformance + cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/experimental_conformance_test.go >> $@ + go fmt $@ -# Run the conformance test suite -.PHONY: conformance +CONFORMANCE_ARGS := -gateway-class=gloo-gateway -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror + +.PHONY: conformance ## Run the conformance test suite conformance: $(TEST_ASSET_DIR)/conformance/conformance_test.go go test -ldflags=$(LDFLAGS) -tags conformance -test.v $(TEST_ASSET_DIR)/conformance/... -args $(CONFORMANCE_ARGS) @@ -1202,6 +1211,14 @@ conformance-%: $(TEST_ASSET_DIR)/conformance/conformance_test.go go test -ldflags=$(LDFLAGS) -tags conformance -test.v $(TEST_ASSET_DIR)/conformance/... -args $(CONFORMANCE_ARGS) \ -run-test=$* +.PHONY: conformance-extended ## Run the extended conformance test suite +conformance-extended: CONFORMANCE_ARGS += -conformance-profiles=HTTP -report-output=$(TEST_ASSET_DIR)/conformance/report.yaml -organization=solo.io -project=gloo -version=$(VERSION) -url=github.com/solo-io/gloo -contact=https://solo.io +conformance-extended: $(TEST_ASSET_DIR)/conformance/conformance_test.go $(TEST_ASSET_DIR)/conformance/experimental_conformance_test.go + go test -ldflags=$(LDFLAGS) \ + -run TestExperimentalConformance \ + -test.v $(TEST_ASSET_DIR)/conformance/... \ + -args $(CONFORMANCE_ARGS) \ + #---------------------------------------------------------------------------------- # Security Scan #----------------------------------------------------------------------------------