.github,Makefile: Add support for generating conformance reports

Signed-off-by: timflannagan <[email protected]>

.github/workflows/composite-actions/kube-gateway-api-conformance-tests/action.yaml

@@ -17,23 +17,67 @@ runs:
   - uses: azure/setup-helm@v4
     with:
       version: ${{ matrix.kube-version.helm }}
+  - name: Set and retrieve environment variables
+    shell: bash
+    run: |
+      # We want to conditionally set the VERSION variable based on the matrix value
+      if [[ -z "${{ matrix.version }}" ]]; then
+        echo "VERSION=$(make print-VERSION)" >> $GITHUB_ENV
+      else
+        echo "VERSION=${{ matrix.version }}" >> $GITHUB_ENV
+      fi
+  - name: Print out version variable
+    shell: bash
+    run: |
+      echo "VERSION is $VERSION"
+      echo "VERSION is ${{ env.VERSION }}"
+      echo "VERSION is $(make print-VERSION)"
   - name: Setup test env
     shell: bash
     env:
       CLUSTER_NODE_VERSION: ${{ matrix.kube-version.node }}
       IMAGE_VARIANT: ${{ matrix.image-variant }}
       CONFORMANCE: "true"
     run: ./ci/kind/setup-kind.sh
-  - name: Install Gloo Gateway Enterprise with k8s gateway integration enabled
+  - name: Install Gloo Gateway with the k8s gateway integration enabled
     shell: bash
     run: |
-      helm upgrade -i -n gloo-system gloo ./_test/gloo-1.0.0-ci1.tgz --create-namespace \
-        --set kubeGateway.enabled=true \
-        --set global.image.variant=${{ matrix.image-variant }}
-  - name: Run kubernetes gateway api conformance tests
+      # TODO(tim): I think we want to interface with helm repositories even
+      # when the chart path is locally defined. This is the pattern that
+      # GME uses as well.
+
+      if [[ -z "${{ matrix.version }}" ]]; then
+        echo "Input version is empty, defaulting to local chart path"
+        # If matrix.version is empty, use the local chart path specified in the Makefile
+        helm upgrade -i -n gloo-system gloo ./_test/gloo-${VERSION}.tgz --create-namespace \
+          --set kubeGateway.enabled=true \
+          --set global.image.variant=${{ matrix.image-variant }}
+      else
+        echo "Input version is not empty, installing from helm repository"
+        helm repo add gloo https://storage.googleapis.com/solo-public-helm
+        helm repo update
+
+        # Use the provided version to install Gloo from the helm repository
+        helm upgrade -i -n gloo-system gloo gloo/gloo --version ${{ matrix.version }} --create-namespace \
+          --set kubeGateway.enabled=true \
+          --set global.image.variant=${{ matrix.image-variant }}
+      fi
+  - name: Run the kubernetes gateway api conformance tests
     shell: bash
-    run: make conformance
-  - name: print events
+    run: |
+      echo "Running conformance tests with the $VERSION version"
+      if [[ ${{ matrix.profile }} == "experimental" ]]; then
+        make conformance-experimental
+      else
+        make conformance
+      fi
+  - name: Capture debug information when tests fail
     if: ${{ failure() }}
     shell: bash
     run: kubectl -n gloo-system  get events --sort-by='{.lastTimestamp}'
+  - name: Upload report artifacts
+    if: "${{ matrix.profile == 'experimental' }}"
+    uses: actions/upload-artifact@v4
+    with:
+      name: conformance-report.yaml
+      path: _test/conformance/report.yaml

.github/workflows/conformance-tests.yaml

@@ -0,0 +1,64 @@
+name: Run Conformance Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      image-variant:
+        description: "Image variant to use"
+        required: true
+        type: choice
+        options:
+          - standard
+          - distroless
+      profile:
+        description: "Conformance profile to run"
+        required: true
+        type: choice
+        options:
+          - standard
+          - experimental
+      version:
+        description: "Gloo Gateway version to deploy and test (e.g. 1.17.3)"
+        required: false
+        type: string
+
+jobs:
+  run-conformance-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # kube-version:
+        #   - kind: ${{ inputs.kube-version }}
+        #     kubectl: ${{ inputs.kube-version }}
+        #     helm: v3.8.0
+        #     node: v16.13.0
+        kube-version:
+        - node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245'
+          kubectl: 'v1.29.2'
+          kind: 'v0.20.0'
+          helm: 'v3.14.4'
+        image-variant:
+          - ${{ inputs.image-variant }}
+        profile:
+          - ${{ inputs.profile }}
+        version:
+          - ${{ inputs.version }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          repository: timflannagan/gloo
+
+      # TODO: provide version input to this action
+      - name: Run Conformance Tests
+        uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
+
+      # TODO: produce slack notification?
+
+      # TODO: generate date-specific report files
+      # See https://github.com/projectcontour/contour/blob/main/test/conformance/gatewayapi/gateway_conformance_test.go
+      # for an example of how to generate a report file.
+
+      # TODO: copy artifacts and create a PR against the gateway-api repo.
+      # Note: it would be best if we forked that repo and tested automation
+      # against that fork first.

.github/workflows/nightly-tests.yaml

@@ -404,13 +404,15 @@ jobs:
                         { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' }]
         image-variant:
           - standard
+        profile:
+        - standard
+        - experimental
     steps:
     - uses: actions/checkout@v4
       with:
         ref: main
     - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
 
-
   kube_gateway_api_conformance_tests_17:
     name: Conformance (branch=v1.17.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} )
     if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }}

.github/workflows/regression-tests.yaml

@@ -82,6 +82,8 @@ jobs:
         # (see https://github.com/solo-io/solo-projects/issues/6094)
         image-variant:
           - standard
+        profiles:
+        - standard
     steps:
     - uses: actions/checkout@v4
     - id: auto-succeed-tests
@@ -90,4 +92,4 @@ jobs:
         echo "Kubernetes Gateway API conformance tests auto-succeeded"
     - id: run-tests
       if: needs.prepare_env.outputs.should-auto-succeed-regression-tests != 'true'
-      uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
+      uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests

Makefile

@@ -167,7 +167,6 @@ install-go-tools: mod-download ## Download and install Go dependencies
 	go install github.com/cratonica/2goarray
 	go install github.com/golang/mock/mockgen
 	go install github.com/saiskee/gettercheck
-	go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
 	# This version must stay in sync with the version used in CI: .github/workflows/static-analysis.yaml
 	go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(LINTER_VERSION)
 	go install github.com/quasilyte/go-ruleguard/cmd/[email protected]
@@ -1187,17 +1186,26 @@ build-test-chart: ## Build the Helm chart and place it in the _test directory
 # Targets for running Kubernetes Gateway API conformance tests
 #----------------------------------------------------------------------------------
 
+# FIXME(tim): Re-evaluate the --version & --contact flags. I think both are fine, but need to validate.
+# TODO(tim): We should have a single Makefile target to drive all of this...
+# Note(tim): Looks like adding the "TLS" profile breaks everything. We obviously don't support the "MESH" profile either.
+# Note(tim): All of the profiles and suites were refactored in the release-1.1 branch. The profiles became more granular
+# 			 among other things like the experimental_conformance_test.go file being removed.
+
 # Pull the conformance test suite from the k8s gateway api repo and copy it into the test dir.
 $(TEST_ASSET_DIR)/conformance/conformance_test.go:
 	mkdir -p $(TEST_ASSET_DIR)/conformance
-	echo "//go:build conformance" > $@
 	cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/conformance_test.go >> $@
 	go fmt $@
 
-CONFORMANCE_ARGS:=-gateway-class=gloo-gateway -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror
+$(TEST_ASSET_DIR)/conformance/experimental_conformance_test.go:
+	mkdir -p $(TEST_ASSET_DIR)/conformance
+	cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/experimental_conformance_test.go >> $@
+	go fmt $@
+
+CONFORMANCE_ARGS := -gateway-class=gloo-gateway -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror
 
-# Run the conformance test suite
-.PHONY: conformance
+.PHONY: conformance ## Run the conformance test suite
 conformance: $(TEST_ASSET_DIR)/conformance/conformance_test.go
 	go test -ldflags=$(LDFLAGS) -tags conformance -test.v $(TEST_ASSET_DIR)/conformance/... -args $(CONFORMANCE_ARGS)
 
@@ -1207,6 +1215,14 @@ conformance-%: $(TEST_ASSET_DIR)/conformance/conformance_test.go
 	go test -ldflags=$(LDFLAGS) -tags conformance -test.v $(TEST_ASSET_DIR)/conformance/... -args $(CONFORMANCE_ARGS) \
 	-run-test=$*
 
+.PHONY: conformance-experimental ## Run the extended conformance test suite
+conformance-experimental: CONFORMANCE_ARGS += -conformance-profiles=HTTP -report-output=$(TEST_ASSET_DIR)/conformance/report.yaml -organization=solo.io -project=gloo -version=$(VERSION) -url=github.com/solo-io/gloo -contact=https://solo.io
+conformance-experimental: $(TEST_ASSET_DIR)/conformance/conformance_test.go $(TEST_ASSET_DIR)/conformance/experimental_conformance_test.go
+	go test -ldflags=$(LDFLAGS) \
+		-run TestExperimentalConformance \
+		-test.v $(TEST_ASSET_DIR)/conformance/... \
+		-args $(CONFORMANCE_ARGS) \
+
 #----------------------------------------------------------------------------------
 # Security Scan
 #----------------------------------------------------------------------------------