WIP

Signed-off-by: timflannagan <[email protected]>

.github/workflows/composite-actions/kube-gateway-api-conformance-tests/action.yaml

@@ -32,8 +32,20 @@ runs:
         --set global.image.variant=${{ matrix.image-variant }}
   - name: Run kubernetes gateway api conformance tests
     shell: bash
-    run: make conformance
+    run: |
+      if [[ ${{ matrix.profile }} == "experimental" ]]; then
+        make conformance-experimental
+      else
+        make conformance
+      fi
   - name: print events
     if: ${{ failure() }}
     shell: bash
     run: kubectl -n gloo-system  get events --sort-by='{.lastTimestamp}'
+  - name: Upload report artifacts
+    if: "{{ success && matrix.profile == 'experimental' }}"
+    uses: actions/upload-artifact@v4
+    with:
+      name: _test/conformance/report.yaml
+      path: _test/conformance/report.yaml
+      if-no-files-found: ignore

.github/workflows/conformance-tests.yaml

@@ -0,0 +1,53 @@
+name: Run Conformance Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      # kube-version:
+      #   description: 'Kubernetes version to test'
+      #   required: true
+      #   type: choice
+      #   options:
+      #     - v1.26.0
+      #     - v1.25.0
+      #     - v1.24.0
+      image-variant:
+        description: 'Image variant to use'
+        required: true
+        type: choice
+        options:
+          - standard
+          - distroless
+      profile:
+        description: 'Conformance profile to run'
+        required: true
+        type: choice
+        options:
+          - standard
+          - experimental
+
+jobs:
+  run-conformance-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # kube-version:
+        #   - kind: ${{ inputs.kube-version }}
+        #     kubectl: ${{ inputs.kube-version }}
+        #     helm: v3.8.0
+        #     node: v16.13.0
+        kube-version:
+        - node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245'
+          kubectl: 'v1.29.2'
+          kind: 'v0.20.0'
+          helm: 'v3.14.4'
+        image-variant:
+          - ${{ inputs.image-variant }}
+        profile:
+          - ${{ inputs.profile }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Run Conformance Tests
+        uses: ./.github/workflows/composite-actions/conformance-tests

.github/workflows/nightly-tests.yaml

@@ -404,13 +404,15 @@ jobs:
                         { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' }]
         image-variant:
           - standard
+        profile:
+        - standard
+        - experimental
     steps:
     - uses: actions/checkout@v4
       with:
         ref: main
     - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
 
-
   kube_gateway_api_conformance_tests_17:
     name: Conformance (branch=v1.17.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} )
     if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }}

.github/workflows/regression-tests.yaml

@@ -82,6 +82,8 @@ jobs:
         # (see https://github.com/solo-io/solo-projects/issues/6094)
         image-variant:
           - standard
+        profiles:
+        - standard
     steps:
     - uses: actions/checkout@v4
     - id: auto-succeed-tests
@@ -90,4 +92,4 @@ jobs:
         echo "Kubernetes Gateway API conformance tests auto-succeeded"
     - id: run-tests
       if: needs.prepare_env.outputs.should-auto-succeed-regression-tests != 'true'
-      uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests
+      uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests

Makefile

@@ -1182,17 +1182,26 @@ build-test-chart: ## Build the Helm chart and place it in the _test directory
 # Targets for running Kubernetes Gateway API conformance tests
 #----------------------------------------------------------------------------------
 
+# FIXME(tim): Re-evaluate the --version & --contact flags. I think both are fine, but need to validate.
+# TODO(tim): We should have a single Makefile target to drive all of this...
+# Note(tim): Looks like adding the "TLS" profile breaks everything. We obviously don't support the "MESH" profile either.
+# Note(tim): All of the profiles and suites were refactored in the release-1.1 branch. The profiles became more granular
+# 			 among other things like the experimental_conformance_test.go file being removed.
+
 # Pull the conformance test suite from the k8s gateway api repo and copy it into the test dir.
 $(TEST_ASSET_DIR)/conformance/conformance_test.go:
 	mkdir -p $(TEST_ASSET_DIR)/conformance
-	echo "//go:build conformance" > $@
 	cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/conformance_test.go >> $@
 	go fmt $@
 
-CONFORMANCE_ARGS:=-gateway-class=gloo-gateway -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror
+$(TEST_ASSET_DIR)/conformance/experimental_conformance_test.go:
+	mkdir -p $(TEST_ASSET_DIR)/conformance
+	cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/experimental_conformance_test.go >> $@
+	go fmt $@
 
-# Run the conformance test suite
-.PHONY: conformance
+CONFORMANCE_ARGS := -gateway-class=gloo-gateway -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror
+
+.PHONY: conformance ## Run the conformance test suite
 conformance: $(TEST_ASSET_DIR)/conformance/conformance_test.go
 	go test -ldflags=$(LDFLAGS) -tags conformance -test.v $(TEST_ASSET_DIR)/conformance/... -args $(CONFORMANCE_ARGS)
 
@@ -1202,6 +1211,14 @@ conformance-%: $(TEST_ASSET_DIR)/conformance/conformance_test.go
 	go test -ldflags=$(LDFLAGS) -tags conformance -test.v $(TEST_ASSET_DIR)/conformance/... -args $(CONFORMANCE_ARGS) \
 	-run-test=$*
 
+.PHONY: conformance-extended ## Run the extended conformance test suite
+conformance-extended: CONFORMANCE_ARGS += -conformance-profiles=HTTP -report-output=$(TEST_ASSET_DIR)/conformance/report.yaml -organization=solo.io -project=gloo -version=$(VERSION) -url=github.com/solo-io/gloo -contact=https://solo.io
+conformance-extended: $(TEST_ASSET_DIR)/conformance/conformance_test.go $(TEST_ASSET_DIR)/conformance/experimental_conformance_test.go
+	go test -ldflags=$(LDFLAGS) \
+		-run TestExperimentalConformance \
+		-test.v $(TEST_ASSET_DIR)/conformance/... \
+		-args $(CONFORMANCE_ARGS) \
+
 #----------------------------------------------------------------------------------
 # Security Scan
 #----------------------------------------------------------------------------------