Nightly #1095
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # necessary to pass upgrade tests | |
| on: | |
| # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 | |
| # Minute [0,59] | |
| # Hour [0,23] | |
| # Day of the month [1,31] | |
| # Month of the year [1,12] | |
| # Day of the week ([0,6] with 0=Sunday) | |
| schedule: | |
| - cron: "0 5 * * *" # every day @ 05:00 UTC, run tests against latest main | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: "The branch to run tests against" | |
| type: choice | |
| options: | |
| - main | |
| - workflow_initiating_branch | |
| run-conformance: | |
| description: "Run conformance tests" | |
| type: boolean | |
| default: false | |
| # run-regression: | |
| # description: "Run regression tests" | |
| # type: boolean | |
| # default: false | |
| # run-performance: | |
| # description: "Run performance tests" | |
| # type: boolean | |
| # default: false | |
| # run-kubernetes-end-to-end: | |
| # # Runs all tests in /tests/kubernetes/e2e/... | |
| # description: "Run Kubernetes e2e tests" | |
| # type: boolean | |
| # default: false | |
| # kubernetes-end-to-end-run-regex: | |
| # # The regex that will be passed to the go test -run invocation | |
| # # This allows users to run just the subset of tests that they care about | |
| # description: "Kubernetes e2e tests -run regex" | |
| # type: string | |
| # required: false | |
| # default: '^Test' | |
| # TODO(tim): Add back in main branch tests once they're green. | |
| # TODO(tim): Evaluate whether we want to publish nightly results to Slack. | |
| jobs: | |
| kube_gateway_api_conformance_tests_main: | |
| name: Conformance (branch=main, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} ) | |
| if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' }, | |
| { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }] | |
| image-variant: | |
| - standard | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - uses: ./.github/actions/kube-gateway-api-conformance-tests | |
| # end_to_end_tests_on_demand: | |
| # name: End-to-End (branch=${{ github.ref_name }}, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} ) | |
| # if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'workflow_initiating_branch' }} | |
| # runs-on: ubuntu-22.04 | |
| # timeout-minutes: 180 | |
| # strategy: | |
| # # Since we are running these on a schedule, there is no value in failing fast | |
| # # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking | |
| # fail-fast: false | |
| # matrix: | |
| # test: | |
| # # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel. | |
| # # As a result, we increase the threshold for the tests, since they all run serially on a single cluster | |
| # - cluster-name: 'cluster-one' | |
| # go-test-args: '-v -timeout=150m' | |
| # go-test-run-regex: ${{ inputs.kubernetes-end-to-end-run-regex }} | |
| # # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support | |
| # # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/ | |
| # version-files: | |
| # - label: 'min' | |
| # file: './.github/workflows/.env/nightly-tests/min_versions.env' | |
| # - label: 'max' | |
| # file: './.github/workflows/.env/nightly-tests/max_versions.env' | |
| # steps: | |
| # # Checkout the branch that initiated the action | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # ref: ${{ github.ref_name }} | |
| # # The dotenv action is used to load key-value pairs from files. | |
| # # In this case, the file is specified in the matrix and will contain the versions of the tools to use | |
| # - name: Dotenv Action | |
| # uses: falti/[email protected] | |
| # id: dotenv | |
| # with: | |
| # path: ${{ matrix.version-files.file }} | |
| # log-variables: true | |
| # - name: Prep Go Runner | |
| # uses: ./.github/actions/prep-go-runner | |
| # # Set up the KinD cluster that the tests will use | |
| # - id: setup-kind-cluster | |
| # name: Setup KinD Cluster | |
| # uses: ./.github/actions/setup-kind-cluster | |
| # with: | |
| # cluster-name: ${{ matrix.test.cluster-name }} | |
| # kind-node-version: ${{ steps.dotenv.outputs.node_version }} | |
| # kind-version: ${{ steps.dotenv.outputs.kind_version }} | |
| # kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} | |
| # helm-version: ${{ steps.dotenv.outputs.helm_version }} | |
| # istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| # kgateway-api-version: ${{ steps.dotenv.outputs.kgateway_api_version }} | |
| # # Run the tests | |
| # - id: run-tests | |
| # name: Run Kubernetes e2e Tests | |
| # uses: ./.github/actions/kubernetes-e2e-tests | |
| # with: | |
| # cluster-name: ${{ matrix.test.cluster-name }} | |
| # test-args: ${{ matrix.test.go-test-args }} | |
| # run-regex: ${{ matrix.test.go-test-run-regex }} | |
| # istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| # matrix-label: ${{ matrix.version-files.label }} | |
| # end_to_end_tests_main: | |
| # name: End-to-End (branch=main, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} ) | |
| # if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }} | |
| # runs-on: ubuntu-22.04 | |
| # timeout-minutes: 180 | |
| # strategy: | |
| # # Since we are running these on a schedule, there is no value in failing fast | |
| # # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking | |
| # fail-fast: false | |
| # matrix: | |
| # test: | |
| # # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel. | |
| # # As a result, we increase the threshold for the tests, since they all run serially on a single cluster | |
| # - cluster-name: 'cluster-one' | |
| # go-test-args: '-v -timeout=150m' | |
| # # Specifying an empty regex means all tests will be run. | |
| # go-test-run-regex: "" | |
| # # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support | |
| # # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/ | |
| # version-files: | |
| # - label: 'min' | |
| # file: './.github/workflows/.env/nightly-tests/min_versions.env' | |
| # - label: 'max' | |
| # file: './.github/workflows/.env/nightly-tests/max_versions.env' | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # ref: main | |
| # # The dotenv action is used to load key-value pairs from files. | |
| # # In this case, the file is specified in the matrix and will contain the versions of the tools to use | |
| # - name: Dotenv Action | |
| # uses: falti/[email protected] | |
| # id: dotenv | |
| # with: | |
| # path: ${{ matrix.version-files.file }} | |
| # log-variables: true | |
| # - name: Prep Go Runner | |
| # uses: ./.github/actions/prep-go-runner | |
| # # Set up the KinD cluster that the tests will use | |
| # - id: setup-kind-cluster | |
| # name: Setup KinD Cluster | |
| # uses: ./.github/actions/setup-kind-cluster | |
| # with: | |
| # cluster-name: ${{ matrix.test.cluster-name }} | |
| # kind-node-version: ${{ steps.dotenv.outputs.node_version }} | |
| # kind-version: ${{ steps.dotenv.outputs.kind_version }} | |
| # kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} | |
| # helm-version: ${{ steps.dotenv.outputs.helm_version }} | |
| # istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| # kgateway-api-version: ${{ steps.dotenv.outputs.kgateway_api_version }} | |
| # # Run the tests | |
| # - id: run-tests | |
| # name: Run Kubernetes e2e Tests | |
| # uses: ./.github/actions/kubernetes-e2e-tests | |
| # with: | |
| # cluster-name: ${{ matrix.test.cluster-name }} | |
| # test-args: ${{ matrix.test.go-test-args }} | |
| # run-regex: ${{ matrix.test.go-test-run-regex }} | |
| # istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| # matrix-label: ${{ matrix.version-files.label }} | |
| # regression_tests_on_demand: | |
| # name: on demand regression tests | |
| # if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'workflow_initiating_branch' }} | |
| # runs-on: ubuntu-22.04 | |
| # timeout-minutes: 60 | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # kube-e2e-test-type: ['gateway', 'gloo', 'upgrade'] | |
| # kube-version: [ { node: 'v1.25.16@sha256:5da57dfc290ac3599e775e63b8b6c49c0c85d3fec771cd7d55b45fae14b38d3b', kubectl: 'v1.25.16', kind: 'v0.20.0', helm: 'v3.13.2' }, | |
| # { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }] | |
| # image-variant: | |
| # - standard | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # ref: ${{ github.ref_name }} | |
| # - uses: ./.github/actions/regression-tests | |
| # regression_tests_main: | |
| # name: main regression tests | |
| # if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }} | |
| # runs-on: ubuntu-22.04 | |
| # timeout-minutes: 60 | |
| # # Previously, there was an environment variable, RELEASED_VERSION="LATEST" set. This made use of some internal code: | |
| # # https://github.com/solo-io/gloo/blob/main/test/kube2e/util.go#L229-L241 | |
| # # which modified our testing process to pull the latest beta release. | |
| # # | |
| # # NOW, however, running this job is the same as normal CI. (building a local chart, then using it) | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # # TODO: | |
| # # As part of the end_to_end_tests_main job, we added support for importing versions from a .env file | |
| # # We should extend the support/usage of those .env files to these other jobs. | |
| # # The tests are currently in flux, and some of these regression tests are being migrated, so we decided | |
| # # to limit the scope (and potentially unnecessary work) for now | |
| # kube-e2e-test-type: ['gateway', 'gloo', 'upgrade'] | |
| # kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' }, | |
| # { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }] | |
| # image-variant: | |
| # - standard | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # ref: main | |
| # - uses: ./.github/actions/regression-tests | |
| # performance_tests_on_demand: | |
| # name: on demand performance tests | |
| # if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'workflow_initiating_branch' }} | |
| # runs-on: ubuntu-22.04 | |
| # timeout-minutes: 60 | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # ref: ${{ github.ref_name }} | |
| # - uses: ./.github/actions/prep-go-runner | |
| # - uses: ./.github/actions/performance-tests | |
| # performance_tests_main: | |
| # name: main performance tests | |
| # if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }} | |
| # runs-on: ubuntu-22.04 | |
| # timeout-minutes: 60 | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # ref: main | |
| # - uses: ./.github/actions/prep-go-runner | |
| # - uses: ./.github/actions/performance-tests |