-
Notifications
You must be signed in to change notification settings - Fork 462
262 lines (252 loc) · 12.4 KB
/
nightly-tests.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
name: Nightly
env:
GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # necessary to pass upgrade tests
on:
# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07
# Minute [0,59]
# Hour [0,23]
# Day of the month [1,31]
# Month of the year [1,12]
# Day of the week ([0,6] with 0=Sunday)
schedule:
- cron: "0 5 * * *" # every day @ 05:00 UTC, run tests against latest main
workflow_dispatch:
inputs:
branch:
description: "The branch to run tests against"
type: choice
options:
- main
- workflow_initiating_branch
run-conformance:
description: "Run conformance tests"
type: boolean
default: false
# run-regression:
# description: "Run regression tests"
# type: boolean
# default: false
# run-performance:
# description: "Run performance tests"
# type: boolean
# default: false
# run-kubernetes-end-to-end:
# # Runs all tests in /tests/kubernetes/e2e/...
# description: "Run Kubernetes e2e tests"
# type: boolean
# default: false
# kubernetes-end-to-end-run-regex:
# # The regex that will be passed to the go test -run invocation
# # This allows users to run just the subset of tests that they care about
# description: "Kubernetes e2e tests -run regex"
# type: string
# required: false
# default: '^Test'
# TODO(tim): Add back in main branch tests once they're green.
# TODO(tim): Evaluate whether we want to publish nightly results to Slack.
jobs:
kube_gateway_api_conformance_tests_main:
name: Conformance (branch=main, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} )
if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }}
runs-on: ubuntu-22.04
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' },
{ node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }]
image-variant:
- standard
steps:
- uses: actions/checkout@v4
with:
ref: main
- uses: ./.github/actions/kube-gateway-api-conformance-tests
# end_to_end_tests_on_demand:
# name: End-to-End (branch=${{ github.ref_name }}, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
# if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'workflow_initiating_branch' }}
# runs-on: ubuntu-22.04
# timeout-minutes: 180
# strategy:
# # Since we are running these on a schedule, there is no value in failing fast
# # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
# fail-fast: false
# matrix:
# test:
# # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
# # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
# - cluster-name: 'cluster-one'
# go-test-args: '-v -timeout=150m'
# go-test-run-regex: ${{ inputs.kubernetes-end-to-end-run-regex }}
# # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
# # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
# version-files:
# - label: 'min'
# file: './.github/workflows/.env/nightly-tests/min_versions.env'
# - label: 'max'
# file: './.github/workflows/.env/nightly-tests/max_versions.env'
# steps:
# # Checkout the branch that initiated the action
# - uses: actions/checkout@v4
# with:
# ref: ${{ github.ref_name }}
# # The dotenv action is used to load key-value pairs from files.
# # In this case, the file is specified in the matrix and will contain the versions of the tools to use
# - name: Dotenv Action
# uses: falti/[email protected]
# id: dotenv
# with:
# path: ${{ matrix.version-files.file }}
# log-variables: true
# - name: Prep Go Runner
# uses: ./.github/actions/prep-go-runner
# # Set up the KinD cluster that the tests will use
# - id: setup-kind-cluster
# name: Setup KinD Cluster
# uses: ./.github/actions/setup-kind-cluster
# with:
# cluster-name: ${{ matrix.test.cluster-name }}
# kind-node-version: ${{ steps.dotenv.outputs.node_version }}
# kind-version: ${{ steps.dotenv.outputs.kind_version }}
# kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
# helm-version: ${{ steps.dotenv.outputs.helm_version }}
# istio-version: ${{ steps.dotenv.outputs.istio_version }}
# kgateway-api-version: ${{ steps.dotenv.outputs.kgateway_api_version }}
# # Run the tests
# - id: run-tests
# name: Run Kubernetes e2e Tests
# uses: ./.github/actions/kubernetes-e2e-tests
# with:
# cluster-name: ${{ matrix.test.cluster-name }}
# test-args: ${{ matrix.test.go-test-args }}
# run-regex: ${{ matrix.test.go-test-run-regex }}
# istio-version: ${{ steps.dotenv.outputs.istio_version }}
# matrix-label: ${{ matrix.version-files.label }}
# end_to_end_tests_main:
# name: End-to-End (branch=main, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
# if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }}
# runs-on: ubuntu-22.04
# timeout-minutes: 180
# strategy:
# # Since we are running these on a schedule, there is no value in failing fast
# # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
# fail-fast: false
# matrix:
# test:
# # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
# # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
# - cluster-name: 'cluster-one'
# go-test-args: '-v -timeout=150m'
# # Specifying an empty regex means all tests will be run.
# go-test-run-regex: ""
# # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
# # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
# version-files:
# - label: 'min'
# file: './.github/workflows/.env/nightly-tests/min_versions.env'
# - label: 'max'
# file: './.github/workflows/.env/nightly-tests/max_versions.env'
# steps:
# - uses: actions/checkout@v4
# with:
# ref: main
# # The dotenv action is used to load key-value pairs from files.
# # In this case, the file is specified in the matrix and will contain the versions of the tools to use
# - name: Dotenv Action
# uses: falti/[email protected]
# id: dotenv
# with:
# path: ${{ matrix.version-files.file }}
# log-variables: true
# - name: Prep Go Runner
# uses: ./.github/actions/prep-go-runner
# # Set up the KinD cluster that the tests will use
# - id: setup-kind-cluster
# name: Setup KinD Cluster
# uses: ./.github/actions/setup-kind-cluster
# with:
# cluster-name: ${{ matrix.test.cluster-name }}
# kind-node-version: ${{ steps.dotenv.outputs.node_version }}
# kind-version: ${{ steps.dotenv.outputs.kind_version }}
# kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
# helm-version: ${{ steps.dotenv.outputs.helm_version }}
# istio-version: ${{ steps.dotenv.outputs.istio_version }}
# kgateway-api-version: ${{ steps.dotenv.outputs.kgateway_api_version }}
# # Run the tests
# - id: run-tests
# name: Run Kubernetes e2e Tests
# uses: ./.github/actions/kubernetes-e2e-tests
# with:
# cluster-name: ${{ matrix.test.cluster-name }}
# test-args: ${{ matrix.test.go-test-args }}
# run-regex: ${{ matrix.test.go-test-run-regex }}
# istio-version: ${{ steps.dotenv.outputs.istio_version }}
# matrix-label: ${{ matrix.version-files.label }}
# regression_tests_on_demand:
# name: on demand regression tests
# if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'workflow_initiating_branch' }}
# runs-on: ubuntu-22.04
# timeout-minutes: 60
# strategy:
# fail-fast: false
# matrix:
# kube-e2e-test-type: ['gateway', 'gloo', 'upgrade']
# kube-version: [ { node: 'v1.25.16@sha256:5da57dfc290ac3599e775e63b8b6c49c0c85d3fec771cd7d55b45fae14b38d3b', kubectl: 'v1.25.16', kind: 'v0.20.0', helm: 'v3.13.2' },
# { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }]
# image-variant:
# - standard
# steps:
# - uses: actions/checkout@v4
# with:
# ref: ${{ github.ref_name }}
# - uses: ./.github/actions/regression-tests
# regression_tests_main:
# name: main regression tests
# if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }}
# runs-on: ubuntu-22.04
# timeout-minutes: 60
# # Previously, there was an environment variable, RELEASED_VERSION="LATEST" set. This made use of some internal code:
# # https://github.com/kgateway-dev/kgateway/blob/main/test/kube2e/util.go#L229-L241
# # which modified our testing process to pull the latest beta release.
# #
# # NOW, however, running this job is the same as normal CI. (building a local chart, then using it)
# strategy:
# fail-fast: false
# matrix:
# # TODO:
# # As part of the end_to_end_tests_main job, we added support for importing versions from a .env file
# # We should extend the support/usage of those .env files to these other jobs.
# # The tests are currently in flux, and some of these regression tests are being migrated, so we decided
# # to limit the scope (and potentially unnecessary work) for now
# kube-e2e-test-type: ['gateway', 'gloo', 'upgrade']
# kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' },
# { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }]
# image-variant:
# - standard
# steps:
# - uses: actions/checkout@v4
# with:
# ref: main
# - uses: ./.github/actions/regression-tests
# performance_tests_on_demand:
# name: on demand performance tests
# if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'workflow_initiating_branch' }}
# runs-on: ubuntu-22.04
# timeout-minutes: 60
# steps:
# - uses: actions/checkout@v4
# with:
# ref: ${{ github.ref_name }}
# - uses: ./.github/actions/prep-go-runner
# - uses: ./.github/actions/performance-tests
# performance_tests_main:
# name: main performance tests
# if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }}
# runs-on: ubuntu-22.04
# timeout-minutes: 60
# steps:
# - uses: actions/checkout@v4
# with:
# ref: main
# - uses: ./.github/actions/prep-go-runner
# - uses: ./.github/actions/performance-tests