Merge pull request #1136 from NeerajNagure/migration-e2e

Migration for dryrun tests

.github/workflows/test.yaml

@@ -258,6 +258,12 @@ jobs:
       run: |
         ./hack/e2e-test.sh kwokctl/kwokctl_with_dryrun
 
+    - name: Test e2e dry run
+      if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime == 'binary' }}
+      shell: bash
+      run: |
+        ./hack/e2e-test.sh e2e/kwokctl/dryrun
+
     - name: Test e2e
       shell: bash
       run: |

hack/update-all.sh

@@ -88,6 +88,11 @@ if [[ "${UPDATE_HELM_CHARTS:-true}" == "true" ]]; then
   "${ROOT_DIR}"/hack/update-helm-charts.sh || failed+=(helm-charts)
 fi
 
+if [[ "${UPDATE_DRY_RUN_TESTDATA:-true}" == "true" ]]; then
+  echo "[*] Update testdata..."
+  "${ROOT_DIR}"/hack/update-testdata.sh || failed+=(testdata)
+fi
+
 if [[ "${#failed[@]}" != 0 ]]; then
   echo "Update failed for: ${failed[*]}"
   exit 1

hack/update-testdata.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+DIR="$(dirname "${BASH_SOURCE[0]}")"
+
+ROOT_DIR="$(realpath "${DIR}/..")"
+
+function update() {
+  go test -v ./test/e2e/kwokctl/dryrun -args --update-testdata
+}
+
+cd "${ROOT_DIR}" && update

hack/verify-all.sh

@@ -108,6 +108,11 @@ if [[ "${VERIFY_DEPENDENCIES_VERSION:-true}" == "true" ]]; then
   "${ROOT_DIR}"/hack/verify-dependencies-version.sh || failed+=(dependencies-version)
 fi
 
+if [[ "${VERIFY_DRY_RUN_TESTDATA:-true}" == "true" ]]; then
+  echo "[*] Verifying testdata..."
+  "${ROOT_DIR}"/hack/verify-testdata.sh || failed+=(testdata)
+fi
+
 # exit based on verify scripts
 if [[ "${#failed[@]}" != 0 ]]; then
   echo "Verify failed for: ${failed[*]}"

hack/verify-testdata.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+DIR="$(dirname "${BASH_SOURCE[0]}")"
+
+ROOT_DIR="$(realpath "${DIR}/..")"
+
+function check() {
+  if [[ "${UPDATE_DRY_RUN_TESTDATA:-"false"}" == "true" ]]; then
+    "${ROOT_DIR}"/hack/update-testdata.sh
+  fi
+  git --no-pager diff --exit-code
+}
+
+cd "${ROOT_DIR}" && check

test/e2e/dryrun.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"os/exec"
+	"regexp"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	"sigs.k8s.io/kwok/pkg/utils/path"
+)
+
+var (
+	emptyLine  = regexp.MustCompile("\n{2,}")
+	homeDir, _ = os.UserHomeDir()
+)
+
+func formatCmdOutput(got, clusterName, rootDir string) string {
+	got = strings.ReplaceAll(got, clusterName, "<CLUSTER_NAME>")
+	got = strings.ReplaceAll(got, rootDir, "<ROOT_DIR>")
+	got = strings.ReplaceAll(got, runtime.GOOS, "<OS>")
+	got = strings.ReplaceAll(got, runtime.GOARCH, "<ARCH>")
+	got = strings.ReplaceAll(got, ".zip", ".<TAR>")
+	got = strings.ReplaceAll(got, ".tar.gz", ".<TAR>")
+	got = strings.ReplaceAll(got, homeDir, "~")
+	got = emptyLine.ReplaceAllLiteralString(got, "\n")
+	return got
+}
+
+func executeCommand(args []string, absPath string, clusterName string, kwokctlPath string, rootDir string, updateTestdata bool) (string, error) {
+	testdataPath := path.Join(rootDir, absPath)
+	expected, err := os.ReadFile(testdataPath)
+	if err != nil {
+		return "", err
+	}
+	cmd := exec.Command(kwokctlPath, args...)
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	want := string(expected)
+	got := string(output)
+	got = formatCmdOutput(got, clusterName, rootDir)
+	if diff := cmp.Diff(got, want); diff != "" {
+		if updateTestdata {
+			err = os.WriteFile(testdataPath, []byte(got), fs.FileMode(0644))
+			if err != nil {
+				return "", err
+			}
+		} else {
+			return diff, nil
+		}
+	}
+	return "", nil
+}
+
+func CaseDryrun(clusterName string, kwokctlPath string, rootDir string, clusterRuntime string, updateTestdata bool) *features.FeatureBuilder {
+	f := features.New("Dry run")
+	f = f.Assess("test cluster dryrun", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		absPath := "test/e2e/kwokctl/dryrun/testdata/" + clusterRuntime + "/create_cluster.txt"
+		args := []string{
+			"create", "cluster", "--dry-run", "--name", clusterName, "--timeout=30m",
+			"--wait=30m", "--quiet-pull", "--disable-qps-limits", "--kube-authorization=false",
+			"--runtime", clusterRuntime,
+		}
+		diff, err := executeCommand(args, absPath, clusterName, kwokctlPath, rootDir, updateTestdata)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff != "" {
+			t.Fatalf("Expected vs got:\n%s", diff)
+		}
+		return ctx
+	})
+	return f
+}
+
+func CaseDryrunWithExtra(clusterName string, kwokctlPath string, rootDir string, clusterRuntime string, updateTestdata bool) *features.FeatureBuilder {
+	f := features.New("Dry run with extra")
+	f = f.Assess("test cluster dryrun with extra", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		absPath := "test/e2e/kwokctl/dryrun/testdata/" + clusterRuntime + "/create_cluster_with_extra.txt"
+		extraPath := path.Join(rootDir, "test/kwokctl/testdata/extra.yaml")
+		args := []string{
+			"create", "cluster", "--dry-run", "--name", clusterName, "--timeout=30m",
+			"--wait=30m", "--quiet-pull", "--disable-qps-limits", "--runtime", clusterRuntime,
+			"--config", extraPath,
+		}
+		diff, err := executeCommand(args, absPath, clusterName, kwokctlPath, rootDir, updateTestdata)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff != "" {
+			t.Fatalf("Expected vs got:\n%s", diff)
+		}
+		return ctx
+	})
+	return f
+}
+
+func CaseDryrunWithVerbosity(clusterName string, kwokctlPath string, rootDir string, clusterRuntime string, updateTestdata bool) *features.FeatureBuilder {
+	f := features.New("Dry run with verbosity")
+	f = f.Assess("test cluster dryrun with verbosity", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		absPath := "test/e2e/kwokctl/dryrun/testdata/" + clusterRuntime + "/create_cluster_with_verbosity.txt"
+		kubeAuditPath := path.Join(rootDir, "test/kwokctl/audit-policy.yaml")
+		schedulerConfigPath := path.Join(rootDir, "test/kwokctl/scheduler-config.yaml")
+		args := []string{
+			"create", "cluster", "--dry-run", "--name", clusterName, "--timeout=30m", "--wait=30m",
+			"--quiet-pull", "--disable-qps-limits", "--runtime", clusterRuntime,
+			"--prometheus-port=9090", "--jaeger-port=16686", "--dashboard-port=8000",
+			"--enable-metrics-server", "--kube-audit-policy", kubeAuditPath,
+			"--kube-scheduler-config", schedulerConfigPath,
+		}
+		diff, err := executeCommand(args, absPath, clusterName, kwokctlPath, rootDir, updateTestdata)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff != "" {
+			t.Fatalf("Expected vs got:\n%s", diff)
+		}
+		return ctx
+	})
+	return f
+}

test/e2e/kwokctl/dryrun/dryrun_binary_test.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dryrun_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kwok/test/e2e"
+)
+
+func TestBinaryDryRun(t *testing.T) {
+	f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "binary", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}
+
+func TestBinaryDryRunWithExtra(t *testing.T) {
+	f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "binary", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}
+
+func TestBinaryDryRunWithVerbosity(t *testing.T) {
+	f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "binary", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}

test/e2e/kwokctl/dryrun/dryrun_docker_test.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dryrun_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kwok/test/e2e"
+)
+
+func TestDockerDryRun(t *testing.T) {
+	f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "docker", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}
+
+func TestDockerDryRunWithExtra(t *testing.T) {
+	f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "docker", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}
+
+func TestDockerDryRunWithVerbosity(t *testing.T) {
+	f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "docker", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}

test/e2e/kwokctl/dryrun/dryrun_kind_podman_test.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dryrun_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kwok/test/e2e"
+)
+
+func TestKindPodmanDryRun(t *testing.T) {
+	f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "kind-podman", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}
+
+func TestKindPodmanDryRunWithExtra(t *testing.T) {
+	f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "kind-podman", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}
+
+func TestKindPodmanDryRunWithVerbosity(t *testing.T) {
+	f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "kind-podman", updateTestdata).Feature()
+	testEnv.Test(t, f0)
+}