From 1ea67426ab59b594d2811893e16757fa1273c81d Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 30 Jan 2025 12:26:07 -0800 Subject: [PATCH] Support multiple DB, push hydrophone logs to test output Signed-off-by: Derek Nola --- tests/docker/conformance/conformance_test.go | 32 ++++++++-- tests/docker/test-helpers.go | 65 ++++++++++++++++++-- 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/tests/docker/conformance/conformance_test.go b/tests/docker/conformance/conformance_test.go index 650e021ca413..98ed8fa242a8 100644 --- a/tests/docker/conformance/conformance_test.go +++ b/tests/docker/conformance/conformance_test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "os/exec" "path/filepath" "runtime" "testing" @@ -14,6 +15,7 @@ import ( ) var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") +var db = flag.String("db", "etcd", "The database to use for the tests") var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests") var config *tester.TestConfig @@ -30,6 +32,7 @@ var _ = Describe("Conformance Tests", Ordered, func() { var err error config, err = tester.NewTestConfig(*k3sImage) Expect(err).NotTo(HaveOccurred()) + config.DBType = *db Expect(config.ProvisionServers(1)).To(Succeed()) Expect(config.ProvisionAgents(1)).To(Succeed()) Eventually(func() error { @@ -62,20 +65,25 @@ var _ = Describe("Conformance Tests", Ordered, func() { cmd := fmt.Sprintf("%s --focus=\"Conformance\" --skip=\"Serial|Flaky\" -p %d --extra-ginkgo-args=\"%s\" --kubeconfig %s", filepath.Join(config.TestDir, "hydrophone"), runtime.NumCPU()/2, - "--poll-progress-after=60s,--poll-progress-interval=30s", + "--poll-progress-after=60s,--poll-progress-interval=120s", config.KubeconfigFile) By("Hydrophone: " + cmd) - res, err := tester.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred(), res) + hc, err := StartCmd(cmd) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + if hc.ProcessState == nil { + return false + } + return hc.ProcessState.Success() + }, "40m", "1m").Should(BeTrue()) }) It("should run serial conformance tests", func() { if !*serial { Skip("Skipping serial conformance tests") } - cmd := fmt.Sprintf("%s -o %s --focus=\"Serial\" --skip=\"Flaky\" --extra-ginkgo-args=\"%s\" --kubeconfig %s", + cmd := fmt.Sprintf("%s --focus=\"Serial\" --skip=\"Flaky\" --extra-ginkgo-args=\"%s\" --kubeconfig %s", filepath.Join(config.TestDir, "hydrophone"), - filepath.Join(config.TestDir, "logs"), - "--poll-progress-after=60s,--poll-progress-interval=30s", + "--poll-progress-after=60s,--poll-progress-interval=120s", config.KubeconfigFile) By("Hydrophone: " + cmd) res, err := tester.RunCommand(cmd) @@ -95,3 +103,15 @@ var _ = AfterSuite(func() { config.Cleanup() } }) + +// StartCmd starts a command and pipes its output to +// the ginkgo Writr, with the expectation to poll the progress of the command +func StartCmd(cmd string) (*exec.Cmd, error) { + c := exec.Command("sh", "-c", cmd) + c.Stdout = GinkgoWriter + c.Stderr = GinkgoWriter + if err := c.Start(); err != nil { + return c, err + } + return c, nil +} diff --git a/tests/docker/test-helpers.go b/tests/docker/test-helpers.go index 9b265ea68b84..a9a91fa183ee 100644 --- a/tests/docker/test-helpers.go +++ b/tests/docker/test-helpers.go @@ -27,6 +27,7 @@ type TestConfig struct { KubeconfigFile string Token string K3sImage string + DBType string Servers []Server Agents []DockerNode ServerYaml string @@ -121,15 +122,24 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error { yamlMount = fmt.Sprintf("--mount type=bind,src=%s,dst=/etc/rancher/k3s/config.yaml", filepath.Join(config.TestDir, fmt.Sprintf("server-%d.yaml", i))) } - var joinOrStart string + var joinServer string + var dbConnect string + var err error if numOfServers > 0 { if i == 0 { - joinOrStart = "--cluster-init" + dbConnect, err = config.setupDatabase(true) + if err != nil { + return err + } } else { + dbConnect, err = config.setupDatabase(false) + if err != nil { + return err + } if config.Servers[0].URL == "" { return fmt.Errorf("first server URL is empty") } - joinOrStart = fmt.Sprintf("--server %s", config.Servers[0].URL) + joinServer = fmt.Sprintf("--server %s", config.Servers[0].URL) } } newServer := Server{ @@ -169,7 +179,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error { } // The pipe requires that we use sh -c with "" to run the command cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"", - joinOrStart+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))) + dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))) if out, err := newServer.RunCmdOnNode(cmd); err != nil { return fmt.Errorf("failed to start server: %s: %v", out, err) } @@ -189,7 +199,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error { os.Getenv("REGISTRY_CLUSTER_ARGS"), yamlMount, config.K3sImage, - "server", joinOrStart, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ") + "server", dbConnect, joinServer, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ") if out, err := RunCommand(dRun); err != nil { return fmt.Errorf("failed to run server container: %s: %v", out, err) } @@ -220,6 +230,39 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error { return copyAndModifyKubeconfig(config) } +// setupDatabase will start the configured database if startDB is true, +// and return the correct flag to join the configured database +func (config *TestConfig) setupDatabase(startDB bool) (string, error) { + + joinFlag := "" + startCmd := "" + switch config.DBType { + case "mysql": + startCmd = "docker run -d --name mysql -e MYSQL_ROOT_PASSWORD=docker -p 3306:3306 mysql:8.4" + joinFlag = "--datastore-endpoint='mysql://root:docker@tcp(172.17.0.1:3306)/k3s'" + case "postgres": + startCmd = "docker run -d --name postgres -e POSTGRES_PASSWORD=docker -p 5432:5432 postgres:16-alpine" + joinFlag = "--datastore-endpoint='postgres://postgres:docker@tcp(172.17.0.1:5432)/k3s'" + case "etcd": + fallthrough + default: + if startDB { + joinFlag = "--cluster-init" + } + config.DBType = "etcd" + } + + if startDB { + if out, err := RunCommand(startCmd); err != nil { + return "", fmt.Errorf("failed to start %s container: %s: %v", config.DBType, out, err) + } + // Wait for DB to start + time.Sleep(10 * time.Second) + } + return joinFlag, nil + +} + func (config *TestConfig) ProvisionAgents(numOfAgents int) error { if err := checkVersionSkew(config); err != nil { return err @@ -346,6 +389,18 @@ func (config *TestConfig) Cleanup() error { } } + // Stop DB if it was started + if config.DBType != "etcd" { + cmd := fmt.Sprintf("docker stop %s", config.DBType) + if _, err := RunCommand(cmd); err != nil { + errs = append(errs, fmt.Errorf("failed to stop %s: %v", config.DBType, err)) + } + cmd = fmt.Sprintf("docker rm %s", config.DBType) + if _, err := RunCommand(cmd); err != nil { + errs = append(errs, fmt.Errorf("failed to remove %s: %v", config.DBType, err)) + } + } + // Error out if we hit any issues if len(errs) > 0 { return fmt.Errorf("cleanup failed: %v", errs)