Support multiple DB, push hydrophone logs to test output

Signed-off-by: Derek Nola <[email protected]>

tests/docker/conformance/conformance_test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -14,6 +15,7 @@ import (
 )
 
 var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var db = flag.String("db", "etcd", "The database to use for the tests")
 var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests")
 var config *tester.TestConfig
 
@@ -30,6 +32,7 @@ var _ = Describe("Conformance Tests", Ordered, func() {
 			var err error
 			config, err = tester.NewTestConfig(*k3sImage)
 			Expect(err).NotTo(HaveOccurred())
+			config.DBType = *db
 			Expect(config.ProvisionServers(1)).To(Succeed())
 			Expect(config.ProvisionAgents(1)).To(Succeed())
 			Eventually(func() error {
@@ -62,20 +65,25 @@ var _ = Describe("Conformance Tests", Ordered, func() {
 			cmd := fmt.Sprintf("%s --focus=\"Conformance\" --skip=\"Serial|Flaky\" -p %d --extra-ginkgo-args=\"%s\" --kubeconfig %s",
 				filepath.Join(config.TestDir, "hydrophone"),
 				runtime.NumCPU()/2,
-				"--poll-progress-after=60s,--poll-progress-interval=30s",
+				"--poll-progress-after=60s,--poll-progress-interval=120s",
 				config.KubeconfigFile)
 			By("Hydrophone: " + cmd)
-			res, err := tester.RunCommand(cmd)
-			Expect(err).NotTo(HaveOccurred(), res)
+			hc, err := StartCmd(cmd)
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(func() bool {
+				if hc.ProcessState == nil {
+					return false
+				}
+				return hc.ProcessState.Success()
+			}, "40m", "1m").Should(BeTrue())
 		})
 		It("should run serial conformance tests", func() {
 			if !*serial {
 				Skip("Skipping serial conformance tests")
 			}
-			cmd := fmt.Sprintf("%s -o %s --focus=\"Serial\" --skip=\"Flaky\"  --extra-ginkgo-args=\"%s\" --kubeconfig %s",
+			cmd := fmt.Sprintf("%s --focus=\"Serial\" --skip=\"Flaky\"  --extra-ginkgo-args=\"%s\" --kubeconfig %s",
 				filepath.Join(config.TestDir, "hydrophone"),
-				filepath.Join(config.TestDir, "logs"),
-				"--poll-progress-after=60s,--poll-progress-interval=30s",
+				"--poll-progress-after=60s,--poll-progress-interval=120s",
 				config.KubeconfigFile)
 			By("Hydrophone: " + cmd)
 			res, err := tester.RunCommand(cmd)
@@ -95,3 +103,15 @@ var _ = AfterSuite(func() {
 		config.Cleanup()
 	}
 })
+
+// StartCmd starts a command and pipes its output to
+// the ginkgo Writr, with the expectation to poll the progress of the command
+func StartCmd(cmd string) (*exec.Cmd, error) {
+	c := exec.Command("sh", "-c", cmd)
+	c.Stdout = GinkgoWriter
+	c.Stderr = GinkgoWriter
+	if err := c.Start(); err != nil {
+		return c, err
+	}
+	return c, nil
+}

tests/docker/test-helpers.go

@@ -27,6 +27,7 @@ type TestConfig struct {
 	KubeconfigFile string
 	Token          string
 	K3sImage       string
+	DBType         string
 	Servers        []Server
 	Agents         []DockerNode
 	ServerYaml     string
@@ -121,15 +122,24 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			yamlMount = fmt.Sprintf("--mount type=bind,src=%s,dst=/etc/rancher/k3s/config.yaml", filepath.Join(config.TestDir, fmt.Sprintf("server-%d.yaml", i)))
 		}
 
-		var joinOrStart string
+		var joinServer string
+		var dbConnect string
+		var err error
 		if numOfServers > 0 {
 			if i == 0 {
-				joinOrStart = "--cluster-init"
+				dbConnect, err = config.setupDatabase(true)
+				if err != nil {
+					return err
+				}
 			} else {
+				dbConnect, err = config.setupDatabase(false)
+				if err != nil {
+					return err
+				}
 				if config.Servers[0].URL == "" {
 					return fmt.Errorf("first server URL is empty")
 				}
-				joinOrStart = fmt.Sprintf("--server %s", config.Servers[0].URL)
+				joinServer = fmt.Sprintf("--server %s", config.Servers[0].URL)
 			}
 		}
 		newServer := Server{
@@ -169,7 +179,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			}
 			// The pipe requires that we use sh -c with "" to run the command
 			cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
-				joinOrStart+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
+				dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
 			if out, err := newServer.RunCmdOnNode(cmd); err != nil {
 				return fmt.Errorf("failed to start server: %s: %v", out, err)
 			}
@@ -189,7 +199,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 				os.Getenv("REGISTRY_CLUSTER_ARGS"),
 				yamlMount,
 				config.K3sImage,
-				"server", joinOrStart, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
+				"server", dbConnect, joinServer, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
 			if out, err := RunCommand(dRun); err != nil {
 				return fmt.Errorf("failed to run server container: %s: %v", out, err)
 			}
@@ -220,6 +230,39 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 	return copyAndModifyKubeconfig(config)
 }
 
+// setupDatabase will start the configured database if startDB is true,
+// and return the correct flag to join the configured database
+func (config *TestConfig) setupDatabase(startDB bool) (string, error) {
+
+	joinFlag := ""
+	startCmd := ""
+	switch config.DBType {
+	case "mysql":
+		startCmd = "docker run -d --name mysql -e MYSQL_ROOT_PASSWORD=docker -p 3306:3306 mysql:8.4"
+		joinFlag = "--datastore-endpoint='mysql://root:docker@tcp(172.17.0.1:3306)/k3s'"
+	case "postgres":
+		startCmd = "docker run -d --name postgres -e POSTGRES_PASSWORD=docker -p 5432:5432 postgres:16-alpine"
+		joinFlag = "--datastore-endpoint='postgres://postgres:docker@tcp(172.17.0.1:5432)/k3s'"
+	case "etcd":
+		fallthrough
+	default:
+		if startDB {
+			joinFlag = "--cluster-init"
+		}
+		config.DBType = "etcd"
+	}
+
+	if startDB {
+		if out, err := RunCommand(startCmd); err != nil {
+			return "", fmt.Errorf("failed to start %s container: %s: %v", config.DBType, out, err)
+		}
+		// Wait for DB to start
+		time.Sleep(10 * time.Second)
+	}
+	return joinFlag, nil
+
+}
+
 func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
 	if err := checkVersionSkew(config); err != nil {
 		return err
@@ -346,6 +389,18 @@ func (config *TestConfig) Cleanup() error {
 		}
 	}
 
+	// Stop DB if it was started
+	if config.DBType != "etcd" {
+		cmd := fmt.Sprintf("docker stop %s", config.DBType)
+		if _, err := RunCommand(cmd); err != nil {
+			errs = append(errs, fmt.Errorf("failed to stop %s: %v", config.DBType, err))
+		}
+		cmd = fmt.Sprintf("docker rm %s", config.DBType)
+		if _, err := RunCommand(cmd); err != nil {
+			errs = append(errs, fmt.Errorf("failed to remove %s: %v", config.DBType, err))
+		}
+	}
+
 	// Error out if we hit any issues
 	if len(errs) > 0 {
 		return fmt.Errorf("cleanup failed: %v", errs)