Integration Test: Startup (k3s-io#5630)

* New startup integration test
* Add testing section to PR template
* Move helper functions to direct k8s client calls

Signed-off-by: Derek Nola <[email protected]>

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,6 +13,11 @@
 
 <!-- How can the changes be verified? Please provide whatever additional information necessary to help verify the proposed changes. -->
 
+#### Testing ####
+
+<!-- Is this change covered by testing? If not, consider adding a Unit or Integration test. -->
+<!-- See https://github.com/k3s-io/k3s/blob/master/tests/TESTING.md for more info -->
+
 #### Linked Issues ####
 
 <!-- Link any related issues, pull-requests, or commit hashes that are relevant to this pull request. If you are opening a PR without a corresponding issue please consider creating one first, at https://github.com/k3s-io/k3s/issues . A functional example will greatly help QA with verifying/reproducing a bug or testing new features. -->

tests/integration/certrotation/certrotation_int_test.go

@@ -34,9 +34,9 @@ var _ = Describe("certificate rotation", func() {
 	})
 	When("a new server is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get pods -A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("get certificate hash", func() {
 			// get md5sum of the CA certs
@@ -60,9 +60,9 @@ var _ = Describe("certificate rotation", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "360s", "5s").Should(Succeed())
 		})
 		It("get certificate hash", func() {
 			// get md5sum of the CA certs

tests/integration/dualstack/dualstack_int_test.go

@@ -39,9 +39,9 @@ var _ = Describe("dual stack", func() {
 	})
 	When("a ipv4 and ipv6 cidr is present", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "10s").Should(Succeed())
 		})
 		It("creates pods with two IPs", func() {
 			podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}")

tests/integration/etcdrestore/etcd_restore_int_test.go

@@ -32,9 +32,9 @@ var _ = Describe("etcd snapshot restore", func() {
 	})
 	When("a snapshot is restored on existing node", func() {
 		It("etcd starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("create a workload", func() {
 			result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml")
@@ -79,9 +79,9 @@ var _ = Describe("etcd snapshot restore", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "360s", "5s").Should(Succeed())
 		})
 		It("Make sure Workload 1 exists", func() {
 			Eventually(func() (string, error) {

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -33,9 +33,9 @@ var _ = Describe("etcd snapshots", func() {
 	})
 	When("a new etcd is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get pods -A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "10s").Should(Succeed())
 		})
 		It("saves an etcd snapshot", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save")).

tests/integration/integration.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -15,6 +16,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 // Compile-time variable
@@ -120,6 +125,67 @@ func K3sServerArgs() []string {
 	return args
 }
 
+// K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error
+func K3sDefaultDeployments() error {
+	return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"})
+}
+
+// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error
+func CheckDeployments(deployments []string) error {
+
+	deploymentSet := make(map[string]bool)
+	for _, d := range deployments {
+		deploymentSet[d] = false
+	}
+
+	client, err := k8sClient()
+	if err != nil {
+		return err
+	}
+	deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, deployment := range deploymentList.Items {
+		if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas {
+			deploymentSet[deployment.Name] = true
+		}
+	}
+	for d, found := range deploymentSet {
+		if !found {
+			return fmt.Errorf("failed to deploy %s", d)
+		}
+	}
+
+	return nil
+}
+
+func ParsePods() ([]corev1.Pod, error) {
+	clientSet, err := k8sClient()
+	if err != nil {
+		return nil, err
+	}
+	pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return pods.Items, nil
+}
+
+func ParseNodes() ([]corev1.Node, error) {
+	clientSet, err := k8sClient()
+	if err != nil {
+		return nil, err
+	}
+	nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return nodes.Items, nil
+}
+
 func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool {
 	for scanner.Scan() {
 		if strings.Contains(scanner.Text(), target) {
@@ -158,7 +224,6 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 }
 
 // K3sKillServer terminates the running K3s server and its children
-// and unlocks the file for other tests
 func K3sKillServer(server *K3sServer) error {
 	pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
 	if err != nil {
@@ -180,9 +245,10 @@ func K3sKillServer(server *K3sServer) error {
 	return nil
 }
 
-// K3sCleanup attempts to cleanup networking and files leftover from an integration test
-// this is similar to the k3s-killall.sh script, but we dynamically generate that on
-// install, so we don't have access to it in testing.
+// K3sCleanup unlocks the test-lock and
+// attempts to cleanup networking and files leftover from an integration test.
+// This is similar to the k3s-killall.sh script, but we dynamically generate that on
+// install, so we don't have access to it during testing.
 func K3sCleanup(k3sTestLock int, dataDir string) error {
 	if cni0Link, err := netlink.LinkByName("cni0"); err == nil {
 		links, _ := netlink.LinkList()
@@ -206,7 +272,10 @@ func K3sCleanup(k3sTestLock int, dataDir string) error {
 	if err := os.RemoveAll(dataDir); err != nil {
 		return err
 	}
-	return flock.Release(k3sTestLock)
+	if k3sTestLock != -1 {
+		return flock.Release(k3sTestLock)
+	}
+	return nil
 }
 
 // RunCommand Runs command on the host
@@ -220,3 +289,15 @@ func RunCommand(cmd string) (string, error) {
 	}
 	return out.String(), nil
 }
+
+func k8sClient() (*kubernetes.Clientset, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml")
+	if err != nil {
+		return nil, err
+	}
+	clientSet, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return clientSet, nil
+}

tests/integration/localstorage/localstorage_int_test.go

@@ -34,9 +34,9 @@ var _ = Describe("local storage", func() {
 	})
 	When("a new local storage is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
 		})
 		It("creates a new pvc", func() {
 			result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml")

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -34,9 +34,9 @@ var _ = Describe("secrets encryption rotation", func() {
 	})
 	When("A server starts with secrets encryption", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("it creates a encryption key", func() {
 			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
@@ -65,9 +65,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("rotates the keys", func() {
 			Eventually(func() (string, error) {
@@ -89,9 +89,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {
@@ -123,9 +123,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {