run many reset tests

.github/workflows/pull-request.yaml

@@ -165,30 +165,39 @@ jobs:
       fail-fast: false
       matrix:
         tests:
-          - TestSingleNodeInstallation
-          - TestSingleNodeInstallationAlmaLinux8
-          - TestSingleNodeInstallationDebian12
-          - TestSingleNodeInstallationCentos8Stream
-          - TestVersion
-          - TestHostPreflight
-          - TestUnsupportedOverrides
-          - TestMultiNodeInstallation
+#          - TestSingleNodeInstallation
+#          - TestSingleNodeInstallationAlmaLinux8
+#          - TestSingleNodeInstallationDebian12
+#          - TestSingleNodeInstallationCentos8Stream
+#          - TestVersion
+#          - TestHostPreflight
+#          - TestUnsupportedOverrides
+#          - TestMultiNodeInstallation
           - TestMultiNodeReset
-          - TestCommandsRequireSudo
-          - TestInstallWithoutEmbed
-          - TestInstallFromReplicatedApp
+          - TestMultiNodeReset1
+          - TestMultiNodeReset2
+          - TestMultiNodeReset3
+          - TestMultiNodeReset4
+          - TestMultiNodeReset5
+          - TestMultiNodeReset6
+          - TestMultiNodeReset7
+          - TestMultiNodeReset8
+          - TestMultiNodeReset9
+#          - TestCommandsRequireSudo
+#          - TestInstallWithoutEmbed
+#          - TestInstallFromReplicatedApp
           - TestResetAndReinstall
           - TestResetAndReinstallAirgap
-          - TestCollectSupportBundle
-          - TestOldVersionUpgrade
-          - TestMaterialize
-          - TestLocalArtifactMirror
-          - TestSingleNodeAirgapUpgradeUbuntuJammy
-          - TestInstallSnapshotFromReplicatedApp
-          - TestMultiNodeAirgapUpgradeUbuntuJammy
-          - TestSingleNodeDisasterRecovery
-          - TestSingleNodeResumeDisasterRecovery
-          - TestSingleNodeAirgapDisasterRecovery
+#          - TestCollectSupportBundle
+#          - TestOldVersionUpgrade
+#          - TestMaterialize
+#          - TestLocalArtifactMirror
+#          - TestSingleNodeAirgapUpgradeUbuntuJammy
+#          - TestInstallSnapshotFromReplicatedApp
+#          - TestMultiNodeAirgapUpgradeUbuntuJammy
+#          - TestSingleNodeDisasterRecovery
+#          - TestSingleNodeResumeDisasterRecovery
+#          - TestSingleNodeAirgapDisasterRecovery
     steps:
     - name: Checkout
       uses: actions/checkout@v4

cmd/embedded-cluster/uninstall.go

@@ -14,6 +14,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -200,26 +201,51 @@ func (h *hostInfo) checkResetSafety(c *cli.Context) (bool, string, error) {
 	return true, "", nil
 }
 
-// leaveEtcdcluster uses k0s to attempt to leave the etcd cluster
-func (h *hostInfo) leaveEtcdcluster() error {
-
-	// if we're the only etcd member we don't need to leave the cluster
+func (h *hostInfo) isLastEtcdMember() (bool, error) {
 	out, err := exec.Command(k0s, "etcd", "member-list").Output()
 	if err != nil {
-		return err
+		return false, err
 	}
 	memberlist := etcdMembers{}
 	err = json.Unmarshal(out, &memberlist)
 	if err != nil {
-		return err
+		return false, err
+	}
+
+	if memberlist.Members[h.Hostname] == "" {
+		return false, nil // this is not a etcd member
+	}
+	if len(memberlist.Members) == 1 {
+		return true, nil // we are an etcd member and the only one
+	}
+	return false, nil // there is more than one etcd member remaining
+}
+
+// leaveEtcdcluster uses k0s to attempt to leave the etcd cluster
+func (h *hostInfo) leaveEtcdcluster() error {
+	// if we're the only etcd member we don't need to leave the cluster
+	lastMember, err := h.isLastEtcdMember()
+	if err != nil {
+		return fmt.Errorf("unable to check if last etcd member: %w", err)
 	}
-	if len(memberlist.Members) == 1 && memberlist.Members[h.Hostname] != "" {
+	if lastMember {
 		return nil
 	}
 
-	out, err = exec.Command(k0s, "etcd", "leave").CombinedOutput()
+	var lasterr error
+	var out []byte
+	backoff := wait.Backoff{Steps: 5, Duration: time.Second, Factor: 2, Jitter: 0.1}
+	err = wait.ExponentialBackoff(backoff, func() (bool, error) {
+		out, err = exec.Command(k0s, "etcd", "leave").CombinedOutput()
+		if err != nil {
+			lasterr = fmt.Errorf("%w: %s", err, string(out))
+			return false, nil
+		}
+		return true, nil
+
+	})
 	if err != nil {
-		return fmt.Errorf("unable to leave etcd cluster: %w, %s", err, string(out))
+		return fmt.Errorf("unable to leave etcd cluster: %w", lasterr)
 	}
 	return nil
 }

e2e/reset_test.go

@@ -116,3 +116,39 @@ func TestMultiNodeReset(t *testing.T) {
 
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
+
+func TestMultiNodeReset1(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset2(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset3(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset4(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset5(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset6(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset7(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset8(t *testing.T) {
+	TestMultiNodeReset(t)
+}
+
+func TestMultiNodeReset9(t *testing.T) {
+	TestMultiNodeReset(t)
+}