diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index d5f58484e..62c47cd31 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -165,30 +165,39 @@ jobs: fail-fast: false matrix: tests: - - TestSingleNodeInstallation - - TestSingleNodeInstallationAlmaLinux8 - - TestSingleNodeInstallationDebian12 - - TestSingleNodeInstallationCentos8Stream - - TestVersion - - TestHostPreflight - - TestUnsupportedOverrides - - TestMultiNodeInstallation +# - TestSingleNodeInstallation +# - TestSingleNodeInstallationAlmaLinux8 +# - TestSingleNodeInstallationDebian12 +# - TestSingleNodeInstallationCentos8Stream +# - TestVersion +# - TestHostPreflight +# - TestUnsupportedOverrides +# - TestMultiNodeInstallation - TestMultiNodeReset - - TestCommandsRequireSudo - - TestInstallWithoutEmbed - - TestInstallFromReplicatedApp + - TestMultiNodeReset1 + - TestMultiNodeReset2 + - TestMultiNodeReset3 + - TestMultiNodeReset4 + - TestMultiNodeReset5 + - TestMultiNodeReset6 + - TestMultiNodeReset7 + - TestMultiNodeReset8 + - TestMultiNodeReset9 +# - TestCommandsRequireSudo +# - TestInstallWithoutEmbed +# - TestInstallFromReplicatedApp - TestResetAndReinstall - TestResetAndReinstallAirgap - - TestCollectSupportBundle - - TestOldVersionUpgrade - - TestMaterialize - - TestLocalArtifactMirror - - TestSingleNodeAirgapUpgradeUbuntuJammy - - TestInstallSnapshotFromReplicatedApp - - TestMultiNodeAirgapUpgradeUbuntuJammy - - TestSingleNodeDisasterRecovery - - TestSingleNodeResumeDisasterRecovery - - TestSingleNodeAirgapDisasterRecovery +# - TestCollectSupportBundle +# - TestOldVersionUpgrade +# - TestMaterialize +# - TestLocalArtifactMirror +# - TestSingleNodeAirgapUpgradeUbuntuJammy +# - TestInstallSnapshotFromReplicatedApp +# - TestMultiNodeAirgapUpgradeUbuntuJammy +# - TestSingleNodeDisasterRecovery +# - TestSingleNodeResumeDisasterRecovery +# - TestSingleNodeAirgapDisasterRecovery steps: - name: Checkout uses: actions/checkout@v4 diff --git a/cmd/embedded-cluster/uninstall.go b/cmd/embedded-cluster/uninstall.go index bc30fb772..38dda2cb7 100644 --- a/cmd/embedded-cluster/uninstall.go +++ b/cmd/embedded-cluster/uninstall.go @@ -14,6 +14,7 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -200,26 +201,51 @@ func (h *hostInfo) checkResetSafety(c *cli.Context) (bool, string, error) { return true, "", nil } -// leaveEtcdcluster uses k0s to attempt to leave the etcd cluster -func (h *hostInfo) leaveEtcdcluster() error { - - // if we're the only etcd member we don't need to leave the cluster +func (h *hostInfo) isLastEtcdMember() (bool, error) { out, err := exec.Command(k0s, "etcd", "member-list").Output() if err != nil { - return err + return false, err } memberlist := etcdMembers{} err = json.Unmarshal(out, &memberlist) if err != nil { - return err + return false, err + } + + if memberlist.Members[h.Hostname] == "" { + return false, nil // this is not a etcd member + } + if len(memberlist.Members) == 1 { + return true, nil // we are an etcd member and the only one + } + return false, nil // there is more than one etcd member remaining +} + +// leaveEtcdcluster uses k0s to attempt to leave the etcd cluster +func (h *hostInfo) leaveEtcdcluster() error { + // if we're the only etcd member we don't need to leave the cluster + lastMember, err := h.isLastEtcdMember() + if err != nil { + return fmt.Errorf("unable to check if last etcd member: %w", err) } - if len(memberlist.Members) == 1 && memberlist.Members[h.Hostname] != "" { + if lastMember { return nil } - out, err = exec.Command(k0s, "etcd", "leave").CombinedOutput() + var lasterr error + var out []byte + backoff := wait.Backoff{Steps: 5, Duration: time.Second, Factor: 2, Jitter: 0.1} + err = wait.ExponentialBackoff(backoff, func() (bool, error) { + out, err = exec.Command(k0s, "etcd", "leave").CombinedOutput() + if err != nil { + lasterr = fmt.Errorf("%w: %s", err, string(out)) + return false, nil + } + return true, nil + + }) if err != nil { - return fmt.Errorf("unable to leave etcd cluster: %w, %s", err, string(out)) + return fmt.Errorf("unable to leave etcd cluster: %w", lasterr) } return nil } diff --git a/e2e/reset_test.go b/e2e/reset_test.go index 54d15eb43..4b61a37b2 100644 --- a/e2e/reset_test.go +++ b/e2e/reset_test.go @@ -116,3 +116,39 @@ func TestMultiNodeReset(t *testing.T) { t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } + +func TestMultiNodeReset1(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset2(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset3(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset4(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset5(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset6(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset7(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset8(t *testing.T) { + TestMultiNodeReset(t) +} + +func TestMultiNodeReset9(t *testing.T) { + TestMultiNodeReset(t) +}