From af8099f0d853451524ab3252428f9d34b0b2659a Mon Sep 17 00:00:00 2001
From: Nesma Badr <nesmabadr@aucegypt.edu>
Date: Thu, 17 Aug 2023 16:48:03 +0200
Subject: [PATCH] Client cache invalidation when remote cluster is unreachable
 (#806)

* invalidate client cache on remote cluster connection failure

* Add E2E test

* fix typo

* fix test

* fix test

* fix test

* Empty commit

* debug test

* Empty commit

* Empty commit

* debug test

* debug test

* Empty commit

* Empty commit

* debug test

* debug test

* debug test

* Empty commit

* Empty commit

* debug test

* debug test

* debug test

* Empty commit

* Empty commit

* debug test

* Empty commit

* Empty commit

* debug test

* debug test

* Empty commit

* remove merge command

* remove rewriting the skr config to github env file

* code review

* code review

* code review

* code review

* code review

* Empty commit

* code review

* code review

* fix

* fix

* fix
---
 controllers/kyma_controller.go       | 29 +++++++++++-------------
 tests/e2e_test/kyma_deletion_test.go | 34 ++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/controllers/kyma_controller.go b/controllers/kyma_controller.go
index b0e063cdf8..6e92625bbf 100644
--- a/controllers/kyma_controller.go
+++ b/controllers/kyma_controller.go
@@ -121,25 +121,22 @@ func (r *KymaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	return r.reconcile(ctx, kyma)
 }
 
-func (r *KymaReconciler) handleRemoteClusterConnectionErrorOnDeletion(
-	ctx context.Context, kyma *v1beta2.Kyma, err error) (
-	ctrl.Result, error,
-) {
+func (r *KymaReconciler) handleRemoteClusterConnectionError(
+	ctx context.Context, kyma *v1beta2.Kyma, err error,
+) error {
+	if util.IsConnectionRefused(err) {
+		r.RemoteClientCache.Del(client.ObjectKeyFromObject(kyma))
+		return err
+	}
+
 	if !kyma.DeletionTimestamp.IsZero() {
-		if util.IsConnectionRefused(err) {
-			r.RemoteClientCache.Del(client.ObjectKeyFromObject(kyma))
-			return r.requeueWithError(ctx, kyma, err)
-		}
 		if util.IsNotFound(err) {
-			if err = r.removeFinalizerAndUpdateKyma(ctx, kyma); err != nil {
-				return r.requeueWithError(ctx, kyma, err)
-			}
-			return ctrl.Result{}, nil
+			return r.removeFinalizerAndUpdateKyma(ctx, kyma)
 		}
 	}
 
 	r.enqueueWarningEvent(kyma, syncContextError, err)
-	return r.requeueWithError(ctx, kyma, err)
+	return err
 }
 
 func (r *KymaReconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctrl.Result, error) {
@@ -147,8 +144,9 @@ func (r *KymaReconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctr
 		var err error
 		remoteClient := remote.NewClientWithConfig(r.Client, r.KcpRestConfig)
 		if ctx, err = remote.InitializeSyncContext(ctx, kyma,
-			r.RemoteSyncNamespace, remoteClient, r.RemoteClientCache); err != nil {
-			return r.handleRemoteClusterConnectionErrorOnDeletion(ctx, kyma, err)
+			r.RemoteSyncNamespace, remoteClient, r.RemoteClientCache); err != nil &&
+			r.handleRemoteClusterConnectionError(ctx, kyma, err) != nil {
+			return r.requeueWithError(ctx, kyma, err)
 		}
 	}
 
@@ -156,7 +154,6 @@ func (r *KymaReconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctr
 		if err := r.deleteRemoteKyma(ctx, kyma); err != nil {
 			return r.requeueWithError(ctx, kyma, err)
 		}
-
 		if err := r.updateStatus(ctx, kyma, v1beta2.StateDeleting, "waiting for modules to be deleted"); err != nil {
 			return r.requeueWithError(ctx, kyma, fmt.Errorf("could not update kyma status after triggering deletion: %w", err))
 		}
diff --git a/tests/e2e_test/kyma_deletion_test.go b/tests/e2e_test/kyma_deletion_test.go
index 161ceb9398..bf06a67891 100644
--- a/tests/e2e_test/kyma_deletion_test.go
+++ b/tests/e2e_test/kyma_deletion_test.go
@@ -72,15 +72,45 @@ var _ = Describe("KCP Kyma CR should be deleted successfully when SKR cluster ge
 				WithContext(ctx).
 				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient).
 				Should(Succeed())
-		})
 
-		It("Should delete KCP Kyma", func() {
 			By("Kyma should be in Error state")
 			Eventually(CheckKymaIsInState, readyTimeout, interval).
 				WithContext(ctx).
 				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient, v1beta2.StateError).
 				Should(Succeed())
+		})
+
+		It("Should recreate SKR cluster and its secret", func() {
+			By("Creating SKR cluster")
+			cmd := exec.Command("k3d", "cluster", "create", "skr")
+			out, err := cmd.CombinedOutput()
+			Expect(err).NotTo(HaveOccurred())
+			GinkgoWriter.Printf(string(out))
+
+			cmd = exec.Command("k3d", "kubeconfig", "write", "skr")
+			out, err = cmd.CombinedOutput()
+			Expect(err).NotTo(HaveOccurred())
+			GinkgoWriter.Printf(string(out))
+
+			controlPlaneConfig, runtimeConfig, err = getKubeConfigs()
+			Expect(err).ToNot(HaveOccurred())
+
+			By("Creating Kyma secret")
+			Eventually(CreateKymaSecret, timeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient).
+				Should(Succeed())
+		})
+
+		It("Kyma should be in Ready state after secret is re-fetched", func() {
+			By("Kyma should be in Ready state")
+			Eventually(CheckKymaIsInState, readyTimeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient, v1beta2.StateReady).
+				Should(Succeed())
+		})
 
+		It("Should delete KCP Kyma", func() {
 			By("Deleting KCP Kyma")
 			Eventually(controlPlaneClient.Delete, readyTimeout, interval).
 				WithContext(ctx).