Bug 2304182: Disallow relocation execution when a cluster is unreachable

internal/controller/drplacementcontrol.go

@@ -825,6 +825,14 @@ func (d *DRPCInstance) RunRelocate() (bool, error) {
 
 	const done = true
 
+	if d.reconciler.numClustersQueriedSuccessfully != len(d.drPolicy.Spec.DRClusters) {
+		d.log.Info("Can't progress with relocation -- Not all clusters are reachable",
+			"numClustersQueriedSuccessfully", d.reconciler.numClustersQueriedSuccessfully,
+			"NumOfClusters", len(d.drPolicy.Spec.DRClusters))
+
+		return !done, nil
+	}
+
 	preferredCluster := d.instance.Spec.PreferredCluster
 	preferredClusterNamespace := d.instance.Spec.PreferredCluster
 

internal/controller/drplacementcontrol_controller.go

@@ -75,15 +75,16 @@ type ProgressCallback func(string, string)
 // DRPlacementControlReconciler reconciles a DRPlacementControl object
 type DRPlacementControlReconciler struct {
 	client.Client
-	APIReader           client.Reader
-	Log                 logr.Logger
-	MCVGetter           rmnutil.ManagedClusterViewGetter
-	Scheme              *runtime.Scheme
-	Callback            ProgressCallback
-	eventRecorder       *rmnutil.EventReporter
-	savedInstanceStatus rmn.DRPlacementControlStatus
-	ObjStoreGetter      ObjectStoreGetter
-	RateLimiter         *workqueue.TypedRateLimiter[reconcile.Request]
+	APIReader                      client.Reader
+	Log                            logr.Logger
+	MCVGetter                      rmnutil.ManagedClusterViewGetter
+	Scheme                         *runtime.Scheme
+	Callback                       ProgressCallback
+	eventRecorder                  *rmnutil.EventReporter
+	savedInstanceStatus            rmn.DRPlacementControlStatus
+	ObjStoreGetter                 ObjectStoreGetter
+	RateLimiter                    *workqueue.TypedRateLimiter[reconcile.Request]
+	numClustersQueriedSuccessfully int
 }
 
 func ManifestWorkPredicateFunc() predicate.Funcs {
@@ -939,11 +940,13 @@ func (r *DRPlacementControlReconciler) createDRPCInstance(
 		return nil, err
 	}
 
-	vrgs, _, _, err := getVRGsFromManagedClusters(r.MCVGetter, drpc, drClusters, vrgNamespace, log)
+	vrgs, cqs, _, err := getVRGsFromManagedClusters(r.MCVGetter, drpc, drClusters, vrgNamespace, log)
 	if err != nil {
 		return nil, err
 	}
 
+	r.numClustersQueriedSuccessfully = cqs
+
 	d := &DRPCInstance{
 		reconciler:      r,
 		ctx:             ctx,
@@ -1663,7 +1666,7 @@ func getVRGsFromManagedClusters(
 	annotations[DRPCNameAnnotation] = drpc.Name
 	annotations[DRPCNamespaceAnnotation] = drpc.Namespace
 
-	var clustersQueriedSuccessfully int
+	var numClustersQueriedSuccessfully int
 
 	var failedCluster string
 
@@ -1675,7 +1678,7 @@ func getVRGsFromManagedClusters(
 			// Only NotFound error is accepted
 			if errors.IsNotFound(err) {
 				log.Info(fmt.Sprintf("VRG not found on %q", drCluster.Name))
-				clustersQueriedSuccessfully++
+				numClustersQueriedSuccessfully++
 
 				continue
 			}
@@ -1687,7 +1690,7 @@ func getVRGsFromManagedClusters(
 			continue
 		}
 
-		clustersQueriedSuccessfully++
+		numClustersQueriedSuccessfully++
 
 		if rmnutil.ResourceIsDeleted(drCluster) {
 			log.Info("Skipping VRG on deleted drcluster", "drcluster", drCluster.Name, "vrg", vrg.Name)
@@ -1701,15 +1704,15 @@ func getVRGsFromManagedClusters(
 	}
 
 	// We are done if we successfully queried all drClusters
-	if clustersQueriedSuccessfully == len(drClusters) {
-		return vrgs, clustersQueriedSuccessfully, "", nil
+	if numClustersQueriedSuccessfully == len(drClusters) {
+		return vrgs, numClustersQueriedSuccessfully, "", nil
 	}
 
-	if clustersQueriedSuccessfully == 0 {
+	if numClustersQueriedSuccessfully == 0 {
 		return vrgs, 0, "", fmt.Errorf("failed to retrieve VRGs from clusters")
 	}
 
-	return vrgs, clustersQueriedSuccessfully, failedCluster, nil
+	return vrgs, numClustersQueriedSuccessfully, failedCluster, nil
 }
 
 func (r *DRPlacementControlReconciler) deleteClonedPlacementRule(ctx context.Context,

internal/controller/drplacementcontrol_controller_test.go

@@ -2479,8 +2479,8 @@ var _ = Describe("DRPlacementControl Reconciler", func() {
 				clearFakeUserPlacementRuleStatus(UserPlacementRuleName, DefaultDRPCNamespace)
 				clearDRPCStatus()
 				expectedAction := rmn.ActionRelocate
-				expectedPhase := rmn.Relocated
-				exptectedPorgression := rmn.ProgressionCleaningUp
+				expectedPhase := rmn.DRState("")
+				exptectedPorgression := rmn.ProgressionStatus("")
 				verifyDRPCStateAndProgression(expectedAction, expectedPhase, exptectedPorgression)
 
 				// User intervention is required (simulate user intervention)