From d78eb0b3870203efbd77198739d88f8b38f3c40b Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Fri, 4 Aug 2023 08:03:00 -0400 Subject: [PATCH] vrg: annotate pvc and pv when they are archived We annotate the PV and PVC after we upload them to the s3 store. Use the annotation to skip upload if annotation exists. Signed-off-by: Raghavendra Talur (cherry picked from commit c52f352d746620ad486d2d1fa0c50efb82dacf29) --- controllers/status.go | 35 +++--- .../volumereplicationgroup_controller.go | 14 ++- controllers/vrg_volrep.go | 115 +++++++++++++++--- 3 files changed, 124 insertions(+), 40 deletions(-) diff --git a/controllers/status.go b/controllers/status.go index 10a80d54c..44eec5e8d 100644 --- a/controllers/status.go +++ b/controllers/status.go @@ -50,23 +50,24 @@ const ( // VRG condition reasons const ( - VRGConditionReasonInitializing = "Initializing" - VRGConditionReasonReplicating = "Replicating" - VRGConditionReasonReplicated = "Replicated" - VRGConditionReasonReady = "Ready" - VRGConditionReasonDataProtected = "DataProtected" - VRGConditionReasonProgressing = "Progressing" - VRGConditionReasonClusterDataRestored = "Restored" - VRGConditionReasonError = "Error" - VRGConditionReasonErrorUnknown = "UnknownError" - VRGConditionReasonUploading = "Uploading" - VRGConditionReasonUploaded = "Uploaded" - VRGConditionReasonUploadError = "UploadError" - VRGConditionReasonVolSyncRepSourceInited = "SourceInitialized" - VRGConditionReasonVolSyncRepDestInited = "DestinationInitialized" - VRGConditionReasonVolSyncPVsRestored = "Restored" - VRGConditionReasonVolSyncFinalSyncInProgress = "Syncing" - VRGConditionReasonVolSyncFinalSyncComplete = "Synced" + VRGConditionReasonInitializing = "Initializing" + VRGConditionReasonReplicating = "Replicating" + VRGConditionReasonReplicated = "Replicated" + VRGConditionReasonReady = "Ready" + VRGConditionReasonDataProtected = "DataProtected" + VRGConditionReasonProgressing = "Progressing" + VRGConditionReasonClusterDataRestored = "Restored" + VRGConditionReasonError = "Error" + VRGConditionReasonErrorUnknown = "UnknownError" + VRGConditionReasonUploading = "Uploading" + VRGConditionReasonUploaded = "Uploaded" + VRGConditionReasonUploadError = "UploadError" + VRGConditionReasonVolSyncRepSourceInited = "SourceInitialized" + VRGConditionReasonVolSyncRepDestInited = "DestinationInitialized" + VRGConditionReasonVolSyncPVsRestored = "Restored" + VRGConditionReasonVolSyncFinalSyncInProgress = "Syncing" + VRGConditionReasonVolSyncFinalSyncComplete = "Synced" + VRGConditionReasonClusterDataAnnotationFailed = "AnnotationFailed" ) const clusterDataProtectedTrueMessage = "Kube objects protected" diff --git a/controllers/volumereplicationgroup_controller.go b/controllers/volumereplicationgroup_controller.go index 3c210db07..7039bf3ce 100644 --- a/controllers/volumereplicationgroup_controller.go +++ b/controllers/volumereplicationgroup_controller.go @@ -449,12 +449,14 @@ const ( pvcInUse = "kubernetes.io/pvc-protection" // Annotations - pvcVRAnnotationProtectedKey = "volumereplicationgroups.ramendr.openshift.io/vr-protected" - pvcVRAnnotationProtectedValue = "protected" - pvVRAnnotationRetentionKey = "volumereplicationgroups.ramendr.openshift.io/vr-retained" - pvVRAnnotationRetentionValue = "retained" - RestoreAnnotation = "volumereplicationgroups.ramendr.openshift.io/ramen-restore" - RestoredByRamen = "True" + pvcVRAnnotationProtectedKey = "volumereplicationgroups.ramendr.openshift.io/vr-protected" + pvcVRAnnotationProtectedValue = "protected" + pvcVRAnnotationArchivedKey = "volumereplicationgroups.ramendr.openshift.io/vr-archived" + pvcVRAnnotationArchivedVersionV1 = "archiveV1" + pvVRAnnotationRetentionKey = "volumereplicationgroups.ramendr.openshift.io/vr-retained" + pvVRAnnotationRetentionValue = "retained" + RestoreAnnotation = "volumereplicationgroups.ramendr.openshift.io/ramen-restore" + RestoredByRamen = "True" // StorageClass label StorageIDLabel = "ramendr.openshift.io/storageid" diff --git a/controllers/vrg_volrep.go b/controllers/vrg_volrep.go index fa0b30771..c68746c9f 100644 --- a/controllers/vrg_volrep.go +++ b/controllers/vrg_volrep.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" "github.com/aws/aws-sdk-go/aws/awserr" @@ -508,17 +509,43 @@ func (v *VRGInstance) undoPVRetentionForPVC(pvc corev1.PersistentVolumeClaim, lo return nil } +func (v *VRGInstance) generateArchiveAnnotation(gen int64) string { + return fmt.Sprintf("%s-%s", pvcVRAnnotationArchivedVersionV1, strconv.Itoa(int(gen))) +} + +func (v *VRGInstance) isArchivedAlready(pvc *corev1.PersistentVolumeClaim, log logr.Logger) bool { + pvHasAnnotation := false + pvcHasAnnotation := false + + pv, err := v.getPVFromPVC(pvc) + if err != nil { + log.Error(err, "Failed to get PV from PVC, PVC:%v, err: %w", pvc.Name, err) + + return false + } + + pvcDesiredValue := v.generateArchiveAnnotation(pvc.Generation) + if v, ok := pvc.ObjectMeta.Annotations[pvcVRAnnotationArchivedKey]; ok && (v == pvcDesiredValue) { + pvcHasAnnotation = true + } + + pvDesiredValue := v.generateArchiveAnnotation(pv.Generation) + if v, ok := pv.ObjectMeta.Annotations[pvcVRAnnotationArchivedKey]; ok && (v == pvDesiredValue) { + pvHasAnnotation = true + } + + if !pvHasAnnotation || !pvcHasAnnotation { + return false + } + + return true +} + // Upload PV to the list of S3 stores in the VRG spec func (v *VRGInstance) uploadPVandPVCtoS3Stores(pvc *corev1.PersistentVolumeClaim, log logr.Logger) (err error) { - // Find the ProtectedPVC of the given PVC in v.instance.Status.ProtectedPVCs[] - protectedPVC := v.findProtectedPVC(pvc.Name) - // Find the ClusterDataProtected condition of the given PVC in ProtectedPVC.Conditions - clusterDataProtected := findCondition(protectedPVC.Conditions, VRGConditionTypeClusterDataProtected) - - // Optimization: skip uploading the PV of this PVC if it was uploaded previously - if clusterDataProtected != nil && clusterDataProtected.Status == metav1.ConditionTrue && - clusterDataProtected.ObservedGeneration == v.instance.Generation { - // v.log.Info("PV cluster data already protected") + if v.isArchivedAlready(pvc, log) { + v.log.Info("PV cluster data already protected for PVC", "PVC", pvc.Name) + return nil } @@ -540,22 +567,34 @@ func (v *VRGInstance) uploadPVandPVCtoS3Stores(pvc *corev1.PersistentVolumeClaim } numProfilesUploaded := len(s3Profiles) - // Set ClusterDataProtected condition to true if PV was uploaded to all the profiles - if numProfilesUploaded == numProfilesToUpload { - msg := fmt.Sprintf("Done uploading PV/PVC cluster data to %d of %d S3 profile(s): %v", - numProfilesUploaded, numProfilesToUpload, s3Profiles) - v.log.Info(msg) - v.updatePVCClusterDataProtectedCondition(pvc.Name, - VRGConditionReasonUploaded, msg) - } else { + + if numProfilesUploaded != numProfilesToUpload { // Merely defensive as we don't expect to reach here msg := fmt.Sprintf("Uploaded PV/PVC cluster data to only %d of %d S3 profile(s): %v", numProfilesUploaded, numProfilesToUpload, s3Profiles) v.log.Info(msg) v.updatePVCClusterDataProtectedCondition(pvc.Name, VRGConditionReasonUploadError, msg) + + return fmt.Errorf(msg) + } + + if err := v.addArchivedAnnotationForPVC(pvc, log); err != nil { + msg := fmt.Sprintf("failed to add archived annotation for PVC (%s/%s) with error (%v)", + pvc.Namespace, pvc.Name, err) + v.log.Info(msg) + v.updatePVCClusterDataProtectedCondition(pvc.Name, + VRGConditionReasonClusterDataAnnotationFailed, msg) + + return fmt.Errorf(msg) } + msg := fmt.Sprintf("Done uploading PV/PVC cluster data to %d of %d S3 profile(s): %v", + numProfilesUploaded, numProfilesToUpload, s3Profiles) + v.log.Info(msg) + v.updatePVCClusterDataProtectedCondition(pvc.Name, + VRGConditionReasonUploaded, msg) + return nil } @@ -1556,6 +1595,8 @@ func setPVCClusterDataProtectedCondition(protectedPVC *ramendrv1alpha1.Protected setVRGClusterDataProtectingCondition(&protectedPVC.Conditions, observedGeneration, message) case reason == VRGConditionReasonUploadError: setVRGClusterDataUnprotectedCondition(&protectedPVC.Conditions, observedGeneration, message) + case reason == VRGConditionReasonClusterDataAnnotationFailed: + setVRGClusterDataUnprotectedCondition(&protectedPVC.Conditions, observedGeneration, message) default: // if appropriate reason is not provided, then treat it as an unknown condition. message = "Unknown reason: " + reason @@ -1686,6 +1727,46 @@ func (v *VRGInstance) removeFinalizerFromPVC(pvc *corev1.PersistentVolumeClaim, return nil } +func (v *VRGInstance) addArchivedAnnotationForPVC(pvc *corev1.PersistentVolumeClaim, log logr.Logger) error { + if pvc.ObjectMeta.Annotations == nil { + pvc.ObjectMeta.Annotations = map[string]string{} + } + + pvc.ObjectMeta.Annotations[pvcVRAnnotationArchivedKey] = v.generateArchiveAnnotation(pvc.Generation) + + if err := v.reconciler.Update(v.ctx, pvc); err != nil { + log.Error(err, "Failed to update PersistentVolumeClaim annotation") + + return fmt.Errorf("failed to update PersistentVolumeClaim (%s/%s) annotation (%s) belonging to"+ + "VolumeReplicationGroup (%s/%s), %w", + pvc.Namespace, pvc.Name, pvcVRAnnotationArchivedKey, v.instance.Namespace, v.instance.Name, err) + } + + pv, err := v.getPVFromPVC(pvc) + if err != nil { + log.Error(err, "Failed to get PV from PVC") + + return fmt.Errorf("failed to update PersistentVolume (%s) annotation (%s) belonging to"+ + "VolumeReplicationGroup (%s/%s), %w", + pv.Name, pvcVRAnnotationArchivedKey, v.instance.Namespace, v.instance.Name, err) + } + + if pv.ObjectMeta.Annotations == nil { + pv.ObjectMeta.Annotations = map[string]string{} + } + + pv.ObjectMeta.Annotations[pvcVRAnnotationArchivedKey] = v.generateArchiveAnnotation(pv.Generation) + if err := v.reconciler.Update(v.ctx, &pv); err != nil { + log.Error(err, "Failed to update PersistentVolume annotation") + + return fmt.Errorf("failed to update PersistentVolume (%s) annotation (%s) belonging to"+ + "VolumeReplicationGroup (%s/%s), %w", + pvc.Name, pvcVRAnnotationArchivedKey, v.instance.Namespace, v.instance.Name, err) + } + + return nil +} + // findProtectedPVC returns the &VRG.Status.ProtectedPVC[x] for the given pvcName func (v *VRGInstance) findProtectedPVC(pvcName string) *ramendrv1alpha1.ProtectedPVC { for index := range v.instance.Status.ProtectedPVCs {