diff --git a/go.mod b/go.mod index 4f4d975..ffa8f0f 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,12 @@ module github.com/rook/kubectl-rook-ceph go 1.22.0 +toolchain go1.22.3 + require ( github.com/fatih/color v1.17.0 github.com/golang/mock v1.6.0 + github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 github.com/pkg/errors v0.9.1 github.com/rook/rook v1.14.8 github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7 diff --git a/go.sum b/go.sum index 1ee04c6..15b1571 100644 --- a/go.sum +++ b/go.sum @@ -521,6 +521,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 h1:dQEHhTfi+bSIOSViQrKY9PqJvZenD6tFz+3lPzux58o= github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1/go.mod h1:my+EVjOJLeQ9lUR9uVkxRvNNkhO2saSGIgzV8GZT9HY= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= github.com/libopenstorage/autopilot-api v0.6.1-0.20210128210103-5fbb67948648/go.mod h1:6JLrPbR3ZJQFbUY/+QJMl/aF00YdIrLf8/GWAplgvJs= github.com/libopenstorage/openstorage v8.0.0+incompatible/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= github.com/libopenstorage/operator v0.0.0-20200725001727-48d03e197117/go.mod h1:Qh+VXOB6hj60VmlgsmY+R1w+dFuHK246UueM4SAqZG0= diff --git a/pkg/filesystem/subvolume.go b/pkg/filesystem/subvolume.go index 107b430..8928013 100644 --- a/pkg/filesystem/subvolume.go +++ b/pkg/filesystem/subvolume.go @@ -24,6 +24,7 @@ import ( "strings" "syscall" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" "github.com/rook/kubectl-rook-ceph/pkg/exec" "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" "github.com/rook/kubectl-rook-ceph/pkg/logging" @@ -42,6 +43,11 @@ type subVolumeInfo struct { state string } +type snapshotInfo struct { + volumeHandle string + snapshotHandle string +} + type monitor struct { ClusterID string Monitors []string @@ -51,12 +57,14 @@ const ( inUse = "in-use" stale = "stale" staleWithSnapshot = "stale-with-snapshot" + snapshotRetained = "snapshot-retained" ) func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) { subvolumeNames := getK8sRefSubvolume(ctx, clientsets) - listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames) + snapshotHandles := getK8sRefSnapshotHandle(ctx, clientsets) + listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames, snapshotHandles) } // checkForExternalStorage checks if the external mode is enabled. @@ -152,6 +160,48 @@ func getSubvolumeNameFromPath(path string) (string, error) { return name, nil } +// getk8sRefSnapshotHandle returns the snapshothandle for k8s ref of the volume snapshots +func getK8sRefSnapshotHandle(ctx context.Context, clientsets *k8sutil.Clientsets) map[string]snapshotInfo { + + snapConfig, err := snapclient.NewForConfig(clientsets.KubeConfig) + if err != nil { + logging.Fatal(err) + } + snapList, err := snapConfig.VolumeSnapshotContents().List(ctx, v1.ListOptions{}) + if err != nil { + logging.Fatal(fmt.Errorf("Error fetching volumesnapshotcontents: %v\n", err)) + } + + snapshotHandles := make(map[string]snapshotInfo) + for _, snap := range snapList.Items { + driverName := snap.Spec.Driver + if snap.Status != nil && snap.Status.SnapshotHandle != nil && strings.Contains(driverName, "cephfs.csi.ceph.com") { + + snapshotHandleId := getSnapshotHandleId(*snap.Status.SnapshotHandle) + // map the snapshotHandle id to later lookup for the subvol id and + // match the subvolume snapshot. + snapshotHandles[snapshotHandleId] = snapshotInfo{} + } + } + + return snapshotHandles +} + +// getSnapshotHandleId gets the id from snapshothandle +// SnapshotHandle: 0001-0009-rook-ceph-0000000000000001-17b95621- +// 58e8-4676-bc6a-39e928f19d23 +// SnapshotHandleId: 17b95621-58e8-4676-bc6a-39e928f19d23 +func getSnapshotHandleId(snapshotHandle string) string { + // get the snaps id from snapshot handle + splitSnapshotHandle := strings.SplitAfterN(snapshotHandle, "-", 6) + if len(splitSnapshotHandle) < 6 { + return "" + } + snapshotHandleId := splitSnapshotHandle[len(splitSnapshotHandle)-1] + + return snapshotHandleId +} + // runCommand checks for the presence of externalcluster and runs the command accordingly. func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, cmd string, args []string) (string, error) { if checkForExternalStorage(ctx, clientsets, clusterNamespace) { @@ -164,7 +214,7 @@ func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNam } // listCephFSSubvolumes list all the subvolumes -func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo) { +func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo, snapshotHandles map[string]snapshotInfo) { // getFilesystem gets the filesystem fsstruct, err := getFileSystem(ctx, clientsets, operatorNamespace, clusterNamespace) @@ -226,11 +276,12 @@ func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, o } else { // check the state of the stale subvolume // if it is snapshot-retained then skip listing it. - if state == "snapshot-retained" { + if state == snapshotRetained { + status = snapshotRetained continue } // check if the stale subvolume has snapshots. - if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name) { + if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name, snapshotHandles) { status = staleWithSnapshot } @@ -283,7 +334,8 @@ func getFileSystem(ctx context.Context, clientsets *k8sutil.Clientsets, operator } // checkSnapshot checks if there are any snapshots in the subvolume -func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string) bool { +// it also check for the stale snapshot and if found, deletes the snapshot. +func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string, snapshotHandles map[string]snapshotInfo) bool { cmd := "ceph" args := []string{"fs", "subvolume", "snapshot", "ls", fs, sv, svg, "--format", "json"} @@ -294,6 +346,19 @@ func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operator return false } snap := unMarshaljson(snapList) + // check for stale subvolume snapshot + // we have the list of snapshothandleid's from the + // volumesnapshotcontent. Looking up for snapid in it + // will confirm if we have stale snapshot or not. + for _, s := range snap { + _, snapId := getSnapOmapVal(s.Name) + // lookup for the snapid in the k8s snapshot handle list + _, ok := snapshotHandles[snapId] + if !ok { + // delete stale snapshot + deleteSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs, sv, svg, s.Name) + } + } if len(snap) == 0 { return false } @@ -327,6 +392,19 @@ func unMarshaljson(list string) []fsStruct { return unmarshal } +// deleteSnapshot deletes the subvolume snapshot +func deleteSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, cephClusterNamespace, fs, subvol, svg, snap string) { + + deleteOmapForSnapshot(ctx, clientsets, operatorNamespace, cephClusterNamespace, snap, fs) + cmd := "ceph" + args := []string{"fs", "subvolume", "snapshot", "rm", fs, subvol, snap, svg} + + _, err := runCommand(ctx, clientsets, operatorNamespace, cephClusterNamespace, cmd, args) + if err != nil { + logging.Fatal(err, "failed to delete subvolume snapshot of %s/%s/%s/%s", fs, svg, subvol, snap) + } +} + func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg string) { k8sSubvolume := getK8sRefSubvolume(ctx, clientsets) _, check := k8sSubvolume[subvol] @@ -410,6 +488,40 @@ func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, } } +// deleteOmapForSnapshot deletes omap object and key for the given snapshot. +func deleteOmapForSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, snap, fs string) { + logging.Info("Deleting the omap object and key for snapshot %q", snap) + snapomapkey := getSnapOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, snap, fs) + snapomapval, _ := getSnapOmapVal(snap) + poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs) + if err != nil || poolName == "" { + logging.Fatal(fmt.Errorf("pool name not found: %q", err)) + } + cmd := "rados" + if snapomapval != "" { + args := []string{"rm", snapomapval, "-p", poolName, "--namespace", "csi"} + + // remove omap object. + _, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args) + if err != nil { + logging.Fatal(err, "failed to remove omap object for snapshot %q", snap) + } + logging.Info("omap object:%q deleted", snapomapval) + + } + if snapomapkey != "" { + args := []string{"rmomapkey", "csi.snaps.default", snapomapkey, "-p", poolName, "--namespace", "csi"} + + // remove omap key. + _, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args) + if err != nil { + logging.Fatal(err, "failed to remove omap key for snapshot %q", snap) + } + logging.Info("omap key:%q deleted", snapomapkey) + + } +} + // getOmapKey gets the omap key and value details for a given subvolume. // Deletion of omap object required the subvolumeName which is of format // csi.volume.subvolume, where subvolume is the name of subvolume that needs to be @@ -438,6 +550,36 @@ func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNam return omapkey } +// getSnapOmapKey gets the omap key and value details for a given snapshot. +// Deletion of omap object required the snapshotName which is of format +// csi.snap.snapid. +// similarly to delete of omap key requires csi.snap.ompakey, where +// omapkey is the snapshotcontent name which is extracted the omap object. +func getSnapOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, cephClusterNamespace, snap, fs string) string { + + poolName, err := getMetadataPoolName(ctx, clientsets, operatorNamespace, cephClusterNamespace, fs) + if err != nil || poolName == "" { + logging.Fatal(fmt.Errorf("pool name not found: %q", err)) + } + snapomapval, _ := getSnapOmapVal(snap) + + args := []string{"getomapval", snapomapval, "csi.snapname", "-p", poolName, "--namespace", "csi", "/dev/stdout"} + cmd := "rados" + snapshotcontentname, err := runCommand(ctx, clientsets, operatorNamespace, cephClusterNamespace, cmd, args) + if snapshotcontentname == "" && err == nil { + logging.Info("No snapshot content found for snapshot") + return "" + } + if err != nil { + logging.Fatal(fmt.Errorf("Error getting snapshot content for snapshot %s: %s", snap, err)) + } + // omap key is for format csi.snap.snapshotcontent-fca205e5-8788-4132-979c-e210c0133182 + // hence, attaching pvname to required prefix. + snapomapkey := "csi.snap." + snapshotcontentname + + return snapomapkey +} + // getNfsClusterName returns the cluster name from the omap. // csi.nfs.cluster // value (26 bytes) : @@ -511,3 +653,19 @@ func getOmapVal(subVol string) (string, string) { return omapval, subvolId } + +// func getSnapOmapVal is used to get the omapval from the given snapshot +// omapval is of format csi.snap.427774b4-340b-11ed-8d66-0242ac110005 +// which is similar to volume name csi-snap-427774b4-340b-11ed-8d66-0242ac110005 +// hence, replacing 'csi-snap-' to 'csi.snap.' +func getSnapOmapVal(snap string) (string, string) { + + splitSnap := strings.SplitAfterN(snap, "-", 3) + if len(splitSnap) < 3 { + return "", "" + } + snapId := splitSnap[len(splitSnap)-1] + snapomapval := "csi.snap." + snapId + + return snapomapval, snapId +} diff --git a/pkg/filesystem/subvolume_test.go b/pkg/filesystem/subvolume_test.go index 3c2d9c6..bc10b56 100644 --- a/pkg/filesystem/subvolume_test.go +++ b/pkg/filesystem/subvolume_test.go @@ -101,3 +101,72 @@ func TestGetSubvolumeNameFromPath(t *testing.T) { }) } } + +func TestGetSnapOmapVal(t *testing.T) { + + tests := []struct { + name string + val string + snapid string + }{ + { + name: "csi-snap-427774b4-340b-11ed-8d66-0242ac110005", + val: "csi.snap.427774b4-340b-11ed-8d66-0242ac110005", + snapid: "427774b4-340b-11ed-8d66-0242ac110005", + }, + { + name: "", + val: "", + snapid: "", + }, + { + name: "csi-427774b4-340b-11ed-8d66-0242ac11000", + val: "csi.snap.340b-11ed-8d66-0242ac11000", + snapid: "340b-11ed-8d66-0242ac11000", + }, + { + name: "csi-427774b440b11ed8d660242ac11000", + val: "", + snapid: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if val, snapid := getSnapOmapVal(tt.name); val != tt.val && snapid != tt.snapid { + t.Errorf("getSnapOmapVal()= got val %v, want val %v,got snapid %v want snapid %v", val, tt.val, snapid, tt.snapid) + } + }) + } +} + +func TestGetSnapshotHandleId(t *testing.T) { + + tests := []struct { + name string + val string + }{ + { + name: "0001-0009-rook-ceph-0000000000000001-17b95621-58e8-4676-bc6a-39e928f19d23", + val: "17b95621-58e8-4676-bc6a-39e928f19d23", + }, + { + name: "", + val: "", + }, + { + name: "0001-0009-rook-0000000000000001-17b95621-58e8-4676-bc6a-39e928f19d23", + val: "58e8-4676-bc6a-39e928f19d23", + }, + { + name: "rook-427774b440b11ed8d660242ac11000", + val: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if val := getSnapshotHandleId(tt.name); val != tt.val { + assert.Equal(t, val, tt.val) + } + }) + } +} diff --git a/tests/github-action-helper.sh b/tests/github-action-helper.sh index 9d49bd7..e423c6c 100755 --- a/tests/github-action-helper.sh +++ b/tests/github-action-helper.sh @@ -241,7 +241,7 @@ install_minikube_with_none_driver() { } install_external_snapshotter() { - EXTERNAL_SNAPSHOTTER_VERSION=7.0.2 + EXTERNAL_SNAPSHOTTER_VERSION=8.0.1 curl -L "https://github.com/kubernetes-csi/external-snapshotter/archive/refs/tags/v${EXTERNAL_SNAPSHOTTER_VERSION}.zip" -o external-snapshotter.zip unzip -d /tmp external-snapshotter.zip cd "/tmp/external-snapshotter-${EXTERNAL_SNAPSHOTTER_VERSION}"