Support shredding in VDisk (ydb-platform#14117)

asmyasnikov · Feb 7, 2025 · a557da3 · a557da3
1 parent b4eb8a5
commit a557da3
Show file tree

Hide file tree

Showing 49 changed files with 996 additions and 427 deletions.
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
@@ -747,6 +747,11 @@ struct TEvBlobStorage {
         EvPreShredCompactVDisk,
         EvShredVDisk,
         EvMarkDirty,
+        EvHullShredDefrag,
+        EvHullShredDefragResult,
+        EvHugeShredNotify,
+        EvHugeShredNotifyResult,
+        EvNotifyChunksDeleted,
 
         EvYardInitResult = EvPut + 9 * 512,                     /// 268 636 672
         EvLogResult,

diff --git a/ydb/core/blobstorage/vdisk/balance/handoff_map.h b/ydb/core/blobstorage/vdisk/balance/handoff_map.h
@@ -110,16 +110,12 @@ namespace NKikimr {
         }
 
         // update merger with the filtered parts (only remaining local parts are kept)
-        TBlobType::EType type;
-        ui32 inplacedDataSize = 0;
-        dataMerger.FilterLocalParts(ingress.LocalParts(Top->GType), key.LogoBlobID(), &type, &inplacedDataSize);
+        dataMerger.FilterLocalParts(ingress.LocalParts(Top->GType));
 
         // reinstate memRec
         memRec = TMemRecLogoBlob(ingress);
-        memRec.SetType(type);
-        if (type == TBlobType::DiskBlob && inplacedDataSize) { // update inplaced data size for inplace blob
-            memRec.SetDiskBlob(TDiskPart(0, 0, inplacedDataSize));
-        }
+        memRec.SetDiskBlob(TDiskPart(0, 0, dataMerger.GetInplacedBlobSize(key.LogoBlobID())));
+        memRec.SetType(dataMerger.GetType());
 
         Y_ABORT_UNLESS(memRec.GetLocalParts(Top->GType) == dataMerger.GetParts());
     }

diff --git a/ydb/core/blobstorage/vdisk/balance/utils.cpp b/ydb/core/blobstorage/vdisk/balance/utils.cpp
@@ -18,7 +18,7 @@ namespace NBalancing {
     {
     }
 
-    void TPartsCollectorMerger::AddFromSegment(const TMemRecLogoBlob& memRec, const TDiskPart *outbound, const TKeyLogoBlob& /*key*/, ui64 /*lsn*/) {
+    void TPartsCollectorMerger::AddFromSegment(const TMemRecLogoBlob& memRec, const TDiskPart *outbound, const TKeyLogoBlob& /*key*/, ui64 /*lsn*/, const void* /*sst*/) {
         Ingress.Merge(memRec.GetIngress());
 
         const NMatrix::TVectorType local = memRec.GetLocalParts(GType);

diff --git a/ydb/core/blobstorage/vdisk/balance/utils.h b/ydb/core/blobstorage/vdisk/balance/utils.h
@@ -21,7 +21,7 @@ namespace NBalancing {
 
         static bool HaveToMergeData() { return true; }
 
-        void AddFromSegment(const TMemRecLogoBlob& memRec, const TDiskPart *outbound, const TKeyLogoBlob&, ui64);
+        void AddFromSegment(const TMemRecLogoBlob& memRec, const TDiskPart *outbound, const TKeyLogoBlob&, ui64, const void*);
         void AddFromFresh(const TMemRecLogoBlob& memRec, const TRope* data, const TKeyLogoBlob& key, ui64 /*lsn*/);
         void Clear();
     };

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_events.h
@@ -771,6 +771,8 @@ namespace NKikimr {
         // LWTrace doesn't support distributed shuttels yet
         mutable NLWTrace::TOrbit Orbit;
 
+        TDiskPart WrittenLocation;
+
         TEvVPutResult();
 
         TEvVPutResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &logoBlobId, const TVDiskID &vdisk,

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_private_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_private_events.h
@@ -85,17 +85,23 @@ namespace NKikimr {
         // mask of databases to compact in terms of EHullDbType type (can compact several databases at once)
         const ui32 Mask;
         const EMode Mode;
+        THashSet<ui64> TablesToCompact;
 
-        TEvCompactVDisk(ui32 mask, EMode mode = EMode::FULL)
+        TEvCompactVDisk(ui32 mask, EMode mode = EMode::FULL, THashSet<ui64> tablesToCompact = {})
             : Mask(mask)
             , Mode(mode)
+            , TablesToCompact(std::move(tablesToCompact))
         {}
 
         // create a message for compaction one database of type 'type'
         static TEvCompactVDisk *Create(EHullDbType type, EMode mode = EMode::FULL) {
             return new TEvCompactVDisk(::NKikimr::Mask(type), mode);
         }
 
+        static TEvCompactVDisk *Create(EHullDbType type, THashSet<ui64> tablesToCompact) {
+            return new TEvCompactVDisk(::NKikimr::Mask(type), EMode::FULL, std::move(tablesToCompact));
+        }
+
         static const char *ModeToString(EMode mode) {
             switch (mode) {
                 case EMode::FULL: return "FULL";
@@ -121,11 +127,13 @@ namespace NKikimr {
         const EHullDbType Type;
         const ui64 RequestId;
         const TEvCompactVDisk::EMode Mode;
+        THashSet<ui64> TablesToCompact;
 
-        TEvHullCompact(EHullDbType type, ui64 requestId, TEvCompactVDisk::EMode mode)
+        TEvHullCompact(EHullDbType type, ui64 requestId, TEvCompactVDisk::EMode mode, THashSet<ui64> tablesToCompact)
             : Type(type)
             , RequestId(requestId)
             , Mode(mode)
+            , TablesToCompact(std::move(tablesToCompact))
         {}
 
         TString ToString() const {
@@ -147,5 +155,28 @@ namespace NKikimr {
         {}
     };
 
+    struct TEvHullShredDefrag : TEventLocal<TEvHullShredDefrag, TEvBlobStorage::EvHullShredDefrag> {
+        std::vector<TChunkIdx> ChunksToShred;
+        TEvHullShredDefrag(std::vector<TChunkIdx> chunksToShred) : ChunksToShred(std::move(chunksToShred)) {}
+    };
+
+    struct TEvHullShredDefragResult : TEventLocal<TEvHullShredDefragResult, TEvBlobStorage::EvHullShredDefragResult> {
+    };
+
+    struct TEvNotifyChunksDeleted : TEventLocal<TEvNotifyChunksDeleted, TEvBlobStorage::EvNotifyChunksDeleted> {
+        ui64 Lsn;
+        std::vector<TChunkIdx> Chunks;
+
+        TEvNotifyChunksDeleted(ui64 lsn, std::vector<TChunkIdx> chunks)
+            : Lsn(lsn)
+            , Chunks(std::move(chunks))
+        {}
+
+        TEvNotifyChunksDeleted(const TEvNotifyChunksDeleted& x)
+            : Lsn(x.Lsn)
+            , Chunks(x.Chunks)
+        {}
+    };
+
 } // NKikimr