From d8b4e0391a2a189a02885c6776b9d227e03ccc81 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 14 Oct 2024 16:18:30 +0300 Subject: [PATCH] Add backup collection schema path item (#10252) --- ydb/core/driver_lib/cli_base/cli_cmds_db.cpp | 6 + ydb/core/protos/counters_schemeshard.proto | 12 + ydb/core/protos/flat_scheme_op.proto | 49 ++++ ydb/core/tx/scheme_board/cache.cpp | 13 + ydb/core/tx/scheme_cache/scheme_cache.h | 7 + .../schemeshard__backup_collection_common.cpp | 79 ++++++ .../schemeshard__backup_collection_common.h | 24 ++ ydb/core/tx/schemeshard/schemeshard__init.cpp | 23 ++ .../tx/schemeshard/schemeshard__operation.cpp | 17 ++ ...rd__operation_create_backup_collection.cpp | 250 ++++++++++++++++++ ...hard__operation_drop_backup_collection.cpp | 221 ++++++++++++++++ .../schemeshard__operation_memory_changes.cpp | 4 + .../schemeshard__operation_memory_changes.h | 5 + .../schemeshard/schemeshard__operation_part.h | 9 + ...hemeshard__operation_upgrade_subdomain.cpp | 1 + .../schemeshard_audit_log_fragment.cpp | 16 ++ ydb/core/tx/schemeshard/schemeshard_impl.cpp | 32 +++ ydb/core/tx/schemeshard/schemeshard_impl.h | 5 + .../tx/schemeshard/schemeshard_info_types.h | 19 ++ ydb/core/tx/schemeshard/schemeshard_path.cpp | 13 + ydb/core/tx/schemeshard/schemeshard_path.h | 1 + .../schemeshard_path_describer.cpp | 16 ++ .../schemeshard/schemeshard_path_describer.h | 1 + .../schemeshard/schemeshard_path_element.cpp | 6 +- .../tx/schemeshard/schemeshard_path_element.h | 1 + ydb/core/tx/schemeshard/schemeshard_schema.h | 13 +- .../tx/schemeshard/schemeshard_tx_infly.h | 15 ++ .../ut_backup_collection.cpp | 248 +++++++++++++++++ .../schemeshard/ut_backup_collection/ya.make | 27 ++ .../ut_backup_collection_reboots.cpp | 233 ++++++++++++++++ .../ut_backup_collection_reboots/ya.make | 26 ++ .../tx/schemeshard/ut_helpers/helpers.cpp | 5 + ydb/core/tx/schemeshard/ut_helpers/helpers.h | 5 + .../tx/schemeshard/ut_helpers/ls_checks.cpp | 7 + .../tx/schemeshard/ut_helpers/ls_checks.h | 1 + .../tx/schemeshard/ut_helpers/test_env.cpp | 1 + ydb/core/tx/schemeshard/ut_helpers/test_env.h | 1 + ydb/core/tx/schemeshard/ya.make | 121 +++++---- ydb/core/tx/tx_proxy/schemereq.cpp | 13 + ydb/core/viewer/browse.h | 1 + ydb/public/lib/deprecated/kicli/kicli.h | 3 +- ydb/public/lib/deprecated/kicli/schema.cpp | 6 + .../flat_schemeshard.schema | 54 ++++ 43 files changed, 1549 insertions(+), 61 deletions(-) create mode 100644 ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp create mode 100644 ydb/core/tx/schemeshard/schemeshard__backup_collection_common.h create mode 100644 ydb/core/tx/schemeshard/schemeshard__operation_create_backup_collection.cpp create mode 100644 ydb/core/tx/schemeshard/schemeshard__operation_drop_backup_collection.cpp create mode 100644 ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp create mode 100644 ydb/core/tx/schemeshard/ut_backup_collection/ya.make create mode 100644 ydb/core/tx/schemeshard/ut_backup_collection_reboots/ut_backup_collection_reboots.cpp create mode 100644 ydb/core/tx/schemeshard/ut_backup_collection_reboots/ya.make diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index e0b2079c7070..6690ac62c135 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -275,6 +275,9 @@ class TClientCommandSchemaDescribe : public TClientCommand { case NKikimrSchemeOp::EPathTypePersQueueGroup: type = ""; break; + case NKikimrSchemeOp::EPathTypeBackupCollection: + type = ""; + break; default: type = ""; break; @@ -487,6 +490,9 @@ class TClientCommandSchemaLs : public TClientCommand { case NKikimrSchemeOp::EPathTypeReplication: type = ""; break; + case NKikimrSchemeOp::EPathTypeBackupCollection: + type = ""; + break; default: type = ""; break; diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index 82ab38ce01db..fe09f910663a 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -222,6 +222,10 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxAlterResourcePool = 176 [(CounterOpts) = {Name: "InFlightOps/AlterResourcePool"}]; COUNTER_IN_FLIGHT_OPS_TxRestoreIncrementalBackupAtTable = 177 [(CounterOpts) = {Name: "InFlightOps/RestoreIncrementalBackupAtTable"}]; + COUNTER_BACKUP_COLLECTION_COUNT = 178 [(CounterOpts) = {Name: "BackupCollectionCount"}]; + COUNTER_IN_FLIGHT_OPS_TxCreateBackupCollection = 179 [(CounterOpts) = {Name: "InFlightOps/CreateBackupCollection"}]; + COUNTER_IN_FLIGHT_OPS_TxAlterBackupCollection = 180 [(CounterOpts) = {Name: "InFlightOps/AlterBackupCollection"}]; + COUNTER_IN_FLIGHT_OPS_TxDropBackupCollection = 181 [(CounterOpts) = {Name: "InFlightOps/DropBackupCollection"}]; } enum ECumulativeCounters { @@ -355,6 +359,10 @@ enum ECumulativeCounters { COUNTER_FINISHED_OPS_TxAlterResourcePool = 105 [(CounterOpts) = {Name: "FinishedOps/AlterResourcePool"}]; COUNTER_FINISHED_OPS_TxRestoreIncrementalBackupAtTable = 106 [(CounterOpts) = {Name: "FinishedOps/RestoreIncrementalBackupAtTable"}]; + + COUNTER_FINISHED_OPS_TxCreateBackupCollection = 107 [(CounterOpts) = {Name: "FinishedOps/CreateBackupCollection"}]; + COUNTER_FINISHED_OPS_TxAlterBackupCollection = 108 [(CounterOpts) = {Name: "FinishedOps/AlterBackupCollection"}]; + COUNTER_FINISHED_OPS_TxDropBackupCollection = 109 [(CounterOpts) = {Name: "FinishedOps/DropBackupCollection"}]; } enum EPercentileCounters { @@ -578,4 +586,8 @@ enum ETxTypes { TXTYPE_UPDATE_DOMAIN_REPLY = 85 [(TxTypeOpts) = {Name: "TxUpdateDomainReply"}]; TXTYPE_SEQUENCESHARD_GET_SEQUENCE_RESULT = 86 [(TxTypeOpts) = {Name: "TxSequenceShardGetSequenceResult"}]; + + TXTYPE_CREATE_BACKUP_COLLECTION_RESULT = 87 [(TxTypeOpts) = {Name: "TxCreateBackupCollectionResult"}]; + TXTYPE_ALTER_BACKUP_COLLECTION_RESULT = 88 [(TxTypeOpts) = {Name: "TxAlterBackupCollectionResult"}]; + TXTYPE_DROP_BACKUP_COLLECTION_RESULT = 89 [(TxTypeOpts) = {Name: "TxDropBackupCollectionResult"}]; } diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 058077f071f9..3ca951287947 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -23,6 +23,7 @@ import "ydb/core/tx/columnshard/engines/scheme/defaults/protos/data.proto"; import "ydb/core/tx/columnshard/common/protos/snapshot.proto"; import "ydb/library/formats/arrow/protos/accessor.proto"; +import "google/protobuf/empty.proto"; import "google/protobuf/struct.proto"; package NKikimrSchemeOp; @@ -1669,6 +1670,11 @@ enum EOperationType { ESchemeOpRestoreIncrementalBackup = 103; ESchemeOpRestoreIncrementalBackupAtTable = 104; + + // BackupCollection + ESchemeOpCreateBackupCollection = 105; + ESchemeOpAlterBackupCollection = 106; + ESchemeOpDropBackupCollection = 107; } message TApplyIf { @@ -1852,6 +1858,10 @@ message TModifyScheme { optional TResourcePoolDescription CreateResourcePool = 72; optional TRestoreIncrementalBackup RestoreIncrementalBackup = 73; + + optional TBackupCollectionDescription CreateBackupCollection = 74; + optional TBackupCollectionDescription AlterBackupCollection = 75; + optional TBackupCollectionDescription DropBackupCollection = 76; } message TCopySequence { @@ -1917,6 +1927,7 @@ enum EPathType { EPathTypeExternalDataSource = 19; EPathTypeView = 20; EPathTypeResourcePool = 21; + EPathTypeBackupCollection = 22; } enum EPathSubType { @@ -1973,6 +1984,7 @@ message TPathVersion { optional uint64 ExternalDataSourceVersion = 28; optional uint64 ViewVersion = 29; optional uint64 ResourcePoolVersion = 30; + optional uint64 BackupCollectionVersion = 31; } // Describes single path @@ -2062,6 +2074,7 @@ message TPathDescription { optional TExternalDataSourceDescription ExternalDataSourceDescription = 28; optional TViewDescription ViewDescription = 29; optional TResourcePoolDescription ResourcePoolDescription = 30; + optional TBackupCollectionDescription BackupCollectionDescription = 31; } // For persisting AlterTable Tx description in Schemeshard internal DB @@ -2202,3 +2215,39 @@ message TResourcePoolDescription { optional uint64 Version = 3; optional TResourcePoolProperties Properties = 4; } + +message TBackupCollectionDescription { + optional string Name = 1; + optional NKikimrProto.TPathID PathId = 2; + optional uint64 Version = 3; + + message TBackupEntry { + enum EType { + ETypeInvalid = 0; + ETypeTable = 1; + } + + optional EType Type = 1; + optional string Path = 2; + } + + message TExplicitEntryList { + repeated TBackupEntry Entries = 1; + } + + message TIncrementalBackupConfig { + + } + + oneof Entries { + google.protobuf.Empty Database = 4; + TExplicitEntryList ExplicitEntryList = 5; + } + + // non-empty enables data-collection + optional TIncrementalBackupConfig IncrementalBackupConfig = 6; + + oneof Storage { + google.protobuf.Empty Cluster = 7; + } +} diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index c9a6c70bd961..9dfa774e27b6 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -244,6 +244,7 @@ namespace { entry.BlobDepotInfo.Drop(); entry.BlockStoreVolumeInfo.Drop(); entry.FileStoreInfo.Drop(); + entry.BackupCollectionInfo.Drop(); } static void SetErrorAndClear(TResolveContext* context, TResolve::TEntry& entry, const bool isDescribeDenied) { @@ -763,6 +764,7 @@ class TSchemeCache: public TMonitorableActor { FileStoreInfo.Drop(); ViewInfo.Drop(); ResourcePoolInfo.Drop(); + BackupCollectionInfo.Drop(); } void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) { @@ -1278,6 +1280,7 @@ class TSchemeCache: public TMonitorableActor { DESCRIPTION_PART(FileStoreInfo); DESCRIPTION_PART(ViewInfo); DESCRIPTION_PART(ResourcePoolInfo); + DESCRIPTION_PART(BackupCollectionInfo); #undef DESCRIPTION_PART @@ -1610,6 +1613,10 @@ class TSchemeCache: public TMonitorableActor { Kind = TNavigate::KindResourcePool; FillInfo(Kind, ResourcePoolInfo, std::move(*pathDesc.MutableResourcePoolDescription())); break; + case NKikimrSchemeOp::EPathTypeBackupCollection: + Kind = TNavigate::KindBackupCollection; + FillInfo(Kind, BackupCollectionInfo, std::move(*pathDesc.MutableBackupCollectionDescription())); + break; case NKikimrSchemeOp::EPathTypeInvalid: Y_DEBUG_ABORT("Invalid path type"); break; @@ -1683,6 +1690,9 @@ class TSchemeCache: public TMonitorableActor { case NKikimrSchemeOp::EPathTypeResourcePool: ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindResourcePool); break; + case NKikimrSchemeOp::EPathTypeBackupCollection: + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindBackupCollection); + break; case NKikimrSchemeOp::EPathTypeTableIndex: case NKikimrSchemeOp::EPathTypeInvalid: Y_DEBUG_ABORT("Invalid path type"); @@ -1904,6 +1914,7 @@ class TSchemeCache: public TMonitorableActor { entry.FileStoreInfo = FileStoreInfo; entry.ViewInfo = ViewInfo; entry.ResourcePoolInfo = ResourcePoolInfo; + entry.BackupCollectionInfo = BackupCollectionInfo; } bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry, @@ -2202,6 +2213,8 @@ class TSchemeCache: public TMonitorableActor { // ResourcePool specific TIntrusivePtr ResourcePoolInfo; + // BackupCollection specific + TIntrusivePtr BackupCollectionInfo; }; // TCacheItem struct TMerger { diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index 92c74e03cff4..987acf44650e 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -159,6 +159,7 @@ struct TSchemeCacheNavigate { KindFileStore = 20, KindView = 21, KindResourcePool = 22, + KindBackupCollection = 23, }; struct TListNodeEntry : public TAtomicRefCount { @@ -275,6 +276,11 @@ struct TSchemeCacheNavigate { NKikimrSchemeOp::TResourcePoolDescription Description; }; + struct TBackupCollectionInfo : public TAtomicRefCount { + EKind Kind = KindUnknown; + NKikimrSchemeOp::TBackupCollectionDescription Description; + }; + struct TEntry { enum class ERequestType : ui8 { ByPath, @@ -327,6 +333,7 @@ struct TSchemeCacheNavigate { TIntrusiveConstPtr FileStoreInfo; TIntrusiveConstPtr ViewInfo; TIntrusiveConstPtr ResourcePoolInfo; + TIntrusiveConstPtr BackupCollectionInfo; TString ToString() const; TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; diff --git a/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp b/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp new file mode 100644 index 000000000000..f1bfde2490c6 --- /dev/null +++ b/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp @@ -0,0 +1,79 @@ +#include "schemeshard__backup_collection_common.h" + +namespace NKikimr::NSchemeShard { + +std::optional ResolveBackupCollectionPaths( + const TString& rootPathStr, + const TString& name, + bool validateFeatureFlag, + TOperationContext& context, + THolder& result) +{ + bool backupServiceEnabled = AppData()->FeatureFlags.GetEnableBackupService(); + if (!backupServiceEnabled && validateFeatureFlag) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, "Backup collections are disabled. Please contact your system administrator to enable it"); + return std::nullopt; + } + + const TPath& rootPath = TPath::Resolve(rootPathStr, context.SS); + { + const auto checks = rootPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() + .IsLikeDirectory() + .FailOnRestrictedCreateInTempZone(); + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + return std::nullopt; + } + } + + const TString& backupCollectionsDir = JoinPath({rootPath.GetDomainPathString(), ".backups/collections"}); + + TPathSplitUnix absPathSplit(name); + + if (absPathSplit.size() > 1 && !absPathSplit.IsAbsolute) { + result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Backup collections must be placed directly in " << backupCollectionsDir); + return std::nullopt; + } + + const TPath& backupCollectionsPath = TPath::Resolve(backupCollectionsDir, context.SS); + { + const auto checks = backupCollectionsPath.Check(); + checks.NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() + .IsLikeDirectory(); + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + return std::nullopt; + } + } + + std::optional parentPath; + if (absPathSplit.size() > 1) { + TString realParent = "/" + JoinRange("/", absPathSplit.begin(), absPathSplit.end() - 1); + parentPath = TPath::Resolve(realParent, context.SS); + } + + TPath dstPath = absPathSplit.IsAbsolute && parentPath ? parentPath->Child(TString(absPathSplit.back())) : rootPath.Child(name); + if (!dstPath.PathString().StartsWith(backupCollectionsDir + "/")) { + result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Backup collections must be placed in " << backupCollectionsDir); + return std::nullopt; + } + + return TBackupCollectionPaths{rootPath, dstPath}; +} + +} // namespace NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.h b/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.h new file mode 100644 index 000000000000..0283d58f46c9 --- /dev/null +++ b/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.h @@ -0,0 +1,24 @@ +#pragma once + +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + +#include + +#include + +namespace NKikimr::NSchemeShard { + +struct TBackupCollectionPaths { + TPath RootPath; + TPath DstPath; +}; + +std::optional ResolveBackupCollectionPaths( + const TString& rootPathStr, + const TString& name, + bool preValidateDst, + TOperationContext& context, + THolder& result); + +} // namespace NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 30cc2265cc28..0f5914c3ef4f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1956,6 +1956,29 @@ struct TSchemeShard::TTxInit : public TTransactionBase { } } + // Read backup collections + { + auto rowset = db.Table().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + TOwnerId ownerPathId = rowset.GetValue(); + TLocalPathId localPathId = rowset.GetValue(); + TPathId pathId(ownerPathId, localPathId); + + auto& backupCollection = Self->BackupCollections[pathId] = new TBackupCollectionInfo(); + backupCollection->AlterVersion = rowset.GetValue(); + Y_PROTOBUF_SUPPRESS_NODISCARD backupCollection->Description.ParseFromString(rowset.GetValue()); + Self->IncrementPathDbRefCount(pathId); + + if (!rowset.Next()) { + return false; + } + } + } + // Read table columns { TColumnRows columnRows; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp index 8d2b3fed10d3..fe1e96d1276c 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp @@ -1193,6 +1193,14 @@ ISubOperation::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxState:: case TTxState::ETxType::TxRestoreIncrementalBackupAtTable: return CreateRestoreIncrementalBackupAtTable(NextPartId(), txState); + // BackupCollection + case TTxState::ETxType::TxCreateBackupCollection: + return CreateNewBackupCollection(NextPartId(), txState); + case TTxState::ETxType::TxAlterBackupCollection: + Y_ABORT("TODO: implement"); + case TTxState::ETxType::TxDropBackupCollection: + return CreateDropBackupCollection(NextPartId(), txState); + case TTxState::ETxType::TxInvalid: Y_UNREACHABLE(); } @@ -1448,6 +1456,15 @@ TVector TOperation::ConstructParts(const TTxTransaction& tx return CreateRestoreIncrementalBackup(NextPartId(), tx, context); case NKikimrSchemeOp::EOperationType::ESchemeOpRestoreIncrementalBackupAtTable: Y_ABORT("multipart operations are handled before, also they require transaction details"); + + // BackupCollection + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateBackupCollection: + return {CreateNewBackupCollection(NextPartId(), tx)}; + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterBackupCollection: + Y_ABORT("TODO: implement"); + case NKikimrSchemeOp::EOperationType::ESchemeOpDropBackupCollection: + return {CreateDropBackupCollection(NextPartId(), tx)}; + } Y_UNREACHABLE(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup_collection.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup_collection.cpp new file mode 100644 index 000000000000..60177768275b --- /dev/null +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup_collection.cpp @@ -0,0 +1,250 @@ +#include "schemeshard__operation_common.h" +#include "schemeshard__backup_collection_common.h" +#include "schemeshard_impl.h" + +#define LOG_I(stream) LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr::NSchemeShard { + +namespace { + +class TPropose: public TSubOperationState { +private: + const TOperationId OperationId; + + TString DebugHint() const override { + return TStringBuilder() + << "TCreateBackupCollection TPropose" + << ", operationId: " << OperationId; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(std::move(id)) + { + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const TTxState* txState = context.SS->FindTx(OperationId); + Y_ABORT_UNLESS(txState); + Y_ABORT_UNLESS(txState->TxType == TTxState::TxCreateBackupCollection); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const TStepId step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + const TTxState* txState = context.SS->FindTx(OperationId); + Y_ABORT_UNLESS(txState); + Y_ABORT_UNLESS(txState->TxType == TTxState::TxCreateBackupCollection); + + const auto pathId = txState->TargetPathId; + const auto path = TPath::Init(pathId, context.SS); + const TPathElement::TPtr pathPtr = context.SS->PathsById.at(pathId); + + context.SS->TabletCounters->Simple()[COUNTER_BACKUP_COLLECTION_COUNT].Add(1); + + NIceDb::TNiceDb db(context.GetDB()); + + path->StepCreated = step; + context.SS->PersistCreateStep(db, pathId, step); + + IncParentDirAlterVersionWithRepublish(OperationId, path, context); + + context.SS->ClearDescribePathCaches(pathPtr); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } +}; + +class TCreateBackupCollection : public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::Propose; + } + + TTxState::ETxState NextState(TTxState::ETxState state) const override { + switch (state) { + case TTxState::Waiting: + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override { + switch (state) { + case TTxState::Waiting: + case TTxState::Propose: + return MakeHolder(OperationId); + case TTxState::Done: + return MakeHolder(OperationId); + default: + return nullptr; + } + } + + static void AddPathInSchemeShard(const THolder& result, TPath& dstPath, const TString& owner) { + dstPath.MaterializeLeaf(owner); + result->SetPathId(dstPath.Base()->PathId.LocalPathId); + } + + TPathElement::TPtr CreateBackupCollectionPathElement(const TPath& dstPath) const { + TPathElement::TPtr backupCollection = dstPath.Base(); + + backupCollection->CreateTxId = OperationId.GetTxId(); + backupCollection->PathType = TPathElement::EPathType::EPathTypeBackupCollection; + backupCollection->PathState = TPathElement::EPathState::EPathStateCreate; + backupCollection->LastTxId = OperationId.GetTxId(); + + return backupCollection; + } + + static void UpdatePathSizeCounts(const TPath& parentPath, const TPath& dstPath) { + dstPath.DomainInfo()->IncPathsInside(); + parentPath.Base()->IncAliveChildren(); + } + +public: + using TSubOperation::TSubOperation; + + THolder Propose(const TString& owner, TOperationContext& context) override { + const TString& rootPathStr = Transaction.GetWorkingDir(); + const auto& desc = Transaction.GetCreateBackupCollection(); + const TString& name = desc.GetName(); + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + + LOG_N("TCreateBackupCollection Propose: opId# " << OperationId << ", path# " << rootPathStr << "/" << name); + + auto result = MakeHolder(NKikimrScheme::StatusAccepted, + static_cast(OperationId.GetTxId()), + static_cast(context.SS->SelfTabletId())); + + auto bcPaths = ResolveBackupCollectionPaths(rootPathStr, name, true, context, result); + if (!bcPaths) { + return result; + } + + auto& [rootPath, dstPath] = *bcPaths; + + { + const auto checks = dstPath.Check(); + checks.IsAtLocalSchemeShard(); + if (dstPath.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() + .FailOnExist(TPathElement::EPathType::EPathTypeBackupCollection, false); + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks) { + checks + .IsValidLeafName() + .DepthLimit() + .PathsLimit() + .DirChildrenLimit() + .IsValidACL(acl); + } + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + return result; + } + } + + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); + return result; + } + + AddPathInSchemeShard(result, dstPath, owner); + auto pathEl = CreateBackupCollectionPathElement(dstPath); + + rootPath->IncAliveChildren(); + rootPath.DomainInfo()->IncPathsInside(); + + auto backupCollection = TBackupCollectionInfo::Create(desc); + context.SS->BackupCollections[dstPath->PathId] = backupCollection; + context.SS->TabletCounters->Simple()[COUNTER_BACKUP_COLLECTION_COUNT].Add(1); + context.SS->CreateTx( + OperationId, + TTxState::TxCreateBackupCollection, + pathEl->PathId); + + NIceDb::TNiceDb db(context.GetDB()); + + if (rootPath.Base()->HasActiveChanges()) { + const TTxId parentTxId = rootPath.Base()->PlannedToCreate() + ? rootPath.Base()->CreateTxId + : rootPath.Base()->LastTxId; + context.OnComplete.Dependence(parentTxId, OperationId.GetTxId()); + } + + context.SS->ChangeTxState(db, OperationId, TTxState::Propose); + context.OnComplete.ActivateTx(OperationId); + + const auto& backupCollectionPathId = pathEl->PathId; + + context.SS->BackupCollections[dstPath->PathId] = backupCollection; + context.SS->IncrementPathDbRefCount(backupCollectionPathId); + + if (!acl.empty()) { + pathEl->ApplyACL(acl); + } + context.SS->PersistPath(db, backupCollectionPathId); + + context.SS->PersistBackupCollection(db, + backupCollectionPathId, + backupCollection); + context.SS->PersistTxState(db, OperationId); + + IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, + dstPath, + context.SS, + context.OnComplete); + + UpdatePathSizeCounts(rootPath, dstPath); + + SetState(NextState()); + return result; + } + + void AbortPropose(TOperationContext& context) override { + LOG_N("TCreateBackupCollection AbortPropose: opId# " << OperationId); + Y_ABORT("no AbortPropose for TCreateBackupCollection"); + } + + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { + LOG_N("TCreateBackupCollection AbortUnsafe: opId# " << OperationId << ", txId# " << forceDropTxId); + context.OnComplete.DoneOperation(OperationId); + } +}; + +} // anonymous namespace + +ISubOperation::TPtr CreateNewBackupCollection(TOperationId id, const TTxTransaction& tx) { + return MakeSubOperation(id, tx); +} + +ISubOperation::TPtr CreateNewBackupCollection(TOperationId id, TTxState::ETxState state) { + Y_VERIFY(state != TTxState::Invalid); + return MakeSubOperation(id, state); +} + +} // namespace NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_backup_collection.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_backup_collection.cpp new file mode 100644 index 000000000000..53c9e6277e7d --- /dev/null +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_backup_collection.cpp @@ -0,0 +1,221 @@ +#include "schemeshard__operation_common.h" +#include "schemeshard__backup_collection_common.h" +#include "schemeshard_impl.h" + +#define LOG_I(stream) LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr::NSchemeShard { + +namespace { + +class TPropose : public TSubOperationState { +public: + explicit TPropose(TOperationId id) + : OperationId(std::move(id)) + {} + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_ABORT_UNLESS(txState); + Y_ABORT_UNLESS(txState->TxType == TTxState::TxDropBackupCollection); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const TStepId step = TStepId(ev->Get()->StepId); + LOG_I(DebugHint() << "HandleReply TEvOperationPlan: step# " << step); + + const TTxState* txState = context.SS->FindTx(OperationId); + Y_ABORT_UNLESS(txState); + Y_ABORT_UNLESS(txState->TxType == TTxState::TxDropBackupCollection); + + const TPathId& pathId = txState->TargetPathId; + const TPathElement::TPtr pathPtr = context.SS->PathsById.at(pathId); + const TPathElement::TPtr parentDirPtr = context.SS->PathsById.at(pathPtr->ParentPathId); + + NIceDb::TNiceDb db(context.GetDB()); + + Y_ABORT_UNLESS(!pathPtr->Dropped()); + pathPtr->SetDropped(step, OperationId.GetTxId()); + context.SS->PersistDropStep(db, pathId, step, OperationId); + context.SS->PersistRemoveBackupCollection(db, pathId); + + auto domainInfo = context.SS->ResolveDomainInfo(pathId); + domainInfo->DecPathsInside(); + parentDirPtr->DecAliveChildren(); + context.SS->TabletCounters->Simple()[COUNTER_BACKUP_COLLECTION_COUNT].Sub(1); + + ++parentDirPtr->DirAlterVersion; + context.SS->PersistPathDirAlterVersion(db, parentDirPtr); + context.SS->ClearDescribePathCaches(parentDirPtr); + context.SS->ClearDescribePathCaches(pathPtr); + + if (!context.SS->DisablePublicationsOfDropping) { + context.OnComplete.PublishToSchemeBoard(OperationId, parentDirPtr->PathId); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + } + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + +private: + TString DebugHint() const override { + return TStringBuilder() << "TDropBackupCollection TPropose, operationId: " << OperationId << ", "; + } + +private: + const TOperationId OperationId; +}; + +class TDropBackupCollection : public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::Propose; + } + + TTxState::ETxState NextState(TTxState::ETxState state) const override { + switch (state) { + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override { + switch (state) { + case TTxState::Propose: + return MakeHolder(OperationId); + case TTxState::Done: + return MakeHolder(OperationId); + default: + return nullptr; + } + } + + void DropBackupCollectionPathElement(const TPath& dstPath) const { + TPathElement::TPtr backupCollection = dstPath.Base(); + + backupCollection->PathState = TPathElement::EPathState::EPathStateDrop; + backupCollection->DropTxId = OperationId.GetTxId(); + backupCollection->LastTxId = OperationId.GetTxId(); + } + + void PersistDropBackupCollection(const TOperationContext& context, const TPath& dstPath) const { + const TPathId& pathId = dstPath.Base()->PathId; + + context.MemChanges.GrabNewTxState(context.SS, OperationId); + context.MemChanges.GrabPath(context.SS, pathId); + context.MemChanges.GrabPath(context.SS, dstPath->ParentPathId); + context.MemChanges.GrabBackupCollection(context.SS, pathId); + + context.DbChanges.PersistTxState(OperationId); + context.DbChanges.PersistPath(pathId); + context.DbChanges.PersistPath(dstPath->ParentPathId); + } + +public: + using TSubOperation::TSubOperation; + + THolder Propose(const TString&, TOperationContext& context) override { + const TString& rootPathStr = Transaction.GetWorkingDir(); + const auto& dropDescription = Transaction.GetDropBackupCollection(); + const TString& name = dropDescription.GetName(); + LOG_N("TDropBackupCollection Propose: opId# " << OperationId << ", path# " << rootPathStr << "/" << name); + + auto result = MakeHolder(NKikimrScheme::StatusAccepted, + static_cast(OperationId.GetTxId()), + static_cast(context.SS->SelfTabletId())); + + auto bcPaths = ResolveBackupCollectionPaths(rootPathStr, name, false, context, result); + if (!bcPaths) { + return result; + } + + auto& [_, dstPath] = *bcPaths; + + { + auto checks = dstPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsBackupCollection() + .NotUnderOperation() + .IsCommonSensePath(); + + if (checks) { + const TBackupCollectionInfo::TPtr backupCollection = context.SS->BackupCollections.Value(dstPath->PathId, nullptr); + if (!backupCollection) { + result->SetError(NKikimrScheme::StatusSchemeError, "Backup collection doesn't exist"); + + return result; + } + } + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + if (dstPath.IsResolved() && dstPath.Base()->IsBackupCollection() && (dstPath.Base()->PlannedToDrop() || dstPath.Base()->Dropped())) { + result->SetPathDropTxId(ui64(dstPath.Base()->DropTxId)); + result->SetPathId(dstPath.Base()->PathId.LocalPathId); + } + + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); + return result; + } + + result->SetPathId(dstPath.Base()->PathId.LocalPathId); + + auto guard = context.DbGuard(); + PersistDropBackupCollection(context, dstPath); + context.SS->CreateTx( + OperationId, + TTxState::TxDropBackupCollection, + dstPath.Base()->PathId); + + DropBackupCollectionPathElement(dstPath); + + context.OnComplete.ActivateTx(OperationId); + + IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, dstPath, context.SS, context.OnComplete); + + SetState(NextState()); + return result; + } + + void AbortPropose(TOperationContext& context) override { + LOG_N("TDropBackupCollection AbortPropose: opId# " << OperationId); + } + + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { + LOG_N("TDropBackupCollection AbortUnsafe: opId# " << OperationId << ", txId# " << forceDropTxId); + context.OnComplete.DoneOperation(OperationId); + } +}; + +} // anonymous namespace + +ISubOperation::TPtr CreateDropBackupCollection(TOperationId id, const TTxTransaction& tx) { + return MakeSubOperation(id, tx); +} + +ISubOperation::TPtr CreateDropBackupCollection(TOperationId id, TTxState::ETxState state) { + Y_ABORT_UNLESS(state != TTxState::Invalid); + return MakeSubOperation(id, state); +} + +} // namespace NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp index 999a610355e9..8cab5f3983df 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp @@ -108,6 +108,10 @@ void TMemoryChanges::GrabResourcePool(TSchemeShard* ss, const TPathId& pathId) { Grab(pathId, ss->ResourcePools, ResourcePools); } +void TMemoryChanges::GrabBackupCollection(TSchemeShard* ss, const TPathId& pathId) { + Grab(pathId, ss->BackupCollections, BackupCollections); +} + void TMemoryChanges::UnDo(TSchemeShard* ss) { // be aware of the order of grab & undo ops // stack is the best way to manage it right diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h index 48a67e8396bf..ce9914de8cca 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h @@ -55,6 +55,9 @@ class TMemoryChanges: public TSimpleRefCount { using TResourcePoolState = std::pair; TStack ResourcePools; + using TBackupCollectionState = std::pair; + TStack BackupCollections; + public: ~TMemoryChanges() = default; @@ -91,6 +94,8 @@ class TMemoryChanges: public TSimpleRefCount { void GrabResourcePool(TSchemeShard* ss, const TPathId& pathId); + void GrabBackupCollection(TSchemeShard* ss, const TPathId& pathId); + void UnDo(TSchemeShard* ss); }; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index 88e57aa3c38e..91d6fc1ff21b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -645,5 +645,14 @@ ISubOperation::TPtr CascadeDropTableChildren(TVector& resul TVector CreateRestoreIncrementalBackup(TOperationId opId, const TTxTransaction& tx, TOperationContext& context); +// BackupCollection +// Create +ISubOperation::TPtr CreateNewBackupCollection(TOperationId id, const TTxTransaction& tx); +ISubOperation::TPtr CreateNewBackupCollection(TOperationId id, TTxState::ETxState state); +// Drop +ISubOperation::TPtr CreateDropBackupCollection(TOperationId id, const TTxTransaction& tx); +ISubOperation::TPtr CreateDropBackupCollection(TOperationId id, TTxState::ETxState state); + + } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp index fcf9bbb4ef40..e5e4bd42ab03 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp @@ -362,6 +362,7 @@ class TConfigure: public TSubOperationState { case NKikimrSchemeOp::EPathType::EPathTypeSequence: case NKikimrSchemeOp::EPathType::EPathTypeReplication: case NKikimrSchemeOp::EPathType::EPathTypeBlobDepot: + case NKikimrSchemeOp::EPathType::EPathTypeBackupCollection: Y_ABORT("UNIMPLEMENTED"); case NKikimrSchemeOp::EPathType::EPathTypeInvalid: Y_UNREACHABLE(); diff --git a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp index 0a074e30ddc8..dba2fa631434 100644 --- a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp @@ -246,6 +246,13 @@ TString DefineUserOperationName(const NKikimrSchemeOp::TModifyScheme& tx) { case NKikimrSchemeOp::EOperationType::ESchemeOpRestoreIncrementalBackup: case NKikimrSchemeOp::EOperationType::ESchemeOpRestoreIncrementalBackupAtTable: return "RESTORE"; + // backup collection + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateBackupCollection: + return "CREATE BACKUP COLLECTION"; + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterBackupCollection: + return "ALTER BACKUP COLLECTION"; + case NKikimrSchemeOp::EOperationType::ESchemeOpDropBackupCollection: + return "DROP BACKUP COLLECTION"; } Y_ABORT("switch should cover all operation types"); } @@ -554,6 +561,15 @@ TVector ExtractChangingPaths(const NKikimrSchemeOp::TModifyScheme& tx) result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetRestoreIncrementalBackup().GetSrcTableName()})); result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetRestoreIncrementalBackup().GetDstTableName()})); break; + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateBackupCollection: + result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetCreateBackupCollection().GetName()})); + break; + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterBackupCollection: + result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetAlterBackupCollection().GetName()})); + break; + case NKikimrSchemeOp::EOperationType::ESchemeOpDropBackupCollection: + result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetDropBackupCollection().GetName()})); + break; } return result; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index e314857b0db0..d1e7fd93a18e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -1519,6 +1519,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxCreateView: case TTxState::TxCreateContinuousBackup: case TTxState::TxCreateResourcePool: + case TTxState::TxCreateBackupCollection: return TPathElement::EPathState::EPathStateCreate; case TTxState::TxAlterPQGroup: case TTxState::TxAlterTable: @@ -1554,6 +1555,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxAlterView: case TTxState::TxAlterContinuousBackup: case TTxState::TxAlterResourcePool: + case TTxState::TxAlterBackupCollection: return TPathElement::EPathState::EPathStateAlter; case TTxState::TxDropTable: case TTxState::TxDropPQGroup: @@ -1578,6 +1580,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxDropView: case TTxState::TxDropContinuousBackup: case TTxState::TxDropResourcePool: + case TTxState::TxDropBackupCollection: return TPathElement::EPathState::EPathStateDrop; case TTxState::TxBackup: return TPathElement::EPathState::EPathStateBackup; @@ -2998,6 +3001,25 @@ void TSchemeShard::PersistRemoveResourcePool(NIceDb::TNiceDb& db, TPathId pathId db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); } +void TSchemeShard::PersistBackupCollection(NIceDb::TNiceDb& db, TPathId pathId, const TBackupCollectionInfo::TPtr backupCollection) { + Y_ABORT_UNLESS(IsLocalId(pathId)); + + db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate{backupCollection->AlterVersion}, + NIceDb::TUpdate{backupCollection->Description.SerializeAsString()} + ); +} + +void TSchemeShard::PersistRemoveBackupCollection(NIceDb::TNiceDb& db, TPathId pathId) { + Y_ABORT_UNLESS(IsLocalId(pathId)); + if (BackupCollections.contains(pathId)) { + BackupCollections.erase(pathId); + DecrementPathDbRefCount(pathId); + } + + db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); +} + void TSchemeShard::PersistRemoveRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId) { Y_ABORT_UNLESS(IsLocalId(pathId)); @@ -4246,6 +4268,13 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co generalVersion += result.GetResourcePoolVersion(); break; } + case NKikimrSchemeOp::EPathType::EPathTypeBackupCollection: { + auto it = BackupCollections.find(pathId); + Y_ABORT_UNLESS(it != BackupCollections.end()); + result.SetBackupCollectionVersion(it->second->AlterVersion); + generalVersion += result.GetBackupCollectionVersion(); + break; + } case NKikimrSchemeOp::EPathType::EPathTypeInvalid: { Y_UNREACHABLE(); @@ -5062,6 +5091,9 @@ void TSchemeShard::UncountNode(TPathElement::TPtr node) { case TPathElement::EPathType::EPathTypeResourcePool: TabletCounters->Simple()[COUNTER_RESOURCE_POOL_COUNT].Sub(1); break; + case TPathElement::EPathType::EPathTypeBackupCollection: + TabletCounters->Simple()[COUNTER_BACKUP_COLLECTION_COUNT].Sub(1); + break; case TPathElement::EPathType::EPathTypeInvalid: Y_ABORT("impossible path type"); } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 87fb104f3293..edf8e9c94e41 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -253,6 +253,7 @@ class TSchemeShard THashMap ExternalDataSources; THashMap Views; THashMap ResourcePools; + THashMap BackupCollections; TTempDirsState TempDirsState; @@ -823,6 +824,10 @@ class TSchemeShard void PersistResourcePool(NIceDb::TNiceDb& db, TPathId pathId, const TResourcePoolInfo::TPtr resourcePool); void PersistRemoveResourcePool(NIceDb::TNiceDb& db, TPathId pathId); + // BackupCollection + void PersistBackupCollection(NIceDb::TNiceDb& db, TPathId pathId, const TBackupCollectionInfo::TPtr backupCollection); + void PersistRemoveBackupCollection(NIceDb::TNiceDb& db, TPathId pathId); + TTabletId GetGlobalHive(const TActorContext& ctx) const; enum class EHiveSelection : uint8_t { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 46de9b7432af..68bbcb0072c5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3417,6 +3417,25 @@ struct TResourcePoolInfo : TSimpleRefCount { NKikimrSchemeOp::TResourcePoolProperties Properties; }; +struct TBackupCollectionInfo : TSimpleRefCount { + using TPtr = TIntrusivePtr; + + static TPtr New() { + return new TBackupCollectionInfo(); + } + + static TPtr Create(const NKikimrSchemeOp::TBackupCollectionDescription& desc) { + TPtr result = New(); + + result->Description = desc; + + return result; + } + + ui64 AlterVersion = 0; + NKikimrSchemeOp::TBackupCollectionDescription Description; +}; + bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, const THashMap& sourceColumns, const THashMap& alterColumns, diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index c871e57c2b12..a317da864533 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -896,6 +896,19 @@ const TPath::TChecker& TPath::TChecker::IsResourcePool(EStatus status) const { << " (" << BasicPathInfo(Path.Base()) << ")"); } +const TPath::TChecker& TPath::TChecker::IsBackupCollection(EStatus status) const { + if (Failed) { + return *this; + } + + if (Path.Base()->IsBackupCollection()) { + return *this; + } + + return Fail(status, TStringBuilder() << "path is not a backup collection" + << " (" << BasicPathInfo(Path.Base()) << ")"); +} + const TPath::TChecker& TPath::TChecker::PathShardsLimit(ui64 delta, EStatus status) const { if (Failed) { return *this; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index 5b674581fbe8..1b85c4d6e359 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -102,6 +102,7 @@ class TPath { const TChecker& IsView(EStatus status = EStatus::StatusNameConflict) const; const TChecker& FailOnRestrictedCreateInTempZone(bool allowCreateInTemporaryDir = false, EStatus status = EStatus::StatusPreconditionFailed) const; const TChecker& IsResourcePool(EStatus status = EStatus::StatusNameConflict) const; + const TChecker& IsBackupCollection(EStatus status = EStatus::StatusNameConflict) const; }; public: diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 416954aa4b8c..2b6aa5b594cb 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -1067,6 +1067,18 @@ void TPathDescriber::DescribeResourcePool(TPathId pathId, TPathElement::TPtr pat entry->MutableProperties()->CopyFrom(resourcePoolInfo->Properties); } +void TPathDescriber::DescribeBackupCollection(TPathId pathId, TPathElement::TPtr pathEl) { + auto it = Self->BackupCollections.FindPtr(pathId); + Y_ABORT_UNLESS(it, "BackupCollections is not found"); + TBackupCollectionInfo::TPtr backupCollectionInfo = *it; + + auto entry = Result->Record.MutablePathDescription()->MutableBackupCollectionDescription(); + entry->SetName(pathEl->Name); + PathIdFromPathId(pathId, entry->MutablePathId()); + entry->SetVersion(backupCollectionInfo->AlterVersion); + entry->CopyFrom(backupCollectionInfo->Description); +} + static bool ConsiderAsDropped(const TPath& path) { Y_ABORT_UNLESS(path.IsResolved()); @@ -1221,6 +1233,10 @@ THolder TPathDescriber::Describe case NKikimrSchemeOp::EPathTypeResourcePool: DescribeResourcePool(base->PathId, base); break; + case NKikimrSchemeOp::EPathTypeBackupCollection: + DescribeDir(path); + DescribeBackupCollection(base->PathId, base); + break; case NKikimrSchemeOp::EPathTypeInvalid: Y_UNREACHABLE(); } diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.h b/ydb/core/tx/schemeshard/schemeshard_path_describer.h index 3aa655831be9..317fbef59150 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.h @@ -47,6 +47,7 @@ class TPathDescriber { void DescribeExternalDataSource(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl); void DescribeView(const TActorContext&, TPathId pathId, TPathElement::TPtr pathEl); void DescribeResourcePool(TPathId pathId, TPathElement::TPtr pathEl); + void DescribeBackupCollection(TPathId pathId, TPathElement::TPtr pathEl); public: explicit TPathDescriber(TSchemeShard* self, NKikimrSchemeOp::TDescribePath&& params) diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp index f2f30095173f..30727dd64599 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp @@ -187,7 +187,7 @@ bool TPathElement::IsContainer() const { } bool TPathElement::IsLikeDirectory() const { - return IsDirectory() || IsDomainRoot() || IsOlapStore() || IsTableIndex(); + return IsDirectory() || IsDomainRoot() || IsOlapStore() || IsTableIndex() || IsBackupCollection(); } bool TPathElement::HasActiveChanges() const { @@ -235,6 +235,10 @@ bool TPathElement::IsResourcePool() const { return PathType == EPathType::EPathTypeResourcePool; } +bool TPathElement::IsBackupCollection() const { + return PathType == EPathType::EPathTypeBackupCollection; +} + void TPathElement::SetDropped(TStepId step, TTxId txId) { PathState = EPathState::EPathStateNotExist; StepDropped = step; diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h index 61e5b3b40c53..ce19247a4963 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h @@ -136,6 +136,7 @@ struct TPathElement : TSimpleRefCount { bool IsView() const; bool IsTemporary() const; bool IsResourcePool() const; + bool IsBackupCollection() const; TVirtualTimestamp GetCreateTS() const; TVirtualTimestamp GetDropTS() const; void SetDropped(TStepId step, TTxId txId); diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index ca918c26c579..d9df1b67edc1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1823,6 +1823,16 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns; }; + struct BackupCollection : Table<111> { + struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; + struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {}; + struct Description: Column<4, NScheme::NTypeIds::String> {}; + + using TKey = TableKey; + using TColumns = TableColumns; + }; + using TTables = SchemaTables< Paths, TxInFlight, @@ -1932,7 +1942,8 @@ struct Schema : NIceDb::Schema { BuildColumnOperationSettings, View, BackgroundSessions, - ResourcePool + ResourcePool, + BackupCollection >; static constexpr ui64 SysParam_NextPathId = 1; diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h index f2ab0e5744cb..186d0f2ce18e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h +++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h @@ -138,6 +138,9 @@ struct TTxState { item(TxDropResourcePool, 92) \ item(TxAlterResourcePool, 93) \ item(TxRestoreIncrementalBackupAtTable, 94) \ + item(TxCreateBackupCollection, 95) \ + item(TxDropBackupCollection, 96) \ + item(TxAlterBackupCollection, 97) \ // TX_STATE_TYPE_ENUM @@ -355,6 +358,7 @@ struct TTxState { case TxCopySequence: case TxCreateContinuousBackup: case TxCreateResourcePool: + case TxCreateBackupCollection: return true; case TxInitializeBuildIndex: //this is more like alter case TxCreateCdcStreamAtTable: @@ -391,6 +395,7 @@ struct TTxState { case TxDropView: case TxDropContinuousBackup: case TxDropResourcePool: + case TxDropBackupCollection: return false; case TxAlterPQGroup: case TxAlterTable: @@ -425,6 +430,7 @@ struct TTxState { case TxAlterContinuousBackup: case TxAlterResourcePool: case TxRestoreIncrementalBackupAtTable: + case TxAlterBackupCollection: return false; case TxMoveTable: case TxMoveTableIndex: @@ -461,6 +467,7 @@ struct TTxState { case TxDropView: case TxDropContinuousBackup: case TxDropResourcePool: + case TxDropBackupCollection: return true; case TxMkDir: case TxCreateTable: @@ -498,6 +505,7 @@ struct TTxState { case TxCreateContinuousBackup: case TxCreateResourcePool: case TxRestoreIncrementalBackupAtTable: + case TxCreateBackupCollection: return false; case TxAlterPQGroup: case TxAlterTable: @@ -531,6 +539,7 @@ struct TTxState { case TxAlterView: case TxAlterContinuousBackup: case TxAlterResourcePool: + case TxAlterBackupCollection: return false; case TxMoveTable: case TxMoveTableIndex: @@ -563,6 +572,7 @@ struct TTxState { case TxDropReplicationCascade: case TxDropBlobDepot: case TxDropContinuousBackup: + case TxDropBackupCollection: return true; case TxDropTableIndex: case TxRmDir: @@ -606,6 +616,7 @@ struct TTxState { case TxCreateContinuousBackup: case TxCreateResourcePool: case TxRestoreIncrementalBackupAtTable: + case TxCreateBackupCollection: return false; case TxAlterPQGroup: case TxAlterTable: @@ -640,6 +651,7 @@ struct TTxState { case TxAlterView: case TxAlterContinuousBackup: case TxAlterResourcePool: + case TxAlterBackupCollection: return false; case TxInvalid: case TxAllocatePQ: @@ -751,6 +763,9 @@ struct TTxState { case NKikimrSchemeOp::ESchemeOpAlterContinuousBackup: return TxInvalid; case NKikimrSchemeOp::ESchemeOpDropContinuousBackup: return TxInvalid; case NKikimrSchemeOp::ESchemeOpRestoreIncrementalBackup: return TxInvalid; + case NKikimrSchemeOp::ESchemeOpCreateBackupCollection: return TxCreateBackupCollection; + case NKikimrSchemeOp::ESchemeOpAlterBackupCollection: return TxAlterBackupCollection; + case NKikimrSchemeOp::ESchemeOpDropBackupCollection: return TxDropBackupCollection; default: return TxInvalid; } } diff --git a/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp b/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp new file mode 100644 index 000000000000..13d65bacb749 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp @@ -0,0 +1,248 @@ +#include + +#define DEFAULT_NAME_1 "MyCollection1" +#define DEFAULT_NAME_2 "MyCollection2" + +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TBackupCollectionTests) { + void SetupLogging(TTestActorRuntimeBase& runtime) { + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + } + + TString DefaultCollectionSettings() { + return R"( + Name: ")" DEFAULT_NAME_1 R"(" + + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/Table1" + } + } + Cluster: {} + )"; + } + + TString CollectionSettings(const TString& name) { + return Sprintf(R"( + Name: "%s" + + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/Table1" + } + } + Cluster: {} + )", name.c_str()); + } + + void PrepareDirs(TTestBasicRuntime& runtime, TTestEnv& env, ui64& txId) { + TestMkDir(runtime, ++txId, "/MyRoot", ".backups"); + env.TestWaitNotification(runtime, txId); + TestMkDir(runtime, ++txId, "/MyRoot/.backups", "collections"); + env.TestWaitNotification(runtime, txId); + } + + Y_UNIT_TEST(HiddenByFeatureFlag) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions()); + ui64 txId = 100; + + SetupLogging(runtime); + + PrepareDirs(runtime, env, txId); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot", DefaultCollectionSettings(), {NKikimrScheme::StatusPreconditionFailed}); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathNotExist, + }); + + // must not be there in any case, smoke test + TestDescribeResult(DescribePath(runtime, "/MyRoot/" DEFAULT_NAME_1), { + NLs::PathNotExist, + }); + } + + Y_UNIT_TEST(DisallowedPath) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + + PrepareDirs(runtime, env, txId); + + { + TestCreateBackupCollection(runtime, ++txId, "/MyRoot", DefaultCollectionSettings(), {NKikimrScheme::EStatus::StatusSchemeError}); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathNotExist, + }); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/" DEFAULT_NAME_1), { + NLs::PathNotExist, + }); + } + + { + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups", DefaultCollectionSettings(), {NKikimrScheme::EStatus::StatusSchemeError}); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathNotExist, + }); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/" DEFAULT_NAME_1), { + NLs::PathNotExist, + }); + } + + { + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", CollectionSettings("SomePrefix/MyCollection1"), {NKikimrScheme::EStatus::StatusSchemeError}); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/SomePrefix/MyCollection1"), { + NLs::PathNotExist, + }); + } + } + + Y_UNIT_TEST(CreateAbsolutePath) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + + PrepareDirs(runtime, env, txId); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot", CollectionSettings("/MyRoot/.backups/collections/" DEFAULT_NAME_1)); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathExist, + NLs::IsBackupCollection, + }); + } + + Y_UNIT_TEST(Create) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + + PrepareDirs(runtime, env, txId); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", DefaultCollectionSettings()); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathExist, + NLs::IsBackupCollection, + }); + } + + Y_UNIT_TEST(CreateTwice) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + + PrepareDirs(runtime, env, txId); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", DefaultCollectionSettings()); + + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathExist, + NLs::IsBackupCollection, + }); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", DefaultCollectionSettings(), {NKikimrScheme::EStatus::StatusSchemeError}); + + env.TestWaitNotification(runtime, txId); + } + + Y_UNIT_TEST(ParallelCreate) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + PrepareDirs(runtime, env, txId); + + AsyncCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", CollectionSettings(DEFAULT_NAME_1)); + AsyncCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", CollectionSettings(DEFAULT_NAME_2)); + TestModificationResult(runtime, txId - 1, NKikimrScheme::StatusAccepted); + TestModificationResult(runtime, txId, NKikimrScheme::StatusAccepted); + + env.TestWaitNotification(runtime, {txId, txId - 1}); + + TestDescribe(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1); + TestDescribe(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_2); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections"), + {NLs::PathVersionEqual(7)}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathVersionEqual(1)}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_2), + {NLs::PathVersionEqual(1)}); + } + + Y_UNIT_TEST(Drop) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + PrepareDirs(runtime, env, txId); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + env.TestWaitNotification(runtime, txId); + + TestLs(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1, false, NLs::PathExist); + + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + env.TestWaitNotification(runtime, txId); + + TestLs(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1, false, NLs::PathNotExist); + } + + Y_UNIT_TEST(DropTwice) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + PrepareDirs(runtime, env, txId); + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + env.TestWaitNotification(runtime, txId); + + AsyncDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + AsyncDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + TestModificationResult(runtime, txId - 1); + + auto ev = runtime.GrabEdgeEvent(); + UNIT_ASSERT(ev); + + const auto& record = ev->Record; + UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusMultipleModifications); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 1); + + env.TestWaitNotification(runtime, txId - 1); + TestLs(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1, false, NLs::PathNotExist); + } +} // TBackupCollectionTests diff --git a/ydb/core/tx/schemeshard/ut_backup_collection/ya.make b/ydb/core/tx/schemeshard/ut_backup_collection/ya.make new file mode 100644 index 000000000000..278f89e60115 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_backup_collection/ya.make @@ -0,0 +1,27 @@ +UNITTEST_FOR(ydb/core/tx/schemeshard) + +FORK_SUBTESTS() + +SPLIT_FACTOR(2) + +IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/protos + ydb/core/tx/schemeshard/ut_helpers +) + +SRCS( + ut_backup_collection.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/ut_backup_collection_reboots/ut_backup_collection_reboots.cpp b/ydb/core/tx/schemeshard/ut_backup_collection_reboots/ut_backup_collection_reboots.cpp new file mode 100644 index 000000000000..e1a4008bcb52 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_backup_collection_reboots/ut_backup_collection_reboots.cpp @@ -0,0 +1,233 @@ +#include + +#define DEFAULT_NAME_1 "MyCollection1" +#define DEFAULT_NAME_2 "MyCollection2" + +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TBackupCollectionWithRebootsTests) { + TString DefaultCollectionSettings() { + return R"( + Name: ")" DEFAULT_NAME_1 R"(" + + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/Table1" + } + } + Cluster {} + )"; + } + + TString CollectionSettings(const TString& name) { + return Sprintf(R"( + Name: "%s" + + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/Table1" + } + } + Cluster {} + )", name.c_str()); + } + + Y_UNIT_TEST(CreateWithReboots) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + AsyncMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + + AsyncCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + + t.TestEnv->TestWaitNotification(runtime, {t.TxId, t.TxId - 1}); + + NKikimrSchemeOp::TBackupCollectionDescription properties; + properties.SetName(DEFAULT_NAME_1); + auto& entry = *properties.MutableExplicitEntryList()->AddEntries(); + entry.SetType(::NKikimrSchemeOp::TBackupCollectionDescription_TBackupEntry_EType_ETypeTable); + entry.SetPath("/MyRoot/Table1"); + properties.MutableCluster(); + + { + TInactiveZone inactive(activeZone); + auto describeResult = DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1); + TestDescribeResult(describeResult, {NLs::Finished}); + + UNIT_ASSERT(describeResult.GetPathDescription().HasBackupCollectionDescription()); + const auto& backupCollectionDescription = describeResult.GetPathDescription().GetBackupCollectionDescription(); + UNIT_ASSERT_VALUES_EQUAL(backupCollectionDescription.GetName(), DEFAULT_NAME_1); + UNIT_ASSERT_VALUES_EQUAL(backupCollectionDescription.GetVersion(), 0); + UNIT_ASSERT_VALUES_EQUAL(backupCollectionDescription.DebugString(), properties.DebugString()); + } + }); + } + + Y_UNIT_TEST(ParallelCreateDrop) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + TestMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + AsyncCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + AsyncDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId - 1); + + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(SimpleDropWithReboots) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TestMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TInactiveZone inactive(activeZone); + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(SimpleDropWithReboots2) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(DropWithReboots) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathNotExist}); + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(CreateDroppedWithReboots) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + }); + } + + Y_UNIT_TEST(CreateDroppedAndDropWithReboots) { + TTestWithReboots t; + t.EnvOpts = TTestEnvOptions().EnableBackupService(true); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestMkDir(runtime, ++t.TxId, "/MyRoot", ".backups/collections"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", DefaultCollectionSettings()); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropBackupCollection(runtime, ++t.TxId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), + {NLs::PathNotExist}); + } + }); + } +} // TBackupCollectionWithRebootsTests diff --git a/ydb/core/tx/schemeshard/ut_backup_collection_reboots/ya.make b/ydb/core/tx/schemeshard/ut_backup_collection_reboots/ya.make new file mode 100644 index 000000000000..5acc5035eb99 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_backup_collection_reboots/ya.make @@ -0,0 +1,26 @@ +UNITTEST_FOR(ydb/core/tx/schemeshard) + +FORK_SUBTESTS() + +SPLIT_FACTOR(20) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/tx/schemeshard/ut_helpers +) + +SRCS( + ut_backup_collection_reboots.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 7fe813667928..eb6cf1c79a3b 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -920,6 +920,11 @@ namespace NSchemeShardUT_Private { GENERIC_HELPERS(DropResourcePool, NKikimrSchemeOp::EOperationType::ESchemeOpDropResourcePool, &NKikimrSchemeOp::TModifyScheme::MutableDrop) DROP_BY_PATH_ID_HELPERS(DropResourcePool, NKikimrSchemeOp::EOperationType::ESchemeOpDropResourcePool) + // backup collection + GENERIC_HELPERS(CreateBackupCollection, NKikimrSchemeOp::EOperationType::ESchemeOpCreateBackupCollection, &NKikimrSchemeOp::TModifyScheme::MutableCreateBackupCollection) + GENERIC_HELPERS(DropBackupCollection, NKikimrSchemeOp::EOperationType::ESchemeOpDropBackupCollection, &NKikimrSchemeOp::TModifyScheme::MutableDropBackupCollection) + DROP_BY_PATH_ID_HELPERS(DropBackupCollection, NKikimrSchemeOp::EOperationType::ESchemeOpDropBackupCollection) + #undef DROP_BY_PATH_ID_HELPERS #undef GENERIC_WITH_ATTRS_HELPERS #undef GENERIC_HELPERS diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index eb4feee35b56..93a7ad7617fb 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -294,6 +294,11 @@ namespace NSchemeShardUT_Private { GENERIC_HELPERS(DropResourcePool); DROP_BY_PATH_ID_HELPERS(DropResourcePool); + // backup collection + GENERIC_HELPERS(CreateBackupCollection); + GENERIC_HELPERS(DropBackupCollection); + DROP_BY_PATH_ID_HELPERS(DropBackupCollection); + #undef DROP_BY_PATH_ID_HELPERS #undef GENERIC_WITH_ATTRS_HELPERS #undef GENERIC_HELPERS diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 36334aca0de1..43452de738ff 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -471,6 +471,13 @@ void IsResourcePool(const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeResourcePool); } +void IsBackupCollection(const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + const auto& pathDescr = record.GetPathDescription(); + const auto& selfPath = pathDescr.GetSelf(); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeBackupCollection); +} + TCheckFunc CheckColumns(const TString& name, const TSet& columns, const TSet& droppedColumns, const TSet keyColumns, bool strictCount) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT(record.HasPathDescription()); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 6e49cb379413..67709f1ef6e8 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -96,6 +96,7 @@ namespace NLs { void IsExternalDataSource(const NKikimrScheme::TEvDescribeSchemeResult& record); void IsView(const NKikimrScheme::TEvDescribeSchemeResult& record); void IsResourcePool(const NKikimrScheme::TEvDescribeSchemeResult& record); + void IsBackupCollection(const NKikimrScheme::TEvDescribeSchemeResult& record); TCheckFunc CheckColumns(const TString& name, const TSet& columns, const TSet& droppedColumns, const TSet keyColumns, bool strictCount = false); TCheckFunc CheckColumnType(const ui64 columnIndex, const TString& columnTypename); void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult& record); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 1aaaf05901ed..c50856e832dd 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -546,6 +546,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe app.SetEnableTieringInColumnShard(opts.EnableTieringInColumnShard_); app.SetEnableParameterizedDecimal(opts.EnableParameterizedDecimal_); app.SetEnableTopicAutopartitioningForCDC(opts.EnableTopicAutopartitioningForCDC_); + app.SetEnableBackupService(opts.EnableBackupService_); app.ColumnShardConfig.SetDisabledOnSchemeShard(false); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index 44a88dca509d..11b031913172 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -69,6 +69,7 @@ namespace NSchemeShardUT_Private { OPTION(std::optional, EnableTieringInColumnShard, std::nullopt); OPTION(std::optional, EnableParameterizedDecimal, std::nullopt); OPTION(std::optional, EnableTopicAutopartitioningForCDC, std::nullopt); + OPTION(std::optional, EnableBackupService, std::nullopt); #undef OPTION }; diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index c0f4082cbccf..7e0a802b9f23 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -2,6 +2,8 @@ RECURSE_FOR_TESTS( ut_auditsettings ut_background_cleaning ut_backup + ut_backup_collection + ut_backup_collection_reboots ut_base ut_base_reboots ut_bsvolume @@ -56,33 +58,31 @@ LIBRARY() SRCS( defs.h + operation_queue_timer.h schemeshard.cpp schemeshard__background_cleaning.cpp + schemeshard__backup_collection_common.cpp schemeshard__borrowed_compaction.cpp - schemeshard__compaction.cpp schemeshard__clean_pathes.cpp + schemeshard__compaction.cpp schemeshard__conditional_erase.cpp - schemeshard__describe_scheme.cpp schemeshard__delete_tablet_reply.cpp + schemeshard__describe_scheme.cpp schemeshard__find_subdomain_path_id.cpp schemeshard__fix_bad_paths.cpp schemeshard__init.cpp schemeshard__init_populator.cpp schemeshard__init_root.cpp schemeshard__init_schema.cpp - schemeshard__serverless_storage_billing.cpp - schemeshard__sync_update_tenants.cpp schemeshard__login.cpp + schemeshard__make_access_database_no_inheritable.cpp schemeshard__monitoring.cpp schemeshard__notify.cpp schemeshard__operation.cpp schemeshard__operation.h - schemeshard__operation_blob_depot.cpp - schemeshard__operation_side_effects.cpp - schemeshard__operation_side_effects.h - schemeshard__operation_memory_changes.cpp - schemeshard__operation_db_changes.cpp schemeshard__operation_alter_bsv.cpp + schemeshard__operation_alter_cdc_stream.cpp + schemeshard__operation_alter_continuous_backup.cpp schemeshard__operation_alter_external_data_source.cpp schemeshard__operation_alter_external_table.cpp schemeshard__operation_alter_extsubdomain.cpp @@ -98,8 +98,11 @@ SRCS( schemeshard__operation_alter_subdomain.cpp schemeshard__operation_alter_table.cpp schemeshard__operation_alter_user_attrs.cpp + schemeshard__operation_apply_build_index.cpp schemeshard__operation_assign_bsv.cpp + schemeshard__operation_blob_depot.cpp schemeshard__operation_cancel_tx.cpp + schemeshard__operation_cansel_build_index.cpp schemeshard__operation_common.cpp schemeshard__operation_common.h schemeshard__operation_common_external_data_source.cpp @@ -110,7 +113,11 @@ SRCS( schemeshard__operation_copy_sequence.cpp schemeshard__operation_copy_table.cpp schemeshard__operation_create_backup.cpp + schemeshard__operation_create_backup_collection.cpp schemeshard__operation_create_bsv.cpp + schemeshard__operation_create_build_index.cpp + schemeshard__operation_create_cdc_stream.cpp + schemeshard__operation_create_continuous_backup.cpp schemeshard__operation_create_external_data_source.cpp schemeshard__operation_create_external_table.cpp schemeshard__operation_create_extsubdomain.cpp @@ -130,11 +137,16 @@ SRCS( schemeshard__operation_create_subdomain.cpp schemeshard__operation_create_table.cpp schemeshard__operation_create_view.cpp + schemeshard__operation_db_changes.cpp + schemeshard__operation_drop_backup_collection.cpp schemeshard__operation_drop_bsv.cpp + schemeshard__operation_drop_cdc_stream.cpp + schemeshard__operation_drop_continuous_backup.cpp schemeshard__operation_drop_external_data_source.cpp schemeshard__operation_drop_external_table.cpp schemeshard__operation_drop_extsubdomain.cpp schemeshard__operation_drop_fs.cpp + schemeshard__operation_drop_index.cpp schemeshard__operation_drop_indexed_table.cpp schemeshard__operation_drop_kesus.cpp schemeshard__operation_drop_lock.cpp @@ -147,92 +159,85 @@ SRCS( schemeshard__operation_drop_table.cpp schemeshard__operation_drop_unsafe.cpp schemeshard__operation_drop_view.cpp + schemeshard__operation_finalize_build_index.cpp + schemeshard__operation_initiate_build_index.cpp + schemeshard__operation_just_reject.cpp + schemeshard__operation_memory_changes.cpp schemeshard__operation_mkdir.cpp schemeshard__operation_modify_acl.cpp schemeshard__operation_move_index.cpp schemeshard__operation_move_table.cpp - schemeshard__operation_move_tables.cpp schemeshard__operation_move_table_index.cpp + schemeshard__operation_move_tables.cpp schemeshard__operation_part.cpp schemeshard__operation_part.h schemeshard__operation_rmdir.cpp + schemeshard__operation_side_effects.cpp + schemeshard__operation_side_effects.h schemeshard__operation_split_merge.cpp - schemeshard__operation_just_reject.cpp schemeshard__operation_upgrade_subdomain.cpp - schemeshard__operation_initiate_build_index.cpp - schemeshard__operation_finalize_build_index.cpp - schemeshard__operation_create_build_index.cpp - schemeshard__operation_apply_build_index.cpp - schemeshard__operation_cansel_build_index.cpp - schemeshard__operation_drop_index.cpp - schemeshard__operation_create_cdc_stream.cpp - schemeshard__operation_alter_cdc_stream.cpp - schemeshard__operation_drop_cdc_stream.cpp - schemeshard__operation_create_continuous_backup.cpp - schemeshard__operation_alter_continuous_backup.cpp - schemeshard__operation_drop_continuous_backup.cpp schemeshard__pq_stats.cpp schemeshard__publish_to_scheme_board.cpp + schemeshard__serverless_storage_billing.cpp schemeshard__state_changed_reply.cpp + schemeshard__sync_update_tenants.cpp schemeshard__table_stats.cpp schemeshard__table_stats_histogram.cpp - schemeshard__upgrade_schema.cpp schemeshard__upgrade_access_database.cpp - schemeshard__make_access_database_no_inheritable.cpp - schemeshard_audit_log_fragment.cpp + schemeshard__upgrade_schema.cpp schemeshard_audit_log.cpp - schemeshard_impl.cpp + schemeshard_audit_log_fragment.cpp schemeshard_backup.cpp - schemeshard_impl.h + schemeshard_bg_tasks__list.cpp schemeshard_billing_helpers.cpp + schemeshard_build_index.cpp + schemeshard_build_index__cancel.cpp + schemeshard_build_index__create.cpp + schemeshard_build_index__forget.cpp + schemeshard_build_index__get.cpp + schemeshard_build_index__list.cpp + schemeshard_build_index__progress.cpp + schemeshard_build_index_tx_base.cpp schemeshard_cdc_stream_scan.cpp - schemeshard_domain_links.h schemeshard_domain_links.cpp - schemeshard_effective_acl.h + schemeshard_domain_links.h schemeshard_effective_acl.cpp + schemeshard_effective_acl.h + schemeshard_export.cpp + schemeshard_export__cancel.cpp + schemeshard_export__create.cpp + schemeshard_export__forget.cpp + schemeshard_export__get.cpp + schemeshard_export__list.cpp + schemeshard_export_flow_proposals.cpp schemeshard_identificators.cpp + schemeshard_impl.cpp + schemeshard_impl.h + schemeshard_import.cpp + schemeshard_import__cancel.cpp + schemeshard_import__create.cpp + schemeshard_import__forget.cpp + schemeshard_import__get.cpp + schemeshard_import__list.cpp + schemeshard_import_flow_proposals.cpp schemeshard_info_types.cpp schemeshard_info_types.h + schemeshard_path.cpp + schemeshard_path.h schemeshard_path_describer.cpp schemeshard_path_element.cpp schemeshard_path_element.h - schemeshard_path.cpp - schemeshard_path.h schemeshard_schema.h - schemeshard_svp_migration.h schemeshard_svp_migration.cpp + schemeshard_svp_migration.h schemeshard_tx_infly.h schemeshard_types.cpp schemeshard_types.h schemeshard_user_attr_limits.h schemeshard_utils.cpp schemeshard_utils.h - schemeshard_bg_tasks__list.cpp - schemeshard_xxport__helpers.cpp - schemeshard_export__cancel.cpp - schemeshard_export__create.cpp - schemeshard_export__forget.cpp - schemeshard_export__get.cpp - schemeshard_export__list.cpp - schemeshard_export_flow_proposals.cpp - schemeshard_export.cpp - schemeshard_import__cancel.cpp - schemeshard_import__create.cpp - schemeshard_import__forget.cpp - schemeshard_import__get.cpp - schemeshard_import__list.cpp - schemeshard_import_flow_proposals.cpp - schemeshard_import.cpp - schemeshard_build_index.cpp - schemeshard_build_index_tx_base.cpp - schemeshard_build_index__cancel.cpp - schemeshard_build_index__forget.cpp - schemeshard_build_index__list.cpp - schemeshard_build_index__create.cpp - schemeshard_build_index__get.cpp - schemeshard_build_index__progress.cpp schemeshard_validate_ttl.cpp - operation_queue_timer.h + schemeshard_xxport__helpers.cpp user_attributes.cpp ) diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp index b470a0e38889..307a155d169b 100644 --- a/ydb/core/tx/tx_proxy/schemereq.cpp +++ b/ydb/core/tx/tx_proxy/schemereq.cpp @@ -362,6 +362,15 @@ struct TBaseSchemeReq: public TActorBootstrapped { case NKikimrSchemeOp::ESchemeOpRestoreIncrementalBackup: case NKikimrSchemeOp::ESchemeOpRestoreIncrementalBackupAtTable: return *modifyScheme.MutableRestoreIncrementalBackup()->MutableSrcTableName(); + + case NKikimrSchemeOp::ESchemeOpCreateBackupCollection: + return *modifyScheme.MutableCreateBackupCollection()->MutableName(); + + case NKikimrSchemeOp::ESchemeOpAlterBackupCollection: + return *modifyScheme.MutableAlterBackupCollection()->MutableName(); + + case NKikimrSchemeOp::ESchemeOpDropBackupCollection: + return *modifyScheme.MutableDropBackupCollection()->MutableName(); } } @@ -385,6 +394,7 @@ struct TBaseSchemeReq: public TActorBootstrapped { case NKikimrSchemeOp::ESchemeOpCreateExternalDataSource: case NKikimrSchemeOp::ESchemeOpCreateView: case NKikimrSchemeOp::ESchemeOpCreateResourcePool: + case NKikimrSchemeOp::ESchemeOpCreateBackupCollection: return true; default: return false; @@ -622,6 +632,7 @@ struct TBaseSchemeReq: public TActorBootstrapped { case NKikimrSchemeOp::ESchemeOpDropContinuousBackup: case NKikimrSchemeOp::ESchemeOpAlterResourcePool: case NKikimrSchemeOp::ESchemeOpRestoreIncrementalBackup: + case NKikimrSchemeOp::ESchemeOpAlterBackupCollection: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme))); @@ -646,6 +657,7 @@ struct TBaseSchemeReq: public TActorBootstrapped { case NKikimrSchemeOp::ESchemeOpDropExternalDataSource: case NKikimrSchemeOp::ESchemeOpDropView: case NKikimrSchemeOp::ESchemeOpDropResourcePool: + case NKikimrSchemeOp::ESchemeOpDropBackupCollection: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme))); @@ -708,6 +720,7 @@ struct TBaseSchemeReq: public TActorBootstrapped { case NKikimrSchemeOp::ESchemeOpCreateExternalDataSource: case NKikimrSchemeOp::ESchemeOpCreateView: case NKikimrSchemeOp::ESchemeOpCreateResourcePool: + case NKikimrSchemeOp::ESchemeOpCreateBackupCollection: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = workingDir; diff --git a/ydb/core/viewer/browse.h b/ydb/core/viewer/browse.h index 69569a35b61c..32c6799a2473 100644 --- a/ydb/core/viewer/browse.h +++ b/ydb/core/viewer/browse.h @@ -62,6 +62,7 @@ class TBrowse : public TActorBootstrapped { switch (type) { case NKikimrSchemeOp::EPathType::EPathTypeDir: case NKikimrSchemeOp::EPathType::EPathTypeColumnStore: // TODO + case NKikimrSchemeOp::EPathType::EPathTypeBackupCollection: // TODO return NKikimrViewer::EObjectType::Directory; case NKikimrSchemeOp::EPathType::EPathTypeRtmrVolume: return NKikimrViewer::EObjectType::RtmrVolume; diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index 7698ac8780d3..2a722e8cac92 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -587,7 +587,8 @@ class TSchemaObject { ExternalTable, ExternalDataSource, View, - ResourcePool + ResourcePool, + BackupCollection }; TSchemaObject(TSchemaObject&&) = default; diff --git a/ydb/public/lib/deprecated/kicli/schema.cpp b/ydb/public/lib/deprecated/kicli/schema.cpp index 9991f3ddc9b1..0923796761c7 100644 --- a/ydb/public/lib/deprecated/kicli/schema.cpp +++ b/ydb/public/lib/deprecated/kicli/schema.cpp @@ -134,6 +134,10 @@ void TSchemaObject::Drop() { case EPathType::ResourcePool: drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropResourcePool); break; + case EPathType::BackupCollection: + // FIXME(+active) + // drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropBackupCollection); + // break; case EPathType::Unknown: case EPathType::SubDomain: case EPathType::RtmrVolume: @@ -227,6 +231,8 @@ static TSchemaObject::EPathType GetType(const NKikimrSchemeOp::TDirEntry& entry) return TSchemaObject::EPathType::View; case NKikimrSchemeOp::EPathTypeResourcePool: return TSchemaObject::EPathType::ResourcePool; + case NKikimrSchemeOp::EPathTypeBackupCollection: + return TSchemaObject::EPathType::BackupCollection; case NKikimrSchemeOp::EPathTypeTableIndex: case NKikimrSchemeOp::EPathTypeExtSubDomain: case NKikimrSchemeOp::EPathTypeCdcStream: diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 0712e409940a..2d81fdc889b0 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -7488,5 +7488,59 @@ "Blobs": 1 } } + }, + { + "TableId": 111, + "TableName": "BackupCollection", + "TableKey": [ + 1, + 2 + ], + "ColumnsAdded": [ + { + "ColumnId": 1, + "ColumnName": "OwnerPathId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 2, + "ColumnName": "LocalPathId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 3, + "ColumnName": "AlterVersion", + "ColumnType": "Uint64" + }, + { + "ColumnId": 4, + "ColumnName": "Description", + "ColumnType": "String" + } + ], + "ColumnsDropped": [], + "ColumnFamilies": { + "0": { + "Columns": [ + 1, + 2, + 3, + 4 + ], + "RoomID": 0, + "Codec": 0, + "InMemory": false, + "Cache": 0, + "Small": 4294967295, + "Large": 4294967295 + } + }, + "Rooms": { + "0": { + "Main": 1, + "Outer": 1, + "Blobs": 1 + } + } } ] \ No newline at end of file