Skip to content

Commit

Permalink
Add backup collection schema path item (ydb-platform#10252)
Browse files Browse the repository at this point in the history
  • Loading branch information
Enjection authored and nikvas0 committed Oct 15, 2024
1 parent 5cf87f5 commit d8b4e03
Show file tree
Hide file tree
Showing 43 changed files with 1,549 additions and 61 deletions.
6 changes: 6 additions & 0 deletions ydb/core/driver_lib/cli_base/cli_cmds_db.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,9 @@ class TClientCommandSchemaDescribe : public TClientCommand {
case NKikimrSchemeOp::EPathTypePersQueueGroup:
type = "<pq group>";
break;
case NKikimrSchemeOp::EPathTypeBackupCollection:
type = "<backup collection>";
break;
default:
type = "<unknown>";
break;
Expand Down Expand Up @@ -487,6 +490,9 @@ class TClientCommandSchemaLs : public TClientCommand {
case NKikimrSchemeOp::EPathTypeReplication:
type = "<replication>";
break;
case NKikimrSchemeOp::EPathTypeBackupCollection:
type = "<backup collection>";
break;
default:
type = "<unknown>";
break;
Expand Down
12 changes: 12 additions & 0 deletions ydb/core/protos/counters_schemeshard.proto
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,10 @@ enum ESimpleCounters {
COUNTER_IN_FLIGHT_OPS_TxAlterResourcePool = 176 [(CounterOpts) = {Name: "InFlightOps/AlterResourcePool"}];

COUNTER_IN_FLIGHT_OPS_TxRestoreIncrementalBackupAtTable = 177 [(CounterOpts) = {Name: "InFlightOps/RestoreIncrementalBackupAtTable"}];
COUNTER_BACKUP_COLLECTION_COUNT = 178 [(CounterOpts) = {Name: "BackupCollectionCount"}];
COUNTER_IN_FLIGHT_OPS_TxCreateBackupCollection = 179 [(CounterOpts) = {Name: "InFlightOps/CreateBackupCollection"}];
COUNTER_IN_FLIGHT_OPS_TxAlterBackupCollection = 180 [(CounterOpts) = {Name: "InFlightOps/AlterBackupCollection"}];
COUNTER_IN_FLIGHT_OPS_TxDropBackupCollection = 181 [(CounterOpts) = {Name: "InFlightOps/DropBackupCollection"}];
}

enum ECumulativeCounters {
Expand Down Expand Up @@ -355,6 +359,10 @@ enum ECumulativeCounters {
COUNTER_FINISHED_OPS_TxAlterResourcePool = 105 [(CounterOpts) = {Name: "FinishedOps/AlterResourcePool"}];

COUNTER_FINISHED_OPS_TxRestoreIncrementalBackupAtTable = 106 [(CounterOpts) = {Name: "FinishedOps/RestoreIncrementalBackupAtTable"}];

COUNTER_FINISHED_OPS_TxCreateBackupCollection = 107 [(CounterOpts) = {Name: "FinishedOps/CreateBackupCollection"}];
COUNTER_FINISHED_OPS_TxAlterBackupCollection = 108 [(CounterOpts) = {Name: "FinishedOps/AlterBackupCollection"}];
COUNTER_FINISHED_OPS_TxDropBackupCollection = 109 [(CounterOpts) = {Name: "FinishedOps/DropBackupCollection"}];
}

enum EPercentileCounters {
Expand Down Expand Up @@ -578,4 +586,8 @@ enum ETxTypes {
TXTYPE_UPDATE_DOMAIN_REPLY = 85 [(TxTypeOpts) = {Name: "TxUpdateDomainReply"}];

TXTYPE_SEQUENCESHARD_GET_SEQUENCE_RESULT = 86 [(TxTypeOpts) = {Name: "TxSequenceShardGetSequenceResult"}];

TXTYPE_CREATE_BACKUP_COLLECTION_RESULT = 87 [(TxTypeOpts) = {Name: "TxCreateBackupCollectionResult"}];
TXTYPE_ALTER_BACKUP_COLLECTION_RESULT = 88 [(TxTypeOpts) = {Name: "TxAlterBackupCollectionResult"}];
TXTYPE_DROP_BACKUP_COLLECTION_RESULT = 89 [(TxTypeOpts) = {Name: "TxDropBackupCollectionResult"}];
}
49 changes: 49 additions & 0 deletions ydb/core/protos/flat_scheme_op.proto
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import "ydb/core/tx/columnshard/engines/scheme/defaults/protos/data.proto";
import "ydb/core/tx/columnshard/common/protos/snapshot.proto";
import "ydb/library/formats/arrow/protos/accessor.proto";

import "google/protobuf/empty.proto";
import "google/protobuf/struct.proto";

package NKikimrSchemeOp;
Expand Down Expand Up @@ -1669,6 +1670,11 @@ enum EOperationType {

ESchemeOpRestoreIncrementalBackup = 103;
ESchemeOpRestoreIncrementalBackupAtTable = 104;

// BackupCollection
ESchemeOpCreateBackupCollection = 105;
ESchemeOpAlterBackupCollection = 106;
ESchemeOpDropBackupCollection = 107;
}

message TApplyIf {
Expand Down Expand Up @@ -1852,6 +1858,10 @@ message TModifyScheme {
optional TResourcePoolDescription CreateResourcePool = 72;

optional TRestoreIncrementalBackup RestoreIncrementalBackup = 73;

optional TBackupCollectionDescription CreateBackupCollection = 74;
optional TBackupCollectionDescription AlterBackupCollection = 75;
optional TBackupCollectionDescription DropBackupCollection = 76;
}

message TCopySequence {
Expand Down Expand Up @@ -1917,6 +1927,7 @@ enum EPathType {
EPathTypeExternalDataSource = 19;
EPathTypeView = 20;
EPathTypeResourcePool = 21;
EPathTypeBackupCollection = 22;
}

enum EPathSubType {
Expand Down Expand Up @@ -1973,6 +1984,7 @@ message TPathVersion {
optional uint64 ExternalDataSourceVersion = 28;
optional uint64 ViewVersion = 29;
optional uint64 ResourcePoolVersion = 30;
optional uint64 BackupCollectionVersion = 31;
}

// Describes single path
Expand Down Expand Up @@ -2062,6 +2074,7 @@ message TPathDescription {
optional TExternalDataSourceDescription ExternalDataSourceDescription = 28;
optional TViewDescription ViewDescription = 29;
optional TResourcePoolDescription ResourcePoolDescription = 30;
optional TBackupCollectionDescription BackupCollectionDescription = 31;
}

// For persisting AlterTable Tx description in Schemeshard internal DB
Expand Down Expand Up @@ -2202,3 +2215,39 @@ message TResourcePoolDescription {
optional uint64 Version = 3;
optional TResourcePoolProperties Properties = 4;
}

message TBackupCollectionDescription {
optional string Name = 1;
optional NKikimrProto.TPathID PathId = 2;
optional uint64 Version = 3;

message TBackupEntry {
enum EType {
ETypeInvalid = 0;
ETypeTable = 1;
}

optional EType Type = 1;
optional string Path = 2;
}

message TExplicitEntryList {
repeated TBackupEntry Entries = 1;
}

message TIncrementalBackupConfig {

}

oneof Entries {
google.protobuf.Empty Database = 4;
TExplicitEntryList ExplicitEntryList = 5;
}

// non-empty enables data-collection
optional TIncrementalBackupConfig IncrementalBackupConfig = 6;

oneof Storage {
google.protobuf.Empty Cluster = 7;
}
}
13 changes: 13 additions & 0 deletions ydb/core/tx/scheme_board/cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ namespace {
entry.BlobDepotInfo.Drop();
entry.BlockStoreVolumeInfo.Drop();
entry.FileStoreInfo.Drop();
entry.BackupCollectionInfo.Drop();
}

static void SetErrorAndClear(TResolveContext* context, TResolve::TEntry& entry, const bool isDescribeDenied) {
Expand Down Expand Up @@ -763,6 +764,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
FileStoreInfo.Drop();
ViewInfo.Drop();
ResourcePoolInfo.Drop();
BackupCollectionInfo.Drop();
}

void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) {
Expand Down Expand Up @@ -1278,6 +1280,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
DESCRIPTION_PART(FileStoreInfo);
DESCRIPTION_PART(ViewInfo);
DESCRIPTION_PART(ResourcePoolInfo);
DESCRIPTION_PART(BackupCollectionInfo);

#undef DESCRIPTION_PART

Expand Down Expand Up @@ -1610,6 +1613,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
Kind = TNavigate::KindResourcePool;
FillInfo(Kind, ResourcePoolInfo, std::move(*pathDesc.MutableResourcePoolDescription()));
break;
case NKikimrSchemeOp::EPathTypeBackupCollection:
Kind = TNavigate::KindBackupCollection;
FillInfo(Kind, BackupCollectionInfo, std::move(*pathDesc.MutableBackupCollectionDescription()));
break;
case NKikimrSchemeOp::EPathTypeInvalid:
Y_DEBUG_ABORT("Invalid path type");
break;
Expand Down Expand Up @@ -1683,6 +1690,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
case NKikimrSchemeOp::EPathTypeResourcePool:
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindResourcePool);
break;
case NKikimrSchemeOp::EPathTypeBackupCollection:
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindBackupCollection);
break;
case NKikimrSchemeOp::EPathTypeTableIndex:
case NKikimrSchemeOp::EPathTypeInvalid:
Y_DEBUG_ABORT("Invalid path type");
Expand Down Expand Up @@ -1904,6 +1914,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
entry.FileStoreInfo = FileStoreInfo;
entry.ViewInfo = ViewInfo;
entry.ResourcePoolInfo = ResourcePoolInfo;
entry.BackupCollectionInfo = BackupCollectionInfo;
}

bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry,
Expand Down Expand Up @@ -2202,6 +2213,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
// ResourcePool specific
TIntrusivePtr<TNavigate::TResourcePoolInfo> ResourcePoolInfo;

// BackupCollection specific
TIntrusivePtr<TNavigate::TBackupCollectionInfo> BackupCollectionInfo;
}; // TCacheItem

struct TMerger {
Expand Down
7 changes: 7 additions & 0 deletions ydb/core/tx/scheme_cache/scheme_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ struct TSchemeCacheNavigate {
KindFileStore = 20,
KindView = 21,
KindResourcePool = 22,
KindBackupCollection = 23,
};

struct TListNodeEntry : public TAtomicRefCount<TListNodeEntry> {
Expand Down Expand Up @@ -275,6 +276,11 @@ struct TSchemeCacheNavigate {
NKikimrSchemeOp::TResourcePoolDescription Description;
};

struct TBackupCollectionInfo : public TAtomicRefCount<TBackupCollectionInfo> {
EKind Kind = KindUnknown;
NKikimrSchemeOp::TBackupCollectionDescription Description;
};

struct TEntry {
enum class ERequestType : ui8 {
ByPath,
Expand Down Expand Up @@ -327,6 +333,7 @@ struct TSchemeCacheNavigate {
TIntrusiveConstPtr<TFileStoreInfo> FileStoreInfo;
TIntrusiveConstPtr<TViewInfo> ViewInfo;
TIntrusiveConstPtr<TResourcePoolInfo> ResourcePoolInfo;
TIntrusiveConstPtr<TBackupCollectionInfo> BackupCollectionInfo;

TString ToString() const;
TString ToString(const NScheme::TTypeRegistry& typeRegistry) const;
Expand Down
79 changes: 79 additions & 0 deletions ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#include "schemeshard__backup_collection_common.h"

namespace NKikimr::NSchemeShard {

std::optional<TBackupCollectionPaths> ResolveBackupCollectionPaths(
const TString& rootPathStr,
const TString& name,
bool validateFeatureFlag,
TOperationContext& context,
THolder<TProposeResponse>& result)
{
bool backupServiceEnabled = AppData()->FeatureFlags.GetEnableBackupService();
if (!backupServiceEnabled && validateFeatureFlag) {
result->SetError(NKikimrScheme::StatusPreconditionFailed, "Backup collections are disabled. Please contact your system administrator to enable it");
return std::nullopt;
}

const TPath& rootPath = TPath::Resolve(rootPathStr, context.SS);
{
const auto checks = rootPath.Check();
checks
.NotEmpty()
.NotUnderDomainUpgrade()
.IsAtLocalSchemeShard()
.IsResolved()
.NotDeleted()
.NotUnderDeleting()
.IsCommonSensePath()
.IsLikeDirectory()
.FailOnRestrictedCreateInTempZone();

if (!checks) {
result->SetError(checks.GetStatus(), checks.GetError());
return std::nullopt;
}
}

const TString& backupCollectionsDir = JoinPath({rootPath.GetDomainPathString(), ".backups/collections"});

TPathSplitUnix absPathSplit(name);

if (absPathSplit.size() > 1 && !absPathSplit.IsAbsolute) {
result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Backup collections must be placed directly in " << backupCollectionsDir);
return std::nullopt;
}

const TPath& backupCollectionsPath = TPath::Resolve(backupCollectionsDir, context.SS);
{
const auto checks = backupCollectionsPath.Check();
checks.NotUnderDomainUpgrade()
.IsAtLocalSchemeShard()
.IsResolved()
.NotDeleted()
.NotUnderDeleting()
.IsCommonSensePath()
.IsLikeDirectory();

if (!checks) {
result->SetError(checks.GetStatus(), checks.GetError());
return std::nullopt;
}
}

std::optional<TPath> parentPath;
if (absPathSplit.size() > 1) {
TString realParent = "/" + JoinRange("/", absPathSplit.begin(), absPathSplit.end() - 1);
parentPath = TPath::Resolve(realParent, context.SS);
}

TPath dstPath = absPathSplit.IsAbsolute && parentPath ? parentPath->Child(TString(absPathSplit.back())) : rootPath.Child(name);
if (!dstPath.PathString().StartsWith(backupCollectionsDir + "/")) {
result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Backup collections must be placed in " << backupCollectionsDir);
return std::nullopt;
}

return TBackupCollectionPaths{rootPath, dstPath};
}

} // namespace NKikimr::NSchemeShard
24 changes: 24 additions & 0 deletions ydb/core/tx/schemeshard/schemeshard__backup_collection_common.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#pragma once

#include "schemeshard__operation_common.h"
#include "schemeshard_impl.h"

#include <util/generic/string.h>

#include <optional>

namespace NKikimr::NSchemeShard {

struct TBackupCollectionPaths {
TPath RootPath;
TPath DstPath;
};

std::optional<TBackupCollectionPaths> ResolveBackupCollectionPaths(
const TString& rootPathStr,
const TString& name,
bool preValidateDst,
TOperationContext& context,
THolder<TProposeResponse>& result);

} // namespace NKikimr::NSchemeShard
23 changes: 23 additions & 0 deletions ydb/core/tx/schemeshard/schemeshard__init.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1956,6 +1956,29 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
}
}

// Read backup collections
{
auto rowset = db.Table<Schema::BackupCollection>().Range().Select();
if (!rowset.IsReady()) {
return false;
}

while (!rowset.EndOfSet()) {
TOwnerId ownerPathId = rowset.GetValue<Schema::BackupCollection::OwnerPathId>();
TLocalPathId localPathId = rowset.GetValue<Schema::BackupCollection::LocalPathId>();
TPathId pathId(ownerPathId, localPathId);

auto& backupCollection = Self->BackupCollections[pathId] = new TBackupCollectionInfo();
backupCollection->AlterVersion = rowset.GetValue<Schema::BackupCollection::AlterVersion>();
Y_PROTOBUF_SUPPRESS_NODISCARD backupCollection->Description.ParseFromString(rowset.GetValue<Schema::BackupCollection::Description>());
Self->IncrementPathDbRefCount(pathId);

if (!rowset.Next()) {
return false;
}
}
}

// Read table columns
{
TColumnRows columnRows;
Expand Down
Loading

0 comments on commit d8b4e03

Please sign in to comment.