Skip to content

Commit

Permalink
backup.cpp refactoring to unify the code a bit
Browse files Browse the repository at this point in the history
  • Loading branch information
jepett0 committed Jan 31, 2025
1 parent 1298f3b commit 941e3b1
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 26 deletions.
51 changes: 28 additions & 23 deletions ydb/library/backup/backup.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -479,24 +479,36 @@ void WriteProtoToFile(const google::protobuf::Message& proto, const TFsPath& fol
outFile.Write(protoStr.data(), protoStr.size());
}

void BackupPermissions(TDriver driver, const TString& dbPrefix, const TString& path, const TFsPath& folderPath) {
auto entry = DescribePath(driver, JoinDatabasePath(dbPrefix, path));
void WriteCreationQueryToFile(const TString& creationQuery, const TFsPath& folderPath, const NDump::NFiles::TFileInfo& fileInfo) {
LOG_D("Write " << fileInfo.LogObjectType << " into " << folderPath.Child(fileInfo.FileName).GetPath().Quote());
TFile outFile(folderPath.Child(fileInfo.FileName), CreateAlways | WrOnly);
outFile.Write(creationQuery.data(), creationQuery.size());
}

void BackupPermissions(TDriver driver, const TString& dbPath, const TFsPath& folderPath) {
auto entry = DescribePath(driver, dbPath);
Ydb::Scheme::ModifyPermissionsRequest proto;
entry.SerializeTo(proto);
WriteProtoToFile(proto, folderPath, NDump::NFiles::Permissions());
}

void BackupPermissions(TDriver driver, const TString& dbPrefix, const TString& path, const TFsPath& folderPath) {
BackupPermissions(driver, JoinDatabasePath(dbPrefix, path), folderPath);
}

Ydb::Table::ChangefeedDescription ProtoFromChangefeedDesc(const NTable::TChangefeedDescription& changefeedDesc) {
Ydb::Table::ChangefeedDescription protoChangeFeedDesc;
changefeedDesc.SerializeTo(protoChangeFeedDesc);
return protoChangeFeedDesc;
}

NTopic::TDescribeTopicResult DescribeTopic(TDriver driver, const TString& path) {
NYdb::NTopic::TTopicClient client(driver);
return NConsoleClient::RetryFunction([&]() {
return client.DescribeTopic(path).GetValueSync();
NTopic::TTopicDescription DescribeTopic(TDriver driver, const TString& path) {
NTopic::TTopicClient client(driver);
const auto result = NConsoleClient::RetryFunction([&]() {
return client.DescribeTopic(path).ExtractValueSync();
});
VerifyStatus(result, "describe topic to build a backup");
return result.GetTopicDescription();
}

void BackupChangefeeds(TDriver driver, const TString& tablePath, const TFsPath& folderPath) {
Expand All @@ -506,16 +518,14 @@ void BackupChangefeeds(TDriver driver, const TString& tablePath, const TFsPath&
TFsPath changefeedDirPath = CreateDirectory(folderPath, TString{changefeedDesc.GetName()});

auto protoChangeFeedDesc = ProtoFromChangefeedDesc(changefeedDesc);
const auto descTopicResult = DescribeTopic(driver, JoinDatabasePath(tablePath, TString{changefeedDesc.GetName()}));
VerifyStatus(descTopicResult);
const auto& topicDescription = descTopicResult.GetTopicDescription();
const auto topicDescription = DescribeTopic(driver, JoinDatabasePath(tablePath, TString{changefeedDesc.GetName()}));
auto protoTopicDescription = NYdb::TProtoAccessor::GetProto(topicDescription);
// Unnecessary fields
protoTopicDescription.clear_self();
protoTopicDescription.clear_topic_stats();

WriteProtoToFile(protoChangeFeedDesc, changefeedDirPath, NDump::NFiles::Changefeed());
WriteProtoToFile(protoTopicDescription, changefeedDirPath, NDump::NFiles::Topic());
WriteProtoToFile(protoTopicDescription, changefeedDirPath, NDump::NFiles::TopicDescription());
}
}

Expand All @@ -542,7 +552,8 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP

namespace {

NView::TViewDescription DescribeView(NView::TViewClient& client, const TString& path) {
NView::TViewDescription DescribeView(TDriver driver, const TString& path) {
NView::TViewClient client(driver);
auto status = NConsoleClient::RetryFunction([&]() {
return client.DescribeView(path).ExtractValueSync();
});
Expand All @@ -559,22 +570,18 @@ and writes it to the backup folder designated for this view.
\param dbBackupRoot the root of the backup in the database
\param dbPathRelativeToBackupRoot the path to the view in the database relative to the backup root
\param fsBackupDir the path on the file system to write the file with the CREATE VIEW statement to
\param fsBackupFolder the path on the file system to write the file with the CREATE VIEW statement to
\param issues the accumulated backup issues container
*/
void BackupView(TDriver driver, const TString& dbBackupRoot, const TString& dbPathRelativeToBackupRoot,
const TFsPath& fsBackupDir, NYql::TIssues& issues
const TFsPath& fsBackupFolder, NYql::TIssues& issues
) {
Y_ENSURE(!dbPathRelativeToBackupRoot.empty());
const auto dbPath = JoinDatabasePath(dbBackupRoot, dbPathRelativeToBackupRoot);

LOG_I("Backup view " << dbPath.Quote() << " to " << fsBackupDir.GetPath().Quote());
LOG_I("Backup view " << dbPath.Quote() << " to " << fsBackupFolder.GetPath().Quote());

NView::TViewClient client(driver);
const auto viewDescription = DescribeView(client, dbPath);

const auto fsPath = fsBackupDir.Child(NDump::NFiles::CreateView().FileName);
LOG_D("Write view creation query to " << fsPath.GetPath().Quote());
const auto viewDescription = DescribeView(driver, dbPath);

const auto creationQuery = NDump::BuildCreateViewQuery(
TFsPath(dbPathRelativeToBackupRoot).GetName(),
Expand All @@ -585,10 +592,8 @@ void BackupView(TDriver driver, const TString& dbBackupRoot, const TString& dbPa
);
Y_ENSURE(creationQuery, issues.ToString());

TFileOutput output(fsPath);
output << creationQuery;

BackupPermissions(driver, dbBackupRoot, dbPathRelativeToBackupRoot, fsBackupDir);
WriteCreationQueryToFile(creationQuery, fsBackupFolder, NDump::NFiles::CreateView());
BackupPermissions(driver, dbPath, fsBackupFolder);
}

void CreateClusterDirectory(const TDriver& driver, const TString& path, bool rootBackupDir = false) {
Expand Down
8 changes: 7 additions & 1 deletion ydb/public/lib/ydb_cli/dump/files/files.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ enum EFilesType {
PERMISSIONS,
CHANGEFEED_DESCRIPTION,
TOPIC_DESCRIPTION,
CREATE_TOPIC,
INCOMPLETE_DATA,
INCOMPLETE,
EMPTY,
Expand All @@ -18,6 +19,7 @@ static constexpr TFileInfo FILES_INFO[] = {
{"permissions.pb", "ACL"},
{"changefeed_description.pb", "changefeed"},
{"topic_description.pb", "topic"},
{"create_topic.pb", "topic"},
{"incomplete.csv", "incomplete"},
{"incomplete", "incomplete"},
{"empty_dir", "empty_dir"},
Expand All @@ -36,10 +38,14 @@ const TFileInfo& Changefeed() {
return FILES_INFO[CHANGEFEED_DESCRIPTION];
}

const TFileInfo& Topic() {
const TFileInfo& TopicDescription() {
return FILES_INFO[TOPIC_DESCRIPTION];
}

const TFileInfo& CreateTopic() {
return FILES_INFO[CREATE_TOPIC];
}

const TFileInfo& IncompleteData() {
return FILES_INFO[INCOMPLETE_DATA];
}
Expand Down
3 changes: 2 additions & 1 deletion ydb/public/lib/ydb_cli/dump/files/files.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ struct TFileInfo {
const TFileInfo& TableScheme();
const TFileInfo& Permissions();
const TFileInfo& Changefeed();
const TFileInfo& Topic();
const TFileInfo& TopicDescription();
const TFileInfo& CreateTopic();
const TFileInfo& IncompleteData();
const TFileInfo& Incomplete();
const TFileInfo& Empty();
Expand Down
2 changes: 1 addition & 1 deletion ydb/public/lib/ydb_cli/dump/restore_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ Ydb::Table::ChangefeedDescription ReadChangefeedDescription(const TFsPath& fsDir
}

Ydb::Topic::DescribeTopicResult ReadTopicDescription(const TFsPath& fsDirPath, const TLog* log) {
return ReadProtoFromFile<Ydb::Topic::DescribeTopicResult>(fsDirPath, log, NDump::NFiles::Topic());
return ReadProtoFromFile<Ydb::Topic::DescribeTopicResult>(fsDirPath, log, NDump::NFiles::TopicDescription());
}

TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) {
Expand Down

0 comments on commit 941e3b1

Please sign in to comment.