From 06878e559b711211dcec4129f8844d9e9b998cc4 Mon Sep 17 00:00:00 2001 From: Daniil Demin Date: Tue, 14 Jan 2025 15:26:23 +0000 Subject: [PATCH] import: dummy metadata for views --- .../schemeshard_export__create.cpp | 8 ++- .../schemeshard_export_scheme_uploader.cpp | 57 +++++++++++++++++-- .../schemeshard_export_scheme_uploader.h | 2 +- ...eshard_export_scheme_uploader_fallback.cpp | 3 +- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index 9ade9972898e..358a5ac726b5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -334,8 +334,14 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(exportSettings.ParseFromString(exportInfo->Settings)); const auto databaseRoot = TStringBuilder() << '/' << JoinSeq('/', Self->RootPathElements); + NBackup::TMetadata metadata; + // do not check hashes of views in backups + constexpr bool EnableChecksums = false; + metadata.SetVersion(EnableChecksums ? 1 : 0); + item.SchemeUploader = ctx.Register(CreateSchemeUploader( - Self->SelfId(), exportInfo->Id, itemIdx, item.SourcePathId, exportSettings, databaseRoot + Self->SelfId(), exportInfo->Id, itemIdx, item.SourcePathId, + exportSettings, databaseRoot, metadata.Serialize() )); } } diff --git a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp index d660fbbdd725..25f0bcc3cf69 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp @@ -45,7 +45,7 @@ class TSchemeUploader: public TActorBootstrapped { void HandleSchemeDescription(TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { const auto& describeResult = ev->Get()->GetRecord(); - LOG_D("HandleSchemeDescription TEvSchemeShard::TEvDescribeSchemeResult" + LOG_D("HandleSchemeDescription" << ", self: " << this->SelfId() << ", status: " << describeResult.GetStatus() ); @@ -109,7 +109,7 @@ class TSchemeUploader: public TActorBootstrapped { void HandleSchemePutResponse(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - LOG_D("HandleSchemePutResponse TEvExternalStorage::TEvPutObjectResponse" + LOG_D("HandleSchemePutResponse" << ", self: " << this->SelfId() << ", result: " << result ); @@ -137,7 +137,7 @@ class TSchemeUploader: public TActorBootstrapped { void HandlePermissionsPutResponse(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - LOG_D("HandlePermissionsPutResponse TEvExternalStorage::TEvPutObjectResponse" + LOG_D("HandlePermissionsPutResponse" << ", self: " << this->SelfId() << ", result: " << result ); @@ -146,6 +146,34 @@ class TSchemeUploader: public TActorBootstrapped { return; } PermissionsUploaded = true; + UploadMetadata(); + } + + void UploadMetadata() { + Y_ABORT_UNLESS(!MetadataUploaded); + + if (!Metadata) { + return Finish(false, "empty metadata"); + } + auto request = Aws::S3::Model::PutObjectRequest() + .WithKey(Sprintf("%s/metadata.json", DestinationPrefix.c_str())); + + this->Send(StorageOperator, new TEvExternalStorage::TEvPutObjectRequest(request, TString(Metadata))); + this->Become(&TThis::StateUploadMetadata); + } + + void HandleMetadataPutResponse(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + LOG_D("HandleMetadataPutResponse" + << ", self: " << this->SelfId() + << ", result: " << result + ); + + if (!CheckResult(result, TStringBuf("PutObject (metadata)"))) { + return; + } + MetadataUploaded = true; Finish(); } @@ -212,7 +240,8 @@ class TSchemeUploader: public TActorBootstrapped { ui32 itemIdx, TPathId sourcePathId, const Ydb::Export::ExportToS3Settings& settings, - const TString& databaseRoot + const TString& databaseRoot, + const TString& metadata ) : SchemeShard(schemeShard) , ExportId(exportId) @@ -221,6 +250,7 @@ class TSchemeUploader: public TActorBootstrapped { , ExternalStorageConfig(new TS3ExternalStorageConfig(settings)) , Retries(settings.number_of_retries()) , DatabaseRoot(databaseRoot) + , Metadata(metadata) { if (itemIdx < ui32(settings.items_size())) { DestinationPrefix = settings.items(itemIdx).destination_prefix(); @@ -245,6 +275,10 @@ class TSchemeUploader: public TActorBootstrapped { UploadPermissions(); return; } + if (!MetadataUploaded) { + UploadMetadata(); + return; + } Finish(); } @@ -279,6 +313,14 @@ class TSchemeUploader: public TActorBootstrapped { } } + STATEFN(StateUploadMetadata) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleMetadataPutResponse); + default: + return StateBase(ev); + } + } + private: TActorId SchemeShard; @@ -306,12 +348,15 @@ class TSchemeUploader: public TActorBootstrapped { TString Permissions; bool PermissionsUploaded = false; + TString Metadata; + bool MetadataUploaded = false; + }; // TSchemeUploader IActor* CreateSchemeUploader(TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId, - const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot + const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot, const TString& metadata ) { - return new TSchemeUploader(schemeShard, exportId, itemIdx, sourcePathId, settings, databaseRoot); + return new TSchemeUploader(schemeShard, exportId, itemIdx, sourcePathId, settings, databaseRoot, metadata); } } // NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h index 2262ab4cf674..d64d53a318c0 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h +++ b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h @@ -10,7 +10,7 @@ namespace Ydb::Export { namespace NKikimr::NSchemeShard { NActors::IActor* CreateSchemeUploader(NActors::TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId, - const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot + const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot, const TString& metadata ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp index 7ecc6ee95d12..030551ed79eb 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp @@ -31,8 +31,9 @@ class TSchemeUploaderFallback: public TActorBootstrapped