diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.h b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.h index db43cdf4d0f..31a09cc106d 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.h +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.h @@ -516,7 +516,7 @@ class TDiskRegistryState /// Mark selected device as clean and remove it /// from lists of suspended/dirty/pending cleanup devices - /// @return deallocated disk id of where selected device was deallocated + /// @return disk id where selected device was deallocated TDiskId MarkDeviceAsClean( TInstant now, TDiskRegistryDatabase& db, diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_create.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_create.cpp index 52db53ee190..568935722a5 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_create.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_create.cpp @@ -492,6 +492,283 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateCreateTest) } } + Y_UNIT_TEST(ShouldAllocateLocalWithDirtyDevices) + { + TTestExecutor executor; + executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); }); + + constexpr ui64 LocalDeviceSize = 99999997952; // ~ 93.13 GiB + + auto makeLocalDevice = [](const auto* name, const auto* uuid) + { + return Device(name, uuid) | + WithPool("local-ssd", NProto::DEVICE_POOL_KIND_LOCAL) | + WithTotalSize(LocalDeviceSize); + }; + + auto agentConfig = AgentConfig( + 1, + { + makeLocalDevice("NVMELOCAL01", "uuid-1"), + makeLocalDevice("NVMELOCAL02", "uuid-2"), + makeLocalDevice("NVMELOCAL03", "uuid-3"), + }); + const TVector agents{ + agentConfig, + }; + + TDiskRegistryState state = + TDiskRegistryStateBuilder() + .WithConfig( + [&] + { + auto config = MakeConfig(0, agents); + + auto* pool = config.AddDevicePoolConfigs(); + pool->SetName("local-ssd"); + pool->SetKind(NProto::DEVICE_POOL_KIND_LOCAL); + pool->SetAllocationUnit(LocalDeviceSize); + + return config; + }()) + .WithAgents(agents) + .WithDirtyDevices( + {TDirtyDevice{"uuid-1", {}}, + TDirtyDevice{"uuid-2", {}}, + TDirtyDevice{"uuid-3", {}}}) + .Build(); + + auto allocate = [&](auto db, ui32 deviceCount) + { + TDiskRegistryState::TAllocateDiskResult result; + + auto error = state.AllocateDisk( + TInstant::Zero(), + db, + TDiskRegistryState::TAllocateDiskParams{ + .DiskId = "local0", + .BlockSize = DefaultLogicalBlockSize, + .BlocksCount = + deviceCount * LocalDeviceSize / DefaultLogicalBlockSize, + .MediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL}, + &result); + + return std::make_pair(std::move(result), error); + }; + + // Register agents. + executor.WriteTx( + [&](TDiskRegistryDatabase db) { + UNIT_ASSERT_SUCCESS( + RegisterAgent(state, db, agentConfig, Now())); + }); + + UNIT_ASSERT_VALUES_EQUAL(1, state.GetConfig().KnownAgentsSize()); + UNIT_ASSERT_VALUES_EQUAL(1, state.GetAgents().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetSuspendedDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(3, state.GetDirtyDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetBrokenDevices().size()); + + // Allocate disk with dirty devices + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + auto [result, error] = allocate(db, 3U); + UNIT_ASSERT_SUCCESS(error); + UNIT_ASSERT_VALUES_EQUAL(3U, result.Devices.size()); + UNIT_ASSERT_VALUES_EQUAL(3U, result.DirtyDevices.size()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL01", + result.Devices[0].GetDeviceName()); + }); + } + + Y_UNIT_TEST(ShouldFavorCleanLocalDevices) + { + TTestExecutor executor; + executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); }); + + constexpr ui64 LocalDeviceSize = 99999997952; // ~ 93.13 GiB + + auto makeLocalDevice = [](const auto* name, const auto* uuid) + { + return Device(name, uuid) | + WithPool("local-ssd", NProto::DEVICE_POOL_KIND_LOCAL) | + WithTotalSize(LocalDeviceSize); + }; + + auto agentConfig = AgentConfig( + 1, + { + makeLocalDevice("NVMELOCAL01", "uuid-1"), + makeLocalDevice("NVMELOCAL02", "uuid-2"), + makeLocalDevice("NVMELOCAL03", "uuid-3"), + }); + const TVector agents{ + agentConfig, + }; + + TDiskRegistryState state = + TDiskRegistryStateBuilder() + .WithConfig( + [&] + { + auto config = MakeConfig(0, agents); + + auto* pool = config.AddDevicePoolConfigs(); + pool->SetName("local-ssd"); + pool->SetKind(NProto::DEVICE_POOL_KIND_LOCAL); + pool->SetAllocationUnit(LocalDeviceSize); + + return config; + }()) + .WithAgents(agents) + .WithDirtyDevices( + {TDirtyDevice{"uuid-1", {}}, + TDirtyDevice{"uuid-3", {}}}) + .Build(); + + auto allocate = [&](auto db, ui32 deviceCount) + { + TDiskRegistryState::TAllocateDiskResult result; + + auto error = state.AllocateDisk( + TInstant::Zero(), + db, + TDiskRegistryState::TAllocateDiskParams{ + .DiskId = "local0", + .BlockSize = DefaultLogicalBlockSize, + .BlocksCount = + deviceCount * LocalDeviceSize / DefaultLogicalBlockSize, + .MediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL}, + &result); + + return std::make_pair(std::move(result), error); + }; + + // Register agents. + executor.WriteTx( + [&](TDiskRegistryDatabase db) { + UNIT_ASSERT_SUCCESS( + RegisterAgent(state, db, agentConfig, Now())); + }); + + UNIT_ASSERT_VALUES_EQUAL(1, state.GetConfig().KnownAgentsSize()); + UNIT_ASSERT_VALUES_EQUAL(1, state.GetAgents().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetSuspendedDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(2, state.GetDirtyDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetBrokenDevices().size()); + + // Allocate disk with dirty devices + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + auto [result, error] = allocate(db, 1U); + UNIT_ASSERT_SUCCESS(error); + UNIT_ASSERT_VALUES_EQUAL(1U, result.Devices.size()); + UNIT_ASSERT_VALUES_EQUAL(0U, result.DirtyDevices.size()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL02", + result.Devices[0].GetDeviceName()); + }); + } + + Y_UNIT_TEST(ShouldFavorCleanDevicesAndAllocateDirty) + { + TTestExecutor executor; + executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); }); + + constexpr ui64 LocalDeviceSize = 99999997952; // ~ 93.13 GiB + + auto makeLocalDevice = [](const auto* name, const auto* uuid) + { + return Device(name, uuid) | + WithPool("local-ssd", NProto::DEVICE_POOL_KIND_LOCAL) | + WithTotalSize(LocalDeviceSize); + }; + + auto agentConfig = AgentConfig( + 1, + { + makeLocalDevice("NVMELOCAL01", "uuid-1"), + makeLocalDevice("NVMELOCAL02", "uuid-2"), + makeLocalDevice("NVMELOCAL03", "uuid-3"), + }); + const TVector agents{ + agentConfig, + }; + + TDiskRegistryState state = + TDiskRegistryStateBuilder() + .WithConfig( + [&] + { + auto config = MakeConfig(0, agents); + + auto* pool = config.AddDevicePoolConfigs(); + pool->SetName("local-ssd"); + pool->SetKind(NProto::DEVICE_POOL_KIND_LOCAL); + pool->SetAllocationUnit(LocalDeviceSize); + + return config; + }()) + .WithAgents(agents) + .WithDirtyDevices( + {TDirtyDevice{"uuid-1", {}}, + TDirtyDevice{"uuid-3", {}}}) + .Build(); + + auto allocate = [&](auto db, ui32 deviceCount) + { + TDiskRegistryState::TAllocateDiskResult result; + + auto error = state.AllocateDisk( + TInstant::Zero(), + db, + TDiskRegistryState::TAllocateDiskParams{ + .DiskId = "local0", + .BlockSize = DefaultLogicalBlockSize, + .BlocksCount = + deviceCount * LocalDeviceSize / DefaultLogicalBlockSize, + .MediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL}, + &result); + + return std::make_pair(std::move(result), error); + }; + + // Register agents. + executor.WriteTx( + [&](TDiskRegistryDatabase db) { + UNIT_ASSERT_SUCCESS( + RegisterAgent(state, db, agentConfig, Now())); + }); + + UNIT_ASSERT_VALUES_EQUAL(1, state.GetConfig().KnownAgentsSize()); + UNIT_ASSERT_VALUES_EQUAL(1, state.GetAgents().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetSuspendedDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(2, state.GetDirtyDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetBrokenDevices().size()); + + // Allocate disk with dirty devices + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + auto [result, error] = allocate(db, 2U); + UNIT_ASSERT_SUCCESS(error); + UNIT_ASSERT_VALUES_EQUAL(2U, result.Devices.size()); + UNIT_ASSERT_VALUES_EQUAL(1U, result.DirtyDevices.size()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL01", + result.DirtyDevices[0].GetDeviceName()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL02", + result.Devices[0].GetDeviceName()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL01", + result.Devices[1].GetDeviceName()); + }); + } + Y_UNIT_TEST(ShouldAllocateDiskWithAdjustedBlockCount) { TTestExecutor executor; diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_pending_cleanup.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_pending_cleanup.cpp index eba9d0bd536..5ae7a1590c8 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_pending_cleanup.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_pending_cleanup.cpp @@ -209,6 +209,116 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStatePendingCleanupTest) UNIT_ASSERT_VALUES_EQUAL("vol0", diskId); }); } + + Y_UNIT_TEST(ShouldCreateLocalDiskFromDirtyDevicesAndWaitForCleanup) + { + TTestExecutor executor; + executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); }); + + constexpr ui64 LocalDeviceSize = 99999997952; // ~ 93.13 GiB + + auto makeLocalDevice = [](const auto* name, const auto* uuid) + { + return Device(name, uuid) | + WithPool("local-ssd", NProto::DEVICE_POOL_KIND_LOCAL) | + WithTotalSize(LocalDeviceSize); + }; + + const TVector agents{ + AgentConfig( + 1, + { + makeLocalDevice("NVMELOCAL01", "uuid-1"), + makeLocalDevice("NVMELOCAL02", "uuid-2"), + makeLocalDevice("NVMELOCAL03", "uuid-3"), + }), + }; + + TDiskRegistryState state = + TDiskRegistryStateBuilder() + .WithConfig( + [&] + { + auto config = MakeConfig(0, agents); + + auto* pool = config.AddDevicePoolConfigs(); + pool->SetName("local-ssd"); + pool->SetKind(NProto::DEVICE_POOL_KIND_LOCAL); + pool->SetAllocationUnit(LocalDeviceSize); + + return config; + }()) + .WithAgents(agents) + .WithDirtyDevices( + {TDirtyDevice{"uuid-1", {}}, + TDirtyDevice{"uuid-3", {}},}) + .Build(); + + auto allocate = [&](auto db, ui32 deviceCount) + { + TDiskRegistryState::TAllocateDiskResult result; + + auto error = state.AllocateDisk( + TInstant::Zero(), + db, + TDiskRegistryState::TAllocateDiskParams{ + .DiskId = "local0", + .BlockSize = DefaultLogicalBlockSize, + .BlocksCount = + deviceCount * LocalDeviceSize / DefaultLogicalBlockSize, + .MediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL}, + &result); + + return std::make_pair(std::move(result), error); + }; + + // Register agents. + executor.WriteTx( + [&](TDiskRegistryDatabase db) { + UNIT_ASSERT_SUCCESS( + RegisterAgent(state, db, agents[0], Now())); + }); + + UNIT_ASSERT_VALUES_EQUAL(1, state.GetConfig().KnownAgentsSize()); + UNIT_ASSERT_VALUES_EQUAL(1, state.GetAgents().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetSuspendedDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(2, state.GetDirtyDevices().size()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetBrokenDevices().size()); + + // Create a disk creates pending allocation with the disk + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + auto [result, error] = allocate(db, 3U); + UNIT_ASSERT_SUCCESS(error); + UNIT_ASSERT_VALUES_EQUAL(3U, result.Devices.size()); + UNIT_ASSERT_VALUES_EQUAL(2U, result.DirtyDevices.size()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL02", + result.Devices[0].GetDeviceName()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL01", + result.DirtyDevices[0].GetDeviceName()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL03", + result.DirtyDevices[1].GetDeviceName()); + }); + + // Marking devices as clean removes the disk from PendingCleanup. + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + auto [allocatingDisks, deallocatingDisks] = state.MarkDevicesAsClean(Now(), db, {"uuid-1"}); + TVector empty; + UNIT_ASSERT_VALUES_EQUAL(empty, allocatingDisks); + UNIT_ASSERT_VALUES_EQUAL(empty, deallocatingDisks); + + std::tie(allocatingDisks, deallocatingDisks) = state.MarkDevicesAsClean(Now(), db, {"uuid-3"}); + TVector expectedAllocatedDisks {"local0",}; + UNIT_ASSERT_VALUES_EQUAL(empty, deallocatingDisks); + UNIT_ASSERT_VALUES_EQUAL(expectedAllocatedDisks, allocatingDisks); + }); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_allocation.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_allocation.cpp index 4c02565fdac..3dd66bd48a0 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_allocation.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_allocation.cpp @@ -26,6 +26,39 @@ using namespace NKikimr; using namespace NDiskRegistryTest; using namespace std::chrono_literals; +namespace { + +constexpr ui64 LOCAL_DEVICE_SIZE = 99999997952; // ~ 93.13 GiB + +auto GetBackup(TDiskRegistryClient& dr) + -> NProto::TDiskRegistryStateBackup +{ + auto response = dr.BackupDiskRegistryState( + false // localDB + ); + + return response->Record.GetBackup(); +} + +auto GetDirtyDeviceCount(TDiskRegistryClient& dr) +{ + return GetBackup(dr).DirtyDevicesSize(); +} + +auto GetSuspendedDeviceCount(TDiskRegistryClient& dr) +{ + return GetBackup(dr).SuspendedDevicesSize(); +} + +auto MakeLocalDevice(const auto* name, const auto* uuid) +{ + return Device(name, uuid) | + WithPool("local-ssd", NProto::DEVICE_POOL_KIND_LOCAL) | + WithTotalSize(LOCAL_DEVICE_SIZE); +}; + +} // namespace + //////////////////////////////////////////////////////////////////////////////// Y_UNIT_TEST_SUITE(TDiskRegistryTest) @@ -1643,6 +1676,181 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) UNIT_ASSERT(tryRecvDeallocResponse() != nullptr); } + Y_UNIT_TEST(ShouldAllocateWithDirtyLocalDevices) + { + const TVector agents{ + CreateAgentConfig( + "agent-1", + { + MakeLocalDevice("NVMELOCAL01", "uuid-1"), + MakeLocalDevice("NVMELOCAL02", "uuid-2"), + MakeLocalDevice("NVMELOCAL03", "uuid-3"), + }), + }; + + auto runtime = TTestRuntimeBuilder() + .With([] { + auto config = CreateDefaultStorageConfig(); + // disable secure erase timeout + config.SetNonReplicatedSecureEraseTimeout(Max()); + config.SetNonReplicatedDontSuspendDevices(true); + + return config; + }()) + .WithAgents(agents) + .Build(); + + TDiskRegistryClient diskRegistry(*runtime); + diskRegistry.WaitReady(); + diskRegistry.SetWritableState(true); + + diskRegistry.UpdateConfig([&] { + auto config = CreateRegistryConfig(0, agents); + + auto* ssd = config.AddDevicePoolConfigs(); + ssd->SetName("local-ssd"); + ssd->SetKind(NProto::DEVICE_POOL_KIND_LOCAL); + ssd->SetAllocationUnit(LOCAL_DEVICE_SIZE); + + return config; + }()); + + RegisterAgents(*runtime, agents.size()); + WaitForAgents(*runtime, agents.size()); + + UNIT_ASSERT_VALUES_EQUAL(0, GetSuspendedDeviceCount(diskRegistry)); + UNIT_ASSERT_VALUES_EQUAL(3, GetDirtyDeviceCount(diskRegistry)); + WaitForSecureErase(*runtime, GetDirtyDeviceCount(diskRegistry)); + + // Disk allocation to mark all devices as dirty + const TString makeDirty = "make_dirty"; + const ui64 diskSize = LOCAL_DEVICE_SIZE * 3; + { + auto request = + diskRegistry.CreateAllocateDiskRequest(makeDirty, diskSize); + + request->Record.SetStorageMediaKind( + NProto::STORAGE_MEDIA_SSD_LOCAL); + + diskRegistry.SendRequest(std::move(request)); + + auto response = diskRegistry.RecvAllocateDiskResponse(); + UNIT_ASSERT(!HasError(response->GetError())); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + + auto& msg = response->Record; + SortBy(*msg.MutableDevices(), TByUUID()); + + UNIT_ASSERT_VALUES_EQUAL(3, msg.DevicesSize()); + + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL01", + msg.GetDevices(0).GetDeviceName()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL02", + msg.GetDevices(1).GetDeviceName()); + UNIT_ASSERT_VALUES_EQUAL( + "NVMELOCAL03", + msg.GetDevices(2).GetDeviceName()); + } + + // Intercept all secure erase requests to keep devices dirty + TVector> secureEraseDeviceRequests; + auto oldOfn = runtime->SetObserverFunc( + [&](TAutoPtr& event) + { + if (event->GetTypeRewrite() == + TEvDiskAgent::EvSecureEraseDeviceRequest) + { + event->DropRewrite(); + secureEraseDeviceRequests.push_back( + std::unique_ptr{event.Release()}); + + return TTestActorRuntime::EEventAction::DROP; + } + + return TTestActorRuntime::DefaultObserverFunc(event); + }); + + // Disk deallocation to mark all devices as dirty + { + diskRegistry.MarkDiskForCleanup(makeDirty); + + auto response = diskRegistry.DeallocateDisk( + makeDirty, + false // sync + ); + + UNIT_ASSERT(!HasError(response->GetError())); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + } + + // Try to get allocation response + auto tryRecvAllocResponse = [&] + { + TAutoPtr handle; + runtime->GrabEdgeEventRethrow< + TEvDiskRegistry::TEvAllocateDiskResponse>(handle, WaitTimeout); + + std::unique_ptr ptr; + + if (handle) { + ptr.reset( + handle->Release() + .Release()); + } + + return ptr; + }; + + // Allocate disk with dirty devices. We'll get a response when all + // disks' dirty devices get clean + const TString diskId = "local0"; + { + auto request = + diskRegistry.CreateAllocateDiskRequest(diskId, diskSize); + request->Record.SetStorageMediaKind( + NProto::STORAGE_MEDIA_SSD_LOCAL); + *request->Record.AddAgentIds() = agents[0].GetAgentId(); + + diskRegistry.SendRequest(std::move(request)); + + UNIT_ASSERT(tryRecvAllocResponse() == nullptr); + } + + // Secure erase all devices except one + runtime->DispatchEvents([&] { + TDispatchOptions options; + options.CustomFinalCondition = [&] { + return secureEraseDeviceRequests.size() == 3; + }; + return options; + }()); + + runtime->SetObserverFunc(oldOfn); + + UNIT_ASSERT_EQUAL(secureEraseDeviceRequests.size(), 3UL); + while (secureEraseDeviceRequests.size() != 1) { + std::cout << secureEraseDeviceRequests.back()->ToString() << std::endl; + runtime->AdvanceCurrentTime(5min); + runtime->Send(secureEraseDeviceRequests.back().release()); + + secureEraseDeviceRequests.pop_back(); + + runtime->AdvanceCurrentTime(5min); + runtime->DispatchEvents({}, 10ms); + } + + // Still no allocation + UNIT_ASSERT(tryRecvAllocResponse() == nullptr); + + // Secure erase last device + runtime->Send(secureEraseDeviceRequests.back().release()); + + // When the last dirty device of allocated disk is clean, we get allocation response + UNIT_ASSERT(tryRecvAllocResponse() != nullptr); + } + Y_UNIT_TEST(ShouldWaitSecureEraseAfterRegularDeallocation) { const TVector agents {