Merge pull request #12215 from ydb-platform/mergelibs-241202-1453

Library import 241202-1453
ydb-platform · Dec 2, 2024 · d3b1607 · d3b1607
2 parents 76b069c + c98e881
commit d3b1607
Show file tree

Hide file tree

Showing 406 changed files with 1,079 additions and 2,950 deletions.
diff --git a/build/mapping.conf.json b/build/mapping.conf.json
@@ -450,6 +450,8 @@
         "7480276291": "https://devtools-registry.s3.yandex.net/7480276291",
         "7537937986": "https://devtools-registry.s3.yandex.net/7537937986",
         "7538005601": "https://devtools-registry.s3.yandex.net/7538005601",
+        "7573694143": "https://devtools-registry.s3.yandex.net/7573694143",
+        "7573699373": "https://devtools-registry.s3.yandex.net/7573699373",
         "5486731632": "https://devtools-registry.s3.yandex.net/5486731632",
         "5514350352": "https://devtools-registry.s3.yandex.net/5514350352",
         "5514360398": "https://devtools-registry.s3.yandex.net/5514360398",
@@ -1514,6 +1516,8 @@
         "7480276291": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "7537937986": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "7538005601": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
+        "7573694143": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
+        "7573699373": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",

diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc
@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7538003967)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573697889)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7538003163)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573697448)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7538005601)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573699373)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7538002456)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573696869)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7538004884)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573698723)
 
 ENDIF()
diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc
@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7537936527)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573693387)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7537935343)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573692847)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7537937986)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573694143)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7537934380)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573692312)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7537937248)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7573693816)
 
 ENDIF()
diff --git a/contrib/libs/clang18-rt/lib/msan/msan_flags.inc b/contrib/libs/clang18-rt/lib/msan/msan_flags.inc
@@ -25,7 +25,7 @@ MSAN_FLAG(bool, poison_stack_with_zeroes, false, "")
 MSAN_FLAG(bool, poison_in_malloc, true, "")
 MSAN_FLAG(bool, poison_in_free, true, "")
 MSAN_FLAG(bool, poison_in_dtor, true, "")
-MSAN_FLAG(bool, report_umrs, false, "")
+MSAN_FLAG(bool, report_umrs, true, "")
 MSAN_FLAG(bool, wrap_signals, true, "")
 MSAN_FLAG(bool, print_stats, false, "")
 MSAN_FLAG(bool, halt_on_error, !&__msan_keep_going, "")

diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt
@@ -1,4 +1,4 @@
-# Version 2024112800, Last Updated Thu Nov 28 07:07:01 2024 UTC
+# Version 2024120100, Last Updated Sun Dec  1 07:07:01 2024 UTC
 AAA
 AARP
 ABB

diff --git a/library/cpp/yt/stockpile/stockpile.h b/library/cpp/yt/stockpile/stockpile.h
@@ -39,9 +39,13 @@ struct TStockpileOptions
 
 ////////////////////////////////////////////////////////////////////////////////
 
-void RunStockpileThread(TStockpileOptions options, std::atomic<bool>* shouldProceed);
-
-void RunDetachedStockpileThreads(TStockpileOptions options);
+class TStockpileManager
+{
+public:
+    //! Configures the background stockpile threads.
+    //! Safe to call multiple times.
+    static void Reconfigure(TStockpileOptions options);
+};
 
 ////////////////////////////////////////////////////////////////////////////////
 

diff --git a/library/cpp/yt/stockpile/stockpile_linux.cpp b/library/cpp/yt/stockpile/stockpile_linux.cpp
@@ -1,119 +1,156 @@
 #include "stockpile.h"
 
-#include "library/cpp/yt/logging/logger.h"
+#include <library/cpp/yt/threading/spin_lock.h>
+
+#include <library/cpp/yt/misc/global.h>
+
+#include <library/cpp/yt/memory/leaky_singleton.h>
+
+#include <library/cpp/yt/logging/logger.h>
 
 #include <thread>
-#include <mutex>
 
 #include <sys/mman.h>
 
 #include <util/system/thread.h>
+
 #include <string.h>
 
 namespace NYT {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-static const auto Logger = NLogging::TLogger("Stockpile");
+namespace {
 
+YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "Stockpile");
 constexpr int MADV_STOCKPILE = 0x59410004;
 
-////////////////////////////////////////////////////////////////////////////////
-
-namespace {
+} // namespace
 
-void RunWithFixedBreaks(i64 bufferSize, TDuration period)
+class TStockpileManagerImpl
 {
-    auto returnCode = -::madvise(nullptr, bufferSize, MADV_STOCKPILE);
-    YT_LOG_DEBUG_IF(returnCode, "System call \"madvise\" failed: %v", strerror(returnCode));
-    Sleep(period);
-}
+public:
+    static TStockpileManagerImpl* Get()
+    {
+        return LeakySingleton<TStockpileManagerImpl>();
+    }
 
-void RunWithCappedLoad(i64 bufferSize, TDuration period)
-{
-    auto started = GetApproximateCpuInstant();
-    auto returnCode = -::madvise(nullptr, bufferSize, MADV_STOCKPILE);
-    YT_LOG_DEBUG_IF(returnCode, "System call \"madvise\" failed: %v", strerror(returnCode));
-    auto duration = CpuDurationToDuration(GetApproximateCpuInstant() - started);
+    void Reconfigure(TStockpileOptions options)
+    {
+        auto guard = Guard(SpinLock_);
+
+        Run_.store(false);
+        for (const auto& thread : Threads_) {
+            thread->join();
+        }
+
+        Threads_.clear();
+        Run_.store(true);
 
-    if (duration < period) {
-        Sleep(period - duration);
+        Options_ = options;
+
+        for (int threadIndex = 0; threadIndex < Options_.ThreadCount; ++threadIndex) {
+            Threads_.push_back(std::make_unique<std::thread>(&TStockpileManagerImpl::ThreadMain, this));
+        }
     }
-}
 
-std::pair<i64, TDuration> RunWithBackoffs(
-    i64 adjustedBufferSize,
-    TDuration adjustedPeriod,
-    const TStockpileOptions& options,
-    i64 pageSize)
-{
-    int returnCode = -::madvise(nullptr, adjustedBufferSize, MADV_STOCKPILE);
-    YT_LOG_DEBUG_IF(returnCode, "System call \"madvise\" failed: %v", strerror(returnCode));
-
-    switch(returnCode) {
-        case 0:
-            Sleep(options.Period);
-            return {options.BufferSize, options.Period};
-
-        case ENOMEM:
-            if (adjustedBufferSize / 2 >= pageSize) {
-                // Immediately make an attempt to reclaim half as much.
-                adjustedBufferSize = adjustedBufferSize / 2;
-            } else {
-                // Unless there is not even a single reclaimable page.
-                Sleep(options.Period);
-            }
-            return {adjustedBufferSize, options.Period};
+private:
+    DECLARE_LEAKY_SINGLETON_FRIEND();
+
+    const i64 PageSize_ = sysconf(_SC_PAGESIZE);
+
+    YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_);
+    std::vector<std::unique_ptr<std::thread>> Threads_;
+    TStockpileOptions Options_;
+    std::atomic<bool> Run_ = false;
+
+    void ThreadMain()
+    {
+        TThread::SetCurrentThreadName("Stockpile");
+
+        auto bufferSize = Options_.BufferSize;
+        auto period = Options_.Period;
 
-        case EAGAIN:
-        case EINTR:
-            Sleep(adjustedPeriod);
-            return {options.BufferSize, adjustedPeriod + options.Period};
+        while (Run_.load()) {
+            switch (Options_.Strategy) {
+                case EStockpileStrategy::FixedBreaks:
+                    RunWithFixedBreaks(Options_.BufferSize, Options_.Period);
+                    break;
 
-        default:
-            Sleep(options.Period);
-            return {options.BufferSize, options.Period};
+                case EStockpileStrategy::FlooredLoad:
+                    RunWithCappedLoad(Options_.BufferSize, Options_.Period);
+                    break;
+
+                case EStockpileStrategy::ProgressiveBackoff:
+                    std::tie(bufferSize, period) = RunWithBackoffs(bufferSize, period);
+                    break;
+
+                default:
+                    YT_ABORT();
+            }
+        }
     }
-}
 
-} // namespace
+    void RunWithFixedBreaks(i64 bufferSize, TDuration period)
+    {
+        auto returnCode = -::madvise(nullptr, bufferSize, MADV_STOCKPILE);
+        YT_LOG_DEBUG_IF(returnCode != 0, "System call \"madvise\" failed: %v", strerror(returnCode));
 
-void RunStockpileThread(TStockpileOptions options, std::atomic<bool>* shouldProceed)
-{
-    TThread::SetCurrentThreadName("Stockpile");
+        Sleep(period);
+    }
 
-    const i64 pageSize = sysconf(_SC_PAGESIZE);
-    auto bufferSize = options.BufferSize;
-    auto period = options.Period;
+    void RunWithCappedLoad(i64 bufferSize, TDuration period)
+    {
+        auto started = GetApproximateCpuInstant();
 
-    while (!shouldProceed || shouldProceed->load()) {
-        switch (options.Strategy) {
-            case EStockpileStrategy::FixedBreaks:
-                RunWithFixedBreaks(options.BufferSize, options.Period);
-                break;
+        auto returnCode = -::madvise(nullptr, bufferSize, MADV_STOCKPILE);
+        YT_LOG_DEBUG_IF(returnCode != 0, "System call \"madvise\" failed: %v", strerror(returnCode));
 
-            case EStockpileStrategy::FlooredLoad:
-                RunWithCappedLoad(options.BufferSize, options.Period);
-                break;
+        auto duration = CpuDurationToDuration(GetApproximateCpuInstant() - started);
+        if (duration < period) {
+            Sleep(period - duration);
+        }
+    }
 
-            case EStockpileStrategy::ProgressiveBackoff:
-                std::tie(bufferSize, period) = RunWithBackoffs(bufferSize, period, options, pageSize);
-                break;
+    std::pair<i64, TDuration> RunWithBackoffs(
+        i64 adjustedBufferSize,
+        TDuration adjustedPeriod)
+    {
+        int returnCode = -::madvise(nullptr, adjustedBufferSize, MADV_STOCKPILE);
+        YT_LOG_DEBUG_IF(returnCode != 0, "System call \"madvise\" failed: %v", strerror(returnCode));
+
+        switch(returnCode) {
+            case 0:
+                Sleep(Options_.Period);
+                return {Options_.BufferSize, Options_.Period};
+
+            case ENOMEM:
+                if (adjustedBufferSize / 2 >= PageSize_) {
+                    // Immediately make an attempt to reclaim half as much.
+                    adjustedBufferSize = adjustedBufferSize / 2;
+                } else {
+                    // Unless there is not even a single reclaimable page.
+                    Sleep(Options_.Period);
+                }
+                return {adjustedBufferSize, Options_.Period};
+
+            case EAGAIN:
+            case EINTR:
+                Sleep(adjustedPeriod);
+                return {Options_.BufferSize, adjustedPeriod + Options_.Period};
 
             default:
-                YT_ABORT();
+                Sleep(Options_.Period);
+                return {Options_.BufferSize, Options_.Period};
         }
     }
-}
+};
+
+////////////////////////////////////////////////////////////////////////////////
 
-void RunDetachedStockpileThreads(TStockpileOptions options)
+void TStockpileManager::Reconfigure(TStockpileOptions options)
 {
-    static std::once_flag OnceFlag;
-    std::call_once(OnceFlag, [options = std::move(options)] {
-        for (int i = 0; i < options.ThreadCount; ++i) {
-            std::thread(RunStockpileThread, options, nullptr).detach();
-        }
-    });
+    TStockpileManagerImpl::Get()->Reconfigure(std::move(options));
 }
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/library/cpp/yt/stockpile/stockpile_other.cpp b/library/cpp/yt/stockpile/stockpile_other.cpp
@@ -4,10 +4,7 @@ namespace NYT {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-void RunStockpileThread(TStockpileOptions /*options*/, std::atomic<bool>* /*shouldProceed*/)
-{ }
-
-void RunDetachedStockpileThreads(TStockpileOptions /*options*/)
+void TStockpileManager::Reconfigure(TStockpileOptions /*options*/)
 { }
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/library/cpp/yt/stockpile/ya.make b/library/cpp/yt/stockpile/ya.make
@@ -8,4 +8,11 @@ ELSE()
     SRCS(stockpile_other.cpp)
 ENDIF()
 
+PEERDIR(
+    library/cpp/yt/misc
+    library/cpp/yt/threading
+    library/cpp/yt/logging
+    library/cpp/yt/memory
+)
+
 END()
diff --git a/ya b/ya
@@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg
 PLATFORM_MAP = {
     "data": {
         "darwin": {
-            "md5": "0315330907e0eea53b99727ff66e5cc7",
+            "md5": "8f73547bbb30217587e27e30be05bca6",
             "urls": [
-                f"{REGISTRY_ENDPOINT}/7537984539"
+                f"{REGISTRY_ENDPOINT}/7573718003"
             ]
         },
         "darwin-arm64": {
-            "md5": "9f26f851069215b6c7d230250e341ddc",
+            "md5": "7609c99e83652c94a8be59194f895992",
             "urls": [
-                f"{REGISTRY_ENDPOINT}/7537983776"
+                f"{REGISTRY_ENDPOINT}/7573716990"
             ]
         },
         "linux-aarch64": {
-            "md5": "fd9640906f78e85bcda2fb9c39b77933",
+            "md5": "df6d7d3ae6830d5d1383746b9dba2e21",
             "urls": [
-                f"{REGISTRY_ENDPOINT}/7537982986"
+                f"{REGISTRY_ENDPOINT}/7573715788"
             ]
         },
         "win32-clang-cl": {
-            "md5": "9734b0100024a13d731199606eabcfce",
+            "md5": "a48d5651a2c9aa7dbbf58384584af599",
             "urls": [
-                f"{REGISTRY_ENDPOINT}/7537985111"
+                f"{REGISTRY_ENDPOINT}/7573718783"
             ]
         },
         "linux": {
-            "md5": "ee84a2e88d0d892d4f451d650b90da8e",
+            "md5": "ad166d3e87a875422d6073227cf5800a",
             "urls": [
-                f"{REGISTRY_ENDPOINT}/7537985768"
+                f"{REGISTRY_ENDPOINT}/7573719525"
             ]
         }
     }

diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt
@@ -1 +1 @@
-21adcc74febab524dedf75a02d887e6f507d0b7e
+24211947581091f2f54c8e50e3427e6621dfec3d
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		21adcc74febab524dedf75a02d887e6f507d0b7e
		24211947581091f2f54c8e50e3427e6621dfec3d