add scheme object: tiering rules

ydb/core/kqp/federated_query/kqp_federated_query_actors.cpp

@@ -2,6 +2,7 @@
 
 #include <ydb/services/metadata/secret/fetcher.h>
 #include <ydb/services/metadata/secret/snapshot.h>
+#include <ydb/services/metadata/service.h>
 
 
 namespace NKikimr::NKqp {

ydb/core/kqp/gateway/behaviour/resource_pool_classifier/behaviour.h

@@ -4,6 +4,7 @@
 
 #include <ydb/services/metadata/abstract/initialization.h>
 #include <ydb/services/metadata/abstract/kqp_common.h>
+#include <ydb/services/metadata/manager/abstract.h>
 
 
 namespace NKikimr::NKqp {

ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.cpp

@@ -1,6 +1,8 @@
 #include "object.h"
 #include "behaviour.h"
 
+#include <ydb/services/metadata/manager/ydb_value_operator.h>
+
 #include <library/cpp/json/json_reader.h>
 
 

ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.h

@@ -3,6 +3,7 @@
 #include <ydb/core/resource_pools/resource_pool_classifier_settings.h>
 
 #include <ydb/services/metadata/abstract/decoder.h>
+#include <ydb/services/metadata/abstract/kqp_common.h>
 #include <ydb/services/metadata/manager/object.h>
 
 

ydb/core/kqp/proxy_service/kqp_proxy_service_impl.h

@@ -11,6 +11,7 @@
 #include <ydb/core/protos/feature_flags.pb.h>
 
 #include <ydb/library/actors/core/actorid.h>
+#include <ydb/services/metadata/service.h>
 
 #include <util/datetime/base.h>
 

ydb/core/kqp/ut/olap/tiering_ut.cpp

@@ -12,25 +12,39 @@
 
 namespace NKikimr::NKqp {
 
-Y_UNIT_TEST_SUITE(KqpOlapTiering) {
-    Y_UNIT_TEST(Eviction) {
+class TTestEvictionBase {
+protected:
+    std::optional<TTestHelper> TestHelper;
+    TString TieringRule;
+
+protected:
+    virtual void UnevictAll() = 0;
+
+public:
+    void RunTest() {
         auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>();
 
         TKikimrSettings runnerSettings;
         runnerSettings.WithSampleTables = false;
-        TTestHelper testHelper(runnerSettings);
-        TLocalHelper localHelper(testHelper.GetKikimr());
-        NYdb::NTable::TTableClient tableClient = testHelper.GetKikimr().GetTableClient();
-        Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize();
+        TestHelper.emplace(runnerSettings);
+        TLocalHelper localHelper(TestHelper->GetKikimr());
+        // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
+        // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG);
+        TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_TIERING, NActors::NLog::PRI_DEBUG);
+        // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG);
+        // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NActors::NLog::PRI_DEBUG);
+        // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_PROXY, NActors::NLog::PRI_DEBUG);
+        NYdb::NTable::TTableClient tableClient = TestHelper->GetKikimr().GetTableClient();
+        Tests::NCommon::TLoggerInit(TestHelper->GetKikimr()).Initialize();
         Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->SetSecretKey("fakeSecret");
 
         localHelper.CreateTestOlapTable();
-        testHelper.CreateTier("tier1");
-        const TString tieringRule = testHelper.CreateTieringRule("tier1", "timestamp");
+        TestHelper->CreateTier("tier1");
+        TieringRule = TestHelper->CreateTieringRule("tier1", "timestamp");
 
         for (ui64 i = 0; i < 100; ++i) {
-            WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000);
-            WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000);
+            WriteTestData(TestHelper->GetKikimr(), "/Root/olapStore/olapTable", 0, 3600000000 + i * 10000, 1000);
+            WriteTestData(TestHelper->GetKikimr(), "/Root/olapStore/olapTable", 0, 3600000000 + i * 10000, 1000);
         }
 
         csController->WaitCompactions(TDuration::Seconds(5));
@@ -54,7 +68,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
             UNIT_ASSERT_GT(columnRawBytes, 0);
         }
 
-        testHelper.SetTiering("/Root/olapStore/olapTable", tieringRule);
+        TestHelper->SetTiering("/Root/olapStore/olapTable", TieringRule);
         csController->WaitActualization(TDuration::Seconds(5));
 
         {
@@ -71,10 +85,10 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
             UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1");
             UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes,
                 TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes
-                                 << " after=" << GetUint64(rows[0].at("RawBytes")));
+                                << " after=" << GetUint64(rows[0].at("RawBytes")));
         }
 
-        testHelper.ResetTiering("/Root/olapStore/olapTable");
+        UnevictAll();
         csController->WaitCompactions(TDuration::Seconds(5));
 
         {
@@ -91,8 +105,44 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
             UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT");
             UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes,
                 TStringBuilder() << "RawBytes changed after resetting tiering: before=" << columnRawBytes
-                                 << " after=" << GetUint64(rows[0].at("RawBytes")));
+                                << " after=" << GetUint64(rows[0].at("RawBytes")));
         }
+
+    }
+};
+
+class TTestEvictionResetTiering : public TTestEvictionBase {
+    private:
+    void UnevictAll() {
+        TestHelper->ResetTiering("/Root/olapStore/olapTable");
+    }
+};
+
+class TTestEvictionIncreaseDuration : public TTestEvictionBase {
+    private:
+    void UnevictAll() {
+        const TString query = R"(ALTER OBJECT )" + TieringRule + R"( (TYPE TIERING_RULE)
+            SET (description = `{
+                "rules" : [
+                    {
+                        "tierName" : "tier1",
+                        "durationForEvict" : "100000000000d"
+                    }
+                ]
+            }`))";
+        auto result = TestHelper->GetSession().ExecuteSchemeQuery(query).GetValueSync();
+        UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
+    }
+};
+
+Y_UNIT_TEST_SUITE(KqpOlapTiering) {
+
+    Y_UNIT_TEST(EvictionResetTiering) {
+        TTestEvictionResetTiering().RunTest();
+    }
+
+    Y_UNIT_TEST(EvictionIncreaseDuration) {
+        TTestEvictionIncreaseDuration().RunTest();
     }
 
     Y_UNIT_TEST(TieringRuleValidation) {
@@ -102,13 +152,24 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
         runnerSettings.WithSampleTables = false;
         TTestHelper testHelper(runnerSettings);
         TLocalHelper localHelper(testHelper.GetKikimr());
+        // testHelper.GetRuntime().SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
+        testHelper.GetRuntime().SetLogPriority(NKikimrServices::TX_TIERING, NActors::NLog::PRI_DEBUG);
+        // testHelper.GetRuntime().SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG);
+        // testHelper.GetRuntime().SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NActors::NLog::PRI_DEBUG);
+        // testHelper.GetRuntime().SetLogPriority(NKikimrServices::TX_PROXY, NActors::NLog::PRI_DEBUG);
         NYdb::NTable::TTableClient tableClient = testHelper.GetKikimr().GetTableClient();
         Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize();
         Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->SetSecretKey("fakeSecret");
 
         localHelper.CreateTestOlapTable();
         testHelper.CreateTier("tier1");
 
+        {
+            const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TIERING = "unknown_tiering";)";
+            auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
+            UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
+        }
+
         {
             const TString query = R"(
             CREATE OBJECT IF NOT EXISTS empty_tiering_rule (TYPE TIERING_RULE)
@@ -133,6 +194,18 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
             UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
         }
 
+        {
+            TString query = R"(
+            CREATE OBJECT IF NOT EXISTS wrong_default_column (TYPE TIERING_RULE)
+                WITH (defaultColumn = `unknown_column`, description = `{"rules": [{ "tierName" : "tier1", "durationForEvict" : "10d" }]}`))";
+            auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
+            UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
+
+            query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TIERING = "wrong_default_column";)";
+            result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
+            UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
+        }
+
         const TString correctTieringRule = testHelper.CreateTieringRule("tier1", "timestamp");
         {
             const TString query = "ALTER OBJECT " + correctTieringRule + R"( (TYPE TIERING_RULE) SET description `{"rules": []}`)";
@@ -157,6 +230,12 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
             auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
             UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
         }
+
+        {
+            const TString query = "DROP OBJECT tier1 (TYPE TIER)";
+            auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
+            UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
+        }
     }
 }
 

ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

@@ -5135,11 +5135,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
         TKikimrSettings runnerSettings;
         runnerSettings.WithSampleTables = false;
         runnerSettings.SetEnableTieringInColumnShard(true);
-        TKikimrRunner kikimr(runnerSettings);
-        auto db = kikimr.GetTableClient();
+        TTestHelper testHelper(runnerSettings);
+        auto db = testHelper.GetKikimr().GetTableClient();
         auto session = db.CreateSession().GetValueSync().GetSession();
         TString tableName = "/Root/ColumnTableTest";
 
+        testHelper.CreateTier("tier1");
+        testHelper.CreateTier("tier2");
+        const auto tieringRule1 = testHelper.CreateTieringRule("tier1", "Key");
+        const auto tieringRule2 = testHelper.CreateTieringRule("tier2", "Key");
+
         auto query = TStringBuilder() << R"(
             --!syntax_v1
             CREATE TABLE `)" << tableName << R"(` (
@@ -5152,7 +5157,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
             WITH (
                 STORE = COLUMN,
                 AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10,
-                TIERING = 'tiering1'
+                TIERING = ')" << tieringRule1 << R"('
             );)";
         auto result = session.ExecuteSchemeQuery(query).GetValueSync();
         UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -5168,7 +5173,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
 #endif
         auto query2 = TStringBuilder() << R"(
             --!syntax_v1
-            ALTER TABLE `)" << tableName << R"(` SET(TIERING = 'tiering2');)";
+            ALTER TABLE `)" << tableName << R"(` SET(TIERING = ')" << tieringRule2 << R"(');)";
         result = session.ExecuteSchemeQuery(query2).GetValueSync();
         UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
 
@@ -5178,7 +5183,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
 
             auto tiering = desc.GetTableDescription().GetTiering();
             UNIT_ASSERT(tiering);
-            UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering2");
+            UNIT_ASSERT_VALUES_EQUAL(*tiering, tieringRule2);
         }
 
         auto query3 = TStringBuilder() << R"(
@@ -5197,7 +5202,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
 
         auto query4 = TStringBuilder() << R"(
             --!syntax_v1
-            ALTER TABLE `)" << tableName << R"(` SET (TIERING = 'tiering1');)";
+            ALTER TABLE `)" << tableName << R"(` SET (TIERING = ')" << tieringRule1 << R"(');)";
         result = session.ExecuteSchemeQuery(query4).GetValueSync();
         UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
 
@@ -5207,7 +5212,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
 
             auto tiering = desc.GetTableDescription().GetTiering();
             UNIT_ASSERT(tiering);
-            UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering1");
+            UNIT_ASSERT_VALUES_EQUAL(*tiering, tieringRule1);
         }
 
         auto query5 = TStringBuilder() << R"(
@@ -7772,6 +7777,8 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
         TTestHelper::TColumnTable testTable;
         testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id", "id_second"}).SetSharding({"id"}).SetSchema(schema);
         testHelper.CreateTable(testTable);
+        testHelper.CreateTier("tier1");
+        const TString tieringName = testHelper.CreateTieringRule("tier1", "created_at");
 
         {
             auto alterQuery = TStringBuilder() << R"(
@@ -7813,15 +7820,15 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
             UNIT_ASSERT_VALUES_EQUAL(columns.size(), 5);
             UNIT_ASSERT_VALUES_EQUAL(description.GetTtlSettings()->GetDateTypeColumn().GetExpireAfter(), TDuration::Hours(1));
         }
-        testHelper.SetTiering("/Root/ColumnTableTest", "tiering1");
+        testHelper.SetTiering("/Root/ColumnTableTest", tieringName);
         {
             auto settings = TDescribeTableSettings().WithTableStatistics(true);
             auto describeResult = testHelper.GetSession().DescribeTable("/Root/ColumnTableTest", settings).GetValueSync();
             UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
 
             const auto& description = describeResult.GetTableDescription();
             UNIT_ASSERT(description.GetTiering());
-            UNIT_ASSERT_VALUES_EQUAL(*description.GetTiering(), "tiering1");
+            UNIT_ASSERT_VALUES_EQUAL(*description.GetTiering(), tieringName);
             UNIT_ASSERT_VALUES_EQUAL(description.GetTtlSettings()->GetDateTypeColumn().GetExpireAfter(), TDuration::Hours(1));
         }
         {
@@ -7836,7 +7843,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
 
             const auto& description = describeResult.GetTableDescription();
             UNIT_ASSERT(description.GetTiering());
-            UNIT_ASSERT_VALUES_EQUAL(*description.GetTiering(), "tiering1");
+            UNIT_ASSERT_VALUES_EQUAL(*description.GetTiering(), tieringName);
             UNIT_ASSERT(!description.GetTtlSettings());
         }
         testHelper.ResetTiering("/Root/ColumnTableTest");

ydb/core/protos/counters_schemeshard.proto

@@ -226,6 +226,12 @@ enum ESimpleCounters {
     COUNTER_IN_FLIGHT_OPS_TxCreateBackupCollection = 179       [(CounterOpts) = {Name: "InFlightOps/CreateBackupCollection"}];
     COUNTER_IN_FLIGHT_OPS_TxAlterBackupCollection = 180        [(CounterOpts) = {Name: "InFlightOps/AlterBackupCollection"}];
     COUNTER_IN_FLIGHT_OPS_TxDropBackupCollection = 181         [(CounterOpts) = {Name: "InFlightOps/DropBackupCollection"}];
+
+    COUNTER_IN_FLIGHT_OPS_TxCreateMetadataObject = 182    [(CounterOpts) = {Name: "InFlightOps/CreateMetadataObject"}];
+    COUNTER_IN_FLIGHT_OPS_TxDropMetadataObject = 183      [(CounterOpts) = {Name: "InFlightOps/DropMetadataObject"}];
+    COUNTER_IN_FLIGHT_OPS_TxAlterMetadataObject = 184     [(CounterOpts) = {Name: "InFlightOps/AlterMetadataObject"}];
+
+    COUNTER_TIERING_RULE_COUNT = 185                      [(CounterOpts) = {Name: "TieringRuleCount"}];
 }
 
 enum ECumulativeCounters {
@@ -363,6 +369,10 @@ enum ECumulativeCounters {
     COUNTER_FINISHED_OPS_TxCreateBackupCollection = 107         [(CounterOpts) = {Name: "FinishedOps/CreateBackupCollection"}];
     COUNTER_FINISHED_OPS_TxAlterBackupCollection = 108          [(CounterOpts) = {Name: "FinishedOps/AlterBackupCollection"}];
     COUNTER_FINISHED_OPS_TxDropBackupCollection = 109           [(CounterOpts) = {Name: "FinishedOps/DropBackupCollection"}];
+
+    COUNTER_FINISHED_OPS_TxCreateMetadataObject = 110     [(CounterOpts) = {Name: "FinishedOps/CreateMetadataObject"}];
+    COUNTER_FINISHED_OPS_TxDropMetadataObject = 111       [(CounterOpts) = {Name: "FinishedOps/DropMetadataObject"}];
+    COUNTER_FINISHED_OPS_TxAlterMetadataObject = 112      [(CounterOpts) = {Name: "FinishedOps/AlterMetadataObject"}];
 }
 
 enum EPercentileCounters {