Merge branch 'master' into more_rules

.github/workflows/cpp-test.yaml

@@ -14,7 +14,7 @@ on:
       - release-4.0
 jobs:
   cpp-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

.github/workflows/rust-test.yaml

@@ -16,7 +16,7 @@ on:
       - release-3.0
 jobs:
   rust-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

OWNERS

@@ -99,6 +99,7 @@ reviewers:
   - ChenPeng2013
   - dhysum
   - fzzf678
+  - ghazalfamilyusa
   - JmPotato
   - js00070
   - lamxTyler
@@ -115,6 +116,7 @@ reviewers:
   - shihongzhi
   - spongedu
   - tangwz
+  - terry1purcell
   - Tjianke
   - TonsnakeLin
   - tsthght

proto/brpb.proto

@@ -147,6 +147,15 @@ message BackupRange {
     repeated File files = 3;
 }
 
+// The message saves the physical table checksum
+message TableMeta {
+    // physical id is either table id or partition id
+    int64 physical_id = 1;
+    uint64 crc64xor = 2;
+    uint64 total_kvs = 3;
+    uint64 total_bytes = 4;
+}
+
 message File {
     string name = 1;
     bytes sha256 = 2;
@@ -156,9 +165,12 @@ message File {
     uint64 start_version = 5;
     uint64 end_version = 6;
 
+    // recorded the total statistics across multiple tables
     uint64 crc64xor = 7;
     uint64 total_kvs = 8;
     uint64 total_bytes = 9;
+    // recorded when the SST kvs are from different tables
+    repeated TableMeta table_metas = 13;
 
     string cf = 10;
 
@@ -311,6 +323,10 @@ message MasterKeyConfig {
     repeated encryptionpb.MasterKey master_keys = 2;
 }
 
+message SortedSubRanges {
+    repeated kvrpcpb.KeyRange sub_ranges = 1;
+}
+
 message BackupRequest {
     uint64 cluster_id = 1;
 
@@ -352,6 +368,10 @@ message BackupRequest {
     // only the remaining sub ranges of the range need to be backed up this time.
     repeated kvrpcpb.KeyRange sub_ranges = 16;
 
+    // only ranges in the same groups can be merged into one SST file.
+    // the ranges in each groups are sorted.
+    repeated SortedSubRanges sorted_sub_ranges_groups = 21;
+
     // replica_read indicates whether to use replica read for backup.
     // If it is false, the backup will only read data from leader.
     // If it is true, the backup will read data from both leader and follower.

proto/import_sstpb.proto

@@ -216,6 +216,7 @@ message DownloadRequest {
     // You need to ensure that the keys before and after rewriting are in the
     // same order, otherwise the RPC request will fail.
     RewriteRule rewrite_rule = 13 [(gogoproto.nullable) = false];
+    repeated RewriteRule sorted_rewrite_rules = 20;
 
     backup.StorageBackend storage_backend = 14;
     // The identity for the stroage backend.

proto/logbackuppb.proto

@@ -52,11 +52,25 @@ message FlushEvent {
     uint64 checkpoint = 3;
 }
 
+message FlushNowRequest {
+}
+
+message FlushNowResponse {
+    repeated FlushResult results = 1;
+}
+
+message FlushResult {
+    string task_name = 1;
+    bool success = 2;
+    string error_message = 3;
+}
+
 // The log backup service.
 // Generally, most essential interfaces of log backup (say, checkpoint management, task management) are
 // provided by adding some key in the embed etcd of PD.
 // This interface is mainly provided for the checkpoint advancer and debug usage.
 service LogBackup {
     rpc GetLastFlushTSOfRegion(GetLastFlushTSOfRegionRequest) returns (GetLastFlushTSOfRegionResponse) {}
     rpc SubscribeFlushEvent(SubscribeFlushEventRequest) returns (stream SubscribeFlushEventResponse) {}
+    rpc FlushNow(FlushNowRequest) returns (FlushNowResponse) {}
 }

scripts/proto.lock

@@ -475,6 +475,31 @@
               }
             ]
           },
+          {
+            "name": "TableMeta",
+            "fields": [
+              {
+                "id": 1,
+                "name": "physical_id",
+                "type": "int64"
+              },
+              {
+                "id": 2,
+                "name": "crc64xor",
+                "type": "uint64"
+              },
+              {
+                "id": 3,
+                "name": "total_kvs",
+                "type": "uint64"
+              },
+              {
+                "id": 4,
+                "name": "total_bytes",
+                "type": "uint64"
+              }
+            ]
+          },
           {
             "name": "File",
             "fields": [
@@ -523,6 +548,12 @@
                 "name": "total_bytes",
                 "type": "uint64"
               },
+              {
+                "id": 13,
+                "name": "table_metas",
+                "type": "TableMeta",
+                "is_repeated": true
+              },
               {
                 "id": 10,
                 "name": "cf",
@@ -866,6 +897,17 @@
               }
             ]
           },
+          {
+            "name": "SortedSubRanges",
+            "fields": [
+              {
+                "id": 1,
+                "name": "sub_ranges",
+                "type": "kvrpcpb.KeyRange",
+                "is_repeated": true
+              }
+            ]
+          },
           {
             "name": "BackupRequest",
             "fields": [
@@ -945,6 +987,12 @@
                 "type": "kvrpcpb.KeyRange",
                 "is_repeated": true
               },
+              {
+                "id": 21,
+                "name": "sorted_sub_ranges_groups",
+                "type": "SortedSubRanges",
+                "is_repeated": true
+              },
               {
                 "id": 17,
                 "name": "replica_read",
@@ -7683,6 +7731,12 @@
                   }
                 ]
               },
+              {
+                "id": 20,
+                "name": "sorted_rewrite_rules",
+                "type": "RewriteRule",
+                "is_repeated": true
+              },
               {
                 "id": 14,
                 "name": "storage_backend",
@@ -12742,6 +12796,40 @@
                 "type": "uint64"
               }
             ]
+          },
+          {
+            "name": "FlushNowRequest"
+          },
+          {
+            "name": "FlushNowResponse",
+            "fields": [
+              {
+                "id": 1,
+                "name": "results",
+                "type": "FlushResult",
+                "is_repeated": true
+              }
+            ]
+          },
+          {
+            "name": "FlushResult",
+            "fields": [
+              {
+                "id": 1,
+                "name": "task_name",
+                "type": "string"
+              },
+              {
+                "id": 2,
+                "name": "success",
+                "type": "bool"
+              },
+              {
+                "id": 3,
+                "name": "error_message",
+                "type": "string"
+              }
+            ]
           }
         ],
         "services": [
@@ -12758,6 +12846,11 @@
                 "in_type": "SubscribeFlushEventRequest",
                 "out_type": "SubscribeFlushEventResponse",
                 "out_streamed": true
+              },
+              {
+                "name": "FlushNow",
+                "in_type": "FlushNowRequest",
+                "out_type": "FlushNowResponse"
               }
             ]
           }