HBASE-29151 add getTime and scanTime metrics to each region to identify heavily read region

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -448,6 +448,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String DELETE_BATCH_KEY = "deleteBatch";
   String GET_SIZE_KEY = "getSize";
   String GET_KEY = "get";
+  String GET_TIME_KEY = "getTime";
   String INCREMENT_KEY = "increment";
   String PUT_KEY = "put";
   String PUT_BATCH_KEY = "putBatch";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java

@@ -73,14 +73,15 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource> {
   void updateDelete();
 
   /**
-   * Update related counts of gets
+   * Update time of gets
+   * @param mills time for this get operation.
    */
-  void updateGet();
+  void updateGet(long mills);
 
   /**
-   * Update related counts of resultScanner.next().
+   * Update time used of resultScanner.next().
    */
-  void updateScan();
+  void updateScanTime(long mills);
 
   /**
    * Update related counts of increments.

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java

@@ -52,9 +52,11 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
   private final String regionPutKey;
   private final String regionDeleteKey;
   private final String regionGetKey;
+  private final String regionGetTimeKey;
   private final String regionIncrementKey;
   private final String regionAppendKey;
   private final String regionScanKey;
+  private final String regionScanTimeKey;
 
   /*
    * Implementation note: Do not put histograms per region. With hundreds of regions in a server
@@ -66,6 +68,8 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
   private final MutableFastCounter regionAppend;
   private final MutableFastCounter regionGet;
   private final MutableFastCounter regionScan;
+  private final MutableFastCounter regionGetTime;
+  private final MutableFastCounter regionScanTime;
 
   private final int hashCode;
 
@@ -105,6 +109,12 @@ public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper,
 
     regionScanKey = regionNamePrefix + MetricsRegionServerSource.SCAN_KEY + suffix;
     regionScan = registry.getCounter(regionScanKey, 0L);
+
+    regionGetTimeKey = regionNamePrefix + MetricsRegionServerSource.GET_TIME_KEY;
+    regionGetTime = registry.getCounter(regionGetTimeKey, 0L);
+
+    regionScanTimeKey = regionNamePrefix + MetricsRegionServerSource.SCAN_TIME_KEY;
+    regionScanTime = registry.getCounter(regionScanTimeKey, 0L);
   }
 
   @Override
@@ -133,6 +143,8 @@ public void close() {
       registry.removeMetric(regionAppendKey);
       registry.removeMetric(regionGetKey);
       registry.removeMetric(regionScanKey);
+      registry.removeMetric(regionGetTimeKey);
+      registry.removeMetric(regionScanTimeKey);
 
       regionWrapper = null;
     }
@@ -149,13 +161,15 @@ public void updateDelete() {
   }
 
   @Override
-  public void updateGet() {
+  public void updateGet(long mills) {
     regionGet.incr();
+    regionGetTime.incr(mills);
   }
 
   @Override
-  public void updateScan() {
+  public void updateScanTime(long mills) {
     regionScan.incr();
+    regionScanTime.incr(mills);
   }
 
   @Override

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

@@ -7937,11 +7937,12 @@ private List<Cell> get(Get get, boolean withCoprocessor, long nonceGroup, long n
   private List<Cell> getInternal(Get get, boolean withCoprocessor, long nonceGroup, long nonce)
     throws IOException {
     List<Cell> results = new ArrayList<>();
+    long before = EnvironmentEdgeManager.currentTime();
 
     // pre-get CP hook
     if (withCoprocessor && (coprocessorHost != null)) {
       if (coprocessorHost.preGet(get, results)) {
-        metricsUpdateForGet();
+        metricsUpdateForGet(before);
         return results;
       }
     }
@@ -7965,14 +7966,14 @@ private List<Cell> getInternal(Get get, boolean withCoprocessor, long nonceGroup
       coprocessorHost.postGet(get, results);
     }
 
-    metricsUpdateForGet();
+    metricsUpdateForGet(before);
 
     return results;
   }
 
-  void metricsUpdateForGet() {
+  void metricsUpdateForGet(long before) {
     if (this.metricsRegion != null) {
-      this.metricsRegion.updateGet();
+      this.metricsRegion.updateGet(EnvironmentEdgeManager.currentTime() - before);
     }
     if (this.rsServices != null && this.rsServices.getMetrics() != null) {
       rsServices.getMetrics().updateReadQueryMeter(this, 1);

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java

@@ -50,12 +50,12 @@ public void updateDelete() {
     source.updateDelete();
   }
 
-  public void updateGet() {
-    source.updateGet();
+  public void updateGet(final long t) {
+    source.updateGet(t);
   }
 
-  public void updateScan() {
-    source.updateScan();
+  public void updateScanTime(final long t) {
+    source.updateScanTime(t);
   }
 
   public void updateFilteredRecords() {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

@@ -2554,10 +2554,11 @@ private Result get(Get get, HRegion region, RegionScannersCloseCallBack closeCal
 
     // This method is almost the same as HRegion#get.
     List<Cell> results = new ArrayList<>();
+    long before = EnvironmentEdgeManager.currentTime();
     // pre-get CP hook
     if (region.getCoprocessorHost() != null) {
       if (region.getCoprocessorHost().preGet(get, results)) {
-        region.metricsUpdateForGet();
+        region.metricsUpdateForGet(before);
         return Result.create(results, get.isCheckExistenceOnly() ? !results.isEmpty() : null,
           stale);
       }
@@ -2592,7 +2593,7 @@ private Result get(Get get, HRegion region, RegionScannersCloseCallBack closeCal
     if (region.getCoprocessorHost() != null) {
       region.getCoprocessorHost().postGet(get, results);
     }
-    region.metricsUpdateForGet();
+    region.metricsUpdateForGet(before);
 
     return Result.create(results, get.isCheckExistenceOnly() ? !results.isEmpty() : null, stale);
   }
@@ -3509,15 +3510,14 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan
       region.closeRegionOperation();
       // Update serverside metrics, even on error.
       long end = EnvironmentEdgeManager.currentTime();
-
       long responseCellSize = 0;
       long blockBytesScanned = 0;
       if (rpcCall != null) {
         responseCellSize = rpcCall.getResponseCellSize();
         blockBytesScanned = rpcCall.getBlockBytesScanned();
         rsh.updateBlockBytesScanned(blockBytesScanned);
       }
-      region.getMetrics().updateScan();
+      region.getMetrics().updateScanTime(end - before);
       final MetricsRegionServer metricsRegionServer = server.getMetrics();
       if (metricsRegionServer != null) {
         metricsRegionServer.updateScan(region, end - before, responseCellSize, blockBytesScanned);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java

@@ -39,6 +39,8 @@ public class TestMetricsRegion {
   @Test
   public void testRegionWrapperMetrics() {
     MetricsRegion mr = new MetricsRegion(new MetricsRegionWrapperStub(), new Configuration());
+    mr.updateGet(1);
+    mr.updateScanTime(2);
     MetricsRegionAggregateSource agg = mr.getSource().getAggregateSource();
 
     HELPER.assertGauge(
@@ -71,6 +73,18 @@ public void testRegionWrapperMetrics() {
     HELPER.assertCounter(
       "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_replicaid", 0,
       agg);
+    HELPER.assertCounter(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + "_metric_getCount", 1,
+      agg);
+    HELPER.assertCounter(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + "_metric_getTime", 1,
+      agg);
+    HELPER.assertCounter(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + "_metric_scanCount", 1,
+      agg);
+    HELPER.assertCounter(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + "_metric_scanTime", 2,
+      agg);
     mr.close();
 
     // test region with replica id > 0