From 3aee32f59a959d39a60b663d0548d670f90af201 Mon Sep 17 00:00:00 2001
From: Cai Yudong <yudong.cai@zilliz.com>
Date: Thu, 4 Jan 2024 12:06:39 +0800
Subject: [PATCH] Enable to set labels for all prometheus metrics (#325)

Signed-off-by: Yudong Cai <yudong.cai@zilliz.com>
---
 include/knowhere/prometheus_client.h | 91 +++++++++++++++++-----------
 src/common/prometheus_client.cc      | 53 +++++++++++-----
 2 files changed, 96 insertions(+), 48 deletions(-)

diff --git a/include/knowhere/prometheus_client.h b/include/knowhere/prometheus_client.h
index a9c938ac8..67f3e32ea 100644
--- a/include/knowhere/prometheus_client.h
+++ b/include/knowhere/prometheus_client.h
@@ -55,38 +55,61 @@ class PrometheusClient {
 extern const prometheus::Histogram::BucketBoundaries defaultBuckets;
 extern const std::unique_ptr<PrometheusClient> prometheusClient;
 
-#define DEFINE_PROMETHEUS_GAUGE(name, desc)                                                                  \
-    prometheus::Family<prometheus::Gauge>& name##_family =                                                   \
-        prometheus::BuildGauge().Name(#name).Help(desc).Register(knowhere::prometheusClient->GetRegistry()); \
-    prometheus::Gauge& name = name##_family.Add({});
-
-#define DEFINE_PROMETHEUS_COUNTER(name, desc)                                                                  \
-    prometheus::Family<prometheus::Counter>& name##_family =                                                   \
-        prometheus::BuildCounter().Name(#name).Help(desc).Register(knowhere::prometheusClient->GetRegistry()); \
-    prometheus::Counter& name = name##_family.Add({});
-
-#define DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(name, desc, buckets)                                            \
-    prometheus::Family<prometheus::Histogram>& name##_family =                                                   \
-        prometheus::BuildHistogram().Name(#name).Help(desc).Register(knowhere::prometheusClient->GetRegistry()); \
-    prometheus::Histogram& name = name##_family.Add({}, buckets);
-
-#define DEFINE_PROMETHEUS_HISTOGRAM(name, desc) DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(name, desc, defaultBuckets)
-
-#define DECLARE_PROMETHEUS_GAUGE(name_gauge) extern prometheus::Gauge& name_gauge;
-#define DECLARE_PROMETHEUS_COUNTER(name_counter) extern prometheus::Counter& name_counter;
-#define DECLARE_PROMETHEUS_HISTOGRAM(name_histogram) extern prometheus::Histogram& name_histogram;
-
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_build_latency);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_load_latency);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_search_latency);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_range_search_latency);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_ann_iterator_init_latency);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_search_topk);
-
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_hnsw_bitset_ratio);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_hnsw_search_hops);
-
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_diskann_bitset_ratio);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_diskann_search_hops);
-DECLARE_PROMETHEUS_HISTOGRAM(knowhere_diskann_range_search_iters);
+#define CONCATENATE(x, y) x##_##y
+#define PROMETHEUS_LABEL_KNOWHERE knowhere
+#define PROMETHEUS_LABEL_CARDINAL cardinal
+
+#define DEFINE_PROMETHEUS_GAUGE_FAMILY(name, desc)                     \
+    prometheus::Family<prometheus::Gauge>& CONCATENATE(name, family) = \
+        prometheus::BuildGauge().Name(#name).Help(desc).Register(knowhere::prometheusClient->GetRegistry());
+
+#define DEFINE_PROMETHEUS_GAUGE(name, module) \
+    prometheus::Gauge& CONCATENATE(module, name) = CONCATENATE(name, family).Add({{"module", #module}});
+
+#define DEFINE_PROMETHEUS_COUNTER_FAMILY(name, desc)                     \
+    prometheus::Family<prometheus::Counter>& CONCATENATE(name, family) = \
+        prometheus::BuildCounter().Name(#name).Help(desc).Register(knowhere::prometheusClient->GetRegistry());
+
+#define DEFINE_PROMETHEUS_COUNTER(name, module) \
+    prometheus::Counter& CONCATENATE(module, name) = CONCATENATE(name, family).Add({{"module", #module}});
+
+#define DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(name, desc)                     \
+    prometheus::Family<prometheus::Histogram>& CONCATENATE(name, family) = \
+        prometheus::BuildHistogram().Name(#name).Help(desc).Register(knowhere::prometheusClient->GetRegistry());
+
+#define DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(name, module, buckets) \
+    prometheus::Histogram& CONCATENATE(module, name) = CONCATENATE(name, family).Add({{"module", #module}}, buckets);
+
+#define DEFINE_PROMETHEUS_HISTOGRAM(name, module) DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(name, module, defaultBuckets)
+
+#define DECLARE_PROMETHEUS_GAUGE(name, module) extern prometheus::Gauge& CONCATENATE(module, name);
+#define DECLARE_PROMETHEUS_COUNTER(name, module) extern prometheus::Counter& CONCATENATE(module, name);
+#define DECLARE_PROMETHEUS_HISTOGRAM(name, module) extern prometheus::Histogram& CONCATENATE(module, name);
+
+DECLARE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(load_latency, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(load_latency, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(search_latency, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(search_latency, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(range_search_latency, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(range_search_latency, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(ann_iterator_init_latency, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(ann_iterator_init_latency, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(search_topk, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(search_topk, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(bitset_ratio, PROMETHEUS_LABEL_CARDINAL);
+
+DECLARE_PROMETHEUS_HISTOGRAM(hnsw_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(hnsw_search_hops, PROMETHEUS_LABEL_KNOWHERE);
+
+DECLARE_PROMETHEUS_HISTOGRAM(diskann_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(diskann_search_hops, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE);
 }  // namespace knowhere
diff --git a/src/common/prometheus_client.cc b/src/common/prometheus_client.cc
index ce48f165f..7772e30d1 100644
--- a/src/common/prometheus_client.cc
+++ b/src/common/prometheus_client.cc
@@ -28,23 +28,48 @@ const std::unique_ptr<PrometheusClient> prometheusClient = std::make_unique<Prom
  *   An error has occurred while serving metrics:
  *   text format parsing error in line 50: expected float as value, got "=\"0.9\"}"
  ******************************************************************************/
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_build_latency, "index build latency in knowhere (s)")
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_load_latency, "index load latency in knowhere (ms)")
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_search_latency, "search latency in knowhere (ms)")
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_range_search_latency, "range search latency in knowhere (ms)")
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_ann_iterator_init_latency, "ann iterator init latency in knowhere (ms)")
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_search_topk, "knowhere search topk")
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(build_latency, "index build latency (s)")
+DEFINE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_CARDINAL)
 
-DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(knowhere_hnsw_bitset_ratio,
-                                         "knowhere HNSW bitset ratio for search and range search", ratioBuckets)
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_hnsw_search_hops, "knowhere HNSW search hops in layer 0")
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(load_latency, "index load latency (ms)")
+DEFINE_PROMETHEUS_HISTOGRAM(load_latency, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM(load_latency, PROMETHEUS_LABEL_CARDINAL)
 
-DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(knowhere_diskann_bitset_ratio,
-                                         "knowhere DISKANN bitset ratio for search and range search", ratioBuckets)
-DEFINE_PROMETHEUS_HISTOGRAM(knowhere_diskann_search_hops, "knowhere DISKANN search hops")
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(search_latency, "search latency (ms)")
+DEFINE_PROMETHEUS_HISTOGRAM(search_latency, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM(search_latency, PROMETHEUS_LABEL_CARDINAL)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(range_search_latency, "range search latency (ms)")
+DEFINE_PROMETHEUS_HISTOGRAM(range_search_latency, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM(range_search_latency, PROMETHEUS_LABEL_CARDINAL)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(ann_iterator_init_latency, "ann iterator init latency (ms)")
+DEFINE_PROMETHEUS_HISTOGRAM(ann_iterator_init_latency, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM(ann_iterator_init_latency, PROMETHEUS_LABEL_CARDINAL)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(search_topk, "search topk")
+DEFINE_PROMETHEUS_HISTOGRAM(search_topk, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM(search_topk, PROMETHEUS_LABEL_CARDINAL)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(bitset_ratio, "bitset ratio")
+DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(bitset_ratio, PROMETHEUS_LABEL_CARDINAL, ratioBuckets)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(hnsw_bitset_ratio, "HNSW bitset ratio for search and range search")
+DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(hnsw_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE, ratioBuckets)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(hnsw_search_hops, "HNSW search hops in layer 0")
+DEFINE_PROMETHEUS_HISTOGRAM(hnsw_search_hops, PROMETHEUS_LABEL_KNOWHERE)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(diskann_bitset_ratio, "DISKANN bitset ratio for search and range search")
+DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(diskann_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE, ratioBuckets)
+
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(diskann_search_hops, "DISKANN search hops")
+DEFINE_PROMETHEUS_HISTOGRAM(diskann_search_hops, PROMETHEUS_LABEL_KNOWHERE)
 
 const prometheus::Histogram::BucketBoundaries diskannRangeSearchIterBuckets = {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22};
-DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(knowhere_diskann_range_search_iters,
-                                         "knowhere DISKANN range search iterations", diskannRangeSearchIterBuckets)
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(diskann_range_search_iters, "DISKANN range search iterations")
+DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE,
+                                         diskannRangeSearchIterBuckets)
 
 }  // namespace knowhere