From 031caf3d7dbcfe4f4de480231036285f3ab3de06 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Thu, 26 Dec 2024 14:03:05 +0800 Subject: [PATCH] [KYUUBI #6867] Refactor the grafana dashboard template ### Why are the changes needed? The original dashboard dashboard template does not work for my use case: In this PR: 1. I add the `baseFilter` variable, default value is `job=~"kyuubi"`. 2. apply the `baseFilter` for all expressions 3. add `baseLegend` for the legendFormat 4. Fix hardcode typos 5. remove the `sum` from the expressions for JVM ### How was this patch tested? image ### Was this patch authored or co-authored using generative AI tooling? No. Closes #6867 from turboFei/dash_board. Closes #6867 d0d1a3612 [Wang, Fei] comments 9fc3b5c20 [Wang, Fei] use kyuubi_jvm_uptime to get instance 280f97c59 [Wang, Fei] green f8fc35193 [Wang, Fei] all filter 8d1ac2fdf [Wang, Fei] base legend 9c18bfcb0 [Wang, Fei] save 1a4799013 [Wang, Fei] job=kyuubi ea7634580 [Wang, Fei] instant datasource and remove cluster 19db2b3e1 [Wang, Fei] add filter Authored-by: Wang, Fei Signed-off-by: Cheng Pan --- grafana/dashboard-template.json | 263 +++++++++++++++++++------------- 1 file changed, 160 insertions(+), 103 deletions(-) diff --git a/grafana/dashboard-template.json b/grafana/dashboard-template.json index de5048caac3..48be1496b14 100644 --- a/grafana/dashboard-template.json +++ b/grafana/dashboard-template.json @@ -1,9 +1,43 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.1.4" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -59,7 +93,7 @@ "mode": "absolute", "steps": [ { - "color": "dark-blue", + "color": "green", "value": null } ] @@ -99,10 +133,10 @@ "datasource": "${DS_PROMETHEUS}", "editorMode": "code", "exemplar": false, - "expr": "kyuubi_jvm_uptime{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_jvm_uptime{$baseFilter,instance=~\"$instance\"}", "format": "time_series", - "instant": true, - "legendFormat": "{{instance}}", + "instant": false, + "legendFormat": "${baseLegend}", "range": false, "refId": "A" } @@ -113,7 +147,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -191,13 +225,13 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "kyuubi_connection_total_BATCH{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_connection_total_BATCH{$baseFilter,instance=~\"$instance\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", - "legendFormat": "{{instance}}-batch", + "legendFormat": "${baseLegend}-batch", "range": true, "refId": "A", "useBackend": false @@ -205,13 +239,13 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "kyuubi_connection_total_INTERACTIVE{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_connection_total_INTERACTIVE{$baseFilter,instance=~\"$instance\"}", "hide": false, "instant": false, - "legendFormat": "{{instance}}-interactive", + "legendFormat": "${baseLegend}-interactive", "range": true, "refId": "B" } @@ -527,9 +561,9 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": " kyuubi_memory_usage_total_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": " kyuubi_memory_usage_total_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "{{instance}}", + "legendFormat": "${baseLegend}", "range": true, "refId": "B" } @@ -684,27 +718,27 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_memory_usage_pools_PS_Eden_Space_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_memory_usage_pools_PS_Eden_Space_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Eden Space Used", + "legendFormat": "${baseLegend} Eden Space Used", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Par_Eden_Space_committed)", + "expr": "kyuubi_memory_usage_pools_Par_Eden_Space_committed{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Eden Space Committed", + "legendFormat": "${baseLegend} Eden Space Committed", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Par_Eden_Space_max)", + "expr": "kyuubi_memory_usage_pools_Par_Eden_Space_max{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Eden Space Max", + "legendFormat": "${baseLegend} Eden Space Max", "range": true, "refId": "C" } @@ -846,27 +880,27 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_memory_usage_pools_PS_Old_Gen_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_memory_usage_pools_PS_Old_Gen_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Old Gen Used", + "legendFormat": "${baseLegend} Old Gen Used", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_CMS_Old_Gen_committed)", + "expr": "kyuubi_memory_usage_pools_CMS_Old_Gen_committed{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Old Gen Committed", + "legendFormat": "${baseLegend} Old Gen Committed", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_CMS_Old_Gen_max)", + "expr": "kyuubi_memory_usage_pools_CMS_Old_Gen_max{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Old Gen Max", + "legendFormat": "${baseLegend} Old Gen Max", "range": true, "refId": "C" } @@ -1008,27 +1042,27 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_memory_usage_pools_PS_Survivor_Space_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_memory_usage_pools_PS_Survivor_Space_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Survivor Space Used", + "legendFormat": "${baseLegend} Survivor Space Used", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Par_Survivor_Space_committed)", + "expr": "kyuubi_memory_usage_pools_Par_Survivor_Space_committed{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Survivor Space Committed", + "legendFormat": "${baseLegend} Survivor Space Committed", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Par_Survivor_Space_max)", + "expr": "kyuubi_memory_usage_pools_Par_Survivor_Space_max{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Survivor Space Max", + "legendFormat": "${baseLegend} Survivor Space Max", "range": true, "refId": "C" } @@ -1170,27 +1204,27 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_memory_usage_pools_Metaspace_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_memory_usage_pools_Metaspace_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Metaspace Used", + "legendFormat": "${baseLegend} Metaspace Used", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Metaspace_committed)", + "expr": "kyuubi_memory_usage_pools_Metaspace_committed{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Metaspace Committed", + "legendFormat": "${baseLegend} Metaspace Committed", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Metaspace_max)", + "expr": "kyuubi_memory_usage_pools_Metaspace_max{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Metaspace Max", + "legendFormat": "${baseLegend} Metaspace Max", "range": true, "refId": "C" } @@ -1332,27 +1366,27 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_memory_usage_pools_Compressed_Class_Space_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_memory_usage_pools_Compressed_Class_Space_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Compressed Class Used", + "legendFormat": "${baseLegend} Compressed Class Used", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Compressed_Class_Space_committed)", + "expr": "kyuubi_memory_usage_pools_Compressed_Class_Space_committed{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Compressed Class Committed", + "legendFormat": "${baseLegend} Compressed Class Committed", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Compressed_Class_Space_max)", + "expr": "kyuubi_memory_usage_pools_Compressed_Class_Space_max{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Compressed Class Max", + "legendFormat": "${baseLegend} Compressed Class Max", "range": true, "refId": "C" } @@ -1494,27 +1528,27 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_memory_usage_pools_Code_Cache_used{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_memory_usage_pools_Code_Cache_used{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Code Cache Used", + "legendFormat": "${baseLegend} Code Cache Used", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Code_Cache_committed)", + "expr": "kyuubi_memory_usage_pools_Code_Cache_committed{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Code Cache Committed", + "legendFormat": "${baseLegend} Code Cache Committed", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "sum(kyuubi_memory_usage_pools_Code_Cache_max)", + "expr": "kyuubi_memory_usage_pools_Code_Cache_max{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Code Cache Max", + "legendFormat": "${baseLegend} Code Cache Max", "range": true, "refId": "C" } @@ -1598,8 +1632,8 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_exec_pool_threads_alive{instance=~\"$instance\",job=~\"$job\"}", - "legendFormat": "{{instance}}", + "expr": "kyuubi_exec_pool_threads_alive{$baseFilter,instance=~\"$instance\"}", + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } @@ -1670,8 +1704,8 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_exec_pool_threads_active{instance=~\"$instance\",job=~\"$job\"}", - "legendFormat": "{{instance}}", + "expr": "kyuubi_exec_pool_threads_active{$baseFilter,instance=~\"$instance\"}", + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } @@ -1756,8 +1790,8 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_engine_total{instance=~\"$instance\",job=~\"$job\"}", - "legendFormat": "{{instance}}", + "expr": "kyuubi_engine_total{$baseFilter,instance=~\"$instance\"}", + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } @@ -1843,17 +1877,17 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_operation_state_ExecuteStatement_running_total{instance=~\"$instance\",job=~\"$job\"}", - "legendFormat": "Running", + "expr": "kyuubi_operation_state_ExecuteStatement_running_total{$baseFilter,instance=~\"$instance\"}", + "legendFormat": "${baseLegend} Running", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_operation_state_ExecuteStatement_finished_total{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_operation_state_ExecuteStatement_finished_total{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Finished", + "legendFormat": "${baseLegend} Finished", "range": true, "refId": "B" }, @@ -1861,11 +1895,11 @@ "datasource": "${DS_PROMETHEUS}", "editorMode": "code", "exemplar": false, - "expr": "kyuubi_operation_state_ExecuteStatement_error_total{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_operation_state_ExecuteStatement_error_total{$baseFilter,instance=~\"$instance\"}", "format": "time_series", "hide": false, "instant": false, - "legendFormat": "Error", + "legendFormat": "${baseLegend} Error", "range": true, "refId": "C" }, @@ -1873,10 +1907,10 @@ "datasource": "${DS_PROMETHEUS}", "editorMode": "code", "exemplar": false, - "expr": "kyuubi_operation_state_ExecuteStatement_pending_total{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_operation_state_ExecuteStatement_pending_total{$baseFilter,instance=~\"$instance\"}", "hide": false, "instant": false, - "legendFormat": "Pending", + "legendFormat": "${baseLegend} Pending", "range": true, "refId": "D" } @@ -1966,10 +2000,10 @@ "datasource": "${DS_PROMETHEUS}", "editorMode": "code", "exemplar": false, - "expr": "kyuubi_connection_opened{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_connection_opened{$baseFilter,instance=~\"$instance\"}", "format": "time_series", "instant": false, - "legendFormat": "{{instance}}", + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } @@ -2058,8 +2092,8 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_thrift_binary_connection_total{instance=~\"$instance\",job=~\"$job\"}", - "legendFormat": "{{instance}}", + "expr": "kyuubi_thrift_binary_connection_total{$baseFilter,instance=~\"$instance\"}", + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } @@ -2148,26 +2182,26 @@ { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_thread_state_runnable_count{instance=~\"$instance\",job=~\"$job\"}", - "legendFormat": "Runnable", + "expr": "kyuubi_thread_state_runnable_count{$baseFilter,instance=~\"$instance\"}", + "legendFormat": "${baseLegend} Runnable", "range": true, "refId": "A" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_thread_state_blocked_count{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_thread_state_blocked_count{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Blocked", + "legendFormat": "${baseLegend} Blocked", "range": true, "refId": "B" }, { "datasource": "${DS_PROMETHEUS}", "editorMode": "code", - "expr": "kyuubi_thread_state_waiting_count{instance=~\"$instance\",job=~\"$job\"}", + "expr": "kyuubi_thread_state_waiting_count{$baseFilter,instance=~\"$instance\"}", "hide": false, - "legendFormat": "Waiting", + "legendFormat": "${baseLegend} Waiting", "range": true, "refId": "C" } @@ -2183,52 +2217,75 @@ "templating": { "list": [ { - "current": {}, + "current": { + "selected": false, + "text": "job=~\"kyuubi\"", + "value": "job=~\"kyuubi\"" + }, + "description": "The base filter to apply for all metrics", + "hide": 0, "includeAll": false, - "label": "datasource", - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" + "label": "baseFilter", + "multi": false, + "name": "baseFilter", + "options": [ + { + "selected": true, + "text": "job=~\"kyuubi\"", + "value": "job=~\"kyuubi\"" + } + ], + "query": "job=~\"kyuubi\"", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" }, { - "current": {}, - "datasource": "$DS_PROMETHEUS", - "definition": "label_values(job)", + "current": { + "selected": false, + "text": "{{instance}}", + "value": "{{instance}}" + }, + "description": "The base legend format to apply for all metrics", + "hide": 0, "includeAll": false, - "label": "Cluster", - "name": "job", - "options": [], - "query": "label_values(job)", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" + "label": "baseLegend", + "multi": false, + "name": "baseLegend", + "options": [ + { + "selected": true, + "text": "{{instance}}", + "value": "{{instance}}" + } + ], + "query": "{{instance}}", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" }, { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "datasource": "$DS_PROMETHEUS", - "definition": "label_values(kyuubi_memory_usage_total_init,instance)", + "definition": "label_values(kyuubi_jvm_uptime, instance)", + "hide": 0, "includeAll": true, "label": "instance", + "mapping": "", + "mappingOnLegend": true, "multi": true, "name": "instance", "options": [], "query": { - "query": "label_values(kyuubi_memory_usage_total_init,instance)", + "query": "label_values(kyuubi_jvm_uptime, instance)", "refId": "StandardVariableQuery" }, - "refresh": 2, + "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "type": "query" }