From 4a7cbcc87d7c1c30f76a5a3bd456a5fce04d8ca3 Mon Sep 17 00:00:00 2001 From: v-zhuravlev Date: Thu, 31 Oct 2024 18:29:26 +0800 Subject: [PATCH 1/2] jvm-observ-lib: Add java_micrometer_with_suffixes option There is yet another metric pattern combinatino. When micrometer metrics are received through OTEL bridge then prometheus suffixes might not be added. Adding this option to jvm/process lib --- jvm-observ-lib/README.md | 5 +- jvm-observ-lib/main.libsonnet | 3 +- jvm-observ-lib/signals/buffers.libsonnet | 17 ++++- jvm-observ-lib/signals/classes.libsonnet | 6 +- jvm-observ-lib/signals/gc.libsonnet | 70 +++++++++++++++++++- jvm-observ-lib/signals/hikari.libsonnet | 30 +++++++++ jvm-observ-lib/signals/logback.libsonnet | 14 ++++ jvm-observ-lib/signals/memory.libsonnet | 18 ++++- jvm-observ-lib/signals/threads.libsonnet | 16 ++++- process-observ-lib/README.md | 25 +++++-- process-observ-lib/config.libsonnet | 2 +- process-observ-lib/signals/process.libsonnet | 16 ++++- process-observ-lib/signals/system.libsonnet | 6 +- 13 files changed, 210 insertions(+), 18 deletions(-) diff --git a/jvm-observ-lib/README.md b/jvm-observ-lib/README.md index 4dbb6c4ae..2d07da1c3 100644 --- a/jvm-observ-lib/README.md +++ b/jvm-observ-lib/README.md @@ -7,7 +7,8 @@ Supports the following sources: - `prometheus` (https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics). This also works for jmx_exporter (javaagent mode) starting from 1.0.1 release. - `otel` (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/jvm.md) - `otel_with_suffixes` same as otel with add_metric_suffixes=true in otelcollector -- `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java) +- `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java). Can be seen when Micrometer to OTEL bridge is used. +- `java_micrometer_with_suffixes` (springboot) same, but with prometheus suffixes - `prometheus_old` client_java instrumentation prior to 1.0.0 release: (https://github.com/prometheus/client_java/releases/tag/v1.0.0-alpha-4). This also works for jmx_exporter (javaagent mode) prior to 1.0.1 release. - `jmx_exporter`. Works with jmx_exporter (both http and javaagent modes) and the folllowing snippet: @@ -45,7 +46,7 @@ local jvm = uid: 'jvm-sample', dashboardNamePrefix: 'JVM', dashboardTags: ['java', 'jvm'], - metricsSource: 'java_micrometer', // or java_otel, prometheus, + metricsSource: 'java_micrometer_with_suffixes', // or java_otel, prometheus, } ); jvm.asMonitoringMixin() diff --git a/jvm-observ-lib/main.libsonnet b/jvm-observ-lib/main.libsonnet index a6e29366f..f98df5d80 100644 --- a/jvm-observ-lib/main.libsonnet +++ b/jvm-observ-lib/main.libsonnet @@ -15,7 +15,7 @@ local processlib = import 'process-observ-lib/main.libsonnet'; filteringSelector: this.config.filteringSelector, groupLabels: this.config.groupLabels, instanceLabels: this.config.instanceLabels, - uid: this.config.uid, + uid: this.config.uid - '-jvm', dashboardNamePrefix: this.config.dashboardNamePrefix, dashboardTags: this.config.dashboardTags, metricsSource: @@ -25,6 +25,7 @@ local processlib = import 'process-observ-lib/main.libsonnet'; + (if std.member(this.config.metricsSource, 'prometheus') then ['prometheus'] else []) + (if std.member(this.config.metricsSource, 'jmx_exporter') then ['jmx_exporter'] else []) + (if std.member(this.config.metricsSource, 'prometheus_old') then ['prometheus'] else []) + + (if std.member(this.config.metricsSource, 'java_micrometer_with_suffixes') then ['java_micrometer_with_suffixes'] else []) + (if std.member(this.config.metricsSource, 'java_micrometer') then ['java_micrometer'] else []), } ), diff --git a/jvm-observ-lib/signals/buffers.libsonnet b/jvm-observ-lib/signals/buffers.libsonnet index fd78fab45..a81536b99 100644 --- a/jvm-observ-lib/signals/buffers.libsonnet +++ b/jvm-observ-lib/signals/buffers.libsonnet @@ -8,10 +8,11 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_buffer_memory_used_bytes', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java - prometheus: 'jvm_buffer_pool_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-buffer-pool-metrics + java_micrometer: 'jvm_buffer_memory_used', // can be seen when otel micrometer bridge is used + java_micrometer_with_suffixes: 'jvm_buffer_memory_used_bytes', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java otel: 'process_runtime_jvm_buffer_usage', otel_with_suffixes: 'process_runtime_jvm_buffer_usage_bytes', + prometheus: 'jvm_buffer_pool_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-buffer-pool-metrics prometheus_old: 'jvm_buffer_pool_used_bytes', }, signals: { @@ -23,6 +24,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_memory_used{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_memory_used_bytes{%(queriesSelector)s}', }, prometheus: { @@ -47,6 +51,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_total_capacity{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_total_capacity_bytes{%(queriesSelector)s}', }, prometheus: { @@ -71,6 +78,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_memory_used{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_memory_used_bytes{%(queriesSelector)s}', }, prometheus: { @@ -95,6 +105,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_total_capacity{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_total_capacity_bytes{%(queriesSelector)s}', }, prometheus: { diff --git a/jvm-observ-lib/signals/classes.libsonnet b/jvm-observ-lib/signals/classes.libsonnet index ddbc809d2..42b398d82 100644 --- a/jvm-observ-lib/signals/classes.libsonnet +++ b/jvm-observ-lib/signals/classes.libsonnet @@ -8,7 +8,8 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_classes_loaded_classes', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java + java_micrometer: 'jvm_classes_loaded', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java + java_micrometer_with_suffixes: 'jvm_classes_loaded_classes', prometheus: 'jvm_classes_loaded', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-class-loading-metrics otel: 'process_runtime_jvm_classes_loaded', otel_with_suffixes: 'process_runtime_jvm_classes_loaded_total', @@ -23,6 +24,9 @@ function(this) unit: 'short', sources: { java_micrometer: { + expr: 'jvm_classes_loaded{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_classes_loaded_classes{%(queriesSelector)s}', }, prometheus: { diff --git a/jvm-observ-lib/signals/gc.libsonnet b/jvm-observ-lib/signals/gc.libsonnet index dde4d70ad..d1f0ddfd5 100644 --- a/jvm-observ-lib/signals/gc.libsonnet +++ b/jvm-observ-lib/signals/gc.libsonnet @@ -6,7 +6,8 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_memory_used_bytes', + java_micrometer: 'jvm_memory_used', + java_micrometer_with_suffixes: 'jvm_memory_used_bytes', prometheus: 'jvm_memory_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics prometheus_old: 'jvm_memory_bytes_used', otel: 'process_runtime_jvm_memory_usage', //https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ @@ -28,6 +29,14 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_gc_pause_count{%(queriesSelector)s}', + aggKeepLabels: ['action', 'cause'], + legendCustomTemplate: '%(aggLegend)s', + exprWrappers: [ + actionLabelPrettify, + ], + }, + java_micrometer_with_suffixes: { expr: 'jvm_gc_pause_seconds_count{%(queriesSelector)s}', aggKeepLabels: ['action', 'cause'], legendCustomTemplate: '%(aggLegend)s', @@ -63,6 +72,13 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'avg by (action,cause, %(agg)s) (rate(jvm_gc_pause_max{%(queriesSelector)s}[$__rate_interval]))', + legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (max)', + exprWrappers: [ + actionLabelPrettify, + ], + }, + java_micrometer_with_suffixes: { expr: 'avg by (action,cause, %(agg)s) (rate(jvm_gc_pause_seconds_max{%(queriesSelector)s}[$__rate_interval]))', legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (max)', exprWrappers: [ @@ -79,6 +95,19 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + avg by (%(agg)s, action, cause) ( + rate(jvm_gc_pause_sum{%(queriesSelector)s}[$__rate_interval]) + / + rate(jvm_gc_pause_count{%(queriesSelector)s}[$__rate_interval]) + ) + |||, + legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (avg)', + exprWrappers: [ + actionLabelPrettify, + ], + }, + java_micrometer_with_suffixes: { expr: ||| avg by (%(agg)s, action, cause) ( rate(jvm_gc_pause_seconds_sum{%(queriesSelector)s}[$__rate_interval]) @@ -138,6 +167,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_gc_memory_promoted{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_gc_memory_promoted_bytes_total{%(queriesSelector)s}', }, }, @@ -151,6 +183,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_gc_memory_allocated{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_gc_memory_allocated_bytes_total{%(queriesSelector)s}', }, prometheus: { @@ -192,6 +227,9 @@ function(this) java_micrometer: { expr: 'jvm_memory_used_bytes{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', }, + java_micrometer_with_suffixes: { + expr: 'jvm_memory_used{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', + }, prometheus: { expr: 'jvm_memory_pool_used_bytes{pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, @@ -223,6 +261,12 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_max{id=~"(G1 )?Eden Space", area="heap", %(queriesSelector)s}', + exprWrappers: [ + ['', ' != -1'], + ], + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_max_bytes{id=~"(G1 )?Eden Space", area="heap", %(queriesSelector)s}', exprWrappers: [ ['', ' != -1'], @@ -255,6 +299,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_committed{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_committed_bytes{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -283,6 +330,9 @@ function(this) sources: { //spring java_micrometer: { + expr: 'jvm_memory_used{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_used_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -316,6 +366,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_max{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s} != -1', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_max_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s} != -1', }, prometheus: { @@ -342,6 +395,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_committed{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_committed_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -369,6 +425,9 @@ function(this) sources: { //spring java_micrometer: { + expr: 'jvm_memory_used{id="Tenured Gen", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_used_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -402,6 +461,12 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_max{id="Tenured Gen", area="heap", %(queriesSelector)s}', + exprWrappers: [ + ['', ' != -1'], + ], + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_max_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}', exprWrappers: [ ['', ' != -1'], @@ -440,6 +505,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_committed{id="Tenured Gen", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_committed_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}', }, prometheus: { diff --git a/jvm-observ-lib/signals/hikari.libsonnet b/jvm-observ-lib/signals/hikari.libsonnet index d0ec495a7..1f95ec6be 100644 --- a/jvm-observ-lib/signals/hikari.libsonnet +++ b/jvm-observ-lib/signals/hikari.libsonnet @@ -24,6 +24,7 @@ function(this) expr: 'hikaricp_connections{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, + java_micrometer_with_suffixes: self.java_micrometer, otel: { expr: 'hikaricp_connections{%(queriesSelector)s}', aggKeepLabels: ['pool'], @@ -42,6 +43,10 @@ function(this) expr: 'hikaricp_connections_timeout_total{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, + java_micrometer_with_suffixes: { + expr: 'hikaricp_connections_timeout{%(queriesSelector)s}', + aggKeepLabels: ['pool'], + }, }, }, connectionsActive: { @@ -55,6 +60,7 @@ function(this) expr: 'hikaricp_connections_active{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, + java_micrometer_with_suffixes: self.java_micrometer, otel: { expr: 'hikaricp_connections_active{%(queriesSelector)s}', aggKeepLabels: ['pool'], @@ -73,6 +79,7 @@ function(this) expr: 'hikaricp_connections_idle{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, + java_micrometer_with_suffixes: self.java_micrometer, otel: { expr: 'hikaricp_connections_idle{%(queriesSelector)s}', aggKeepLabels: ['pool'], @@ -91,6 +98,7 @@ function(this) expr: 'hikaricp_connections_pending{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, + java_micrometer_with_suffixes: self.java_micrometer, otel: { expr: 'hikaricp_connections_pending{%(queriesSelector)s}', aggKeepLabels: ['pool'], @@ -107,12 +115,20 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + rate(hikaricp_connections_creation_sum{%(queriesSelector)s}[$__rate_interval]) + /rate(hikaricp_connections_creation_count{%(queriesSelector)s}[$__rate_interval]) + |||, + aggKeepLabels: ['pool'], + }, + java_micrometer_with_suffixes: { expr: ||| rate(hikaricp_connections_creation_seconds_sum{%(queriesSelector)s}[$__rate_interval]) /rate(hikaricp_connections_creation_seconds_count{%(queriesSelector)s}[$__rate_interval]) |||, aggKeepLabels: ['pool'], }, + }, }, connectionsUsageDurationAvg: { @@ -123,6 +139,13 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + rate(hikaricp_connections_usage_sum{%(queriesSelector)s}[$__rate_interval]) + /rate(hikaricp_connections_usage_count{%(queriesSelector)s}[$__rate_interval]) + |||, + aggKeepLabels: ['pool'], + }, + java_micrometer_with_suffixes: { expr: ||| rate(hikaricp_connections_usage_seconds_sum{%(queriesSelector)s}[$__rate_interval]) /rate(hikaricp_connections_usage_seconds_count{%(queriesSelector)s}[$__rate_interval]) @@ -139,6 +162,13 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + rate(hikaricp_connections_acquire_sum{%(queriesSelector)s}[$__rate_interval]) + /rate(hikaricp_connections_acquire_count{%(queriesSelector)s}[$__rate_interval]) + |||, + aggKeepLabels: ['pool'], + }, + java_micrometer_with_suffixes: { expr: ||| rate(hikaricp_connections_acquire_seconds_sum{%(queriesSelector)s}[$__rate_interval]) /rate(hikaricp_connections_acquire_seconds_count{%(queriesSelector)s}[$__rate_interval]) diff --git a/jvm-observ-lib/signals/logback.libsonnet b/jvm-observ-lib/signals/logback.libsonnet index 30f1333aa..63ff23a4e 100644 --- a/jvm-observ-lib/signals/logback.libsonnet +++ b/jvm-observ-lib/signals/logback.libsonnet @@ -20,6 +20,14 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + logback_events{%(queriesSelector)s} + |||, + exprWrappers: [ + ['topk(10,', ')'], + ], + }, + java_micrometer_with_suffixes: { expr: ||| logback_events_total{%(queriesSelector)s} |||, @@ -37,6 +45,12 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'logback_events{level="error", %(queriesSelector)s}', + exprWrappers: [ + ['topk(10,', ')'], + ], + }, + java_micrometer_with_suffixes: { expr: 'logback_events_total{level="error", %(queriesSelector)s}', exprWrappers: [ ['topk(10,', ')'], diff --git a/jvm-observ-lib/signals/memory.libsonnet b/jvm-observ-lib/signals/memory.libsonnet index 1faf44545..052d5736d 100644 --- a/jvm-observ-lib/signals/memory.libsonnet +++ b/jvm-observ-lib/signals/memory.libsonnet @@ -6,7 +6,8 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_memory_used_bytes', + java_micrometer: 'jvm_memory_used', + java_micrometer_with_suffixes: 'jvm_memory_used_bytes', prometheus: 'jvm_memory_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics otel: 'process_runtime_jvm_memory_usage', otel_with_suffixes: 'process_runtime_jvm_memory_usage_bytes', @@ -23,6 +24,9 @@ function(this) sources: { //spring java_micrometer: { + expr: 'sum without (id) (jvm_memory_used{area="heap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_used_bytes{area="heap", %(queriesSelector)s})', }, prometheus: { @@ -58,6 +62,9 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_max{area="heap", %(queriesSelector)s} != -1)', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_max_bytes{area="heap", %(queriesSelector)s} != -1)', }, prometheus: self.java_micrometer, @@ -83,6 +90,9 @@ function(this) sources: { //spring java_micrometer: { + expr: 'sum without (id) (jvm_memory_used{area="nonheap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_used_bytes{area="nonheap", %(queriesSelector)s})', }, prometheus: self.java_micrometer, @@ -107,6 +117,9 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_max{area="nonheap", %(queriesSelector)s} != -1)', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_max_bytes{area="nonheap", %(queriesSelector)s} != -1)', }, prometheus: self.java_micrometer, @@ -133,6 +146,9 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_committed{area="heap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_committed_bytes{area="heap", %(queriesSelector)s})', }, prometheus: self.java_micrometer, diff --git a/jvm-observ-lib/signals/threads.libsonnet b/jvm-observ-lib/signals/threads.libsonnet index 99e5eb68a..71ccba637 100644 --- a/jvm-observ-lib/signals/threads.libsonnet +++ b/jvm-observ-lib/signals/threads.libsonnet @@ -8,7 +8,8 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_threads_live_threads', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmThreadMetrics.java + java_micrometer: 'jvm_threads_live', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmThreadMetrics.java + java_micrometer_with_suffixes: 'jvm_threads_live_threads', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmThreadMetrics.java prometheus: 'jvm_threads_current', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics otel: 'process_runtime_jvm_threads_count', otel_with_suffixes: self.otel, @@ -23,6 +24,9 @@ function(this) unit: 'short', sources: { java_micrometer: { + expr: 'jvm_threads_live{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_threads_live_threads{%(queriesSelector)s}', }, prometheus: { @@ -45,6 +49,9 @@ function(this) unit: 'short', sources: { java_micrometer: { + expr: 'jvm_threads_daemon{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_threads_daemon_threads{%(queriesSelector)s}', }, prometheus: { @@ -70,6 +77,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_threads_peak{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_threads_peak_threads{%(queriesSelector)s}', }, prometheus: { @@ -102,6 +112,10 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'sum by (state, %(agg)s) (jvm_threads_states{%(queriesSelector)s})', + legendCustomTemplate: '{{ state }}', + }, + java_micrometer_with_suffixes: { expr: 'sum by (state, %(agg)s) (jvm_threads_states_threads{%(queriesSelector)s})', legendCustomTemplate: '{{ state }}', }, diff --git a/process-observ-lib/README.md b/process-observ-lib/README.md index 5005d5e9d..9212c657d 100644 --- a/process-observ-lib/README.md +++ b/process-observ-lib/README.md @@ -4,12 +4,25 @@ This lib can be used to generate dashboards, rows, panels for generic OS process Supports the following sources: -- prometheus -- otel -- java_otel -- java_otel_with_suffixes (add_metric_suffixes=true in otelcollector) -- java_micrometer (springboot) -- jmx_exporter +- `prometheus`. Should also work for jmx_exporter in javaagent mode +- `otel` +- `java_otel` +- `java_micrometer` (springboot, can be seen when Micromter to OTEL bridge is used) +- `java_micrometer_with_suffixes` (springboot when scraped from prometheus endpoint) +- `jmx_exporter`. Works for both jmx_exporter modes: http and javaagent + +If you pick `jmx_exporter` option, make sure you add the following snippet to your jmx_exporter config: + +```yaml +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + - pattern: java.lang<(.+)>(\w+) + name: java_lang_$1_$4_$3_$2 + - pattern: java.lang<>(\w+) + name: java_lang_$1_$3_$2 + - pattern : java.lang +``` ## Import diff --git a/process-observ-lib/config.libsonnet b/process-observ-lib/config.libsonnet index d353e9b55..7e2ddea4e 100644 --- a/process-observ-lib/config.libsonnet +++ b/process-observ-lib/config.libsonnet @@ -18,7 +18,7 @@ dashboardTimezone: 'default', dashboardRefresh: '1m', - metricsSource: ['prometheus'], // or any combination of: prometheus, otel, otel_with_suffixes, java_otel, java_otel_with_suffixes, java_micrometer. + metricsSource: ['prometheus'], // or any combination of: prometheus, otel, otel_with_suffixes, java_otel, java_otel_with_suffixes, java_micrometer, java_micrometer_with_suffixes. signals+: { system: (import './signals/system.libsonnet')(this), diff --git a/process-observ-lib/signals/process.libsonnet b/process-observ-lib/signals/process.libsonnet index 1548a18f7..78f3c2a09 100644 --- a/process-observ-lib/signals/process.libsonnet +++ b/process-observ-lib/signals/process.libsonnet @@ -5,7 +5,8 @@ function(this) otel: 'runtime_uptime', otel_with_suffixes: 'runtime_uptime_milliseconds_total', java_otel: self.otel, // some system metrics are calculated differently for java. (see system.libsonnet) - java_micrometer: 'process_uptime_seconds', // https://docs.spring.io/spring-boot/docs/1.3.3.RELEASE/reference/html/production-ready-metrics.html + java_micrometer: 'process_uptime', // https://docs.spring.io/spring-boot/docs/1.3.3.RELEASE/reference/html/production-ready-metrics.html + java_micrometer_with_suffixes: 'process_uptime_seconds', // https://docs.spring.io/spring-boot/docs/1.3.3.RELEASE/reference/html/production-ready-metrics.html java_otel_with_suffixes: self.otel_with_suffixes, // some system metrics are calculated differently for java. (see system.libsonnet), // https://prometheus.github.io/client_java/instrumentation/jvm/#process-metrics // https://github.com/prometheus/client_golang @@ -41,6 +42,9 @@ function(this) java_otel: self.otel, java_otel_with_suffixes: self.otel_with_suffixes, java_micrometer: { + expr: 'process_uptime{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'process_uptime_seconds{%(queriesSelector)s}', }, jmx_exporter: { @@ -59,6 +63,9 @@ function(this) unit: 'dateTimeAsIso', sources: { java_micrometer: { + expr: 'process_start_time{%(queriesSelector)s} * 1000', + }, + java_micrometer_with_suffixes: { expr: 'process_start_time_seconds{%(queriesSelector)s} * 1000', }, prometheus: @@ -92,6 +99,7 @@ function(this) java_micrometer: { expr: 'process_cpu_usage{%(queriesSelector)s} * 100', }, + java_micrometer_with_suffixes: self.java_micrometer, java_otel: { expr: 'process_runtime_jvm_cpu_utilization{%(queriesSelector)s} * 100', }, @@ -162,6 +170,9 @@ function(this) java_otel_with_suffixes: self.otel, java_otel: self.otel, java_micrometer: { + expr: 'process_files_open{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'process_files_open_files{%(queriesSelector)s}', }, prometheus: { @@ -182,6 +193,9 @@ function(this) otel_with_suffixes: self.otel, java_otel_with_suffixes: self.otel, java_micrometer: { + expr: 'process_files_max{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'process_files_max_files{%(queriesSelector)s}', }, prometheus: { diff --git a/process-observ-lib/signals/system.libsonnet b/process-observ-lib/signals/system.libsonnet index 9fa8c4a63..68d118c9a 100644 --- a/process-observ-lib/signals/system.libsonnet +++ b/process-observ-lib/signals/system.libsonnet @@ -6,7 +6,8 @@ function(this) otel_with_suffixes: 'runtime_uptime_milliseconds_total', java_otel: self.otel, // some system metrics are calculated differently for java java_otel_with_suffixes: self.otel_with_suffixes, - java_micrometer: 'process_uptime_seconds', + java_micrometer: 'process_uptime', + java_micrometer_with_suffixes: 'process_uptime_seconds', // https://prometheus.github.io/client_java/instrumentation/jvm/#process-metrics // https://github.com/prometheus/client_golang prometheus: 'process_start_time_seconds', @@ -34,6 +35,7 @@ function(this) java_micrometer: { expr: 'system_load_average_1m{%(queriesSelector)s}', }, + java_micrometer_with_suffixes: self.java_micrometer, jmx_exporter: { expr: 'java_lang_operatingsystem_systemloadaverage{%(queriesSelector)s}', }, @@ -49,6 +51,7 @@ function(this) java_micrometer: { expr: 'system_cpu_usage{%(queriesSelector)s} * 100', }, + java_micrometer_with_suffixes: self.java_micrometer, java_otel: { expr: 'process_runtime_jvm_system_cpu_utilization{%(queriesSelector)s}', }, @@ -76,6 +79,7 @@ function(this) java_micrometer: { expr: 'system_cpu_count{%(queriesSelector)s}', }, + java_micrometer_with_suffixes: self.java_micrometer, }, }, }, From b4dc84d566e5a516c93c89455eaabcd3206d567b Mon Sep 17 00:00:00 2001 From: v-zhuravlev Date: Thu, 31 Oct 2024 18:44:40 +0800 Subject: [PATCH 2/2] Update memory --- jvm-observ-lib/signals/memory.libsonnet | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jvm-observ-lib/signals/memory.libsonnet b/jvm-observ-lib/signals/memory.libsonnet index 052d5736d..cab0a5f5d 100644 --- a/jvm-observ-lib/signals/memory.libsonnet +++ b/jvm-observ-lib/signals/memory.libsonnet @@ -175,6 +175,9 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_committed{area="nonheap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_committed_bytes{area="nonheap", %(queriesSelector)s})', }, prometheus: self.java_micrometer,