From 44cdd431091e0066c72f5150a5663b0e4db27066 Mon Sep 17 00:00:00 2001 From: Mathieu Durand <1435391+matdurand@users.noreply.github.com> Date: Wed, 13 Sep 2023 22:23:21 -0400 Subject: [PATCH 1/2] feat: allow for custom labels on prometheus metrics --- components/metrics/builder.go | 28 ++++++++++++++++++++++++---- components/metrics/handler.go | 10 ++++++++-- components/metrics/labels.go | 14 ++++++++++++++ components/metrics/publisher.go | 4 ++++ components/metrics/subscriber.go | 4 ++++ 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/components/metrics/builder.go b/components/metrics/builder.go index c70e4755a..1639cd899 100644 --- a/components/metrics/builder.go +++ b/components/metrics/builder.go @@ -7,12 +7,28 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func NewPrometheusMetricsBuilder(prometheusRegistry prometheus.Registerer, namespace string, subsystem string) PrometheusMetricsBuilder { - return PrometheusMetricsBuilder{ +type Option func(b *PrometheusMetricsBuilder) + +// CustomLabel allow to provide a custom metric label and a function to compute the value +func CustomLabel(label string, fn LabelComputeFn) Option { + return func(b *PrometheusMetricsBuilder) { + b.customLabels = append(b.customLabels, metricLabel{ + label: label, + computeFn: fn, + }) + } +} + +func NewPrometheusMetricsBuilder(prometheusRegistry prometheus.Registerer, namespace string, subsystem string, opts ...Option) PrometheusMetricsBuilder { + builder := PrometheusMetricsBuilder{ Namespace: namespace, Subsystem: subsystem, PrometheusRegistry: prometheusRegistry, } + for _, opt := range opts { + opt(&builder) + } + return builder } // PrometheusMetricsBuilder provides methods to decorate publishers, subscribers and handlers. @@ -22,6 +38,8 @@ type PrometheusMetricsBuilder struct { Namespace string Subsystem string + + customLabels []metricLabel } // AddPrometheusRouterMetrics is a convenience function that acts on the message router to add the metrics middleware @@ -38,6 +56,7 @@ func (b PrometheusMetricsBuilder) DecoratePublisher(pub message.Publisher) (mess d := PublisherPrometheusMetricsDecorator{ pub: pub, publisherName: internal.StructName(pub), + customLabels: b.customLabels, } d.publishTimeSeconds, err = b.registerHistogramVec(prometheus.NewHistogramVec( @@ -47,7 +66,7 @@ func (b PrometheusMetricsBuilder) DecoratePublisher(pub message.Publisher) (mess Name: "publish_time_seconds", Help: "The time that a publishing attempt (success or not) took in seconds", }, - publisherLabelKeys, + appendCustomLabels(publisherLabelKeys, b.customLabels), )) if err != nil { return nil, errors.Wrap(err, "could not register publish time metric") @@ -61,6 +80,7 @@ func (b PrometheusMetricsBuilder) DecorateSubscriber(sub message.Subscriber) (me d := &SubscriberPrometheusMetricsDecorator{ closing: make(chan struct{}), subscriberName: internal.StructName(sub), + customLabels: b.customLabels, } d.subscriberMessagesReceivedTotal, err = b.registerCounterVec(prometheus.NewCounterVec( @@ -70,7 +90,7 @@ func (b PrometheusMetricsBuilder) DecorateSubscriber(sub message.Subscriber) (me Name: "subscriber_messages_received_total", Help: "The total number of messages received by the subscriber", }, - append(subscriberLabelKeys, labelAcked), + appendCustomLabels(append(subscriberLabelKeys, labelAcked), b.customLabels), )) if err != nil { return nil, errors.Wrap(err, "could not register time to ack metric") diff --git a/components/metrics/handler.go b/components/metrics/handler.go index b9240426f..a487e5add 100644 --- a/components/metrics/handler.go +++ b/components/metrics/handler.go @@ -35,6 +35,7 @@ var ( // HandlerPrometheusMetricsMiddleware is a middleware that captures Prometheus metrics. type HandlerPrometheusMetricsMiddleware struct { handlerExecutionTimeSeconds *prometheus.HistogramVec + customLabels []metricLabel } // Middleware returns the middleware ready to be used with watermill's Router. @@ -45,6 +46,9 @@ func (m HandlerPrometheusMetricsMiddleware) Middleware(h message.HandlerFunc) me labels := prometheus.Labels{ labelKeyHandlerName: message.HandlerNameFromCtx(ctx), } + for _, customLabel := range m.customLabels { + labels[customLabel.label] = customLabel.computeFn(ctx) + } defer func() { if err != nil { @@ -62,7 +66,9 @@ func (m HandlerPrometheusMetricsMiddleware) Middleware(h message.HandlerFunc) me // NewRouterMiddleware returns new middleware. func (b PrometheusMetricsBuilder) NewRouterMiddleware() HandlerPrometheusMetricsMiddleware { var err error - m := HandlerPrometheusMetricsMiddleware{} + m := HandlerPrometheusMetricsMiddleware{ + customLabels: b.customLabels, + } m.handlerExecutionTimeSeconds, err = b.registerHistogramVec(prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -72,7 +78,7 @@ func (b PrometheusMetricsBuilder) NewRouterMiddleware() HandlerPrometheusMetrics Help: "The total time elapsed while executing the handler function in seconds", Buckets: handlerExecutionTimeBuckets, }, - handlerLabelKeys, + appendCustomLabels(handlerLabelKeys, b.customLabels), )) if err != nil { panic(errors.Wrap(err, "could not register handler execution time metric")) diff --git a/components/metrics/labels.go b/components/metrics/labels.go index 6b928c9c0..ac10b77f3 100644 --- a/components/metrics/labels.go +++ b/components/metrics/labels.go @@ -46,3 +46,17 @@ func labelsFromCtx(ctx context.Context, labels ...string) prometheus.Labels { return ctxLabels } + +type LabelComputeFn func(msgCtx context.Context) string + +type metricLabel struct { + label string + computeFn LabelComputeFn +} + +func appendCustomLabels(labels []string, customs []metricLabel) []string { + for _, label := range customs { + labels = append(labels, label.label) + } + return labels +} diff --git a/components/metrics/publisher.go b/components/metrics/publisher.go index 2110429b0..acd23569f 100644 --- a/components/metrics/publisher.go +++ b/components/metrics/publisher.go @@ -20,6 +20,7 @@ type PublisherPrometheusMetricsDecorator struct { pub message.Publisher publisherName string publishTimeSeconds *prometheus.HistogramVec + customLabels []metricLabel } // Publish updates the relevant publisher metrics and calls the wrapped publisher's Publish. @@ -37,6 +38,9 @@ func (m PublisherPrometheusMetricsDecorator) Publish(topic string, messages ...* if labels[labelKeyHandlerName] == "" { labels[labelKeyHandlerName] = labelValueNoHandler } + for _, customLabel := range m.customLabels { + labels[customLabel.label] = customLabel.computeFn(ctx) + } start := time.Now() defer func() { diff --git a/components/metrics/subscriber.go b/components/metrics/subscriber.go index c439a53a4..8ce4d9418 100644 --- a/components/metrics/subscriber.go +++ b/components/metrics/subscriber.go @@ -18,6 +18,7 @@ type SubscriberPrometheusMetricsDecorator struct { subscriberName string subscriberMessagesReceivedTotal *prometheus.CounterVec closing chan struct{} + customLabels []metricLabel } func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message) { @@ -33,6 +34,9 @@ func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message if labels[labelKeyHandlerName] == "" { labels[labelKeyHandlerName] = labelValueNoHandler } + for _, customLabel := range s.customLabels { + labels[customLabel.label] = customLabel.computeFn(ctx) + } go func() { if subscribeAlreadyObserved(ctx) { From cb73514fb90420180f25f14807ca037ee9b6c9c5 Mon Sep 17 00:00:00 2001 From: Mathieu Durand <1435391+matdurand@users.noreply.github.com> Date: Wed, 20 Sep 2023 20:27:22 -0400 Subject: [PATCH 2/2] fix: move to a config struct pattern --- components/metrics/builder.go | 49 ++++++++++++++++---------------- components/metrics/handler.go | 10 +++---- components/metrics/labels.go | 23 +++++++++++---- components/metrics/publisher.go | 6 ++-- components/metrics/subscriber.go | 6 ++-- 5 files changed, 53 insertions(+), 41 deletions(-) diff --git a/components/metrics/builder.go b/components/metrics/builder.go index 1639cd899..32a10d38a 100644 --- a/components/metrics/builder.go +++ b/components/metrics/builder.go @@ -7,30 +7,29 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -type Option func(b *PrometheusMetricsBuilder) - -// CustomLabel allow to provide a custom metric label and a function to compute the value -func CustomLabel(label string, fn LabelComputeFn) Option { - return func(b *PrometheusMetricsBuilder) { - b.customLabels = append(b.customLabels, metricLabel{ - label: label, - computeFn: fn, - }) - } +type PrometheusMetricsBuilderConfig struct { + Namespace string + Subsystem string + AdditionalLabels []MetricLabel } -func NewPrometheusMetricsBuilder(prometheusRegistry prometheus.Registerer, namespace string, subsystem string, opts ...Option) PrometheusMetricsBuilder { +func NewPrometheusMetricsBuilderWithConfig(prometheusRegistry prometheus.Registerer, config PrometheusMetricsBuilderConfig) PrometheusMetricsBuilder { builder := PrometheusMetricsBuilder{ - Namespace: namespace, - Subsystem: subsystem, + Namespace: config.Namespace, + Subsystem: config.Subsystem, PrometheusRegistry: prometheusRegistry, - } - for _, opt := range opts { - opt(&builder) + additionalLabels: config.AdditionalLabels, } return builder } +func NewPrometheusMetricsBuilder(prometheusRegistry prometheus.Registerer, namespace string, subsystem string) PrometheusMetricsBuilder { + return NewPrometheusMetricsBuilderWithConfig(prometheusRegistry, PrometheusMetricsBuilderConfig{ + Namespace: namespace, + Subsystem: subsystem, + }) +} + // PrometheusMetricsBuilder provides methods to decorate publishers, subscribers and handlers. type PrometheusMetricsBuilder struct { // PrometheusRegistry may be filled with a pre-existing Prometheus registry, or left empty for the default registry. @@ -39,7 +38,7 @@ type PrometheusMetricsBuilder struct { Namespace string Subsystem string - customLabels []metricLabel + additionalLabels []MetricLabel } // AddPrometheusRouterMetrics is a convenience function that acts on the message router to add the metrics middleware @@ -54,9 +53,9 @@ func (b PrometheusMetricsBuilder) AddPrometheusRouterMetrics(r *message.Router) func (b PrometheusMetricsBuilder) DecoratePublisher(pub message.Publisher) (message.Publisher, error) { var err error d := PublisherPrometheusMetricsDecorator{ - pub: pub, - publisherName: internal.StructName(pub), - customLabels: b.customLabels, + pub: pub, + publisherName: internal.StructName(pub), + additionalLabels: b.additionalLabels, } d.publishTimeSeconds, err = b.registerHistogramVec(prometheus.NewHistogramVec( @@ -66,7 +65,7 @@ func (b PrometheusMetricsBuilder) DecoratePublisher(pub message.Publisher) (mess Name: "publish_time_seconds", Help: "The time that a publishing attempt (success or not) took in seconds", }, - appendCustomLabels(publisherLabelKeys, b.customLabels), + toLabelsSlice(publisherLabelKeys, b.additionalLabels), )) if err != nil { return nil, errors.Wrap(err, "could not register publish time metric") @@ -78,9 +77,9 @@ func (b PrometheusMetricsBuilder) DecoratePublisher(pub message.Publisher) (mess func (b PrometheusMetricsBuilder) DecorateSubscriber(sub message.Subscriber) (message.Subscriber, error) { var err error d := &SubscriberPrometheusMetricsDecorator{ - closing: make(chan struct{}), - subscriberName: internal.StructName(sub), - customLabels: b.customLabels, + closing: make(chan struct{}), + subscriberName: internal.StructName(sub), + additionalLabels: b.additionalLabels, } d.subscriberMessagesReceivedTotal, err = b.registerCounterVec(prometheus.NewCounterVec( @@ -90,7 +89,7 @@ func (b PrometheusMetricsBuilder) DecorateSubscriber(sub message.Subscriber) (me Name: "subscriber_messages_received_total", Help: "The total number of messages received by the subscriber", }, - appendCustomLabels(append(subscriberLabelKeys, labelAcked), b.customLabels), + toLabelsSlice(append(subscriberLabelKeys, labelAcked), b.additionalLabels), )) if err != nil { return nil, errors.Wrap(err, "could not register time to ack metric") diff --git a/components/metrics/handler.go b/components/metrics/handler.go index a487e5add..aa983a6e9 100644 --- a/components/metrics/handler.go +++ b/components/metrics/handler.go @@ -35,7 +35,7 @@ var ( // HandlerPrometheusMetricsMiddleware is a middleware that captures Prometheus metrics. type HandlerPrometheusMetricsMiddleware struct { handlerExecutionTimeSeconds *prometheus.HistogramVec - customLabels []metricLabel + additionalLabels []MetricLabel } // Middleware returns the middleware ready to be used with watermill's Router. @@ -46,8 +46,8 @@ func (m HandlerPrometheusMetricsMiddleware) Middleware(h message.HandlerFunc) me labels := prometheus.Labels{ labelKeyHandlerName: message.HandlerNameFromCtx(ctx), } - for _, customLabel := range m.customLabels { - labels[customLabel.label] = customLabel.computeFn(ctx) + for _, lb := range m.additionalLabels { + labels[lb.Label] = lb.ComputeFn(ctx) } defer func() { @@ -67,7 +67,7 @@ func (m HandlerPrometheusMetricsMiddleware) Middleware(h message.HandlerFunc) me func (b PrometheusMetricsBuilder) NewRouterMiddleware() HandlerPrometheusMetricsMiddleware { var err error m := HandlerPrometheusMetricsMiddleware{ - customLabels: b.customLabels, + additionalLabels: b.additionalLabels, } m.handlerExecutionTimeSeconds, err = b.registerHistogramVec(prometheus.NewHistogramVec( @@ -78,7 +78,7 @@ func (b PrometheusMetricsBuilder) NewRouterMiddleware() HandlerPrometheusMetrics Help: "The total time elapsed while executing the handler function in seconds", Buckets: handlerExecutionTimeBuckets, }, - appendCustomLabels(handlerLabelKeys, b.customLabels), + toLabelsSlice(handlerLabelKeys, b.additionalLabels), )) if err != nil { panic(errors.Wrap(err, "could not register handler execution time metric")) diff --git a/components/metrics/labels.go b/components/metrics/labels.go index ac10b77f3..f5acf7892 100644 --- a/components/metrics/labels.go +++ b/components/metrics/labels.go @@ -49,14 +49,27 @@ func labelsFromCtx(ctx context.Context, labels ...string) prometheus.Labels { type LabelComputeFn func(msgCtx context.Context) string -type metricLabel struct { - label string - computeFn LabelComputeFn +type MetricLabel struct { + Label string + ComputeFn LabelComputeFn } -func appendCustomLabels(labels []string, customs []metricLabel) []string { +func toLabelsSlice(baseLabels []string, customs []MetricLabel) []string { + labels := make([]string, len(baseLabels), len(baseLabels)+len(customs)) + copy(labels, baseLabels) for _, label := range customs { - labels = append(labels, label.label) + //Check if the additional label is already in the base labels. We cannot have duplicate labels + //If it's in the base, just skip it as the compute function is going to overwrite the default value + contains := false + for _, baseLabel := range baseLabels { + if baseLabel == label.Label { + contains = true + break + } + } + if !contains { + labels = append(labels, label.Label) + } } return labels } diff --git a/components/metrics/publisher.go b/components/metrics/publisher.go index acd23569f..55f433bf4 100644 --- a/components/metrics/publisher.go +++ b/components/metrics/publisher.go @@ -20,7 +20,7 @@ type PublisherPrometheusMetricsDecorator struct { pub message.Publisher publisherName string publishTimeSeconds *prometheus.HistogramVec - customLabels []metricLabel + additionalLabels []MetricLabel } // Publish updates the relevant publisher metrics and calls the wrapped publisher's Publish. @@ -38,8 +38,8 @@ func (m PublisherPrometheusMetricsDecorator) Publish(topic string, messages ...* if labels[labelKeyHandlerName] == "" { labels[labelKeyHandlerName] = labelValueNoHandler } - for _, customLabel := range m.customLabels { - labels[customLabel.label] = customLabel.computeFn(ctx) + for _, lb := range m.additionalLabels { + labels[lb.Label] = lb.ComputeFn(ctx) } start := time.Now() diff --git a/components/metrics/subscriber.go b/components/metrics/subscriber.go index 8ce4d9418..bb1569432 100644 --- a/components/metrics/subscriber.go +++ b/components/metrics/subscriber.go @@ -18,7 +18,7 @@ type SubscriberPrometheusMetricsDecorator struct { subscriberName string subscriberMessagesReceivedTotal *prometheus.CounterVec closing chan struct{} - customLabels []metricLabel + additionalLabels []MetricLabel } func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message) { @@ -34,8 +34,8 @@ func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message if labels[labelKeyHandlerName] == "" { labels[labelKeyHandlerName] = labelValueNoHandler } - for _, customLabel := range s.customLabels { - labels[customLabel.label] = customLabel.computeFn(ctx) + for _, lb := range s.additionalLabels { + labels[lb.Label] = lb.ComputeFn(ctx) } go func() {