Reimplement PrometheusMetric to use slices for label pairs

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/storagegateway v1.33.1
 	github.com/aws/aws-sdk-go-v2/service/sts v1.31.1
 	github.com/aws/smithy-go v1.21.0
+	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/go-kit/log v0.2.1
 	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
 	github.com/prometheus/client_golang v1.20.4
@@ -28,7 +29,6 @@ require (
 	github.com/r3labs/diff/v3 v3.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.4
-	golang.org/x/exp v0.0.0-20240823005443-9b4947da3948
 	golang.org/x/sync v0.8.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -43,7 +43,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.23.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect

go.sum

@@ -108,8 +108,6 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
-golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA=
-golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=

pkg/exporter.go

@@ -196,7 +196,7 @@ func UpdateMetrics(
 		return nil
 	}
 	metrics, observedMetricLabels = promutil.BuildNamespaceInfoMetrics(tagsData, metrics, observedMetricLabels, options.labelsSnakeCase, logger)
-	metrics = promutil.EnsureLabelConsistencyAndRemoveDuplicates(metrics, observedMetricLabels)
+	metrics = promutil.EnsureLabelConsistencyAndRemoveDuplicates(metrics, observedMetricLabels, logger)
 
 	registry.MustRegister(promutil.NewPrometheusCollector(metrics))
 	return nil

pkg/promutil/migrate.go

@@ -2,14 +2,13 @@ package promutil
 
 import (
 	"fmt"
-	"maps"
 	"math"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/grafana/regexp"
-	prom_model "github.com/prometheus/common/model"
 
 	"github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging"
 	"github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model"
@@ -46,30 +45,30 @@ func BuildMetricName(namespace, metricName, statistic string) string {
 
 func BuildNamespaceInfoMetrics(tagData []model.TaggedResourceResult, metrics []*PrometheusMetric, observedMetricLabels map[string]model.LabelSet, labelsSnakeCase bool, logger logging.Logger) ([]*PrometheusMetric, map[string]model.LabelSet) {
 	for _, tagResult := range tagData {
-		contextLabels := contextToLabels(tagResult.Context, labelsSnakeCase, logger)
+		contextLabelKeys, contextLabelValues := contextToLabels(tagResult.Context, labelsSnakeCase, logger)
 		for _, d := range tagResult.Data {
-			metricName := BuildMetricName(d.Namespace, "info", "")
+			size := len(d.Tags) + len(contextLabelKeys) + 1
+			promLabelKeys, promLabelValues := make([]string, 0, size), make([]string, 0, size)
+
+			promLabelKeys = append(promLabelKeys, "name")
+			promLabelKeys = append(promLabelKeys, contextLabelKeys...)
+			promLabelValues = append(promLabelValues, d.ARN)
+			promLabelValues = append(promLabelValues, contextLabelValues...)
 
-			promLabels := make(map[string]string, len(d.Tags)+len(contextLabels)+1)
-			maps.Copy(promLabels, contextLabels)
-			promLabels["name"] = d.ARN
 			for _, tag := range d.Tags {
 				ok, promTag := PromStringTag(tag.Key, labelsSnakeCase)
 				if !ok {
 					logger.Warn("tag name is an invalid prometheus label name", "tag", tag.Key)
 					continue
 				}
 
-				labelName := "tag_" + promTag
-				promLabels[labelName] = tag.Value
+				promLabelKeys = append(promLabelKeys, "tag_"+promTag)
+				promLabelValues = append(promLabelValues, tag.Value)
 			}
 
-			observedMetricLabels = recordLabelsForMetric(metricName, promLabels, observedMetricLabels)
-			metrics = append(metrics, &PrometheusMetric{
-				Name:   metricName,
-				Labels: promLabels,
-				Value:  0,
-			})
+			metricName := BuildMetricName(d.Namespace, "info", "")
+			observedMetricLabels = recordLabelsForMetric(metricName, promLabelKeys, observedMetricLabels)
+			metrics = append(metrics, NewPrometheusMetric(metricName, promLabelKeys, promLabelValues, 0))
 		}
 	}
 
@@ -81,7 +80,7 @@ func BuildMetrics(results []model.CloudwatchMetricResult, labelsSnakeCase bool,
 	observedMetricLabels := make(map[string]model.LabelSet)
 
 	for _, result := range results {
-		contextLabels := contextToLabels(result.Context, labelsSnakeCase, logger)
+		contextLabelKeys, contextLabelValues := contextToLabels(result.Context, labelsSnakeCase, logger)
 		for _, metric := range result.Data {
 			// This should not be possible but check just in case
 			if metric.GetMetricStatisticsResult == nil && metric.GetMetricDataResult == nil {
@@ -112,17 +111,17 @@ func BuildMetrics(results []model.CloudwatchMetricResult, labelsSnakeCase bool,
 
 				name := BuildMetricName(metric.Namespace, metric.MetricName, statistic)
 
-				promLabels := createPrometheusLabels(metric, labelsSnakeCase, contextLabels, logger)
-				observedMetricLabels = recordLabelsForMetric(name, promLabels, observedMetricLabels)
-
-				output = append(output, &PrometheusMetric{
-					Name:             name,
-					Labels:           promLabels,
-					Value:            exportedDatapoint,
-					Timestamp:        ts,
-					IncludeTimestamp: metric.MetricMigrationParams.AddCloudwatchTimestamp,
-				})
-
+				labelKeys, labelValues := createPrometheusLabels(metric, labelsSnakeCase, contextLabelKeys, contextLabelValues, logger)
+				observedMetricLabels = recordLabelsForMetric(name, labelKeys, observedMetricLabels)
+
+				output = append(output, NewPrometheusMetricWithTimestamp(
+					name,
+					labelKeys,
+					labelValues,
+					exportedDatapoint,
+					metric.MetricMigrationParams.AddCloudwatchTimestamp,
+					ts,
+				))
 			}
 		}
 	}
@@ -209,9 +208,12 @@ func sortByTimestamp(datapoints []*model.Datapoint) []*model.Datapoint {
 	return datapoints
 }
 
-func createPrometheusLabels(cwd *model.CloudwatchData, labelsSnakeCase bool, contextLabels map[string]string, logger logging.Logger) map[string]string {
-	labels := make(map[string]string, len(cwd.Dimensions)+len(cwd.Tags)+len(contextLabels))
-	labels["name"] = cwd.ResourceName
+func createPrometheusLabels(cwd *model.CloudwatchData, labelsSnakeCase bool, contextLabelsKeys []string, contextLabelsValues []string, logger logging.Logger) ([]string, []string) {
+	size := len(cwd.Dimensions) + len(cwd.Tags) + len(contextLabelsKeys) + 1
+	labelKeys, labelValues := make([]string, 0, size), make([]string, 0, size)
+
+	labelKeys = append(labelKeys, "name")
+	labelValues = append(labelValues, cwd.ResourceName)
 
 	// Inject the sfn name back as a label
 	for _, dimension := range cwd.Dimensions {
@@ -220,7 +222,8 @@ func createPrometheusLabels(cwd *model.CloudwatchData, labelsSnakeCase bool, con
 			logger.Warn("dimension name is an invalid prometheus label name", "dimension", dimension.Name)
 			continue
 		}
-		labels["dimension_"+promTag] = dimension.Value
+		labelKeys = append(labelKeys, "dimension_"+promTag)
+		labelValues = append(labelValues, dimension.Value)
 	}
 
 	for _, tag := range cwd.Tags {
@@ -229,25 +232,31 @@ func createPrometheusLabels(cwd *model.CloudwatchData, labelsSnakeCase bool, con
 			logger.Warn("metric tag name is an invalid prometheus label name", "tag", tag.Key)
 			continue
 		}
-		labels["tag_"+promTag] = tag.Value
+		labelKeys = append(labelKeys, "tag_"+promTag)
+		labelValues = append(labelValues, tag.Value)
 	}
 
-	maps.Copy(labels, contextLabels)
+	labelKeys = append(labelKeys, contextLabelsKeys...)
+	labelValues = append(labelValues, contextLabelsValues...)
 
-	return labels
+	return labelKeys, labelValues
 }
 
-func contextToLabels(context *model.ScrapeContext, labelsSnakeCase bool, logger logging.Logger) map[string]string {
+func contextToLabels(context *model.ScrapeContext, labelsSnakeCase bool, logger logging.Logger) ([]string, []string) {
 	if context == nil {
-		return map[string]string{}
+		return []string{}, []string{}
 	}
 
-	labels := make(map[string]string, 2+len(context.CustomTags))
-	labels["region"] = context.Region
-	labels["account_id"] = context.AccountID
+	size := 3 + len(context.CustomTags)
+	keys, values := make([]string, 0, size), make([]string, 0, size)
+
+	keys = append(keys, "region", "account_id")
+	values = append(values, context.Region, context.AccountID)
+
 	// If there's no account alias, omit adding an extra label in the series, it will work either way query wise
 	if context.AccountAlias != "" {
-		labels["account_alias"] = context.AccountAlias
+		keys = append(keys, "account_alias")
+		values = append(values, context.AccountAlias)
 	}
 
 	for _, label := range context.CustomTags {
@@ -256,19 +265,20 @@ func contextToLabels(context *model.ScrapeContext, labelsSnakeCase bool, logger
 			logger.Warn("custom tag name is an invalid prometheus label name", "tag", label.Key)
 			continue
 		}
-		labels["custom_tag_"+promTag] = label.Value
+		keys = append(keys, "custom_tag_"+promTag)
+		values = append(values, label.Value)
 	}
 
-	return labels
+	return keys, values
 }
 
 // recordLabelsForMetric adds any missing labels from promLabels in to the LabelSet for the metric name and returns
 // the updated observedMetricLabels
-func recordLabelsForMetric(metricName string, promLabels map[string]string, observedMetricLabels map[string]model.LabelSet) map[string]model.LabelSet {
+func recordLabelsForMetric(metricName string, labelKeys []string, observedMetricLabels map[string]model.LabelSet) map[string]model.LabelSet {
 	if _, ok := observedMetricLabels[metricName]; !ok {
-		observedMetricLabels[metricName] = make(model.LabelSet, len(promLabels))
+		observedMetricLabels[metricName] = make(model.LabelSet, len(labelKeys))
 	}
-	for label := range promLabels {
+	for _, label := range labelKeys {
 		if _, ok := observedMetricLabels[metricName][label]; !ok {
 			observedMetricLabels[metricName][label] = struct{}{}
 		}
@@ -280,18 +290,22 @@ func recordLabelsForMetric(metricName string, promLabels map[string]string, obse
 // EnsureLabelConsistencyAndRemoveDuplicates ensures that every metric has the same set of labels based on the data
 // in observedMetricLabels and that there are no duplicate metrics.
 // Prometheus requires that all metrics with the same name have the same set of labels and that no duplicates are registered
-func EnsureLabelConsistencyAndRemoveDuplicates(metrics []*PrometheusMetric, observedMetricLabels map[string]model.LabelSet) []*PrometheusMetric {
+func EnsureLabelConsistencyAndRemoveDuplicates(metrics []*PrometheusMetric, observedMetricLabels map[string]model.LabelSet, logger logging.Logger) []*PrometheusMetric {
 	metricKeys := make(map[string]struct{}, len(metrics))
 	output := make([]*PrometheusMetric, 0, len(metrics))
 
 	for _, metric := range metrics {
-		for observedLabels := range observedMetricLabels[metric.Name] {
-			if _, ok := metric.Labels[observedLabels]; !ok {
-				metric.Labels[observedLabels] = ""
-			}
+		observedLabels := observedMetricLabels[metric.Name()]
+		for label := range observedLabels {
+			metric.AddIfMissingLabelPair(label, "")
+		}
+
+		if len(observedLabels) != metric.LabelsLen() {
+			duplicates := metric.RemoveDuplicateLabels()
+			logger.Warn("metric has duplicate labels", "metric_name", metric.Name(), "observed_labels", len(observedLabels), "labels_len", metric.LabelsLen(), "duplicated_labels", duplicates)
 		}
 
-		metricKey := fmt.Sprintf("%s-%d", metric.Name, prom_model.LabelsToSignature(metric.Labels))
+		metricKey := metric.Name() + "-" + strconv.FormatUint(metric.LabelsSignature(), 10)
 		if _, exists := metricKeys[metricKey]; !exists {
 			metricKeys[metricKey] = struct{}{}
 			output = append(output, metric)