diff --git a/go.mod b/go.mod index 4d956098..606e4e5e 100644 --- a/go.mod +++ b/go.mod @@ -23,10 +23,12 @@ require ( github.com/go-kit/log v0.2.1 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db github.com/prometheus/client_golang v1.20.2 + github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.57.0 github.com/r3labs/diff/v3 v3.0.1 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.4 + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 golang.org/x/sync v0.8.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -50,7 +52,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect diff --git a/go.sum b/go.sum index 9c277aa8..b7bdf223 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= diff --git a/pkg/promutil/migrate.go b/pkg/promutil/migrate.go index 78115971..0db1dc84 100644 --- a/pkg/promutil/migrate.go +++ b/pkg/promutil/migrate.go @@ -66,7 +66,7 @@ func BuildNamespaceInfoMetrics(tagData []model.TaggedResourceResult, metrics []* observedMetricLabels = recordLabelsForMetric(metricName, promLabels, observedMetricLabels) metrics = append(metrics, &PrometheusMetric{ - Name: &metricName, + Name: metricName, Labels: promLabels, Value: 0, }) @@ -116,7 +116,7 @@ func BuildMetrics(results []model.CloudwatchMetricResult, labelsSnakeCase bool, observedMetricLabels = recordLabelsForMetric(name, promLabels, observedMetricLabels) output = append(output, &PrometheusMetric{ - Name: &name, + Name: name, Labels: promLabels, Value: exportedDatapoint, Timestamp: ts, @@ -285,13 +285,13 @@ func EnsureLabelConsistencyAndRemoveDuplicates(metrics []*PrometheusMetric, obse output := make([]*PrometheusMetric, 0, len(metrics)) for _, metric := range metrics { - for observedLabels := range observedMetricLabels[*metric.Name] { + for observedLabels := range observedMetricLabels[metric.Name] { if _, ok := metric.Labels[observedLabels]; !ok { metric.Labels[observedLabels] = "" } } - metricKey := fmt.Sprintf("%s-%d", *metric.Name, prom_model.LabelsToSignature(metric.Labels)) + metricKey := fmt.Sprintf("%s-%d", metric.Name, prom_model.LabelsToSignature(metric.Labels)) if _, exists := metricKeys[metricKey]; !exists { metricKeys[metricKey] = struct{}{} output = append(output, metric) diff --git a/pkg/promutil/migrate_test.go b/pkg/promutil/migrate_test.go index 59a3331b..57dd3fbb 100644 --- a/pkg/promutil/migrate_test.go +++ b/pkg/promutil/migrate_test.go @@ -48,7 +48,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { labelsSnakeCase: false, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_info"), + Name: "aws_elasticache_info", Labels: map[string]string{ "name": "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster", "tag_CustomTag": "tag_Value", @@ -88,7 +88,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_info"), + Name: "aws_elasticache_info", Labels: map[string]string{ "name": "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster", "tag_custom_tag": "tag_Value", @@ -125,7 +125,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { }, metrics: []*PrometheusMetric{ { - Name: aws.String("aws_ec2_cpuutilization_maximum"), + Name: "aws_ec2_cpuutilization_maximum", Labels: map[string]string{ "name": "arn:aws:ec2:us-east-1:123456789012:instance/i-abc123", "dimension_InstanceId": "i-abc123", @@ -142,7 +142,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_ec2_cpuutilization_maximum"), + Name: "aws_ec2_cpuutilization_maximum", Labels: map[string]string{ "name": "arn:aws:ec2:us-east-1:123456789012:instance/i-abc123", "dimension_InstanceId": "i-abc123", @@ -150,7 +150,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { Value: 0, }, { - Name: aws.String("aws_elasticache_info"), + Name: "aws_elasticache_info", Labels: map[string]string{ "name": "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster", "tag_custom_tag": "tag_Value", @@ -201,7 +201,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_info"), + Name: "aws_elasticache_info", Labels: map[string]string{ "name": "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster", "tag_cache_name": "cache_instance_1", @@ -247,7 +247,7 @@ func TestBuildNamespaceInfoMetrics(t *testing.T) { labelsSnakeCase: false, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_sagemaker_trainingjobs_info"), + Name: "aws_sagemaker_trainingjobs_info", Labels: map[string]string{ "name": "arn:aws:sagemaker:us-east-1:123456789012:training-job/sagemaker-xgboost", "tag_CustomTag": "tag_Value", @@ -380,7 +380,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: false, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_cpuutilization_average"), + Name: "aws_elasticache_cpuutilization_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -391,7 +391,7 @@ func TestBuildMetrics(t *testing.T) { }, }, { - Name: aws.String("aws_elasticache_freeable_memory_average"), + Name: "aws_elasticache_freeable_memory_average", Value: 2, Timestamp: ts, Labels: map[string]string{ @@ -402,7 +402,7 @@ func TestBuildMetrics(t *testing.T) { }, }, { - Name: aws.String("aws_elasticache_network_bytes_in_average"), + Name: "aws_elasticache_network_bytes_in_average", Value: 3, Timestamp: ts, Labels: map[string]string{ @@ -413,7 +413,7 @@ func TestBuildMetrics(t *testing.T) { }, }, { - Name: aws.String("aws_elasticache_network_bytes_out_average"), + Name: "aws_elasticache_network_bytes_out_average", Value: 4, Timestamp: ts, IncludeTimestamp: true, @@ -548,7 +548,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: false, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_cpuutilization_average"), + Name: "aws_elasticache_cpuutilization_average", Value: 0, Timestamp: ts, Labels: map[string]string{ @@ -560,7 +560,7 @@ func TestBuildMetrics(t *testing.T) { IncludeTimestamp: false, }, { - Name: aws.String("aws_elasticache_freeable_memory_average"), + Name: "aws_elasticache_freeable_memory_average", Value: math.NaN(), Timestamp: ts, Labels: map[string]string{ @@ -572,7 +572,7 @@ func TestBuildMetrics(t *testing.T) { IncludeTimestamp: false, }, { - Name: aws.String("aws_elasticache_network_bytes_in_average"), + Name: "aws_elasticache_network_bytes_in_average", Value: 0, Timestamp: ts, Labels: map[string]string{ @@ -640,7 +640,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_cpuutilization_average"), + Name: "aws_elasticache_cpuutilization_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -695,7 +695,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_sagemaker_trainingjobs_cpuutilization_average"), + Name: "aws_sagemaker_trainingjobs_cpuutilization_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -750,7 +750,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_glue_driver_aggregate_bytes_read_average"), + Name: "aws_glue_driver_aggregate_bytes_read_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -805,7 +805,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_glue_aggregate_glue_jobs_bytes_read_average"), + Name: "aws_glue_aggregate_glue_jobs_bytes_read_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -863,7 +863,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_cpuutilization_average"), + Name: "aws_elasticache_cpuutilization_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -920,7 +920,7 @@ func TestBuildMetrics(t *testing.T) { labelsSnakeCase: true, expectedMetrics: []*PrometheusMetric{ { - Name: aws.String("aws_elasticache_cpuutilization_average"), + Name: "aws_elasticache_cpuutilization_average", Value: 1, Timestamp: ts, Labels: map[string]string{ @@ -1170,17 +1170,17 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "adds missing labels", metrics: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, Value: 1.0, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label2": "value2"}, Value: 2.0, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{}, Value: 3.0, }, @@ -1188,17 +1188,17 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { observedLabels: map[string]model.LabelSet{"metric1": {"label1": {}, "label2": {}, "label3": {}}}, output: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1", "label2": "", "label3": ""}, Value: 1.0, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "", "label3": "", "label2": "value2"}, Value: 2.0, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "", "label2": "", "label3": ""}, Value: 3.0, }, @@ -1208,18 +1208,18 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "duplicate metric", metrics: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, }, observedLabels: map[string]model.LabelSet{}, output: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, }, @@ -1228,18 +1228,18 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "duplicate metric, multiple labels", metrics: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1", "label2": "value2"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label2": "value2", "label1": "value1"}, }, }, observedLabels: map[string]model.LabelSet{}, output: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1", "label2": "value2"}, }, }, @@ -1248,22 +1248,22 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "metric with different labels", metrics: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label2": "value2"}, }, }, observedLabels: map[string]model.LabelSet{}, output: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label2": "value2"}, }, }, @@ -1272,22 +1272,22 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "two metrics", metrics: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label1": "value1"}, }, }, observedLabels: map[string]model.LabelSet{}, output: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label1": "value1"}, }, }, @@ -1296,22 +1296,22 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "two metrics with different labels", metrics: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label2": "value2"}, }, }, observedLabels: map[string]model.LabelSet{}, output: []*PrometheusMetric{ { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label2": "value2"}, }, }, @@ -1320,38 +1320,38 @@ func Test_EnsureLabelConsistencyAndRemoveDuplicates(t *testing.T) { name: "multiple duplicates and non-duplicates", metrics: []*PrometheusMetric{ { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label2": "value2"}, }, { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, }, observedLabels: map[string]model.LabelSet{}, output: []*PrometheusMetric{ { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label2": "value2"}, }, { - Name: aws.String("metric2"), + Name: "metric2", Labels: map[string]string{"label1": "value1"}, }, { - Name: aws.String("metric1"), + Name: "metric1", Labels: map[string]string{"label1": "value1"}, }, }, diff --git a/pkg/promutil/prometheus.go b/pkg/promutil/prometheus.go index 13c77a38..c47b8c98 100644 --- a/pkg/promutil/prometheus.go +++ b/pkg/promutil/prometheus.go @@ -6,6 +6,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "golang.org/x/exp/maps" ) var ( @@ -91,7 +92,7 @@ var replacer = strings.NewReplacer( ) type PrometheusMetric struct { - Name *string + Name string Labels map[string]string Value float64 IncludeTimestamp bool @@ -99,12 +100,12 @@ type PrometheusMetric struct { } type PrometheusCollector struct { - metrics []*PrometheusMetric + metrics []prometheus.Metric } func NewPrometheusCollector(metrics []*PrometheusMetric) *PrometheusCollector { return &PrometheusCollector{ - metrics: metrics, + metrics: toConstMetrics(metrics), } } @@ -118,24 +119,48 @@ func (p *PrometheusCollector) Describe(_ chan<- *prometheus.Desc) { func (p *PrometheusCollector) Collect(metrics chan<- prometheus.Metric) { for _, metric := range p.metrics { - metrics <- createMetric(metric) + metrics <- metric } } -func createMetric(metric *PrometheusMetric) prometheus.Metric { - gauge := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: *metric.Name, - Help: "Help is not implemented yet.", - ConstLabels: metric.Labels, - }) +func toConstMetrics(metrics []*PrometheusMetric) []prometheus.Metric { + // We keep two fast lookup maps here one for the prometheus.Desc of a metric which can be reused for each metric with + // the same name and the expected label key order of a particular metric name. + // The prometheus.Desc object is expensive to create and being able to reuse it for all metrics with the same name + // results in large performance gain. We use the other map because metrics created using the Desc only provide label + // values and they must be provided in the exact same order as registered in the Desc. + metricToDesc := map[string]*prometheus.Desc{} + metricToExpectedLabelOrder := map[string][]string{} + + result := make([]prometheus.Metric, 0, len(metrics)) + for _, metric := range metrics { + metricName := metric.Name + if _, ok := metricToDesc[metricName]; !ok { + labelKeys := maps.Keys(metric.Labels) + metricToDesc[metricName] = prometheus.NewDesc(metricName, "Help is not implemented yet.", labelKeys, nil) + metricToExpectedLabelOrder[metricName] = labelKeys + } + metricsDesc := metricToDesc[metricName] - gauge.Set(metric.Value) + // Create the label values using the label order of the Desc + labelValues := make([]string, 0, len(metric.Labels)) + for _, labelKey := range metricToExpectedLabelOrder[metricName] { + labelValues = append(labelValues, metric.Labels[labelKey]) + } + + promMetric, err := prometheus.NewConstMetric(metricsDesc, prometheus.GaugeValue, metric.Value, labelValues...) + if err != nil { + // If for whatever reason the metric or metricsDesc is considered invalid this will ensure the error is + // reported through the collector + promMetric = prometheus.NewInvalidMetric(metricsDesc, err) + } else if metric.IncludeTimestamp { + promMetric = prometheus.NewMetricWithTimestamp(metric.Timestamp, promMetric) + } - if !metric.IncludeTimestamp { - return gauge + result = append(result, promMetric) } - return prometheus.NewMetricWithTimestamp(metric.Timestamp, gauge) + return result } func PromString(text string) string { diff --git a/pkg/promutil/prometheus_test.go b/pkg/promutil/prometheus_test.go index ecc1ac8a..fd4c33c4 100644 --- a/pkg/promutil/prometheus_test.go +++ b/pkg/promutil/prometheus_test.go @@ -2,8 +2,12 @@ package promutil import ( "testing" + "time" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSplitString(t *testing.T) { @@ -116,3 +120,114 @@ func TestPromStringTag(t *testing.T) { }) } } + +func TestNewPrometheusCollector_CanReportMetricsAndErrors(t *testing.T) { + metrics := []*PrometheusMetric{ + { + Name: "this*is*not*valid", + Labels: map[string]string{}, + Value: 0, + IncludeTimestamp: false, + }, + { + Name: "this_is_valid", + Labels: map[string]string{"key": "value1"}, + Value: 0, + IncludeTimestamp: false, + }, + } + collector := NewPrometheusCollector(metrics) + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(collector)) + families, err := registry.Gather() + assert.Error(t, err) + assert.Len(t, families, 1) + family := families[0] + assert.Equal(t, "this_is_valid", family.GetName()) +} + +func TestNewPrometheusCollector_CanReportMetrics(t *testing.T) { + ts := time.Now() + + labelSet1 := map[string]string{"key1": "value", "key2": "value", "key3": "value"} + labelSet2 := map[string]string{"key2": "out", "key3": "of", "key1": "order"} + labelSet3 := map[string]string{"key2": "out", "key1": "of", "key3": "order"} + metrics := []*PrometheusMetric{ + { + Name: "metric_with_labels", + Labels: labelSet1, + Value: 1, + IncludeTimestamp: false, + }, + { + Name: "metric_with_labels", + Labels: labelSet2, + Value: 2, + IncludeTimestamp: false, + }, + { + Name: "metric_with_labels", + Labels: labelSet3, + Value: 3, + IncludeTimestamp: false, + }, + { + Name: "metric_with_timestamp", + Labels: map[string]string{}, + Value: 1, + IncludeTimestamp: true, + Timestamp: ts, + }, + } + + collector := NewPrometheusCollector(metrics) + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(collector)) + families, err := registry.Gather() + assert.NoError(t, err) + assert.Len(t, families, 2) + + var metricWithLabels *dto.MetricFamily + var metricWithTs *dto.MetricFamily + + for _, metricFamily := range families { + assert.Equal(t, dto.MetricType_GAUGE, metricFamily.GetType()) + + switch { + case metricFamily.GetName() == "metric_with_labels": + metricWithLabels = metricFamily + case metricFamily.GetName() == "metric_with_timestamp": + metricWithTs = metricFamily + default: + require.Failf(t, "Encountered an unexpected metric family %s", metricFamily.GetName()) + } + } + require.NotNil(t, metricWithLabels) + require.NotNil(t, metricWithTs) + + assert.Len(t, metricWithLabels.Metric, 3) + for _, metric := range metricWithLabels.Metric { + assert.Len(t, metric.Label, 3) + var labelSetToMatch map[string]string + switch *metric.Gauge.Value { + case 1.0: + labelSetToMatch = labelSet1 + case 2.0: + labelSetToMatch = labelSet2 + case 3.0: + labelSetToMatch = labelSet3 + default: + require.Fail(t, "Encountered an metric value value %v", *metric.Gauge.Value) + } + + for _, labelPairs := range metric.Label { + require.Contains(t, labelSetToMatch, *labelPairs.Name) + require.Equal(t, labelSetToMatch[*labelPairs.Name], *labelPairs.Value) + } + } + + require.Len(t, metricWithTs.Metric, 1) + tsMetric := metricWithTs.Metric[0] + assert.Equal(t, ts.UnixMilli(), *tsMetric.TimestampMs) + assert.Equal(t, 1.0, *tsMetric.Gauge.Value) +}