diff --git a/slo/rules.go b/slo/rules.go index ac469fac..5ac832d5 100644 --- a/slo/rules.go +++ b/slo/rules.go @@ -512,7 +512,7 @@ func (o Objective) Burnrate(timerange time.Duration) string { return expr.String() case LatencyNative: - expr, err := parser.ParseExpr(`1 - histogram_fraction(0,0.696969, rate(metric{matchers="total"}[1s]))`) + expr, err := parser.ParseExpr(`1 - histogram_fraction(0,0.696969, sum(rate(metric{matchers="total"}[1s])))`) if err != nil { return err.Error() } @@ -911,7 +911,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { } } - expr, err := parser.ParseExpr(`histogram_count(increase(metric{matchers="total"}[1s]))`) + expr, err := parser.ParseExpr(`histogram_count(sum(increase(metric{matchers="total"}[1s])))`) if err != nil { return monitoringv1.RuleGroup{}, err } @@ -929,7 +929,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { Labels: ruleLabels, }) - expr, err = parser.ParseExpr(`histogram_fraction(0, 0.696969, increase(metric{matchers="total"}[1s])) * histogram_count(increase(metric{matchers="total"}[1s]))`) + expr, err = parser.ParseExpr(`histogram_fraction(0, 0.696969, sum(increase(metric{matchers="total"}[1s]))) * histogram_count(sum(increase(metric{matchers="total"}[1s])))`) if err != nil { return monitoringv1.RuleGroup{}, err } @@ -1384,6 +1384,47 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { Labels: ruleLabels, }) } + case LatencyNative: + latencySeconds := time.Duration(o.Indicator.LatencyNative.Latency).Seconds() + + // availability + { + expr, err := parser.ParseExpr(`sum(metric{matchers="errors"} or vector(0)) / sum(metric{matchers="total"})`) + if err != nil { + return monitoringv1.RuleGroup{}, err + } + + metric := increaseName(o.Indicator.LatencyNative.Total.Name, o.Window) + matchers := o.Indicator.LatencyNative.Total.LabelMatchers + for _, m := range matchers { + if m.Name == labels.MetricName { + m.Value = metric + break + } + } + matchers = append(matchers, &labels.Matcher{ + Type: labels.MatchEqual, + Name: "slo", + Value: o.Name(), + }) + + errorMatchers := slices.Clone(matchers) + errorMatchers = append(errorMatchers, &labels.Matcher{Type: labels.MatchEqual, Name: "le", Value: fmt.Sprintf("%g", latencySeconds)}) + matchers = append(matchers, &labels.Matcher{Type: labels.MatchEqual, Name: "le", Value: ""}) + + objectiveReplacer{ + metric: metric, + matchers: matchers, + errorMatchers: errorMatchers, + window: time.Duration(o.Window), + }.replace(expr) + + rules = append(rules, monitoringv1.Rule{ + Record: "pyrra_availability", + Expr: intstr.FromString(expr.String()), + Labels: ruleLabels, + }) + } case BoolGauge: if len(o.Indicator.BoolGauge.Grouping) > 0 { diff --git a/slo/rules_test.go b/slo/rules_test.go index 65d617e5..24e10133 100644 --- a/slo/rules_test.go +++ b/slo/rules_test.go @@ -359,31 +359,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: monitoringDuration("30s"), Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:burnrate5m", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[5m]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[5m])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate30m", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[30m]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[30m])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate1h", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[1h]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate2h", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[2h]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[2h])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate6h", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[6h]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[6h])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate1d", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[1d]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[1d])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate4d", - Expr: intstr.FromString(`1 - histogram_fraction(0, 1, rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4d]))`), + Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4d])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Alert: "ErrorBudgetBurn", @@ -1369,11 +1369,11 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: monitoringDuration("2m30s"), Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`histogram_count(increase(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`histogram_count(sum(increase(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4w])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`histogram_fraction(0, 1, increase(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4w])) * histogram_count(increase(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`histogram_fraction(0, 1, sum(increase(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))) * histogram_count(sum(increase(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4w])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "le": "1"}, //}, { // Alert: "SLOMetricAbsent", @@ -1774,6 +1774,26 @@ func TestObjective_GrafanaRules(t *testing.T) { Labels: map[string]string{"slo": "monitoring-http-latency"}, }}, }, + }, { + name: "http-latency-native", + slo: objectiveHTTPNativeLatency(), + rules: monitoringv1.RuleGroup{ + Name: "monitoring-http-latency-generic", + Interval: monitoringDuration("30s"), + Rules: []monitoringv1.Rule{{ + Record: "pyrra_objective", + Expr: intstr.FromString("0.995"), + Labels: map[string]string{"slo": "monitoring-http-latency"}, + }, { + Record: "pyrra_window", + Expr: intstr.FromString("2419200"), + Labels: map[string]string{"slo": "monitoring-http-latency"}, + }, { + Record: "pyrra_availability", + Expr: intstr.FromString(`sum(http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="1",slo="monitoring-http-latency"} or vector(0)) / sum(http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="",slo="monitoring-http-latency"})`), + Labels: map[string]string{"slo": "monitoring-http-latency"}, + }}, + }, }, { name: "http-latency-grouping", slo: objectiveHTTPLatencyGrouping(), @@ -1912,7 +1932,7 @@ func TestObjective_GrafanaRules(t *testing.T) { err: ErrGroupingUnsupported, }} - require.Len(t, testcases, 16) + require.Len(t, testcases, 17) for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) {