feat: Add Manifest Reconcile Duration Metric (#1321)

* feat: Add Manifest Reconcile Duration Metric

* fix: lint

* fix: remove unnecessary call

* test: Add Metric Check to E2E Metrics Test

* fix: Lint Issue

* fix: Metric Check in E2E Metrics Test

* chore: Update Buckets for Metrics

* fix:Lint Issue

* fix: FDescribe

internal/declarative/v2/reconciler.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	apicorev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -92,6 +93,9 @@ func newResourcesCondition(obj Object) apimetav1.Condition {
 
 //nolint:funlen,cyclop,gocognit // Declarative pkg will be removed soon
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	reconcileStart := time.Now()
+	defer r.Metrics.ObserveReconcileDuration(reconcileStart, req.NamespacedName.Name)
+
 	obj, ok := r.prototype.DeepCopyObject().(Object)
 	if !ok {
 		r.Metrics.RecordRequeueReason(metrics.ManifestTypeCast, queue.UnexpectedRequeue)

internal/pkg/metrics/manifest.go

@@ -1,10 +1,19 @@
 package metrics
 
-import "github.com/kyma-project/lifecycle-manager/pkg/queue"
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	"github.com/kyma-project/lifecycle-manager/pkg/queue"
+)
 
 type ManifestRequeueReason string
 
 const (
+	MetricReconcileDuration               string                = "reconcile_duration_seconds"
+	MetricLabelModule                     string                = "module_name"
 	ManifestTypeCast                      ManifestRequeueReason = "manifest_type_cast"
 	ManifestRetrieval                     ManifestRequeueReason = "manifest_retrieval"
 	ManifestInit                          ManifestRequeueReason = "manifest_initialize"
@@ -28,12 +37,35 @@ const (
 
 type ManifestMetrics struct {
 	*SharedMetrics
+	reconcileDurationHistogram *prometheus.HistogramVec
 }
 
 func NewManifestMetrics(sharedMetrics *SharedMetrics) *ManifestMetrics {
-	return &ManifestMetrics{SharedMetrics: sharedMetrics}
+	metrics := &ManifestMetrics{
+		SharedMetrics: sharedMetrics,
+		reconcileDurationHistogram: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name: MetricReconcileDuration,
+				Help: "Histogram of reconcile duration for manifest reconciliation in seconds",
+				Buckets: []float64{
+					0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6,
+					0.7, 0.8, 0.9, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20,
+					25, 30, 40, 50, 60,
+				},
+			},
+			[]string{MetricLabelModule},
+		),
+	}
+	ctrlmetrics.Registry.MustRegister(metrics.reconcileDurationHistogram)
+	return metrics
 }
 
 func (k *ManifestMetrics) RecordRequeueReason(requeueReason ManifestRequeueReason, requeueType queue.RequeueType) {
 	k.requeueReasonCounter.WithLabelValues(string(requeueReason), string(requeueType)).Inc()
 }
+
+func (k *ManifestMetrics) ObserveReconcileDuration(reconcileStart time.Time, moduleName string) {
+	durationInSeconds := time.Since(reconcileStart).Seconds()
+	k.reconcileDurationHistogram.WithLabelValues(moduleName).
+		Observe(durationInSeconds)
+}

pkg/testutils/metrics.go

@@ -12,6 +12,8 @@ import (
 	"github.com/kyma-project/lifecycle-manager/internal/pkg/metrics"
 )
 
+const reMatchCount = `"} (\d+)`
+
 var ErrMetricNotFound = errors.New("metric was not found")
 
 func GetKymaStateMetricCount(ctx context.Context, kymaName, state string) (int, error) {
@@ -27,8 +29,7 @@ func GetKymaStateMetricCount(ctx context.Context, kymaName, state string) (int,
 func getKymaStateMetricRegex(kymaName, state string) *regexp.Regexp {
 	return regexp.MustCompile(
 		metrics.MetricKymaState + `{instance_id="[^"]+",kyma_name="` + kymaName +
-			`",shoot="[^"]+",state="` + state +
-			`"} (\d+)`)
+			`",shoot="[^"]+",state="` + state + reMatchCount)
 }
 
 func AssertKymaStateMetricNotFound(ctx context.Context, kymaName, state string) error {
@@ -55,8 +56,7 @@ func GetRequeueReasonCount(ctx context.Context,
 	}
 	re := regexp.MustCompile(
 		metrics.MetricRequeueReason + `{requeue_reason="` + requeueReason +
-			`",requeue_type="` + requeueType +
-			`"}` + ` (\d+)`)
+			`",requeue_type="` + requeueType + reMatchCount)
 	return parseCount(re, bodyString)
 }
 
@@ -69,12 +69,25 @@ func IsManifestRequeueReasonCountIncreased(ctx context.Context, requeueReason, r
 	}
 	re := regexp.MustCompile(
 		metrics.MetricRequeueReason + `{requeue_reason="` + requeueReason +
-			`",requeue_type="` + requeueType +
-			`"}` + ` (\d+)`)
+			`",requeue_type="` + requeueType + reMatchCount)
 	count, err := parseCount(re, bodyString)
 	return count >= 1, err
 }
 
+func IsManifestReconcileDurationCountNonZero(ctx context.Context, manifestName string) (bool,
+	error,
+) {
+	bodyString, err := getMetricsBody(ctx)
+	if err != nil {
+		return false, err
+	}
+	re := regexp.MustCompile(
+		metrics.MetricReconcileDuration + `_count{` + metrics.MetricLabelModule +
+			`="` + manifestName + reMatchCount)
+	count, err := parseCount(re, bodyString)
+	return count > 0, err
+}
+
 func GetModuleStateMetricCount(ctx context.Context, kymaName, moduleName, state string) (int, error) {
 	bodyString, err := getMetricsBody(ctx)
 	if err != nil {
@@ -83,8 +96,7 @@ func GetModuleStateMetricCount(ctx context.Context, kymaName, moduleName, state
 	re := regexp.MustCompile(
 		metrics.MetricModuleState + `{instance_id="[^"]+",kyma_name="` + kymaName +
 			`",module_name="` + moduleName +
-			`",shoot="[^"]+",state="` + state +
-			`"} (\d+)`)
+			`",shoot="[^"]+",state="` + state + reMatchCount)
 	return parseCount(re, bodyString)
 }
 

tests/e2e/kyma_metrics_test.go

@@ -61,6 +61,16 @@ var _ = Describe("Manage Module Metrics", Ordered, func() {
 				Should(BeTrue())
 		})
 
+		It("Then Related Manifest Duration Metric Is Non Zero", func() {
+			manifest, err := GetManifest(ctx, controlPlaneClient, kyma.GetName(), kyma.GetNamespace(), module.Name)
+			Expect(err).ToNot(HaveOccurred())
+
+			Eventually(IsManifestReconcileDurationCountNonZero).
+				WithContext(ctx).
+				WithArguments(manifest.Name).
+				Should(BeTrue())
+		})
+
 		It("When Kyma Module is disabled", func() {
 			Eventually(DisableModule).
 				WithContext(ctx).