feat(cli): add clean up moira pattern metrics key (#1011)

cmd/cli/main.go

@@ -216,14 +216,15 @@ func main() { //nolint
 
 		log.Info().
 			Interface("user_whitelist", confCleanup.Whitelist).
-			Msg("Cleanup started")
+			Msg("Cleanup users started")
 
 		if err := handleCleanup(logger, database, confCleanup); err != nil {
 			log.Error().
 				Error(err).
 				Msg("Failed to cleanup")
 		}
-		log.Info().Msg("Cleanup finished")
+
+		log.Info().Msg("Cleanup users finished")
 	}
 
 	if *cleanupMetrics {
@@ -236,6 +237,19 @@ func main() { //nolint
 				Error(err).
 				Msg("Failed to cleanup outdated metrics")
 		}
+
+		count, err := handleCleanUpOutdatedPatternMetrics(database)
+		if err != nil {
+			log.Error().
+				Error(err).
+				Msg("Failed to cleanup outdated pattern metrics")
+		}
+
+		log.Info().
+			Int64("deleted_pattern_metrics", count).
+			Msg("Cleaned up outdated pattern metrics")
+
+		log.Info().Msg("Cleanup of outdated metrics finished")
 	}
 
 	if *cleanupLastChecks {
@@ -248,6 +262,7 @@ func main() { //nolint
 				Error(err).
 				Msg("Failed to cleanup abandoned triggers last checks")
 		}
+
 		log.Info().Msg("Cleanup abandoned triggers last checks finished")
 	}
 

cmd/cli/metrics.go

@@ -39,3 +39,7 @@ func handleRemoveMetricsByPrefix(database moira.Database, prefix string) error {
 func handleRemoveAllMetrics(database moira.Database) error {
 	return database.RemoveAllMetrics()
 }
+
+func handleCleanUpOutdatedPatternMetrics(database moira.Database) (int64, error) {
+	return database.CleanupOutdatedPatternMetrics()
+}

database/redis/metric.go

@@ -18,13 +18,27 @@ import (
 	"gopkg.in/tomb.v2"
 )
 
+func (connector *DbConnector) addPatterns(patterns ...string) error {
+	ctx := connector.context
+	client := *connector.client
+
+	if _, err := client.SAdd(ctx, patternsListKey, patterns).Result(); err != nil {
+		return fmt.Errorf("failed to add moira patterns, error: %w", err)
+	}
+
+	return nil
+}
+
 // GetPatterns gets updated patterns array.
 func (connector *DbConnector) GetPatterns() ([]string, error) {
-	c := *connector.client
-	patterns, err := c.SMembers(connector.context, patternsListKey).Result()
+	ctx := connector.context
+	client := *connector.client
+
+	patterns, err := client.SMembers(ctx, patternsListKey).Result()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get moira patterns, error: %w", err)
 	}
+
 	return patterns, nil
 }
 
@@ -239,6 +253,7 @@ func (connector *DbConnector) AddPatternMetric(pattern, metric string) error {
 	if _, err := c.SAdd(connector.context, patternMetricsKey(pattern), metric).Result(); err != nil {
 		return fmt.Errorf("failed to SADD pattern-metrics, pattern: %s, metric: %s, error: %w", pattern, metric, err)
 	}
+
 	return nil
 }
 
@@ -396,6 +411,80 @@ func (connector *DbConnector) CleanUpOutdatedMetrics(duration time.Duration) err
 	})
 }
 
+// CleanupOutdatedPatternMetrics removes already deleted metrics from the moira-pattern-metrics key.
+func (connector *DbConnector) CleanupOutdatedPatternMetrics() (int64, error) {
+	var count int64
+
+	ctx := connector.context
+	client := *connector.client
+
+	patterns, err := connector.GetPatterns()
+	if err != nil {
+		return count, fmt.Errorf("failed to get patterns: %w", err)
+	}
+
+	pipe := client.TxPipeline()
+
+	for _, pattern := range patterns {
+		nonExistentMetrics, err := connector.getNonExistentPatternMetrics(pattern)
+		if err != nil {
+			return count, fmt.Errorf("failed to get non existent metrics by pattern: %w", err)
+		}
+
+		for _, metric := range nonExistentMetrics {
+			pipe.SRem(ctx, patternMetricsKey(pattern), metric)
+			count++
+		}
+	}
+
+	if _, err := pipe.Exec(ctx); err != nil {
+		return count, fmt.Errorf("failed to remove outdated pattern metrics: %w", err)
+	}
+
+	return count, nil
+}
+
+func (connector *DbConnector) getNonExistentPatternMetrics(pattern string) ([]string, error) {
+	ctx := connector.context
+	client := *connector.client
+
+	metrics, err := connector.GetPatternMetrics(pattern)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get pattern metrics: %w", err)
+	}
+
+	pipe := client.TxPipeline()
+
+	for _, metric := range metrics {
+		pipe.Exists(ctx, metricDataKey(metric))
+	}
+
+	exec, err := pipe.Exec(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to Exec Exists metric by pattern: %w", err)
+	}
+
+	nonExistentMetrics := make([]string, 0)
+
+	for i, cmder := range exec {
+		cmd, ok := cmder.(*redis.IntCmd)
+		if !ok {
+			return nil, fmt.Errorf("failed to convert cmder to intcmd result: %w", err)
+		}
+
+		res, err := cmd.Result()
+		if err != nil {
+			return nil, err
+		}
+
+		if res == 0 {
+			nonExistentMetrics = append(nonExistentMetrics, metrics[i])
+		}
+	}
+
+	return nonExistentMetrics, nil
+}
+
 // CleanUpAbandonedRetentions removes metric retention keys that have no corresponding metric data.
 func (connector *DbConnector) CleanUpAbandonedRetentions() error {
 	return connector.callFunc(cleanUpAbandonedRetentionsOnRedisNode)