feature(metrics): add fetch notifications metric (#941)

moira-alert · Oct 30, 2023 · d7c2a23 · d7c2a23
1 parent 1a59ff3
commit d7c2a23
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 2 deletions.
diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go
@@ -137,6 +137,7 @@ func main() {
 		Logger:   logger,
 		Database: database,
 		Notifier: sender,
+		Metrics:  notifierMetrics,
 	}
 	fetchNotificationsWorker.Start()
 	defer stopNotificationsFetcher(fetchNotificationsWorker)

diff --git a/integration_tests/notifier/notifier_test.go b/integration_tests/notifier/notifier_test.go
@@ -119,6 +119,7 @@ func TestNotifier(t *testing.T) {
 	fetchNotificationsWorker := notifications.FetchNotificationsWorker{
 		Database: database,
 		Logger:   logger,
+		Metrics:  notifierMetrics,
 		Notifier: notifierInstance,
 	}
 

diff --git a/metrics/notifier.go b/metrics/notifier.go
@@ -1,6 +1,8 @@
 package metrics
 
-// NotifierMetrics is a collection of metrics used in notifier.
+import "time"
+
+// NotifierMetrics is a collection of metrics used in notifier
 type NotifierMetrics struct {
 	SubsMalformed                  Meter
 	EventsReceived                 Meter
@@ -13,6 +15,7 @@ type NotifierMetrics struct {
 	SendersDroppedNotifications    MetersCollection
 	PlotsBuildDurationMs           Histogram
 	PlotsEvaluateTriggerDurationMs Histogram
+	fetchNotificationsDurationMs   Histogram
 }
 
 // ConfigureNotifierMetrics is notifier metrics configurator.
@@ -29,9 +32,15 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics
 		SendersDroppedNotifications:    NewMetersCollection(registry),
 		PlotsBuildDurationMs:           registry.NewHistogram("plots", "build", "duration", "ms"),
 		PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"),
+		fetchNotificationsDurationMs:   registry.NewHistogram("fetch", "notifications", "duration", "ms"),
 	}
 }
 
+// UpdateFetchNotificationsDurationMs - counts how much time has passed since fetchNotificationsStartTime in ms and updates the metric
+func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime time.Time) {
+	metrics.fetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds())
+}
+
 // MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications.
 func (metrics *NotifierMetrics) MarkSendersDroppedNotifications(contactType string) {
 	if metric, found := metrics.SendersDroppedNotifications.GetRegisteredMeter(contactType); found {

diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go
@@ -8,6 +8,7 @@ import (
 	"gopkg.in/tomb.v2"
 
 	"github.com/moira-alert/moira"
+	"github.com/moira-alert/moira/metrics"
 	"github.com/moira-alert/moira/notifier"
 )
 
@@ -18,9 +19,19 @@ type FetchNotificationsWorker struct {
 	Logger   moira.Logger
 	Database moira.Database
 	Notifier notifier.Notifier
+	Metrics  *metrics.NotifierMetrics
 	tomb     tomb.Tomb
 }
 
+func (worker *FetchNotificationsWorker) updateFetchNotificationsMetric(fetchNotificationsStartTime time.Time) {
+	if worker.Metrics == nil {
+		worker.Logger.Warning().Msg("Cannot update fetch notifications metric because Metrics is nil")
+		return
+	}
+
+	worker.Metrics.UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime)
+}
+
 // Start is a cycle that fetches scheduled notifications from database
 func (worker *FetchNotificationsWorker) Start() {
 	worker.tomb.Go(func() error {
@@ -63,14 +74,18 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error {
 	if err != nil {
 		return notifierInBadStateError("can't get current notifier state")
 	}
+
 	if state != moira.SelfStateOK {
 		return notifierInBadStateError(fmt.Sprintf("notifier in a bad state: %v", state))
 	}
-	notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize())
 
+	fetchNotificationsStartTime := time.Now()
+	notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize())
 	if err != nil {
 		return err
 	}
+	worker.updateFetchNotificationsMetric(fetchNotificationsStartTime)
+
 	notificationPackages := make(map[string]*notifier.NotificationPackage)
 	for _, notification := range notifications {
 		packageKey := fmt.Sprintf("%s:%s:%s", notification.Contact.Type, notification.Contact.Value, notification.Event.TriggerID)

diff --git a/notifier/notifications/notifications_test.go b/notifier/notifications/notifications_test.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	logging "github.com/moira-alert/moira/logging/zerolog_adapter"
+	"github.com/moira-alert/moira/metrics"
 	. "github.com/smartystreets/goconvey/convey"
 
 	"github.com/moira-alert/moira"
@@ -14,6 +15,8 @@ import (
 	notifier2 "github.com/moira-alert/moira/notifier"
 )
 
+var notifierMetrics = metrics.ConfigureNotifierMetrics(metrics.NewDummyRegistry(), "notifier")
+
 func TestProcessScheduledEvent(t *testing.T) {
 	subID2 := "subscriptionID-00000000000002"
 	subID5 := "subscriptionID-00000000000005"
@@ -60,6 +63,7 @@ func TestProcessScheduledEvent(t *testing.T) {
 		Database: dataBase,
 		Logger:   logger,
 		Notifier: notifier,
+		Metrics:  notifierMetrics,
 	}
 
 	Convey("Two different notifications, should send two packages", t, func() {
@@ -159,6 +163,7 @@ func TestGoRoutine(t *testing.T) {
 		Database: dataBase,
 		Logger:   logger,
 		Notifier: notifier,
+		Metrics:  notifierMetrics,
 	}
 
 	shutdown := make(chan struct{})