From f1a8fb81006f93ca2e650b0cfc00cab5c743f7bf Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 17 Oct 2023 21:03:37 +0300 Subject: [PATCH 1/6] add new fetchNotifications metric --- cmd/notifier/main.go | 1 + metrics/notifier.go | 2 ++ notifier/notifications/notifications.go | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 8a75bb410..4408deadb 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -137,6 +137,7 @@ func main() { Logger: logger, Database: database, Notifier: sender, + Metrics: notifierMetrics, } fetchNotificationsWorker.Start() defer stopNotificationsFetcher(fetchNotificationsWorker) diff --git a/metrics/notifier.go b/metrics/notifier.go index a2b5889e9..100bea39f 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -12,6 +12,7 @@ type NotifierMetrics struct { SendersFailedMetrics MetersCollection PlotsBuildDurationMs Histogram PlotsEvaluateTriggerDurationMs Histogram + FetchNotificationsDurationMs Histogram } // ConfigureNotifierMetrics is notifier metrics configurator @@ -27,5 +28,6 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics SendersFailedMetrics: NewMetersCollection(registry), PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), + FetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), } } diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 7c573aad1..19cb0750c 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -8,6 +8,7 @@ import ( "gopkg.in/tomb.v2" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira/notifier" ) @@ -18,6 +19,7 @@ type FetchNotificationsWorker struct { Logger moira.Logger Database moira.Database Notifier notifier.Notifier + Metrics *metrics.NotifierMetrics tomb tomb.Tomb } @@ -63,14 +65,18 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error { if err != nil { return notifierInBadStateError("can't get current notifier state") } + if state != moira.SelfStateOK { return notifierInBadStateError(fmt.Sprintf("notifier in a bad state: %v", state)) } - notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) + fetchNotificationsStartTime := time.Now() + notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) if err != nil { return err } + worker.Metrics.FetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) + notificationPackages := make(map[string]*notifier.NotificationPackage) for _, notification := range notifications { packageKey := fmt.Sprintf("%s:%s:%s", notification.Contact.Type, notification.Contact.Value, notification.Event.TriggerID) From e7ed590cf43a2b1c34907bdee8a4a5962d891353 Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 17 Oct 2023 21:46:21 +0300 Subject: [PATCH 2/6] fix notifier tests --- notifier/notifications/notifications_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/notifier/notifications/notifications_test.go b/notifier/notifications/notifications_test.go index d97aaa890..66f488001 100644 --- a/notifier/notifications/notifications_test.go +++ b/notifier/notifications/notifications_test.go @@ -6,6 +6,7 @@ import ( "github.com/golang/mock/gomock" logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" "github.com/moira-alert/moira" @@ -14,6 +15,8 @@ import ( notifier2 "github.com/moira-alert/moira/notifier" ) +var notifierMetrics = metrics.ConfigureNotifierMetrics(metrics.NewDummyRegistry(), "notifier") + func TestProcessScheduledEvent(t *testing.T) { subID2 := "subscriptionID-00000000000002" subID5 := "subscriptionID-00000000000005" @@ -60,6 +63,7 @@ func TestProcessScheduledEvent(t *testing.T) { Database: dataBase, Logger: logger, Notifier: notifier, + Metrics: notifierMetrics, } Convey("Two different notifications, should send two packages", t, func() { @@ -159,6 +163,7 @@ func TestGoRoutine(t *testing.T) { Database: dataBase, Logger: logger, Notifier: notifier, + Metrics: notifierMetrics, } shutdown := make(chan struct{}) From 36728071cd8f2f31ee136a9de18154c92724fb51 Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 17 Oct 2023 22:26:27 +0300 Subject: [PATCH 3/6] fix integration tests --- integration_tests/notifier/notifier_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/notifier/notifier_test.go b/integration_tests/notifier/notifier_test.go index a77e4dd47..51bc894a9 100644 --- a/integration_tests/notifier/notifier_test.go +++ b/integration_tests/notifier/notifier_test.go @@ -119,6 +119,7 @@ func TestNotifier(t *testing.T) { fetchNotificationsWorker := notifications.FetchNotificationsWorker{ Database: database, Logger: logger, + Metrics: notifierMetrics, Notifier: notifierInstance, } From 395f33122c065ca1b9b625b07f122e28c373dbb0 Mon Sep 17 00:00:00 2001 From: almostinf Date: Wed, 25 Oct 2023 16:00:30 +0300 Subject: [PATCH 4/6] add updateFetchNotificationsMetric function --- notifier/notifications/notifications.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 19cb0750c..40c74a89c 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -23,6 +23,15 @@ type FetchNotificationsWorker struct { tomb tomb.Tomb } +func (worker *FetchNotificationsWorker) updateFetchNotificationsMetric(fetchNotificationsStartTime time.Time) { + if worker.Metrics == nil { + worker.Logger.Warning().Msg("Cannot update fetch notifications metric because Metrics is nil") + return + } + + worker.Metrics.FetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) +} + // Start is a cycle that fetches scheduled notifications from database func (worker *FetchNotificationsWorker) Start() { worker.tomb.Go(func() error { @@ -75,7 +84,7 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error { if err != nil { return err } - worker.Metrics.FetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) + worker.updateFetchNotificationsMetric(fetchNotificationsStartTime) notificationPackages := make(map[string]*notifier.NotificationPackage) for _, notification := range notifications { From a8e30c3e2dd899cb42082b041d4bdc63a3485e9e Mon Sep 17 00:00:00 2001 From: almostinf Date: Fri, 27 Oct 2023 23:17:59 +0300 Subject: [PATCH 5/6] refactor update fetch notifications metric --- metrics/notifier.go | 7 +++++++ notifier/notifications/notifications.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/metrics/notifier.go b/metrics/notifier.go index 100bea39f..377f8f619 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -1,5 +1,7 @@ package metrics +import "time" + // NotifierMetrics is a collection of metrics used in notifier type NotifierMetrics struct { SubsMalformed Meter @@ -31,3 +33,8 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics FetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), } } + +// UpdateFetchNotificationsDurationMs - counts how much time has passed since fetchNotificationsStartTime in ms and updates the metric +func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime time.Time) { + metrics.FetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) +} diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 40c74a89c..e7cd5be9d 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -29,7 +29,7 @@ func (worker *FetchNotificationsWorker) updateFetchNotificationsMetric(fetchNoti return } - worker.Metrics.FetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) + worker.Metrics.UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime) } // Start is a cycle that fetches scheduled notifications from database From 4847fa33bac00d1138f7e6a6711f92decc4e2bb7 Mon Sep 17 00:00:00 2001 From: almostinf Date: Sat, 28 Oct 2023 15:12:59 +0300 Subject: [PATCH 6/6] make field private --- metrics/notifier.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics/notifier.go b/metrics/notifier.go index 83941a039..9149eee2b 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -15,7 +15,7 @@ type NotifierMetrics struct { SendersDroppedNotifications MetersCollection PlotsBuildDurationMs Histogram PlotsEvaluateTriggerDurationMs Histogram - FetchNotificationsDurationMs Histogram + fetchNotificationsDurationMs Histogram } // ConfigureNotifierMetrics is notifier metrics configurator. @@ -32,13 +32,13 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics SendersDroppedNotifications: NewMetersCollection(registry), PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), - FetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), + fetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), } } // UpdateFetchNotificationsDurationMs - counts how much time has passed since fetchNotificationsStartTime in ms and updates the metric func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime time.Time) { - metrics.FetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) + metrics.fetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) } // MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications.