diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 8a75bb410..4408deadb 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -137,6 +137,7 @@ func main() { Logger: logger, Database: database, Notifier: sender, + Metrics: notifierMetrics, } fetchNotificationsWorker.Start() defer stopNotificationsFetcher(fetchNotificationsWorker) diff --git a/integration_tests/notifier/notifier_test.go b/integration_tests/notifier/notifier_test.go index a77e4dd47..51bc894a9 100644 --- a/integration_tests/notifier/notifier_test.go +++ b/integration_tests/notifier/notifier_test.go @@ -119,6 +119,7 @@ func TestNotifier(t *testing.T) { fetchNotificationsWorker := notifications.FetchNotificationsWorker{ Database: database, Logger: logger, + Metrics: notifierMetrics, Notifier: notifierInstance, } diff --git a/metrics/notifier.go b/metrics/notifier.go index bcf61daa3..9149eee2b 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -1,6 +1,8 @@ package metrics -// NotifierMetrics is a collection of metrics used in notifier. +import "time" + +// NotifierMetrics is a collection of metrics used in notifier type NotifierMetrics struct { SubsMalformed Meter EventsReceived Meter @@ -13,6 +15,7 @@ type NotifierMetrics struct { SendersDroppedNotifications MetersCollection PlotsBuildDurationMs Histogram PlotsEvaluateTriggerDurationMs Histogram + fetchNotificationsDurationMs Histogram } // ConfigureNotifierMetrics is notifier metrics configurator. @@ -29,9 +32,15 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics SendersDroppedNotifications: NewMetersCollection(registry), PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), + fetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), } } +// UpdateFetchNotificationsDurationMs - counts how much time has passed since fetchNotificationsStartTime in ms and updates the metric +func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime time.Time) { + metrics.fetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) +} + // MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications. func (metrics *NotifierMetrics) MarkSendersDroppedNotifications(contactType string) { if metric, found := metrics.SendersDroppedNotifications.GetRegisteredMeter(contactType); found { diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 7c573aad1..e7cd5be9d 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -8,6 +8,7 @@ import ( "gopkg.in/tomb.v2" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira/notifier" ) @@ -18,9 +19,19 @@ type FetchNotificationsWorker struct { Logger moira.Logger Database moira.Database Notifier notifier.Notifier + Metrics *metrics.NotifierMetrics tomb tomb.Tomb } +func (worker *FetchNotificationsWorker) updateFetchNotificationsMetric(fetchNotificationsStartTime time.Time) { + if worker.Metrics == nil { + worker.Logger.Warning().Msg("Cannot update fetch notifications metric because Metrics is nil") + return + } + + worker.Metrics.UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime) +} + // Start is a cycle that fetches scheduled notifications from database func (worker *FetchNotificationsWorker) Start() { worker.tomb.Go(func() error { @@ -63,14 +74,18 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error { if err != nil { return notifierInBadStateError("can't get current notifier state") } + if state != moira.SelfStateOK { return notifierInBadStateError(fmt.Sprintf("notifier in a bad state: %v", state)) } - notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) + fetchNotificationsStartTime := time.Now() + notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) if err != nil { return err } + worker.updateFetchNotificationsMetric(fetchNotificationsStartTime) + notificationPackages := make(map[string]*notifier.NotificationPackage) for _, notification := range notifications { packageKey := fmt.Sprintf("%s:%s:%s", notification.Contact.Type, notification.Contact.Value, notification.Event.TriggerID) diff --git a/notifier/notifications/notifications_test.go b/notifier/notifications/notifications_test.go index d97aaa890..66f488001 100644 --- a/notifier/notifications/notifications_test.go +++ b/notifier/notifications/notifications_test.go @@ -6,6 +6,7 @@ import ( "github.com/golang/mock/gomock" logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" "github.com/moira-alert/moira" @@ -14,6 +15,8 @@ import ( notifier2 "github.com/moira-alert/moira/notifier" ) +var notifierMetrics = metrics.ConfigureNotifierMetrics(metrics.NewDummyRegistry(), "notifier") + func TestProcessScheduledEvent(t *testing.T) { subID2 := "subscriptionID-00000000000002" subID5 := "subscriptionID-00000000000005" @@ -60,6 +63,7 @@ func TestProcessScheduledEvent(t *testing.T) { Database: dataBase, Logger: logger, Notifier: notifier, + Metrics: notifierMetrics, } Convey("Two different notifications, should send two packages", t, func() { @@ -159,6 +163,7 @@ func TestGoRoutine(t *testing.T) { Database: dataBase, Logger: logger, Notifier: notifier, + Metrics: notifierMetrics, } shutdown := make(chan struct{})