diff --git a/database/redis/notification.go b/database/redis/notification.go index aab10b38e..d46a433c5 100644 --- a/database/redis/notification.go +++ b/database/redis/notification.go @@ -1,6 +1,7 @@ package redis import ( + "context" "errors" "fmt" "strconv" @@ -101,16 +102,15 @@ func (connector *DbConnector) RemoveNotification(notificationKey string) (int64, foundNotifications = append(foundNotifications, notification) } } - return connector.removeNotifications(foundNotifications) + + return connector.removeNotifications(connector.context, (*connector.client).TxPipeline(), foundNotifications) } -func (connector *DbConnector) removeNotifications(notifications []*moira.ScheduledNotification) (int64, error) { +func (connector *DbConnector) removeNotifications(ctx context.Context, pipe redis.Pipeliner, notifications []*moira.ScheduledNotification) (int64, error) { if len(notifications) == 0 { return 0, nil } - ctx := connector.context - pipe := (*connector.client).TxPipeline() for _, notification := range notifications { notificationString, err := reply.GetNotificationBytes(*notification) if err != nil { @@ -121,7 +121,7 @@ func (connector *DbConnector) removeNotifications(notifications []*moira.Schedul response, err := pipe.Exec(ctx) if err != nil { - return 0, fmt.Errorf("failed to remove notifier-notification: %s", err.Error()) + return 0, fmt.Errorf("failed to remove notifications: %w", err) } total := int64(0) @@ -133,6 +133,151 @@ func (connector *DbConnector) removeNotifications(notifications []*moira.Schedul return total, nil } +// GetDelayedTimeInSeconds returns the time, if the difference between notification +// creation and sending time is greater than this time, the notification will be considered delayed +func (connector *DbConnector) GetDelayedTimeInSeconds() int64 { + return int64(connector.notification.DelayedTime.Seconds()) +} + +// filterNotificationsByDelay filters notifications into delayed and not delayed notifications +func filterNotificationsByDelay(notifications []*moira.ScheduledNotification, delayedTime int64) (delayedNotifications []*moira.ScheduledNotification, notDelayedNotifications []*moira.ScheduledNotification) { + delayedNotifications = make([]*moira.ScheduledNotification, 0, len(notifications)) + notDelayedNotifications = make([]*moira.ScheduledNotification, 0, len(notifications)) + + for _, notification := range notifications { + if notification == nil { + continue + } + + if notification.IsDelayed(delayedTime) { + delayedNotifications = append(delayedNotifications, notification) + } else { + notDelayedNotifications = append(notDelayedNotifications, notification) + } + } + + return delayedNotifications, notDelayedNotifications +} + +// getNotificationsTriggerChecks returns notifications trigger checks, optimized for the case when there are many notifications at one trigger +func (connector *DbConnector) getNotificationsTriggerChecks(notifications []*moira.ScheduledNotification) ([]*moira.CheckData, error) { + checkDataMap := make(map[string]*moira.CheckData, len(notifications)) + for _, notification := range notifications { + if notification != nil { + checkDataMap[notification.Trigger.ID] = nil + } + } + + triggerIDs := make([]string, 0, len(checkDataMap)) + for triggerID := range checkDataMap { + triggerIDs = append(triggerIDs, triggerID) + } + + triggersLastCheck, err := connector.getTriggersLastCheck(triggerIDs) + if err != nil { + return nil, err + } + + for i, triggerID := range triggerIDs { + checkDataMap[triggerID] = triggersLastCheck[i] + } + + result := make([]*moira.CheckData, 0, len(notifications)) + for _, notification := range notifications { + result = append(result, checkDataMap[notification.Trigger.ID]) + } + + return result, nil +} + +// filterNotificationsByState filters notifications based on their state to the corresponding arrays +func (connector *DbConnector) filterNotificationsByState(notifications []*moira.ScheduledNotification) (validNotifications []*moira.ScheduledNotification, toRemoveNotifications []*moira.ScheduledNotification, err error) { + validNotifications = make([]*moira.ScheduledNotification, 0, len(notifications)) + toRemoveNotifications = make([]*moira.ScheduledNotification, 0, len(notifications)) + + triggerChecks, err := connector.getNotificationsTriggerChecks(notifications) + if err != nil { + return nil, nil, fmt.Errorf("failed to get notifications trigger checks: %w", err) + } + + for i, notification := range notifications { + if notification != nil { + switch notification.GetState(triggerChecks[i]) { + case moira.ValidNotification: + validNotifications = append(validNotifications, notification) + + case moira.RemovedNotification: + toRemoveNotifications = append(toRemoveNotifications, notification) + } + } + } + + return validNotifications, toRemoveNotifications, nil +} + +// Helper function for logging information on to remove notifications +func logToRemoveNotifications(logger moira.Logger, toRemoveNotifications []*moira.ScheduledNotification) { + if len(toRemoveNotifications) == 0 { + return + } + + triggerIDsSet := make(map[string]struct{}, len(toRemoveNotifications)) + for _, removedNotification := range toRemoveNotifications { + if removedNotification == nil { + continue + } + + if _, ok := triggerIDsSet[removedNotification.Trigger.ID]; !ok { + triggerIDsSet[removedNotification.Trigger.ID] = struct{}{} + } + } + + triggerIDs := make([]string, 0, len(triggerIDsSet)) + for triggerID := range triggerIDsSet { + triggerIDs = append(triggerIDs, triggerID) + } + + logger.Info(). + Interface("notification_trigger_ids", triggerIDs). + Int("to_remove_count", len(toRemoveNotifications)). + Msg("To remove notifications") +} + +/* +handleNotifications filters notifications into delayed and not delayed, +then filters delayed notifications by notification state, then merges the 2 arrays +of not delayed and valid delayed notifications into a single sorted array + +Returns valid notifications in sorted order by timestamp and notifications to remove +*/ +func (connector *DbConnector) handleNotifications(notifications []*moira.ScheduledNotification) ([]*moira.ScheduledNotification, []*moira.ScheduledNotification, error) { + if len(notifications) == 0 { + return notifications, nil, nil + } + + delayedNotifications, notDelayedNotifications := filterNotificationsByDelay(notifications, connector.GetDelayedTimeInSeconds()) + + if len(delayedNotifications) == 0 { + return notDelayedNotifications, notDelayedNotifications, nil + } + + validNotifications, toRemoveNotifications, err := connector.filterNotificationsByState(delayedNotifications) + if err != nil { + return nil, nil, fmt.Errorf("failed to filter delayed notifications by state: %w", err) + } + + logToRemoveNotifications(connector.logger, toRemoveNotifications) + + validNotifications, err = moira.MergeToSorted[*moira.ScheduledNotification](validNotifications, notDelayedNotifications) + if err != nil { + return nil, nil, fmt.Errorf("failed to merge valid and not delayed notifications into sorted array: %w", err) + } + + toRemoveNotifications = append(toRemoveNotifications, validNotifications...) + + return validNotifications, toRemoveNotifications, nil +} + // FetchNotifications fetch notifications by given timestamp and delete it func (connector *DbConnector) FetchNotifications(to int64, limit int64) ([]*moira.ScheduledNotification, error) { if limit == 0 { @@ -141,7 +286,7 @@ func (connector *DbConnector) FetchNotifications(to int64, limit int64) ([]*moir // No limit if limit == notifier.NotificationsLimitUnlimited { - return connector.fetchNotificationsNoLimit(to) + return connector.fetchNotifications(to, notifier.NotificationsLimitUnlimited) } count, err := connector.notificationsCount(to) @@ -151,10 +296,10 @@ func (connector *DbConnector) FetchNotifications(to int64, limit int64) ([]*moir // Hope count will be not greater then limit when we call fetchNotificationsNoLimit if limit > connector.notification.TransactionHeuristicLimit && count < limit/2 { - return connector.fetchNotificationsNoLimit(to) + return connector.fetchNotifications(to, notifier.NotificationsLimitUnlimited) } - return connector.fetchNotificationsWithLimit(to, limit) + return connector.fetchNotifications(to, limit) } func (connector *DbConnector) notificationsCount(to int64) (int64, error) { @@ -171,102 +316,151 @@ func (connector *DbConnector) notificationsCount(to int64) (int64, error) { } // fetchNotificationsWithLimit reads and drops notifications from DB with limit -func (connector *DbConnector) fetchNotificationsWithLimit(to int64, limit int64) ([]*moira.ScheduledNotification, error) { - // fetchNotifecationsWithLimitDo uses WATCH, so transaction may fail and will retry it +func (connector *DbConnector) fetchNotifications(to int64, limit int64) ([]*moira.ScheduledNotification, error) { + // fetchNotificationsDo uses WATCH, so transaction may fail and will retry it // see https://redis.io/topics/transactions for i := 0; i < connector.notification.TransactionMaxRetries; i++ { - res, err := connector.fetchNotificationsWithLimitDo(to, limit) + res, err := connector.fetchNotificationsDo(to, limit) if err == nil { return res, nil } - if !errors.As(err, &transactionError{}) { + if !errors.Is(err, &transactionError{}) { return nil, err } - time.Sleep(200 * time.Millisecond) //nolint + connector.logger.Info(). + Int("transaction_retries", i+1). + Msg("Transaction error. Retry") + + time.Sleep(connector.notification.TransactionTimeout) } return nil, fmt.Errorf("transaction tries limit exceeded") } -// same as fetchNotificationsWithLimit, but only once -func (connector *DbConnector) fetchNotificationsWithLimitDo(to int64, limit int64) ([]*moira.ScheduledNotification, error) { - // see https://redis.io/topics/transactions - - ctx := connector.context - c := *connector.client - - // start optimistic transaction and get notifications with LIMIT - var response *redis.StringSliceCmd - - err := c.Watch(ctx, func(tx *redis.Tx) error { - rng := &redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(to, 10), Offset: 0, Count: limit} - response = tx.ZRangeByScore(ctx, notifierNotificationsKey, rng) - - return response.Err() - }, notifierNotificationsKey) - - if err != nil { - return nil, fmt.Errorf("failed to ZRANGEBYSCORE: %s", err) +// getNotificationsInTxWithLimit receives notifications from the database by a certain time +// sorted by timestamp in one transaction with or without limit, depending on whether limit is nil +func getNotificationsInTxWithLimit(ctx context.Context, tx *redis.Tx, to int64, limit int64) ([]*moira.ScheduledNotification, error) { + var rng *redis.ZRangeBy + if limit != notifier.NotificationsLimitUnlimited { + rng = &redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(to, 10), Offset: 0, Count: limit} + } else { + rng = &redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(to, 10)} } - notifications, err := reply.Notifications(response) - if err != nil { - return nil, fmt.Errorf("failed to EXEC: %s", err) + response := tx.ZRangeByScore(ctx, notifierNotificationsKey, rng) + if response.Err() != nil { + if errors.Is(response.Err(), redis.TxFailedErr) { + return nil, &transactionError{} + } + return nil, fmt.Errorf("failed to ZRANGEBYSCORE: %w", response.Err()) } + return reply.Notifications(response) +} + +/* +getLimitedNotifications restricts the list of notifications by last timestamp. There are two possible cases +with arrays of notifications with timestamps: + + - [1, 2, 3, 3], after limitNotifications we get the array [1, 2], + further, since the array size is not equal to the passed one, we return [1, 2] + + - [1, 1, 1], after limitNotifications we will get array [1, 1, 1], its size is equal to the initial one, + so we will get all notifications from the database with the last timestamp <= 1, i.e., + if the database at this moment has [1, 1, 1, 1, 1], then the output will be [1, 1, 1, 1, 1] + +This is to ensure that notifications with the same timestamp are always clumped into a single stack +*/ +func getLimitedNotifications( + ctx context.Context, + tx *redis.Tx, + limit int64, + notifications []*moira.ScheduledNotification, +) ([]*moira.ScheduledNotification, error) { if len(notifications) == 0 { return notifications, nil } - // ZRANGEBYSCORE with LIMIT may return not all notification with last timestamp - // (e.g. if we have notifications with timestamps [1, 2, 3, 3, 3] and limit == 3) - // but ZREMRANGEBYSCORE does not have LIMIT, so will delete all notifications with last timestamp - // (ts 3 in our example) and then run ZRANGEBYSCORE with our new last timestamp (2 in our example). + limitedNotifications := notifications - notificationsLimited := limitNotifications(notifications) - lastTs := notificationsLimited[len(notificationsLimited)-1].Timestamp + if limit != notifier.NotificationsLimitUnlimited { + limitedNotifications = limitNotifications(notifications) + lastTs := limitedNotifications[len(limitedNotifications)-1].Timestamp - if len(notifications) == len(notificationsLimited) { - // this means that all notifications have same timestamp, - // we hope that all notifications with same timestamp should fit our memory - return connector.fetchNotificationsNoLimit(lastTs) + if len(notifications) == len(limitedNotifications) { + // this means that all notifications have same timestamp, + // we hope that all notifications with same timestamp should fit our memory + var err error + limitedNotifications, err = getNotificationsInTxWithLimit(ctx, tx, lastTs, notifier.NotificationsLimitUnlimited) + if err != nil { + return nil, fmt.Errorf("failed to get notification without limit in transaction: %w", err) + } + } } - pipe := c.TxPipeline() - pipe.ZRemRangeByScore(ctx, notifierNotificationsKey, "-inf", strconv.FormatInt(lastTs, 10)) - rangeResponse, errDelete := pipe.Exec(ctx) + return limitedNotifications, nil +} - if errDelete != nil { - return nil, fmt.Errorf("failed to EXEC: %w", errDelete) - } +// fetchNotificationsDo performs fetching of notifications within a single transaction +func (connector *DbConnector) fetchNotificationsDo(to int64, limit int64) ([]*moira.ScheduledNotification, error) { + // See https://redis.io/topics/transactions - // someone has changed notifierNotificationsKey while we do our job - // and transaction fail (no notifications were deleted) :( - deleteCount, errConvert := rangeResponse[0].(*redis.IntCmd).Result() - if errConvert != nil || deleteCount == 0 { - return nil, &transactionError{} - } + ctx := connector.context + c := *connector.client - return notificationsLimited, nil -} + result := make([]*moira.ScheduledNotification, 0) -// FetchNotifications fetch notifications by given timestamp and delete it -func (connector *DbConnector) fetchNotificationsNoLimit(to int64) ([]*moira.ScheduledNotification, error) { - ctx := connector.context - pipe := (*connector.client).TxPipeline() - pipe.ZRangeByScore(ctx, notifierNotificationsKey, &redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(to, 10)}) - pipe.ZRemRangeByScore(ctx, notifierNotificationsKey, "-inf", strconv.FormatInt(to, 10)) - response, err := pipe.Exec(ctx) + // it is necessary to do these operations in one transaction to avoid data race + err := c.Watch(ctx, func(tx *redis.Tx) error { + notifications, err := getNotificationsInTxWithLimit(ctx, tx, to, limit) + if err != nil { + return fmt.Errorf("failed to get notifications with limit in transaction: %w", err) + } + + if len(notifications) == 0 { + return nil + } + + // ZRANGEBYSCORE with LIMIT may return not all notifications with last timestamp + // (e.g. we have notifications with timestamps [1, 2, 3, 3, 3] and limit = 3, + // we will get [1, 2, 3]) other notifications with timestamp 3 remain in the database, so then + // limit notifications by last timestamp and return and delete valid notifications with our new timestamp [1, 2] + limitedNotifications, err := getLimitedNotifications(ctx, tx, limit, notifications) + if err != nil { + return fmt.Errorf("failed to get limited notifications: %w", err) + } + + validNotifications, toRemoveNotifications, err := connector.handleNotifications(limitedNotifications) + if err != nil { + return fmt.Errorf("failed to validate notifications: %w", err) + } + result = validNotifications + + _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + // someone has changed notifierNotificationsKey while we do our job + // and transaction fail (no notifications were deleted) :( + if _, err = connector.removeNotifications(ctx, pipe, toRemoveNotifications); err != nil { + if errors.Is(err, redis.TxFailedErr) { + return &transactionError{} + } + return fmt.Errorf("failed to remove notifications in transaction: %w", err) + } + + return nil + }) + + return err + }, notifierNotificationsKey) if err != nil { - return nil, fmt.Errorf("failed to EXEC: %s", err) + return nil, err } - return reply.Notifications(response[0].(*redis.StringSliceCmd)) + return result, nil } // AddNotification store notification at given timestamp diff --git a/database/redis/notification_test.go b/database/redis/notification_test.go index 7b262ea5d..9cacc9f3f 100644 --- a/database/redis/notification_test.go +++ b/database/redis/notification_test.go @@ -6,9 +6,11 @@ import ( "testing" "time" + "github.com/go-redis/redis/v8" "github.com/moira-alert/moira" logging "github.com/moira-alert/moira/logging/zerolog_adapter" "github.com/moira-alert/moira/notifier" + "github.com/stretchr/testify/assert" . "github.com/smartystreets/goconvey/convey" ) @@ -20,18 +22,21 @@ func TestScheduledNotification(t *testing.T) { defer dataBase.Flush() Convey("ScheduledNotification manipulation", t, func() { - now := time.Now().Unix() + now = time.Now().Unix() notificationNew := moira.ScheduledNotification{ SendFail: 1, - Timestamp: now + 3600, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } notification := moira.ScheduledNotification{ SendFail: 2, Timestamp: now, + CreatedAt: now, } notificationOld := moira.ScheduledNotification{ SendFail: 3, - Timestamp: now - 3600, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } Convey("Test add and get by pages", func() { @@ -53,7 +58,7 @@ func TestScheduledNotification(t *testing.T) { }) Convey("Test fetch notifications", func() { - actual, err := dataBase.FetchNotifications(now-3600, notifier.NotificationsLimitUnlimited) //nolint + actual, err := dataBase.FetchNotifications(now-dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld}) @@ -62,7 +67,7 @@ func TestScheduledNotification(t *testing.T) { So(total, ShouldEqual, 2) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ification, ¬ificationNew}) - actual, err = dataBase.FetchNotifications(now+3600, notifier.NotificationsLimitUnlimited) //nolint + actual, err = dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ification, ¬ificationNew}) @@ -73,7 +78,7 @@ func TestScheduledNotification(t *testing.T) { }) Convey("Test fetch notifications limit 0", func() { - actual, err := dataBase.FetchNotifications(now-3600, 0) //nolint + actual, err := dataBase.FetchNotifications(now-dataBase.GetDelayedTimeInSeconds(), 0) //nolint So(err, ShouldBeError) So(actual, ShouldBeNil) //nolint }) @@ -96,7 +101,7 @@ func TestScheduledNotification(t *testing.T) { Contact: moira.ContactData{ID: id1}, Event: moira.NotificationEvent{SubscriptionID: &id1}, SendFail: 3, - Timestamp: now + 3600, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), } addNotifications(dataBase, []moira.ScheduledNotification{notification1, notification2, notification3}) actual, total, err := dataBase.GetNotifications(0, -1) @@ -113,7 +118,7 @@ func TestScheduledNotification(t *testing.T) { So(total, ShouldEqual, 1) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ification3}) - total, err = dataBase.RemoveNotification(strings.Join([]string{fmt.Sprintf("%v", now+3600), id1, id1}, "")) //nolint + total, err = dataBase.RemoveNotification(strings.Join([]string{fmt.Sprintf("%v", now+dataBase.GetDelayedTimeInSeconds()), id1, id1}, "")) //nolint So(err, ShouldBeNil) So(total, ShouldEqual, 1) @@ -122,7 +127,7 @@ func TestScheduledNotification(t *testing.T) { So(total, ShouldEqual, 0) So(actual, ShouldResemble, []*moira.ScheduledNotification{}) - actual, err = dataBase.FetchNotifications(now+3600, notifier.NotificationsLimitUnlimited) //nolint + actual, err = dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{}) }) @@ -145,7 +150,7 @@ func TestScheduledNotification(t *testing.T) { Contact: moira.ContactData{ID: id1}, Event: moira.NotificationEvent{SubscriptionID: &id1}, SendFail: 3, - Timestamp: now + 3600, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), } addNotifications(dataBase, []moira.ScheduledNotification{notification1, notification2, notification3}) actual, total, err := dataBase.GetNotifications(0, -1) @@ -161,7 +166,7 @@ func TestScheduledNotification(t *testing.T) { So(total, ShouldEqual, 0) So(actual, ShouldResemble, []*moira.ScheduledNotification{}) - actual, err = dataBase.FetchNotifications(now+3600, notifier.NotificationsLimitUnlimited) //nolint + actual, err = dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{}) }) @@ -214,23 +219,26 @@ func TestFetchNotifications(t *testing.T) { defer dataBase.Flush() Convey("FetchNotifications manipulation", t, func() { - now := time.Now().Unix() + now = time.Now().Unix() notificationNew := moira.ScheduledNotification{ SendFail: 1, - Timestamp: now + 3600, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } notification := moira.ScheduledNotification{ SendFail: 2, Timestamp: now, + CreatedAt: now, } notificationOld := moira.ScheduledNotification{ SendFail: 3, - Timestamp: now - 3600, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } Convey("Test fetch notifications with limit if all notifications has diff timestamp", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.FetchNotifications(now+6000, 1) //nolint + actual, err := dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), 1) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld}) @@ -245,7 +253,7 @@ func TestFetchNotifications(t *testing.T) { Convey("Test fetch notifications with limit little bit greater than count if all notifications has diff timestamp", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.FetchNotifications(now+6000, 4) //nolint + actual, err := dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), 4) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification}) @@ -261,7 +269,7 @@ func TestFetchNotifications(t *testing.T) { Convey("Test fetch notifications with limit greater than count if all notifications has diff timestamp", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.FetchNotifications(now+6000, 200000) //nolint + actual, err := dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), 200000) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) @@ -276,7 +284,7 @@ func TestFetchNotifications(t *testing.T) { Convey("Test fetch notifications without limit", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.FetchNotifications(now+6000, notifier.NotificationsLimitUnlimited) //nolint + actual, err := dataBase.FetchNotifications(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) @@ -291,117 +299,633 @@ func TestFetchNotifications(t *testing.T) { }) } -func TestNotificationsCount(t *testing.T) { +func TestGetNotificationsInTxWithLimit(t *testing.T) { logger, _ := logging.GetLogger("dataBase") dataBase := NewTestDatabase(logger) dataBase.Flush() defer dataBase.Flush() - Convey("notificationsCount in db", t, func() { - now := time.Now().Unix() + client := *dataBase.client + ctx := dataBase.context + + Convey("Test getNotificationsInTxWithLimit", t, func() { + var limit int64 = 0 + now = time.Now().Unix() notificationNew := moira.ScheduledNotification{ SendFail: 1, - Timestamp: now + 3600, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } notification := moira.ScheduledNotification{ SendFail: 2, Timestamp: now, + CreatedAt: now, } notificationOld := moira.ScheduledNotification{ SendFail: 3, - Timestamp: now - 3600, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } - Convey("Test all notification with different ts in db", func() { + Convey("Test with zero notifications without limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{}) + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getNotificationsInTxWithLimit(ctx, tx, now+dataBase.GetDelayedTimeInSeconds()*2, notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{}) + return nil + }, notifierNotificationsKey) + So(err, ShouldBeNil) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Test all notifications without limit", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.notificationsCount(now + 6000) + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getNotificationsInTxWithLimit(ctx, tx, now+dataBase.GetDelayedTimeInSeconds()*2, notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) + return nil + }, notifierNotificationsKey) So(err, ShouldBeNil) - So(actual, ShouldResemble, int64(3)) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) - Convey("Test get 0 notification with ts in db", func() { + Convey("Test all notifications with limit != count", func() { + limit = 1 addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.notificationsCount(now - 7000) + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getNotificationsInTxWithLimit(ctx, tx, now+dataBase.GetDelayedTimeInSeconds()*2, limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld}) + return nil + }, notifierNotificationsKey) So(err, ShouldBeNil) - So(actual, ShouldResemble, int64(0)) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) - Convey("Test part notification in db with ts", func() { + Convey("Test all notifications with limit = count", func() { + limit = 3 addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.notificationsCount(now) + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getNotificationsInTxWithLimit(ctx, tx, now+dataBase.GetDelayedTimeInSeconds()*2, limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) + return nil + }, notifierNotificationsKey) So(err, ShouldBeNil) - So(actual, ShouldResemble, int64(2)) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) + }) +} - Convey("Test 0 notification in db", func() { - addNotifications(dataBase, []moira.ScheduledNotification{}) - actual, err := dataBase.notificationsCount(now) +func TestGetLimitedNotifications(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + dataBase.Flush() + defer dataBase.Flush() + + client := *dataBase.client + ctx := dataBase.context + + Convey("Test getLimitedNotifications", t, func() { + var limit int64 + now = time.Now().Unix() + notificationNew := moira.ScheduledNotification{ + SendFail: 1, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, + } + notification := moira.ScheduledNotification{ + SendFail: 2, + Timestamp: now, + CreatedAt: now, + } + notificationOld := moira.ScheduledNotification{ + SendFail: 3, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, + } + + Convey("Test all notifications with different timestamps without limit", func() { + notifications := []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew} + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getLimitedNotifications(ctx, tx, notifier.NotificationsLimitUnlimited, notifications) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) + return nil + }, notifierNotificationsKey) So(err, ShouldBeNil) - So(actual, ShouldResemble, int64(0)) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) + + Convey("Test all notifications with different timestamps and limit", func() { + notifications := []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew} + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getLimitedNotifications(ctx, tx, limit, notifications) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification}) + return nil + }, notifierNotificationsKey) + So(err, ShouldBeNil) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Test all notifications with same timestamp and limit", func() { + notification.Timestamp = now + notificationNew.Timestamp = now + notificationOld.Timestamp = now + defer func() { + notificationNew.Timestamp = now + dataBase.GetDelayedTimeInSeconds() + notificationOld.Timestamp = now - dataBase.GetDelayedTimeInSeconds() + }() + + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notification, notificationNew}) + notifications := []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew} + expected := []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew} + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getLimitedNotifications(ctx, tx, limit, notifications) + So(err, ShouldBeNil) + assert.ElementsMatch(t, actual, expected) + return nil + }, notifierNotificationsKey) + So(err, ShouldBeNil) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Test not all notifications with same timestamp and limit", func() { + notification.Timestamp = now + notificationNew.Timestamp = now + notificationOld.Timestamp = now + defer func() { + notificationNew.Timestamp = now + dataBase.GetDelayedTimeInSeconds() + notificationOld.Timestamp = now - dataBase.GetDelayedTimeInSeconds() + }() + + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notification, notificationNew}) + notifications := []*moira.ScheduledNotification{¬ificationOld, ¬ification} + expected := []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew} + err := client.Watch(ctx, func(tx *redis.Tx) error { + actual, err := getLimitedNotifications(ctx, tx, limit, notifications) + So(err, ShouldBeNil) + assert.ElementsMatch(t, actual, expected) + return nil + }, notifierNotificationsKey) + So(err, ShouldBeNil) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) +} + +func TestFilterNotificationsByState(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + dataBase.Flush() + defer dataBase.Flush() + + notificationOld := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + Event: moira.NotificationEvent{ + Metric: "test", + }, + SendFail: 1, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now - dataBase.GetDelayedTimeInSeconds(), + } + notification := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + Event: moira.NotificationEvent{ + Metric: "test1", + }, + SendFail: 2, + Timestamp: now, + CreatedAt: now, + } + notificationNew := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, + Event: moira.NotificationEvent{ + Metric: "test1", + }, + SendFail: 3, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, + } + + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{}, moira.TriggerSourceNotSet) + + _ = dataBase.SetTriggerLastCheck("test2", &moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "test": {}, + }, + }, moira.TriggerSourceNotSet) + + Convey("Test filter notifications by state", t, func() { + Convey("With empty notifications", func() { + validNotifications, removedNotifications, err := dataBase.filterNotificationsByState([]*moira.ScheduledNotification{}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("With zero removed notifications", func() { + validNotifications, removedNotifications, err := dataBase.filterNotificationsByState([]*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("With removed check data", func() { + dataBase.RemoveTriggerLastCheck("test1") //nolint + defer func() { + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{}, moira.TriggerSourceNotSet) + }() + + validNotifications, removedNotifications, err := dataBase.filterNotificationsByState([]*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationNew}) + }) + + Convey("With metric on maintenance", func() { + dataBase.SetTriggerCheckMaintenance("test2", map[string]int64{"test": time.Now().Add(time.Hour).Unix()}, nil, "test", 100) //nolint + defer dataBase.SetTriggerCheckMaintenance("test2", map[string]int64{"test": 0}, nil, "test", 100) //nolint + + validNotifications, removedNotifications, err := dataBase.filterNotificationsByState([]*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notification, notificationNew}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("With trigger on maintenance", func() { + var triggerMaintenance int64 = time.Now().Add(time.Hour).Unix() + dataBase.SetTriggerCheckMaintenance("test1", map[string]int64{}, &triggerMaintenance, "test", 100) //nolint + defer func() { + triggerMaintenance = 0 + dataBase.SetTriggerCheckMaintenance("test1", map[string]int64{}, &triggerMaintenance, "test", 100) //nolint + }() + + validNotifications, removedNotifications, err := dataBase.filterNotificationsByState([]*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + }) + }) +} + +func TestHandleNotifications(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + dataBase.Flush() + defer dataBase.Flush() + + notificationOld := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + SendFail: 1, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now - dataBase.GetDelayedTimeInSeconds(), + } + notification := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, + SendFail: 2, + Timestamp: now, + CreatedAt: now, + } + notificationNew := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, + SendFail: 3, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, + } + + // create delayed notifications + notificationOld2 := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + Event: moira.NotificationEvent{ + Metric: "test", + }, + SendFail: 4, + Timestamp: now - dataBase.GetDelayedTimeInSeconds() + 1, + CreatedAt: now - 2*dataBase.GetDelayedTimeInSeconds(), + } + notificationNew2 := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, + Event: moira.NotificationEvent{ + Metric: "test", + }, + SendFail: 5, + Timestamp: now + dataBase.GetDelayedTimeInSeconds() + 1, + CreatedAt: now, + } + notificationNew3 := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + Event: moira.NotificationEvent{ + Metric: "test2", + }, + SendFail: 6, + Timestamp: now + dataBase.GetDelayedTimeInSeconds() + 2, + CreatedAt: now, + } + + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{}, moira.TriggerSourceNotSet) + + _ = dataBase.SetTriggerLastCheck("test2", &moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "test": {}, + }, + }, moira.TriggerSourceNotSet) + + Convey("Test handle notifications", t, func() { + Convey("Without delayed notifications", func() { + validNotifications, removedNotifications, err := dataBase.handleNotifications([]*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification, notificationNew}) + }) + + Convey("With both delayed and not delayed valid notifications", func() { + validNotifications, removedNotifications, err := dataBase.handleNotifications([]*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + }) + + Convey("With both delayed and not delayed notifications and removed check data", func() { + dataBase.RemoveTriggerLastCheck("test1") //nolint + defer func() { + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{}, moira.TriggerSourceNotSet) + }() + + validNotifications, removedNotifications, err := dataBase.handleNotifications([]*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew3}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationNew2, notificationOld, notificationOld2, notification, notificationNew, notificationNew3}) + }) + + Convey("With both delayed and not delayed valid notifications and metric on maintenance", func() { + dataBase.SetTriggerCheckMaintenance("test2", map[string]int64{"test": time.Now().Add(time.Hour).Unix()}, nil, "test", 100) //nolint + defer dataBase.SetTriggerCheckMaintenance("test2", map[string]int64{"test": 0}, nil, "test", 100) //nolint + + validNotifications, removedNotifications, err := dataBase.handleNotifications([]*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification, notificationNew, notificationNew2, notificationNew3}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notification, notificationNew, notificationNew2, notificationNew3}) + }) + + Convey("With both delayed and not delayed valid notifications and trigger on maintenance", func() { + var triggerMaintenance int64 = time.Now().Add(time.Hour).Unix() + dataBase.SetTriggerCheckMaintenance("test1", map[string]int64{}, &triggerMaintenance, "test", 100) //nolint + defer func() { + triggerMaintenance = 0 + dataBase.SetTriggerCheckMaintenance("test1", map[string]int64{}, &triggerMaintenance, "test", 100) //nolint + }() + + validNotifications, removedNotifications, err := dataBase.handleNotifications([]*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + So(err, ShouldBeNil) + So(validNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew3}) + So(removedNotifications, ShouldResemble, []*moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew3}) + }) }) } -func TestFetchNotificationsWithLimitDo(t *testing.T) { +func TestNotificationsCount(t *testing.T) { logger, _ := logging.GetLogger("dataBase") dataBase := NewTestDatabase(logger) dataBase.Flush() defer dataBase.Flush() Convey("notificationsCount in db", t, func() { - now := time.Now().Unix() + now = time.Now().Unix() notificationNew := moira.ScheduledNotification{ SendFail: 1, - Timestamp: now + 3600, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, } notification := moira.ScheduledNotification{ SendFail: 2, Timestamp: now, + CreatedAt: now, } notificationOld := moira.ScheduledNotification{ SendFail: 3, - Timestamp: now - 3600, - } - notification4 := moira.ScheduledNotification{ - SendFail: 4, - Timestamp: now - 3600, + Timestamp: now - dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now - dataBase.GetDelayedTimeInSeconds(), } - Convey("Test all notification with ts and limit in db", func() { + Convey("Test all notification with different ts in db", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.fetchNotificationsWithLimitDo(now+6000, 1) //nolint + actual, err := dataBase.notificationsCount(now + dataBase.GetDelayedTimeInSeconds()*2) So(err, ShouldBeNil) - So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld}) + So(actual, ShouldResemble, int64(3)) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Test get 0 notification with ts in db", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) + actual, err := dataBase.notificationsCount(now - dataBase.GetDelayedTimeInSeconds()*2) + So(err, ShouldBeNil) + So(actual, ShouldResemble, int64(0)) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Test part notification in db with ts", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) + actual, err := dataBase.notificationsCount(now) + So(err, ShouldBeNil) + So(actual, ShouldResemble, int64(2)) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) - Convey("Test 0 notification with ts and limit in empty db", func() { + Convey("Test 0 notification in db", func() { addNotifications(dataBase, []moira.ScheduledNotification{}) - actual, err := dataBase.fetchNotificationsWithLimitDo(now+6000, 10) //nolint + actual, err := dataBase.notificationsCount(now) So(err, ShouldBeNil) - So(actual, ShouldResemble, []*moira.ScheduledNotification{}) + So(actual, ShouldResemble, int64(0)) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) +} + +func TestFetchNotificationsDo(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + dataBase.Flush() + defer dataBase.Flush() + + var limit int64 + + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "test1": {}, + }, + }, moira.TriggerSourceNotSet) + + _ = dataBase.SetTriggerLastCheck("test2", &moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "test1": {}, + "test2": {}, + }, + }, moira.TriggerSourceNotSet) + + now := time.Now().Unix() + notificationOld := moira.ScheduledNotification{ + SendFail: 1, + Timestamp: now - dataBase.GetDelayedTimeInSeconds() + 1, + CreatedAt: now - dataBase.GetDelayedTimeInSeconds() + 1, + } + notification4 := moira.ScheduledNotification{ + SendFail: 2, + Timestamp: now - dataBase.GetDelayedTimeInSeconds() + 2, + CreatedAt: now - dataBase.GetDelayedTimeInSeconds() + 2, + } + notificationNew := moira.ScheduledNotification{ + SendFail: 3, + Timestamp: now + dataBase.GetDelayedTimeInSeconds(), + CreatedAt: now, + } + notification := moira.ScheduledNotification{ + SendFail: 4, + Timestamp: now, + CreatedAt: now, + } + + // create delayed notifications + notificationOld2 := moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + Event: moira.NotificationEvent{ + Metric: "test1", + }, + SendFail: 5, + Timestamp: now - dataBase.GetDelayedTimeInSeconds() + 3, + CreatedAt: now - 2*dataBase.GetDelayedTimeInSeconds(), + } + notificationNew2 := moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, + Event: moira.NotificationEvent{ + Metric: "test1", + }, + SendFail: 6, + Timestamp: now + dataBase.GetDelayedTimeInSeconds() + 1, + CreatedAt: now, + } + notificationNew3 := moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, + Event: moira.NotificationEvent{ + Metric: "test2", + }, + SendFail: 7, + Timestamp: now + dataBase.GetDelayedTimeInSeconds() + 2, + CreatedAt: now, + } + + Convey("Test fetchNotificationsDo", t, func() { + Convey("Test all notifications with diff ts in db", func() { + Convey("With limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) + limit = 1 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld}) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Without limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) + + Convey("Test zero notifications with ts in empty db", func() { + Convey("With limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{}) + limit = 10 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{}) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Without limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{}) + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{}) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) + + Convey("Test all notification with ts and without limit in db", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld, notification4}) + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification4, ¬ification, ¬ificationNew}) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) - Convey("Test all notification with ts and big limit in db", func() { + Convey("Test all notifications with ts and big limit in db", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.fetchNotificationsWithLimitDo(now+6000, 100) //nolint + limit = 100 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), limit) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification}) @@ -409,9 +933,10 @@ func TestFetchNotificationsWithLimitDo(t *testing.T) { So(err, ShouldBeNil) }) - Convey("Test notification with ts and small limit in db", func() { + Convey("Test notifications with ts and small limit in db", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld, notification4}) - actual, err := dataBase.fetchNotificationsWithLimitDo(now+6000, 3) //nolint + limit = 3 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), limit) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification4}) @@ -419,15 +944,134 @@ func TestFetchNotificationsWithLimitDo(t *testing.T) { So(err, ShouldBeNil) }) - Convey("Test notification with ts and limit = count", func() { + Convey("Test notifications with ts and limit = count", func() { addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.fetchNotificationsWithLimitDo(now+6000, 3) //nolint + limit = 3 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds(), limit) //nolint So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification}) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) + + Convey("Test delayed notifications with ts and deleted trigger", func() { + dataBase.RemoveTriggerLastCheck("test1") //nolint + defer func() { + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "test1": {}, + }, + }, moira.TriggerSourceNotSet) + }() + + Convey("With big limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + limit = 100 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds()+3, limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ificationOld2, ¬ification, ¬ificationNew}) + + allNotifications, count, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{¬ificationNew3}) + So(count, ShouldEqual, 1) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Without limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds()+3, notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ificationOld2, ¬ification, ¬ificationNew, ¬ificationNew3}) + + allNotifications, count, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + So(count, ShouldEqual, 0) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) + + Convey("Test notifications with ts and metric on maintenance", func() { + dataBase.SetTriggerCheckMaintenance("test2", map[string]int64{"test2": time.Now().Add(time.Hour).Unix()}, nil, "test", 100) //nolint + defer dataBase.SetTriggerCheckMaintenance("test2", map[string]int64{"test2": 0}, nil, "test", 100) //nolint + + Convey("With limit = count", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + limit = 6 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds()+3, limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ificationOld2, ¬ification, ¬ificationNew, ¬ificationNew2}) + + allNotifications, count, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{¬ificationNew3}) + So(count, ShouldEqual, 1) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("Without limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds()+3, notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ificationOld2, ¬ification, ¬ificationNew, ¬ificationNew2}) + + allNotifications, count, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{¬ificationNew3}) + So(count, ShouldEqual, 1) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) + + Convey("Test delayed notifications with ts and trigger on maintenance", func() { + var triggerMaintenance int64 = time.Now().Add(time.Hour).Unix() + dataBase.SetTriggerCheckMaintenance("test1", map[string]int64{}, &triggerMaintenance, "test", 100) //nolint + defer func() { + triggerMaintenance = 0 + dataBase.SetTriggerCheckMaintenance("test1", map[string]int64{}, &triggerMaintenance, "test", 100) //nolint + }() + + Convey("With small limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + limit = 3 + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds()+3, limit) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ificationOld2}) + + allNotifications, count, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{¬ification, ¬ificationNew, ¬ificationNew2, ¬ificationNew3}) + So(count, ShouldEqual, 4) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + + Convey("without limit", func() { + addNotifications(dataBase, []moira.ScheduledNotification{notificationOld, notificationOld2, notification, notificationNew, notificationNew2, notificationNew3}) + actual, err := dataBase.fetchNotificationsDo(now+dataBase.GetDelayedTimeInSeconds()+3, notifier.NotificationsLimitUnlimited) + So(err, ShouldBeNil) + So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ificationOld2, ¬ification, ¬ificationNew, ¬ificationNew3}) + + allNotifications, count, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{¬ificationNew2}) + So(count, ShouldEqual, 1) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) }) } @@ -487,59 +1131,218 @@ func TestLimitNotifications(t *testing.T) { //nolint }) } -func TestFetchNotificationsNoLimit(t *testing.T) { +func TestFilterNotificationsByDelay(t *testing.T) { + Convey("filterNotificationsByDelay manipulations", t, func() { + notification1 := &moira.ScheduledNotification{ + Timestamp: 105, + CreatedAt: 100, + } + notification2 := &moira.ScheduledNotification{ + Timestamp: 110, + CreatedAt: 100, + } + notification3 := &moira.ScheduledNotification{ + Timestamp: 120, + CreatedAt: 100, + } + + Convey("Test with zero notifications", func() { + notifications := []*moira.ScheduledNotification{} + delayed, notDelayed := filterNotificationsByDelay(notifications, 1) + So(delayed, ShouldResemble, []*moira.ScheduledNotification{}) + So(notDelayed, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("Test with zero delayed notifications", func() { + notifications := []*moira.ScheduledNotification{notification1, notification2, notification3} + delayed, notDelayed := filterNotificationsByDelay(notifications, 50) + So(delayed, ShouldResemble, []*moira.ScheduledNotification{}) + So(notDelayed, ShouldResemble, notifications) + }) + + Convey("Test with zero not delayed notifications", func() { + notifications := []*moira.ScheduledNotification{notification1, notification2, notification3} + delayed, notDelayed := filterNotificationsByDelay(notifications, 2) + So(delayed, ShouldResemble, notifications) + So(notDelayed, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("Test with one delayed and two not delayed notifications", func() { + notifications := []*moira.ScheduledNotification{notification1, notification2, notification3} + delayed, notDelayed := filterNotificationsByDelay(notifications, 15) + So(delayed, ShouldResemble, []*moira.ScheduledNotification{notification3}) + So(notDelayed, ShouldResemble, []*moira.ScheduledNotification{notification1, notification2}) + }) + }) +} + +func TestGetNotificationsTriggerChecks(t *testing.T) { logger, _ := logging.GetLogger("dataBase") dataBase := NewTestDatabase(logger) dataBase.Flush() defer dataBase.Flush() - Convey("fetchNotificationsNoLimit manipulation", t, func() { - now := time.Now().Unix() - notificationNew := moira.ScheduledNotification{ - SendFail: 1, - Timestamp: now + 3600, + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{ + Timestamp: 1, + }, moira.TriggerSourceNotSet) + _ = dataBase.SetTriggerLastCheck("test2", &moira.CheckData{ + Timestamp: 2, + }, moira.TriggerSourceNotSet) + + Convey("getNotificationsTriggerChecks manipulations", t, func() { + notification1 := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, } - notification := moira.ScheduledNotification{ - SendFail: 2, - Timestamp: now, + notification2 := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test1", + }, } - notificationOld := moira.ScheduledNotification{ - SendFail: 3, - Timestamp: now - 3600, - } - notification4 := moira.ScheduledNotification{ - SendFail: 4, - Timestamp: now - 3600, + notification3 := &moira.ScheduledNotification{ + Trigger: moira.TriggerData{ + ID: "test2", + }, } - Convey("Test all notifications with diff ts in db", func() { - addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld}) - actual, err := dataBase.fetchNotificationsNoLimit(now + 6000) + Convey("Test with zero notifications", func() { + notifications := []*moira.ScheduledNotification{} + triggerChecks, err := dataBase.getNotificationsTriggerChecks(notifications) So(err, ShouldBeNil) - So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification, ¬ificationNew}) + So(triggerChecks, ShouldResemble, []*moira.CheckData{}) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) - Convey("Test zero notifications in db", func() { - addNotifications(dataBase, []moira.ScheduledNotification{}) - actual, err := dataBase.fetchNotificationsNoLimit(now + 6000) + Convey("Test with correct notifications", func() { + notifications := []*moira.ScheduledNotification{notification1, notification2, notification3} + triggerChecks, err := dataBase.getNotificationsTriggerChecks(notifications) So(err, ShouldBeNil) - So(actual, ShouldResemble, []*moira.ScheduledNotification{}) + So(triggerChecks, ShouldResemble, []*moira.CheckData{ + { + Timestamp: 1, + MetricsToTargetRelation: map[string]string{}, + }, + { + Timestamp: 1, + MetricsToTargetRelation: map[string]string{}, + }, + { + Timestamp: 2, + MetricsToTargetRelation: map[string]string{}, + }, + }) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) - Convey("Test all notifications with various ts in db", func() { - addNotifications(dataBase, []moira.ScheduledNotification{notification, notificationNew, notificationOld, notification4}) - actual, err := dataBase.fetchNotificationsNoLimit(now + 6000) + Convey("Test notifications with removed test1 trigger check", func() { + dataBase.RemoveTriggerLastCheck("test1") //nolint + defer func() { + _ = dataBase.SetTriggerLastCheck("test1", &moira.CheckData{ + Timestamp: 1, + }, moira.TriggerSourceNotSet) + }() + + notifications := []*moira.ScheduledNotification{notification1, notification2, notification3} + triggerChecks, err := dataBase.getNotificationsTriggerChecks(notifications) So(err, ShouldBeNil) - So(actual, ShouldResemble, []*moira.ScheduledNotification{¬ificationOld, ¬ification4, ¬ification, ¬ificationNew}) + So(triggerChecks, ShouldResemble, []*moira.CheckData{nil, nil, {Timestamp: 2, MetricsToTargetRelation: map[string]string{}}}) err = dataBase.RemoveAllNotifications() So(err, ShouldBeNil) }) + + Convey("Test notifications with removed all trigger checks", func() { + dataBase.RemoveTriggerLastCheck("test1") //nolint + dataBase.RemoveTriggerLastCheck("test2") //nolint + + notifications := []*moira.ScheduledNotification{notification1, notification2, notification3} + triggerChecks, err := dataBase.getNotificationsTriggerChecks(notifications) + So(err, ShouldBeNil) + So(triggerChecks, ShouldResemble, []*moira.CheckData{nil, nil, nil}) + + err = dataBase.RemoveAllNotifications() + So(err, ShouldBeNil) + }) + }) +} + +func TestRemoveNotifications(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + dataBase.Flush() + defer dataBase.Flush() + + client := dataBase.client + ctx := dataBase.context + pipe := (*client).TxPipeline() + + notification1 := &moira.ScheduledNotification{ + Timestamp: 1, + } + notification2 := &moira.ScheduledNotification{ + Timestamp: 2, + } + notification3 := &moira.ScheduledNotification{ + Timestamp: 3, + } + + Convey("Test removeNotifications", t, func() { + Convey("Test remove empty notifications", func() { + count, err := dataBase.removeNotifications(ctx, pipe, []*moira.ScheduledNotification{}) + So(err, ShouldBeNil) + So(count, ShouldEqual, 0) + + allNotifications, countAllNotifications, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(countAllNotifications, ShouldEqual, 0) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("Test remove one notification", func() { + addNotifications(dataBase, []moira.ScheduledNotification{*notification1, *notification2, *notification3}) + + count, err := dataBase.removeNotifications(ctx, pipe, []*moira.ScheduledNotification{notification2}) + So(err, ShouldBeNil) + So(count, ShouldEqual, 1) + + allNotifications, countAllNotifications, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(countAllNotifications, ShouldEqual, 2) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{notification1, notification3}) + }) + + Convey("Test remove all notifications", func() { + addNotifications(dataBase, []moira.ScheduledNotification{*notification1, *notification2, *notification3}) + + count, err := dataBase.removeNotifications(ctx, pipe, []*moira.ScheduledNotification{notification1, notification2, notification3}) + So(err, ShouldBeNil) + So(count, ShouldEqual, 3) + + allNotifications, countAllNotifications, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(countAllNotifications, ShouldEqual, 0) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{}) + }) + + Convey("Test remove a nonexistent notification", func() { + notification4 := &moira.ScheduledNotification{ + Timestamp: 4, + } + addNotifications(dataBase, []moira.ScheduledNotification{*notification1, *notification2, *notification3}) + + count, err := dataBase.removeNotifications(ctx, pipe, []*moira.ScheduledNotification{notification4}) + So(err, ShouldBeNil) + So(count, ShouldEqual, 0) + + allNotifications, countAllNotifications, err := dataBase.GetNotifications(0, -1) + So(err, ShouldBeNil) + So(countAllNotifications, ShouldEqual, 3) + So(allNotifications, ShouldResemble, []*moira.ScheduledNotification{notification1, notification2, notification3}) + }) }) } diff --git a/datatypes.go b/datatypes.go index 543dc88bc..51c15f9ab 100644 --- a/datatypes.go +++ b/datatypes.go @@ -245,6 +245,14 @@ type ScheduledNotification struct { CreatedAt int64 `json:"created_at" example:"1594471900" format:"int64"` } +type scheduledNotificationState int + +const ( + IgnoredNotification scheduledNotificationState = iota + ValidNotification + RemovedNotification +) + // Less is needed for the ScheduledNotification to match the Comparable interface func (notification *ScheduledNotification) Less(other Comparable) (bool, error) { otherNotification, ok := other.(*ScheduledNotification) @@ -255,6 +263,30 @@ func (notification *ScheduledNotification) Less(other Comparable) (bool, error) return notification.Timestamp < otherNotification.Timestamp, nil } +// IsDelayed checks if the notification is delayed, the difference between the send time and the create time +// is greater than the delayedTime +func (notification *ScheduledNotification) IsDelayed(delayedTime int64) bool { + return notification.CreatedAt != 0 && notification.Timestamp-notification.CreatedAt > delayedTime +} + +/* +GetState checks: + - If the trigger for which the notification was generated has been deleted, returns Removed state + - If the metric is on Maintenance, returns Ignored state + - If the trigger is on Maintenance, returns Ignored state + +Otherwise returns Valid state +*/ +func (notification *ScheduledNotification) GetState(triggerCheck *CheckData) scheduledNotificationState { + if triggerCheck == nil { + return RemovedNotification + } + if !triggerCheck.IsMetricOnMaintenance(notification.Event.Metric) && !triggerCheck.IsTriggerOnMaintenance() { + return ValidNotification + } + return IgnoredNotification +} + // MatchedMetric represents parsed and matched metric data type MatchedMetric struct { Metric string @@ -381,7 +413,7 @@ type CheckData struct { State State `json:"state" example:"OK"` Maintenance int64 `json:"maintenance,omitempty" example:"0" format:"int64"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` - // Timestamp - time, which means when the checker last checked this trigger, updated every checkInterval seconds + // Timestamp - time, which means when the checker last checked this trigger, this value stops updating if the trigger does not receive metrics Timestamp int64 `json:"timestamp,omitempty" example:"1590741916" format:"int64"` EventTimestamp int64 `json:"event_timestamp,omitempty" example:"1590741878" format:"int64"` // LastSuccessfulCheckTimestamp - time of the last check of the trigger, during which there were no errors @@ -401,6 +433,25 @@ func (checkData *CheckData) RemoveMetricsToTargetRelation() { checkData.MetricsToTargetRelation = make(map[string]string) } +// IsTriggerOnMaintenance checks if the trigger is on Maintenance +func (checkData *CheckData) IsTriggerOnMaintenance() bool { + return time.Now().Unix() <= checkData.Maintenance +} + +// IsMetricOnMaintenance checks if the metric of the given trigger is on Maintenance +func (checkData *CheckData) IsMetricOnMaintenance(metric string) bool { + if checkData.Metrics == nil { + return false + } + + metricState, ok := checkData.Metrics[metric] + if !ok { + return false + } + + return time.Now().Unix() <= metricState.Maintenance +} + // MetricState represents metric state data for given timestamp type MetricState struct { EventTimestamp int64 `json:"event_timestamp" example:"1590741878" format:"int64"` diff --git a/datatypes_test.go b/datatypes_test.go index f55c1d21b..2d7b33472 100644 --- a/datatypes_test.go +++ b/datatypes_test.go @@ -296,6 +296,53 @@ func TestScheduledNotification_GetKey(t *testing.T) { }) } +func TestScheduledNotification_GetState(t *testing.T) { + Convey("Test get state of scheduled notifications", t, func() { + notification := ScheduledNotification{ + Event: NotificationEvent{ + Metric: "test", + }, + } + + Convey("Get Removed state with nil check data", func() { + state := notification.GetState(nil) + So(state, ShouldEqual, RemovedNotification) + }) + + Convey("Get Ignored state with metric on maintenance", func() { + state := notification.GetState(&CheckData{ + Metrics: map[string]MetricState{ + "test": { + Maintenance: time.Now().Add(time.Hour).Unix(), + }, + }, + }) + So(state, ShouldEqual, IgnoredNotification) + }) + + Convey("Get Ignored state with trigger on maintenance", func() { + state := notification.GetState(&CheckData{ + Maintenance: time.Now().Add(time.Hour).Unix(), + }) + So(state, ShouldEqual, IgnoredNotification) + }) + + Convey("Get Valid state with trigger without metrics", func() { + state := notification.GetState(&CheckData{}) + So(state, ShouldEqual, ValidNotification) + }) + + Convey("Get Valid state with trigger with test metric", func() { + state := notification.GetState(&CheckData{ + Metrics: map[string]MetricState{ + "test": {}, + }, + }) + So(state, ShouldEqual, ValidNotification) + }) + }) +} + func TestCheckData_GetOrCreateMetricState(t *testing.T) { Convey("Test no metric", t, func() { checkData := CheckData{ @@ -373,6 +420,86 @@ func TestTrigger_IsSimple(t *testing.T) { }) } +func TestCheckData_IsTriggerOnMaintenance(t *testing.T) { + Convey("IsTriggerOnMaintenance manipulations", t, func() { + checkData := &CheckData{ + Maintenance: time.Now().Add(time.Hour).Unix(), + } + + Convey("Test with trigger check Maintenance more than time now", func() { + actual := checkData.IsTriggerOnMaintenance() + So(actual, ShouldBeTrue) + }) + + Convey("Test with trigger check Maintenance less than time now", func() { + checkData.Maintenance = time.Now().Add(-time.Hour).Unix() + defer func() { + checkData.Maintenance = time.Now().Add(time.Hour).Unix() + }() + + actual := checkData.IsTriggerOnMaintenance() + So(actual, ShouldBeFalse) + }) + }) +} + +func TestCheckData_IsMetricOnMaintenance(t *testing.T) { + Convey("isMetricOnMaintenance manipulations", t, func() { + checkData := &CheckData{ + Metrics: map[string]MetricState{ + "test1": { + Maintenance: time.Now().Add(time.Hour).Unix(), + }, + "test2": {}, + }, + } + + Convey("Test with a metric that is not in the trigger", func() { + actual := checkData.IsMetricOnMaintenance("") + So(actual, ShouldBeFalse) + }) + + Convey("Test with metrics that are in the trigger but not on maintenance", func() { + actual := checkData.IsMetricOnMaintenance("test2") + So(actual, ShouldBeFalse) + }) + + Convey("Test with metrics that are in the trigger and on maintenance", func() { + actual := checkData.IsMetricOnMaintenance("test1") + So(actual, ShouldBeTrue) + }) + + Convey("Test with the metric that is in the trigger, but the time now is more than Maintenance", func() { + metric := checkData.Metrics["test1"] + metric.Maintenance = time.Now().Add(-time.Hour).Unix() + checkData.Metrics["test1"] = metric + defer func() { + metric := checkData.Metrics["test1"] + metric.Maintenance = time.Now().Add(time.Hour).Unix() + checkData.Metrics["test1"] = metric + }() + + actual := checkData.IsMetricOnMaintenance("test1") + So(actual, ShouldBeFalse) + }) + + Convey("Test with trigger without metrics", func() { + checkData.Metrics = make(map[string]MetricState) + defer func() { + checkData.Metrics = map[string]MetricState{ + "test1": { + Maintenance: time.Now().Add(time.Hour).Unix(), + }, + "test2": {}, + } + }() + + actual := checkData.IsMetricOnMaintenance("test1") + So(actual, ShouldBeFalse) + }) + }) +} + func TestCheckData_GetEventTimestamp(t *testing.T) { Convey("Get event timestamp", t, func() { checkData := CheckData{Timestamp: 800, EventTimestamp: 0} @@ -750,3 +877,32 @@ func TestScheduledNotificationLess(t *testing.T) { }) }) } + +func TestScheduledNotificationIsDelayed(t *testing.T) { + Convey("Test Scheduled notification IsDelayed function", t, func() { + notification := &ScheduledNotification{ + Timestamp: 5, + } + + var delayedTime int64 = 2 + + Convey("Test notification with empty created at field", func() { + actual := notification.IsDelayed(delayedTime) + So(actual, ShouldBeFalse) + }) + + notification.CreatedAt = 1 + + Convey("Test notification which is to be defined as delayed", func() { + actual := notification.IsDelayed(delayedTime) + So(actual, ShouldBeTrue) + }) + + notification.CreatedAt = 4 + + Convey("Test notification which is to be defined as not delayed", func() { + actual := notification.IsDelayed(delayedTime) + So(actual, ShouldBeFalse) + }) + }) +}