Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(notifier): refactor selfstate heartbeaters #1117

Open
wants to merge 11 commits into
base: feat/add-emergency-contacts-allowed-list
Choose a base branch
from
73 changes: 45 additions & 28 deletions notifier/selfstate/heartbeat/database.go
Original file line number Diff line number Diff line change
@@ -1,52 +1,69 @@
package heartbeat

import (
"fmt"
"time"

"github.com/moira-alert/moira"
"github.com/moira-alert/moira/datatypes"
)

type databaseHeartbeat struct{ heartbeat }
// Verify that databaseHeartbeater matches the Heartbeater interface.
var _ Heartbeater = (*databaseHeartbeater)(nil)

func GetDatabase(delay int64, logger moira.Logger, database moira.Database) Heartbeater {
if delay > 0 {
return &databaseHeartbeat{heartbeat{
logger: logger,
database: database,
delay: delay,
lastSuccessfulCheck: time.Now().Unix(),
}}
// DatabaseHeartbeaterConfig structure describing the databaseHeartbeater configuration.
type DatabaseHeartbeaterConfig struct {
HeartbeaterBaseConfig

RedisDisconnectDelay time.Duration `validate:"required,gt=0"`
}

type databaseHeartbeater struct {
*heartbeaterBase

cfg DatabaseHeartbeaterConfig
}

// NewDatabaseHeartbeater is a function that creates a new databaseHeartbeater.
func NewDatabaseHeartbeater(cfg DatabaseHeartbeaterConfig, base *heartbeaterBase) (*databaseHeartbeater, error) {
AleksandrMatsko marked this conversation as resolved.
Show resolved Hide resolved
if err := moira.ValidateStruct(cfg); err != nil {
return nil, fmt.Errorf("database heartbeater configuration error: %w", err)
}
return nil

return &databaseHeartbeater{
heartbeaterBase: base,
cfg: cfg,
}, nil
}

func (check *databaseHeartbeat) Check(nowTS int64) (int64, bool, error) {
_, err := check.database.GetChecksUpdatesCount()
// Check is a function that checks if the database is working correctly.
func (heartbeater *databaseHeartbeater) Check() (State, error) {
now := heartbeater.clock.NowUTC()

_, err := heartbeater.database.GetChecksUpdatesCount()
if err == nil {
check.lastSuccessfulCheck = nowTS
return 0, false, nil
heartbeater.lastSuccessfulCheck = now
return StateOK, nil
}

if check.lastSuccessfulCheck < nowTS-check.delay {
check.logger.Error().
String("error", check.GetErrorMessage()).
Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck).
Msg("Send message")

return nowTS - check.lastSuccessfulCheck, true, nil
if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.RedisDisconnectDelay {
return StateError, nil
}

return 0, false, nil
return StateOK, err
}

func (databaseHeartbeat) NeedTurnOffNotifier() bool {
return true
// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off.
func (heartbeater databaseHeartbeater) NeedTurnOffNotifier() bool {
return heartbeater.cfg.NeedTurnOffNotifier
}

func (databaseHeartbeat) NeedToCheckOthers() bool {
return false
// Type is a function that returns the current heartbeat type.
func (databaseHeartbeater) Type() datatypes.HeartbeatType {
return datatypes.HeartbeatTypeNotSet
}

func (databaseHeartbeat) GetErrorMessage() string {
return "Redis disconnected"
// AlertSettings is a function that returns the current settings for alerts.
func (heartbeater databaseHeartbeater) AlertSettings() AlertConfig {
return heartbeater.cfg.AlertCfg
}
180 changes: 137 additions & 43 deletions notifier/selfstate/heartbeat/database_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,69 +5,163 @@ import (
"testing"
"time"

mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert"

logging "github.com/moira-alert/moira/logging/zerolog_adapter"
"github.com/go-playground/validator/v10"
"github.com/moira-alert/moira/datatypes"
. "github.com/smartystreets/goconvey/convey"
"go.uber.org/mock/gomock"
)

func TestDatabaseHeartbeat(t *testing.T) {
Convey("Test database heartbeat", t, func() {
now := time.Now().Unix()
err := errors.New("test database error")
check := createRedisDelayTest(t)
database := check.database.(*mock_moira_alert.MockDatabase)
const (
defaultRedisDisconnectDelay = time.Minute
)

func TestNewDatabaseHeartbeater(t *testing.T) {
_, _, _, heartbeaterBase := heartbeaterHelper(t)

validationErr := validator.ValidationErrors{}

Convey("Checking the created heartbeat database", func() {
expected := &databaseHeartbeat{heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}}
Convey("Test NewDatabaseHeartbeater", t, func() {
Convey("With too low redis disconnect delay", func() {
cfg := DatabaseHeartbeaterConfig{
HeartbeaterBaseConfig: HeartbeaterBaseConfig{
Enabled: true,
},
RedisDisconnectDelay: -1,
}

So(GetDatabase(0, check.logger, check.database), ShouldBeNil)
So(GetDatabase(1, check.logger, check.database), ShouldResemble, expected)
databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase)
So(errors.As(err, &validationErr), ShouldBeTrue)
So(databaseHeartbeater, ShouldBeNil)
})

Convey("Test update lastSuccessfulCheck", func() {
now += 1000
database.EXPECT().GetChecksUpdatesCount().Return(int64(1), nil)
Convey("Without redis disconnect delay", func() {
cfg := DatabaseHeartbeaterConfig{}

databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase)
So(errors.As(err, &validationErr), ShouldBeTrue)
So(databaseHeartbeater, ShouldBeNil)
})

Convey("With correct database heartbeater config", func() {
cfg := DatabaseHeartbeaterConfig{
RedisDisconnectDelay: 1,
}

expected := &databaseHeartbeater{
heartbeaterBase: heartbeaterBase,
cfg: cfg,
}

value, needSend, errActual := check.Check(now)
So(errActual, ShouldBeNil)
So(needSend, ShouldBeFalse)
So(value, ShouldEqual, 0)
So(check.lastSuccessfulCheck, ShouldResemble, now)
databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase)
So(err, ShouldBeNil)
So(databaseHeartbeater, ShouldResemble, expected)
})
})
}

func TestDatabaseHeartbeaterCheck(t *testing.T) {
database, clock, testTime, heartbeaterBase := heartbeaterHelper(t)

cfg := DatabaseHeartbeaterConfig{
RedisDisconnectDelay: defaultRedisDisconnectDelay,
}

Convey("Database error handling test", func() {
database.EXPECT().GetChecksUpdatesCount().Return(int64(1), err)
databaseHeartbeater, _ := NewDatabaseHeartbeater(cfg, heartbeaterBase)

value, needSend, errActual := check.Check(now)
So(errActual, ShouldBeNil)
So(needSend, ShouldBeFalse)
So(value, ShouldEqual, 0)
So(check.lastSuccessfulCheck, ShouldResemble, now)
var (
testErr = errors.New("test error")
checkUpdates int64
)

Convey("Test databaseHeartbeater.Check", t, func() {
Convey("With nil error in GetCheckUpdatedCount", func() {
database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, nil)
clock.EXPECT().NowUTC().Return(testTime)

state, err := databaseHeartbeater.Check()
So(state, ShouldResemble, StateOK)
So(err, ShouldBeNil)
})

Convey("Check for notification", func() {
check.lastSuccessfulCheck = now - check.delay - 1
Convey("With too much time elapsed since the last successful check", func() {
heartbeaterBase.lastSuccessfulCheck = testTime.Add(-10 * defaultRedisDisconnectDelay)
defer func() {
heartbeaterBase.lastSuccessfulCheck = testTime
}()

database.EXPECT().GetChecksUpdatesCount().Return(int64(0), err)
database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, testErr)
clock.EXPECT().NowUTC().Return(testTime)

value, needSend, errActual := check.Check(now)
So(errActual, ShouldBeNil)
So(needSend, ShouldBeTrue)
So(value, ShouldEqual, now-check.lastSuccessfulCheck)
state, err := databaseHeartbeater.Check()
So(state, ShouldResemble, StateError)
So(err, ShouldBeNil)
})

Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() {
So(check.NeedTurnOffNotifier(), ShouldBeTrue)
So(check.NeedToCheckOthers(), ShouldBeFalse)
Convey("With only error from GetChecksUpdateCount", func() {
database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, testErr)
clock.EXPECT().NowUTC().Return(testTime)

state, err := databaseHeartbeater.Check()
So(state, ShouldResemble, StateOK)
So(err, ShouldResemble, testErr)
})
})
}

func createRedisDelayTest(t *testing.T) *databaseHeartbeat {
mockCtrl := gomock.NewController(t)
logger, _ := logging.GetLogger("CheckDelay")
func TestDatabaseHeartbeaterNeedTurnOffNotifier(t *testing.T) {
_, _, _, heartbeaterBase := heartbeaterHelper(t)

Convey("Test databaseHeartbeater.TurnOffNotifier", t, func() {
cfg := DatabaseHeartbeaterConfig{
HeartbeaterBaseConfig: HeartbeaterBaseConfig{
NeedTurnOffNotifier: true,
},
RedisDisconnectDelay: defaultRedisDisconnectDelay,
}

return GetDatabase(10, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*databaseHeartbeat)
databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase)
So(err, ShouldBeNil)

needTurnOffNotifier := databaseHeartbeater.NeedTurnOffNotifier()
So(needTurnOffNotifier, ShouldBeTrue)
})
}

func TestDatabaseHeartbeaterType(t *testing.T) {
_, _, _, heartbeaterBase := heartbeaterHelper(t)

Convey("Test databaseHeartbeater.Type", t, func() {
cfg := DatabaseHeartbeaterConfig{
RedisDisconnectDelay: defaultRedisDisconnectDelay,
}

databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase)
So(err, ShouldBeNil)

databaseHeartbeaterType := databaseHeartbeater.Type()
So(databaseHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet)
})
}

func TestDatabaseHeartbeaterAlertSettings(t *testing.T) {
_, _, _, heartbeaterBase := heartbeaterHelper(t)

Convey("Test databaseHeartbeater.AlertSettings", t, func() {
alertCfg := AlertConfig{
Name: "test name",
Desc: "test desc",
}

cfg := DatabaseHeartbeaterConfig{
HeartbeaterBaseConfig: HeartbeaterBaseConfig{
AlertCfg: alertCfg,
},
RedisDisconnectDelay: defaultRedisDisconnectDelay,
}

databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase)
So(err, ShouldBeNil)

alertSettings := databaseHeartbeater.AlertSettings()
So(alertSettings, ShouldResemble, alertCfg)
})
}
Loading
Loading