Skip to content

Commit

Permalink
add heartbeat metrics to controller
Browse files Browse the repository at this point in the history
  • Loading branch information
@almostinf
almostinf committed Nov 6, 2024
1 parent b53b2eb commit 48a97cc
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 26 deletions.
3 changes: 2 additions & 1 deletion cmd/notifier/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,13 @@ func main() {
}

selfstateCfg := config.Notifier.Selfstate.getSettings()
heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(telemetry.Metrics)

// Start moira selfstate checker
if selfstateCfg.Enabled {
fmt.Println(selfstateCfg)
logger.Info().Msg("Selfstate enabled")
selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock)
selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock, heartbeatMetrics)
if err != nil {
logger.Fatal().
Error(err).
Expand Down
31 changes: 22 additions & 9 deletions notifier/selfstate/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"time"

"github.com/moira-alert/moira"
"github.com/moira-alert/moira/datatypes"
"github.com/moira-alert/moira/metrics"
"github.com/moira-alert/moira/notifier/selfstate/heartbeat"
w "github.com/moira-alert/moira/worker"
"gopkg.in/tomb.v2"
Expand Down Expand Up @@ -33,11 +35,12 @@ type ControllerConfig struct {
}

type controller struct {
cfg ControllerConfig
tomb tomb.Tomb
logger moira.Logger
database moira.Database
heartbeaters []heartbeat.Heartbeater
cfg ControllerConfig
tomb tomb.Tomb
logger moira.Logger
database moira.Database
heartbeaters []heartbeat.Heartbeater
heartbeatMetrics *metrics.HeartBeatMetrics
}

// NewController is a function to create a new selfstate controller.
Expand All @@ -46,6 +49,7 @@ func NewController(
logger moira.Logger,
database moira.Database,
clock moira.Clock,
heartbeatMetrics *metrics.HeartBeatMetrics,
) (*controller, error) {
if err := moira.ValidateStruct(cfg); err != nil {
return nil, fmt.Errorf("controller configuration error: %w", err)
Expand All @@ -54,10 +58,11 @@ func NewController(
heartbeaters := createHeartbeaters(cfg.HeartbeatersCfg, logger, database, clock)

return &controller{
cfg: cfg,
logger: logger,
database: database,
heartbeaters: heartbeaters,
cfg: cfg,
logger: logger,
database: database,
heartbeaters: heartbeaters,
heartbeatMetrics: heartbeatMetrics,
}, nil
}

Expand Down Expand Up @@ -177,6 +182,14 @@ func (c *controller) checkHeartbeats() {
Msg("Heartbeat check failed")
}

if heartbeater.Type() == datatypes.HeartbeatNotifier {
if heartbeatState == heartbeat.StateError {
c.heartbeatMetrics.MarkNotifierIsAlive(false)
} else {
c.heartbeatMetrics.MarkNotifierIsAlive(true)
}
}

if heartbeatState == heartbeat.StateError {
if err = c.database.SetNotifierState(moira.SelfStateERROR); err != nil {
c.logger.Error().
Expand Down
42 changes: 31 additions & 11 deletions notifier/selfstate/controller/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/moira-alert/moira"
"github.com/moira-alert/moira/datatypes"
logging "github.com/moira-alert/moira/logging/zerolog_adapter"
"github.com/moira-alert/moira/metrics"
mock_clock "github.com/moira-alert/moira/mock/clock"
mock_heartbeat "github.com/moira-alert/moira/mock/heartbeat"
mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert"
Expand All @@ -24,20 +25,23 @@ func TestNewController(t *testing.T) {
mockClock := mock_clock.NewMockClock(mockCtrl)
logger, _ := logging.GetLogger("Test")
testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC)
dummySource := metrics.NewDummyRegistry()
heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummySource)

Convey("Test NewController", t, func() {
mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()

Convey("With disabled config", func() {
cfg := ControllerConfig{}

c, err := NewController(cfg, logger, mockDatabase, mockClock)
c, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics)
So(err, ShouldBeNil)
So(c, ShouldResemble, &controller{
cfg: cfg,
logger: logger,
database: mockDatabase,
heartbeaters: make([]heartbeat.Heartbeater, 0),
cfg: cfg,
logger: logger,
database: mockDatabase,
heartbeaters: make([]heartbeat.Heartbeater, 0),
heartbeatMetrics: heartbeatMetrics,
})
})

Expand All @@ -46,7 +50,7 @@ func TestNewController(t *testing.T) {
Enabled: true,
}

c, err := NewController(cfg, logger, mockDatabase, mockClock)
c, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics)
So(err, ShouldNotBeNil)
So(c, ShouldBeNil)
})
Expand All @@ -64,7 +68,7 @@ func TestNewController(t *testing.T) {
},
}

c, err := NewController(cfg, logger, mockDatabase, mockClock)
c, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics)
So(err, ShouldNotBeNil)
So(c, ShouldBeNil)
})
Expand All @@ -83,7 +87,7 @@ func TestNewController(t *testing.T) {
CheckInterval: time.Minute,
}

_, err := NewController(cfg, logger, mockDatabase, mockClock)
_, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics)
So(err, ShouldBeNil)
})
})
Expand Down Expand Up @@ -274,6 +278,8 @@ func TestCheckHeartbeats(t *testing.T) {
logger, _ := logging.GetLogger("Test")
mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl)
testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC)
dummySource := metrics.NewDummyRegistry()
heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummySource)

Convey("Test checkHeartbeats", t, func() {
mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
Expand All @@ -283,22 +289,32 @@ func TestCheckHeartbeats(t *testing.T) {
notifierHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl)

c := &controller{
heartbeaters: []heartbeat.Heartbeater{databaseHeartbeater, localCheckerHeartbeater, notifierHeartbeater},
logger: logger,
database: mockDatabase,
heartbeaters: []heartbeat.Heartbeater{databaseHeartbeater, localCheckerHeartbeater, notifierHeartbeater},
logger: logger,
database: mockDatabase,
heartbeatMetrics: heartbeatMetrics,
}

Convey("Without error heartbeat states", func() {
databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil)
databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase)

localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil)
localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker)

notifierHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil)
notifierHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatNotifier)

c.checkHeartbeats()
})

Convey("With heartbeat error state", func() {
databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil)
databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase)

localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil)
localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker)

mockDatabase.EXPECT().SetNotifierState(moira.SelfStateERROR).Return(nil)

c.checkHeartbeats()
Expand All @@ -307,7 +323,11 @@ func TestCheckHeartbeats(t *testing.T) {
Convey("With heartbeat error state and error while set notifier state", func() {
dbErr := errors.New("test database error")
databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil)
databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase)

localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil)
localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker)

mockDatabase.EXPECT().SetNotifierState(moira.SelfStateERROR).Return(dbErr)
localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker)

Expand Down
7 changes: 5 additions & 2 deletions notifier/selfstate/selfstate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"

"github.com/moira-alert/moira"
"github.com/moira-alert/moira/metrics"
"github.com/moira-alert/moira/notifier"
"github.com/moira-alert/moira/notifier/selfstate/controller"
"github.com/moira-alert/moira/notifier/selfstate/monitor"
Expand All @@ -30,9 +31,10 @@ func NewSelfstateWorker(
database moira.Database,
notifier notifier.Notifier,
clock moira.Clock,
heartbeatMetrics *metrics.HeartBeatMetrics,
) (*selfstateWorker, error) {
monitors := createMonitors(cfg.MonitorCfg, logger, database, clock, notifier)
controller := createController(cfg.ControllerCfg, logger, database, clock)
controller := createController(cfg.ControllerCfg, logger, database, clock, heartbeatMetrics)

return &selfstateWorker{
monitors: monitors,
Expand Down Expand Up @@ -94,12 +96,13 @@ func createController(
logger moira.Logger,
database moira.Database,
clock moira.Clock,
heartbeatMetrics *metrics.HeartBeatMetrics,
) controller.Controller {
var c controller.Controller
var err error

if controllerCfg.Enabled {
c, err = controller.NewController(controllerCfg, logger, database, clock)
c, err = controller.NewController(controllerCfg, logger, database, clock, heartbeatMetrics)
if err != nil {
logger.Error().
Error(err).
Expand Down
11 changes: 8 additions & 3 deletions notifier/selfstate/selfstate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"go.uber.org/mock/gomock"

logging "github.com/moira-alert/moira/logging/zerolog_adapter"
"github.com/moira-alert/moira/metrics"
mock_clock "github.com/moira-alert/moira/mock/clock"
mock_controller "github.com/moira-alert/moira/mock/controller"
mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert"
Expand All @@ -27,6 +28,8 @@ func TestNewSelfstateWorker(t *testing.T) {
mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl)
logger, _ := logging.GetLogger("Test")
mockNotifier := mock_notifier.NewMockNotifier(mockCtrl)
dummyRegistry := metrics.NewDummyRegistry()
heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry)

cfg := Config{
Enabled: true,
Expand All @@ -35,7 +38,7 @@ func TestNewSelfstateWorker(t *testing.T) {
}

Convey("Test NewSelfstateWorker", t, func() {
worker, err := NewSelfstateWorker(cfg, logger, mockDatabase, mockNotifier, mockClock)
worker, err := NewSelfstateWorker(cfg, logger, mockDatabase, mockNotifier, mockClock, heartbeatMetrics)
So(err, ShouldBeNil)
So(worker.monitors, ShouldHaveLength, 0)
})
Expand Down Expand Up @@ -133,12 +136,14 @@ func TestCreateController(t *testing.T) {
mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl)
logger, _ := logging.GetLogger("Test")
testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC)
dummyRegistry := metrics.NewDummyRegistry()
heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry)

Convey("Test createController", t, func() {
mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()

Convey("With disabled controller", func() {
controller := createController(controller.ControllerConfig{}, logger, mockDatabase, mockClock)
controller := createController(controller.ControllerConfig{}, logger, mockDatabase, mockClock, heartbeatMetrics)
So(controller, ShouldBeNil)
})

Expand All @@ -156,7 +161,7 @@ func TestCreateController(t *testing.T) {
CheckInterval: time.Minute,
}

controller := createController(cfg, logger, mockDatabase, mockClock)
controller := createController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics)
So(controller, ShouldNotBeNil)
})
})
Expand Down

0 comments on commit 48a97cc

Please sign in to comment.