From ab16e551d5996853baa31c329e8b04131a4cb4f7 Mon Sep 17 00:00:00 2001 From: Aleksandr Matsko <90016901+AleksandrMatsko@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:43:56 +0700 Subject: [PATCH] feat(api): trigger noisiness (#1131) --- api/controller/paginate.go | 26 +++ api/controller/paginate_test.go | 122 ++++++++++ api/controller/triggers.go | 107 +++++++++ api/controller/triggers_test.go | 264 ++++++++++++++++++++++ api/dto/triggers.go | 23 ++ api/handler/constants.go | 7 + api/handler/event.go | 29 +-- api/handler/triggers.go | 48 ++++ api/handler/triggers_test.go | 92 ++++++++ api/handler/validate.go | 61 +++++ database/redis/notification_event.go | 4 +- database/redis/notification_event_test.go | 37 +-- interfaces.go | 2 +- mock/moira-alert/database.go | 8 +- notifier/scheduler.go | 5 +- notifier/scheduler_test.go | 15 +- 16 files changed, 799 insertions(+), 51 deletions(-) create mode 100644 api/controller/paginate.go create mode 100644 api/controller/paginate_test.go create mode 100644 api/handler/validate.go diff --git a/api/controller/paginate.go b/api/controller/paginate.go new file mode 100644 index 000000000..e40b7451a --- /dev/null +++ b/api/controller/paginate.go @@ -0,0 +1,26 @@ +package controller + +// applyPagination returns entities[page*size:(page+1)*size] if possible. +// If bad page and size are given or out of range than empty slice []T is returned. +func applyPagination[T any](page, size, total int64, entities []T) []T { + if page < 0 || (page > 0 && size < 0) { + return make([]T, 0) + } + + if page >= 0 && size >= 0 { + start := page * size + end := start + size + + if start >= total { + return make([]T, 0) + } else { + if end > total { + end = total + } + + return entities[start:end] + } + } + + return entities +} diff --git a/api/controller/paginate_test.go b/api/controller/paginate_test.go new file mode 100644 index 000000000..99d061dbe --- /dev/null +++ b/api/controller/paginate_test.go @@ -0,0 +1,122 @@ +package controller + +import ( + "fmt" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func Test_applyPaginate(t *testing.T) { + Convey("Test paginating", t, func() { + entities := make([]int, 0, 'z'-'a'+1) + for i := 0; i < 40; i++ { + entities = append(entities, i) + } + + type testcase struct { + page int64 + size int64 + total int64 + givenEntities []int + expectedEntities []int + desc string + } + + cases := []testcase{ + { + page: -1, + givenEntities: entities, + expectedEntities: []int{}, + desc: "with negative page", + }, + { + page: 1, + size: -1, + givenEntities: entities, + expectedEntities: []int{}, + desc: "with positive page and negative size", + }, + { + page: 1, + size: 10, + total: 7, + givenEntities: entities, + expectedEntities: []int{}, + desc: "out of range", + }, + { + page: 0, + size: -1, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: entities, + desc: "fetch all entities", + }, + { + page: 0, + size: -2, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: entities, + desc: "again fetch all entities", + }, + { + page: 0, + size: 7, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: []int{0, 1, 2, 3, 4, 5, 6}, + desc: "first page with size 7 (page = 0)", + }, + { + page: 1, + size: 7, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: []int{7, 8, 9, 10, 11, 12, 13}, + desc: "second page with size 7 (page = 1)", + }, + { + page: 2, + size: 7, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: []int{14, 15, 16, 17, 18, 19, 20}, + desc: "third page with size 7 (page = 2)", + }, + { + page: 3, + size: 7, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: []int{21, 22, 23, 24, 25, 26, 27}, + desc: "forth page with size 7 (page = 3)", + }, + { + page: 4, + size: 7, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: []int{28, 29, 30, 31, 32, 33, 34}, + desc: "fifth page with size 7 (page = 4)", + }, + { + page: 5, + size: 7, + total: int64(len(entities)), + givenEntities: entities, + expectedEntities: []int{35, 36, 37, 38, 39}, + desc: "last page with size 7 (page = 5)", + }, + } + + for i, tcase := range cases { + Convey(fmt.Sprintf("Case %v: %s", i+1, tcase.desc), func() { + gotEntities := applyPagination[int](tcase.page, tcase.size, tcase.total, tcase.givenEntities) + + So(gotEntities, ShouldResemble, tcase.expectedEntities) + }) + } + }) +} diff --git a/api/controller/triggers.go b/api/controller/triggers.go index de97e8d3e..b701ee547 100644 --- a/api/controller/triggers.go +++ b/api/controller/triggers.go @@ -5,6 +5,8 @@ import ( "fmt" "math" "regexp" + "slices" + "strings" "github.com/gofrs/uuid" @@ -213,3 +215,108 @@ func triggerExists(database moira.Database, triggerID string) (bool, error) { } return true, nil } + +// GetTriggerNoisiness get triggers with amount of events (within time range [from, to]) +// and sorts by events_count according to sortOrder. +func GetTriggerNoisiness( + database moira.Database, + page, size int64, + from, to string, + sortOrder api.SortOrder, +) (*dto.TriggerNoisinessList, *api.ErrorResponse) { + triggerIDs, err := database.GetAllTriggerIDs() + if err != nil { + return nil, api.ErrorInternalServer(err) + } + + triggerIDsWithEventsCount := getTriggerIDsWithEventsCount(database, triggerIDs, from, to) + + sortTriggerIDsByEventsCount(triggerIDsWithEventsCount, sortOrder) + + total := int64(len(triggerIDsWithEventsCount)) + + resDto := dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{}, + Page: page, + Size: size, + Total: total, + } + + triggerIDsWithEventsCount = applyPagination[triggerIDWithEventsCount](page, size, total, triggerIDsWithEventsCount) + if len(triggerIDsWithEventsCount) == 0 { + return &resDto, nil + } + + triggers, err := getTriggerChecks(database, onlyTriggerIDs(triggerIDsWithEventsCount)) + if err != nil { + return nil, api.ErrorInternalServer(err) + } + + if len(triggers) != len(triggerIDsWithEventsCount) { + return nil, api.ErrorInternalServer(fmt.Errorf("failed to fetch triggers for such range")) + } + + resDto.List = make([]dto.TriggerNoisiness, 0, len(triggers)) + for i := range triggers { + resDto.List = append(resDto.List, dto.TriggerNoisiness{ + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggers[i].Trigger), + Throttling: triggers[i].Throttling, + }, + EventsCount: triggerIDsWithEventsCount[i].eventsCount, + }) + } + + return &resDto, nil +} + +type triggerIDWithEventsCount struct { + triggerID string + eventsCount int64 +} + +func getTriggerIDsWithEventsCount( + database moira.Database, + triggerIDs []string, + from, to string, +) []triggerIDWithEventsCount { + resultTriggerIDs := make([]triggerIDWithEventsCount, 0, len(triggerIDs)) + + for _, triggerID := range triggerIDs { + eventsCount := database.GetNotificationEventCount(triggerID, from, to) + resultTriggerIDs = append(resultTriggerIDs, triggerIDWithEventsCount{ + triggerID: triggerID, + eventsCount: eventsCount, + }) + } + + return resultTriggerIDs +} + +func sortTriggerIDsByEventsCount(idsWithCount []triggerIDWithEventsCount, sortOrder api.SortOrder) { + if sortOrder == api.AscSortOrder || sortOrder == api.DescSortOrder { + slices.SortFunc(idsWithCount, func(first, second triggerIDWithEventsCount) int { + cmpRes := first.eventsCount - second.eventsCount + + if cmpRes == 0 { + return strings.Compare(first.triggerID, second.triggerID) + } + + if sortOrder == api.DescSortOrder { + cmpRes *= -1 + } + + return int(cmpRes) + }) + } +} + +func onlyTriggerIDs(idsWithCount []triggerIDWithEventsCount) []string { + triggerIDs := make([]string, 0, len(idsWithCount)) + + for _, idWithCount := range idsWithCount { + triggerIDs = append(triggerIDs, idWithCount.triggerID) + } + + return triggerIDs +} diff --git a/api/controller/triggers_test.go b/api/controller/triggers_test.go index a70ca9d35..6842c0c4d 100644 --- a/api/controller/triggers_test.go +++ b/api/controller/triggers_test.go @@ -1192,3 +1192,267 @@ func TestGetUnusedTriggerIDs(t *testing.T) { So(actual, ShouldResemble, expected) }) } + +func TestGetTriggerNoisiness(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) + + const ( + defaultFrom = "-inf" + defaultTo = "+inf" + zeroPage = int64(0) + allEventsSize = int64(-1) + ) + + Convey("TestGetTriggerNoisiness", t, func() { + Convey("with error from db", func() { + errFromDB := errors.New("some err") + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{}, errFromDB) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, allEventsSize, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldResemble, api.ErrorInternalServer(errFromDB)) + So(triggerNoisinessList, ShouldBeNil) + }) + + Convey("with no triggers", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{}, nil) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, allEventsSize, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{}, + Page: zeroPage, + Size: allEventsSize, + Total: 0, + }) + }) + + const ( + triggerID1 = "first-trigger" + triggerID2 = "second-trigger" + ) + + triggerCheck1 := moira.TriggerCheck{ + Trigger: moira.Trigger{ + ID: triggerID1, + }, + } + + triggerCheck2 := moira.TriggerCheck{ + Trigger: moira.Trigger{ + ID: triggerID2, + }, + } + + Convey("with triggers no events", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(0)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(0)) + dataBase.EXPECT().GetTriggerChecks([]string{triggerID1, triggerID2}). + Return([]*moira.TriggerCheck{&triggerCheck1, &triggerCheck2}, nil) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, allEventsSize, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{ + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck1.Trigger), + }, + EventsCount: 0, + }, + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck2.Trigger), + }, + EventsCount: 0, + }, + }, + Page: zeroPage, + Size: allEventsSize, + Total: 2, + }) + }) + + Convey("with triggers and some events", func() { + Convey("with asc sort order", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + dataBase.EXPECT().GetTriggerChecks([]string{triggerID1, triggerID2}). + Return([]*moira.TriggerCheck{&triggerCheck1, &triggerCheck2}, nil) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, allEventsSize, defaultFrom, defaultTo, api.AscSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{ + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck1.Trigger), + }, + EventsCount: 1, + }, + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck2.Trigger), + }, + EventsCount: 2, + }, + }, + Page: zeroPage, + Size: allEventsSize, + Total: 2, + }) + }) + + Convey("with desc sort order", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + dataBase.EXPECT().GetTriggerChecks([]string{triggerID2, triggerID1}). + Return([]*moira.TriggerCheck{&triggerCheck2, &triggerCheck1}, nil) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, allEventsSize, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{ + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck2.Trigger), + }, + EventsCount: 2, + }, + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck1.Trigger), + }, + EventsCount: 1, + }, + }, + Page: zeroPage, + Size: allEventsSize, + Total: 2, + }) + }) + }) + + Convey("with paginating", func() { + Convey("with page >= 0, size >= 0", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, 0, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{}, + Page: zeroPage, + Size: 0, + Total: 2, + }) + + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + dataBase.EXPECT().GetTriggerChecks([]string{triggerID2}). + Return([]*moira.TriggerCheck{&triggerCheck2}, nil) + + triggerNoisinessList, err = GetTriggerNoisiness(dataBase, zeroPage, 1, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{ + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck2.Trigger), + }, + EventsCount: 2, + }, + }, + Page: zeroPage, + Size: 1, + Total: 2, + }) + + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + dataBase.EXPECT().GetTriggerChecks([]string{triggerID1}). + Return([]*moira.TriggerCheck{&triggerCheck1}, nil) + + triggerNoisinessList, err = GetTriggerNoisiness(dataBase, 1, 1, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{ + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&triggerCheck1.Trigger), + }, + EventsCount: 1, + }, + }, + Page: 1, + Size: 1, + Total: 2, + }) + }) + + Convey("with page > 0, size < 0", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, 1, -1, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{}, + Page: 1, + Size: -1, + Total: 2, + }) + }) + + Convey("with page < 0, size < 0", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, -1, -1, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{}, + Page: -1, + Size: -1, + Total: 2, + }) + }) + + Convey("with page < 0, size > 0", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, -1, 1, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldBeNil) + So(triggerNoisinessList, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{}, + Page: -1, + Size: 1, + Total: 2, + }) + }) + }) + + Convey("error when len of fetched trigger checks != len of fetched ids", func() { + dataBase.EXPECT().GetAllTriggerIDs().Return([]string{triggerID1, triggerID2}, nil) + dataBase.EXPECT().GetNotificationEventCount(triggerID1, defaultFrom, defaultTo).Return(int64(1)) + dataBase.EXPECT().GetNotificationEventCount(triggerID2, defaultFrom, defaultTo).Return(int64(2)) + dataBase.EXPECT().GetTriggerChecks([]string{triggerID2, triggerID1}). + Return([]*moira.TriggerCheck{&triggerCheck2}, nil) + + triggerNoisinessList, err := GetTriggerNoisiness(dataBase, zeroPage, allEventsSize, defaultFrom, defaultTo, api.DescSortOrder) + So(err, ShouldResemble, api.ErrorInternalServer(fmt.Errorf("failed to fetch triggers for such range"))) + So(triggerNoisinessList, ShouldBeNil) + }) + }) +} diff --git a/api/dto/triggers.go b/api/dto/triggers.go index 46aa866b4..d0429b5be 100644 --- a/api/dto/triggers.go +++ b/api/dto/triggers.go @@ -562,3 +562,26 @@ type TriggersSearchResultDeleteResponse struct { func (TriggersSearchResultDeleteResponse) Render(http.ResponseWriter, *http.Request) error { return nil } + +// TriggerNoisiness represents TriggerCheck with amount of events for this trigger. +type TriggerNoisiness struct { + Trigger + // EventsCount for the trigger. + EventsCount int64 `json:"events_count"` +} + +func (*TriggerNoisiness) Render(http.ResponseWriter, *http.Request) error { + return nil +} + +// TriggerNoisinessList represents list of TriggerNoisiness. +type TriggerNoisinessList struct { + List []TriggerNoisiness `json:"list"` + Page int64 `json:"page" example:"0" format:"int64"` + Size int64 `json:"size" example:"100" format:"int64"` + Total int64 `json:"total" example:"10" format:"int64"` +} + +func (*TriggerNoisinessList) Render(http.ResponseWriter, *http.Request) error { + return nil +} diff --git a/api/handler/constants.go b/api/handler/constants.go index 4d2f55bad..80181e1d3 100644 --- a/api/handler/constants.go +++ b/api/handler/constants.go @@ -22,3 +22,10 @@ const ( getAllTeamsDefaultSize = -1 getAllTeamsDefaultRegexTemplate = ".*" ) + +const ( + getTriggerNoisinessDefaultPage = 0 + getTriggerNoisinessDefaultSize = -1 + getTriggerNoisinessDefaultFrom = "-3hour" + getTriggerNoisinessDefaultTo = "now" +) diff --git a/api/handler/event.go b/api/handler/event.go index 61ef68104..8eca5b6fc 100644 --- a/api/handler/event.go +++ b/api/handler/event.go @@ -4,10 +4,6 @@ import ( "fmt" "net/http" "regexp" - "strconv" - "time" - - "github.com/go-graphite/carbonapi/date" "github.com/go-chi/chi" "github.com/go-chi/render" @@ -53,22 +49,11 @@ func getEventsList(writer http.ResponseWriter, request *http.Request) { fromStr := middleware.GetFromStr(request) toStr := middleware.GetToStr(request) - if fromStr != "-inf" { - from := date.DateParamToEpoch(fromStr, "UTC", 0, time.UTC) - if from == 0 { - render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("can not parse from: %s", fromStr))) //nolint - return - } - fromStr = strconv.FormatInt(from, 10) - } - - if toStr != "+inf" { - to := date.DateParamToEpoch(toStr, "UTC", 0, time.UTC) - if to == 0 { - render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("can not parse to: %v", to))) //nolint - return - } - toStr = strconv.FormatInt(to, 10) + validator := DateRangeValidator{AllowInf: true} + fromStr, toStr, err := validator.ValidateDateRangeStrings(fromStr, toStr) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return } metricStr := middleware.GetMetric(request) @@ -80,9 +65,9 @@ func getEventsList(writer http.ResponseWriter, request *http.Request) { states := middleware.GetStates(request) - eventsList, err := controller.GetTriggerEvents(database, triggerID, page, size, fromStr, toStr, metricRegexp, states) + eventsList, errRsp := controller.GetTriggerEvents(database, triggerID, page, size, fromStr, toStr, metricRegexp, states) if err != nil { - render.Render(writer, request, err) //nolint + render.Render(writer, request, errRsp) //nolint return } if err := render.Render(writer, request, eventsList); err != nil { diff --git a/api/handler/triggers.go b/api/handler/triggers.go index d13e19bfb..f4c7581d4 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -32,6 +32,12 @@ func triggers(metricSourceProvider *metricSource.SourceProvider, searcher moira. router.With(middleware.AdminOnlyMiddleware()).Get("/", getAllTriggers) router.With(middleware.AdminOnlyMiddleware()).Get("/unused", getUnusedTriggers) + router.With( + middleware.AdminOnlyMiddleware(), + middleware.Paginate(getTriggerNoisinessDefaultPage, getTriggerNoisinessDefaultSize), + middleware.DateRange(getTriggerNoisinessDefaultFrom, getTriggerNoisinessDefaultTo), + middleware.SortOrderContext(api.DescSortOrder), + ).Get("/noisiness", getTriggerNoisiness) router.Put("/", createTrigger) router.Put("/check", triggerCheck) @@ -404,3 +410,45 @@ func getSearchRequestString(request *http.Request) string { searchText, _ = url.PathUnescape(searchText) return searchText } + +// nolint: gofmt,goimports +// +// @summary Get triggers noisiness +// @id get-triggers-noisiness +// @tags trigger +// @produce json +// @param size query int false "Number of items to be displayed on one page. if size = -1 then all events returned" default(100) +// @param p query int false "Defines the number of the displayed page. E.g, p=2 would display the 2nd page" default(0) +// @param from query string false "Start time of the time range" default(-3hours) +// @param to query string false "End time of the time range" default(now) +// @param sort query string false "String to set sort order (by events_count). On empty - no order, asc - ascending, desc - descending" default(desc) +// @success 200 {object} dto.TriggerNoisinessList "Get noisiness for triggers in range" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/noisiness [get] +func getTriggerNoisiness(writer http.ResponseWriter, request *http.Request) { + size := middleware.GetSize(request) + page := middleware.GetPage(request) + fromStr := middleware.GetFromStr(request) + toStr := middleware.GetToStr(request) + sort := middleware.GetSortOrder(request) + + validator := DateRangeValidator{AllowInf: true} + fromStr, toStr, err := validator.ValidateDateRangeStrings(fromStr, toStr) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + triggersNoisinessList, errorResponse := controller.GetTriggerNoisiness(database, page, size, fromStr, toStr, sort) + if errorResponse != nil { + render.Render(writer, request, errorResponse) //nolint + return + } + + if err := render.Render(writer, request, triggersNoisinessList); err != nil { + render.Render(writer, request, api.ErrorRender(err)) //nolint + return + } +} diff --git a/api/handler/triggers_test.go b/api/handler/triggers_test.go index ea558c2bf..c65101d77 100644 --- a/api/handler/triggers_test.go +++ b/api/handler/triggers_test.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -886,3 +888,93 @@ func isTriggerCreated(response *http.Response) bool { return actual.Message == expected } + +func TestGetTriggerNoisiness(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDB := mock_moira_alert.NewMockDatabase(mockCtrl) + database = mockDB + + getRequestTriggerNoisiness := func(from, to string) *http.Request { + request := httptest.NewRequest("GET", "/trigger/noisiness", nil) + request.Header.Add("content-type", "application/json") + + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "page", int64(0))) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "size", int64(-1))) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "from", from)) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "to", to)) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "sort", api.AscSortOrder)) + + return request + } + + testTriggerCheck := moira.TriggerCheck{ + Trigger: moira.Trigger{ + ID: "triggerID", + }, + } + + Convey("Test get trigger noisiness", t, func() { + now := time.Now() + + from := strconv.FormatInt(now.Add(time.Second*-3).Unix(), 10) + to := strconv.FormatInt(now.Unix(), 10) + + Convey("with ok", func() { + responseWriter := httptest.NewRecorder() + + mockDB.EXPECT().GetAllTriggerIDs().Return([]string{testTriggerCheck.ID}, nil) + mockDB.EXPECT().GetNotificationEventCount(testTriggerCheck.ID, from, to).Return(int64(1)) + mockDB.EXPECT().GetTriggerChecks([]string{testTriggerCheck.ID}).Return([]*moira.TriggerCheck{&testTriggerCheck}, nil) + + getTriggerNoisiness(responseWriter, getRequestTriggerNoisiness(from, to)) + + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + + var gotDTO dto.TriggerNoisinessList + err = json.Unmarshal(contentBytes, &gotDTO) + So(err, ShouldBeNil) + So(&gotDTO, ShouldResemble, &dto.TriggerNoisinessList{ + List: []dto.TriggerNoisiness{ + { + Trigger: dto.Trigger{ + TriggerModel: dto.CreateTriggerModel(&testTriggerCheck.Trigger), + }, + EventsCount: 1, + }, + }, + Page: 0, + Size: -1, + Total: 1, + }) + }) + + Convey("with error from db", func() { + responseWriter := httptest.NewRecorder() + errFromDB := errors.New("some DB error") + + mockDB.EXPECT().GetAllTriggerIDs().Return(nil, errFromDB) + + getTriggerNoisiness(responseWriter, getRequestTriggerNoisiness(from, to)) + + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + + expectedContentBytes, err := json.Marshal(api.ErrorInternalServer(errFromDB)) + So(err, ShouldBeNil) + So(string(contentBytes), ShouldResemble, string(expectedContentBytes)+"\n") + }) + }) +} diff --git a/api/handler/validate.go b/api/handler/validate.go new file mode 100644 index 000000000..85ad901ff --- /dev/null +++ b/api/handler/validate.go @@ -0,0 +1,61 @@ +package handler + +import ( + "fmt" + "strconv" + "time" + + "github.com/go-graphite/carbonapi/date" +) + +// DateRangeValidator for query parameters from and to. +type DateRangeValidator struct { + AllowInf bool +} + +// ValidateDateRangeStrings validates and converts both from and to query params. +// Returns converted from and to, or error if there was any. +// If AllowInf is true "-inf" is allowed for from, "+inf" for to. +func (d DateRangeValidator) ValidateDateRangeStrings(fromStr, toStr string) (string, string, error) { + fromStr, err := d.validateFromStr(fromStr) + if err != nil { + return "", "", err + } + + toStr, err = d.validateToStr(toStr) + if err != nil { + return "", "", err + } + + return fromStr, toStr, nil +} + +// validateFromStr by trying to parse date with carbonapi/date package. Also converts to proper format. +// If AllowInf is true, then "-inf" is also allowed. +func (d DateRangeValidator) validateFromStr(fromStr string) (string, error) { + if d.AllowInf && fromStr == "-inf" { + return fromStr, nil + } + + from := date.DateParamToEpoch(fromStr, "UTC", 0, time.UTC) + if from == 0 { + return "", fmt.Errorf("can not parse from: %s", fromStr) + } + + return strconv.FormatInt(from, 10), nil +} + +// validateToStr by trying to parse date with carbonapi/date package. Also converts to proper format. +// If AllowInf is true, then "+inf" is also allowed. +func (d DateRangeValidator) validateToStr(toStr string) (string, error) { + if d.AllowInf && toStr == "+inf" { + return toStr, nil + } + + to := date.DateParamToEpoch(toStr, "UTC", 0, time.UTC) + if to == 0 { + return "", fmt.Errorf("can not parse to: %v", to) + } + + return strconv.FormatInt(to, 10), nil +} diff --git a/database/redis/notification_event.go b/database/redis/notification_event.go index c094794e4..8b4053a1b 100644 --- a/database/redis/notification_event.go +++ b/database/redis/notification_event.go @@ -68,11 +68,11 @@ func (connector *DbConnector) PushNotificationEvent(event *moira.NotificationEve } // GetNotificationEventCount returns planned notifications count from given timestamp. -func (connector *DbConnector) GetNotificationEventCount(triggerID string, from int64) int64 { +func (connector *DbConnector) GetNotificationEventCount(triggerID string, from, to string) int64 { ctx := connector.context c := *connector.client - count, _ := c.ZCount(ctx, triggerEventsKey(triggerID), strconv.FormatInt(from, 10), "+inf").Result() + count, _ := c.ZCount(ctx, triggerEventsKey(triggerID), from, to).Result() return count } diff --git a/database/redis/notification_event_test.go b/database/redis/notification_event_test.go index 0266ed24e..cb9fc42de 100644 --- a/database/redis/notification_event_test.go +++ b/database/redis/notification_event_test.go @@ -19,11 +19,14 @@ const ( triggerID3 = "F0F4A5B9-637C-4933-AA0D-88B9798A2630" //nolint ) -var ( +const ( allTimeFrom = "-inf" allTimeTo = "+inf" - now = time.Now().Unix() - value = float64(0) +) + +var ( + now = time.Now().Unix() + value = float64(0) ) // nolint @@ -40,7 +43,7 @@ func TestNotificationEvents(t *testing.T) { So(err, ShouldBeNil) So(actual, ShouldResemble, make([]*moira.NotificationEvent, 0)) - total := dataBase.GetNotificationEventCount(triggerID, 0) + total := dataBase.GetNotificationEventCount(triggerID, allTimeFrom, allTimeTo) So(total, ShouldEqual, 0) actual1, err := dataBase.FetchNotificationEvent() @@ -73,7 +76,7 @@ func TestNotificationEvents(t *testing.T) { }, }) - total := dataBase.GetNotificationEventCount(triggerID, 0) + total := dataBase.GetNotificationEventCount(triggerID, allTimeFrom, allTimeTo) So(total, ShouldEqual, 1) actual1, err := dataBase.FetchNotificationEvent() @@ -102,7 +105,7 @@ func TestNotificationEvents(t *testing.T) { }, }) - total := dataBase.GetNotificationEventCount(triggerID, 0) + total := dataBase.GetNotificationEventCount(triggerID, allTimeFrom, allTimeTo) So(total, ShouldEqual, 1) }) @@ -148,7 +151,7 @@ func TestNotificationEvents(t *testing.T) { }, }) - total := dataBase.GetNotificationEventCount(triggerID1, 0) + total := dataBase.GetNotificationEventCount(triggerID1, allTimeFrom, allTimeTo) So(total, ShouldEqual, 1) actual, err = dataBase.GetNotificationEvents(triggerID2, 0, 1, allTimeFrom, allTimeTo) @@ -164,7 +167,7 @@ func TestNotificationEvents(t *testing.T) { }, }) - total = dataBase.GetNotificationEventCount(triggerID2, 0) + total = dataBase.GetNotificationEventCount(triggerID2, allTimeFrom, allTimeTo) So(total, ShouldEqual, 1) }) @@ -193,7 +196,7 @@ func TestNotificationEvents(t *testing.T) { }, }) - total := dataBase.GetNotificationEventCount(triggerID1, 0) + total := dataBase.GetNotificationEventCount(triggerID1, allTimeFrom, allTimeTo) So(total, ShouldEqual, 1) }) @@ -239,16 +242,16 @@ func TestNotificationEvents(t *testing.T) { }, }) - total := dataBase.GetNotificationEventCount(triggerID3, 0) + total := dataBase.GetNotificationEventCount(triggerID3, allTimeFrom, allTimeTo) So(total, ShouldEqual, 1) - total = dataBase.GetNotificationEventCount(triggerID3, now-1) + total = dataBase.GetNotificationEventCount(triggerID3, strconv.FormatInt(now-1, 10), allTimeTo) So(total, ShouldEqual, 1) - total = dataBase.GetNotificationEventCount(triggerID3, now) + total = dataBase.GetNotificationEventCount(triggerID3, strconv.FormatInt(now, 10), allTimeTo) So(total, ShouldEqual, 1) - total = dataBase.GetNotificationEventCount(triggerID3, now+1) + total = dataBase.GetNotificationEventCount(triggerID3, strconv.FormatInt(now+1, 10), allTimeTo) So(total, ShouldEqual, 0) actual, err = dataBase.GetNotificationEvents(triggerID3, 1, 1, allTimeFrom, allTimeTo) @@ -285,6 +288,9 @@ func TestNotificationEvents(t *testing.T) { actual, err := dataBase.GetNotificationEvents(triggerID3, 0, 1, strconv.FormatInt(now-2, 10), strconv.FormatInt(now-1, 10)) So(err, ShouldBeNil) So(actual, ShouldResemble, []*moira.NotificationEvent{}) + + eventCount := dataBase.GetNotificationEventCount(triggerID3, strconv.FormatInt(now-2, 10), strconv.FormatInt(now-1, 10)) + So(eventCount, ShouldEqual, 0) }) Convey("returns event in time range", func() { @@ -300,6 +306,9 @@ func TestNotificationEvents(t *testing.T) { Values: map[string]float64{}, }, }) + + eventCount := dataBase.GetNotificationEventCount(triggerID3, strconv.FormatInt(now-1, 10), strconv.FormatInt(now+1, 10)) + So(eventCount, ShouldEqual, 1) }) }) @@ -369,7 +378,7 @@ func TestNotificationEventErrorConnection(t *testing.T) { err = dataBase.PushNotificationEvent(&newNotificationEvent, true) So(err, ShouldNotBeNil) - total := dataBase.GetNotificationEventCount("123", 0) + total := dataBase.GetNotificationEventCount("123", allTimeFrom, allTimeFrom) So(total, ShouldEqual, 0) actual2, err := dataBase.FetchNotificationEvent() diff --git a/interfaces.go b/interfaces.go index fa4f4d751..1e935a9bf 100644 --- a/interfaces.go +++ b/interfaces.go @@ -62,7 +62,7 @@ type Database interface { // NotificationEvent storing GetNotificationEvents(triggerID string, page, size int64, from, to string) ([]*NotificationEvent, error) PushNotificationEvent(event *NotificationEvent, ui bool) error - GetNotificationEventCount(triggerID string, from int64) int64 + GetNotificationEventCount(triggerID string, from, to string) int64 FetchNotificationEvent() (NotificationEvent, error) RemoveAllNotificationEvents() error diff --git a/mock/moira-alert/database.go b/mock/moira-alert/database.go index 47bdf9228..da4befd6f 100644 --- a/mock/moira-alert/database.go +++ b/mock/moira-alert/database.go @@ -491,17 +491,17 @@ func (mr *MockDatabaseMockRecorder) GetMetricsValues(arg0, arg1, arg2 any) *gomo } // GetNotificationEventCount mocks base method. -func (m *MockDatabase) GetNotificationEventCount(arg0 string, arg1 int64) int64 { +func (m *MockDatabase) GetNotificationEventCount(arg0, arg1, arg2 string) int64 { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetNotificationEventCount", arg0, arg1) + ret := m.ctrl.Call(m, "GetNotificationEventCount", arg0, arg1, arg2) ret0, _ := ret[0].(int64) return ret0 } // GetNotificationEventCount indicates an expected call of GetNotificationEventCount. -func (mr *MockDatabaseMockRecorder) GetNotificationEventCount(arg0, arg1 any) *gomock.Call { +func (mr *MockDatabaseMockRecorder) GetNotificationEventCount(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotificationEventCount", reflect.TypeOf((*MockDatabase)(nil).GetNotificationEventCount), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotificationEventCount", reflect.TypeOf((*MockDatabase)(nil).GetNotificationEventCount), arg0, arg1, arg2) } // GetNotificationEvents mocks base method. diff --git a/notifier/scheduler.go b/notifier/scheduler.go index 446c1d376..b36ff3062 100644 --- a/notifier/scheduler.go +++ b/notifier/scheduler.go @@ -2,6 +2,7 @@ package notifier import ( "fmt" + "strconv" "time" "github.com/moira-alert/moira" @@ -84,6 +85,8 @@ func (scheduler *StandardScheduler) ScheduleNotification(params moira.SchedulerP return notification } +const allTimeTo = "+inf" + func (scheduler *StandardScheduler) calculateNextDelivery(now time.Time, event *moira.NotificationEvent, logger moira.Logger, ) (time.Time, bool) { @@ -124,7 +127,7 @@ func (scheduler *StandardScheduler) calculateNextDelivery(now time.Time, event * if from.Before(beginning) { from = beginning } - count := scheduler.database.GetNotificationEventCount(event.TriggerID, from.Unix()) + count := scheduler.database.GetNotificationEventCount(event.TriggerID, strconv.FormatInt(from.Unix(), 10), allTimeTo) if count >= level.count { next = now.Add(level.delay) logger.Debug(). diff --git a/notifier/scheduler_test.go b/notifier/scheduler_test.go index 79d3b9681..bd090835d 100644 --- a/notifier/scheduler_test.go +++ b/notifier/scheduler_test.go @@ -3,6 +3,7 @@ package notifier import ( "errors" "fmt" + "strconv" "testing" "time" @@ -97,8 +98,8 @@ func TestThrottling(t *testing.T) { systemClock.EXPECT().NowUTC().Return(now).Times(1) dataBase.EXPECT().GetTriggerThrottling(params2.Event.TriggerID).Return(now, now) dataBase.EXPECT().GetSubscription(*params2.Event.SubscriptionID).Return(subscription, nil) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Unix()).Return(int64(0)) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Unix()).Return(int64(0)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Unix(), 10), allTimeTo).Return(int64(0)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Unix(), 10), allTimeTo).Return(int64(0)) notification := scheduler.ScheduleNotification(params2, logger) So(notification, ShouldResemble, &expected2) @@ -273,8 +274,8 @@ func TestSubscriptionSchedule(t *testing.T) { Convey("Has trigger events count slightly less than low throttling level, should next timestamp now minutes, but throttling", func() { dataBase.EXPECT().GetTriggerThrottling(event.TriggerID).Return(time.Unix(0, 0), time.Unix(0, 0)) dataBase.EXPECT().GetSubscription(*event.SubscriptionID).Return(subscription, nil) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Add(-time.Hour*3).Unix()).Return(int64(13)) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Add(-time.Hour).Unix()).Return(int64(9)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Add(-time.Hour*3).Unix(), 10), allTimeTo).Return(int64(13)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Add(-time.Hour).Unix(), 10), allTimeTo).Return(int64(9)) next, throttled := scheduler.calculateNextDelivery(now, &event, logger) So(next, ShouldResemble, now) @@ -284,8 +285,8 @@ func TestSubscriptionSchedule(t *testing.T) { Convey("Has trigger events count event more than low throttling level, should next timestamp in 30 minutes", func() { dataBase.EXPECT().GetTriggerThrottling(event.TriggerID).Return(time.Unix(0, 0), time.Unix(0, 0)) dataBase.EXPECT().GetSubscription(*event.SubscriptionID).Return(subscription, nil) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Add(-time.Hour*3).Unix()).Return(int64(10)) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Add(-time.Hour).Unix()).Return(int64(10)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Add(-time.Hour*3).Unix(), 10), allTimeTo).Return(int64(10)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Add(-time.Hour).Unix(), 10), allTimeTo).Return(int64(10)) dataBase.EXPECT().SetTriggerThrottling(event.TriggerID, now.Add(time.Hour/2)).Return(nil) next, throttled := scheduler.calculateNextDelivery(now, &event, logger) @@ -296,7 +297,7 @@ func TestSubscriptionSchedule(t *testing.T) { Convey("Has trigger event more than high throttling level, should next timestamp in 1 hour", func() { dataBase.EXPECT().GetTriggerThrottling(event.TriggerID).Return(time.Unix(0, 0), time.Unix(0, 0)) dataBase.EXPECT().GetSubscription(*event.SubscriptionID).Return(subscription, nil) - dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, now.Add(-time.Hour*3).Unix()).Return(int64(20)) + dataBase.EXPECT().GetNotificationEventCount(event.TriggerID, strconv.FormatInt(now.Add(-time.Hour*3).Unix(), 10), allTimeTo).Return(int64(20)) dataBase.EXPECT().SetTriggerThrottling(event.TriggerID, now.Add(time.Hour)).Return(nil) next, throttled := scheduler.calculateNextDelivery(now, &event, logger)