From c17a2ebe9c9b6bc0e4d178e7ea89961d561eda84 Mon Sep 17 00:00:00 2001 From: Dmitry Redkin <48379797+Dimedrolity@users.noreply.github.com> Date: Tue, 8 Aug 2023 17:57:46 +0500 Subject: [PATCH 01/46] Refactored cluster specific code (#778) refactor(database): extracted method for redis --- database/redis/contact.go | 27 +++------- database/redis/database.go | 31 +++++++----- database/redis/last_check.go | 23 +-------- database/redis/metric.go | 97 +++--------------------------------- 4 files changed, 36 insertions(+), 142 deletions(-) diff --git a/database/redis/contact.go b/database/redis/contact.go index 5cce4f669..ebf5d735b 100644 --- a/database/redis/contact.go +++ b/database/redis/contact.go @@ -83,29 +83,18 @@ func getContactsKeysOnRedisNode(ctx context.Context, client redis.UniversalClien // GetAllContacts returns full contact list func (connector *DbConnector) GetAllContacts() ([]*moira.ContactData, error) { - client := *connector.client var keys []string - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - keysResult, err := getContactsKeysOnRedisNode(ctx, shard) - if err != nil { - return err - } - keys = append(keys, keysResult...) - return nil - }) - - if err != nil { - return nil, err - } - default: - keysResult, err := getContactsKeysOnRedisNode(connector.context, c) + err := connector.callFunc(func(connector *DbConnector, client redis.UniversalClient) error { + keysResult, err := getContactsKeysOnRedisNode(connector.context, client) if err != nil { - return nil, err + return err } - keys = keysResult + keys = append(keys, keysResult...) + return nil + }) + if err != nil { + return nil, err } contactIDs := make([]string, 0, len(keys)) diff --git a/database/redis/database.go b/database/redis/database.go index d72236963..6c7afc5e4 100644 --- a/database/redis/database.go +++ b/database/redis/database.go @@ -94,18 +94,11 @@ func NewTestDatabaseWithIncorrectConfig(logger moira.Logger) *DbConnector { // Flush deletes all the keys of the DB, use it only for tests func (connector *DbConnector) Flush() { - client := *connector.client - - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - return shard.FlushDB(ctx).Err() - }) - if err != nil { - return - } - default: - (*connector.client).FlushDB(connector.context) + err := connector.callFunc(func(connector *DbConnector, client redis.UniversalClient) error { + return client.FlushDB(connector.context).Err() + }) + if err != nil { + return } } @@ -126,3 +119,17 @@ func (connector *DbConnector) Client() redis.UniversalClient { func (connector *DbConnector) Context() context.Context { return connector.context } + +// callFunc calls the fn dependent of Redis client type (cluster or standalone). +func (connector *DbConnector) callFunc(fn func(connector *DbConnector, client redis.UniversalClient) error) error { + client := *connector.client + + switch c := client.(type) { + case *redis.ClusterClient: + return c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { + return fn(connector, shard) + }) + default: + return fn(connector, client) + } +} diff --git a/database/redis/last_check.go b/database/redis/last_check.go index 48c34f303..031d68e07 100644 --- a/database/redis/last_check.go +++ b/database/redis/last_check.go @@ -131,28 +131,7 @@ func cleanUpAbandonedTriggerLastCheckOnRedisNode(connector *DbConnector, client // CleanUpAbandonedTriggerLastCheck cleans up abandoned triggers last check. func (connector *DbConnector) CleanUpAbandonedTriggerLastCheck() error { - client := *connector.client - - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - err := cleanUpAbandonedTriggerLastCheckOnRedisNode(connector, shard) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - default: - err := cleanUpAbandonedTriggerLastCheckOnRedisNode(connector, c) - if err != nil { - return err - } - } - - return nil + return connector.callFunc(cleanUpAbandonedTriggerLastCheckOnRedisNode) } // SetTriggerCheckMaintenance sets maintenance for whole trigger and to given metrics, diff --git a/database/redis/metric.go b/database/redis/metric.go index 2379189a2..cdf3e6e15 100644 --- a/database/redis/metric.go +++ b/database/redis/metric.go @@ -1,7 +1,6 @@ package redis import ( - "context" "encoding/json" "errors" "fmt" @@ -386,58 +385,18 @@ func cleanUpAbandonedRetentionsOnRedisNode(connector *DbConnector, client redis. } func (connector *DbConnector) CleanUpOutdatedMetrics(duration time.Duration) error { - client := *connector.client - if duration >= 0 { return errors.New("clean up duration value must be less than zero, otherwise all metrics will be removed") } - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - err := cleanUpOutdatedMetricsOnRedisNode(connector, shard, duration) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - default: - err := cleanUpOutdatedMetricsOnRedisNode(connector, c, duration) - if err != nil { - return err - } - } - - return nil + return connector.callFunc(func(connector *DbConnector, client redis.UniversalClient) error { + return cleanUpOutdatedMetricsOnRedisNode(connector, client, duration) + }) } // CleanUpAbandonedRetentions removes metric retention keys that have no corresponding metric data. func (connector *DbConnector) CleanUpAbandonedRetentions() error { - client := *connector.client - - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - err := cleanUpAbandonedRetentionsOnRedisNode(connector, shard) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - default: - err := cleanUpAbandonedRetentionsOnRedisNode(connector, c) - if err != nil { - return err - } - } - - return nil + return connector.callFunc(cleanUpAbandonedRetentionsOnRedisNode) } func removeMetricsByPrefixOnRedisNode(connector *DbConnector, client redis.UniversalClient, prefix string) error { @@ -476,28 +435,9 @@ func removeMetricsByPrefixOnRedisNode(connector *DbConnector, client redis.Unive // RemoveMetricsByPrefix removes metrics by their prefix e.g. "my.super.metric.". func (connector *DbConnector) RemoveMetricsByPrefix(prefix string) error { - client := *connector.client - - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - err := removeMetricsByPrefixOnRedisNode(connector, shard, prefix) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - default: - err := removeMetricsByPrefixOnRedisNode(connector, c, prefix) - if err != nil { - return err - } - } - - return nil + return connector.callFunc(func(connector *DbConnector, client redis.UniversalClient) error { + return removeMetricsByPrefixOnRedisNode(connector, client, prefix) + }) } func removeAllMetricsOnRedisNode(connector *DbConnector, client redis.UniversalClient) error { @@ -530,28 +470,7 @@ func removeAllMetricsOnRedisNode(connector *DbConnector, client redis.UniversalC // RemoveAllMetrics removes all metrics. func (connector *DbConnector) RemoveAllMetrics() error { - client := *connector.client - - switch c := client.(type) { - case *redis.ClusterClient: - err := c.ForEachMaster(connector.context, func(ctx context.Context, shard *redis.Client) error { - err := removeAllMetricsOnRedisNode(connector, shard) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - default: - err := removeAllMetricsOnRedisNode(connector, c) - if err != nil { - return err - } - } - - return nil + return connector.callFunc(removeAllMetricsOnRedisNode) } func flushMetric(database moira.Database, metric string, duration time.Duration) (int64, error) { From 4b24fdc4620aa5568eace34eaa75174c221f3db6 Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Thu, 10 Aug 2023 12:53:16 +0300 Subject: [PATCH 02/46] fix(api): add the parse query function instead of query (#889) --- api/handler/notification.go | 22 ++- api/handler/notification_test.go | 115 +++++++++++++++ api/handler/trigger_metric_test.go | 64 +++++++++ api/handler/trigger_metrics.go | 13 +- api/handler/trigger_render.go | 23 ++- api/handler/trigger_render_test.go | 130 +++++++++++++++++ api/middleware/context.go | 50 +++++-- api/middleware/context_test.go | 219 +++++++++++++++++++++++++++++ go.sum | 3 - 9 files changed, 621 insertions(+), 18 deletions(-) create mode 100644 api/handler/notification_test.go create mode 100644 api/handler/trigger_metric_test.go create mode 100644 api/handler/trigger_render_test.go create mode 100644 api/middleware/context_test.go diff --git a/api/handler/notification.go b/api/handler/notification.go index 856a67afe..1ab60441f 100644 --- a/api/handler/notification.go +++ b/api/handler/notification.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "net/http" + "net/url" "strconv" "github.com/go-chi/chi" @@ -18,11 +19,18 @@ func notification(router chi.Router) { } func getNotification(writer http.ResponseWriter, request *http.Request) { - start, err := strconv.ParseInt(request.URL.Query().Get("start"), 10, 64) + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + start, err := strconv.ParseInt(urlValues.Get("start"), 10, 64) if err != nil { start = 0 } - end, err := strconv.ParseInt(request.URL.Query().Get("end"), 10, 64) + + end, err := strconv.ParseInt(urlValues.Get("end"), 10, 64) if err != nil { end = -1 } @@ -32,13 +40,20 @@ func getNotification(writer http.ResponseWriter, request *http.Request) { render.Render(writer, request, errorResponse) //nolint return } + if err := render.Render(writer, request, notifications); err != nil { render.Render(writer, request, api.ErrorRender(err)) //nolint } } func deleteNotification(writer http.ResponseWriter, request *http.Request) { - notificationKey := request.URL.Query().Get("id") + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + notificationKey := urlValues.Get("id") if notificationKey == "" { render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("notification id can not be empty"))) //nolint return @@ -49,6 +64,7 @@ func deleteNotification(writer http.ResponseWriter, request *http.Request) { render.Render(writer, request, errorResponse) //nolint return } + if err := render.Render(writer, request, notifications); err != nil { render.Render(writer, request, api.ErrorRender(err)) //nolint } diff --git a/api/handler/notification_test.go b/api/handler/notification_test.go new file mode 100644 index 000000000..f10200da0 --- /dev/null +++ b/api/handler/notification_test.go @@ -0,0 +1,115 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/moira-alert/moira" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" +) + +func TestGetNotifications(t *testing.T) { + Convey("test get notifications url parameters", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + Convey("with the correct parameters", func() { + parameters := []string{"start=0&end=100", "start=0", "end=100", "", "start=test&end=100", "start=0&end=test"} + for _, param := range parameters { + mockDb.EXPECT().GetNotifications(gomock.Any(), gomock.Any()).Return([]*moira.ScheduledNotification{}, int64(0), nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodGet, "/notifications?"+param, nil) + + getNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + } + }) + + Convey("with the wrong url query string", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/notifications?start=test%&end=100", nil) + + getNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Invalid request","error":"invalid URL escape \"%\""} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestDeleteNotifications(t *testing.T) { + Convey("test delete notifications url parameters", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + Convey("with the empty id parameter", func() { + testRequest := httptest.NewRequest(http.MethodDelete, `/notifications`, nil) + + deleteNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Invalid request","error":"notification id can not be empty"} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("with the correct id", func() { + mockDb.EXPECT().RemoveNotification(gomock.Any()).Return(int64(0), nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, `/notifications?id=test`, nil) + + deleteNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"result":0} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("with the wrong url query string", func() { + testRequest := httptest.NewRequest(http.MethodDelete, `/notifications?id=test%&`, nil) + + deleteNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Invalid request","error":"invalid URL escape \"%\""} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} diff --git a/api/handler/trigger_metric_test.go b/api/handler/trigger_metric_test.go new file mode 100644 index 000000000..8a727ab31 --- /dev/null +++ b/api/handler/trigger_metric_test.go @@ -0,0 +1,64 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/api/middleware" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" +) + +func TestDeleteTriggerMetric(t *testing.T) { + Convey("Delete metric by name", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + Convey("with the correct name", func() { + mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ID: "triggerID-0000000000001"}, nil).Times(1) + mockDb.EXPECT().AcquireTriggerCheckLock(gomock.Any(), gomock.Any()).Return(nil).Times(1) + mockDb.EXPECT().GetTriggerLastCheck(gomock.Any()).Return(moira.CheckData{}, nil).Times(1) + mockDb.EXPECT().DeleteTriggerCheckLock(gomock.Any()).Return(nil).Times(1) + mockDb.EXPECT().RemovePatternsMetrics(gomock.Any()).Return(nil).Times(1) + mockDb.EXPECT().SetTriggerLastCheck(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/trigger/triggerID-0000000000001/metrics?name=test", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "triggerID", "triggerID-0000000000001")) + + deleteTriggerMetric(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + + So(contents, ShouldEqual, "") + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("with wrong name", func() { + testRequest := httptest.NewRequest(http.MethodDelete, "/trigger/triggerID-0000000000001/metrics?name=test%name", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "triggerID", "triggerID-0000000000001")) + testRequest.Header.Add("content-type", "application/json") + + deleteTriggerMetric(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Invalid request","error":"invalid URL escape \"%na\""} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} diff --git a/api/handler/trigger_metrics.go b/api/handler/trigger_metrics.go index 6cf56050a..d48360b29 100644 --- a/api/handler/trigger_metrics.go +++ b/api/handler/trigger_metrics.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "net/http" + "net/url" "time" "github.com/go-chi/chi" @@ -30,16 +31,19 @@ func getTriggerMetrics(writer http.ResponseWriter, request *http.Request) { render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("can not parse from: %s", fromStr))) //nolint return } + to := date.DateParamToEpoch(toStr, "UTC", 0, time.UTC) if to == 0 { render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("can not parse to: %v", to))) //nolint return } + triggerMetrics, err := controller.GetTriggerMetrics(database, metricSourceProvider, from, to, triggerID) if err != nil { render.Render(writer, request, err) //nolint return } + if err := render.Render(writer, request, triggerMetrics); err != nil { render.Render(writer, request, api.ErrorRender(err)) //nolint } @@ -47,7 +51,14 @@ func getTriggerMetrics(writer http.ResponseWriter, request *http.Request) { func deleteTriggerMetric(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) - metricName := request.URL.Query().Get("name") + + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + metricName := urlValues.Get("name") if err := controller.DeleteTriggerMetric(database, metricName, triggerID); err != nil { render.Render(writer, request, err) //nolint } diff --git a/api/handler/trigger_render.go b/api/handler/trigger_render.go index 99f38a72f..c2e961211 100644 --- a/api/handler/trigger_render.go +++ b/api/handler/trigger_render.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "net/http" + "net/url" "strconv" "time" @@ -57,23 +58,31 @@ func getEvaluationParameters(request *http.Request) (sourceProvider *metricSourc fromStr := middleware.GetFromStr(request) toStr := middleware.GetToStr(request) from = date.DateParamToEpoch(fromStr, "UTC", 0, time.UTC) + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + return sourceProvider, "", 0, 0, "", false, fmt.Errorf("failed to parse query string: %w", err) + } if from == 0 { return sourceProvider, "", 0, 0, "", false, fmt.Errorf("can not parse from: %s", fromStr) } from -= from % 60 //nolint + to = date.DateParamToEpoch(toStr, "UTC", 0, time.UTC) if to == 0 { return sourceProvider, "", 0, 0, "", false, fmt.Errorf("can not parse to: %s", fromStr) } - realtime := request.URL.Query().Get("realtime") + + realtime := urlValues.Get("realtime") if realtime == "" { return } + fetchRealtimeData, err = strconv.ParseBool(realtime) if err != nil { return sourceProvider, "", 0, 0, "", false, fmt.Errorf("invalid realtime param: %s", err.Error()) } + return } @@ -87,19 +96,27 @@ func evaluateTargetMetrics(metricSourceProvider *metricSource.SourceProvider, fr } func buildRenderable(request *http.Request, trigger *moira.Trigger, metricsData []metricSource.MetricData, targetName string) (*chart.Chart, error) { - timezone := request.URL.Query().Get("timezone") + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + return nil, fmt.Errorf("failed to parse query string: %w", err) + } + + timezone := urlValues.Get("timezone") location, err := time.LoadLocation(timezone) if err != nil { return nil, fmt.Errorf("failed to load %s timezone: %s", timezone, err.Error()) } - plotTheme := request.URL.Query().Get("theme") + + plotTheme := urlValues.Get("theme") plotTemplate, err := plotting.GetPlotTemplate(plotTheme, location) if err != nil { return nil, fmt.Errorf("can not initialize plot theme %s", err.Error()) } + renderable, err := plotTemplate.GetRenderable(targetName, trigger, metricsData) if err != nil { return nil, err } + return &renderable, err } diff --git a/api/handler/trigger_render_test.go b/api/handler/trigger_render_test.go new file mode 100644 index 000000000..1f7621e85 --- /dev/null +++ b/api/handler/trigger_render_test.go @@ -0,0 +1,130 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/api/middleware" + metricSource "github.com/moira-alert/moira/metric_source" + mock_metric_source "github.com/moira-alert/moira/mock/metric_source" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" +) + +func TestRenderTrigger(t *testing.T) { + Convey("Checking the correctness of parameters", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + localSource := mock_metric_source.NewMockMetricSource(mockCtrl) + remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + Convey("with the wrong realtime parameter", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/trigger/triggerID-0000000000001/render?realtime=test", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "triggerID", "triggerID-0000000000001")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "metricSourceProvider", sourceProvider)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "target", "t1")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "from", "-1hour")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "to", "now")) + + renderTrigger(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Invalid request","error":"invalid realtime param: strconv.ParseBool: parsing \"test\": invalid syntax"} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("with the wrong timezone parameter", func() { + mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ID: "triggerID-0000000000001", Targets: []string{"t1"}}, nil).Times(1) + localSource.EXPECT().IsConfigured().Return(true, nil).AnyTimes().Times(1) + fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) + fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData("", []float64{}, 0, 0)}).Times(1) + localSource.EXPECT().Fetch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fetchResult, nil).Times(1) + + database = mockDb + + testRequest := httptest.NewRequest(http.MethodGet, "/trigger/triggerID-0000000000001/render?timezone=test", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "triggerID", "triggerID-0000000000001")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "metricSourceProvider", sourceProvider)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "target", "t1")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "from", "-1hour")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "to", "now")) + + renderTrigger(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Internal Server Error","error":"failed to load test timezone: unknown time zone test"} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + + Convey("without points for render", func() { + mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ID: "triggerID-0000000000001", Targets: []string{"t1"}}, nil).Times(1) + localSource.EXPECT().IsConfigured().Return(true, nil).Times(1) + fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) + fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData("", []float64{}, 0, 0)}).Times(1) + localSource.EXPECT().Fetch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fetchResult, nil).Times(1) + + database = mockDb + + testRequest := httptest.NewRequest(http.MethodGet, "/trigger/triggerID-0000000000001/render", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "triggerID", "triggerID-0000000000001")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "metricSourceProvider", sourceProvider)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "target", "t1")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "from", "-1hour")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "to", "now")) + + renderTrigger(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Internal Server Error","error":"no points found to render trigger: triggerID-0000000000001"} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + + Convey("with the wrong query string", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/trigger/triggerID-0000000000001/render?realtime=test%rt", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "triggerID", "triggerID-0000000000001")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "metricSourceProvider", sourceProvider)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "target", "t1")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "from", "-1hour")) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), "to", "now")) + + renderTrigger(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + expected := `{"status":"Invalid request","error":"failed to parse query string: invalid URL escape \"%rt\""} +` + + So(contents, ShouldEqual, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} diff --git a/api/middleware/context.go b/api/middleware/context.go index e0e6a07b4..9fe1fd266 100644 --- a/api/middleware/context.go +++ b/api/middleware/context.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "time" @@ -109,11 +110,18 @@ func MetricSourceProvider(sourceProvider *metricSource.SourceProvider) func(next func Paginate(defaultPage, defaultSize int64) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - page, err := strconv.ParseInt(request.URL.Query().Get("p"), 10, 64) + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + page, err := strconv.ParseInt(urlValues.Get("p"), 10, 64) if err != nil { page = defaultPage } - size, err := strconv.ParseInt(request.URL.Query().Get("size"), 10, 64) + + size, err := strconv.ParseInt(urlValues.Get("size"), 10, 64) if err != nil { size = defaultSize } @@ -129,12 +137,18 @@ func Paginate(defaultPage, defaultSize int64) func(next http.Handler) http.Handl func Pager(defaultCreatePager bool, defaultPagerID string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - pagerID := request.URL.Query().Get("pagerID") + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + pagerID := urlValues.Get("pagerID") if pagerID == "" { pagerID = defaultPagerID } - createPager, err := strconv.ParseBool(request.URL.Query().Get("createPager")) + createPager, err := strconv.ParseBool(urlValues.Get("createPager")) if err != nil { createPager = defaultCreatePager } @@ -150,7 +164,13 @@ func Pager(defaultCreatePager bool, defaultPagerID string) func(next http.Handle func Populate(defaultPopulated bool) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - populate, err := strconv.ParseBool(request.URL.Query().Get("populated")) + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + populate, err := strconv.ParseBool(urlValues.Get("populated")) if err != nil { populate = defaultPopulated } @@ -176,11 +196,18 @@ func Triggers(LocalMetricTTL, RemoteMetricTTL time.Duration) func(next http.Hand func DateRange(defaultFrom, defaultTo string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - from := request.URL.Query().Get("from") + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + from := urlValues.Get("from") if from == "" { from = defaultFrom } - to := request.URL.Query().Get("to") + + to := urlValues.Get("to") if to == "" { to = defaultTo } @@ -196,10 +223,17 @@ func DateRange(defaultFrom, defaultTo string) func(next http.Handler) http.Handl func TargetName(defaultTargetName string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - targetName := request.URL.Query().Get("target") + urlValues, err := url.ParseQuery(request.URL.RawQuery) + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } + + targetName := urlValues.Get("target") if targetName == "" { targetName = defaultTargetName } + ctx := context.WithValue(request.Context(), targetNameKey, targetName) next.ServeHTTP(writer, request.WithContext(ctx)) }) diff --git a/api/middleware/context_test.go b/api/middleware/context_test.go new file mode 100644 index 000000000..5535c6c70 --- /dev/null +++ b/api/middleware/context_test.go @@ -0,0 +1,219 @@ +package middleware_test + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/moira-alert/moira/api/middleware" + . "github.com/smartystreets/goconvey/convey" +) + +const expectedBadRequest = `{"status":"Invalid request","error":"invalid URL escape \"%\""} +` + +func TestPaginateMiddleware(t *testing.T) { + Convey("checking correctness of parameters", t, func() { + responseWriter := httptest.NewRecorder() + defaultPage := int64(1) + defaultSize := int64(10) + + Convey("with correct parameters", func() { + parameters := []string{"p=0&size=100", "p=0", "size=100", "", "p=test&size=100", "p=0&size=test"} + + for _, param := range parameters { + testRequest := httptest.NewRequest(http.MethodGet, "/test?"+param, nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.Paginate(defaultPage, defaultSize) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + } + }) + + Convey("with wrong url query parameters", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?p=0%&size=100", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.Paginate(defaultPage, defaultSize) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + + So(contents, ShouldEqual, expectedBadRequest) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestPagerMiddleware(t *testing.T) { + Convey("checking correctness of parameters", t, func() { + responseWriter := httptest.NewRecorder() + defaultCreatePager := false + defaultPagerID := "test" + + Convey("with correct parameters", func() { + parameters := []string{"pagerID=test&createPager=true", "pagerID=test", "createPager=true", "", "pagerID=-1&createPager=true", "pagerID=test&createPager=-1"} + + for _, param := range parameters { + testRequest := httptest.NewRequest(http.MethodGet, "/test?"+param, nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.Pager(defaultCreatePager, defaultPagerID) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + } + }) + + Convey("with wrong url query parameters", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?pagerID=test%&createPager=true", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.Pager(defaultCreatePager, defaultPagerID) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + + So(contents, ShouldEqual, expectedBadRequest) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestPopulateMiddleware(t *testing.T) { + Convey("checking correctness of parameter", t, func() { + responseWriter := httptest.NewRecorder() + defaultPopulated := false + + Convey("with correct parameter", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?populated=true", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.Populate(defaultPopulated) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("with wrong url query parameter", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?populated%=true", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.Populate(defaultPopulated) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + + So(contents, ShouldEqual, expectedBadRequest) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestDateRangeMiddleware(t *testing.T) { + Convey("checking correctness of parameters", t, func() { + responseWriter := httptest.NewRecorder() + defaultFrom := "-1hour" + defaultTo := "now" + + Convey("with correct parameters", func() { + parameters := []string{"from=-2hours&to=now", "from=-2hours", "to=now", "", "from=-2&to=now", "from=-2hours&to=-1"} + + for _, param := range parameters { + testRequest := httptest.NewRequest(http.MethodGet, "/test?"+param, nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.DateRange(defaultFrom, defaultTo) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + } + }) + + Convey("with wrong url query parameters", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?from=-2hours%&to=now", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.DateRange(defaultFrom, defaultTo) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + + So(contents, ShouldEqual, expectedBadRequest) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestTargetNameMiddleware(t *testing.T) { + Convey("checking correctness of parameter", t, func() { + responseWriter := httptest.NewRecorder() + defaultTargetName := "test" + + Convey("with correct parameter", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?target=test", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.TargetName(defaultTargetName) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("with wrong url query parameter", func() { + testRequest := httptest.NewRequest(http.MethodGet, "/test?target%=test", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + + middlewareFunc := middleware.TargetName(defaultTargetName) + wrappedHandler := middlewareFunc(http.HandlerFunc(handler)) + + wrappedHandler.ServeHTTP(responseWriter, testRequest) + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + + So(contents, ShouldEqual, expectedBadRequest) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + }) +} diff --git a/go.sum b/go.sum index d38ec7ff7..09b73f652 100644 --- a/go.sum +++ b/go.sum @@ -444,7 +444,6 @@ github.com/aws/aws-sdk-go v1.44.219/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -689,7 +688,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -1111,7 +1109,6 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= From 0dc273e9f688925d1ead1ad68e02d05f07e04b02 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhuravlev <52780071+ejuravlev@users.noreply.github.com> Date: Tue, 15 Aug 2023 14:19:55 +0500 Subject: [PATCH 03/46] feat: noisy alert analyzer #846 * save info about notifications in db while parsing events before sending notification package to sender * don't use custom struct. Use NotificationEvents instead * add ablility to sent contact and it's recent events via api * comment out plug * create own struct to handle notificaions with events. Ability to seach events by interval. * delete unnesessary code * delete /{contactId}/events endpoint. Use query params only for /{contactId} * remove contact and events api reply from contact api. Leave only 'getContactById' method * separate contacts and events logic. Add /contactevents endpoint * camelCase * notification history settings are in yml settings files now * make fields private * we shouldn't ignore this errors * Update notifier/notifications/notifications.go Co-authored-by: Tetrergeru <41305740+Tetrergeru@users.noreply.github.com> * fix double api error generation * represent timestamp as uint64 not string * no httpin library * generate mock * fix api structure * int64 timestamp * tests * should be not positive too * cosmetics * Split redis configs and notification history configs * fix tests * fix after merge * remove unnecessary setting * remove unnesessary fields from api output, go to ready to use middleware instead of own * fix after merge * fix lint * fix config retrive tests * fix after merge * style(api,cmd): make vars more readable resolve pr 846 reviewers notes. Make short vars longer but more readable. Also fix local linter issue with 'config' var shadowing --------- Co-authored-by: e.juravlev Co-authored-by: Tetrergeru --- api/controller/contact.go | 18 ++ api/controller/contact_events.go | 33 ++++ api/controller/contact_events_test.go | 115 +++++++++++++ api/controller/contact_test.go | 46 ++++++ api/dto/event_history_item.go | 19 +++ api/handler/contact.go | 17 ++ api/handler/contact_events.go | 49 ++++++ api/handler/handler.go | 6 +- cmd/api/config.go | 19 ++- cmd/api/config_test.go | 4 + cmd/api/main.go | 23 +-- cmd/checker/main.go | 2 +- cmd/cli/main.go | 2 +- cmd/config.go | 41 +++-- cmd/filter/main.go | 2 +- cmd/notifier/config.go | 18 +- cmd/notifier/main.go | 3 +- database/redis/config.go | 9 +- .../redis/contact_notification_history.go | 104 ++++++++++++ .../contact_notification_history_test.go | 155 ++++++++++++++++++ database/redis/database.go | 20 ++- datatypes.go | 13 +- interfaces.go | 2 + local/api.yml | 3 + local/notifier.yml | 3 + mock/moira-alert/database.go | 29 ++++ notifier/notifications/notifications.go | 8 + notifier/notifications/notifications_test.go | 5 + 28 files changed, 719 insertions(+), 49 deletions(-) create mode 100644 api/controller/contact_events.go create mode 100644 api/controller/contact_events_test.go create mode 100644 api/dto/event_history_item.go create mode 100644 api/handler/contact_events.go create mode 100644 database/redis/contact_notification_history.go create mode 100644 database/redis/contact_notification_history_test.go diff --git a/api/controller/contact.go b/api/controller/contact.go index c61f3c5d2..1011a3518 100644 --- a/api/controller/contact.go +++ b/api/controller/contact.go @@ -26,6 +26,24 @@ func GetAllContacts(database moira.Database) (*dto.ContactList, *api.ErrorRespon return &contactsList, nil } +// GetContactById gets notification contact by its id string +func GetContactById(database moira.Database, contactID string) (*dto.Contact, *api.ErrorResponse) { + contact, err := database.GetContact(contactID) + if err != nil { + return nil, api.ErrorInternalServer(err) + } + + contactToReturn := &dto.Contact{ + ID: contact.ID, + User: contact.User, + TeamID: contact.Team, + Type: contact.Type, + Value: contact.Value, + } + + return contactToReturn, nil +} + // CreateContact creates new notification contact for current user func CreateContact(dataBase moira.Database, contact *dto.Contact, userLogin, teamID string) *api.ErrorResponse { if userLogin != "" && teamID != "" { diff --git a/api/controller/contact_events.go b/api/controller/contact_events.go new file mode 100644 index 000000000..e204e6e25 --- /dev/null +++ b/api/controller/contact_events.go @@ -0,0 +1,33 @@ +package controller + +import ( + "fmt" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/api" + "github.com/moira-alert/moira/api/dto" +) + +func GetContactEventsByIdWithLimit(database moira.Database, contactID string, from int64, to int64) (*dto.ContactEventItemList, *api.ErrorResponse) { + events, err := database.GetNotificationsByContactIdWithLimit(contactID, from, to) + + if err != nil { + return nil, api.ErrorInternalServer(fmt.Errorf("GetContactEventsByIdWithLimit: can't get notifications for contact with id %v", contactID)) + } + + var eventsList = dto.ContactEventItemList{ + List: make([]dto.ContactEventItem, 0), + } + for _, i := range events { + contactEventItem := &dto.ContactEventItem{ + TimeStamp: i.TimeStamp, + Metric: i.Metric, + State: string(i.State), + OldState: string(i.OldState), + TriggerID: i.TriggerID, + } + eventsList.List = append(eventsList.List, *contactEventItem) + } + + return &eventsList, nil +} diff --git a/api/controller/contact_events_test.go b/api/controller/contact_events_test.go new file mode 100644 index 000000000..cf57ec8ad --- /dev/null +++ b/api/controller/contact_events_test.go @@ -0,0 +1,115 @@ +package controller + +import ( + "testing" + "time" + + "github.com/moira-alert/moira" + + "github.com/moira-alert/moira/api/dto" + + "github.com/golang/mock/gomock" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" +) + +func TestGetContactEventsByIdWithLimit(t *testing.T) { + mockCtrl := gomock.NewController(t) + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) + defer mockCtrl.Finish() + + now := time.Now() + + contact := dto.Contact{ + ID: "some_contact_id", + Type: "telegram", + Value: "#some_tg_channel", + TeamID: "MoiraCoolestTeam", + } + + contactExpect := moira.ContactData{ + ID: contact.ID, + Value: contact.Value, + Type: contact.Type, + User: "", + Team: contact.TeamID, + } + + items := []*moira.NotificationEventHistoryItem{ + { + TimeStamp: now.Unix() - 20, + Metric: "some.metric1", + State: moira.StateOK, + OldState: moira.StateERROR, + TriggerID: "someTriggerId", + ContactID: "some_contact_id", + }, + + { + TimeStamp: now.Unix() - 50, + Metric: "some.metric2", + State: moira.StateWARN, + OldState: moira.StateOK, + TriggerID: "someTriggerId", + ContactID: "some_contact_id", + }, + } + + itemsExpected := dto.ContactEventItemList{ + List: []dto.ContactEventItem{ + { + TimeStamp: now.Unix() - 20, + Metric: "some.metric1", + State: "OK", + OldState: "ERROR", + TriggerID: "someTriggerId", + }, + { + TimeStamp: now.Unix() - 50, + Metric: "some.metric2", + State: "WARN", + OldState: "OK", + TriggerID: "someTriggerId", + }, + }, + } + + defaultToParameter := now.Unix() + defaultFromParameter := defaultToParameter - int64((3 * time.Hour).Seconds()) + + Convey("Ensure that request with default parameters would return both event items (no url params specified)", t, func() { + dataBase.EXPECT().GetContact(contact.ID).Return(contactExpect, nil).AnyTimes() + dataBase.EXPECT().GetNotificationsByContactIdWithLimit(contact.ID, defaultFromParameter, defaultToParameter).Return(items, nil) + + actualEvents, err := GetContactEventsByIdWithLimit(dataBase, contact.ID, defaultFromParameter, defaultToParameter) + + So(err, ShouldBeNil) + So(actualEvents, ShouldResemble, &itemsExpected) + }) + + Convey("Ensure that request with only 'from' parameter given and 'to' default will return only one (newest) event", t, func() { + dataBase.EXPECT().GetContact(contact.ID).Return(contactExpect, nil).AnyTimes() + dataBase.EXPECT().GetNotificationsByContactIdWithLimit(contact.ID, defaultFromParameter-20, defaultToParameter).Return(items[:1], nil) + + actualEvents, err := GetContactEventsByIdWithLimit(dataBase, contact.ID, defaultFromParameter-20, defaultToParameter) + So(err, ShouldBeNil) + So(actualEvents, ShouldResemble, &dto.ContactEventItemList{ + List: []dto.ContactEventItem{ + itemsExpected.List[0], + }, + }) + }) + + Convey("Ensure that request with only 'to' parameter given and 'from' default will return only one (oldest) event", t, func() { + dataBase.EXPECT().GetContact(contact.ID).Return(contactExpect, nil).AnyTimes() + dataBase.EXPECT().GetNotificationsByContactIdWithLimit(contact.ID, defaultFromParameter, defaultToParameter-30).Return(items[1:], nil) + + actualEvents, err := GetContactEventsByIdWithLimit(dataBase, contact.ID, defaultFromParameter, defaultToParameter-30) + So(err, ShouldBeNil) + So(actualEvents, ShouldResemble, &dto.ContactEventItemList{ + List: []dto.ContactEventItem{ + itemsExpected.List[1], + }, + }) + }) +} diff --git a/api/controller/contact_test.go b/api/controller/contact_test.go index 342cd888d..ec749ebac 100644 --- a/api/controller/contact_test.go +++ b/api/controller/contact_test.go @@ -59,6 +59,52 @@ func TestGetAllContacts(t *testing.T) { }) } +func TestGetContactById(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) + + Convey("Get contact by id should be success", t, func() { + contact := moira.ContactData{ + ID: uuid.Must(uuid.NewV4()).String(), + Type: "slack", + User: "awesome_moira_user", + Value: "awesome_moira_user@gmail.com", + } + + dataBase.EXPECT().GetContact(contact.ID).Return(contact, nil) + actual, err := GetContactById(dataBase, contact.ID) + So(err, ShouldBeNil) + So(actual, + ShouldResemble, + &dto.Contact{ + ID: contact.ID, + Type: contact.Type, + User: contact.User, + Value: contact.Value, + }) + }) + + Convey("Get contact with invalid or unexisting guid id should be empty json", t, func() { + const invalidId = "invalidID" + dataBase.EXPECT().GetContact(invalidId).Return(moira.ContactData{}, nil) + actual, err := GetContactById(dataBase, invalidId) + So(err, ShouldBeNil) + So(actual, ShouldResemble, &dto.Contact{}) + }) + + Convey("Error to fetch contact from db should rise api error", t, func() { + const contactID = "no-matter-what-id-is-there" + emptyContact := moira.ContactData{} + dbError := fmt.Errorf("some db internal error here") + + dataBase.EXPECT().GetContact(contactID).Return(emptyContact, dbError) + contact, err := GetContactById(dataBase, contactID) + So(err, ShouldResemble, api.ErrorInternalServer(dbError)) + So(contact, ShouldBeNil) + }) +} + func TestCreateContact(t *testing.T) { mockCtrl := gomock.NewController(t) dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) diff --git a/api/dto/event_history_item.go b/api/dto/event_history_item.go new file mode 100644 index 000000000..26c85175f --- /dev/null +++ b/api/dto/event_history_item.go @@ -0,0 +1,19 @@ +package dto + +import "net/http" + +type ContactEventItem struct { + TimeStamp int64 `json:"timestamp"` + Metric string `json:"metric"` + State string `json:"state"` + OldState string `json:"old_state"` + TriggerID string `json:"trigger_id"` +} + +type ContactEventItemList struct { + List []ContactEventItem `json:"list"` +} + +func (*ContactEventItemList) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} diff --git a/api/handler/contact.go b/api/handler/contact.go index e7db8aed5..b824a9037 100644 --- a/api/handler/contact.go +++ b/api/handler/contact.go @@ -20,6 +20,7 @@ func contact(router chi.Router) { router.Route("/{contactId}", func(router chi.Router) { router.Use(middleware.ContactContext) router.Use(contactFilter) + router.Get("/", getContactById) router.Put("/", updateContact) router.Delete("/", removeContact) router.Post("/test", sendTestContactNotification) @@ -39,6 +40,22 @@ func getAllContacts(writer http.ResponseWriter, request *http.Request) { } } +func getContactById(writer http.ResponseWriter, request *http.Request) { + contactData := request.Context().Value(contactKey).(moira.ContactData) + + contact, apiErr := controller.GetContactById(database, contactData.ID) + + if apiErr != nil { + render.Render(writer, request, apiErr) //nolint + return + } + + if err := render.Render(writer, request, contact); err != nil { + render.Render(writer, request, api.ErrorRender(err)) //nolint + return + } +} + func createNewContact(writer http.ResponseWriter, request *http.Request) { contact := &dto.Contact{} if err := render.Bind(request, contact); err != nil { diff --git a/api/handler/contact_events.go b/api/handler/contact_events.go new file mode 100644 index 000000000..e1192a254 --- /dev/null +++ b/api/handler/contact_events.go @@ -0,0 +1,49 @@ +package handler + +import ( + "fmt" + "net/http" + "time" + + "github.com/go-graphite/carbonapi/date" + + "github.com/go-chi/render" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/api" + "github.com/moira-alert/moira/api/controller" + + "github.com/go-chi/chi" + "github.com/moira-alert/moira/api/middleware" +) + +func contactEvents(router chi.Router) { + router.Route("/{contactId}/events", func(router chi.Router) { + router.Use(middleware.ContactContext) + router.Use(contactFilter) + router.With(middleware.DateRange("-3hour", "now")).Get("/", getContactByIdWithEvents) + }) +} + +func getContactByIdWithEvents(writer http.ResponseWriter, request *http.Request) { + contactData := request.Context().Value(contactKey).(moira.ContactData) + fromStr := middleware.GetFromStr(request) + toStr := middleware.GetToStr(request) + from := date.DateParamToEpoch(fromStr, "UTC", 0, time.UTC) + if from == 0 { + render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("can not parse from: %s", fromStr))) //nolint + return + } + to := date.DateParamToEpoch(toStr, "UTC", 0, time.UTC) + if to == 0 { + render.Render(writer, request, api.ErrorInvalidRequest(fmt.Errorf("can not parse to: %v", to))) //nolint + return + } + contactWithEvents, err := controller.GetContactEventsByIdWithLimit(database, contactData.ID, from, to) + if err != nil { + render.Render(writer, request, err) //nolint + } + if err := render.Render(writer, request, contactWithEvents); err != nil { + render.Render(writer, request, api.ErrorRender(err)) //nolint + return + } +} diff --git a/api/handler/handler.go b/api/handler/handler.go index 55211be7a..cad6f07c9 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -41,11 +41,15 @@ func NewHandler(db moira.Database, log moira.Logger, index moira.Searcher, confi router.Route("/tag", tag) router.Route("/pattern", pattern) router.Route("/event", event) - router.Route("/contact", contact) router.Route("/subscription", subscription) router.Route("/notification", notification) router.Route("/health", health) router.Route("/teams", teams) + + router.Route("/contact", func(router chi.Router) { + contact(router) + contactEvents(router) + }) }) if config.EnableCORS { return cors.AllowAll().Handler(router) diff --git a/cmd/api/config.go b/cmd/api/config.go index 3e231137d..5cd083586 100644 --- a/cmd/api/config.go +++ b/cmd/api/config.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" + "github.com/moira-alert/moira/notifier" + "github.com/xiam/to" "github.com/moira-alert/moira/api" @@ -11,12 +13,13 @@ import ( ) type config struct { - Redis cmd.RedisConfig `yaml:"redis"` - Logger cmd.LoggerConfig `yaml:"log"` - API apiConfig `yaml:"api"` - Web webConfig `yaml:"web"` - Telemetry cmd.TelemetryConfig `yaml:"telemetry"` - Remote cmd.RemoteConfig `yaml:"remote"` + Redis cmd.RedisConfig `yaml:"redis"` + Logger cmd.LoggerConfig `yaml:"log"` + API apiConfig `yaml:"api"` + Web webConfig `yaml:"web"` + Telemetry cmd.TelemetryConfig `yaml:"telemetry"` + Remote cmd.RemoteConfig `yaml:"remote"` + NotificationHistory cmd.NotificationHistoryConfig `yaml:"notification_history"` } type apiConfig struct { @@ -106,6 +109,10 @@ func getDefault() config { DialTimeout: "500ms", MaxRetries: 3, }, + NotificationHistory: cmd.NotificationHistoryConfig{ + NotificationHistoryTTL: "48h", + NotificationHistoryQueryLimit: int(notifier.NotificationsLimitUnlimited), + }, Logger: cmd.LoggerConfig{ LogFile: "stdout", LogLevel: "info", diff --git a/cmd/api/config_test.go b/cmd/api/config_test.go index 1973aff25..8336370a4 100644 --- a/cmd/api/config_test.go +++ b/cmd/api/config_test.go @@ -92,6 +92,10 @@ func Test_webConfig_getDefault(t *testing.T) { Timeout: "60s", MetricsTTL: "7d", }, + NotificationHistory: cmd.NotificationHistoryConfig{ + NotificationHistoryTTL: "48h", + NotificationHistoryQueryLimit: -1, + }, } result := getDefault() diff --git a/cmd/api/main.go b/cmd/api/main.go index 06e1c8f0a..71da5a098 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -48,21 +48,21 @@ func main() { os.Exit(0) } - config := getDefault() + applicationConfig := getDefault() if *printDefaultConfigFlag { - cmd.PrintConfig(config) + cmd.PrintConfig(applicationConfig) os.Exit(0) } - err := cmd.ReadConfig(*configFileName, &config) + err := cmd.ReadConfig(*configFileName, &applicationConfig) if err != nil { fmt.Fprintf(os.Stderr, "Can not read settings: %s\n", err.Error()) os.Exit(1) } - apiConfig := config.API.getSettings(config.Redis.MetricsTTL, config.Remote.MetricsTTL) + apiConfig := applicationConfig.API.getSettings(applicationConfig.Redis.MetricsTTL, applicationConfig.Remote.MetricsTTL) - logger, err := logging.ConfigureLog(config.Logger.LogFile, config.Logger.LogLevel, serviceName, config.Logger.LogPrettyFormat) + logger, err := logging.ConfigureLog(applicationConfig.Logger.LogFile, applicationConfig.Logger.LogLevel, serviceName, applicationConfig.Logger.LogPrettyFormat) if err != nil { fmt.Fprintf(os.Stderr, "Can not configure log: %s\n", err.Error()) @@ -72,7 +72,7 @@ func main() { String("moira_version", MoiraVersion). Msg("Moira API stopped") - telemetry, err := cmd.ConfigureTelemetry(logger, config.Telemetry, serviceName) + telemetry, err := cmd.ConfigureTelemetry(logger, applicationConfig.Telemetry, serviceName) if err != nil { logger.Fatal(). Error(err). @@ -80,8 +80,9 @@ func main() { } defer telemetry.Stop() - databaseSettings := config.Redis.GetSettings() - database := redis.NewDatabase(logger, databaseSettings, redis.API) + databaseSettings := applicationConfig.Redis.GetSettings() + notificationHistorySettings := applicationConfig.NotificationHistory.GetSettings() + database := redis.NewDatabase(logger, databaseSettings, notificationHistorySettings, redis.API) // Start Index right before HTTP listener. Fail if index cannot start searchIndex := index.NewSearchIndex(logger, database, telemetry.Metrics) @@ -114,15 +115,15 @@ func main() { Msg("Start listening") localSource := local.Create(database) - remoteConfig := config.Remote.GetRemoteSourceSettings() + remoteConfig := applicationConfig.Remote.GetRemoteSourceSettings() remoteSource := remote.Create(remoteConfig) metricSourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) - webConfigContent, err := config.Web.getSettings(remoteConfig.Enabled) + webConfigContent, err := applicationConfig.Web.getSettings(remoteConfig.Enabled) if err != nil { logger.Fatal(). Error(err). - Msg("Failed to get web config content ") + Msg("Failed to get web applicationConfig content ") } httpHandler := handler.NewHandler(database, logger, searchIndex, apiConfig, metricSourceProvider, webConfigContent) diff --git a/cmd/checker/main.go b/cmd/checker/main.go index 44de5fd62..64ea4723c 100644 --- a/cmd/checker/main.go +++ b/cmd/checker/main.go @@ -80,7 +80,7 @@ func main() { defer telemetry.Stop() databaseSettings := config.Redis.GetSettings() - database := redis.NewDatabase(logger, databaseSettings, redis.Checker) + database := redis.NewDatabase(logger, databaseSettings, redis.NotificationHistoryConfig{}, redis.Checker) remoteConfig := config.Remote.GetRemoteSourceSettings() localSource := local.Create(database) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index f814aea77..657dc1ce2 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -340,7 +340,7 @@ func initApp() (cleanupConfig, moira.Logger, moira.Database) { } databaseSettings := config.Redis.GetSettings() - dataBase := redis.NewDatabase(logger, databaseSettings, redis.Cli) + dataBase := redis.NewDatabase(logger, databaseSettings, redis.NotificationHistoryConfig{}, redis.Cli) return config.Cleanup, logger, dataBase } diff --git a/cmd/config.go b/cmd/config.go index fbacf1940..94c61d3b4 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -47,19 +47,34 @@ type RedisConfig struct { } // GetSettings returns redis config parsed from moira config files -func (config *RedisConfig) GetSettings() redis.Config { - return redis.Config{ - MasterName: config.MasterName, - Addrs: strings.Split(config.Addrs, ","), - SentinelPassword: config.SentinelPassword, - SentinelUsername: config.SentinelUsername, - Username: config.Username, - Password: config.Password, - MaxRetries: config.MaxRetries, - MetricsTTL: to.Duration(config.MetricsTTL), - DialTimeout: to.Duration(config.DialTimeout), - ReadTimeout: to.Duration(config.ReadTimeout), - WriteTimeout: to.Duration(config.WriteTimeout), +func (config *RedisConfig) GetSettings() redis.DatabaseConfig { + return redis.DatabaseConfig{ + MasterName: config.MasterName, + Addrs: strings.Split(config.Addrs, ","), + Username: config.Username, + Password: config.Password, + MaxRetries: config.MaxRetries, + MetricsTTL: to.Duration(config.MetricsTTL), + DialTimeout: to.Duration(config.DialTimeout), + ReadTimeout: to.Duration(config.ReadTimeout), + WriteTimeout: to.Duration(config.WriteTimeout), + } +} + +// NotificationHistoryConfig is the config which coordinates interaction with notification statistics +// e.g. how much time should we store it, or how many history items can we request from database +type NotificationHistoryConfig struct { + // Time which moira should store contacts and theirs events history + NotificationHistoryTTL string `yaml:"ttl"` + // Max count of events which moira may send as response of contact and its events history + NotificationHistoryQueryLimit int `yaml:"query_limit"` +} + +// GetSettings returns notification history storage policy configuration +func (notificationHistoryConfig *NotificationHistoryConfig) GetSettings() redis.NotificationHistoryConfig { + return redis.NotificationHistoryConfig{ + NotificationHistoryTTL: to.Duration(notificationHistoryConfig.NotificationHistoryTTL), + NotificationHistoryQueryLimit: notificationHistoryConfig.NotificationHistoryQueryLimit, } } diff --git a/cmd/filter/main.go b/cmd/filter/main.go index 5c155a0e6..a4d3773f3 100644 --- a/cmd/filter/main.go +++ b/cmd/filter/main.go @@ -85,7 +85,7 @@ func main() { } filterMetrics := metrics.ConfigureFilterMetrics(telemetry.Metrics) - database := redis.NewDatabase(logger, config.Redis.GetSettings(), redis.Filter) + database := redis.NewDatabase(logger, config.Redis.GetSettings(), redis.NotificationHistoryConfig{}, redis.Filter) retentionConfigFile, err := os.Open(config.Filter.RetentionConfig) if err != nil { diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index a61dbff96..5c725944c 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -13,12 +13,13 @@ import ( ) type config struct { - Redis cmd.RedisConfig `yaml:"redis"` - Logger cmd.LoggerConfig `yaml:"log"` - Notifier notifierConfig `yaml:"notifier"` - Telemetry cmd.TelemetryConfig `yaml:"telemetry"` - Remote cmd.RemoteConfig `yaml:"remote"` - ImageStores cmd.ImageStoreConfig `yaml:"image_store"` + Redis cmd.RedisConfig `yaml:"redis"` + Logger cmd.LoggerConfig `yaml:"log"` + Notifier notifierConfig `yaml:"notifier"` + Telemetry cmd.TelemetryConfig `yaml:"telemetry"` + Remote cmd.RemoteConfig `yaml:"remote"` + ImageStores cmd.ImageStoreConfig `yaml:"image_store"` + NotificationHistory cmd.NotificationHistoryConfig `yaml:"notification_history"` } type entityLogConfig struct { @@ -78,12 +79,15 @@ func getDefault() config { MetricsTTL: "1h", DialTimeout: "500ms", }, - Logger: cmd.LoggerConfig{ LogFile: "stdout", LogLevel: "info", LogPrettyFormat: false, }, + NotificationHistory: cmd.NotificationHistoryConfig{ + NotificationHistoryTTL: "48h", + NotificationHistoryQueryLimit: int(notifier.NotificationsLimitUnlimited), + }, Notifier: notifierConfig{ SenderTimeout: "10s", ResendingTimeout: "1:00", diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index c28737b28..b0bd8415d 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -80,7 +80,8 @@ func main() { notifierMetrics := metrics.ConfigureNotifierMetrics(telemetry.Metrics, serviceName) databaseSettings := config.Redis.GetSettings() - database := redis.NewDatabase(logger, databaseSettings, redis.Notifier) + notificationHistorySettings := config.NotificationHistory.GetSettings() + database := redis.NewDatabase(logger, databaseSettings, notificationHistorySettings, redis.Notifier) localSource := local.Create(database) remoteConfig := config.Remote.GetRemoteSourceSettings() diff --git a/database/redis/config.go b/database/redis/config.go index f9817756f..045ed7b21 100644 --- a/database/redis/config.go +++ b/database/redis/config.go @@ -2,8 +2,8 @@ package redis import "time" -// Config - Redis database connection config -type Config struct { +// DatabaseConfig - Redis database connection config +type DatabaseConfig struct { MasterName string Addrs []string Username string @@ -16,3 +16,8 @@ type Config struct { WriteTimeout time.Duration MaxRetries int } + +type NotificationHistoryConfig struct { + NotificationHistoryTTL time.Duration + NotificationHistoryQueryLimit int +} diff --git a/database/redis/contact_notification_history.go b/database/redis/contact_notification_history.go new file mode 100644 index 000000000..2e5f3ffa6 --- /dev/null +++ b/database/redis/contact_notification_history.go @@ -0,0 +1,104 @@ +package redis + +import ( + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/go-redis/redis/v8" + "github.com/moira-alert/moira" +) + +const contactNotificationKey = "moira-contact-notifications" + +func getNotificationBytes(notification *moira.NotificationEventHistoryItem) ([]byte, error) { + bytes, err := json.Marshal(notification) + if err != nil { + return nil, fmt.Errorf("failed to marshal notification event: %s", err.Error()) + } + return bytes, nil +} + +func getNotificationStruct(notificationString string) (moira.NotificationEventHistoryItem, error) { + var object moira.NotificationEventHistoryItem + err := json.Unmarshal([]byte(notificationString), &object) + if err != nil { + return object, fmt.Errorf("failed to umarshall event: %s", err.Error()) + } + return object, nil +} + +func (connector *DbConnector) GetNotificationsByContactIdWithLimit(contactID string, from int64, to int64) ([]*moira.NotificationEventHistoryItem, error) { + c := *connector.client + var notifications []*moira.NotificationEventHistoryItem + + notificationStings, err := c.ZRangeByScore(connector.context, contactNotificationKey, &redis.ZRangeBy{ + Min: strconv.FormatInt(from, 10), + Max: strconv.FormatInt(to, 10), + Count: int64(connector.notificationHistory.NotificationHistoryQueryLimit), + }).Result() + + if err != nil { + return notifications, err + } + + for _, notification := range notificationStings { + notificationObj, err := getNotificationStruct(notification) + + if err != nil { + return notifications, err + } + + if notificationObj.ContactID == contactID { + notifications = append(notifications, ¬ificationObj) + } + } + + return notifications, nil +} + +// PushContactNotificationToHistory converts ScheduledNotification to NotificationEventHistoryItem and +// saves it, and deletes items older than specified ttl +func (connector *DbConnector) PushContactNotificationToHistory(notification *moira.ScheduledNotification) error { + notificationItemToSave := &moira.NotificationEventHistoryItem{ + Metric: notification.Event.Metric, + State: notification.Event.State, + TriggerID: notification.Trigger.ID, + OldState: notification.Event.OldState, + ContactID: notification.Contact.ID, + TimeStamp: notification.Timestamp, + } + + notificationBytes, serializationErr := getNotificationBytes(notificationItemToSave) + + if serializationErr != nil { + return fmt.Errorf("failed to serialize notification to contact event history item: %s", serializationErr.Error()) + } + + to := int(time.Now().Unix() - int64(connector.notificationHistory.NotificationHistoryTTL.Seconds())) + + pipe := (*connector.client).TxPipeline() + + pipe.ZAdd( + connector.context, + contactNotificationKey, + &redis.Z{ + Score: float64(notification.Timestamp), + Member: notificationBytes}) + + pipe.ZRemRangeByScore( + connector.context, + contactNotificationKey, + "-inf", + strconv.Itoa(to), + ) + + _, err := pipe.Exec(connector.Context()) + + if err != nil { + return fmt.Errorf("failed to push contact event history item: %s", err.Error()) + } + + return nil +} diff --git a/database/redis/contact_notification_history_test.go b/database/redis/contact_notification_history_test.go new file mode 100644 index 000000000..cb1961828 --- /dev/null +++ b/database/redis/contact_notification_history_test.go @@ -0,0 +1,155 @@ +package redis + +import ( + "testing" + "time" + + "github.com/moira-alert/moira" + + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + . "github.com/smartystreets/goconvey/convey" +) + +var inputScheduledNotification = moira.ScheduledNotification{ + Event: moira.NotificationEvent{ + IsTriggerEvent: true, + Timestamp: time.Now().Unix(), + Metric: "some_metric", + State: moira.StateERROR, + OldState: moira.StateOK, + TriggerID: "1111-2222-33-4444-5555", + }, + Trigger: moira.TriggerData{ + ID: "1111-2222-33-4444-5555", + Name: "Awesome Trigger", + Desc: "No desc", + Targets: []string{ + "some.metric.path", + }, + WarnValue: 0.9, + ErrorValue: 1.0, + IsRemote: false, + Tags: []string{ + "TEST_TAG1", + "TEST_TAG2", + }, + }, + Contact: moira.ContactData{ + Type: "slack", + Value: "#auf_channel", + ID: "contact_id", + User: "user", + }, + Plotting: moira.PlottingData{ + Enabled: false, + }, + Throttled: false, + SendFail: 1, + Timestamp: time.Now().Unix(), +} + +var eventsShouldBeInDb = []*moira.NotificationEventHistoryItem{ + { + TimeStamp: inputScheduledNotification.Timestamp, + Metric: inputScheduledNotification.Event.Metric, + State: inputScheduledNotification.Event.State, + OldState: inputScheduledNotification.Event.OldState, + TriggerID: inputScheduledNotification.Trigger.ID, + ContactID: inputScheduledNotification.Contact.ID, + }, +} + +func TestGetNotificationsByContactIdWithLimit(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + + Convey("Notification history items manipulation", t, func() { + dataBase.Flush() + defer dataBase.Flush() + + Convey("While no data then notification items should be empty", func() { + items, err := dataBase.GetNotificationsByContactIdWithLimit( + "id", + eventsShouldBeInDb[0].TimeStamp, + eventsShouldBeInDb[0].TimeStamp) + + So(err, ShouldBeNil) + So(items, ShouldHaveLength, 0) + }) + + Convey("Write event and check for success write", func() { + err := dataBase.PushContactNotificationToHistory(&inputScheduledNotification) + So(err, ShouldBeNil) + + Convey("Ensure that we can find event on +- 5 seconds interval", func() { + eventFromDb, err := dataBase.GetNotificationsByContactIdWithLimit( + eventsShouldBeInDb[0].ContactID, + eventsShouldBeInDb[0].TimeStamp-5, + eventsShouldBeInDb[0].TimeStamp+5) + So(err, ShouldBeNil) + So(eventFromDb, ShouldResemble, eventsShouldBeInDb) + }) + + Convey("Ensure that we can find event exactly by its timestamp", func() { + eventFromDb, err := dataBase.GetNotificationsByContactIdWithLimit( + eventsShouldBeInDb[0].ContactID, + eventsShouldBeInDb[0].TimeStamp, + eventsShouldBeInDb[0].TimeStamp) + So(err, ShouldBeNil) + So(eventFromDb, ShouldResemble, eventsShouldBeInDb) + }) + + Convey("Ensure that we can find event if 'from' border equals its timestamp", func() { + eventFromDb, err := dataBase.GetNotificationsByContactIdWithLimit( + eventsShouldBeInDb[0].ContactID, + eventsShouldBeInDb[0].TimeStamp, + eventsShouldBeInDb[0].TimeStamp+5) + So(err, ShouldBeNil) + So(eventFromDb, ShouldResemble, eventsShouldBeInDb) + }) + + Convey("Ensure that we can find event if 'to' border equals its timestamp", func() { + eventFromDb, err := dataBase.GetNotificationsByContactIdWithLimit( + eventsShouldBeInDb[0].ContactID, + eventsShouldBeInDb[0].TimeStamp-5, + eventsShouldBeInDb[0].TimeStamp) + So(err, ShouldBeNil) + So(eventFromDb, ShouldResemble, eventsShouldBeInDb) + }) + + Convey("Ensure that we can't find event time borders don't fit event timestamp", func() { + eventFromDb, err := dataBase.GetNotificationsByContactIdWithLimit( + eventsShouldBeInDb[0].ContactID, + 928930626, + 992089026) + So(err, ShouldBeNil) + So(eventFromDb, ShouldNotResemble, eventsShouldBeInDb) + }) + }) + }) +} + +func TestPushNotificationToHistory(t *testing.T) { + logger, _ := logging.GetLogger("dataBase") + dataBase := NewTestDatabase(logger) + dataBase.notificationHistory.NotificationHistoryQueryLimit = 500 + + Convey("Ensure that event would not have duplicates", t, func() { + dataBase.Flush() + defer dataBase.Flush() + + err1 := dataBase.PushContactNotificationToHistory(&inputScheduledNotification) + So(err1, ShouldBeNil) + + err2 := dataBase.PushContactNotificationToHistory(&inputScheduledNotification) + So(err2, ShouldBeNil) + + dbContent, err3 := dataBase.GetNotificationsByContactIdWithLimit( + inputScheduledNotification.Contact.ID, + inputScheduledNotification.Timestamp, + inputScheduledNotification.Timestamp) + + So(err3, ShouldBeNil) + So(dbContent, ShouldHaveLength, 1) + }) +} diff --git a/database/redis/database.go b/database/redis/database.go index 6c7afc5e4..f760049b2 100644 --- a/database/redis/database.go +++ b/database/redis/database.go @@ -45,9 +45,10 @@ type DbConnector struct { context context.Context source DBSource clock moira.Clock + notificationHistory NotificationHistoryConfig } -func NewDatabase(logger moira.Logger, config Config, source DBSource) *DbConnector { +func NewDatabase(logger moira.Logger, config DatabaseConfig, nh NotificationHistoryConfig, source DBSource) *DbConnector { client := redis.NewUniversalClient(&redis.UniversalOptions{ MasterName: config.MasterName, Addrs: config.Addrs, @@ -76,20 +77,31 @@ func NewDatabase(logger moira.Logger, config Config, source DBSource) *DbConnect metricsTTLSeconds: int64(config.MetricsTTL.Seconds()), source: source, clock: clock.NewSystemClock(), + notificationHistory: nh, } return &connector } // NewTestDatabase use it only for tests func NewTestDatabase(logger moira.Logger) *DbConnector { - return NewDatabase(logger, Config{ + return NewDatabase(logger, DatabaseConfig{ Addrs: []string{"0.0.0.0:6379"}, - }, testSource) + }, + NotificationHistoryConfig{ + NotificationHistoryTTL: time.Hour * 48, + NotificationHistoryQueryLimit: 1000, + }, testSource) } // NewTestDatabaseWithIncorrectConfig use it only for tests func NewTestDatabaseWithIncorrectConfig(logger moira.Logger) *DbConnector { - return NewDatabase(logger, Config{Addrs: []string{"0.0.0.0:0000"}}, testSource) + return NewDatabase(logger, + DatabaseConfig{Addrs: []string{"0.0.0.0:0000"}}, + NotificationHistoryConfig{ + NotificationHistoryTTL: time.Hour * 48, + NotificationHistoryQueryLimit: 1000, + }, + testSource) } // Flush deletes all the keys of the DB, use it only for tests diff --git a/datatypes.go b/datatypes.go index 62b633f9e..2c37daad9 100644 --- a/datatypes.go +++ b/datatypes.go @@ -50,12 +50,23 @@ type NotificationEvent struct { State State `json:"state"` TriggerID string `json:"trigger_id"` SubscriptionID *string `json:"sub_id,omitempty"` - ContactID string `json:"contactId,omitempty"` + ContactID string `json:"contact_id,omitempty"` OldState State `json:"old_state"` Message *string `json:"msg,omitempty"` MessageEventInfo *EventInfo `json:"event_message"` } +// NotificationEventHistoryItem is in use to store notifications history of channel +// (see database/redis/contact_notifications_history.go +type NotificationEventHistoryItem struct { + TimeStamp int64 `json:"timestamp"` + Metric string `json:"metric"` + State State `json:"state"` + OldState State `json:"old_state"` + TriggerID string `json:"trigger_id"` + ContactID string `json:"contact_id"` +} + // EventInfo - a base for creating messages. type EventInfo struct { Maintenance *MaintenanceInfo `json:"maintenance,omitempty"` diff --git a/interfaces.go b/interfaces.go index 16c7324e3..95904cef3 100644 --- a/interfaces.go +++ b/interfaces.go @@ -83,11 +83,13 @@ type Database interface { // ScheduledNotification storing GetNotifications(start, end int64) ([]*ScheduledNotification, int64, error) + GetNotificationsByContactIdWithLimit(contactID string, from int64, to int64) ([]*NotificationEventHistoryItem, error) RemoveNotification(notificationKey string) (int64, error) RemoveAllNotifications() error FetchNotifications(to int64, limit int64) ([]*ScheduledNotification, error) AddNotification(notification *ScheduledNotification) error AddNotifications(notification []*ScheduledNotification, timestamp int64) error + PushContactNotificationToHistory(notification *ScheduledNotification) error // Patterns and metrics storing GetPatterns() ([]string, error) diff --git a/local/api.yml b/local/api.yml index 93575c500..ccbb28718 100644 --- a/local/api.yml +++ b/local/api.yml @@ -44,6 +44,9 @@ web: is_plotting_available: true is_plotting_default_on: true is_subscription_to_all_tags_available: true +notification_history: + ttl: 48h + query_limit: 10000 log: log_file: stdout log_level: debug diff --git a/local/notifier.yml b/local/notifier.yml index 414d4cf76..c9583d4bd 100644 --- a/local/notifier.yml +++ b/local/notifier.yml @@ -33,6 +33,9 @@ notifier: front_uri: http://localhost timezone: UTC date_time_format: "15:04 02.01.2006" +notification_history: + ttl: 48h + query_limit: 10000 log: log_file: stdout log_level: info diff --git a/mock/moira-alert/database.go b/mock/moira-alert/database.go index fa4c9b782..2f2dc189c 100644 --- a/mock/moira-alert/database.go +++ b/mock/moira-alert/database.go @@ -517,6 +517,21 @@ func (mr *MockDatabaseMockRecorder) GetNotifications(arg0, arg1 interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotifications", reflect.TypeOf((*MockDatabase)(nil).GetNotifications), arg0, arg1) } +// GetNotificationsByContactIdWithLimit mocks base method. +func (m *MockDatabase) GetNotificationsByContactIdWithLimit(arg0 string, arg1, arg2 int64) ([]*moira.NotificationEventHistoryItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNotificationsByContactIdWithLimit", arg0, arg1, arg2) + ret0, _ := ret[0].([]*moira.NotificationEventHistoryItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNotificationsByContactIdWithLimit indicates an expected call of GetNotificationsByContactIdWithLimit. +func (mr *MockDatabaseMockRecorder) GetNotificationsByContactIdWithLimit(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotificationsByContactIdWithLimit", reflect.TypeOf((*MockDatabase)(nil).GetNotificationsByContactIdWithLimit), arg0, arg1, arg2) +} + // GetNotifierState mocks base method. func (m *MockDatabase) GetNotifierState() (string, error) { m.ctrl.T.Helper() @@ -1018,6 +1033,20 @@ func (mr *MockDatabaseMockRecorder) NewLock(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewLock", reflect.TypeOf((*MockDatabase)(nil).NewLock), arg0, arg1) } +// PushContactNotificationToHistory mocks base method. +func (m *MockDatabase) PushContactNotificationToHistory(arg0 *moira.ScheduledNotification) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PushContactNotificationToHistory", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// PushContactNotificationToHistory indicates an expected call of PushContactNotificationToHistory. +func (mr *MockDatabaseMockRecorder) PushContactNotificationToHistory(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PushContactNotificationToHistory", reflect.TypeOf((*MockDatabase)(nil).PushContactNotificationToHistory), arg0) +} + // PushNotificationEvent mocks base method. func (m *MockDatabase) PushNotificationEvent(arg0 *moira.NotificationEvent, arg1 bool) error { m.ctrl.T.Helper() diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 087bef083..8f09b3668 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -67,6 +67,7 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error { return notifierInBadStateError(fmt.Sprintf("notifier in a bad state: %v", state)) } notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) + if err != nil { return err } @@ -85,6 +86,13 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error { } } p.Events = append(p.Events, notification.Event) + + err = worker.Database.PushContactNotificationToHistory(notification) + + if err != nil { + worker.Logger.Warning().Error(err).Msg("Can't save notification to history") + } + notificationPackages[packageKey] = p } var sendingWG sync.WaitGroup diff --git a/notifier/notifications/notifications_test.go b/notifier/notifications/notifications_test.go index a579db9e0..d97aaa890 100644 --- a/notifier/notifications/notifications_test.go +++ b/notifier/notifications/notifications_test.go @@ -88,6 +88,8 @@ func TestProcessScheduledEvent(t *testing.T) { notification2.Event, }, } + dataBase.EXPECT().PushContactNotificationToHistory(¬ification1).Return(nil).AnyTimes() + dataBase.EXPECT().PushContactNotificationToHistory(¬ification2).Return(nil).AnyTimes() notifier.EXPECT().Send(&pkg1, gomock.Any()) notifier.EXPECT().Send(&pkg2, gomock.Any()) notifier.EXPECT().GetReadBatchSize().Return(notifier2.NotificationsLimitUnlimited) @@ -114,6 +116,8 @@ func TestProcessScheduledEvent(t *testing.T) { }, } + dataBase.EXPECT().PushContactNotificationToHistory(¬ification2).Return(nil).AnyTimes() + dataBase.EXPECT().PushContactNotificationToHistory(¬ification3).Return(nil).AnyTimes() notifier.EXPECT().Send(&pkg, gomock.Any()) dataBase.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) notifier.EXPECT().GetReadBatchSize().Return(notifier2.NotificationsLimitUnlimited) @@ -159,6 +163,7 @@ func TestGoRoutine(t *testing.T) { shutdown := make(chan struct{}) dataBase.EXPECT().FetchNotifications(gomock.Any(), notifier2.NotificationsLimitUnlimited).Return([]*moira.ScheduledNotification{¬ification1}, nil) + dataBase.EXPECT().PushContactNotificationToHistory(¬ification1).Return(nil).AnyTimes() notifier.EXPECT().Send(&pkg, gomock.Any()).Do(func(arg0, arg1 interface{}) { close(shutdown) }) notifier.EXPECT().StopSenders() notifier.EXPECT().GetReadBatchSize().Return(notifier2.NotificationsLimitUnlimited) From 31167c655e596c45782e4c11bb4e186df835f509 Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:11:33 +0300 Subject: [PATCH 04/46] feature(api): add swagger specification generation (#891) --- Dockerfile.api | 3 + Makefile | 14 +++ api/dto/contact.go | 8 +- api/dto/events.go | 6 +- api/dto/health.go | 4 +- api/dto/notification.go | 4 +- api/dto/pattern.go | 4 +- api/dto/tag.go | 8 +- api/dto/target.go | 10 +-- api/dto/team.go | 12 +-- api/dto/triggers.go | 44 +++++----- api/dto/user.go | 2 +- api/error_response.go | 32 +++++++ api/handler/config.go | 18 ++++ api/handler/contact.go | 79 +++++++++++++++++ api/handler/contact_events.go | 16 ++++ api/handler/event.go | 24 ++++++ api/handler/handler.go | 55 +++++++++++- api/handler/health.go | 10 +++ api/handler/notification.go | 25 ++++++ api/handler/pattern.go | 21 +++++ api/handler/subscription.go | 63 ++++++++++++++ api/handler/tag.go | 32 +++++++ api/handler/team.go | 139 +++++++++++++++++++++++++++++ api/handler/team_contact.go | 16 ++++ api/handler/team_subscription.go | 16 ++++ api/handler/trigger.go | 96 +++++++++++++++++++++ api/handler/trigger_metrics.go | 40 +++++++++ api/handler/trigger_render.go | 18 ++++ api/handler/triggers.go | 70 ++++++++++++++- api/handler/user.go | 19 ++++ datatypes.go | 144 +++++++++++++++---------------- docs/docs.go | 3 + go.mod | 26 ++++-- go.sum | 61 ++++++++++--- 35 files changed, 1000 insertions(+), 142 deletions(-) create mode 100644 docs/docs.go diff --git a/Dockerfile.api b/Dockerfile.api index 520944ec0..f34cbc4e8 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -3,9 +3,12 @@ FROM golang:1.18 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira RUN go mod download +RUN go install github.com/swaggo/swag/cmd/swag@v1.8.12 COPY . /go/src/github.com/moira-alert/moira/ +RUN /go/bin/swag init -g api/handler/handler.go + ARG GO_VERSION="GoVersion" ARG GIT_COMMIT="git_Commit" ARG MoiraVersion="MoiraVersion" diff --git a/Makefile b/Makefile index 74e3a0e35..97df83ae3 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,20 @@ lint: mock: . ./generate_mocks.sh +.PHONY: install-swag +install-swag: + go install github.com/swaggo/swag/cmd/swag@v1.8.12 + +.PHONY: spec +spec: + echo "Generating Swagger documentation" + swag init -g api/handler/handler.go + swag fmt + +.PHONY: validate-spec +validate-spec: + openapi-generator-cli validate -i docs/swagger.yaml + .PHONY: test test: echo 'mode: atomic' > coverage.txt && go list ./... | xargs -n1 -I{} sh -c 'go test -race -v -bench=. -covermode=atomic -coverprofile=coverage.tmp {} && tail -n +2 coverage.tmp >> coverage.txt' && rm coverage.tmp diff --git a/api/dto/contact.go b/api/dto/contact.go index 4f28adf2b..12e9fd1da 100644 --- a/api/dto/contact.go +++ b/api/dto/contact.go @@ -17,10 +17,10 @@ func (*ContactList) Render(w http.ResponseWriter, r *http.Request) error { } type Contact struct { - Type string `json:"type"` - Value string `json:"value"` - ID string `json:"id,omitempty"` - User string `json:"user,omitempty"` + Type string `json:"type" example:"mail"` + Value string `json:"value" example:"devops@example.com"` + ID string `json:"id,omitempty" example:"1dd38765-c5be-418d-81fa-7a5f879c2315"` + User string `json:"user,omitempty" example:""` TeamID string `json:"team_id,omitempty"` } diff --git a/api/dto/events.go b/api/dto/events.go index e24fddde3..2f704d337 100644 --- a/api/dto/events.go +++ b/api/dto/events.go @@ -8,9 +8,9 @@ import ( ) type EventsList struct { - Page int64 `json:"page"` - Size int64 `json:"size"` - Total int64 `json:"total"` + Page int64 `json:"page" example:"0"` + Size int64 `json:"size" example:"100"` + Total int64 `json:"total" example:"10"` List []moira.NotificationEvent `json:"list"` } diff --git a/api/dto/health.go b/api/dto/health.go index a9fd7eb8d..db8adbfc9 100644 --- a/api/dto/health.go +++ b/api/dto/health.go @@ -13,8 +13,8 @@ const ( ) type NotifierState struct { - State string `json:"state"` - Message string `json:"message,omitempty"` + State string `json:"state" example:"ERROR"` + Message string `json:"message,omitempty" example:"Moira has been turned off for maintenance"` } func (*NotifierState) Render(w http.ResponseWriter, r *http.Request) error { diff --git a/api/dto/notification.go b/api/dto/notification.go index 0c685df32..4a6fa6ef1 100644 --- a/api/dto/notification.go +++ b/api/dto/notification.go @@ -8,7 +8,7 @@ import ( ) type NotificationsList struct { - Total int64 `json:"total"` + Total int64 `json:"total" example:"0"` List []*moira.ScheduledNotification `json:"list"` } @@ -17,7 +17,7 @@ func (*NotificationsList) Render(w http.ResponseWriter, r *http.Request) error { } type NotificationDeleteResponse struct { - Result int64 `json:"result"` + Result int64 `json:"result" example:"0"` } func (*NotificationDeleteResponse) Render(w http.ResponseWriter, r *http.Request) error { diff --git a/api/dto/pattern.go b/api/dto/pattern.go index 6df448512..3fdd4710b 100644 --- a/api/dto/pattern.go +++ b/api/dto/pattern.go @@ -14,7 +14,7 @@ func (*PatternList) Render(w http.ResponseWriter, r *http.Request) error { } type PatternData struct { - Metrics []string `json:"metrics"` - Pattern string `json:"pattern"` + Metrics []string `json:"metrics" example:"DevOps.my_server.hdd.freespace_mbytes, DevOps.my_server.hdd.freespace_mbytes, DevOps.my_server.db.*"` + Pattern string `json:"pattern" example:"Devops.my_server.*"` Triggers []TriggerModel `json:"triggers"` } diff --git a/api/dto/tag.go b/api/dto/tag.go index a6f3ae497..9fd435bf4 100644 --- a/api/dto/tag.go +++ b/api/dto/tag.go @@ -8,7 +8,7 @@ import ( ) type TagsData struct { - TagNames []string `json:"list"` + TagNames []string `json:"list" example:"cpu"` } func (*TagsData) Render(w http.ResponseWriter, r *http.Request) error { @@ -16,7 +16,7 @@ func (*TagsData) Render(w http.ResponseWriter, r *http.Request) error { } type MessageResponse struct { - Message string `json:"message"` + Message string `json:"message" example:"tag deleted"` } func (*MessageResponse) Render(w http.ResponseWriter, r *http.Request) error { @@ -28,8 +28,8 @@ type TagsStatistics struct { } type TagStatistics struct { - TagName string `json:"name"` - Triggers []string `json:"triggers"` + TagName string `json:"name" example:"cpu"` + Triggers []string `json:"triggers" example:"bcba82f5-48cf-44c0-b7d6-e1d32c64a88c"` Subscriptions []moira.SubscriptionData `json:"subscriptions"` } diff --git a/api/dto/target.go b/api/dto/target.go index 135e3b52f..2c6f11afb 100644 --- a/api/dto/target.go +++ b/api/dto/target.go @@ -101,10 +101,10 @@ var ( ) type ProblemOfTarget struct { - Argument string `json:"argument"` - Type typeOfProblem `json:"type,omitempty"` - Description string `json:"description,omitempty"` - Position int `json:"position"` + Argument string `json:"argument" example:"consolidateBy"` + Type typeOfProblem `json:"type,omitempty" example:"warn"` + Description string `json:"description,omitempty" example:"This function affects only visual graph representation. It is meaningless in Moira"` + Position int `json:"position" example:"0"` Problems []ProblemOfTarget `json:"problems,omitempty"` } @@ -123,7 +123,7 @@ func (p *ProblemOfTarget) hasError() bool { } type TreeOfProblems struct { - SyntaxOk bool `json:"syntax_ok"` + SyntaxOk bool `json:"syntax_ok" example:"true"` TreeOfProblems *ProblemOfTarget `json:"tree_of_problems,omitempty"` } diff --git a/api/dto/team.go b/api/dto/team.go index 1bb07c7bb..53b129967 100644 --- a/api/dto/team.go +++ b/api/dto/team.go @@ -15,9 +15,9 @@ const ( // TeamModel is a structure that represents team entity in HTTP transfer type TeamModel struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` + ID string `json:"id" example:"d5d98eb3-ee18-4f75-9364-244f67e23b54"` + Name string `json:"name" example:"Infrastructure Team"` + Description string `json:"description" example:"Team that holds all members of infrastructure division"` } // NewTeamModel is a constructor function that creates a new TeamModel using moira.Team @@ -59,7 +59,7 @@ func (t TeamModel) ToMoiraTeam() moira.Team { // SaveTeamResponse is a structure to return team creation result in HTTP response type SaveTeamResponse struct { - ID string `json:"id"` + ID string `json:"id" example:"d5d98eb3-ee18-4f75-9364-244f67e23b54"` } // Render is a function that implements chi Renderer interface for SaveTeamResponse @@ -79,7 +79,7 @@ func (UserTeams) Render(w http.ResponseWriter, r *http.Request) error { // TeamMembers is a structure that represents a team members in HTTP transfer type TeamMembers struct { - Usernames []string `json:"usernames"` + Usernames []string `json:"usernames" example:"anonymous"` } // Bind is a method that implements Binder interface from chi and checks that validity of data in request @@ -96,7 +96,7 @@ func (TeamMembers) Render(w http.ResponseWriter, r *http.Request) error { } type TeamSettings struct { - TeamID string `json:"team_id"` + TeamID string `json:"team_id" example:"d5d98eb3-ee18-4f75-9364-244f67e23b54"` Contacts []moira.ContactData `json:"contacts"` Subscriptions []moira.SubscriptionData `json:"subscriptions"` } diff --git a/api/dto/triggers.go b/api/dto/triggers.go index 300df70af..c573e1659 100644 --- a/api/dto/triggers.go +++ b/api/dto/triggers.go @@ -36,43 +36,43 @@ func (*TriggersList) Render(http.ResponseWriter, *http.Request) error { type Trigger struct { TriggerModel - Throttling int64 `json:"throttling"` + Throttling int64 `json:"throttling" example:"0"` } // TriggerModel is moira.Trigger api representation type TriggerModel struct { // Trigger unique ID - ID string `json:"id"` + ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` // Trigger name - Name string `json:"name"` + Name string `json:"name" example:"Not enough disk space left"` // Description string - Desc *string `json:"desc,omitempty"` + Desc *string `json:"desc,omitempty" example:"check the size of /var/log"` // Graphite-like targets: t1, t2, ... - Targets []string `json:"targets"` + Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` // WARN threshold - WarnValue *float64 `json:"warn_value"` + WarnValue *float64 `json:"warn_value" example:"500"` // ERROR threshold - ErrorValue *float64 `json:"error_value"` + ErrorValue *float64 `json:"error_value" example:"1000"` // Could be: rising, falling, expression - TriggerType string `json:"trigger_type"` + TriggerType string `json:"trigger_type" example:"rising"` // Set of tags to manipulate subscriptions - Tags []string `json:"tags"` + Tags []string `json:"tags" example:"server,disk"` // When there are no metrics for trigger, Moira will switch metric to TTLState state after TTL seconds - TTLState *moira.TTLState `json:"ttl_state,omitempty"` + TTLState *moira.TTLState `json:"ttl_state,omitempty" example:"NODATA"` // When there are no metrics for trigger, Moira will switch metric to TTLState state after TTL seconds - TTL int64 `json:"ttl,omitempty"` + TTL int64 `json:"ttl,omitempty" example:"600"` // Determines when Moira should monitor trigger Schedule *moira.ScheduleData `json:"sched,omitempty"` // Used if you need more complex logic than provided by WARN/ERROR values - Expression string `json:"expression"` + Expression string `json:"expression" example:""` // Graphite patterns for trigger - Patterns []string `json:"patterns"` + Patterns []string `json:"patterns" example:""` // Shows if trigger is remote (graphite-backend) based or stored inside Moira-Redis DB - IsRemote bool `json:"is_remote"` + IsRemote bool `json:"is_remote" example:"false"` // If true, first event NODATA → OK will be omitted - MuteNewMetrics bool `json:"mute_new_metrics"` + MuteNewMetrics bool `json:"mute_new_metrics" example:"false"` // A list of targets that have only alone metrics - AloneMetrics map[string]bool `json:"alone_metrics"` + AloneMetrics map[string]bool `json:"alone_metrics" example:"t1:true"` // Datetime when the trigger was created CreatedAt *time.Time `json:"created_at"` // Datetime when the trigger was updated @@ -366,7 +366,7 @@ type TriggerCheckResponse struct { type TriggerCheck struct { *moira.CheckData - TriggerID string `json:"trigger_id"` + TriggerID string `json:"trigger_id" example:"trigger_id"` } func (*TriggerCheck) Render(http.ResponseWriter, *http.Request) error { @@ -380,7 +380,7 @@ func (*MetricsMaintenance) Bind(*http.Request) error { } type TriggerMaintenance struct { - Trigger *int64 `json:"trigger"` + Trigger *int64 `json:"trigger" example:"1594225165"` Metrics map[string]int64 `json:"metrics"` } @@ -389,7 +389,7 @@ func (*TriggerMaintenance) Bind(*http.Request) error { } type ThrottlingResponse struct { - Throttling int64 `json:"throttling"` + Throttling int64 `json:"throttling" example:"0"` } func (*ThrottlingResponse) Render(http.ResponseWriter, *http.Request) error { @@ -397,8 +397,8 @@ func (*ThrottlingResponse) Render(http.ResponseWriter, *http.Request) error { } type SaveTriggerResponse struct { - ID string `json:"id"` - Message string `json:"message"` + ID string `json:"id" example:"trigger_id"` + Message string `json:"message" example:"trigger created"` CheckResult TriggerCheckResponse `json:"checkResult,omitempty"` } @@ -426,7 +426,7 @@ type TriggerDump struct { } type TriggersSearchResultDeleteResponse struct { - PagerID string `json:"pager_id"` + PagerID string `json:"pager_id" example:"292516ed-4924-4154-a62c-ebe312431fce"` } func (TriggersSearchResultDeleteResponse) Render(http.ResponseWriter, *http.Request) error { diff --git a/api/dto/user.go b/api/dto/user.go index c03b94bd4..68d45eca2 100644 --- a/api/dto/user.go +++ b/api/dto/user.go @@ -18,7 +18,7 @@ func (*UserSettings) Render(w http.ResponseWriter, r *http.Request) error { } type User struct { - Login string `json:"login"` + Login string `json:"login" example:"john"` } func (*User) Render(w http.ResponseWriter, r *http.Request) error { diff --git a/api/error_response.go b/api/error_response.go index fb4d40658..08dfec426 100644 --- a/api/error_response.go +++ b/api/error_response.go @@ -85,3 +85,35 @@ var ErrNotFound = &ErrorResponse{HTTPStatusCode: http.StatusNotFound, StatusText // ErrMethodNotAllowed is default 405 router method not allowed var ErrMethodNotAllowed = &ErrorResponse{HTTPStatusCode: http.StatusMethodNotAllowed, StatusText: "Method not allowed."} + +// Examples for `swaggo`: + +type ErrorInternalServerExample struct { + StatusText string `json:"status" example:"Internal Server Error"` + ErrorText string `json:"error" example:"server error during request handling"` +} + +type ErrorInvalidRequestExample struct { + StatusText string `json:"status" example:"Invalid request"` + ErrorText string `json:"error" example:"resource with the ID does not exist"` +} + +type ErrorRenderExample struct { + StatusText string `json:"status" example:"Error rendering response"` + ErrorText string `json:"error" example:"rendering error"` +} + +type ErrorNotFoundExample struct { + StatusText string `json:"status" example:"Resource not found"` + ErrorText string `json:"error" example:"resource with ID '66741a8c-c2ba-4357-a2c9-ee78e0e7' does not exist"` +} + +type ErrorForbiddenExample struct { + StatusText string `json:"status" example:"Forbidden"` + ErrorText string `json:"error" example:"you cannot access this resource"` +} + +type ErrorRemoteServerUnavailableExample struct { + StatusText string `json:"status" example:"Remote server unavailable"` + ErrorText string `json:"error" example:"Remote server error, please contact administrator"` +} diff --git a/api/handler/config.go b/api/handler/config.go index c758f2145..ea62edc2b 100644 --- a/api/handler/config.go +++ b/api/handler/config.go @@ -2,6 +2,24 @@ package handler import "net/http" +type ContactExample struct { + Type string `json:"type" example:"telegram"` + Label string `json:"label" example:"Telegram"` +} + +type ConfigurationResponse struct { + RemoteAllowed bool `json:"remoteAllowed" example:"false"` + Contacts []ContactExample `json:"contacts"` +} + +// nolint: gofmt,goimports +// +// @summary Get available configuration +// @id get-web-config +// @tags config +// @produce json +// @success 200 {object} ConfigurationResponse "Configuration fetched successfully" +// @router /config [get] func getWebConfig(configContent []byte) http.HandlerFunc { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.Header().Set("Content-Type", "application/json") diff --git a/api/handler/contact.go b/api/handler/contact.go index b824a9037..9c4f338b5 100644 --- a/api/handler/contact.go +++ b/api/handler/contact.go @@ -27,6 +27,16 @@ func contact(router chi.Router) { }) } +// nolint: gofmt,goimports +// +// @summary Gets all Moira contacts +// @id get-all-contacts +// @tags contact +// @produce json +// @success 200 {object} dto.ContactList "Contacts fetched successfully" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact [get] func getAllContacts(writer http.ResponseWriter, request *http.Request) { contacts, err := controller.GetAllContacts(database) if err != nil { @@ -40,6 +50,19 @@ func getAllContacts(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get contact by ID +// @id get-contact-by-id +// @tags contact +// @produce json +// @param contactID path string true "Contact ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.Contact "Successfully received contact" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact/{contactID} [get] func getContactById(writer http.ResponseWriter, request *http.Request) { contactData := request.Context().Value(contactKey).(moira.ContactData) @@ -56,6 +79,19 @@ func getContactById(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Creates a new contact notification for the current user +// @id create-new-contact +// @tags contact +// @accept json +// @produce json +// @param contact body dto.Contact true "Contact data" +// @success 200 {object} dto.Contact "Contact created successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact [put] func createNewContact(writer http.ResponseWriter, request *http.Request) { contact := &dto.Contact{} if err := render.Bind(request, contact); err != nil { @@ -90,6 +126,22 @@ func contactFilter(next http.Handler) http.Handler { }) } +// nolint: gofmt,goimports +// +// @summary Updates an existing notification contact to the values passed in the request body +// @id update-contact +// @accept json +// @produce json +// @param contactID path string true "ID of the contact to update" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param contact body dto.Contact true "Updated contact data" +// @success 200 {object} dto.Contact "Updated contact" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact/{contactID} [put] +// @tags contact func updateContact(writer http.ResponseWriter, request *http.Request) { contactDTO := dto.Contact{} if err := render.Bind(request, &contactDTO); err != nil { @@ -108,6 +160,20 @@ func updateContact(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Deletes notification contact for the current user and remove the contact ID from all subscriptions +// @id remove-contact +// @accept json +// @produce json +// @tags contact +// @param contactID path string true "ID of the contact to remove" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Contact has been deleted" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact/{contactID} [delete] func removeContact(writer http.ResponseWriter, request *http.Request) { contactData := request.Context().Value(contactKey).(moira.ContactData) err := controller.RemoveContact(database, contactData.ID, contactData.User, "") @@ -116,6 +182,19 @@ func removeContact(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Push a test notification to verify that the contact is properly set up +// @id send-test-contact-notification +// @accept json +// @produce json +// @param contactID path string true "The ID of the target contact" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Test successful" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact/{contactID}/test [post] +// @tags contact func sendTestContactNotification(writer http.ResponseWriter, request *http.Request) { contactID := middleware.GetContactID(request) err := controller.SendTestContactNotification(database, contactID) diff --git a/api/handler/contact_events.go b/api/handler/contact_events.go index e1192a254..7c57b409e 100644 --- a/api/handler/contact_events.go +++ b/api/handler/contact_events.go @@ -24,6 +24,22 @@ func contactEvents(router chi.Router) { }) } +// nolint: gofmt,goimports +// +// @summary Get contact events by ID with time range +// @id get-contact-events-by-id +// @tags contact +// @produce json +// @param contactID path string true "Contact ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param from query string false "Start time of the time range" default(-3hour) +// @param to query string false "End time of the time range" default(now) +// @success 200 {object} dto.ContactEventItemList "Successfully received contact events" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /contact/{contactID}/events [get] func getContactByIdWithEvents(writer http.ResponseWriter, request *http.Request) { contactData := request.Context().Value(contactKey).(moira.ContactData) fromStr := middleware.GetFromStr(request) diff --git a/api/handler/event.go b/api/handler/event.go index 10f162e5c..398ad96a8 100644 --- a/api/handler/event.go +++ b/api/handler/event.go @@ -15,6 +15,21 @@ func event(router chi.Router) { router.Delete("/all", deleteAllEvents) } +// nolint: gofmt,goimports +// +// @summary Gets all trigger events for current page and their count +// @id get-events-list +// @tags event +// @produce json +// @param triggerID path string true "The ID of updated trigger" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param size query int false "Number of items to be displayed on one page" default(100) +// @param p query int false "Defines the number of the displayed page. E.g, p=2 would display the 2nd page" default(0) +// @success 200 {object} dto.EventsList "Events fetched successfully" +// @Failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @Failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @Failure 422 {object} api.ErrorRenderExample "Render error" +// @Failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /event/{triggerID} [get] func getEventsList(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) size := middleware.GetSize(request) @@ -29,6 +44,15 @@ func getEventsList(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Deletes all notification events +// @id delete-all-events +// @tags event +// @produce json +// @success 200 "Events removed successfully" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /event/all [delete] func deleteAllEvents(writer http.ResponseWriter, request *http.Request) { if errorResponse := controller.DeleteAllEvents(database); errorResponse != nil { render.Render(writer, request, errorResponse) //nolint diff --git a/api/handler/handler.go b/api/handler/handler.go index cad6f07c9..54d7e3254 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -8,10 +8,13 @@ import ( "github.com/go-chi/render" metricSource "github.com/moira-alert/moira/metric_source" "github.com/rs/cors" + httpSwagger "github.com/swaggo/http-swagger" "github.com/moira-alert/moira" "github.com/moira-alert/moira/api" moiramiddle "github.com/moira-alert/moira/api/middleware" + + _ "github.com/moira-alert/moira/docs" // docs is generated by Swag CLI, you have to import it. ) var database moira.Database @@ -33,6 +36,53 @@ func NewHandler(db moira.Database, log moira.Logger, index moira.Searcher, confi router.NotFound(notFoundHandler) router.MethodNotAllowed(methodNotAllowedHandler) + // @title Moira Alert + // @version master + // @description This is an API description for [Moira Alert API](https://moira.readthedocs.io/en/latest/overview.html) + // @description Check us out on [Github](https://github.com/moira-alert) or look up our [guide on getting started with Moira](https://moira.readthedocs.io) + // @contact.name Contact Moira Team + // @contact.email opensource@skbkontur.com + // @license.name MIT + // @BasePath /api + // + // @tag.name contact + // @tag.description APIs for working with Moira contacts. For more details, see + // + // @tag.name config + // @tag.description View Moira's runtime configuration. For more details, see + // + // @tag.name event + // @tag.description APIs for interacting with notification events. See for details + // + // @tag.name health + // @tag.description interact with Moira states/health status. See for details + // + // @tag.name notification + // @tag.description manage notifications that are currently in queue. See + // + // @tag.name pattern + // @tag.description APIs for interacting with graphite patterns in Moira. See + // + // @tag.name subscription + // @tag.description APIs for managing a user's subscription(s). See to learn about Moira subscriptions + // + // @tag.name tag + // @tag.description APIs for managing tags (a grouping of tags and subscriptions). See + // + // @tag.name trigger + // @tag.description APIs for interacting with Moira triggers. See to learn about Triggers + // + // @tag.name team + // @tag.description APIs for interacting with Moira teams + // + // @tag.name teamSubscription + // @tag.description APIs for interacting with Moira subscriptions owned by certain team + // + // @tag.name teamContact + // @tag.description APIs for interacting with Moira contacts owned by certain team + // + // @tag.name user + // @tag.description APIs for interacting with Moira users router.Route("/api", func(router chi.Router) { router.Use(moiramiddle.DatabaseContext(database)) router.Get("/config", getWebConfig(webConfigContent)) @@ -45,12 +95,15 @@ func NewHandler(db moira.Database, log moira.Logger, index moira.Searcher, confi router.Route("/notification", notification) router.Route("/health", health) router.Route("/teams", teams) - router.Route("/contact", func(router chi.Router) { contact(router) contactEvents(router) }) + router.Get("/swagger/*", httpSwagger.Handler( + httpSwagger.URL("/api/swagger/doc.json"), + )) }) + if config.EnableCORS { return cors.AllowAll().Handler(router) } diff --git a/api/handler/health.go b/api/handler/health.go index 30eb5c1f2..e8affaa7f 100644 --- a/api/handler/health.go +++ b/api/handler/health.go @@ -15,6 +15,16 @@ func health(router chi.Router) { router.Put("/notifier", setNotifierState) } +// nolint: gofmt,goimports +// +// @summary Get notifier state +// @id get-notifier-state +// @tags health +// @produce json +// @success 200 {object} dto.NotifierState "Notifier state retrieved" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /health/notifier [get] func getNotifierState(writer http.ResponseWriter, request *http.Request) { state, err := controller.GetNotifierState(database) if err != nil { diff --git a/api/handler/notification.go b/api/handler/notification.go index 1ab60441f..92d879f00 100644 --- a/api/handler/notification.go +++ b/api/handler/notification.go @@ -18,6 +18,19 @@ func notification(router chi.Router) { router.Delete("/all", deleteAllNotifications) } +// nolint: gofmt,goimports +// +// @summary Gets a paginated list of notifications, all notifications are fetched if end = -1 and start = 0 +// @id get-notifications +// @tags notification +// @produce json +// @param start query int false "Default Value: 0" default(0) +// @param end query int false "Default Value: -1" default(-1) +// @success 200 {object} dto.NotificationsList "Notifications fetched successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /notification [get] func getNotification(writer http.ResponseWriter, request *http.Request) { urlValues, err := url.ParseQuery(request.URL.RawQuery) if err != nil { @@ -46,6 +59,18 @@ func getNotification(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete a notification by id +// @id delete-notification +// @tags notification +// @param id query string true "The ID of deleted notification" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @produce json +// @success 200 {object} dto.NotificationDeleteResponse "Notification have been deleted" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /notification [delete] func deleteNotification(writer http.ResponseWriter, request *http.Request) { urlValues, err := url.ParseQuery(request.URL.RawQuery) if err != nil { diff --git a/api/handler/pattern.go b/api/handler/pattern.go index ac2c20327..148c6b98f 100644 --- a/api/handler/pattern.go +++ b/api/handler/pattern.go @@ -16,6 +16,16 @@ func pattern(router chi.Router) { router.Delete("/{pattern}", deletePattern) } +// nolint: gofmt,goimports +// +// @summary Get all patterns +// @id get-all-patterns +// @tags pattern +// @produce json +// @success 200 {object} dto.PatternList "Patterns fetched successfully" +// @Failure 422 {object} api.ErrorRenderExample "Render error" +// @Failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /pattern [get] func getAllPatterns(writer http.ResponseWriter, request *http.Request) { logger := middleware.GetLoggerEntry(request) patternsList, err := controller.GetAllPatterns(database, logger) @@ -28,6 +38,17 @@ func getAllPatterns(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Deletes a Moira pattern +// @id delete-pattern +// @tags pattern +// @produce json +// @param pattern path string true "Trigger pattern to operate on" default(DevOps.my_server.hdd.freespace_mbytes) +// @success 200 "Pattern deleted successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /pattern/{pattern} [delete] func deletePattern(writer http.ResponseWriter, request *http.Request) { pattern := chi.URLParam(request, "pattern") if pattern == "" { diff --git a/api/handler/subscription.go b/api/handler/subscription.go index 1a958e513..833abbfb5 100644 --- a/api/handler/subscription.go +++ b/api/handler/subscription.go @@ -26,6 +26,16 @@ func subscription(router chi.Router) { }) } +// nolint: gofmt,goimports +// +// @summary Get all subscriptions +// @id get-user-subscriptions +// @tags subscription +// @produce json +// @success 200 {object} dto.SubscriptionList "Subscriptions fetched successfully" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /subscription [get] func getUserSubscriptions(writer http.ResponseWriter, request *http.Request) { userLogin := middleware.GetLogin(request) contacts, err := controller.GetUserSubscriptions(database, userLogin) @@ -39,6 +49,19 @@ func getUserSubscriptions(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Create a new subscription +// @id create-subscription +// @tags subscription +// @accept json +// @produce json +// @param subscription body dto.Subscription true "Subscription data" +// @success 200 {object} dto.Subscription "Subscription created successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /subscription [put] func createSubscription(writer http.ResponseWriter, request *http.Request) { subscription := &dto.Subscription{} if err := render.Bind(request, subscription); err != nil { @@ -78,6 +101,22 @@ func subscriptionFilter(next http.Handler) http.Handler { }) } +// nolint: gofmt,goimports +// +// @summary Update a subscription +// @id update-subscription +// @tags subscription +// @accept json +// @produce json +// @param subscriptionID path string true "ID of the subscription to update" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param subscription body dto.Subscription true "Updated subscription data" +// @success 200 {object} dto.Subscription "Subscription updated successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /subscription/{subscriptionID} [put] func updateSubscription(writer http.ResponseWriter, request *http.Request) { subscription := &dto.Subscription{} if err := render.Bind(request, subscription); err != nil { @@ -109,6 +148,18 @@ func updateSubscription(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete a subscription +// @id remove-subscription +// @tags subscription +// @produce json +// @param subscriptionID path string true "ID of the subscription to remove" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Subscription deleted" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /subscription/{subscriptionID} [delete] func removeSubscription(writer http.ResponseWriter, request *http.Request) { subscriptionID := middleware.GetSubscriptionID(request) if err := controller.RemoveSubscription(database, subscriptionID); err != nil { @@ -116,6 +167,18 @@ func removeSubscription(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Send a test notification for a subscription +// @id send-test-notification +// @tags subscription +// @produce json +// @param subscriptionID path string true "ID of the subscription to send the test notification" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Test notification sent successfully" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /subscription/{subscriptionID}/test [put] func sendTestNotification(writer http.ResponseWriter, request *http.Request) { subscriptionID := middleware.GetSubscriptionID(request) if err := controller.SendTestNotification(database, subscriptionID); err != nil { diff --git a/api/handler/tag.go b/api/handler/tag.go index 40068dee8..6faaea6ae 100644 --- a/api/handler/tag.go +++ b/api/handler/tag.go @@ -19,6 +19,16 @@ func tag(router chi.Router) { }) } +// nolint: gofmt,goimports +// +// @summary Get all tags +// @id get-all-tags +// @tags tag +// @produce json +// @success 200 {object} dto.TagsData "Tags fetched successfully" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /tag [get] func getAllTags(writer http.ResponseWriter, request *http.Request) { tagData, err := controller.GetAllTags(database) if err != nil { @@ -32,6 +42,16 @@ func getAllTags(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get all tags and their subscriptions +// @id get-all-tags-and-subscriptions +// @tags tag +// @produce json +// @success 200 {object} dto.TagsStatistics "Successful" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /tag/stats [get] func getAllTagsAndSubscriptions(writer http.ResponseWriter, request *http.Request) { logger := middleware.GetLoggerEntry(request) data, err := controller.GetAllTagsAndSubscriptions(database, logger) @@ -45,6 +65,18 @@ func getAllTagsAndSubscriptions(writer http.ResponseWriter, request *http.Reques } } +// nolint: gofmt,goimports +// +// @summary Remove a tag +// @id remove-tag +// @tags tag +// @produce json +// @param tag path string true "Name of the tag to remove" default(cpu) +// @success 200 {object} dto.MessageResponse "Tag removed successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /tag/{tag} [delete] func removeTag(writer http.ResponseWriter, request *http.Request) { tagName := middleware.GetTag(request) response, err := controller.RemoveTag(database, tagName) diff --git a/api/handler/team.go b/api/handler/team.go index 44c644903..bc079e6b4 100644 --- a/api/handler/team.go +++ b/api/handler/team.go @@ -46,6 +46,19 @@ func usersFilterForTeams(next http.Handler) http.Handler { }) } +// nolint: gofmt,goimports +// +// @summary Create a new team +// @id create-team +// @tags team +// @accept json +// @produce json +// @param team body dto.TeamModel true "Team data" +// @success 200 {object} dto.SaveTeamResponse "Team created successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams [post] func createTeam(writer http.ResponseWriter, request *http.Request) { user := middleware.GetLogin(request) team := dto.TeamModel{} @@ -65,6 +78,16 @@ func createTeam(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get all teams +// @id get-all-teams +// @tags team +// @produce json +// @success 200 {object} dto.UserTeams "Teams fetched successfully" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams [get] func getAllTeams(writer http.ResponseWriter, request *http.Request) { user := middleware.GetLogin(request) response, err := controller.GetUserTeams(database, user) @@ -79,6 +102,19 @@ func getAllTeams(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get a team by ID +// @id get-team +// @tags team +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TeamModel "Team updated successfully" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID} [get] func getTeam(writer http.ResponseWriter, request *http.Request) { teamID := middleware.GetTeamID(request) @@ -94,6 +130,22 @@ func getTeam(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Update existing team +// @id update-team +// @tags team +// @accept json +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param team body dto.TeamModel true "Updated team data" +// @success 200 {object} dto.SaveTeamResponse "Team updated successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID} [patch] func updateTeam(writer http.ResponseWriter, request *http.Request) { team := dto.TeamModel{} err := render.Bind(request, &team) @@ -115,6 +167,20 @@ func updateTeam(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete a team +// @id delete-team +// @tags team +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.SaveTeamResponse "Team has been successfully deleted" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID} [delete] func deleteTeam(writer http.ResponseWriter, request *http.Request) { userLogin := middleware.GetLogin(request) teamID := middleware.GetTeamID(request) @@ -130,6 +196,19 @@ func deleteTeam(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get users of a team +// @id get-team-users +// @tags team +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TeamMembers "Users fetched successfully" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/users [get] func getTeamUsers(writer http.ResponseWriter, request *http.Request) { teamID := middleware.GetTeamID(request) @@ -145,6 +224,22 @@ func getTeamUsers(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Set users of a team +// @id set-team-users +// @tags team +// @accept json +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param usernames body dto.TeamMembers true "Usernames to set as team members" +// @success 200 {object} dto.TeamMembers "Team updated successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/users [put] func setTeamUsers(writer http.ResponseWriter, request *http.Request) { members := dto.TeamMembers{} err := render.Bind(request, &members) @@ -167,6 +262,22 @@ func setTeamUsers(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Add users to a team +// @id add-team-users +// @tags team +// @accept json +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param usernames body dto.TeamMembers true "Usernames to add to the team" +// @success 200 {object} dto.TeamMembers "Team updated successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/users [post] func addTeamUsers(writer http.ResponseWriter, request *http.Request) { members := dto.TeamMembers{} err := render.Bind(request, &members) @@ -188,6 +299,21 @@ func addTeamUsers(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete a user from a team +// @id delete-team-user +// @tags team +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param teamUserID path string true "User login in methods related to teams manipulation" default(anonymous) +// @success 200 {object} dto.TeamMembers "Team updated successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/users/{teamUserID} [delete] func deleteTeamUser(writer http.ResponseWriter, request *http.Request) { teamID := middleware.GetTeamID(request) userID := middleware.GetTeamUserID(request) @@ -204,6 +330,19 @@ func deleteTeamUser(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get team settings +// @id get-team-settings +// @tags team +// @produce json +// @param teamID path string true "ID of the team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TeamSettings "Team settings" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/settings [get] func getTeamSettings(writer http.ResponseWriter, request *http.Request) { teamID := middleware.GetTeamID(request) teamSettings, err := controller.GetTeamSettings(database, teamID) diff --git a/api/handler/team_contact.go b/api/handler/team_contact.go index 70f094db2..1ed5874ab 100644 --- a/api/handler/team_contact.go +++ b/api/handler/team_contact.go @@ -15,6 +15,22 @@ func teamContact(router chi.Router) { router.Post("/", createNewTeamContact) } +// nolint: gofmt,goimports +// +// @summary Create a new team contact +// @id create-new-team-contact +// @tags teamContact +// @accept json +// @produce json +// @param teamID path string true "The ID of team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param contact body dto.Contact true "Team contact data" +// @success 200 {object} dto.Contact "Team contact created successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/contacts [post] func createNewTeamContact(writer http.ResponseWriter, request *http.Request) { contact := &dto.Contact{} if err := render.Bind(request, contact); err != nil { diff --git a/api/handler/team_subscription.go b/api/handler/team_subscription.go index 4d7d29939..c913fff79 100644 --- a/api/handler/team_subscription.go +++ b/api/handler/team_subscription.go @@ -16,6 +16,22 @@ func teamSubscription(router chi.Router) { router.Post("/", createTeamSubscription) } +// nolint: gofmt,goimports +// +// @summary Create a new team subscription +// @id create-new-team-subscription +// @tags teamSubscription +// @accept json +// @produce json +// @param teamID path string true "The ID of team" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param subscription body dto.Subscription true "Team subscription data" +// @success 200 {object} dto.Subscription "Team subscription created successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 403 {object} api.ErrorForbiddenExample "Forbidden" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /teams/{teamID}/subscriptions [post] func createTeamSubscription(writer http.ResponseWriter, request *http.Request) { subscription := &dto.Subscription{} if err := render.Bind(request, subscription); err != nil { diff --git a/api/handler/trigger.go b/api/handler/trigger.go index 924f338dd..73d9f574e 100644 --- a/api/handler/trigger.go +++ b/api/handler/trigger.go @@ -30,6 +30,22 @@ func trigger(router chi.Router) { router.Get("/dump", triggerDump) } +// nolint: gofmt,goimports +// +// @summary Update existing trigger +// @id update-trigger +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param validate query bool false "For validating targets" +// @param body body dto.Trigger true "Trigger data" +// @success 200 {object} dto.SaveTriggerResponse "Updated trigger" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @failure 503 {object} api.ErrorRemoteServerUnavailableExample "Remote server unavailable" +// @router /trigger/{triggerID} [put] func updateTrigger(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) @@ -95,6 +111,16 @@ func writeErrorSaveResponse(writer http.ResponseWriter, request *http.Request, t render.JSON(writer, request, response) } +// nolint: gofmt,goimports +// +// @summary Remove trigger +// @id remove-trigger +// @tags trigger +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Successfully removed" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID} [delete] func removeTrigger(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) err := controller.RemoveTrigger(database, triggerID) @@ -103,6 +129,19 @@ func removeTrigger(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get an existing trigger +// @id get-trigger +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param populated query bool false "Populated" default(false) +// @success 200 {object} dto.Trigger "Trigger data" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID} [get] func getTrigger(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) @@ -140,6 +179,18 @@ func checkingTemplateFilling(request *http.Request, trigger dto.Trigger) *api.Er return nil } +// nolint: gofmt,goimports +// +// @summary Get the trigger state as at last check +// @id get-trigger-state +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TriggerCheck "Trigger state fetched successful" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/state [get] func getTriggerState(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) triggerState, err := controller.GetTriggerLastCheck(database, triggerID) @@ -152,6 +203,17 @@ func getTriggerState(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get a trigger with its throttling i.e its next allowed message time +// @id get-trigger-throttling +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.ThrottlingResponse "Trigger throttle info retrieved" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @router /trigger/{triggerID}/throttling [get] func getTriggerThrottling(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) triggerState, err := controller.GetTriggerThrottling(database, triggerID) @@ -164,6 +226,16 @@ func getTriggerThrottling(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Deletes throttling for a trigger +// @id delete-trigger-throttling +// @tags trigger +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Trigger throttling has been deleted" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/throttling [delete] func deleteThrottling(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) err := controller.DeleteTriggerThrottling(database, triggerID) @@ -172,6 +244,19 @@ func deleteThrottling(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Set metrics and the trigger itself to maintenance mode +// @id set-trigger-maintenance +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param body body dto.TriggerMaintenance true "Maintenance data" +// @success 200 "Trigger or metric have been scheduled for maintenance" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/setMaintenance [put] func setTriggerMaintenance(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) triggerMaintenance := dto.TriggerMaintenance{} @@ -188,6 +273,17 @@ func setTriggerMaintenance(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get trigger dump +// @id get-trigger-dump +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TriggerDump "Trigger dump" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/dump [get] func triggerDump(writer http.ResponseWriter, request *http.Request) { triggerID, log := prepareTriggerContext(request) diff --git a/api/handler/trigger_metrics.go b/api/handler/trigger_metrics.go index d48360b29..a1f95fa4a 100644 --- a/api/handler/trigger_metrics.go +++ b/api/handler/trigger_metrics.go @@ -21,6 +21,21 @@ func triggerMetrics(router chi.Router) { router.Delete("/nodata", deleteTriggerNodataMetrics) } +// nolint: gofmt,goimports +// +// @summary Get metrics associated with certain trigger +// @id get-trigger-metrics +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param from query string false "Start time for metrics retrieval" default(-10minutes) +// @param to query string false "End time for metrics retrieval" default(now) +// @success 200 {object} dto.TriggerMetrics "Trigger metrics retrieved successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/metrics [get] func getTriggerMetrics(writer http.ResponseWriter, request *http.Request) { metricSourceProvider := middleware.GetTriggerTargetsSourceProvider(request) triggerID := middleware.GetTriggerID(request) @@ -49,6 +64,19 @@ func getTriggerMetrics(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete metric from last check and all trigger pattern metrics +// @id delete-trigger-metric +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param name query string false "Name of the target metric" default(DevOps.my_server.hdd.freespace_mbytes) +// @success 200 "Trigger metric deleted successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/metrics [delete] func deleteTriggerMetric(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) @@ -64,6 +92,18 @@ func deleteTriggerMetric(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete all metrics from last data which are in NODATA state. It also deletes all trigger patterns of those metrics +// @id delete-trigger-nodata-metrics +// @tags trigger +// @produce json +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 "Trigger nodata metrics deleted successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/metrics/nodata [delete] func deleteTriggerNodataMetrics(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) if err := controller.DeleteTriggerNodataMetrics(database, triggerID); err != nil { diff --git a/api/handler/trigger_render.go b/api/handler/trigger_render.go index c2e961211..02794c759 100644 --- a/api/handler/trigger_render.go +++ b/api/handler/trigger_render.go @@ -18,6 +18,24 @@ import ( "github.com/moira-alert/moira/plotting" ) +// nolint: gofmt,goimports +// +// @summary Render trigger metrics plot +// @id render-trigger-metrics +// @tags trigger +// @produce png +// @param triggerID path string true "Trigger ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param target query string false "Target metric name" default(t1) +// @param from query string false "Start time for metrics retrieval" default(-1hour) +// @param to query string false "End time for metrics retrieval" default(now) +// @param timezone query string false "Timezone for rendering" default(UTC) +// @param theme query string false "Plot theme" default(light) +// @param realtime query bool false "Fetch real-time data" default(false) +// @success 200 "Rendered plot image successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/{triggerID}/render [get] func renderTrigger(writer http.ResponseWriter, request *http.Request) { sourceProvider, targetName, from, to, triggerID, fetchRealtimeData, err := getEvaluationParameters(request) if err != nil { diff --git a/api/handler/triggers.go b/api/handler/triggers.go index fe61fae70..3b78433da 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -38,6 +38,16 @@ func triggers(metricSourceProvider *metricSource.SourceProvider, searcher moira. } } +// nolint: gofmt,goimports +// +// @summary Get all triggers +// @id get-all-triggers +// @tags trigger +// @produce json +// @success 200 {object} dto.TriggersList "Fetched all triggers" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger [get] func getAllTriggers(writer http.ResponseWriter, request *http.Request) { triggersList, errorResponse := controller.GetAllTriggers(database) if errorResponse != nil { @@ -51,7 +61,22 @@ func getAllTriggers(writer http.ResponseWriter, request *http.Request) { } } -// createTrigger handler creates moira.Trigger. +// nolint: gofmt,goimports +// createTrigger handler creates moira.Trigger +// +// @summary Create a new trigger +// @id create-trigger +// @tags trigger +// @accept json +// @produce json +// @param validate query bool false "For validating targets" +// @param trigger body dto.Trigger true "Trigger data" +// @success 200 {object} dto.SaveTriggerResponse "Trigger created successfully" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @failure 503 {object} api.ErrorRemoteServerUnavailableExample "Remote server unavailable" +// @router /trigger [put] func createTrigger(writer http.ResponseWriter, request *http.Request) { trigger, err := getTriggerFromRequest(request) if err != nil { @@ -133,6 +158,18 @@ func getMetricTTLByTrigger(request *http.Request, trigger *dto.Trigger) time.Dur return ttl } +// nolint: gofmt,goimports +// +// @summary Validates trigger target +// @id trigger-check +// @tags trigger +// @accept json +// @produce json +// @param trigger body dto.Trigger true "Trigger data" +// @success 200 {object} dto.TriggerCheckResponse "Validation is done, see response body for validation result" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/check [put] func triggerCheck(writer http.ResponseWriter, request *http.Request) { trigger := &dto.Trigger{} response := dto.TriggerCheckResponse{} @@ -157,6 +194,26 @@ func triggerCheck(writer http.ResponseWriter, request *http.Request) { render.JSON(writer, request, response) } +// nolint: gofmt,goimports +// +// @summary Search triggers. Replaces the deprecated `page` path +// @description You can also add filtering by tags, for this purpose add query parameters tags[0]=test, tags[1]=test1 and so on +// @description For example, `/api/trigger/search?tags[0]=test&tags[1]=test1` +// @id search-triggers +// @tags trigger +// @produce json +// @param onlyProblems query boolean false "Only include problems" default(false) +// @param text query string false "Search text" default(cpu) +// @param p query integer false "Page number" default(0) +// @param size query integer false "Page size" default(10) +// @param createPager query boolean false "Create pager" default(false) +// @param pagerID query string false "Pager ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TriggersList "Successfully fetched matching triggers" +// @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/search [get] func searchTriggers(writer http.ResponseWriter, request *http.Request) { request.ParseForm() //nolint onlyErrors := getOnlyProblemsFlag(request) @@ -181,6 +238,17 @@ func searchTriggers(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Delete triggers pager +// @tags trigger +// @produce json +// @param pagerID path string true "Pager ID to delete" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @success 200 {object} dto.TriggersSearchResultDeleteResponse "Successfully deleted pager" +// @failure 404 {object} api.ErrorNotFoundExample "Resource not found" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger/pagers/{pagerID} [delete] func deletePager(writer http.ResponseWriter, request *http.Request) { pagerID := middleware.GetPagerID(request) diff --git a/api/handler/user.go b/api/handler/user.go index 8432f544f..17c856698 100644 --- a/api/handler/user.go +++ b/api/handler/user.go @@ -16,6 +16,15 @@ func user(router chi.Router) { router.Get("/settings", getUserSettings) } +// nolint: gofmt,goimports +// +// @summary Gets the username of the authenticated user if it is available +// @id get-user-name +// @tags user +// @produce json +// @success 200 {object} dto.User "User name fetched successfully" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @router /user [get] func getUserName(writer http.ResponseWriter, request *http.Request) { userLogin := middleware.GetLogin(request) if err := render.Render(writer, request, &dto.User{Login: userLogin}); err != nil { @@ -24,6 +33,16 @@ func getUserName(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get the user's contacts and subscriptions +// @id get-user-settings +// @tags user +// @produce json +// @success 200 {object} dto.UserSettings "Settings fetched successfully" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /user/settings [get] func getUserSettings(writer http.ResponseWriter, request *http.Request) { userLogin := middleware.GetLogin(request) userSettings, err := controller.GetUserSettings(database, userLogin) diff --git a/datatypes.go b/datatypes.go index 2c37daad9..852cba7bd 100644 --- a/datatypes.go +++ b/datatypes.go @@ -42,16 +42,16 @@ const ( // NotificationEvent represents trigger state changes event type NotificationEvent struct { - IsTriggerEvent bool `json:"trigger_event,omitempty"` - Timestamp int64 `json:"timestamp"` - Metric string `json:"metric"` - Value *float64 `json:"value,omitempty"` + IsTriggerEvent bool `json:"trigger_event,omitempty" example:"true"` + Timestamp int64 `json:"timestamp" example:"1590741878"` + Metric string `json:"metric" example:"carbon.agents.*.metricsReceived"` + Value *float64 `json:"value,omitempty" example:"70"` Values map[string]float64 `json:"values,omitempty"` - State State `json:"state"` - TriggerID string `json:"trigger_id"` + State State `json:"state" example:"OK"` + TriggerID string `json:"trigger_id" example:"5ff37996-8927-4cab-8987-970e80d8e0a8"` SubscriptionID *string `json:"sub_id,omitempty"` ContactID string `json:"contact_id,omitempty"` - OldState State `json:"old_state"` + OldState State `json:"old_state" example:"ERROR"` Message *string `json:"msg,omitempty"` MessageEventInfo *EventInfo `json:"event_message"` } @@ -70,7 +70,7 @@ type NotificationEventHistoryItem struct { // EventInfo - a base for creating messages. type EventInfo struct { Maintenance *MaintenanceInfo `json:"maintenance,omitempty"` - Interval *int64 `json:"interval,omitempty"` + Interval *int64 `json:"interval,omitempty" example:"0"` } // CreateMessage - creates a message based on EventInfo. @@ -157,14 +157,14 @@ func NotificationEventsToTemplatingEvents(events NotificationEvents) []templatin // TriggerData represents trigger object type TriggerData struct { - ID string `json:"id"` - Name string `json:"name"` - Desc string `json:"desc"` - Targets []string `json:"targets"` - WarnValue float64 `json:"warn_value"` - ErrorValue float64 `json:"error_value"` - IsRemote bool `json:"is_remote"` - Tags []string `json:"__notifier_trigger_tags"` + ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` + Name string `json:"name" example:"Not enough disk space left"` + Desc string `json:"desc" example:"check the size of /var/log"` + Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` + WarnValue float64 `json:"warn_value" example:"5000"` + ErrorValue float64 `json:"error_value" example:"1000"` + IsRemote bool `json:"is_remote" example:"false"` + Tags []string `json:"__notifier_trigger_tags" example:"server,disk"` } // GetTriggerURI gets frontUri and returns triggerUrl, returns empty string on selfcheck and test notifications @@ -184,47 +184,47 @@ type Team struct { // ContactData represents contact object type ContactData struct { - Type string `json:"type"` - Value string `json:"value"` - ID string `json:"id"` - User string `json:"user"` + Type string `json:"type" example:"mail"` + Value string `json:"value" example:"devops@example.com"` + ID string `json:"id" example:"1dd38765-c5be-418d-81fa-7a5f879c2315"` + User string `json:"user" example:""` Team string `json:"team"` } // SubscriptionData represents user subscription type SubscriptionData struct { - Contacts []string `json:"contacts"` - Tags []string `json:"tags"` + Contacts []string `json:"contacts" example:"acd2db98-1659-4a2f-b227-52d71f6e3ba1"` + Tags []string `json:"tags" example:"server,cpu"` Schedule ScheduleData `json:"sched"` Plotting PlottingData `json:"plotting"` - ID string `json:"id"` - Enabled bool `json:"enabled"` - AnyTags bool `json:"any_tags"` - IgnoreWarnings bool `json:"ignore_warnings,omitempty"` - IgnoreRecoverings bool `json:"ignore_recoverings,omitempty"` - ThrottlingEnabled bool `json:"throttling"` - User string `json:"user"` - TeamID string `json:"team_id"` + ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` + Enabled bool `json:"enabled" example:"true"` + AnyTags bool `json:"any_tags" example:"false"` + IgnoreWarnings bool `json:"ignore_warnings,omitempty" example:"false"` + IgnoreRecoverings bool `json:"ignore_recoverings,omitempty" example:"false"` + ThrottlingEnabled bool `json:"throttling" example:"false"` + User string `json:"user" example:""` + TeamID string `json:"team_id" example:"324516ed-4924-4154-a62c-eb124234fce"` } // PlottingData represents plotting settings type PlottingData struct { - Enabled bool `json:"enabled"` - Theme string `json:"theme"` + Enabled bool `json:"enabled" example:"true"` + Theme string `json:"theme" example:"dark"` } // ScheduleData represents subscription schedule type ScheduleData struct { Days []ScheduleDataDay `json:"days"` - TimezoneOffset int64 `json:"tzOffset"` - StartOffset int64 `json:"startOffset"` - EndOffset int64 `json:"endOffset"` + TimezoneOffset int64 `json:"tzOffset" example:"-60"` + StartOffset int64 `json:"startOffset" example:"0"` + EndOffset int64 `json:"endOffset" example:"1439"` } // ScheduleDataDay represents week day of schedule type ScheduleDataDay struct { - Enabled bool `json:"enabled"` - Name string `json:"name,omitempty"` + Enabled bool `json:"enabled" example:"true"` + Name string `json:"name,omitempty" example:"Mon"` } // ScheduledNotification represent notification object @@ -233,9 +233,9 @@ type ScheduledNotification struct { Trigger TriggerData `json:"trigger"` Contact ContactData `json:"contact"` Plotting PlottingData `json:"plotting"` - Throttled bool `json:"throttled"` - SendFail int `json:"send_fail"` - Timestamp int64 `json:"timestamp"` + Throttled bool `json:"throttled" example:"false"` + SendFail int `json:"send_fail" example:"0"` + Timestamp int64 `json:"timestamp" example:"1594471927"` } // MatchedMetric represents parsed and matched metric data @@ -266,23 +266,23 @@ const ( // Trigger represents trigger data object type Trigger struct { - ID string `json:"id"` - Name string `json:"name"` - Desc *string `json:"desc,omitempty"` - Targets []string `json:"targets"` - WarnValue *float64 `json:"warn_value"` - ErrorValue *float64 `json:"error_value"` - TriggerType string `json:"trigger_type"` - Tags []string `json:"tags"` - TTLState *TTLState `json:"ttl_state,omitempty"` - TTL int64 `json:"ttl,omitempty"` + ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` + Name string `json:"name" example:"Not enough disk space left"` + Desc *string `json:"desc,omitempty" example:"check the size of /var/log"` + Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` + WarnValue *float64 `json:"warn_value" example:"5000"` + ErrorValue *float64 `json:"error_value" example:"1000"` + TriggerType string `json:"trigger_type" example:"rising"` + Tags []string `json:"tags" example:"server,disk"` + TTLState *TTLState `json:"ttl_state,omitempty" example:"NODATA"` + TTL int64 `json:"ttl,omitempty" example:"600"` Schedule *ScheduleData `json:"sched,omitempty"` - Expression *string `json:"expression,omitempty"` + Expression *string `json:"expression,omitempty" example:""` PythonExpression *string `json:"python_expression,omitempty"` - Patterns []string `json:"patterns"` - IsRemote bool `json:"is_remote"` - MuteNewMetrics bool `json:"mute_new_metrics"` - AloneMetrics map[string]bool `json:"alone_metrics"` + Patterns []string `json:"patterns" example:""` + IsRemote bool `json:"is_remote" example:"false"` + MuteNewMetrics bool `json:"mute_new_metrics" example:"false"` + AloneMetrics map[string]bool `json:"alone_metrics" example:"t1:true"` CreatedAt *int64 `json:"created_at"` UpdatedAt *int64 `json:"updated_at"` CreatedBy string `json:"created_by"` @@ -292,7 +292,7 @@ type Trigger struct { // TriggerCheck represents trigger data with last check data and check timestamp type TriggerCheck struct { Trigger - Throttling int64 `json:"throttling"` + Throttling int64 `json:"throttling" example:"0"` LastCheck CheckData `json:"last_check"` Highlights map[string]string `json:"highlights"` } @@ -309,15 +309,15 @@ type CheckData struct { // MetricsToTargetRelation is a map that holds relation between metric names that was alone during last // check and targets that fetched this metric // {"t1": "metric.name.1", "t2": "metric.name.2"} - MetricsToTargetRelation map[string]string `json:"metrics_to_target_relation"` - Score int64 `json:"score"` - State State `json:"state"` - Maintenance int64 `json:"maintenance,omitempty"` + MetricsToTargetRelation map[string]string `json:"metrics_to_target_relation" example:"t1:metric.name.1,t2:metric.name.2"` + Score int64 `json:"score" example:"100"` + State State `json:"state" example:"OK"` + Maintenance int64 `json:"maintenance,omitempty" example:"0"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` - Timestamp int64 `json:"timestamp,omitempty"` - EventTimestamp int64 `json:"event_timestamp,omitempty"` - LastSuccessfulCheckTimestamp int64 `json:"last_successful_check_timestamp"` - Suppressed bool `json:"suppressed,omitempty"` + Timestamp int64 `json:"timestamp,omitempty" example:"1590741916"` + EventTimestamp int64 `json:"event_timestamp,omitempty" example:"1590741878"` + LastSuccessfulCheckTimestamp int64 `json:"last_successful_check_timestamp" example:"1590741916"` + Suppressed bool `json:"suppressed,omitempty" example:"true"` SuppressedState State `json:"suppressed_state,omitempty"` Message string `json:"msg,omitempty"` } @@ -334,14 +334,14 @@ func (checkData *CheckData) RemoveMetricsToTargetRelation() { // MetricState represents metric state data for given timestamp type MetricState struct { - EventTimestamp int64 `json:"event_timestamp"` - State State `json:"state"` - Suppressed bool `json:"suppressed"` + EventTimestamp int64 `json:"event_timestamp" example:"1590741878"` + State State `json:"state" example:"OK"` + Suppressed bool `json:"suppressed" example:"false"` SuppressedState State `json:"suppressed_state,omitempty"` - Timestamp int64 `json:"timestamp"` - Value *float64 `json:"value,omitempty"` + Timestamp int64 `json:"timestamp" example:"1590741878"` + Value *float64 `json:"value,omitempty" example:"70"` Values map[string]float64 `json:"values,omitempty"` - Maintenance int64 `json:"maintenance,omitempty"` + Maintenance int64 `json:"maintenance,omitempty" example:"0"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` // AloneMetrics map[string]string `json:"alone_metrics"` // represents a relation between name of alone metrics and their targets } @@ -360,9 +360,9 @@ func (metricState *MetricState) GetMaintenance() (MaintenanceInfo, int64) { // MaintenanceInfo represents user and time set/unset maintenance type MaintenanceInfo struct { StartUser *string `json:"setup_user"` - StartTime *int64 `json:"setup_time"` + StartTime *int64 `json:"setup_time" example:"0"` StopUser *string `json:"remove_user"` - StopTime *int64 `json:"remove_time"` + StopTime *int64 `json:"remove_time" example:"0"` } // Set maintanace start and stop users and times diff --git a/docs/docs.go b/docs/docs.go new file mode 100644 index 000000000..a8f0f0241 --- /dev/null +++ b/docs/docs.go @@ -0,0 +1,3 @@ +// Will be generated in `Dockerfile.api` by the `make spec` command for swagger docs. DO NOT EDIT. + +package docs \ No newline at end of file diff --git a/go.mod b/go.mod index 14e7107fe..09559cd88 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,10 @@ require ( gopkg.in/yaml.v2 v2.4.0 ) -require github.com/mitchellh/mapstructure v1.5.0 +require ( + github.com/mitchellh/mapstructure v1.5.0 + github.com/swaggo/http-swagger v1.3.4 +) require ( bitbucket.org/tebeka/strftime v0.0.0-20140926081919-2194253a23c0 // indirect @@ -146,7 +149,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.14.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.1 // indirect + github.com/stretchr/testify v1.8.4 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/tinylib/msgp v1.1.8 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect @@ -158,12 +161,12 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.6.0 // indirect + golang.org/x/crypto v0.12.0 // indirect golang.org/x/exp v0.0.0-20200924195034-c827fd4f18b9 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect gonum.org/v1/gonum v0.12.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect @@ -173,13 +176,24 @@ require ( ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/spec v0.20.9 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.11 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect + github.com/swaggo/files v1.0.1 // indirect + github.com/swaggo/swag v1.8.12 // indirect + golang.org/x/tools v0.11.1 // indirect ) // Have to exclude version that is incorectly retracted by authors diff --git a/go.sum b/go.sum index 09b73f652..b7930c20f 100644 --- a/go.sum +++ b/go.sum @@ -399,13 +399,16 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JaderDias/movingmedian v0.0.0-20220813210630-d8c6b6de8835 h1:mbxQnovjDz5SvlatpxkbiMvybHH1hsSEu6OhPDLlfU8= github.com/JaderDias/movingmedian v0.0.0-20220813210630-d8c6b6de8835/go.mod h1:zsfWLaDctbM7aV1TsQAwkVswuKQ0k7PK4rjC1VZqpbI= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= @@ -602,6 +605,21 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= +github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= @@ -793,6 +811,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -824,7 +844,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -842,6 +862,11 @@ github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 h1:Khvh6waxG1cHc4Cz5ef9n3XVCxRWpAKUtqg9PJl5+y8= @@ -903,6 +928,7 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -1053,12 +1079,19 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= +github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= +github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= +github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= +github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w= +github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= @@ -1131,8 +1164,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1178,6 +1211,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1245,8 +1279,9 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1386,8 +1421,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1409,8 +1444,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1478,6 +1513,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc= +golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1739,6 +1776,7 @@ gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1768,6 +1806,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From db5a20f52fc2597412cb5686bc7932a8414d7850 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Mon, 21 Aug 2023 14:43:41 +0600 Subject: [PATCH 05/46] fix(notifier|mattermost) added plot sending (#897) * fix(notifier|mattermost) added plot sending --- mock/notifier/mattermost/client.go | 16 +++++++ senders/mattermost/client.go | 1 + senders/mattermost/sender.go | 52 +++++++++++++++++++--- senders/mattermost/sender_internal_test.go | 24 +++++++++- 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/mock/notifier/mattermost/client.go b/mock/notifier/mattermost/client.go index 8b066cd49..3407b00bd 100644 --- a/mock/notifier/mattermost/client.go +++ b/mock/notifier/mattermost/client.go @@ -61,3 +61,19 @@ func (mr *MockClientMockRecorder) SetToken(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetToken", reflect.TypeOf((*MockClient)(nil).SetToken), arg0) } + +// UploadFile mocks base method. +func (m *MockClient) UploadFile(arg0 []byte, arg1, arg2 string) (*model.FileUploadResponse, *model.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UploadFile", arg0, arg1, arg2) + ret0, _ := ret[0].(*model.FileUploadResponse) + ret1, _ := ret[1].(*model.Response) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// UploadFile indicates an expected call of UploadFile. +func (mr *MockClientMockRecorder) UploadFile(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadFile", reflect.TypeOf((*MockClient)(nil).UploadFile), arg0, arg1, arg2) +} diff --git a/senders/mattermost/client.go b/senders/mattermost/client.go index 60a0b4e74..b91163b15 100644 --- a/senders/mattermost/client.go +++ b/senders/mattermost/client.go @@ -6,4 +6,5 @@ import "github.com/mattermost/mattermost-server/v6/model" type Client interface { SetToken(token string) CreatePost(post *model.Post) (*model.Post, *model.Response, error) + UploadFile(data []byte, channelId string, filename string) (*model.FileUploadResponse, *model.Response, error) } diff --git a/senders/mattermost/sender.go b/senders/mattermost/sender.go index e4079daa7..52ec2e25e 100644 --- a/senders/mattermost/sender.go +++ b/senders/mattermost/sender.go @@ -28,6 +28,7 @@ type mattermost struct { // You must call Init method before SendEvents method. type Sender struct { frontURI string + logger moira.Logger location *time.Location client Client } @@ -70,17 +71,28 @@ func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, loca } sender.frontURI = frontURI sender.location = location + sender.logger = logger return nil } // SendEvents implements moira.Sender interface. -func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.ContactData, trigger moira.TriggerData, _ [][]byte, throttled bool) error { +func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.ContactData, trigger moira.TriggerData, plots [][]byte, throttled bool) error { message := sender.buildMessage(events, trigger, throttled) - err := sender.sendMessage(message, contact.Value, trigger.ID) + post, err := sender.sendMessage(message, contact.Value, trigger.ID) if err != nil { return err } + if len(plots) > 0 { + err = sender.sendPlots(plots, contact.Value, post.Id, trigger.ID) + if err != nil { + sender.logger.Warning(). + String("trigger_id", trigger.ID). + String("contact_value", contact.Value). + String("contact_type", contact.Type). + Error(err) + } + } return nil } @@ -187,15 +199,45 @@ func (sender *Sender) buildEventsString(events moira.NotificationEvents, charsFo return eventsString } -func (sender *Sender) sendMessage(message string, contact string, triggerID string) error { +func (sender *Sender) sendMessage(message string, contact string, triggerID string) (*model.Post, error) { post := model.Post{ ChannelId: contact, Message: message, } - _, _, err := sender.client.CreatePost(&post) + sentPost, _, err := sender.client.CreatePost(&post) if err != nil { - return fmt.Errorf("failed to send %s event message to Mattermost [%s]: %s", triggerID, contact, err) + return nil, fmt.Errorf("failed to send %s event message to Mattermost [%s]: %s", triggerID, contact, err) + } + + return sentPost, nil +} + +func (sender *Sender) sendPlots(plots [][]byte, channelID, postID, triggerID string) error { + var filesID []string + + filename := fmt.Sprintf("%s.png", triggerID) + for _, plot := range plots { + file, _, err := sender.client.UploadFile(plot, channelID, filename) + if err != nil { + return err + } + for _, info := range file.FileInfos { + filesID = append(filesID, info.Id) + } + } + + if len(filesID) > 0 { + _, _, err := sender.client.CreatePost( + &model.Post{ + ChannelId: channelID, + RootId: postID, + FileIds: filesID, + }, + ) + if err != nil { + return err + } } return nil diff --git a/senders/mattermost/sender_internal_test.go b/senders/mattermost/sender_internal_test.go index 81dfcfa47..18b5950df 100644 --- a/senders/mattermost/sender_internal_test.go +++ b/senders/mattermost/sender_internal_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/mattermost/mattermost-server/v6/model" + "github.com/moira-alert/moira" "github.com/golang/mock/gomock" @@ -39,16 +41,34 @@ func TestSendEvents(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("When client CreatePost is success, SendEvents should not return error", func() { + Convey("When client CreatePost is success and no plots, SendEvents should not return error", func() { ctrl := gomock.NewController(t) client := mock.NewMockClient(ctrl) - client.EXPECT().CreatePost(gomock.Any()).Return(nil, nil, nil) + client.EXPECT().CreatePost(gomock.Any()).Return(&model.Post{Id: "postID"}, nil, nil) sender.client = client events, contact, trigger, plots, throttled := moira.NotificationEvents{}, moira.ContactData{}, moira.TriggerData{}, make([][]byte, 0), false err = sender.SendEvents(events, contact, trigger, plots, throttled) So(err, ShouldBeNil) }) + + Convey("When client CreatePost is success and have succeeded sent plots, SendEvents should not return error", func() { + ctrl := gomock.NewController(t) + client := mock.NewMockClient(ctrl) + client.EXPECT().CreatePost(gomock.Any()).Return(&model.Post{Id: "postID"}, nil, nil).Times(2) + client.EXPECT().UploadFile(gomock.Any(), "contactDataID", "triggerID.png"). + Return( + &model.FileUploadResponse{ + FileInfos: []*model.FileInfo{{Id: "fileID"}}, + }, nil, nil) + sender.client = client + + plots := make([][]byte, 0) + plots = append(plots, []byte("my_awesome_plot")) + events, contact, trigger, throttled := moira.NotificationEvents{}, moira.ContactData{Value: "contactDataID"}, moira.TriggerData{ID: "triggerID"}, false + err = sender.SendEvents(events, contact, trigger, plots, throttled) + So(err, ShouldBeNil) + }) }) } From 27f09ddbe6e1d3378779551719f17c1867f72e3a Mon Sep 17 00:00:00 2001 From: Xenia N Date: Tue, 22 Aug 2023 13:55:43 +0600 Subject: [PATCH 06/46] fix(api) not mock decode error in api response (#899) --- api/middleware/logger.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/middleware/logger.go b/api/middleware/logger.go index a34608758..dd7166ee7 100644 --- a/api/middleware/logger.go +++ b/api/middleware/logger.go @@ -65,7 +65,13 @@ func RequestLogger(logger moira.Logger) func(next http.Handler) http.Handler { func getErrorResponseIfItHas(writer http.ResponseWriter) *api.ErrorResponse { writerWithBody := writer.(*responseWriterWithBody) var errResp = &api.ErrorResponse{} - json.NewDecoder(&writerWithBody.body).Decode(errResp) //nolint + if err := json.NewDecoder(&writerWithBody.body).Decode(errResp); err != nil { + return &api.ErrorResponse{ + HTTPStatusCode: http.StatusInternalServerError, + Err: err, + } + } + return errResp } From 2da331a5e6cd149f819599091c7eeec3b5c65145 Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:00:20 +0300 Subject: [PATCH 07/46] feature: add pre-commit hooks (#894) --- .githooks/pre-commit | 9 +++++++++ .github/CONTRIBUTING.md | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100755 .githooks/pre-commit diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 000000000..5f8e49100 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,9 @@ +#!/bin/bash + +# Check if any of the specified files are staged for commit +if git diff --name-only --cached | grep -E 'api/'; then + echo "Format swaggo annotations (swag fmt)" + swag fmt +fi + +make lint \ No newline at end of file diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 2aaf2d16f..1a467cc5e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -138,8 +138,8 @@ For full local deployment of all services, including web, graphite and metrics r docker-compose up ``` -Before pushing your changes don't forget about linter: +Before pushing, don't forget to write this command please, it will activate the pre-commit hook on the linter and auto formatting swagger documentation: ```bash -make lint +git config --local core.hooksPath .githooks/ ``` From 1aa62b4155b2b4a55266ad67fd66864fbd2b055e Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Tue, 22 Aug 2023 14:27:20 +0300 Subject: [PATCH 08/46] feature: add github jobs for swaggerhub (#895) --- .github/workflows/swagger-delete.yml | 31 +++++++++++++ .github/workflows/swagger-publish.yml | 60 ++++++++++++++++++++++++++ .github/workflows/swagger-validate.yml | 28 ++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 .github/workflows/swagger-delete.yml create mode 100644 .github/workflows/swagger-publish.yml create mode 100644 .github/workflows/swagger-validate.yml diff --git a/.github/workflows/swagger-delete.yml b/.github/workflows/swagger-delete.yml new file mode 100644 index 000000000..d2fceb3c2 --- /dev/null +++ b/.github/workflows/swagger-delete.yml @@ -0,0 +1,31 @@ +name: Delete spec version from SwaggerHub +defaults: + run: + working-directory: . +on: + pull_request: + types: [closed] + branches: + - master + push: + branches: + - release/[0-9]+.[0-9]+.[0-9]+ + - master + +jobs: + removespec: + name: Delete api from SwaggerHub + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: '16.16.0' + - run: npm i --location=global swaggerhub-cli + - run: | + VERSION=`echo ${GITHUB_REF_NAME}| sed 's#[^a-zA-Z0-9_\.\-]#_#g'` + SWAGGERHUB_API_KEY=${{secrets.SWAGGERHUB_TOKEN}} swaggerhub api:unpublish "Moira/moira-alert/${VERSION}" || true + SWAGGERHUB_API_KEY=${{secrets.SWAGGERHUB_TOKEN}} swaggerhub api:delete "Moira/moira-alert/${VERSION}" || true +# The `|| true` at the end of the calls is necessary to keep the job from crashing +# when deleting documentation that hasn't been created yet, but if you see something wrong happening, +# remove `|| true` from the command diff --git a/.github/workflows/swagger-publish.yml b/.github/workflows/swagger-publish.yml new file mode 100644 index 000000000..03e623c36 --- /dev/null +++ b/.github/workflows/swagger-publish.yml @@ -0,0 +1,60 @@ +name: Publish spec version to SwaggerHub + +on: + push: + branches: + - master + - release/[0-9]+.[0-9]+.[0-9]+ + +jobs: + validate-spec: + name: Validate spec file + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: . + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + - run: make install-swag + + - uses: actions/setup-node@v3 + with: + node-version: '16.16.0' + - run: npm install --location=global @openapitools/openapi-generator-cli + - run: make spec + - run: make validate-spec + + - name: Save build artifact + uses: actions/upload-artifact@v3 + with: + name: specfile + path: docs/swagger.yaml + + publishspec: + name: Upload generated OpenAPI description + runs-on: ubuntu-22.04 + needs: validate-spec + defaults: + run: + working-directory: . + + steps: + - uses: actions/checkout@v3 + + - name: Download spec file artifact + uses: actions/download-artifact@v3 + with: + name: specfile + path: docs + + - uses: actions/setup-node@v3 + - run: npm i --location=global swaggerhub-cli + - run: | + VERSION=`echo ${GITHUB_REF_NAME}| sed 's#[^a-zA-Z0-9_\.\-]#_#g'` + SWAGGERHUB_API_KEY=${{secrets.SWAGGERHUB_TOKEN}} swaggerhub api:create "Moira/moira-alert/${VERSION}" -f ./docs/swagger.yaml --published=publish --visibility=public diff --git a/.github/workflows/swagger-validate.yml b/.github/workflows/swagger-validate.yml new file mode 100644 index 000000000..a39d7daba --- /dev/null +++ b/.github/workflows/swagger-validate.yml @@ -0,0 +1,28 @@ +name: Validate OpenAPI on PR + +on: + - pull_request + +jobs: + mergespec: + name: Validate spec file + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: . + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + - run: make install-swag + + - uses: actions/setup-node@v3 + with: + node-version: '16.16.0' + - run: npm install --location=global @openapitools/openapi-generator-cli + - run: make spec + - run: make validate-spec From 0e6fb14bc190547dd1ffe0b2edc96d1f8a7ee30d Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Tue, 22 Aug 2023 15:03:22 +0300 Subject: [PATCH 09/46] fix: hotfix swagger delete job (#900) --- .github/workflows/swagger-delete.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/swagger-delete.yml b/.github/workflows/swagger-delete.yml index d2fceb3c2..63f4e5c44 100644 --- a/.github/workflows/swagger-delete.yml +++ b/.github/workflows/swagger-delete.yml @@ -3,14 +3,10 @@ defaults: run: working-directory: . on: - pull_request: - types: [closed] - branches: - - master push: branches: - - release/[0-9]+.[0-9]+.[0-9]+ - master + - release/[0-9]+.[0-9]+.[0-9]+ jobs: removespec: From 4e9751cb0beb9229a721743d8aed27dc52fe2d7a Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:26:40 +0300 Subject: [PATCH 10/46] fix(api): add int64 annotations in swaggo (#901) --- api/dto/event_history_item.go | 2 +- api/dto/events.go | 6 ++--- api/dto/notification.go | 4 +-- api/dto/triggers.go | 14 +++++------ api/handler/handler.go | 4 +-- datatypes.go | 46 +++++++++++++++++------------------ 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/api/dto/event_history_item.go b/api/dto/event_history_item.go index 26c85175f..74783d284 100644 --- a/api/dto/event_history_item.go +++ b/api/dto/event_history_item.go @@ -3,7 +3,7 @@ package dto import "net/http" type ContactEventItem struct { - TimeStamp int64 `json:"timestamp"` + TimeStamp int64 `json:"timestamp" format:"int64"` Metric string `json:"metric"` State string `json:"state"` OldState string `json:"old_state"` diff --git a/api/dto/events.go b/api/dto/events.go index 2f704d337..603eb9b29 100644 --- a/api/dto/events.go +++ b/api/dto/events.go @@ -8,9 +8,9 @@ import ( ) type EventsList struct { - Page int64 `json:"page" example:"0"` - Size int64 `json:"size" example:"100"` - Total int64 `json:"total" example:"10"` + Page int64 `json:"page" example:"0" format:"int64"` + Size int64 `json:"size" example:"100" format:"int64"` + Total int64 `json:"total" example:"10" format:"int64"` List []moira.NotificationEvent `json:"list"` } diff --git a/api/dto/notification.go b/api/dto/notification.go index 4a6fa6ef1..5e1dbe447 100644 --- a/api/dto/notification.go +++ b/api/dto/notification.go @@ -8,7 +8,7 @@ import ( ) type NotificationsList struct { - Total int64 `json:"total" example:"0"` + Total int64 `json:"total" example:"0" format:"int64"` List []*moira.ScheduledNotification `json:"list"` } @@ -17,7 +17,7 @@ func (*NotificationsList) Render(w http.ResponseWriter, r *http.Request) error { } type NotificationDeleteResponse struct { - Result int64 `json:"result" example:"0"` + Result int64 `json:"result" example:"0" format:"int64"` } func (*NotificationDeleteResponse) Render(w http.ResponseWriter, r *http.Request) error { diff --git a/api/dto/triggers.go b/api/dto/triggers.go index c573e1659..60a9f1bc1 100644 --- a/api/dto/triggers.go +++ b/api/dto/triggers.go @@ -23,9 +23,9 @@ var targetNameRegex = regexp.MustCompile("t(\\d+)") var asteriskPattern = "*" type TriggersList struct { - Page *int64 `json:"page,omitempty"` - Size *int64 `json:"size,omitempty"` - Total *int64 `json:"total,omitempty"` + Page *int64 `json:"page,omitempty" format:"int64"` + Size *int64 `json:"size,omitempty" format:"int64"` + Total *int64 `json:"total,omitempty" format:"int64"` Pager *string `json:"pager,omitempty"` List []moira.TriggerCheck `json:"list"` } @@ -36,7 +36,7 @@ func (*TriggersList) Render(http.ResponseWriter, *http.Request) error { type Trigger struct { TriggerModel - Throttling int64 `json:"throttling" example:"0"` + Throttling int64 `json:"throttling" example:"0" format:"int64"` } // TriggerModel is moira.Trigger api representation @@ -60,7 +60,7 @@ type TriggerModel struct { // When there are no metrics for trigger, Moira will switch metric to TTLState state after TTL seconds TTLState *moira.TTLState `json:"ttl_state,omitempty" example:"NODATA"` // When there are no metrics for trigger, Moira will switch metric to TTLState state after TTL seconds - TTL int64 `json:"ttl,omitempty" example:"600"` + TTL int64 `json:"ttl,omitempty" example:"600" format:"int64"` // Determines when Moira should monitor trigger Schedule *moira.ScheduleData `json:"sched,omitempty"` // Used if you need more complex logic than provided by WARN/ERROR values @@ -380,7 +380,7 @@ func (*MetricsMaintenance) Bind(*http.Request) error { } type TriggerMaintenance struct { - Trigger *int64 `json:"trigger" example:"1594225165"` + Trigger *int64 `json:"trigger" example:"1594225165" format:"int64"` Metrics map[string]int64 `json:"metrics"` } @@ -389,7 +389,7 @@ func (*TriggerMaintenance) Bind(*http.Request) error { } type ThrottlingResponse struct { - Throttling int64 `json:"throttling" example:"0"` + Throttling int64 `json:"throttling" example:"0" format:"int64"` } func (*ThrottlingResponse) Render(http.ResponseWriter, *http.Request) error { diff --git a/api/handler/handler.go b/api/handler/handler.go index 54d7e3254..98d1f8df2 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -39,8 +39,8 @@ func NewHandler(db moira.Database, log moira.Logger, index moira.Searcher, confi // @title Moira Alert // @version master // @description This is an API description for [Moira Alert API](https://moira.readthedocs.io/en/latest/overview.html) - // @description Check us out on [Github](https://github.com/moira-alert) or look up our [guide on getting started with Moira](https://moira.readthedocs.io) - // @contact.name Contact Moira Team + // @description Check us out on [Github](https://github.com/moira-alert) or look up our [guide](https://moira.readthedocs.io) on getting started with Moira + // @contact.name Moira Team // @contact.email opensource@skbkontur.com // @license.name MIT // @BasePath /api diff --git a/datatypes.go b/datatypes.go index 852cba7bd..b725039a3 100644 --- a/datatypes.go +++ b/datatypes.go @@ -43,7 +43,7 @@ const ( // NotificationEvent represents trigger state changes event type NotificationEvent struct { IsTriggerEvent bool `json:"trigger_event,omitempty" example:"true"` - Timestamp int64 `json:"timestamp" example:"1590741878"` + Timestamp int64 `json:"timestamp" example:"1590741878" format:"int64"` Metric string `json:"metric" example:"carbon.agents.*.metricsReceived"` Value *float64 `json:"value,omitempty" example:"70"` Values map[string]float64 `json:"values,omitempty"` @@ -59,7 +59,7 @@ type NotificationEvent struct { // NotificationEventHistoryItem is in use to store notifications history of channel // (see database/redis/contact_notifications_history.go type NotificationEventHistoryItem struct { - TimeStamp int64 `json:"timestamp"` + TimeStamp int64 `json:"timestamp" format:"int64"` Metric string `json:"metric"` State State `json:"state"` OldState State `json:"old_state"` @@ -70,7 +70,7 @@ type NotificationEventHistoryItem struct { // EventInfo - a base for creating messages. type EventInfo struct { Maintenance *MaintenanceInfo `json:"maintenance,omitempty"` - Interval *int64 `json:"interval,omitempty" example:"0"` + Interval *int64 `json:"interval,omitempty" example:"0" format:"int64"` } // CreateMessage - creates a message based on EventInfo. @@ -216,9 +216,9 @@ type PlottingData struct { // ScheduleData represents subscription schedule type ScheduleData struct { Days []ScheduleDataDay `json:"days"` - TimezoneOffset int64 `json:"tzOffset" example:"-60"` - StartOffset int64 `json:"startOffset" example:"0"` - EndOffset int64 `json:"endOffset" example:"1439"` + TimezoneOffset int64 `json:"tzOffset" example:"-60" format:"int64"` + StartOffset int64 `json:"startOffset" example:"0" format:"int64"` + EndOffset int64 `json:"endOffset" example:"1439" format:"int64"` } // ScheduleDataDay represents week day of schedule @@ -235,7 +235,7 @@ type ScheduledNotification struct { Plotting PlottingData `json:"plotting"` Throttled bool `json:"throttled" example:"false"` SendFail int `json:"send_fail" example:"0"` - Timestamp int64 `json:"timestamp" example:"1594471927"` + Timestamp int64 `json:"timestamp" example:"1594471927" format:"int64"` } // MatchedMetric represents parsed and matched metric data @@ -250,8 +250,8 @@ type MatchedMetric struct { // MetricValue represents metric data type MetricValue struct { - RetentionTimestamp int64 `json:"step,omitempty"` - Timestamp int64 `json:"ts"` + RetentionTimestamp int64 `json:"step,omitempty" format:"int64"` + Timestamp int64 `json:"ts" format:"int64"` Value float64 `json:"value"` } @@ -275,7 +275,7 @@ type Trigger struct { TriggerType string `json:"trigger_type" example:"rising"` Tags []string `json:"tags" example:"server,disk"` TTLState *TTLState `json:"ttl_state,omitempty" example:"NODATA"` - TTL int64 `json:"ttl,omitempty" example:"600"` + TTL int64 `json:"ttl,omitempty" example:"600" format:"int64"` Schedule *ScheduleData `json:"sched,omitempty"` Expression *string `json:"expression,omitempty" example:""` PythonExpression *string `json:"python_expression,omitempty"` @@ -283,8 +283,8 @@ type Trigger struct { IsRemote bool `json:"is_remote" example:"false"` MuteNewMetrics bool `json:"mute_new_metrics" example:"false"` AloneMetrics map[string]bool `json:"alone_metrics" example:"t1:true"` - CreatedAt *int64 `json:"created_at"` - UpdatedAt *int64 `json:"updated_at"` + CreatedAt *int64 `json:"created_at" format:"int64"` + UpdatedAt *int64 `json:"updated_at" format:"int64"` CreatedBy string `json:"created_by"` UpdatedBy string `json:"updated_by"` } @@ -292,7 +292,7 @@ type Trigger struct { // TriggerCheck represents trigger data with last check data and check timestamp type TriggerCheck struct { Trigger - Throttling int64 `json:"throttling" example:"0"` + Throttling int64 `json:"throttling" example:"0" format:"int64"` LastCheck CheckData `json:"last_check"` Highlights map[string]string `json:"highlights"` } @@ -310,13 +310,13 @@ type CheckData struct { // check and targets that fetched this metric // {"t1": "metric.name.1", "t2": "metric.name.2"} MetricsToTargetRelation map[string]string `json:"metrics_to_target_relation" example:"t1:metric.name.1,t2:metric.name.2"` - Score int64 `json:"score" example:"100"` + Score int64 `json:"score" example:"100" format:"int64"` State State `json:"state" example:"OK"` - Maintenance int64 `json:"maintenance,omitempty" example:"0"` + Maintenance int64 `json:"maintenance,omitempty" example:"0" format:"int64"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` - Timestamp int64 `json:"timestamp,omitempty" example:"1590741916"` - EventTimestamp int64 `json:"event_timestamp,omitempty" example:"1590741878"` - LastSuccessfulCheckTimestamp int64 `json:"last_successful_check_timestamp" example:"1590741916"` + Timestamp int64 `json:"timestamp,omitempty" example:"1590741916" format:"int64"` + EventTimestamp int64 `json:"event_timestamp,omitempty" example:"1590741878" format:"int64"` + LastSuccessfulCheckTimestamp int64 `json:"last_successful_check_timestamp" example:"1590741916" format:"int64"` Suppressed bool `json:"suppressed,omitempty" example:"true"` SuppressedState State `json:"suppressed_state,omitempty"` Message string `json:"msg,omitempty"` @@ -334,14 +334,14 @@ func (checkData *CheckData) RemoveMetricsToTargetRelation() { // MetricState represents metric state data for given timestamp type MetricState struct { - EventTimestamp int64 `json:"event_timestamp" example:"1590741878"` + EventTimestamp int64 `json:"event_timestamp" example:"1590741878" format:"int64"` State State `json:"state" example:"OK"` Suppressed bool `json:"suppressed" example:"false"` SuppressedState State `json:"suppressed_state,omitempty"` - Timestamp int64 `json:"timestamp" example:"1590741878"` + Timestamp int64 `json:"timestamp" example:"1590741878" format:"int64"` Value *float64 `json:"value,omitempty" example:"70"` Values map[string]float64 `json:"values,omitempty"` - Maintenance int64 `json:"maintenance,omitempty" example:"0"` + Maintenance int64 `json:"maintenance,omitempty" example:"0" format:"int64"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` // AloneMetrics map[string]string `json:"alone_metrics"` // represents a relation between name of alone metrics and their targets } @@ -360,9 +360,9 @@ func (metricState *MetricState) GetMaintenance() (MaintenanceInfo, int64) { // MaintenanceInfo represents user and time set/unset maintenance type MaintenanceInfo struct { StartUser *string `json:"setup_user"` - StartTime *int64 `json:"setup_time" example:"0"` + StartTime *int64 `json:"setup_time" example:"0" format:"int64"` StopUser *string `json:"remove_user"` - StopTime *int64 `json:"remove_time" example:"0"` + StopTime *int64 `json:"remove_time" example:"0" format:"int64"` } // Set maintanace start and stop users and times From 6bb22d41a6c8d62312166d6f613f75a4f87fcc15 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Tue, 29 Aug 2023 10:13:19 +0300 Subject: [PATCH 11/46] feat: prometheus remote (#848) --- api/config.go | 9 +- api/controller/trigger.go | 2 +- api/controller/trigger_metrics.go | 2 +- api/controller/trigger_metrics_test.go | 47 ++-- api/controller/trigger_test.go | 22 +- api/dto/target.go | 31 ++- api/dto/target_test.go | 44 +++- api/dto/triggers.go | 28 ++- api/dto/triggers_test.go | 11 +- api/handler/handler.go | 2 +- api/handler/trigger.go | 20 +- api/handler/trigger_render_test.go | 14 +- api/handler/trigger_test.go | 215 +++++++++++++++--- api/handler/triggers.go | 28 ++- api/handler/triggers_test.go | 235 ++++++++++++++++---- api/middleware/context.go | 12 +- api/middleware/middleware.go | 50 +++-- checker/check.go | 131 ++++++++--- checker/check_test.go | 73 ++++-- checker/config.go | 3 +- checker/event.go | 20 +- checker/trigger_checker.go | 9 +- checker/trigger_checker_test.go | 50 +++-- checker/worker/handler.go | 54 +++-- checker/worker/lazy_triggers.go | 34 +-- checker/worker/local.go | 103 +++++++++ checker/worker/metrics.go | 58 ++--- checker/worker/nodata.go | 62 ------ checker/worker/prometheus.go | 106 +++++++++ checker/worker/remote.go | 91 +++++--- checker/worker/trigger_to_check.go | 41 +++- checker/worker/worker.go | 194 +++++++++------- cmd/api/config.go | 9 +- cmd/api/config_test.go | 8 +- cmd/api/main.go | 18 +- cmd/checker/config.go | 18 +- cmd/checker/main.go | 29 ++- cmd/cli/main.go | 9 +- cmd/config.go | 34 ++- cmd/notifier/config.go | 1 + cmd/notifier/main.go | 14 +- database/redis/last_check.go | 19 +- database/redis/last_check_test.go | 48 ++-- database/redis/reply/trigger.go | 8 +- database/redis/selfstate.go | 11 + database/redis/selfstate_test.go | 8 +- database/redis/trigger.go | 48 +++- database/redis/trigger_test.go | 173 +++++++------- database/redis/triggers_to_check.go | 13 ++ datatypes.go | 61 ++++- go.mod | 7 +- go.sum | 4 + helpers_test.go | 10 +- integration_tests/notifier/notifier_test.go | 40 +++- interfaces.go | 11 +- local/api.yml | 6 + local/checker.yml | 8 +- local/notifier.yml | 8 +- metric_source/local/local.go | 5 + metric_source/prometheus/convert.go | 89 ++++++++ metric_source/prometheus/convert_test.go | 225 +++++++++++++++++++ metric_source/prometheus/fetch.go | 60 +++++ metric_source/prometheus/prometheus.go | 57 +++++ metric_source/prometheus/prometheus_api.go | 37 +++ metric_source/prometheus/prometheus_test.go | 23 ++ metric_source/provider.go | 33 ++- metric_source/remote/remote.go | 2 +- metric_source/remote/remote_test.go | 4 +- metric_source/source.go | 1 + metrics/checker.go | 24 +- mock/metric_source/source.go | 15 ++ mock/moira-alert/database.go | 76 ++++++- notifier/events/event.go | 17 +- notifier/notifier_test.go | 2 +- notifier/plotting.go | 2 +- senders/script/script_test.go | 7 +- support/trigger.go | 11 +- 77 files changed, 2408 insertions(+), 706 deletions(-) create mode 100644 checker/worker/local.go delete mode 100644 checker/worker/nodata.go create mode 100644 checker/worker/prometheus.go create mode 100644 metric_source/prometheus/convert.go create mode 100644 metric_source/prometheus/convert_test.go create mode 100644 metric_source/prometheus/fetch.go create mode 100644 metric_source/prometheus/prometheus.go create mode 100644 metric_source/prometheus/prometheus_api.go create mode 100644 metric_source/prometheus/prometheus_test.go diff --git a/api/config.go b/api/config.go index 8f226d170..a65340214 100644 --- a/api/config.go +++ b/api/config.go @@ -4,10 +4,11 @@ import "time" // Config for api configuration variables. type Config struct { - EnableCORS bool - Listen string - LocalMetricTTL time.Duration - RemoteMetricTTL time.Duration + EnableCORS bool + Listen string + GraphiteLocalMetricTTL time.Duration + GraphiteRemoteMetricTTL time.Duration + PrometheusRemoteMetricTTL time.Duration } // WebConfig is container for web ui configuration parameters. diff --git a/api/controller/trigger.go b/api/controller/trigger.go index 2f4e5ca30..9d0206d1c 100644 --- a/api/controller/trigger.go +++ b/api/controller/trigger.go @@ -55,7 +55,7 @@ func saveTrigger(dataBase moira.Database, trigger *moira.Trigger, triggerID stri lastCheck.UpdateScore() } - if err = dataBase.SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote); err != nil { + if err = dataBase.SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource); err != nil { return nil, api.ErrorInternalServer(err) } diff --git a/api/controller/trigger_metrics.go b/api/controller/trigger_metrics.go index 537fbe0d2..96bd19d18 100644 --- a/api/controller/trigger_metrics.go +++ b/api/controller/trigger_metrics.go @@ -113,7 +113,7 @@ func deleteTriggerMetrics(dataBase moira.Database, metricName string, triggerID if err = dataBase.RemovePatternsMetrics(trigger.Patterns); err != nil { return api.ErrorInternalServer(err) } - if err = dataBase.SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote); err != nil { + if err = dataBase.SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource); err != nil { return api.ErrorInternalServer(err) } return nil diff --git a/api/controller/trigger_metrics_test.go b/api/controller/trigger_metrics_test.go index 89b512847..d96d0d766 100644 --- a/api/controller/trigger_metrics_test.go +++ b/api/controller/trigger_metrics_test.go @@ -40,7 +40,7 @@ func TestDeleteTriggerMetric(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(expectedLastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.IsRemote) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.TriggerSource) err := DeleteTriggerMetric(dataBase, "super.metric1", triggerID) So(err, ShouldBeNil) So(expectedLastCheck, ShouldResemble, emptyLastCheck) @@ -53,7 +53,7 @@ func TestDeleteTriggerMetric(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(expectedLastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.IsRemote) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.TriggerSource) err := DeleteTriggerMetric(dataBase, "super.metric1", triggerID) So(err, ShouldBeNil) So(expectedLastCheck, ShouldResemble, emptyLastCheck) @@ -117,7 +117,7 @@ func TestDeleteTriggerMetric(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(lastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote).Return(expected) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource).Return(expected) err := DeleteTriggerMetric(dataBase, "super.metric1", triggerID) So(err, ShouldResemble, api.ErrorInternalServer(expected)) }) @@ -175,7 +175,7 @@ func TestDeleteTriggerNodataMetrics(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(expectedLastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.IsRemote) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.TriggerSource) err := DeleteTriggerNodataMetrics(dataBase, triggerID) So(err, ShouldBeNil) So(expectedLastCheck, ShouldResemble, emptyLastCheck) @@ -188,7 +188,7 @@ func TestDeleteTriggerNodataMetrics(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(expectedLastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.IsRemote) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.TriggerSource) err := DeleteTriggerNodataMetrics(dataBase, triggerID) So(err, ShouldBeNil) So(expectedLastCheck, ShouldResemble, emptyLastCheck) @@ -201,7 +201,7 @@ func TestDeleteTriggerNodataMetrics(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(expectedLastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheckWithoutNodata, trigger.IsRemote) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheckWithoutNodata, trigger.TriggerSource) err := DeleteTriggerNodataMetrics(dataBase, triggerID) So(err, ShouldBeNil) So(expectedLastCheck, ShouldResemble, lastCheckWithoutNodata) @@ -214,7 +214,7 @@ func TestDeleteTriggerNodataMetrics(t *testing.T) { dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(expectedLastCheck, nil) dataBase.EXPECT().RemovePatternsMetrics(trigger.Patterns).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.IsRemote) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &expectedLastCheck, trigger.TriggerSource) err := DeleteTriggerNodataMetrics(dataBase, triggerID) So(err, ShouldBeNil) So(expectedLastCheck, ShouldResemble, emptyLastCheck) @@ -269,7 +269,7 @@ func TestGetTriggerMetrics(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) pattern := "super.puper.pattern" metric := "super.puper.metric" @@ -278,7 +278,11 @@ func TestGetTriggerMetrics(t *testing.T) { var retention int64 = 10 Convey("Trigger is remote but remote is not configured", t, func() { - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ID: triggerID, Targets: []string{pattern}, IsRemote: true}, nil) + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + ID: triggerID, + Targets: []string{pattern}, + TriggerSource: moira.GraphiteRemote, + }, nil) remoteSource.EXPECT().IsConfigured().Return(false, nil) triggerMetrics, err := GetTriggerMetrics(dataBase, sourceProvider, from, until, triggerID) So(err, ShouldResemble, api.ErrorInternalServer(metricSource.ErrMetricSourceIsNotConfigured)) @@ -286,7 +290,11 @@ func TestGetTriggerMetrics(t *testing.T) { }) Convey("Trigger is remote but remote has bad config", t, func() { - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ID: triggerID, Targets: []string{pattern}, IsRemote: true}, nil) + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + ID: triggerID, + Targets: []string{pattern}, + TriggerSource: moira.GraphiteRemote, + }, nil) remoteSource.EXPECT().IsConfigured().Return(false, remote.ErrRemoteStorageDisabled) triggerMetrics, err := GetTriggerMetrics(dataBase, sourceProvider, from, until, triggerID) So(err, ShouldResemble, api.ErrorInternalServer(remote.ErrRemoteStorageDisabled)) @@ -294,7 +302,11 @@ func TestGetTriggerMetrics(t *testing.T) { }) Convey("Has metrics", t, func() { - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ID: triggerID, Targets: []string{pattern}}, nil) + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + ID: triggerID, + Targets: []string{pattern}, + TriggerSource: moira.GraphiteLocal, + }, nil) localSource.EXPECT().IsConfigured().Return(true, nil) localSource.EXPECT().Fetch(pattern, from, until, false).Return(fetchResult, nil) fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData(metric, []float64{0, 1, 2, 3, 4}, retention, from)}) @@ -319,11 +331,20 @@ func TestGetTriggerMetrics(t *testing.T) { }) Convey("Fetch error", t, func() { - expectedError := remote.ErrRemoteTriggerResponse{InternalError: fmt.Errorf("some error"), Target: pattern} - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ID: triggerID, Targets: []string{pattern}, IsRemote: true}, nil) + expectedError := remote.ErrRemoteTriggerResponse{ + InternalError: fmt.Errorf("some error"), + Target: pattern, + } + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + ID: triggerID, + Targets: []string{pattern}, + TriggerSource: moira.GraphiteRemote, + }, nil) remoteSource.EXPECT().IsConfigured().Return(true, nil) remoteSource.EXPECT().Fetch(pattern, from, until, false).Return(nil, expectedError) + triggerMetrics, err := GetTriggerMetrics(dataBase, sourceProvider, from, until, triggerID) + So(err, ShouldResemble, api.ErrorInternalServer(expectedError)) So(triggerMetrics, ShouldBeNil) }) diff --git a/api/controller/trigger_test.go b/api/controller/trigger_test.go index 0f408d0ae..1c5769e85 100644 --- a/api/controller/trigger_test.go +++ b/api/controller/trigger_test.go @@ -27,7 +27,7 @@ func TestUpdateTrigger(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(gomock.Any(), 10) dataBase.EXPECT().DeleteTriggerCheckLock(gomock.Any()) dataBase.EXPECT().GetTriggerLastCheck(gomock.Any()).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(gomock.Any(), gomock.Any(), trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(gomock.Any(), gomock.Any(), trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(gomock.Any(), trigger).Return(nil) resp, err := UpdateTrigger(dataBase, &triggerModel, triggerModel.ID, make(map[string]bool)) So(err, ShouldBeNil) @@ -77,7 +77,7 @@ func TestSaveTrigger(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) @@ -88,7 +88,7 @@ func TestSaveTrigger(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(actualLastCheck, nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &emptyLastCheck, trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &emptyLastCheck, trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) @@ -101,7 +101,7 @@ func TestSaveTrigger(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, map[string]bool{"super.metric1": true, "super.metric2": true}) So(err, ShouldBeNil) @@ -133,7 +133,7 @@ func TestSaveTrigger(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.IsRemote).Return(expected) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.TriggerSource).Return(expected) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldResemble, api.ErrorInternalServer(expected)) So(resp, ShouldBeNil) @@ -144,7 +144,7 @@ func TestSaveTrigger(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, gomock.Any(), trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(expected) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldResemble, api.ErrorInternalServer(expected)) @@ -175,7 +175,7 @@ func TestVariousTtlState(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) @@ -190,7 +190,7 @@ func TestVariousTtlState(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) @@ -205,7 +205,7 @@ func TestVariousTtlState(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) @@ -220,7 +220,7 @@ func TestVariousTtlState(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) @@ -235,7 +235,7 @@ func TestVariousTtlState(t *testing.T) { dataBase.EXPECT().AcquireTriggerCheckLock(triggerID, 10) dataBase.EXPECT().DeleteTriggerCheckLock(triggerID) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck(triggerID, &lastCheck, trigger.TriggerSource).Return(nil) dataBase.EXPECT().SaveTrigger(triggerID, &trigger).Return(nil) resp, err := saveTrigger(dataBase, &trigger, triggerID, make(map[string]bool)) So(err, ShouldBeNil) diff --git a/api/dto/target.go b/api/dto/target.go index 2c6f11afb..0654b5c3e 100644 --- a/api/dto/target.go +++ b/api/dto/target.go @@ -6,6 +6,7 @@ import ( "github.com/go-graphite/carbonapi/expr/functions" "github.com/go-graphite/carbonapi/expr/metadata" + "github.com/moira-alert/moira" "github.com/go-graphite/carbonapi/pkg/parser" ) @@ -128,7 +129,19 @@ type TreeOfProblems struct { } // TargetVerification validates trigger targets. -func TargetVerification(targets []string, ttl time.Duration, isRemote bool) []TreeOfProblems { +func TargetVerification(targets []string, ttl time.Duration, triggerSource moira.TriggerSource) ([]TreeOfProblems, error) { + switch triggerSource { + case moira.PrometheusRemote: + return []TreeOfProblems{{SyntaxOk: true}}, nil + + case moira.GraphiteLocal, moira.GraphiteRemote: + return graphiteTargetVerification(targets, ttl, triggerSource), nil + } + + return nil, fmt.Errorf("unknown trigger source '%s'", triggerSource) +} + +func graphiteTargetVerification(targets []string, ttl time.Duration, triggerSource moira.TriggerSource) []TreeOfProblems { functionsOfTargets := make([]TreeOfProblems, 0) for _, target := range targets { @@ -147,7 +160,7 @@ func TargetVerification(targets []string, ttl time.Duration, isRemote bool) []Tr continue } - functionsOfTarget.TreeOfProblems = checkExpression(expr, ttl, isRemote) + functionsOfTarget.TreeOfProblems = checkExpression(expr, ttl, triggerSource) functionsOfTargets = append(functionsOfTargets, functionsOfTarget) } @@ -167,13 +180,13 @@ func DoesAnyTreeHaveError(trees []TreeOfProblems) bool { } // checkExpression validates expression. -func checkExpression(expression parser.Expr, ttl time.Duration, isRemote bool) *ProblemOfTarget { +func checkExpression(expression parser.Expr, ttl time.Duration, triggerSource moira.TriggerSource) *ProblemOfTarget { if !expression.IsFunc() { return nil } funcName := expression.Target() - problemFunction := checkFunction(funcName, isRemote) + problemFunction := checkFunction(funcName, triggerSource) if argument, ok := functionArgumentsInTheRangeTTL(expression, ttl); !ok { if problemFunction == nil { @@ -195,7 +208,7 @@ func checkExpression(expression parser.Expr, ttl time.Duration, isRemote bool) * continue } - if badFunc := checkExpression(argument, ttl, isRemote); badFunc != nil { + if badFunc := checkExpression(argument, ttl, triggerSource); badFunc != nil { badFunc.Position = position if problemFunction == nil { @@ -209,7 +222,11 @@ func checkExpression(expression parser.Expr, ttl time.Duration, isRemote bool) * return problemFunction } -func checkFunction(funcName string, isRemote bool) *ProblemOfTarget { +func checkFunction(funcName string, triggerSource moira.TriggerSource) *ProblemOfTarget { + if triggerSource != moira.GraphiteLocal { + return nil + } + if _, isUnstable := unstableFunctions[funcName]; isUnstable { return &ProblemOfTarget{ Argument: funcName, @@ -234,7 +251,7 @@ func checkFunction(funcName string, isRemote bool) *ProblemOfTarget { } } - if !isRemote && !funcIsSupported(funcName) { + if !funcIsSupported(funcName) { return &ProblemOfTarget{ Argument: funcName, Type: isBad, diff --git a/api/dto/target_test.go b/api/dto/target_test.go index 8ccb94930..849527d7c 100644 --- a/api/dto/target_test.go +++ b/api/dto/target_test.go @@ -6,35 +6,47 @@ import ( "time" "github.com/go-graphite/carbonapi/pkg/parser" + "github.com/moira-alert/moira" . "github.com/smartystreets/goconvey/convey" ) func TestTargetVerification(t *testing.T) { Convey("Target verification", t, func() { + Convey("Check unknown trigger type", func() { + targets := []string{`alias(test.one,'One'`} + problems, err := TargetVerification(targets, 10, "random_source") + So(err, ShouldResemble, fmt.Errorf("unknown trigger source '%s'", "random_source")) + So(problems, ShouldBeNil) + }) + Convey("Check bad function", func() { targets := []string{`alias(test.one,'One'`} - problems := TargetVerification(targets, 10, false) + problems, err := TargetVerification(targets, 10, moira.GraphiteLocal) + So(err, ShouldBeNil) So(len(problems), ShouldEqual, 1) So(problems[0].SyntaxOk, ShouldBeFalse) }) Convey("Check correct construction", func() { targets := []string{`alias(test.one,'One')`} - problems := TargetVerification(targets, 10, false) + problems, err := TargetVerification(targets, 10, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) }) Convey("Check correct empty function", func() { targets := []string{`alias(movingSum(),'One')`} - problems := TargetVerification(targets, 10, false) + problems, err := TargetVerification(targets, 10, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems, ShouldBeNil) }) Convey("Check interval larger that TTL", func() { targets := []string{"movingAverage(groupByTags(seriesByTag('project=my-test-project'), 'max'), '10min')"} - problems := TargetVerification(targets, 5*time.Minute, false) + problems, err := TargetVerification(targets, 5*time.Minute, moira.GraphiteLocal) + So(err, ShouldBeNil) // target is not valid because set of metrics by last 5 minutes is not enough for function with 10min interval So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems.Argument, ShouldEqual, "movingAverage") @@ -44,7 +56,8 @@ func TestTargetVerification(t *testing.T) { Convey("Check ttl is 0", func() { targets := []string{"movingAverage(groupByTags(seriesByTag('project=my-test-project'), 'max'), '10min')"} // ttl is 0 means that metrics will persist forever - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) // target is valid because there is enough metrics So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems, ShouldBeNil) @@ -52,49 +65,56 @@ func TestTargetVerification(t *testing.T) { Convey("Check unstable function", func() { targets := []string{"summarize(test.metric, '10min')"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems.Argument, ShouldEqual, "summarize") }) Convey("Check false notifications function", func() { targets := []string{"highest(test.metric)"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems.Argument, ShouldEqual, "highest") }) Convey("Check visual function", func() { targets := []string{"consolidateBy(Servers.web01.sda1.free_space, 'max')"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems.Argument, ShouldEqual, "consolidateBy") }) Convey("Check unsupported function", func() { targets := []string{"myUnsupportedFunction(Servers.web01.sda1.free_space, 'max')"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems.Argument, ShouldEqual, "myUnsupportedFunction") }) Convey("Check nested function", func() { targets := []string{"movingAverage(myUnsupportedFunction(), '10min')"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems.Problems[0].Argument, ShouldEqual, "myUnsupportedFunction") }) Convey("Check target only with metric (without Graphite-function)", func() { targets := []string{"my.metric"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeTrue) So(problems[0].TreeOfProblems, ShouldBeNil) }) Convey("Check target with space symbol in metric name", func() { targets := []string{"a b"} - problems := TargetVerification(targets, 0, false) + problems, err := TargetVerification(targets, 0, moira.GraphiteLocal) + So(err, ShouldBeNil) So(problems[0].SyntaxOk, ShouldBeFalse) So(problems[0].TreeOfProblems, ShouldBeNil) }) diff --git a/api/dto/triggers.go b/api/dto/triggers.go index 60a9f1bc1..30c372b56 100644 --- a/api/dto/triggers.go +++ b/api/dto/triggers.go @@ -68,7 +68,11 @@ type TriggerModel struct { // Graphite patterns for trigger Patterns []string `json:"patterns" example:""` // Shows if trigger is remote (graphite-backend) based or stored inside Moira-Redis DB + // + // Deprecated: Use TriggerSource field instead IsRemote bool `json:"is_remote" example:"false"` + // Shows the source from where the metrics are fetched + TriggerSource moira.TriggerSource `json:"trigger_source" example:"graphite_local"` // If true, first event NODATA → OK will be omitted MuteNewMetrics bool `json:"mute_new_metrics" example:"false"` // A list of targets that have only alone metrics @@ -99,7 +103,7 @@ func (model *TriggerModel) ToMoiraTrigger() *moira.Trigger { Schedule: model.Schedule, Expression: &model.Expression, Patterns: model.Patterns, - IsRemote: model.IsRemote, + TriggerSource: model.TriggerSource, MuteNewMetrics: model.MuteNewMetrics, AloneMetrics: model.AloneMetrics, UpdatedBy: model.UpdatedBy, @@ -122,7 +126,8 @@ func CreateTriggerModel(trigger *moira.Trigger) TriggerModel { Schedule: trigger.Schedule, Expression: moira.UseString(trigger.Expression), Patterns: trigger.Patterns, - IsRemote: trigger.IsRemote, + IsRemote: trigger.TriggerSource == moira.GraphiteRemote, + TriggerSource: trigger.TriggerSource, MuteNewMetrics: trigger.MuteNewMetrics, AloneMetrics: trigger.AloneMetrics, CreatedAt: getDateTime(trigger.CreatedAt), @@ -172,8 +177,10 @@ func (trigger *Trigger) Bind(request *http.Request) error { Expression: &trigger.Expression, } + trigger.TriggerSource = trigger.TriggerSource.FillInIfNotSet(trigger.IsRemote) + metricsSourceProvider := middleware.GetTriggerTargetsSourceProvider(request) - metricsSource, err := metricsSourceProvider.GetMetricSource(trigger.IsRemote) + metricsSource, err := metricsSourceProvider.GetMetricSource(trigger.TriggerSource) if err != nil { return err } @@ -216,10 +223,19 @@ func checkTTLSanity(trigger *Trigger, metricsSource metricSource.MetricSource) e maximumAllowedTTL := metricsSource.GetMetricsTTLSeconds() if trigger.TTL > maximumAllowedTTL { - triggerType := "local" - if trigger.IsRemote { - triggerType = "remote" + var triggerType string + + switch trigger.TriggerSource { + case moira.GraphiteLocal: + triggerType = "graphite local" + + case moira.GraphiteRemote: + triggerType = "graphite remote" + + case moira.PrometheusRemote: + triggerType = "prometheus remote" } + return fmt.Errorf("TTL for %s trigger can't be more than %d seconds", triggerType, maximumAllowedTTL) } return nil diff --git a/api/dto/triggers_test.go b/api/dto/triggers_test.go index 210bc8d5f..ddd710437 100644 --- a/api/dto/triggers_test.go +++ b/api/dto/triggers_test.go @@ -25,7 +25,7 @@ func TestTriggerValidation(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) request, _ := http.NewRequest("PUT", "/api/trigger", nil) request.Header.Set("Content-Type", "application/json") @@ -46,7 +46,7 @@ func TestTriggerValidation(t *testing.T) { Tags: tags, TTLState: &moira.TTLStateNODATA, TTL: 600, - IsRemote: false, + TriggerSource: moira.GraphiteLocal, MuteNewMetrics: false, } @@ -271,7 +271,7 @@ func TestTriggerModel_ToMoiraTrigger(t *testing.T) { }, Expression: expression, Patterns: []string{"pattern-1", "pattern-2"}, - IsRemote: true, + TriggerSource: moira.GraphiteRemote, MuteNewMetrics: true, AloneMetrics: map[string]bool{ "t1": true, @@ -304,7 +304,7 @@ func TestTriggerModel_ToMoiraTrigger(t *testing.T) { }, Expression: &expression, Patterns: []string{"pattern-1", "pattern-2"}, - IsRemote: true, + TriggerSource: moira.GraphiteRemote, MuteNewMetrics: true, AloneMetrics: map[string]bool{ "t1": true, @@ -345,7 +345,7 @@ func TestCreateTriggerModel(t *testing.T) { }, Expression: &expression, Patterns: []string{"pattern-1", "pattern-2"}, - IsRemote: true, + TriggerSource: moira.GraphiteRemote, MuteNewMetrics: true, AloneMetrics: map[string]bool{ "t1": true, @@ -376,6 +376,7 @@ func TestCreateTriggerModel(t *testing.T) { }, Expression: expression, Patterns: []string{"pattern-1", "pattern-2"}, + TriggerSource: moira.GraphiteRemote, IsRemote: true, MuteNewMetrics: true, AloneMetrics: map[string]bool{ diff --git a/api/handler/handler.go b/api/handler/handler.go index 98d1f8df2..332010a30 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -87,7 +87,7 @@ func NewHandler(db moira.Database, log moira.Logger, index moira.Searcher, confi router.Use(moiramiddle.DatabaseContext(database)) router.Get("/config", getWebConfig(webConfigContent)) router.Route("/user", user) - router.With(moiramiddle.Triggers(config.LocalMetricTTL, config.RemoteMetricTTL)).Route("/trigger", triggers(metricSourceProvider, searchIndex)) + router.With(moiramiddle.Triggers(config.GraphiteLocalMetricTTL, config.GraphiteRemoteMetricTTL, config.PrometheusRemoteMetricTTL)).Route("/trigger", triggers(metricSourceProvider, searchIndex)) router.Route("/tag", tag) router.Route("/pattern", pattern) router.Route("/event", event) diff --git a/api/handler/trigger.go b/api/handler/trigger.go index 73d9f574e..a02e59d89 100644 --- a/api/handler/trigger.go +++ b/api/handler/trigger.go @@ -57,7 +57,13 @@ func updateTrigger(writer http.ResponseWriter, request *http.Request) { var problems []dto.TreeOfProblems if needValidate(request) { - problems = validateTargets(request, trigger) + problems, err = validateTargets(request, trigger) + + if err != nil { + render.Render(writer, request, err) //nolint + return + } + if problems != nil && dto.DoesAnyTreeHaveError(problems) { writeErrorSaveResponse(writer, request, problems) return @@ -88,17 +94,21 @@ func needValidate(request *http.Request) bool { // validateTargets checks targets of trigger. // Returns tree of problems if there is any invalid child, else returns nil. -func validateTargets(request *http.Request, trigger *dto.Trigger) (problems []dto.TreeOfProblems) { +func validateTargets(request *http.Request, trigger *dto.Trigger) ([]dto.TreeOfProblems, *api.ErrorResponse) { ttl := getMetricTTLByTrigger(request, trigger) - treesOfProblems := dto.TargetVerification(trigger.Targets, ttl, trigger.IsRemote) + treesOfProblems, err := dto.TargetVerification(trigger.Targets, ttl, trigger.TriggerSource) + + if err != nil { + return nil, api.ErrorInvalidRequest(err) + } for _, tree := range treesOfProblems { if tree.TreeOfProblems != nil { - return treesOfProblems + return treesOfProblems, nil } } - return nil + return nil, nil } func writeErrorSaveResponse(writer http.ResponseWriter, request *http.Request, treesOfProblems []dto.TreeOfProblems) { diff --git a/api/handler/trigger_render_test.go b/api/handler/trigger_render_test.go index 1f7621e85..02efbd345 100644 --- a/api/handler/trigger_render_test.go +++ b/api/handler/trigger_render_test.go @@ -22,7 +22,7 @@ func TestRenderTrigger(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) responseWriter := httptest.NewRecorder() mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) @@ -49,7 +49,11 @@ func TestRenderTrigger(t *testing.T) { }) Convey("with the wrong timezone parameter", func() { - mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ID: "triggerID-0000000000001", Targets: []string{"t1"}}, nil).Times(1) + mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ + ID: "triggerID-0000000000001", + Targets: []string{"t1"}, + TriggerSource: moira.GraphiteLocal, + }, nil).Times(1) localSource.EXPECT().IsConfigured().Return(true, nil).AnyTimes().Times(1) fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData("", []float64{}, 0, 0)}).Times(1) @@ -78,7 +82,11 @@ func TestRenderTrigger(t *testing.T) { }) Convey("without points for render", func() { - mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ID: "triggerID-0000000000001", Targets: []string{"t1"}}, nil).Times(1) + mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ + ID: "triggerID-0000000000001", + Targets: []string{"t1"}, + TriggerSource: moira.GraphiteLocal, + }, nil).Times(1) localSource.EXPECT().IsConfigured().Return(true, nil).Times(1) fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData("", []float64{}, 0, 0)}).Times(1) diff --git a/api/handler/trigger_test.go b/api/handler/trigger_test.go index 70a4224e1..d0ce8b2b9 100644 --- a/api/handler/trigger_test.go +++ b/api/handler/trigger_test.go @@ -21,6 +21,8 @@ import ( "github.com/xiam/to" ) +const triggerIDKey = "triggerID" + func TestGetTrigger(t *testing.T) { Convey("Get trigger by id", t, func() { mockCtrl := gomock.NewController(t) @@ -30,7 +32,10 @@ func TestGetTrigger(t *testing.T) { Convey("When success and have empty created_at & updated_at should return null in response", func() { throttlingTime := time.Date(2022, time.June, 7, 10, 0, 0, 0, time.UTC) - mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ID: "triggerID-0000000000001"}, nil) + mockDb.EXPECT().GetTrigger("triggerID-0000000000001").Return(moira.Trigger{ + ID: "triggerID-0000000000001", + TriggerSource: moira.GraphiteLocal, + }, nil) mockDb.EXPECT().GetTriggerThrottling("triggerID-0000000000001").Return(throttlingTime, throttlingTime) database = mockDb @@ -46,7 +51,7 @@ func TestGetTrigger(t *testing.T) { contentBytes, _ := io.ReadAll(response.Body) contents := string(contentBytes) - expected := "{\"id\":\"triggerID-0000000000001\",\"name\":\"\",\"targets\":null,\"warn_value\":null,\"error_value\":null,\"trigger_type\":\"\",\"tags\":null,\"expression\":\"\",\"patterns\":null,\"is_remote\":false,\"mute_new_metrics\":false,\"alone_metrics\":null,\"created_at\":null,\"updated_at\":null,\"created_by\":\"\",\"updated_by\":\"\",\"throttling\":0}\n" + expected := "{\"id\":\"triggerID-0000000000001\",\"name\":\"\",\"targets\":null,\"warn_value\":null,\"error_value\":null,\"trigger_type\":\"\",\"tags\":null,\"expression\":\"\",\"patterns\":null,\"is_remote\":false,\"trigger_source\":\"graphite_local\",\"mute_new_metrics\":false,\"alone_metrics\":null,\"created_at\":null,\"updated_at\":null,\"created_by\":\"\",\"updated_by\":\"\",\"throttling\":0}\n" So(contents, ShouldEqual, expected) }) @@ -56,9 +61,10 @@ func TestGetTrigger(t *testing.T) { mockDb.EXPECT().GetTrigger("triggerID-0000000000001"). Return( moira.Trigger{ - ID: "triggerID-0000000000001", - CreatedAt: &triggerTime, - UpdatedAt: &triggerTime, + ID: "triggerID-0000000000001", + CreatedAt: &triggerTime, + TriggerSource: moira.GraphiteLocal, + UpdatedAt: &triggerTime, }, nil) mockDb.EXPECT().GetTriggerThrottling("triggerID-0000000000001").Return(throttlingTime, throttlingTime) @@ -76,7 +82,7 @@ func TestGetTrigger(t *testing.T) { contentBytes, _ := io.ReadAll(response.Body) contents := string(contentBytes) - expected := "{\"id\":\"triggerID-0000000000001\",\"name\":\"\",\"targets\":null,\"warn_value\":null,\"error_value\":null,\"trigger_type\":\"\",\"tags\":null,\"expression\":\"\",\"patterns\":null,\"is_remote\":false,\"mute_new_metrics\":false,\"alone_metrics\":null,\"created_at\":\"2022-06-07T10:00:00Z\",\"updated_at\":\"2022-06-07T10:00:00Z\",\"created_by\":\"\",\"updated_by\":\"\",\"throttling\":0}\n" + expected := "{\"id\":\"triggerID-0000000000001\",\"name\":\"\",\"targets\":null,\"warn_value\":null,\"error_value\":null,\"trigger_type\":\"\",\"tags\":null,\"expression\":\"\",\"patterns\":null,\"is_remote\":false,\"trigger_source\":\"graphite_local\",\"mute_new_metrics\":false,\"alone_metrics\":null,\"created_at\":\"2022-06-07T10:00:00Z\",\"updated_at\":\"2022-06-07T10:00:00Z\",\"created_by\":\"\",\"updated_by\":\"\",\"throttling\":0}\n" So(contents, ShouldEqual, expected) }) @@ -107,7 +113,7 @@ func TestUpdateTrigger(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) localSource.EXPECT().IsConfigured().Return(true, nil).AnyTimes() localSource.EXPECT().GetMetricsTTLSeconds().Return(int64(3600)).AnyTimes() @@ -140,12 +146,12 @@ func TestUpdateTrigger(t *testing.T) { triggerWarnValue := float64(10) triggerErrorValue := float64(15) trigger := moira.Trigger{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{"my.metric"}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{"my.metric"}, + TriggerSource: moira.GraphiteLocal, } mockDb.EXPECT().GetTrigger(gomock.Any()).Return(trigger, nil) @@ -163,6 +169,7 @@ func TestUpdateTrigger(t *testing.T) { Convey(fmt.Sprintf("should return success message, url=%s", url), func() { response := responseWriter.Result() defer response.Body.Close() + So(response.StatusCode, ShouldEqual, http.StatusOK) So(isTriggerUpdated(response), ShouldBeTrue) }) @@ -180,12 +187,12 @@ func TestUpdateTrigger(t *testing.T) { triggerErrorValue := float64(15) trigger := dto.Trigger{ TriggerModel: dto.TriggerModel{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{}, + TriggerSource: moira.GraphiteLocal, }, } @@ -211,12 +218,12 @@ func TestUpdateTrigger(t *testing.T) { triggerWarnValue := float64(10) triggerErrorValue := float64(15) trigger := moira.Trigger{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{"alias(consolidateBy(Sales.widgets.largeBlue, 'sum'), 'alias to test nesting')"}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{"alias(consolidateBy(Sales.widgets.largeBlue, 'sum'), 'alias to test nesting')"}, + TriggerSource: moira.GraphiteLocal, } jsonTrigger, _ := json.Marshal(trigger) @@ -300,12 +307,12 @@ func TestUpdateTrigger(t *testing.T) { triggerWarnValue := float64(10) triggerErrorValue := float64(15) trigger := moira.Trigger{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{"alias(summarize(my.metric, '5min'), 'alias to test nesting')"}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{"alias(summarize(my.metric, '5min'), 'alias to test nesting')"}, + TriggerSource: moira.GraphiteLocal, } jsonTrigger, _ := json.Marshal(trigger) @@ -379,6 +386,152 @@ func TestUpdateTrigger(t *testing.T) { }) } +func TestGetTriggerWithTriggerSource(t *testing.T) { + mockCtrl := gomock.NewController(t) + + db := mock_moira_alert.NewMockDatabase(mockCtrl) + database = db + defer func() { database = nil }() + + const triggerId = "TestTriggerId" + + Convey("Given database returns trigger with TriggerSource = GraphiteLocal", t, func() { + request := httptest.NewRequest("GET", "/trigger/"+triggerId, nil) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), triggerIDKey, triggerId)) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "populated", true)) + + trigger := moira.Trigger{ + ID: triggerId, + WarnValue: newFloat64(1.0), + ErrorValue: newFloat64(2.0), + TriggerType: moira.RisingTrigger, + Tags: []string{"test"}, + TTLState: &moira.TTLStateOK, + TTL: 600, + Schedule: &moira.ScheduleData{}, + TriggerSource: moira.GraphiteLocal, + } + + db.EXPECT().GetTrigger(triggerId).Return(trigger, nil) + db.EXPECT().GetTriggerThrottling(triggerId) + db.EXPECT().GetNotificationEvents(triggerId, gomock.Any(), gomock.Any()).Return(make([]*moira.NotificationEvent, 0), nil) + db.EXPECT().GetNotificationEventCount(triggerId, gomock.Any()).Return(int64(0)) + + responseWriter := httptest.NewRecorder() + getTrigger(responseWriter, request) + + result := make(map[string]interface{}) + err := json.Unmarshal(responseWriter.Body.Bytes(), &result) + So(err, ShouldBeNil) + + isRemoteRaw, ok := result["is_remote"] + So(ok, ShouldBeTrue) + + isRemote, ok := isRemoteRaw.(bool) + So(ok, ShouldBeTrue) + So(isRemote, ShouldBeFalse) + + triggerSourceRaw, ok := result["trigger_source"] + So(ok, ShouldBeTrue) + + triggerSource, ok := triggerSourceRaw.(string) + So(ok, ShouldBeTrue) + So(triggerSource, ShouldEqual, moira.GraphiteLocal) + }) + + Convey("Given database returns trigger with TriggerSource = GraphiteRemote", t, func() { + request := httptest.NewRequest("GET", "/trigger/"+triggerId, nil) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), triggerIDKey, triggerId)) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "populated", true)) + + trigger := moira.Trigger{ + ID: triggerId, + WarnValue: newFloat64(1.0), + ErrorValue: newFloat64(2.0), + TriggerType: moira.RisingTrigger, + Tags: []string{"test"}, + TTLState: &moira.TTLStateOK, + TTL: 600, + Schedule: &moira.ScheduleData{}, + TriggerSource: moira.GraphiteRemote, + } + + db.EXPECT().GetTrigger(triggerId).Return(trigger, nil) + db.EXPECT().GetTriggerThrottling(triggerId) + db.EXPECT().GetNotificationEvents(triggerId, gomock.Any(), gomock.Any()).Return(make([]*moira.NotificationEvent, 0), nil) + db.EXPECT().GetNotificationEventCount(triggerId, gomock.Any()).Return(int64(0)) + + responseWriter := httptest.NewRecorder() + getTrigger(responseWriter, request) + + result := make(map[string]interface{}) + err := json.Unmarshal(responseWriter.Body.Bytes(), &result) + So(err, ShouldBeNil) + + isRemoteRaw, ok := result["is_remote"] + So(ok, ShouldBeTrue) + + isRemote, ok := isRemoteRaw.(bool) + So(ok, ShouldBeTrue) + So(isRemote, ShouldBeTrue) + + triggerSourceRaw, ok := result["trigger_source"] + So(ok, ShouldBeTrue) + + triggerSource, ok := triggerSourceRaw.(string) + So(ok, ShouldBeTrue) + So(triggerSource, ShouldEqual, moira.GraphiteRemote) + }) + + Convey("Given database returns trigger with TriggerSource = PrometheusTrigger", t, func() { + request := httptest.NewRequest("GET", "/trigger/"+triggerId, nil) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), triggerIDKey, triggerId)) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "populated", true)) + + trigger := moira.Trigger{ + ID: triggerId, + WarnValue: newFloat64(1.0), + ErrorValue: newFloat64(2.0), + TriggerType: moira.RisingTrigger, + Tags: []string{"test"}, + TTLState: &moira.TTLStateOK, + TTL: 600, + Schedule: &moira.ScheduleData{}, + TriggerSource: moira.PrometheusRemote, + } + + db.EXPECT().GetTrigger(triggerId).Return(trigger, nil) + db.EXPECT().GetTriggerThrottling(triggerId) + db.EXPECT().GetNotificationEvents(triggerId, gomock.Any(), gomock.Any()).Return(make([]*moira.NotificationEvent, 0), nil) + db.EXPECT().GetNotificationEventCount(triggerId, gomock.Any()).Return(int64(0)) + + responseWriter := httptest.NewRecorder() + getTrigger(responseWriter, request) + + result := make(map[string]interface{}) + err := json.Unmarshal(responseWriter.Body.Bytes(), &result) + So(err, ShouldBeNil) + + isRemoteRaw, ok := result["is_remote"] + So(ok, ShouldBeTrue) + + isRemote, ok := isRemoteRaw.(bool) + So(ok, ShouldBeTrue) + So(isRemote, ShouldBeFalse) + + triggerSourceRaw, ok := result["trigger_source"] + So(ok, ShouldBeTrue) + + triggerSource, ok := triggerSourceRaw.(string) + So(ok, ShouldBeTrue) + So(triggerSource, ShouldEqual, moira.PrometheusRemote) + }) +} + +func newFloat64(value float64) *float64 { + return &value +} + func isTriggerUpdated(response *http.Response) bool { contentBytes, _ := io.ReadAll(response.Body) actual := dto.SaveTriggerResponse{} diff --git a/api/handler/triggers.go b/api/handler/triggers.go index 3b78433da..15c8eb095 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -86,7 +86,12 @@ func createTrigger(writer http.ResponseWriter, request *http.Request) { var problems []dto.TreeOfProblems if needValidate(request) { - problems = validateTargets(request, trigger) + problems, err = validateTargets(request, trigger) + if err != nil { + render.Render(writer, request, err) //nolint + return + } + if problems != nil && dto.DoesAnyTreeHaveError(problems) { writeErrorSaveResponse(writer, request, problems) return @@ -150,11 +155,18 @@ func getTriggerFromRequest(request *http.Request) (*dto.Trigger, *api.ErrorRespo // getMetricTTLByTrigger gets metric ttl duration time from request context for local or remote trigger. func getMetricTTLByTrigger(request *http.Request, trigger *dto.Trigger) time.Duration { var ttl time.Duration - if trigger.IsRemote { - ttl = middleware.GetRemoteMetricTTL(request) - } else { + + switch trigger.TriggerSource { + case moira.GraphiteLocal: ttl = middleware.GetLocalMetricTTL(request) + + case moira.GraphiteRemote: + ttl = middleware.GetRemoteMetricTTL(request) + + case moira.PrometheusRemote: + ttl = middleware.GetPrometheusMetricTTL(request) } + return ttl } @@ -188,7 +200,13 @@ func triggerCheck(writer http.ResponseWriter, request *http.Request) { ttl := getMetricTTLByTrigger(request, trigger) if len(trigger.Targets) > 0 { - response.Targets = dto.TargetVerification(trigger.Targets, ttl, trigger.IsRemote) + var err error + response.Targets, err = dto.TargetVerification(trigger.Targets, ttl, trigger.TriggerSource) + + if err != nil { + render.Render(writer, request, api.ErrorInvalidRequest(err)) //nolint + return + } } render.JSON(writer, request, response) diff --git a/api/handler/triggers_test.go b/api/handler/triggers_test.go index 06d049b5e..4246f7de0 100644 --- a/api/handler/triggers_test.go +++ b/api/handler/triggers_test.go @@ -14,7 +14,9 @@ import ( "github.com/golang/mock/gomock" "github.com/moira-alert/moira" "github.com/moira-alert/moira/api" + dataBase "github.com/moira-alert/moira/database" metricSource "github.com/moira-alert/moira/metric_source" + "github.com/moira-alert/moira/metric_source/local" mock_metric_source "github.com/moira-alert/moira/mock/metric_source" mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" @@ -52,7 +54,7 @@ func TestGetTriggerFromRequest(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) localSource.EXPECT().IsConfigured().Return(true, nil).AnyTimes() localSource.EXPECT().GetMetricsTTLSeconds().Return(int64(3600)).AnyTimes() @@ -79,7 +81,7 @@ func TestGetTriggerFromRequest(t *testing.T) { Schedule: &moira.ScheduleData{}, Expression: "", Patterns: []string{}, - IsRemote: false, + TriggerSource: moira.GraphiteLocal, MuteNewMetrics: false, AloneMetrics: map[string]bool{}, CreatedAt: &time.Time{}, @@ -143,7 +145,7 @@ func TestGetMetricTTLByTrigger(t *testing.T) { Convey("Given a local trigger", t, func() { trigger := dto.Trigger{TriggerModel: dto.TriggerModel{ - IsRemote: false, + TriggerSource: moira.GraphiteLocal, }} Convey("It's metric ttl should be equal to local", func() { @@ -153,7 +155,7 @@ func TestGetMetricTTLByTrigger(t *testing.T) { Convey("Given a remote trigger", t, func() { trigger := dto.Trigger{TriggerModel: dto.TriggerModel{ - IsRemote: true, + TriggerSource: moira.GraphiteRemote, }} Convey("It's metric ttl should be equal to remote", func() { @@ -171,7 +173,7 @@ func TestTriggerCheckHandler(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) fetchResult := mock_metric_source.NewMockFetchResult(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) localSource.EXPECT().IsConfigured().Return(true, nil).AnyTimes() localSource.EXPECT().GetMetricsTTLSeconds().Return(int64(3600)).AnyTimes() @@ -184,40 +186,40 @@ func TestTriggerCheckHandler(t *testing.T) { remoteSource.EXPECT().Fetch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fetchResult, nil).AnyTimes() testCases := []struct { - isRemote bool + triggerSource moira.TriggerSource targets []string expectedResponse string }{ { - false, + moira.GraphiteLocal, []string{ "integralByInterval(aliasSub(sum(aliasByNode(my.own.metric, 6)), '(.*)', 'metric'), '1h')", }, "{\"targets\":[{\"syntax_ok\":true}]}\n", }, { - false, + moira.GraphiteLocal, []string{ "integralByInterval(aliasSub(sum(aliasByNode(my.own.metric, 6)), '(.*)', 'metric'), '6h')", }, "{\"targets\":[{\"syntax_ok\":true,\"tree_of_problems\":{\"argument\":\"integralByInterval\",\"position\":0,\"problems\":[{\"argument\":\"6h\",\"type\":\"bad\",\"description\":\"The function integralByInterval has a time sampling parameter 6h larger than allowed by the config:1h5m0s\",\"position\":1}]}}]}\n", }, { - false, + moira.GraphiteLocal, []string{ "my.own.metric", }, "{\"targets\":[{\"syntax_ok\":true}]}\n", }, { - true, + moira.GraphiteRemote, []string{ "integralByInterval(aliasSub(sum(aliasByNode(my.own.metric, 6)), '(.*)', 'metric'), '1h')", }, "{\"targets\":[{\"syntax_ok\":true}]}\n", }, { - true, + moira.GraphiteRemote, []string{ "integralByInterval(aliasSub(sum(aliasByNode(my.own.metric, 6)), '(.*)', 'metric'), '6h')", }, @@ -230,12 +232,12 @@ func TestTriggerCheckHandler(t *testing.T) { triggerErrorValue := float64(15) triggerDTO := dto.Trigger{ TriggerModel: dto.TriggerModel{ - Name: "Test trigger", - Tags: []string{"Normal", "DevOps", "DevOpsGraphite-duty"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: testCase.targets, - IsRemote: testCase.isRemote, + Name: "Test trigger", + Tags: []string{"Normal", "DevOps", "DevOpsGraphite-duty"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: testCase.targets, + TriggerSource: testCase.triggerSource, }, } jsonTrigger, _ := json.Marshal(triggerDTO) @@ -265,7 +267,7 @@ func TestCreateTriggerHandler(t *testing.T) { localSource := mock_metric_source.NewMockMetricSource(mockCtrl) remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) - sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, nil) localSource.EXPECT().IsConfigured().Return(true, nil).AnyTimes() localSource.EXPECT().GetMetricsTTLSeconds().Return(int64(3600)).AnyTimes() @@ -296,12 +298,12 @@ func TestCreateTriggerHandler(t *testing.T) { triggerErrorValue := float64(15) triggerDTO := dto.Trigger{ TriggerModel: dto.TriggerModel{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{"my.metric"}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{"my.metric"}, + TriggerSource: moira.GraphiteLocal, }, } jsonTrigger, _ := json.Marshal(triggerDTO) @@ -333,12 +335,12 @@ func TestCreateTriggerHandler(t *testing.T) { triggerErrorValue := float64(15) triggerDTO := dto.Trigger{ TriggerModel: dto.TriggerModel{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{}, + TriggerSource: moira.GraphiteLocal, }, } jsonTrigger, _ := json.Marshal(triggerDTO) @@ -363,12 +365,12 @@ func TestCreateTriggerHandler(t *testing.T) { triggerErrorValue := float64(15) trigger := dto.Trigger{ TriggerModel: dto.TriggerModel{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{"alias(consolidateBy(Sales.widgets.largeBlue, 'sum'), 'alias to test nesting')"}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{"alias(consolidateBy(Sales.widgets.largeBlue, 'sum'), 'alias to test nesting')"}, + TriggerSource: moira.GraphiteLocal, }, } jsonTrigger, _ := json.Marshal(trigger) @@ -448,12 +450,12 @@ func TestCreateTriggerHandler(t *testing.T) { triggerErrorValue := float64(15) triggerDTO := dto.Trigger{ TriggerModel: dto.TriggerModel{ - Name: "Test trigger", - Tags: []string{"123"}, - WarnValue: &triggerWarnValue, - ErrorValue: &triggerErrorValue, - Targets: []string{"alias(summarize(my.metric, '5min'), 'alias to test nesting')"}, - IsRemote: false, + Name: "Test trigger", + Tags: []string{"123"}, + WarnValue: &triggerWarnValue, + ErrorValue: &triggerErrorValue, + Targets: []string{"alias(summarize(my.metric, '5min'), 'alias to test nesting')"}, + TriggerSource: moira.GraphiteLocal, }, } jsonTrigger, _ := json.Marshal(triggerDTO) @@ -526,6 +528,155 @@ func TestCreateTriggerHandler(t *testing.T) { }) } +func TestTriggersCreatedWithTriggerSource(t *testing.T) { + mockCtrl := gomock.NewController(t) + + localSource := mock_metric_source.NewMockMetricSource(mockCtrl) + remoteSource := mock_metric_source.NewMockMetricSource(mockCtrl) + prometheusSource := mock_metric_source.NewMockMetricSource(mockCtrl) + sourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, prometheusSource) + + db := mock_moira_alert.NewMockDatabase(mockCtrl) + database = db + defer func() { database = nil }() + + triggerId := "test" + target := `test_target_value` + + Convey("Given is_remote flag is false and trigger_source is not set", t, func() { + jsonTrigger := makeTestTriggerJson(target, triggerId, `"is_remote": false`) + request := newTriggerCreateRequest(sourceProvider, triggerId, jsonTrigger) + + Convey("Expect trigger to be graphite local", func() { + setupExpectationsForCreateTrigger(localSource, db, target, triggerId, moira.GraphiteLocal) + + responseWriter := httptest.NewRecorder() + createTrigger(responseWriter, request) + + So(responseWriter.Code, ShouldEqual, 200) + }) + }) + + Convey("Given is_remote flag is true and trigger_source is not set", t, func() { + jsonTrigger := makeTestTriggerJson(target, triggerId, `"is_remote": true`) + request := newTriggerCreateRequest(sourceProvider, triggerId, jsonTrigger) + + Convey("Expect trigger to be graphite remote", func() { + setupExpectationsForCreateTrigger(remoteSource, db, target, triggerId, moira.GraphiteRemote) + + responseWriter := httptest.NewRecorder() + createTrigger(responseWriter, request) + + So(responseWriter.Code, ShouldEqual, 200) + }) + }) + + Convey("Given is_remote flag is not set and trigger_source is graphite_local", t, func() { + jsonTrigger := makeTestTriggerJson(target, triggerId, `"trigger_source": "graphite_local"`) + request := newTriggerCreateRequest(sourceProvider, triggerId, jsonTrigger) + + Convey("Expect trigger to be graphite local", func() { + setupExpectationsForCreateTrigger(localSource, db, target, triggerId, moira.GraphiteLocal) + + responseWriter := httptest.NewRecorder() + createTrigger(responseWriter, request) + + So(responseWriter.Code, ShouldEqual, 200) + }) + }) + + Convey("Given is_remote flag is not set and trigger_source is graphite_remote", t, func() { + jsonTrigger := makeTestTriggerJson(target, triggerId, `"trigger_source": "graphite_remote"`) + request := newTriggerCreateRequest(sourceProvider, triggerId, jsonTrigger) + + Convey("Expect trigger to be graphite remote", func() { + setupExpectationsForCreateTrigger(remoteSource, db, target, triggerId, moira.GraphiteRemote) + + responseWriter := httptest.NewRecorder() + createTrigger(responseWriter, request) + + So(responseWriter.Code, ShouldEqual, 200) + }) + }) + + Convey("Given is_remote flag is not set and trigger_source is prometheus_remote", t, func() { + jsonTrigger := makeTestTriggerJson(target, triggerId, `"trigger_source": "prometheus_remote"`) + request := newTriggerCreateRequest(sourceProvider, triggerId, jsonTrigger) + + Convey("Expect trigger to be prometheus remote", func() { + setupExpectationsForCreateTrigger(prometheusSource, db, target, triggerId, moira.PrometheusRemote) + + responseWriter := httptest.NewRecorder() + createTrigger(responseWriter, request) + + So(responseWriter.Code, ShouldEqual, 200) + }) + }) + + Convey("Given is_remote flag is true and trigger_source is graphite_local", t, func() { + jsonTrigger := makeTestTriggerJson(target, triggerId, `"is_remote": true, "trigger_source": "graphite_local"`) + request := newTriggerCreateRequest(sourceProvider, triggerId, jsonTrigger) + + Convey("Expect trigger to be graphite local", func() { + setupExpectationsForCreateTrigger(localSource, db, target, triggerId, moira.GraphiteLocal) + + responseWriter := httptest.NewRecorder() + createTrigger(responseWriter, request) + + So(responseWriter.Code, ShouldEqual, 200) + }) + }) +} + +func makeTestTriggerJson(target, triggerId, triggerSource string) []byte { + targetJson, _ := json.Marshal(target) + jsonTrigger := fmt.Sprintf(`{ + "name": "Test", + "targets": [ %s ], + "id": "%s", + "warn_value": 100, + "error_value": 200, + "trigger_type": "rising", + "tags": [ "test" ], + "ttl_state": "NODATA", + %s, + "ttl": 600 + }`, targetJson, triggerId, triggerSource) + return []byte(jsonTrigger) +} + +func setupExpectationsForCreateTrigger( + source *mock_metric_source.MockMetricSource, + db *mock_moira_alert.MockDatabase, + target, triggerId string, + triggerSource moira.TriggerSource, +) { + source.EXPECT().IsConfigured().Return(true, nil) + source.EXPECT().GetMetricsTTLSeconds().Return(int64(3600)) + source.EXPECT().Fetch(target, gomock.Any(), gomock.Any(), gomock.Any()).Return(&local.FetchResult{}, nil) + + db.EXPECT().GetTrigger(triggerId).Return(moira.Trigger{}, dataBase.ErrNil) + db.EXPECT().AcquireTriggerCheckLock(triggerId, gomock.Any()).Return(nil) + db.EXPECT().DeleteTriggerCheckLock(triggerId).Return(nil) + db.EXPECT().GetTriggerLastCheck(triggerId).Return(moira.CheckData{}, dataBase.ErrNil) + db.EXPECT().SetTriggerLastCheck(triggerId, gomock.Any(), triggerSource).Return(nil) + db.EXPECT().SaveTrigger(triggerId, gomock.Any()).Return(nil) +} + +func newTriggerCreateRequest( + sourceProvider *metricSource.SourceProvider, + triggerId string, + jsonTrigger []byte, +) *http.Request { + request := httptest.NewRequest("PUT", "/trigger", bytes.NewBuffer(jsonTrigger)) + request.Header.Add("content-type", "application/json") + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "metricSourceProvider", sourceProvider)) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), "localMetricTTL", to.Duration("65m"))) + request = request.WithContext(middleware.SetContextValueForTest(request.Context(), triggerIDKey, triggerId)) + + return request +} + func isTriggerCreated(response *http.Response) bool { contentBytes, _ := io.ReadAll(response.Body) actual := dto.SaveTriggerResponse{} diff --git a/api/middleware/context.go b/api/middleware/context.go index 9fe1fd266..e0020fbd0 100644 --- a/api/middleware/context.go +++ b/api/middleware/context.go @@ -182,12 +182,16 @@ func Populate(defaultPopulated bool) func(next http.Handler) http.Handler { } // Triggers gets string value target from URI query and set it to request context. If query has not values sets given values -func Triggers(LocalMetricTTL, RemoteMetricTTL time.Duration) func(next http.Handler) http.Handler { +func Triggers(localMetricTTL, remoteMetricTTL, prometheusMetricTTL time.Duration) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - localTTL := context.WithValue(request.Context(), localMetricTTLKey, LocalMetricTTL) - remoteTTL := context.WithValue(localTTL, remoteMetricTTLKey, RemoteMetricTTL) - next.ServeHTTP(writer, request.WithContext(remoteTTL)) + ctx := request.Context() + + ctx = context.WithValue(ctx, localMetricTTLKey, localMetricTTL) + ctx = context.WithValue(ctx, remoteMetricTTLKey, remoteMetricTTL) + ctx = context.WithValue(ctx, prometheusMetricTTLKey, prometheusMetricTTL) + + next.ServeHTTP(writer, request.WithContext(ctx)) }) } } diff --git a/api/middleware/middleware.go b/api/middleware/middleware.go index e8877d3e0..b6b088674 100644 --- a/api/middleware/middleware.go +++ b/api/middleware/middleware.go @@ -17,28 +17,29 @@ func (key ContextKey) String() string { } var ( - databaseKey ContextKey = "database" - searcherKey ContextKey = "searcher" - triggerIDKey ContextKey = "triggerID" - localMetricTTLKey ContextKey = "localMetricTTL" - remoteMetricTTLKey ContextKey = "remoteMetricTTL" - populateKey ContextKey = "populated" - contactIDKey ContextKey = "contactID" - tagKey ContextKey = "tag" - subscriptionIDKey ContextKey = "subscriptionID" - pageKey ContextKey = "page" - sizeKey ContextKey = "size" - pagerIDKey ContextKey = "pagerID" - createPagerKey ContextKey = "createPager" - fromKey ContextKey = "from" - toKey ContextKey = "to" - loginKey ContextKey = "login" - timeSeriesNamesKey ContextKey = "timeSeriesNames" - metricSourceProvider ContextKey = "metricSourceProvider" - targetNameKey ContextKey = "target" - teamIDKey ContextKey = "teamID" - teamUserIDKey ContextKey = "teamUserIDKey" - anonymousUser = "anonymous" + databaseKey ContextKey = "database" + searcherKey ContextKey = "searcher" + triggerIDKey ContextKey = "triggerID" + localMetricTTLKey ContextKey = "localMetricTTL" + remoteMetricTTLKey ContextKey = "remoteMetricTTL" + prometheusMetricTTLKey ContextKey = "prometheusMetricTTL" + populateKey ContextKey = "populated" + contactIDKey ContextKey = "contactID" + tagKey ContextKey = "tag" + subscriptionIDKey ContextKey = "subscriptionID" + pageKey ContextKey = "page" + sizeKey ContextKey = "size" + pagerIDKey ContextKey = "pagerID" + createPagerKey ContextKey = "createPager" + fromKey ContextKey = "from" + toKey ContextKey = "to" + loginKey ContextKey = "login" + timeSeriesNamesKey ContextKey = "timeSeriesNames" + metricSourceProvider ContextKey = "metricSourceProvider" + targetNameKey ContextKey = "target" + teamIDKey ContextKey = "teamID" + teamUserIDKey ContextKey = "teamUserIDKey" + anonymousUser = "anonymous" ) // GetDatabase gets moira.Database realization from request context @@ -72,6 +73,11 @@ func GetRemoteMetricTTL(request *http.Request) time.Duration { return request.Context().Value(remoteMetricTTLKey).(time.Duration) } +// GetRemoteMetricTTL gets remote metric ttl duration time from request context, which was sets in TriggerContext middleware +func GetPrometheusMetricTTL(request *http.Request) time.Duration { + return request.Context().Value(prometheusMetricTTLKey).(time.Duration) +} + // GetPopulated get populate bool from request context, which was sets in TriggerContext middleware func GetPopulated(request *http.Request) bool { return request.Context().Value(populateKey).(bool) diff --git a/checker/check.go b/checker/check.go index 46363ad65..f2493729b 100644 --- a/checker/check.go +++ b/checker/check.go @@ -18,9 +18,11 @@ const ( // Check handle trigger and last check and write new state of trigger, if state were change then write new NotificationEvent func (triggerChecker *TriggerChecker) Check() error { - passError := false triggerChecker.logger.Debug().Msg("Checking trigger") + checkData := newCheckData(triggerChecker.lastCheck, triggerChecker.until) + errorSeverity := NoCheckError + triggerMetricsData, err := triggerChecker.fetchTriggerMetrics() if err != nil { return triggerChecker.handleFetchError(checkData, err) @@ -28,21 +30,23 @@ func (triggerChecker *TriggerChecker) Check() error { preparedMetrics, aloneMetrics, err := triggerChecker.prepareMetrics(triggerMetricsData) if err != nil { - passError, checkData, err = triggerChecker.handlePrepareError(checkData, err) - if !passError { + errorSeverity, checkData, err = triggerChecker.handlePrepareError(checkData, err) + if errorSeverity == MustStopCheck { return err } } checkData.MetricsToTargetRelation = conversion.GetRelations(aloneMetrics, triggerChecker.trigger.AloneMetrics) + checkData, err = triggerChecker.check(preparedMetrics, aloneMetrics, checkData, triggerChecker.logger) if err != nil { return triggerChecker.handleUndefinedError(checkData, err) } - if !passError { + if errorSeverity == NoCheckError { checkData.State = moira.StateOK } + checkData.LastSuccessfulCheckTimestamp = checkData.Timestamp if checkData.LastSuccessfulCheckTimestamp != 0 { checkData, err = triggerChecker.compareTriggerStates(checkData) @@ -50,38 +54,60 @@ func (triggerChecker *TriggerChecker) Check() error { return err } } + checkData.UpdateScore() - return triggerChecker.database.SetTriggerLastCheck(triggerChecker.triggerID, &checkData, triggerChecker.trigger.IsRemote) + return triggerChecker.database.SetTriggerLastCheck( + triggerChecker.triggerID, + &checkData, + triggerChecker.trigger.TriggerSource, + ) } +type ErrorSeverity int + +const ( + NoCheckError ErrorSeverity = 0 + CanContinueCheck ErrorSeverity = 1 + MustStopCheck ErrorSeverity = 2 +) + // handlePrepareError is a function that checks error returned from prepareMetrics function. If error -// is not serious and check process can be continued first return value became true and Filled CheckData returned. -// in the other case first return value became true and error passed to this function is handled. -func (triggerChecker *TriggerChecker) handlePrepareError(checkData moira.CheckData, err error) (bool, moira.CheckData, error) { +// is not serious and check process can be continued first return value became CanContinueCheck and Filled CheckData returned. +// in the other case first return value became MustStopCheck and error passed to this function is handled. +func (triggerChecker *TriggerChecker) handlePrepareError(checkData moira.CheckData, err error) (ErrorSeverity, moira.CheckData, error) { switch err.(type) { case ErrTriggerHasSameMetricNames: checkData.State = moira.StateEXCEPTION checkData.Message = err.Error() - return true, checkData, nil + return CanContinueCheck, checkData, nil + case conversion.ErrUnexpectedAloneMetric: checkData.State = moira.StateEXCEPTION checkData.Message = err.Error() logTriggerCheckException(triggerChecker.logger, triggerChecker.triggerID, err) + case conversion.ErrEmptyAloneMetricsTarget: checkData.State = moira.StateNODATA triggerChecker.logger.Warning(). Error(err). Msg("Trigger check failed") + default: - return false, checkData, triggerChecker.handleUndefinedError(checkData, err) + return MustStopCheck, checkData, triggerChecker.handleUndefinedError(checkData, err) } checkData, err = triggerChecker.compareTriggerStates(checkData) if err != nil { - return false, checkData, err + return MustStopCheck, checkData, err } + checkData.UpdateScore() - return false, checkData, triggerChecker.database.SetTriggerLastCheck(triggerChecker.triggerID, &checkData, triggerChecker.trigger.IsRemote) + err = triggerChecker.database.SetTriggerLastCheck( + triggerChecker.triggerID, + &checkData, + triggerChecker.trigger.TriggerSource, + ) + return MustStopCheck, checkData, err } // handleFetchError is a function that checks error returned from fetchTriggerMetrics function. @@ -100,7 +126,11 @@ func (triggerChecker *TriggerChecker) handleFetchError(checkData moira.CheckData // Do not alert when user don't wanna receive // NODATA state alerts, but change trigger status checkData.UpdateScore() - return triggerChecker.database.SetTriggerLastCheck(triggerChecker.triggerID, &checkData, triggerChecker.trigger.IsRemote) + return triggerChecker.database.SetTriggerLastCheck( + triggerChecker.triggerID, + &checkData, + triggerChecker.trigger.TriggerSource, + ) } case remote.ErrRemoteTriggerResponse: timeSinceLastSuccessfulCheck := checkData.Timestamp - checkData.LastSuccessfulCheckTimestamp @@ -122,7 +152,11 @@ func (triggerChecker *TriggerChecker) handleFetchError(checkData moira.CheckData return err } checkData.UpdateScore() - return triggerChecker.database.SetTriggerLastCheck(triggerChecker.triggerID, &checkData, triggerChecker.trigger.IsRemote) + return triggerChecker.database.SetTriggerLastCheck( + triggerChecker.triggerID, + &checkData, + triggerChecker.trigger.TriggerSource, + ) } // handleUndefinedError is a function that check error with undefined type. @@ -141,7 +175,11 @@ func (triggerChecker *TriggerChecker) handleUndefinedError(checkData moira.Check return err } checkData.UpdateScore() - return triggerChecker.database.SetTriggerLastCheck(triggerChecker.triggerID, &checkData, triggerChecker.trigger.IsRemote) + return triggerChecker.database.SetTriggerLastCheck( + triggerChecker.triggerID, + &checkData, + triggerChecker.trigger.TriggerSource, + ) } func logTriggerCheckException(logger moira.Logger, triggerID string, err error) { @@ -248,19 +286,29 @@ func (triggerChecker *TriggerChecker) preparePatternMetrics(fetchedMetrics conve } // check is the function that handles check on prepared metrics. -func (triggerChecker *TriggerChecker) check(metrics map[string]map[string]metricSource.MetricData, - aloneMetrics map[string]metricSource.MetricData, checkData moira.CheckData, logger moira.Logger) (moira.CheckData, error) { - if len(metrics) == 0 { // Case when trigger have only alone metrics +func (triggerChecker *TriggerChecker) check( + metrics map[string]map[string]metricSource.MetricData, + aloneMetrics map[string]metricSource.MetricData, + checkData moira.CheckData, + logger moira.Logger, +) (moira.CheckData, error) { + // Case when trigger have only alone metrics + if len(metrics) == 0 { if metrics == nil { metrics = make(map[string]map[string]metricSource.MetricData, 1) } metricName := conversion.MetricName(aloneMetrics) metrics[metricName] = make(map[string]metricSource.MetricData) } + for metricName, targets := range metrics { - log := logger.Clone().String(moira.LogFieldNameMetricName, metricName) + log := logger.Clone(). + String(moira.LogFieldNameMetricName, metricName) + log.Debug().Msg("Checking metrics") + targets = conversion.Merge(targets, aloneMetrics) + metricState, needToDeleteMetric, err := triggerChecker.checkTargets(metricName, targets, log) if needToDeleteMetric { log.Info().Msg("Remove metric") @@ -269,6 +317,7 @@ func (triggerChecker *TriggerChecker) check(metrics map[string]map[string]metric } else { checkData.Metrics[metricName] = metricState } + if err != nil { return checkData, err } @@ -277,38 +326,56 @@ func (triggerChecker *TriggerChecker) check(metrics map[string]map[string]metric } // checkTargets is a Function that takes a -func (triggerChecker *TriggerChecker) checkTargets(metricName string, metrics map[string]metricSource.MetricData, - logger moira.Logger) (lastState moira.MetricState, needToDeleteMetric bool, err error) { +func (triggerChecker *TriggerChecker) checkTargets( + metricName string, + metrics map[string]metricSource.MetricData, + logger moira.Logger, +) ( + lastState moira.MetricState, + needToDeleteMetric bool, + err error, +) { lastState, metricStates, err := triggerChecker.getMetricStepsStates(metricName, metrics, logger) if err != nil { return lastState, needToDeleteMetric, err } + for _, currentState := range metricStates { lastState, err = triggerChecker.compareMetricStates(metricName, currentState, lastState) if err != nil { return lastState, needToDeleteMetric, err } } + needToDeleteMetric, noDataState := triggerChecker.checkForNoData(lastState, logger) if needToDeleteMetric { return lastState, needToDeleteMetric, err } + if noDataState != nil { lastState, err = triggerChecker.compareMetricStates(metricName, *noDataState, lastState) } + return lastState, needToDeleteMetric, err } -func (triggerChecker *TriggerChecker) checkForNoData(metricLastState moira.MetricState, - logger moira.Logger) (bool, *moira.MetricState) { +func (triggerChecker *TriggerChecker) checkForNoData( + metricLastState moira.MetricState, + logger moira.Logger, +) ( + needToDeleteMetric bool, + noDataState *moira.MetricState, +) { if triggerChecker.ttl == 0 { return false, nil } + lastCheckTimeStamp := triggerChecker.lastCheck.Timestamp if metricLastState.Timestamp+triggerChecker.ttl >= lastCheckTimeStamp { return false, nil } + logger.Debug(). Interface("metric_last_state", metricLastState). Msg("Metric TTL expired for state") @@ -316,6 +383,7 @@ func (triggerChecker *TriggerChecker) checkForNoData(metricLastState moira.Metri if triggerChecker.ttlState == moira.TTLStateDEL && metricLastState.EventTimestamp != 0 { return true, nil } + return false, newMetricState( metricLastState, triggerChecker.ttlState.ToMetricState(), @@ -324,13 +392,24 @@ func (triggerChecker *TriggerChecker) checkForNoData(metricLastState moira.Metri ) } -func (triggerChecker *TriggerChecker) getMetricStepsStates(metricName string, metrics map[string]metricSource.MetricData, - logger moira.Logger) (last moira.MetricState, current []moira.MetricState, err error) { +func (triggerChecker *TriggerChecker) getMetricStepsStates( + metricName string, + metrics map[string]metricSource.MetricData, + logger moira.Logger, +) ( + last moira.MetricState, + current []moira.MetricState, + err error, +) { var startTime int64 var stepTime int64 for _, metric := range metrics { // Taking values from any metric - last = triggerChecker.lastCheck.GetOrCreateMetricState(metricName, metric.StartTime-secondsInHour, triggerChecker.trigger.MuteNewMetrics) + last = triggerChecker.lastCheck.GetOrCreateMetricState( + metricName, + metric.StartTime-secondsInHour, + triggerChecker.trigger.MuteNewMetrics, + ) startTime = metric.StartTime stepTime = metric.StepTime break diff --git a/checker/check_test.go b/checker/check_test.go index 9b5c2807d..75de07472 100644 --- a/checker/check_test.go +++ b/checker/check_test.go @@ -24,7 +24,7 @@ func TestGetMetricDataState(t *testing.T) { logger, _ := logging.GetLogger("Test") var warnValue float64 = 10 var errValue float64 = 20 - checkerMetrics := metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false) + checkerMetrics := metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false, false) triggerChecker := TriggerChecker{ logger: logger, metrics: checkerMetrics.LocalMetrics, @@ -535,7 +535,7 @@ func TestCheckForNODATA(t *testing.T) { var ttl int64 = 600 - checkerMetrics := metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false) + checkerMetrics := metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false, false) triggerChecker := TriggerChecker{ metrics: checkerMetrics.LocalMetrics, logger: logger, @@ -634,7 +634,7 @@ func TestCheck(t *testing.T) { var ttl int64 = 30 - checkerMetrics := metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false) + checkerMetrics := metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false, false) triggerChecker := TriggerChecker{ triggerID: "SuperId", database: dataBase, @@ -688,7 +688,11 @@ func TestCheck(t *testing.T) { Timestamp: int64(67), Metric: triggerChecker.trigger.Name, }, true).Return(nil), - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil), + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil), ) err := triggerChecker.Check() So(err, ShouldBeNil) @@ -719,7 +723,11 @@ func TestCheck(t *testing.T) { gomock.InOrder( source.EXPECT().Fetch(pattern, triggerChecker.from, triggerChecker.until, true).Return(nil, unknownFunctionExc), dataBase.EXPECT().PushNotificationEvent(&event, true).Return(nil), - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil), + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil), ) err := triggerChecker.Check() So(err, ShouldBeNil) @@ -764,7 +772,11 @@ func TestCheck(t *testing.T) { dataBase.EXPECT().GetMetricsTTLSeconds().Return(metricsTTL), dataBase.EXPECT().RemoveMetricsValues([]string{metric}, triggerChecker.until-metricsTTL).Return(nil), dataBase.EXPECT().PushNotificationEvent(&event, true).Return(nil), - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil), + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil), ) err := triggerChecker.Check() So(err, ShouldBeNil) @@ -808,7 +820,11 @@ func TestCheck(t *testing.T) { dataBase.EXPECT().GetMetricsTTLSeconds().Return(metricsTTL), dataBase.EXPECT().RemoveMetricsValues([]string{metric}, triggerChecker.until-metricsTTL).Return(nil), dataBase.EXPECT().PushNotificationEvent(&event, true).Return(nil), - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil), + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil), ) err := triggerChecker.Check() So(err, ShouldBeNil) @@ -850,7 +866,11 @@ func TestCheck(t *testing.T) { }) fetchResult.EXPECT().GetPatternMetrics().Return([]string{metric}, nil) dataBase.EXPECT().PushNotificationEvent(&event, true).Return(nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil) err := triggerChecker.Check() So(err, ShouldBeNil) }) @@ -920,7 +940,11 @@ func TestCheck(t *testing.T) { dataBase.EXPECT().GetMetricsTTLSeconds().Return(metricsTTL), dataBase.EXPECT().RemoveMetricsValues([]string{metricName1, metricNameAlone, metricName2}, triggerChecker.until-metricsTTL).Return(nil), - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil), + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil), ) err := triggerChecker.Check() So(err, ShouldBeNil) @@ -1183,7 +1207,7 @@ func TestTriggerChecker_Check(t *testing.T) { source: source, logger: logger, config: &Config{}, - metrics: metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false).LocalMetrics, + metrics: metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false, false).LocalMetrics, from: 17, until: 67, ttl: ttl, @@ -1234,7 +1258,11 @@ func TestTriggerChecker_Check(t *testing.T) { source.EXPECT().Fetch(pattern, triggerChecker.from, triggerChecker.until, true).Return(fetchResult, nil) fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData(metric, []float64{0, 1, 2, 3, 4}, retention, triggerChecker.from)}) fetchResult.EXPECT().GetPatternMetrics().Return([]string{metric}, nil) - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil) + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil) _ = triggerChecker.Check() } @@ -1262,7 +1290,7 @@ func BenchmarkTriggerChecker_Check(b *testing.B) { source: source, logger: logger, config: &Config{}, - metrics: metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false).LocalMetrics, + metrics: metrics.ConfigureCheckerMetrics(metrics.NewDummyRegistry(), false, false).LocalMetrics, from: 17, until: 67, ttl: ttl, @@ -1313,7 +1341,12 @@ func BenchmarkTriggerChecker_Check(b *testing.B) { source.EXPECT().Fetch(pattern, triggerChecker.from, triggerChecker.until, true).Return(fetchResult, nil).AnyTimes() fetchResult.EXPECT().GetMetricsData().Return([]metricSource.MetricData{*metricSource.MakeMetricData(metric, []float64{0, 1, 2, 3, 4}, retention, triggerChecker.from)}).AnyTimes() fetchResult.EXPECT().GetPatternMetrics().Return([]string{metric}, nil).AnyTimes() - dataBase.EXPECT().SetTriggerLastCheck(triggerChecker.triggerID, &lastCheck, triggerChecker.trigger.IsRemote).Return(nil).AnyTimes() + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &lastCheck, + triggerChecker.trigger.TriggerSource, + ).Return(nil).AnyTimes() + for n := 0; n < b.N; n++ { err := triggerChecker.Check() if err != nil { @@ -1430,7 +1463,9 @@ func TestTriggerChecker_handlePrepareError(t *testing.T) { dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) logger, _ := logging.GetLogger("Test") - trigger := &moira.Trigger{} + trigger := &moira.Trigger{ + TriggerSource: moira.GraphiteLocal, + } triggerChecker := TriggerChecker{ triggerID: "test trigger", trigger: trigger, @@ -1443,7 +1478,7 @@ func TestTriggerChecker_handlePrepareError(t *testing.T) { err := ErrTriggerHasSameMetricNames{} pass, checkDataReturn, errReturn := triggerChecker.handlePrepareError(checkData, err) So(errReturn, ShouldBeNil) - So(pass, ShouldBeTrue) + So(pass, ShouldEqual, CanContinueCheck) So(checkDataReturn, ShouldResemble, moira.CheckData{ State: moira.StateEXCEPTION, Message: err.Error(), @@ -1472,10 +1507,10 @@ func TestTriggerChecker_handlePrepareError(t *testing.T) { Metric: triggerChecker.trigger.Name, MessageEventInfo: nil, }, true) - dataBase.EXPECT().SetTriggerLastCheck("test trigger", &expectedCheckData, false) + dataBase.EXPECT().SetTriggerLastCheck("test trigger", &expectedCheckData, moira.GraphiteLocal) pass, checkDataReturn, errReturn := triggerChecker.handlePrepareError(checkData, err) So(errReturn, ShouldBeNil) - So(pass, ShouldBeFalse) + So(pass, ShouldEqual, MustStopCheck) So(checkDataReturn, ShouldResemble, expectedCheckData) }) Convey("with ErrEmptyAloneMetricsTarget-this error is handled as NODATA", func() { @@ -1489,10 +1524,10 @@ func TestTriggerChecker_handlePrepareError(t *testing.T) { State: moira.StateNODATA, EventTimestamp: 10, } - dataBase.EXPECT().SetTriggerLastCheck("test trigger", &expectedCheckData, false) + dataBase.EXPECT().SetTriggerLastCheck("test trigger", &expectedCheckData, moira.GraphiteLocal) pass, checkDataReturn, errReturn := triggerChecker.handlePrepareError(checkData, err) So(errReturn, ShouldBeNil) - So(pass, ShouldBeFalse) + So(pass, ShouldEqual, MustStopCheck) So(checkDataReturn, ShouldResemble, expectedCheckData) }) }) diff --git a/checker/config.go b/checker/config.go index ad0b903e2..a8ae75cdf 100644 --- a/checker/config.go +++ b/checker/config.go @@ -11,8 +11,9 @@ type Config struct { CheckInterval time.Duration LazyTriggersCheckInterval time.Duration StopCheckingIntervalSeconds int64 - MaxParallelChecks int + MaxParallelLocalChecks int MaxParallelRemoteChecks int + MaxParallelPrometheusChecks int LogFile string LogLevel string LogTriggersToLevel map[string]string diff --git a/checker/event.go b/checker/event.go index dff0fb004..740712f50 100644 --- a/checker/event.go +++ b/checker/event.go @@ -33,7 +33,15 @@ func (triggerChecker *TriggerChecker) compareTriggerStates(currentCheck moira.Ch currentCheck.SuppressedState = lastStateSuppressedValue maintenanceInfo, maintenanceTimestamp := getMaintenanceInfo(lastCheck, nil) - eventInfo, needSend := isStateChanged(currentStateValue, lastStateValue, currentCheckTimestamp, lastCheck.GetEventTimestamp(), lastStateSuppressed, lastStateSuppressedValue, maintenanceInfo) + eventInfo, needSend := isStateChanged( + currentStateValue, + lastStateValue, + currentCheckTimestamp, + lastCheck.GetEventTimestamp(), + lastStateSuppressed, + lastStateSuppressedValue, + maintenanceInfo, + ) if !needSend { if maintenanceTimestamp < currentCheckTimestamp { currentCheck.Suppressed = false @@ -83,7 +91,15 @@ func (triggerChecker *TriggerChecker) compareMetricStates(metric string, current currentState.SuppressedState = lastState.SuppressedState maintenanceInfo, maintenanceTimestamp := getMaintenanceInfo(triggerChecker.lastCheck, ¤tState) - eventInfo, needSend := isStateChanged(currentState.State, lastState.State, currentState.Timestamp, lastState.GetEventTimestamp(), lastState.Suppressed, lastState.SuppressedState, maintenanceInfo) + eventInfo, needSend := isStateChanged( + currentState.State, + lastState.State, + currentState.Timestamp, + lastState.GetEventTimestamp(), + lastState.Suppressed, + lastState.SuppressedState, + maintenanceInfo, + ) if !needSend { if maintenanceTimestamp < currentState.Timestamp { currentState.Suppressed = false diff --git a/checker/trigger_checker.go b/checker/trigger_checker.go index ebdd1a76a..5cb738f0f 100644 --- a/checker/trigger_checker.go +++ b/checker/trigger_checker.go @@ -31,7 +31,14 @@ type TriggerChecker struct { // MakeTriggerChecker initialize new triggerChecker data // if trigger does not exists then return ErrTriggerNotExists error // if trigger metrics source does not configured then return ErrMetricSourceIsNotConfigured error. -func MakeTriggerChecker(triggerID string, dataBase moira.Database, logger moira.Logger, config *Config, sourceProvider *metricSource.SourceProvider, metrics *metrics.CheckerMetrics) (*TriggerChecker, error) { +func MakeTriggerChecker( + triggerID string, + dataBase moira.Database, + logger moira.Logger, + config *Config, + sourceProvider *metricSource.SourceProvider, + metrics *metrics.CheckerMetrics, +) (*TriggerChecker, error) { until := time.Now().Unix() trigger, err := dataBase.GetTrigger(triggerID) if err != nil { diff --git a/checker/trigger_checker_test.go b/checker/trigger_checker_test.go index eba000f7c..91273ad0d 100644 --- a/checker/trigger_checker_test.go +++ b/checker/trigger_checker_test.go @@ -27,24 +27,31 @@ func TestInitTriggerChecker(t *testing.T) { Convey("Test errors", t, func() { Convey("Get trigger error", func() { getTriggerError := fmt.Errorf("Oppps! Can't read trigger") - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{}, getTriggerError) - _, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + TriggerSource: moira.GraphiteLocal, + }, getTriggerError) + _, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) So(err, ShouldBeError) So(err, ShouldResemble, getTriggerError) }) Convey("No trigger error", func() { - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{}, database.ErrNil) - _, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + TriggerSource: moira.GraphiteLocal, + }, database.ErrNil) + _, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) So(err, ShouldBeError) So(err, ShouldResemble, ErrTriggerNotExists) }) Convey("Get lastCheck error", func() { readLastCheckError := fmt.Errorf("Oppps! Can't read last check") - dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{TriggerType: moira.RisingTrigger}, nil) + dataBase.EXPECT().GetTrigger(triggerID).Return(moira.Trigger{ + TriggerType: moira.RisingTrigger, + TriggerSource: moira.GraphiteLocal, + }, nil) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, readLastCheckError) - _, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + _, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) So(err, ShouldBeError) So(err, ShouldResemble, readLastCheckError) }) @@ -54,18 +61,18 @@ func TestInitTriggerChecker(t *testing.T) { var errorValue float64 = 100000 var ttl int64 = 900 var value float64 - trigger := moira.Trigger{ - ID: "d39b8510-b2f4-448c-b881-824658c58128", - Name: "Time", - Targets: []string{"aliasByNode(Metric.*.time, 1)"}, - WarnValue: &warnValue, - ErrorValue: &errorValue, - TriggerType: moira.RisingTrigger, - Tags: []string{"tag1", "tag2"}, - TTLState: &moira.TTLStateOK, - Patterns: []string{"Egais.elasticsearch.*.*.jvm.gc.collection.time"}, - TTL: ttl, + ID: "d39b8510-b2f4-448c-b881-824658c58128", + Name: "Time", + Targets: []string{"aliasByNode(Metric.*.time, 1)"}, + WarnValue: &warnValue, + ErrorValue: &errorValue, + TriggerType: moira.RisingTrigger, + Tags: []string{"tag1", "tag2"}, + TTLState: &moira.TTLStateOK, + Patterns: []string{"Egais.elasticsearch.*.*.jvm.gc.collection.time"}, + TTL: ttl, + TriggerSource: moira.GraphiteLocal, } lastCheck := moira.CheckData{ @@ -100,7 +107,7 @@ func TestInitTriggerChecker(t *testing.T) { Convey("Test trigger checker with lastCheck", t, func() { dataBase.EXPECT().GetTrigger(triggerID).Return(trigger, nil) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(lastCheck, nil) - actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) So(err, ShouldBeNil) expected := TriggerChecker{ @@ -122,7 +129,7 @@ func TestInitTriggerChecker(t *testing.T) { Convey("Test trigger checker without lastCheck", t, func() { dataBase.EXPECT().GetTrigger(triggerID).Return(trigger, nil) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) So(err, ShouldBeNil) expected := TriggerChecker{ @@ -151,7 +158,7 @@ func TestInitTriggerChecker(t *testing.T) { Convey("Test trigger checker without lastCheck and ttl", t, func() { dataBase.EXPECT().GetTrigger(triggerID).Return(trigger, nil) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, database.ErrNil) - actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) So(err, ShouldBeNil) expected := TriggerChecker{ @@ -177,7 +184,8 @@ func TestInitTriggerChecker(t *testing.T) { Convey("Test trigger checker with lastCheck and without ttl", t, func() { dataBase.EXPECT().GetTrigger(triggerID).Return(trigger, nil) dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(lastCheck, nil) - actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil), &metrics.CheckerMetrics{}) + actual, err := MakeTriggerChecker(triggerID, dataBase, logger, config, metricSource.CreateMetricSourceProvider(localSource, nil, nil), &metrics.CheckerMetrics{}) + So(err, ShouldBeNil) expected := TriggerChecker{ diff --git a/checker/worker/handler.go b/checker/worker/handler.go index dba0cde01..24ef5c947 100644 --- a/checker/worker/handler.go +++ b/checker/worker/handler.go @@ -12,18 +12,19 @@ import ( const sleepAfterCheckingError = time.Second * 2 -// startTriggerHandler is blocking func -func (worker *Checker) startTriggerHandler(triggerIDsToCheck <-chan string, metrics *metrics.CheckMetrics) error { +// startTriggerHandler is a blocking func +func (check *Checker) startTriggerHandler(triggerIDsToCheck <-chan string, metrics *metrics.CheckMetrics) error { for { triggerID, ok := <-triggerIDsToCheck if !ok { return nil } - err := worker.handleTrigger(triggerID, metrics) + + err := check.handleTrigger(triggerID, metrics) if err != nil { metrics.HandleError.Mark(1) - worker.Logger.Error(). + check.Logger.Error(). String(moira.LogFieldNameTriggerID, triggerID). Error(err). Msg("Failed to handle trigger") @@ -33,39 +34,50 @@ func (worker *Checker) startTriggerHandler(triggerIDsToCheck <-chan string, metr } } -func (worker *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) error { +func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) error { var err error defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: '%s' stack: %s", r, debug.Stack()) } }() - err = worker.handleTriggerInLock(triggerID, metrics) + err = check.handleTriggerInLock(triggerID, metrics) return err } -func (worker *Checker) handleTriggerInLock(triggerID string, metrics *metrics.CheckMetrics) error { - acquired, err := worker.Database.SetTriggerCheckLock(triggerID) +func (check *Checker) handleTriggerInLock(triggerID string, metrics *metrics.CheckMetrics) error { + acquired, err := check.Database.SetTriggerCheckLock(triggerID) if err != nil { return err } - if acquired { - startedAt := time.Now() - defer metrics.TriggersCheckTime.UpdateSince(startedAt) - if err := worker.checkTrigger(triggerID); err != nil { - return err - } + + if !acquired { + return nil } - return nil + + startedAt := time.Now() + defer metrics.TriggersCheckTime.UpdateSince(startedAt) + + err = check.checkTrigger(triggerID) + return err } -func (worker *Checker) checkTrigger(triggerID string) error { - defer worker.Database.DeleteTriggerCheckLock(triggerID) //nolint - triggerChecker, err := checker.MakeTriggerChecker(triggerID, worker.Database, worker.Logger, worker.Config, worker.SourceProvider, worker.Metrics) +func (check *Checker) checkTrigger(triggerID string) error { + defer check.Database.DeleteTriggerCheckLock(triggerID) //nolint + + triggerChecker, err := checker.MakeTriggerChecker( + triggerID, + check.Database, + check.Logger, + check.Config, + check.SourceProvider, + check.Metrics, + ) + + if err == checker.ErrTriggerNotExists { + return nil + } if err != nil { - if err == checker.ErrTriggerNotExists { - return nil - } return err } return triggerChecker.Check() diff --git a/checker/worker/lazy_triggers.go b/checker/worker/lazy_triggers.go index 3ba4f14c6..2104ae66f 100644 --- a/checker/worker/lazy_triggers.go +++ b/checker/worker/lazy_triggers.go @@ -9,30 +9,30 @@ const ( lazyTriggersWorkerTicker = time.Second * 10 ) -func (worker *Checker) lazyTriggersWorker() error { - if worker.Config.LazyTriggersCheckInterval <= worker.Config.CheckInterval { - worker.Logger.Info(). - Interface("lazy_triggers_check_interval", worker.Config.LazyTriggersCheckInterval). - Interface("check_interval", worker.Config.CheckInterval). +func (check *Checker) lazyTriggersWorker() error { + if check.Config.LazyTriggersCheckInterval <= check.Config.CheckInterval { + check.Logger.Info(). + Interface("lazy_triggers_check_interval", check.Config.LazyTriggersCheckInterval). + Interface("check_interval", check.Config.CheckInterval). Msg("Lazy triggers worker won't start because lazy triggers interval is less or equal to check interval") return nil } checkTicker := time.NewTicker(lazyTriggersWorkerTicker) - worker.Logger.Info(). - Interface("lazy_triggers_check_interval", worker.Config.LazyTriggersCheckInterval). + check.Logger.Info(). + Interface("lazy_triggers_check_interval", check.Config.LazyTriggersCheckInterval). Interface("update_lazy_triggers_every", lazyTriggersWorkerTicker). Msg("Start lazy triggers worker") for { select { - case <-worker.tomb.Dying(): + case <-check.tomb.Dying(): checkTicker.Stop() - worker.Logger.Info().Msg("Lazy triggers worker stopped") + check.Logger.Info().Msg("Lazy triggers worker stopped") return nil case <-checkTicker.C: - err := worker.fillLazyTriggerIDs() + err := check.fillLazyTriggerIDs() if err != nil { - worker.Logger.Error(). + check.Logger.Error(). Error(err). Msg("Failed to get lazy triggers") } @@ -40,8 +40,8 @@ func (worker *Checker) lazyTriggersWorker() error { } } -func (worker *Checker) fillLazyTriggerIDs() error { - triggerIDs, err := worker.Database.GetUnusedTriggerIDs() +func (check *Checker) fillLazyTriggerIDs() error { + triggerIDs, err := check.Database.GetUnusedTriggerIDs() if err != nil { return err } @@ -49,13 +49,13 @@ func (worker *Checker) fillLazyTriggerIDs() error { for _, triggerID := range triggerIDs { newLazyTriggerIDs[triggerID] = true } - worker.lazyTriggerIDs.Store(newLazyTriggerIDs) - worker.Metrics.UnusedTriggersCount.Update(int64(len(newLazyTriggerIDs))) + check.lazyTriggerIDs.Store(newLazyTriggerIDs) + check.Metrics.UnusedTriggersCount.Update(int64(len(newLazyTriggerIDs))) return nil } -func (worker *Checker) getRandomLazyCacheDuration() time.Duration { - maxLazyCacheSeconds := worker.Config.LazyTriggersCheckInterval.Seconds() +func (check *Checker) getRandomLazyCacheDuration() time.Duration { + maxLazyCacheSeconds := check.Config.LazyTriggersCheckInterval.Seconds() min := maxLazyCacheSeconds / 2 //nolint i := rand.Float64()*min + min return time.Duration(i) * time.Second diff --git a/checker/worker/local.go b/checker/worker/local.go new file mode 100644 index 000000000..4febcb5b1 --- /dev/null +++ b/checker/worker/local.go @@ -0,0 +1,103 @@ +package worker + +import ( + "time" + + "github.com/moira-alert/moira/metrics" + w "github.com/moira-alert/moira/worker" +) + +const ( + nodataCheckerLockName = "moira-nodata-checker" + nodataCheckerLockTTL = time.Second * 15 + nodataWorkerName = "NODATA checker" +) + +type localChecker struct { + check *Checker +} + +func newLocalChecker(check *Checker) checkerWorker { + return &localChecker{ + check: check, + } +} + +func (ch *localChecker) Name() string { + return "Local" +} + +func (ch *localChecker) IsEnabled() bool { + return true +} + +func (ch *localChecker) MaxParallelChecks() int { + return ch.check.Config.MaxParallelLocalChecks +} + +func (ch *localChecker) Metrics() *metrics.CheckMetrics { + return ch.check.Metrics.LocalMetrics +} + +// localTriggerGetter starts NODATA checker and manages its subscription in Redis +// to make sure there is always only one working checker +func (ch *localChecker) StartTriggerGetter() error { + w.NewWorker( + nodataWorkerName, + ch.check.Logger, + ch.check.Database.NewLock(nodataCheckerLockName, nodataCheckerLockTTL), + ch.localChecker, + ).Run(ch.check.tomb.Dying()) + + return nil +} + +func (ch *localChecker) GetTriggersToCheck(count int) ([]string, error) { + return ch.check.Database.GetLocalTriggersToCheck(count) +} + +func (ch *localChecker) localChecker(stop <-chan struct{}) error { + checkTicker := time.NewTicker(ch.check.Config.NoDataCheckInterval) + ch.check.Logger.Info().Msg("Local checker started") + for { + select { + case <-stop: + ch.check.Logger.Info().Msg("Local checker stopped") + checkTicker.Stop() + return nil + + case <-checkTicker.C: + if err := ch.addLocalTriggersToCheckQueue(); err != nil { + ch.check.Logger.Error(). + Error(err). + Msg("Local check failed") + } + } + } +} + +func (ch *localChecker) addLocalTriggersToCheckQueue() error { + now := time.Now().UTC().Unix() + if ch.check.lastData+ch.check.Config.StopCheckingIntervalSeconds < now { + ch.check.Logger.Info(). + Int64("no_metrics_for_sec", now-ch.check.lastData). + Msg("Checking Local disabled. No metrics for some seconds") + return nil + } + + ch.check.Logger.Info().Msg("Checking Local") + triggerIds, err := ch.check.Database.GetLocalTriggerIDs() + if err != nil { + return err + } + ch.check.addLocalTriggerIDsIfNeeded(triggerIds) + + return nil +} + +func (check *Checker) addLocalTriggerIDsIfNeeded(triggerIDs []string) { + needToCheckTriggerIDs := check.getTriggerIDsToCheck(triggerIDs) + if len(needToCheckTriggerIDs) > 0 { + check.Database.AddLocalTriggersToCheck(needToCheckTriggerIDs) //nolint + } +} diff --git a/checker/worker/metrics.go b/checker/worker/metrics.go index 2d8b3e19c..086bf674d 100644 --- a/checker/worker/metrics.go +++ b/checker/worker/metrics.go @@ -7,16 +7,16 @@ import ( "github.com/patrickmn/go-cache" ) -func (worker *Checker) newMetricsHandler(metricEventsChannel <-chan *moira.MetricEvent) error { +func (check *Checker) newMetricsHandler(metricEventsChannel <-chan *moira.MetricEvent) error { for { metricEvent, ok := <-metricEventsChannel if !ok { return nil } pattern := metricEvent.Pattern - if worker.needHandlePattern(pattern) { - if err := worker.handleMetricEvent(pattern); err != nil { - worker.Logger.Error(). + if check.needHandlePattern(pattern) { + if err := check.handleMetricEvent(pattern); err != nil { + check.Logger.Error(). Error(err). Msg("Failed to handle metricEvent") } @@ -24,56 +24,28 @@ func (worker *Checker) newMetricsHandler(metricEventsChannel <-chan *moira.Metri } } -func (worker *Checker) needHandlePattern(pattern string) bool { - err := worker.PatternCache.Add(pattern, true, cache.DefaultExpiration) +func (check *Checker) needHandlePattern(pattern string) bool { + err := check.PatternCache.Add(pattern, true, cache.DefaultExpiration) return err == nil } -func (worker *Checker) handleMetricEvent(pattern string) error { +func (check *Checker) handleMetricEvent(pattern string) error { start := time.Now() - defer worker.Metrics.MetricEventsHandleTime.UpdateSince(start) - worker.lastData = time.Now().UTC().Unix() - triggerIds, err := worker.Database.GetPatternTriggerIDs(pattern) + defer check.Metrics.MetricEventsHandleTime.UpdateSince(start) + + check.lastData = time.Now().UTC().Unix() + triggerIds, err := check.Database.GetPatternTriggerIDs(pattern) if err != nil { return err } + // Cleanup pattern and its metrics if this pattern doesn't match to any trigger if len(triggerIds) == 0 { - if err := worker.Database.RemovePatternWithMetrics(pattern); err != nil { + if err := check.Database.RemovePatternWithMetrics(pattern); err != nil { return err } } - worker.addTriggerIDsIfNeeded(triggerIds) - return nil -} - -func (worker *Checker) addTriggerIDsIfNeeded(triggerIDs []string) { - needToCheckTriggerIDs := worker.getTriggerIDsToCheck(triggerIDs) - if len(needToCheckTriggerIDs) > 0 { - worker.Database.AddLocalTriggersToCheck(needToCheckTriggerIDs) //nolint - } -} -func (worker *Checker) addRemoteTriggerIDsIfNeeded(triggerIDs []string) { - needToCheckRemoteTriggerIDs := worker.getTriggerIDsToCheck(triggerIDs) - if len(needToCheckRemoteTriggerIDs) > 0 { - worker.Database.AddRemoteTriggersToCheck(needToCheckRemoteTriggerIDs) //nolint - } -} - -func (worker *Checker) getTriggerIDsToCheck(triggerIDs []string) []string { - lazyTriggerIDs := worker.lazyTriggerIDs.Load().(map[string]bool) - var triggerIDsToCheck []string = make([]string, 0, len(triggerIDs)) - for _, triggerID := range triggerIDs { - if _, ok := lazyTriggerIDs[triggerID]; ok { - randomDuration := worker.getRandomLazyCacheDuration() - if err := worker.LazyTriggersCache.Add(triggerID, true, randomDuration); err != nil { - continue - } - } - if err := worker.TriggerCache.Add(triggerID, true, cache.DefaultExpiration); err == nil { - triggerIDsToCheck = append(triggerIDsToCheck, triggerID) - } - } - return triggerIDsToCheck + check.addLocalTriggerIDsIfNeeded(triggerIds) + return nil } diff --git a/checker/worker/nodata.go b/checker/worker/nodata.go deleted file mode 100644 index 21f7077ab..000000000 --- a/checker/worker/nodata.go +++ /dev/null @@ -1,62 +0,0 @@ -package worker - -import ( - "time" - - w "github.com/moira-alert/moira/worker" -) - -const ( - nodataCheckerLockName = "moira-nodata-checker" - nodataCheckerLockTTL = time.Second * 15 - nodataWorkerName = "NODATA checker" -) - -// localTriggerGetter starts NODATA checker and manages its subscription in Redis -// to make sure there is always only one working checker -func (worker *Checker) localTriggerGetter() error { - w.NewWorker( - nodataWorkerName, - worker.Logger, - worker.Database.NewLock(nodataCheckerLockName, nodataCheckerLockTTL), - worker.noDataChecker, - ).Run(worker.tomb.Dying()) - - return nil -} - -func (worker *Checker) noDataChecker(stop <-chan struct{}) error { - checkTicker := time.NewTicker(worker.Config.NoDataCheckInterval) - worker.Logger.Info().Msg("NODATA checker started") - for { - select { - case <-stop: - worker.Logger.Info().Msg("NODATA checker stopped") - checkTicker.Stop() - return nil - case <-checkTicker.C: - if err := worker.checkNoData(); err != nil { - worker.Logger.Error(). - Error(err). - Msg("NODATA check failed") - } - } - } -} - -func (worker *Checker) checkNoData() error { - now := time.Now().UTC().Unix() - if worker.lastData+worker.Config.StopCheckingIntervalSeconds < now { - worker.Logger.Info(). - Int64("no_metrics_for_sec", now-worker.lastData). - Msg("Checking NODATA disabled. No metrics for some seconds") - } else { - worker.Logger.Info().Msg("Checking NODATA") - triggerIds, err := worker.Database.GetLocalTriggerIDs() - if err != nil { - return err - } - worker.addTriggerIDsIfNeeded(triggerIds) - } - return nil -} diff --git a/checker/worker/prometheus.go b/checker/worker/prometheus.go new file mode 100644 index 000000000..b8fdca815 --- /dev/null +++ b/checker/worker/prometheus.go @@ -0,0 +1,106 @@ +package worker + +import ( + "time" + + "github.com/moira-alert/moira/metrics" + w "github.com/moira-alert/moira/worker" +) + +const ( + prometheusTriggerLockName = "moira-prometheus-checker" + prometheusTriggerName = "Prometheus checker" +) + +type prometheusChecker struct { + check *Checker +} + +func newPrometheusChecker(check *Checker) checkerWorker { + return &prometheusChecker{ + check: check, + } +} + +func (ch *prometheusChecker) Name() string { + return "Prometheus" +} + +func (ch *prometheusChecker) IsEnabled() bool { + return ch.check.PrometheusConfig.Enabled +} + +func (ch *prometheusChecker) MaxParallelChecks() int { + return ch.check.Config.MaxParallelPrometheusChecks +} + +func (ch *prometheusChecker) Metrics() *metrics.CheckMetrics { + return ch.check.Metrics.PrometheusMetrics +} + +func (ch *prometheusChecker) StartTriggerGetter() error { + w.NewWorker( + remoteTriggerName, + ch.check.Logger, + ch.check.Database.NewLock(prometheusTriggerLockName, nodataCheckerLockTTL), + ch.prometheusTriggerChecker, + ).Run(ch.check.tomb.Dying()) + + return nil +} + +func (ch *prometheusChecker) GetTriggersToCheck(count int) ([]string, error) { + return ch.check.Database.GetPrometheusTriggersToCheck(count) +} + +func (ch *prometheusChecker) prometheusTriggerChecker(stop <-chan struct{}) error { + checkTicker := time.NewTicker(ch.check.PrometheusConfig.CheckInterval) + ch.check.Logger.Info().Msg(prometheusTriggerName + " started") + for { + select { + case <-stop: + ch.check.Logger.Info().Msg(prometheusTriggerName + " stopped") + checkTicker.Stop() + return nil + case <-checkTicker.C: + if err := ch.checkPrometheus(); err != nil { + ch.check.Logger.Error(). + Error(err). + Msg("Prometheus trigger failed") + } + } + } +} + +func (ch *prometheusChecker) checkPrometheus() error { + source, err := ch.check.SourceProvider.GetPrometheus() + if err != nil { + return err + } + + available, err := source.IsAvailable() + if !available { + ch.check.Logger.Info(). + Error(err). + Msg("Prometheus API is unavailable. Stop checking prometheus triggers") + return nil + } + + ch.check.Logger.Debug().Msg("Checking prometheus triggers") + triggerIds, err := ch.check.Database.GetPrometheusTriggerIDs() + + if err != nil { + return err + } + + ch.addPrometheusTriggerIDsIfNeeded(triggerIds) + + return nil +} + +func (ch *prometheusChecker) addPrometheusTriggerIDsIfNeeded(triggerIDs []string) { + needToCheckPrometheusTriggerIDs := ch.check.getTriggerIDsToCheck(triggerIDs) + if len(needToCheckPrometheusTriggerIDs) > 0 { + ch.check.Database.AddPrometheusTriggersToCheck(needToCheckPrometheusTriggerIDs) //nolint + } +} diff --git a/checker/worker/remote.go b/checker/worker/remote.go index f0e6b791c..f7f59f07e 100644 --- a/checker/worker/remote.go +++ b/checker/worker/remote.go @@ -3,7 +3,7 @@ package worker import ( "time" - "github.com/moira-alert/moira/metric_source/remote" + "github.com/moira-alert/moira/metrics" w "github.com/moira-alert/moira/worker" ) @@ -12,54 +12,95 @@ const ( remoteTriggerName = "Remote checker" ) -func (worker *Checker) remoteTriggerGetter() error { +type remoteChecker struct { + check *Checker +} + +func newRemoteChecker(check *Checker) checkerWorker { + return &remoteChecker{ + check: check, + } +} + +func (ch *remoteChecker) Name() string { + return "Remote" +} + +func (ch *remoteChecker) IsEnabled() bool { + return ch.check.RemoteConfig.Enabled +} + +func (ch *remoteChecker) MaxParallelChecks() int { + return ch.check.Config.MaxParallelRemoteChecks +} + +func (ch *remoteChecker) Metrics() *metrics.CheckMetrics { + return ch.check.Metrics.RemoteMetrics +} + +func (ch *remoteChecker) StartTriggerGetter() error { w.NewWorker( remoteTriggerName, - worker.Logger, - worker.Database.NewLock(remoteTriggerLockName, nodataCheckerLockTTL), - worker.remoteTriggerChecker, - ).Run(worker.tomb.Dying()) + ch.check.Logger, + ch.check.Database.NewLock(remoteTriggerLockName, nodataCheckerLockTTL), + ch.remoteTriggerChecker, + ).Run(ch.check.tomb.Dying()) return nil } -func (worker *Checker) remoteTriggerChecker(stop <-chan struct{}) error { - checkTicker := time.NewTicker(worker.RemoteConfig.CheckInterval) - worker.Logger.Info().Msg(remoteTriggerName + " started") +func (ch *remoteChecker) GetTriggersToCheck(count int) ([]string, error) { + return ch.check.Database.GetRemoteTriggersToCheck(count) +} + +func (ch *remoteChecker) remoteTriggerChecker(stop <-chan struct{}) error { + checkTicker := time.NewTicker(ch.check.RemoteConfig.CheckInterval) + ch.check.Logger.Info().Msg(remoteTriggerName + " started") for { select { case <-stop: - worker.Logger.Info().Msg(remoteTriggerName + " stopped") + ch.check.Logger.Info().Msg(remoteTriggerName + " stopped") checkTicker.Stop() return nil + case <-checkTicker.C: - if err := worker.checkRemote(); err != nil { - worker.Logger.Error(). + if err := ch.checkRemote(); err != nil { + ch.check.Logger.Error(). Error(err). - String("remote_trigger_name", remoteTriggerName). Msg("Remote trigger failed") } } } } -func (worker *Checker) checkRemote() error { - source, err := worker.SourceProvider.GetRemote() +func (ch *remoteChecker) checkRemote() error { + source, err := ch.check.SourceProvider.GetRemote() if err != nil { return err } - remoteAvailable, err := source.(*remote.Remote).IsRemoteAvailable() - if !remoteAvailable { - worker.Logger.Info(). + + available, err := source.IsAvailable() + if !available { + ch.check.Logger.Info(). Error(err). Msg("Remote API is unavailable. Stop checking remote triggers") - } else { - worker.Logger.Debug().Msg("Checking remote triggers") - triggerIds, err := worker.Database.GetRemoteTriggerIDs() - if err != nil { - return err - } - worker.addRemoteTriggerIDsIfNeeded(triggerIds) + return nil } + + ch.check.Logger.Debug().Msg("Checking remote triggers") + + triggerIds, err := ch.check.Database.GetRemoteTriggerIDs() + if err != nil { + return err + } + ch.addRemoteTriggerIDsIfNeeded(triggerIds) + return nil } + +func (ch *remoteChecker) addRemoteTriggerIDsIfNeeded(triggerIDs []string) { + needToCheckRemoteTriggerIDs := ch.check.getTriggerIDsToCheck(triggerIDs) + if len(needToCheckRemoteTriggerIDs) > 0 { + ch.check.Database.AddRemoteTriggersToCheck(needToCheckRemoteTriggerIDs) //nolint + } +} diff --git a/checker/worker/trigger_to_check.go b/checker/worker/trigger_to_check.go index d4cf3f13c..fc8dbe501 100644 --- a/checker/worker/trigger_to_check.go +++ b/checker/worker/trigger_to_check.go @@ -2,35 +2,39 @@ package worker import ( "time" + + "github.com/patrickmn/go-cache" ) const sleepAfterGetTriggerIDError = time.Second * 1 const sleepWhenNoTriggerToCheck = time.Millisecond * 500 -func (worker *Checker) startTriggerToCheckGetter(fetch func(int) ([]string, error), batchSize int) <-chan string { +func (check *Checker) startTriggerToCheckGetter(fetch func(int) ([]string, error), batchSize int) <-chan string { triggerIDsToCheck := make(chan string, batchSize*2) //nolint - worker.tomb.Go(func() error { return worker.triggerToCheckGetter(fetch, batchSize, triggerIDsToCheck) }) + check.tomb.Go(func() error { + return check.triggerToCheckGetter(fetch, batchSize, triggerIDsToCheck) + }) return triggerIDsToCheck } -func (worker *Checker) triggerToCheckGetter(fetch func(int) ([]string, error), batchSize int, triggerIDsToCheck chan<- string) error { +func (check *Checker) triggerToCheckGetter(fetch func(int) ([]string, error), batchSize int, triggerIDsToCheck chan<- string) error { var fetchDelay time.Duration for { startFetch := time.After(fetchDelay) select { - case <-worker.tomb.Dying(): + case <-check.tomb.Dying(): close(triggerIDsToCheck) return nil case <-startFetch: triggerIDs, err := fetch(batchSize) - fetchDelay = worker.handleFetchResponse(triggerIDs, err, triggerIDsToCheck) + fetchDelay = check.handleFetchResponse(triggerIDs, err, triggerIDsToCheck) } } } -func (worker *Checker) handleFetchResponse(triggerIDs []string, fetchError error, triggerIDsToCheck chan<- string) time.Duration { +func (check *Checker) handleFetchResponse(triggerIDs []string, fetchError error, triggerIDsToCheck chan<- string) time.Duration { if fetchError != nil { - worker.Logger.Error(). + check.Logger.Error(). Error(fetchError). Msg("Failed to handle trigger loop") return sleepAfterGetTriggerIDError @@ -43,3 +47,26 @@ func (worker *Checker) handleFetchResponse(triggerIDs []string, fetchError error } return time.Duration(0) } + +func (check *Checker) getTriggerIDsToCheck(triggerIDs []string) []string { + triggerIDsToCheck := make([]string, 0, len(triggerIDs)) + + lazyTriggerIDs := check.lazyTriggerIDs.Load().(map[string]bool) + for _, triggerID := range triggerIDs { + if _, ok := lazyTriggerIDs[triggerID]; ok { + randomDuration := check.getRandomLazyCacheDuration() + cacheContainsIdErr := check.LazyTriggersCache.Add(triggerID, true, randomDuration) + + if cacheContainsIdErr != nil { + continue + } + } + + cacheContainsIdErr := check.TriggerCache.Add(triggerID, true, cache.DefaultExpiration) + if cacheContainsIdErr == nil { + triggerIDsToCheck = append(triggerIDsToCheck, triggerID) + } + } + + return triggerIDsToCheck +} diff --git a/checker/worker/worker.go b/checker/worker/worker.go index 55d36ae9a..24336cf73 100644 --- a/checker/worker/worker.go +++ b/checker/worker/worker.go @@ -9,6 +9,7 @@ import ( "github.com/moira-alert/moira/metrics" metricSource "github.com/moira-alert/moira/metric_source" + "github.com/moira-alert/moira/metric_source/prometheus" "github.com/moira-alert/moira/metric_source/remote" "github.com/patrickmn/go-cache" "gopkg.in/tomb.v2" @@ -23,6 +24,7 @@ type Checker struct { Database moira.Database Config *checker.Config RemoteConfig *remote.Config + PrometheusConfig *prometheus.Config SourceProvider *metricSource.SourceProvider Metrics *metrics.CheckerMetrics TriggerCache *cache.Cache @@ -31,144 +33,182 @@ type Checker struct { lazyTriggerIDs atomic.Value lastData int64 tomb tomb.Tomb - remoteEnabled bool } // Start start schedule new MetricEvents and check for NODATA triggers -func (worker *Checker) Start() error { - if worker.Config.MaxParallelChecks == 0 { - worker.Config.MaxParallelChecks = runtime.NumCPU() - worker.Logger.Info(). - Int("number_of_cpu", worker.Config.MaxParallelChecks). - Msg("MaxParallelChecks is not configured, set it to the number of CPU") +func (check *Checker) Start() error { + var err error + + err = check.startLocalMetricEvents() + if err != nil { + return err + } + + err = check.startLazyTriggers() + if err != nil { + return err + } + + err = check.startCheckerWorker(newRemoteChecker(check)) + if err != nil { + return err + } + + err = check.startCheckerWorker(newPrometheusChecker(check)) + if err != nil { + return err } - worker.lastData = time.Now().UTC().Unix() + err = check.startCheckerWorker(newLocalChecker(check)) + if err != nil { + return err + } + + return nil +} - if worker.Config.MetricEventPopBatchSize == 0 { - worker.Config.MetricEventPopBatchSize = 100 - } else if worker.Config.MetricEventPopBatchSize < 0 { - return errors.New("MetricEventPopBatchSize param less than zero") +func (check *Checker) startLocalMetricEvents() error { + if check.Config.MetricEventPopBatchSize < 0 { + return errors.New("MetricEventPopBatchSize param was less than zero") + } + + if check.Config.MetricEventPopBatchSize == 0 { + check.Config.MetricEventPopBatchSize = 100 } subscribeMetricEventsParams := moira.SubscribeMetricEventsParams{ - BatchSize: worker.Config.MetricEventPopBatchSize, - Delay: worker.Config.MetricEventPopDelay, + BatchSize: check.Config.MetricEventPopBatchSize, + Delay: check.Config.MetricEventPopDelay, } - metricEventsChannel, err := worker.Database.SubscribeMetricEvents(&worker.tomb, &subscribeMetricEventsParams) + metricEventsChannel, err := check.Database.SubscribeMetricEvents(&check.tomb, &subscribeMetricEventsParams) if err != nil { return err } - worker.lazyTriggerIDs.Store(make(map[string]bool)) - worker.tomb.Go(worker.lazyTriggersWorker) + for i := 0; i < check.Config.MaxParallelLocalChecks; i++ { + check.tomb.Go(func() error { + return check.newMetricsHandler(metricEventsChannel) + }) + } - worker.tomb.Go(worker.localTriggerGetter) + check.tomb.Go(func() error { + return check.checkMetricEventsChannelLen(metricEventsChannel) + }) - _, err = worker.SourceProvider.GetRemote() - worker.remoteEnabled = err == nil + check.Logger.Info().Msg("Checking new events started") - if worker.remoteEnabled && worker.Config.MaxParallelRemoteChecks == 0 { - worker.Config.MaxParallelRemoteChecks = runtime.NumCPU() + go func() { + <-check.tomb.Dying() + check.Logger.Info().Msg("Checking for new events stopped") + }() + + return nil +} + +type checkerWorker interface { + // Returns the name of the worker for logging + Name() string + // Returns true if worker is enabled, false otherwise + IsEnabled() bool + // Returns the max number of parallel checks for this worker + MaxParallelChecks() int + // Returns the metrics for this worker + Metrics() *metrics.CheckMetrics + // Starts separate goroutine that fetches triggers for this worker from database and adds them to the check queue + StartTriggerGetter() error + // Fetches triggers from the queue + GetTriggersToCheck(count int) ([]string, error) +} - worker.Logger.Info(). - Int("number_of_cpu", worker.Config.MaxParallelRemoteChecks). - Msg("MaxParallelRemoteChecks is not configured, set it to the number of CPU") +func (check *Checker) startCheckerWorker(w checkerWorker) error { + if !w.IsEnabled() { + check.Logger.Info().Msg(w.Name() + " checker disabled") + return nil } - if worker.remoteEnabled { - worker.tomb.Go(worker.remoteTriggerGetter) - worker.Logger.Info().Msg("Remote checker started") - } else { - worker.Logger.Info().Msg("Remote checker disabled") + maxParallelChecks := w.MaxParallelChecks() + if maxParallelChecks == 0 { + maxParallelChecks = runtime.NumCPU() + + check.Logger.Info(). + Int("number_of_cpu", maxParallelChecks). + Msg("MaxParallel" + w.Name() + "Checks is not configured, set it to the number of CPU") } const maxParallelChecksMaxValue = 1024 * 8 - if worker.Config.MaxParallelChecks > maxParallelChecksMaxValue { - return errors.New("MaxParallelChecks value is too large") + if maxParallelChecks > maxParallelChecksMaxValue { + return errors.New("MaxParallel" + w.Name() + "Checks value is too large") } - worker.Logger.Info(). - Int("number_of_checkers", worker.Config.MaxParallelChecks). - Msg("Start parallel local checkers") + check.tomb.Go(w.StartTriggerGetter) + check.Logger.Info().Msg(w.Name() + "checker started") - localTriggerIdsToCheckChan := worker.startTriggerToCheckGetter(worker.Database.GetLocalTriggersToCheck, worker.Config.MaxParallelChecks) - for i := 0; i < worker.Config.MaxParallelChecks; i++ { - worker.tomb.Go(func() error { - return worker.newMetricsHandler(metricEventsChannel) - }) - worker.tomb.Go(func() error { - return worker.startTriggerHandler(localTriggerIdsToCheckChan, worker.Metrics.LocalMetrics) + triggerIdsToCheckChan := check.startTriggerToCheckGetter( + w.GetTriggersToCheck, + maxParallelChecks, + ) + + for i := 0; i < maxParallelChecks; i++ { + check.tomb.Go(func() error { + return check.startTriggerHandler( + triggerIdsToCheckChan, + w.Metrics(), + ) }) } - if worker.remoteEnabled { - const maxParallelRemoteChecksMaxValue = 1024 * 8 - if worker.Config.MaxParallelRemoteChecks > maxParallelRemoteChecksMaxValue { - return errors.New("MaxParallelRemoteChecks value is too large") - } + return nil +} - worker.Logger.Info(). - Int("number_of_checkers", worker.Config.MaxParallelChecks). - Msg("Start parallel remote checkers") +func (check *Checker) startLazyTriggers() error { + check.lastData = time.Now().UTC().Unix() - remoteTriggerIdsToCheckChan := worker.startTriggerToCheckGetter(worker.Database.GetRemoteTriggersToCheck, worker.Config.MaxParallelRemoteChecks) - for i := 0; i < worker.Config.MaxParallelRemoteChecks; i++ { - worker.tomb.Go(func() error { - return worker.startTriggerHandler(remoteTriggerIdsToCheckChan, worker.Metrics.RemoteMetrics) - }) - } - } - worker.Logger.Info().Msg("Checking new events started") + check.lazyTriggerIDs.Store(make(map[string]bool)) + check.tomb.Go(check.lazyTriggersWorker) - go func() { - <-worker.tomb.Dying() - worker.Logger.Info().Msg("Checking for new events stopped") - }() + check.tomb.Go(check.checkTriggersToCheckCount) - worker.tomb.Go(func() error { return worker.checkMetricEventsChannelLen(metricEventsChannel) }) - worker.tomb.Go(worker.checkTriggersToCheckCount) return nil } -func (worker *Checker) checkTriggersToCheckCount() error { +func (check *Checker) checkTriggersToCheckCount() error { checkTicker := time.NewTicker(time.Millisecond * 100) //nolint var triggersToCheckCount, remoteTriggersToCheckCount int64 var err error for { select { - case <-worker.tomb.Dying(): + case <-check.tomb.Dying(): return nil case <-checkTicker.C: - triggersToCheckCount, err = worker.Database.GetLocalTriggersToCheckCount() + triggersToCheckCount, err = check.Database.GetLocalTriggersToCheckCount() if err == nil { - worker.Metrics.LocalMetrics.TriggersToCheckCount.Update(triggersToCheckCount) + check.Metrics.LocalMetrics.TriggersToCheckCount.Update(triggersToCheckCount) } - if worker.remoteEnabled { - remoteTriggersToCheckCount, err = worker.Database.GetRemoteTriggersToCheckCount() + if check.RemoteConfig.Enabled { + remoteTriggersToCheckCount, err = check.Database.GetRemoteTriggersToCheckCount() if err == nil { - worker.Metrics.RemoteMetrics.TriggersToCheckCount.Update(remoteTriggersToCheckCount) + check.Metrics.RemoteMetrics.TriggersToCheckCount.Update(remoteTriggersToCheckCount) } } } } } -func (worker *Checker) checkMetricEventsChannelLen(ch <-chan *moira.MetricEvent) error { +func (check *Checker) checkMetricEventsChannelLen(ch <-chan *moira.MetricEvent) error { checkTicker := time.NewTicker(time.Millisecond * 100) //nolint for { select { - case <-worker.tomb.Dying(): + case <-check.tomb.Dying(): return nil case <-checkTicker.C: - worker.Metrics.MetricEventsChannelLen.Update(int64(len(ch))) + check.Metrics.MetricEventsChannelLen.Update(int64(len(ch))) } } } // Stop stops checks triggers -func (worker *Checker) Stop() error { - worker.tomb.Kill(nil) - return worker.tomb.Wait() +func (check *Checker) Stop() error { + check.tomb.Kill(nil) + return check.tomb.Wait() } diff --git a/cmd/api/config.go b/cmd/api/config.go index 5cd083586..032d22a73 100644 --- a/cmd/api/config.go +++ b/cmd/api/config.go @@ -19,6 +19,7 @@ type config struct { Web webConfig `yaml:"web"` Telemetry cmd.TelemetryConfig `yaml:"telemetry"` Remote cmd.RemoteConfig `yaml:"remote"` + Prometheus cmd.PrometheusConfig `yaml:"prometheus"` NotificationHistory cmd.NotificationHistoryConfig `yaml:"notification_history"` } @@ -62,10 +63,10 @@ type featureFlags struct { func (config *apiConfig) getSettings(localMetricTTL, remoteMetricTTL string) *api.Config { return &api.Config{ - EnableCORS: config.EnableCORS, - Listen: config.Listen, - LocalMetricTTL: to.Duration(localMetricTTL), - RemoteMetricTTL: to.Duration(remoteMetricTTL), + EnableCORS: config.EnableCORS, + Listen: config.Listen, + GraphiteLocalMetricTTL: to.Duration(localMetricTTL), + GraphiteRemoteMetricTTL: to.Duration(remoteMetricTTL), } } diff --git a/cmd/api/config_test.go b/cmd/api/config_test.go index 8336370a4..23969361f 100644 --- a/cmd/api/config_test.go +++ b/cmd/api/config_test.go @@ -19,10 +19,10 @@ func Test_apiConfig_getSettings(t *testing.T) { } expectedResult := &api.Config{ - EnableCORS: true, - Listen: "0000", - LocalMetricTTL: time.Hour, - RemoteMetricTTL: 24 * time.Hour, + EnableCORS: true, + Listen: "0000", + GraphiteLocalMetricTTL: time.Hour, + GraphiteRemoteMetricTTL: 24 * time.Hour, } result := apiConf.getSettings("1h", "24h") diff --git a/cmd/api/main.go b/cmd/api/main.go index 71da5a098..db7e47b53 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -19,6 +19,7 @@ import ( logging "github.com/moira-alert/moira/logging/zerolog_adapter" metricSource "github.com/moira-alert/moira/metric_source" "github.com/moira-alert/moira/metric_source/local" + "github.com/moira-alert/moira/metric_source/prometheus" "github.com/moira-alert/moira/metric_source/remote" _ "go.uber.org/automaxprocs" ) @@ -114,10 +115,23 @@ func main() { String("listen_address", apiConfig.Listen). Msg("Start listening") - localSource := local.Create(database) remoteConfig := applicationConfig.Remote.GetRemoteSourceSettings() + prometheusConfig := applicationConfig.Prometheus.GetPrometheusSourceSettings() + + localSource := local.Create(database) remoteSource := remote.Create(remoteConfig) - metricSourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + prometheusSource, err := prometheus.Create(prometheusConfig, logger) + if err != nil { + logger.Fatal(). + Error(err). + Msg("Failed to initialize prometheus metric source") + } + + metricSourceProvider := metricSource.CreateMetricSourceProvider( + localSource, + remoteSource, + prometheusSource, + ) webConfigContent, err := applicationConfig.Web.getSettings(remoteConfig.Enabled) if err != nil { diff --git a/cmd/checker/config.go b/cmd/checker/config.go index ca2c13b2f..4782b3cc2 100644 --- a/cmd/checker/config.go +++ b/cmd/checker/config.go @@ -8,11 +8,12 @@ import ( ) type config struct { - Redis cmd.RedisConfig `yaml:"redis"` - Logger cmd.LoggerConfig `yaml:"log"` - Checker checkerConfig `yaml:"checker"` - Telemetry cmd.TelemetryConfig `yaml:"telemetry"` - Remote cmd.RemoteConfig `yaml:"remote"` + Redis cmd.RedisConfig `yaml:"redis"` + Logger cmd.LoggerConfig `yaml:"log"` + Checker checkerConfig `yaml:"checker"` + Telemetry cmd.TelemetryConfig `yaml:"telemetry"` + Remote cmd.RemoteConfig `yaml:"remote"` + Prometheus cmd.PrometheusConfig `yaml:"prometheus"` } type triggerLogConfig struct { @@ -60,7 +61,7 @@ func (config *checkerConfig) getSettings(logger moira.Logger) *checker.Config { LazyTriggersCheckInterval: to.Duration(config.LazyTriggersCheckInterval), NoDataCheckInterval: to.Duration(config.NoDataCheckInterval), StopCheckingIntervalSeconds: int64(to.Duration(config.StopCheckingInterval).Seconds()), - MaxParallelChecks: config.MaxParallelChecks, + MaxParallelLocalChecks: config.MaxParallelChecks, MaxParallelRemoteChecks: config.MaxParallelRemoteChecks, LogTriggersToLevel: logTriggersToLevel, MetricEventPopBatchSize: int64(config.MetricEventPopBatchSize), @@ -104,5 +105,10 @@ func getDefault() config { Timeout: "60s", MetricsTTL: "7d", }, + Prometheus: cmd.PrometheusConfig{ + CheckInterval: "60s", + Timeout: "60s", + MetricsTTL: "7d", + }, } } diff --git a/cmd/checker/main.go b/cmd/checker/main.go index 64ea4723c..5de14c9ea 100644 --- a/cmd/checker/main.go +++ b/cmd/checker/main.go @@ -8,14 +8,15 @@ import ( "syscall" "time" + "github.com/moira-alert/moira/checker/worker" metricSource "github.com/moira-alert/moira/metric_source" "github.com/moira-alert/moira/metric_source/local" + "github.com/moira-alert/moira/metric_source/prometheus" "github.com/moira-alert/moira/metric_source/remote" "github.com/patrickmn/go-cache" "github.com/moira-alert/moira" "github.com/moira-alert/moira/checker" - "github.com/moira-alert/moira/checker/worker" "github.com/moira-alert/moira/cmd" "github.com/moira-alert/moira/database/redis" logging "github.com/moira-alert/moira/logging/zerolog_adapter" @@ -79,17 +80,36 @@ func main() { } defer telemetry.Stop() + logger.Info().Msg("Debug: checker started") + databaseSettings := config.Redis.GetSettings() database := redis.NewDatabase(logger, databaseSettings, redis.NotificationHistoryConfig{}, redis.Checker) remoteConfig := config.Remote.GetRemoteSourceSettings() + prometheusConfig := config.Prometheus.GetPrometheusSourceSettings() + localSource := local.Create(database) remoteSource := remote.Create(remoteConfig) - metricSourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + prometheusSource, err := prometheus.Create(prometheusConfig, logger) + if err != nil { + logger.Fatal(). + Error(err). + Msg("Failed to initialize prometheus metric source") + } + + // TODO: Abstractions over sources, so that they all are handled the same way + metricSourceProvider := metricSource.CreateMetricSourceProvider( + localSource, + remoteSource, + prometheusSource, + ) - isConfigured, _ := remoteSource.IsConfigured() - checkerMetrics := metrics.ConfigureCheckerMetrics(telemetry.Metrics, isConfigured) + remoteConfigured, _ := remoteSource.IsConfigured() + prometheusConfigured, _ := prometheusSource.IsConfigured() + + checkerMetrics := metrics.ConfigureCheckerMetrics(telemetry.Metrics, remoteConfigured, prometheusConfigured) checkerSettings := config.Checker.getSettings(logger) + if triggerID != nil && *triggerID != "" { checkSingleTrigger(database, checkerMetrics, checkerSettings, metricSourceProvider) } @@ -99,6 +119,7 @@ func main() { Database: database, Config: checkerSettings, RemoteConfig: remoteConfig, + PrometheusConfig: prometheusConfig, SourceProvider: metricSourceProvider, Metrics: checkerMetrics, TriggerCache: cache.New(checkerSettings.CheckInterval, time.Minute*60), //nolint diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 657dc1ce2..01ead7ed6 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -293,8 +293,13 @@ func main() { //nolint Error(err). Msg("Failed to handle push trigger metrics") } - if err := support.HandlePushTriggerLastCheck(logger, dataBase, dump.Trigger.ID, &dump.LastCheck, - dump.Trigger.IsRemote); err != nil { + if err := support.HandlePushTriggerLastCheck( + logger, + dataBase, + dump.Trigger.ID, + &dump.LastCheck, + dump.Trigger.TriggerSource, + ); err != nil { logger.Fatal(). Error(err). Msg("Failed to handle push trigger last check") diff --git a/cmd/config.go b/cmd/config.go index 94c61d3b4..54504735c 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -8,6 +8,7 @@ import ( "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira/image_store/s3" + "github.com/moira-alert/moira/metric_source/prometheus" remoteSource "github.com/moira-alert/moira/metric_source/remote" "github.com/xiam/to" "gopkg.in/yaml.v2" @@ -142,11 +143,6 @@ type RemoteConfig struct { Enabled bool `yaml:"enabled"` } -// ImageStoreConfig defines the configuration for all the image stores to be initialized by InitImageStores -type ImageStoreConfig struct { - S3 s3.Config `yaml:"s3"` -} - // GetRemoteSourceSettings returns remote config parsed from moira config files func (config *RemoteConfig) GetRemoteSourceSettings() *remoteSource.Config { return &remoteSource.Config{ @@ -160,6 +156,34 @@ func (config *RemoteConfig) GetRemoteSourceSettings() *remoteSource.Config { } } +type PrometheusConfig struct { + URL string `yaml:"url"` + CheckInterval string `yaml:"check_interval"` + MetricsTTL string `yaml:"metrics_ttl"` + Timeout string `yaml:"timeout"` + User string `yaml:"user"` + Password string `yaml:"password"` + Enabled bool `yaml:"enabled"` +} + +// GetRemoteSourceSettings returns remote config parsed from moira config files +func (config *PrometheusConfig) GetPrometheusSourceSettings() *prometheus.Config { + return &prometheus.Config{ + Enabled: config.Enabled, + URL: config.URL, + CheckInterval: to.Duration(config.CheckInterval), + MetricsTTL: to.Duration(config.MetricsTTL), + User: config.User, + Password: config.Password, + Timeout: to.Duration(config.Timeout), + } +} + +// ImageStoreConfig defines the configuration for all the image stores to be initialized by InitImageStores +type ImageStoreConfig struct { + S3 s3.Config `yaml:"s3"` +} + // ReadConfig parses config file by the given path into Moira-used type func ReadConfig(configFileName string, config interface{}) error { configYaml, err := os.ReadFile(configFileName) diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index 5c725944c..1ae79ae29 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -18,6 +18,7 @@ type config struct { Notifier notifierConfig `yaml:"notifier"` Telemetry cmd.TelemetryConfig `yaml:"telemetry"` Remote cmd.RemoteConfig `yaml:"remote"` + Prometheus cmd.PrometheusConfig `yaml:"prometheus"` ImageStores cmd.ImageStoreConfig `yaml:"image_store"` NotificationHistory cmd.NotificationHistoryConfig `yaml:"notification_history"` } diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index b0bd8415d..4ecf4155d 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -9,6 +9,7 @@ import ( metricSource "github.com/moira-alert/moira/metric_source" "github.com/moira-alert/moira/metric_source/local" + "github.com/moira-alert/moira/metric_source/prometheus" "github.com/moira-alert/moira/metric_source/remote" "github.com/moira-alert/moira" @@ -83,10 +84,19 @@ func main() { notificationHistorySettings := config.NotificationHistory.GetSettings() database := redis.NewDatabase(logger, databaseSettings, notificationHistorySettings, redis.Notifier) - localSource := local.Create(database) remoteConfig := config.Remote.GetRemoteSourceSettings() + prometheusConfig := config.Prometheus.GetPrometheusSourceSettings() + + localSource := local.Create(database) remoteSource := remote.Create(remoteConfig) - metricSourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource) + prometheusSource, err := prometheus.Create(prometheusConfig, logger) + if err != nil { + logger.Fatal(). + Error(err). + Msg("Failed to initialize prometheus metric source") + } + + metricSourceProvider := metricSource.CreateMetricSourceProvider(localSource, remoteSource, prometheusSource) // Initialize the image store imageStoreMap := cmd.InitImageStores(config.ImageStores, logger) diff --git a/database/redis/last_check.go b/database/redis/last_check.go index 031d68e07..21ac48aeb 100644 --- a/database/redis/last_check.go +++ b/database/redis/last_check.go @@ -28,8 +28,8 @@ func (connector *DbConnector) GetTriggerLastCheck(triggerID string) (moira.Check } // SetTriggerLastCheck sets trigger last check data -func (connector *DbConnector) SetTriggerLastCheck(triggerID string, checkData *moira.CheckData, isRemote bool) error { - selfStateCheckCountKey := connector.getSelfStateCheckCountKey(isRemote) +func (connector *DbConnector) SetTriggerLastCheck(triggerID string, checkData *moira.CheckData, triggerSource moira.TriggerSource) error { + selfStateCheckCountKey := connector.getSelfStateCheckCountKey(triggerSource) bytes, err := reply.GetCheckBytes(*checkData) if err != nil { return err @@ -65,14 +65,23 @@ func (connector *DbConnector) SetTriggerLastCheck(triggerID string, checkData *m return nil } -func (connector *DbConnector) getSelfStateCheckCountKey(isRemote bool) string { +func (connector *DbConnector) getSelfStateCheckCountKey(triggerSource moira.TriggerSource) string { if connector.source != Checker { return "" } - if isRemote { + switch triggerSource { + case moira.GraphiteLocal: + return selfStateChecksCounterKey + + case moira.GraphiteRemote: return selfStateRemoteChecksCounterKey + + case moira.PrometheusRemote: + return selfStatePrometheusChecksCounterKey + + default: + return "" } - return selfStateChecksCounterKey } func appendRemoveTriggerLastCheckToRedisPipeline(ctx context.Context, pipe redis.Pipeliner, triggerID string) redis.Pipeliner { diff --git a/database/redis/last_check_test.go b/database/redis/last_check_test.go index 83ad0407b..5d7cebceb 100644 --- a/database/redis/last_check_test.go +++ b/database/redis/last_check_test.go @@ -22,7 +22,7 @@ func TestLastCheck(t *testing.T) { Convey("LastCheck manipulation", t, func() { Convey("Test read write delete", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) actual, err := dataBase.GetTriggerLastCheck(triggerID) @@ -54,7 +54,7 @@ func TestLastCheck(t *testing.T) { Convey("While no metrics", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, moira.GraphiteLocal) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric1": 1, "metric5": 5}, nil, "", 0) @@ -67,7 +67,7 @@ func TestLastCheck(t *testing.T) { Convey("While no metrics to change", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric11": 1, "metric55": 5}, nil, "", 0) @@ -81,7 +81,7 @@ func TestLastCheck(t *testing.T) { Convey("Has metrics to change", func() { checkData := lastCheckTest triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &checkData, false) + err := dataBase.SetTriggerLastCheck(triggerID, &checkData, moira.GraphiteLocal) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric1": 1, "metric5": 5}, nil, "", 0) @@ -108,7 +108,7 @@ func TestLastCheck(t *testing.T) { Convey("Set metrics maintenance while no metrics", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, moira.GraphiteLocal) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric1": 1, "metric5": 5}, nil, "", 0) @@ -121,7 +121,7 @@ func TestLastCheck(t *testing.T) { Convey("Set trigger maintenance while no metrics", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 1000 @@ -136,7 +136,7 @@ func TestLastCheck(t *testing.T) { Convey("Set metrics maintenance while no metrics to change", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric11": 1, "metric55": 5}, nil, "", 0) @@ -151,7 +151,7 @@ func TestLastCheck(t *testing.T) { newLastCheckTest := lastCheckTest newLastCheckTest.Maintenance = 1000 triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 1000 @@ -166,7 +166,7 @@ func TestLastCheck(t *testing.T) { Convey("Set metrics maintenance while has metrics to change", func() { checkData := lastCheckTest triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &checkData, false) + err := dataBase.SetTriggerLastCheck(triggerID, &checkData, moira.GraphiteLocal) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric1": 1, "metric5": 5}, nil, "", 0) @@ -186,7 +186,7 @@ func TestLastCheck(t *testing.T) { Convey("Set trigger and metrics maintenance while has metrics to change", func() { checkData := lastCheckTest triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &checkData, false) + err := dataBase.SetTriggerLastCheck(triggerID, &checkData, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 1000 @@ -208,7 +208,7 @@ func TestLastCheck(t *testing.T) { Convey("Set trigger maintenance to 0 and metrics maintenance", func() { checkData := lastCheckTest triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &checkData, false) + err := dataBase.SetTriggerLastCheck(triggerID, &checkData, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 0 @@ -230,7 +230,7 @@ func TestLastCheck(t *testing.T) { So(dataBase.checkDataScoreChanged(triggerID, &lastCheckWithNoMetrics), ShouldBeTrue) // set new last check. Should add a trigger to a reindex set - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, moira.GraphiteLocal) So(err, ShouldBeNil) So(dataBase.checkDataScoreChanged(triggerID, &lastCheckWithNoMetrics), ShouldBeFalse) @@ -243,7 +243,7 @@ func TestLastCheck(t *testing.T) { time.Sleep(time.Second) - err = dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err = dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) actual, err = dataBase.FetchTriggersToReindex(time.Now().Unix() - 10) @@ -282,7 +282,7 @@ func TestLastCheck(t *testing.T) { Value: &value, }, }, - }, false) + }, moira.GraphiteLocal) So(err, ShouldBeNil) actual, err := dataBase.GetTriggerLastCheck(triggerID) @@ -326,13 +326,13 @@ func TestCleanUpAbandonedTriggerLastCheck(t *testing.T) { } _ = dataBase.SaveTrigger(trigger.ID, &trigger) - _ = dataBase.SetTriggerLastCheck(trigger.ID, &lastCheckTest, false) + _ = dataBase.SetTriggerLastCheck(trigger.ID, &lastCheckTest, moira.GraphiteLocal) _, err := dataBase.GetTriggerLastCheck(trigger.ID) So(err, ShouldBeNil) Convey("Given abandoned last check (without saved trigger)", func() { removedTriggerID := uuid.Must(uuid.NewV4()).String() - err = dataBase.SetTriggerLastCheck(removedTriggerID, &lastCheckTest, false) + err = dataBase.SetTriggerLastCheck(removedTriggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) _, err = dataBase.GetTriggerLastCheck(removedTriggerID) @@ -364,7 +364,7 @@ func TestRemoteLastCheck(t *testing.T) { Convey("LastCheck manipulation", t, func() { Convey("Test read write delete", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, true) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteRemote) So(err, ShouldBeNil) actual, err := dataBase.GetTriggerLastCheck(triggerID) @@ -396,7 +396,7 @@ func TestRemoteLastCheck(t *testing.T) { Convey("While no metrics", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, true) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckWithNoMetrics, moira.GraphiteRemote) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric1": 1, "metric5": 5}, nil, "", 0) @@ -409,7 +409,7 @@ func TestRemoteLastCheck(t *testing.T) { Convey("While no metrics to change", func() { triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, true) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteRemote) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric11": 1, "metric55": 5}, nil, "", 0) @@ -423,7 +423,7 @@ func TestRemoteLastCheck(t *testing.T) { Convey("Has metrics to change", func() { checkData := lastCheckTest triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &checkData, true) + err := dataBase.SetTriggerLastCheck(triggerID, &checkData, moira.GraphiteRemote) So(err, ShouldBeNil) err = dataBase.SetTriggerCheckMaintenance(triggerID, map[string]int64{"metric1": 1, "metric5": 5}, nil, "", 0) @@ -453,7 +453,7 @@ func TestLastCheckErrorConnection(t *testing.T) { So(actual1, ShouldResemble, moira.CheckData{}) So(err, ShouldNotBeNil) - err = dataBase.SetTriggerLastCheck("123", &lastCheckTest, false) + err = dataBase.SetTriggerLastCheck("123", &lastCheckTest, moira.GraphiteLocal) So(err, ShouldNotBeNil) err = dataBase.RemoveTriggerLastCheck("123") @@ -486,7 +486,7 @@ func TestMaintenanceUserSave(t *testing.T) { newLastCheckTest.MaintenanceInfo.StartUser = &userLogin newLastCheckTest.MaintenanceInfo.StartTime = &startTime triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 1000 @@ -504,7 +504,7 @@ func TestMaintenanceUserSave(t *testing.T) { newLastCheckTest.MaintenanceInfo.StopUser = &userLogin newLastCheckTest.MaintenanceInfo.StopTime = &startTime triggerID := uuid.Must(uuid.NewV4()).String() - err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck(triggerID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 1000 @@ -523,7 +523,7 @@ func TestMaintenanceUserSave(t *testing.T) { checkData.MaintenanceInfo = moira.MaintenanceInfo{} userLogin := "test" var timeCallMaintenance = int64(3) - err := dataBase.SetTriggerLastCheck(triggerID, &checkData, false) + err := dataBase.SetTriggerLastCheck(triggerID, &checkData, moira.GraphiteLocal) So(err, ShouldBeNil) triggerMaintenanceTS = 1000 diff --git a/database/redis/reply/trigger.go b/database/redis/reply/trigger.go index 5132af23d..268ce3537 100644 --- a/database/redis/reply/trigger.go +++ b/database/redis/reply/trigger.go @@ -27,6 +27,7 @@ type triggerStorageElement struct { Patterns []string `json:"patterns"` TTL string `json:"ttl,omitempty"` IsRemote bool `json:"is_remote"` + TriggerSource moira.TriggerSource `json:"trigger_source,omitempty"` MuteNewMetrics bool `json:"mute_new_metrics,omitempty"` AloneMetrics map[string]bool `json:"alone_metrics"` CreatedAt *int64 `json:"created_at"` @@ -46,6 +47,8 @@ func (storageElement *triggerStorageElement) toTrigger() moira.Trigger { } } //TODO(litleleprikon): END remove in moira v2.8.0. Compatibility with moira < v2.6.0 + + triggerSource := storageElement.TriggerSource.FillInIfNotSet(storageElement.IsRemote) return moira.Trigger{ ID: storageElement.ID, Name: storageElement.Name, @@ -61,7 +64,7 @@ func (storageElement *triggerStorageElement) toTrigger() moira.Trigger { PythonExpression: storageElement.PythonExpression, Patterns: storageElement.Patterns, TTL: getTriggerTTL(storageElement.TTL), - IsRemote: storageElement.IsRemote, + TriggerSource: triggerSource, MuteNewMetrics: storageElement.MuteNewMetrics, AloneMetrics: storageElement.AloneMetrics, CreatedAt: storageElement.CreatedAt, @@ -87,7 +90,8 @@ func toTriggerStorageElement(trigger *moira.Trigger, triggerID string) *triggerS PythonExpression: trigger.PythonExpression, Patterns: trigger.Patterns, TTL: getTriggerTTLString(trigger.TTL), - IsRemote: trigger.IsRemote, + IsRemote: trigger.TriggerSource == moira.GraphiteRemote, + TriggerSource: trigger.TriggerSource, MuteNewMetrics: trigger.MuteNewMetrics, AloneMetrics: trigger.AloneMetrics, CreatedAt: trigger.CreatedAt, diff --git a/database/redis/selfstate.go b/database/redis/selfstate.go index 2fe2a1c75..1b6ebf1e6 100644 --- a/database/redis/selfstate.go +++ b/database/redis/selfstate.go @@ -42,6 +42,16 @@ func (connector *DbConnector) GetRemoteChecksUpdatesCount() (int64, error) { return ts, err } +// GetRemoteChecksUpdatesCount return remote checks count by Moira-Checker +func (connector *DbConnector) GetPrometheusChecksUpdatesCount() (int64, error) { + c := *connector.client + ts, err := c.Get(connector.context, selfStatePrometheusChecksCounterKey).Int64() + if err == redis.Nil { + return 0, nil + } + return ts, err +} + // GetNotifierState return current notifier state: func (connector *DbConnector) GetNotifierState() (string, error) { c := *connector.client @@ -64,4 +74,5 @@ func (connector *DbConnector) SetNotifierState(health string) error { var selfStateMetricsHeartbeatKey = "moira-selfstate:metrics-heartbeat" var selfStateChecksCounterKey = "moira-selfstate:checks-counter" var selfStateRemoteChecksCounterKey = "moira-selfstate:remote-checks-counter" +var selfStatePrometheusChecksCounterKey = "moira-selfstate:prometheus-checks-counter" var selfStateNotifierHealth = "moira-selfstate:notifier-health" diff --git a/database/redis/selfstate_test.go b/database/redis/selfstate_test.go index a423f644b..191ab2a62 100644 --- a/database/redis/selfstate_test.go +++ b/database/redis/selfstate_test.go @@ -40,14 +40,14 @@ func TestSelfCheckWithWritesInChecker(t *testing.T) { }) Convey("Update metrics checks updates count", func() { - err := dataBase.SetTriggerLastCheck("123", &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck("123", &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) count, err := dataBase.GetChecksUpdatesCount() So(count, ShouldEqual, 1) So(err, ShouldBeNil) - err = dataBase.SetTriggerLastCheck("12345", &lastCheckTest, true) + err = dataBase.SetTriggerLastCheck("12345", &lastCheckTest, moira.GraphiteRemote) So(err, ShouldBeNil) count, err = dataBase.GetRemoteChecksUpdatesCount() @@ -72,14 +72,14 @@ func testSelfCheckWithWritesInDBSource(t *testing.T, dbSource DBSource) { defer dataBase.Flush() Convey(fmt.Sprintf("Self state triggers manipulation in %s", dbSource), t, func() { Convey("Update metrics checks updates count", func() { - err := dataBase.SetTriggerLastCheck("123", &lastCheckTest, false) + err := dataBase.SetTriggerLastCheck("123", &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) count, err := dataBase.GetChecksUpdatesCount() So(count, ShouldEqual, 0) So(err, ShouldBeNil) - err = dataBase.SetTriggerLastCheck("12345", &lastCheckTest, true) + err = dataBase.SetTriggerLastCheck("12345", &lastCheckTest, moira.GraphiteRemote) So(err, ShouldBeNil) count, err = dataBase.GetRemoteChecksUpdatesCount() diff --git a/database/redis/trigger.go b/database/redis/trigger.go index f86629ac2..34334b205 100644 --- a/database/redis/trigger.go +++ b/database/redis/trigger.go @@ -26,7 +26,7 @@ func (connector *DbConnector) GetAllTriggerIDs() ([]string, error) { // GetLocalTriggerIDs gets moira local triggerIDs func (connector *DbConnector) GetLocalTriggerIDs() ([]string, error) { c := *connector.client - triggerIds, err := c.SDiff(connector.context, triggersListKey, remoteTriggersListKey).Result() + triggerIds, err := c.SDiff(connector.context, triggersListKey, remoteTriggersListKey, prometheusTriggersListKey).Result() if err != nil { return nil, fmt.Errorf("failed to get triggers-list: %s", err.Error()) } @@ -43,6 +43,15 @@ func (connector *DbConnector) GetRemoteTriggerIDs() ([]string, error) { return triggerIds, nil } +func (connector *DbConnector) GetPrometheusTriggerIDs() ([]string, error) { + c := *connector.client + triggerIds, err := c.SMembers(connector.context, prometheusTriggersListKey).Result() + if err != nil { + return nil, fmt.Errorf("failed to get prometheus triggers-list: %s", err.Error()) + } + return triggerIds, nil +} + // GetTrigger gets trigger and trigger tags by given ID and return it in merged object func (connector *DbConnector) GetTrigger(triggerID string) (moira.Trigger, error) { pipe := (*connector.client).TxPipeline() @@ -177,25 +186,39 @@ func (connector *DbConnector) updateTrigger(triggerID string, newTrigger *moira. for _, pattern := range moira.GetStringListsDiff(oldTrigger.Patterns, newTrigger.Patterns) { pipe.SRem(connector.context, patternTriggersKey(pattern), triggerID) } - if oldTrigger.IsRemote && !newTrigger.IsRemote { - pipe.SRem(connector.context, remoteTriggersListKey, triggerID) - } for _, tag := range moira.GetStringListsDiff(oldTrigger.Tags, newTrigger.Tags) { pipe.SRem(connector.context, triggerTagsKey(triggerID), tag) pipe.SRem(connector.context, tagTriggersKey(tag), triggerID) } + + if newTrigger.TriggerSource != oldTrigger.TriggerSource { + switch oldTrigger.TriggerSource { + case moira.GraphiteRemote: + pipe.SRem(connector.context, remoteTriggersListKey, triggerID) + + case moira.PrometheusRemote: + pipe.SRem(connector.context, prometheusTriggersListKey, triggerID) + } + } } pipe.Set(connector.context, triggerKey(triggerID), bytes, redis.KeepTTL) pipe.SAdd(connector.context, triggersListKey, triggerID) - if newTrigger.IsRemote { + + switch newTrigger.TriggerSource { + case moira.GraphiteRemote: pipe.SAdd(connector.context, remoteTriggersListKey, triggerID) - } else { + + case moira.PrometheusRemote: + pipe.SAdd(connector.context, prometheusTriggersListKey, triggerID) + + case moira.GraphiteLocal: for _, pattern := range newTrigger.Patterns { pipe.SAdd(connector.context, patternsListKey, pattern) pipe.SAdd(connector.context, patternTriggersKey(pattern), triggerID) } } + for _, tag := range newTrigger.Tags { pipe.SAdd(connector.context, triggerTagsKey(triggerID), tag) pipe.SAdd(connector.context, tagTriggersKey(tag), triggerID) @@ -212,7 +235,7 @@ func (connector *DbConnector) updateTrigger(triggerID string, newTrigger *moira. } func (connector *DbConnector) preSaveTrigger(newTrigger *moira.Trigger, oldTrigger *moira.Trigger) { - if newTrigger.IsRemote { + if newTrigger.TriggerSource != moira.GraphiteLocal { newTrigger.Patterns = make([]string, 0) } @@ -252,7 +275,15 @@ func (connector *DbConnector) removeTrigger(triggerID string, trigger *moira.Tri pipe.Del(connector.context, triggerTagsKey(triggerID)) pipe.Del(connector.context, triggerEventsKey(triggerID)) pipe.SRem(connector.context, triggersListKey, triggerID) - pipe.SRem(connector.context, remoteTriggersListKey, triggerID) + + switch trigger.TriggerSource { + case moira.GraphiteRemote: + pipe.SRem(connector.context, remoteTriggersListKey, triggerID) + + case moira.PrometheusRemote: + pipe.SRem(connector.context, prometheusTriggersListKey, triggerID) + } + pipe.SRem(connector.context, unusedTriggersKey, triggerID) for _, tag := range trigger.Tags { pipe.SRem(connector.context, tagTriggersKey(tag), triggerID) @@ -381,6 +412,7 @@ func (connector *DbConnector) triggerHasSubscriptions(trigger *moira.Trigger) (b var triggersListKey = "{moira-triggers-list}:moira-triggers-list" var remoteTriggersListKey = "{moira-triggers-list}:moira-remote-triggers-list" +var prometheusTriggersListKey = "{moira-triggers-list}:moira-prometheus-triggers-list" func triggerKey(triggerID string) string { return "moira-trigger:" + triggerID diff --git a/database/redis/trigger_test.go b/database/redis/trigger_test.go index 20b4f1097..ffabae0fc 100644 --- a/database/redis/trigger_test.go +++ b/database/redis/trigger_test.go @@ -248,7 +248,7 @@ func TestTriggerStoring(t *testing.T) { So(actualTriggerChecks, ShouldResemble, []*moira.TriggerCheck{triggerCheck}) //Add check data - err = dataBase.SetTriggerLastCheck(trigger.ID, &lastCheckTest, false) + err = dataBase.SetTriggerLastCheck(trigger.ID, &lastCheckTest, moira.GraphiteLocal) So(err, ShouldBeNil) triggerCheck.LastCheck = lastCheckTest @@ -305,23 +305,25 @@ func TestTriggerStoring(t *testing.T) { metric2 := "my.new.test.super.metric2" triggerVer1 := &moira.Trigger{ - ID: "test-triggerID-id1", - Name: "test trigger 1 v1.0", - Targets: []string{pattern1}, - Tags: []string{"test-tag-1"}, - Patterns: []string{pattern1}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "test-triggerID-id1", + Name: "test trigger 1 v1.0", + Targets: []string{pattern1}, + Tags: []string{"test-tag-1"}, + Patterns: []string{pattern1}, + TriggerType: moira.RisingTrigger, + TriggerSource: moira.GraphiteLocal, + AloneMetrics: map[string]bool{}, } triggerVer2 := &moira.Trigger{ - ID: "test-triggerID-id1", - Name: "test trigger 1 v2.0", - Targets: []string{pattern2}, - Tags: []string{"test-tag-1"}, - Patterns: []string{pattern2}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "test-triggerID-id1", + Name: "test trigger 1 v2.0", + Targets: []string{pattern2}, + Tags: []string{"test-tag-1"}, + Patterns: []string{pattern2}, + TriggerType: moira.RisingTrigger, + TriggerSource: moira.GraphiteLocal, + AloneMetrics: map[string]bool{}, } val1 := &moira.MatchedMetric{ @@ -534,13 +536,13 @@ func TestRemoteTrigger(t *testing.T) { dataBase.clock = systemClock pattern := "test.pattern.remote1" trigger := &moira.Trigger{ - ID: "triggerID-0000000000010", - Name: "remote", - Targets: []string{"test.target.remote1"}, - Patterns: []string{pattern}, - IsRemote: true, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000010", + Name: "remote", + Targets: []string{"test.target.remote1"}, + Patterns: []string{pattern}, + TriggerSource: moira.GraphiteRemote, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, } dataBase.Flush() defer dataBase.Flush() @@ -591,7 +593,7 @@ func TestRemoteTrigger(t *testing.T) { }) Convey("Update remote trigger as local", t, func() { - trigger.IsRemote = false + trigger.TriggerSource = moira.GraphiteLocal trigger.Patterns = []string{pattern} Convey("Trigger should be saved correctly", func() { systemClock.EXPECT().Now().Return(time.Date(2022, time.June, 7, 10, 0, 0, 0, time.UTC)) @@ -634,7 +636,7 @@ func TestRemoteTrigger(t *testing.T) { So(patterns, ShouldResemble, trigger.Patterns) }) - trigger.IsRemote = true + trigger.TriggerSource = moira.GraphiteRemote Convey("Update this trigger as remote", func() { systemClock.EXPECT().Now().Return(time.Date(2022, time.June, 7, 10, 0, 0, 0, time.UTC)) @@ -726,9 +728,10 @@ func TestDbConnector_preSaveTrigger(t *testing.T) { Convey("When a local trigger", t, func() { trigger := &moira.Trigger{ - ID: "trigger-id", - Patterns: patterns, - UpdatedBy: "awesome_user", + ID: "trigger-id", + Patterns: patterns, + UpdatedBy: "awesome_user", + TriggerSource: moira.GraphiteLocal, } Convey("UpdatedAt CreatedAt fields should be set `now` on creation.", func() { @@ -769,7 +772,11 @@ func TestDbConnector_preSaveTrigger(t *testing.T) { }) Convey("When a remote trigger", t, func() { - trigger := &moira.Trigger{ID: "trigger-id", Patterns: patterns, IsRemote: true} + trigger := &moira.Trigger{ + ID: "trigger-id", + Patterns: patterns, + TriggerSource: moira.GraphiteRemote, + } Convey("UpdatedAt CreatedAt fields should be set `now` on creation; patterns should be empty.", func() { connector.preSaveTrigger(trigger, nil) @@ -843,71 +850,81 @@ var testTriggers = []moira.Trigger{ TriggerType: moira.RisingTrigger, TTLState: &moira.TTLStateNODATA, AloneMetrics: map[string]bool{}, + //TODO: Test that empty TriggerSource is filled on getting vale from db + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000001", - Name: "test trigger 1 v2.0", - Targets: []string{"test.target.1", "test.target.2"}, - Tags: []string{"test-tag-2", "test-tag-1"}, - Patterns: []string{"test.pattern.2", "test.pattern.1"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{"t2": true}, + ID: "triggerID-0000000000001", + Name: "test trigger 1 v2.0", + Targets: []string{"test.target.1", "test.target.2"}, + Tags: []string{"test-tag-2", "test-tag-1"}, + Patterns: []string{"test.pattern.2", "test.pattern.1"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{"t2": true}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000001", - Name: "test trigger 1 v3.0", - Targets: []string{"test.target.3"}, - Tags: []string{"test-tag-2", "test-tag-3"}, - Patterns: []string{"test.pattern.3", "test.pattern.2"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000001", + Name: "test trigger 1 v3.0", + Targets: []string{"test.target.3"}, + Tags: []string{"test-tag-2", "test-tag-3"}, + Patterns: []string{"test.pattern.3", "test.pattern.2"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000004", - Name: "test trigger 4", - Targets: []string{"test.target.4"}, - Tags: []string{"test-tag-4"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000004", + Name: "test trigger 4", + Targets: []string{"test.target.4"}, + Tags: []string{"test-tag-4"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000005", - Name: "test trigger 5 (nobody is subscribed)", - Targets: []string{"test.target.5"}, - Tags: []string{"test-tag-nosub"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000005", + Name: "test trigger 5 (nobody is subscribed)", + Targets: []string{"test.target.5"}, + Tags: []string{"test-tag-nosub"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000006", - Name: "test trigger 6 (throttling disabled)", - Targets: []string{"test.target.6"}, - Tags: []string{"test-tag-throttling-disabled"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000006", + Name: "test trigger 6 (throttling disabled)", + Targets: []string{"test.target.6"}, + Tags: []string{"test-tag-throttling-disabled"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000007", - Name: "test trigger 7 (multiple subscribers)", - Targets: []string{"test.target.7"}, - Tags: []string{"test-tag-multiple-subs"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000007", + Name: "test trigger 7 (multiple subscribers)", + Targets: []string{"test.target.7"}, + Tags: []string{"test-tag-multiple-subs"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000008", - Name: "test trigger 8 (duplicated contacts)", - Targets: []string{"test.target.8"}, - Tags: []string{"test-tag-dup-contacts"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000008", + Name: "test trigger 8 (duplicated contacts)", + Targets: []string{"test.target.8"}, + Tags: []string{"test-tag-dup-contacts"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, { - ID: "triggerID-0000000000009", - Name: "test trigger 9 (pseudo tag)", - Targets: []string{"test.target.9"}, - Tags: []string{"test-degradation"}, - TriggerType: moira.RisingTrigger, - AloneMetrics: map[string]bool{}, + ID: "triggerID-0000000000009", + Name: "test trigger 9 (pseudo tag)", + Targets: []string{"test.target.9"}, + Tags: []string{"test-degradation"}, + TriggerType: moira.RisingTrigger, + AloneMetrics: map[string]bool{}, + TriggerSource: moira.GraphiteLocal, }, } diff --git a/database/redis/triggers_to_check.go b/database/redis/triggers_to_check.go index f7105bb84..0281d9453 100644 --- a/database/redis/triggers_to_check.go +++ b/database/redis/triggers_to_check.go @@ -17,6 +17,10 @@ func (connector *DbConnector) AddRemoteTriggersToCheck(triggerIDs []string) erro return connector.addTriggersToCheck(remoteTriggersToCheckKey, triggerIDs) } +func (connector *DbConnector) AddPrometheusTriggersToCheck(triggerIDs []string) error { + return connector.addTriggersToCheck(prometheusTriggersToCheckKey, triggerIDs) +} + // GetLocalTriggersToCheck return random trigger ID from Redis Set func (connector *DbConnector) GetLocalTriggersToCheck(count int) ([]string, error) { return connector.getTriggersToCheck(localTriggersToCheckKey, count) @@ -27,6 +31,10 @@ func (connector *DbConnector) GetRemoteTriggersToCheck(count int) ([]string, err return connector.getTriggersToCheck(remoteTriggersToCheckKey, count) } +func (connector *DbConnector) GetPrometheusTriggersToCheck(count int) ([]string, error) { + return connector.getTriggersToCheck(prometheusTriggersToCheckKey, count) +} + // GetLocalTriggersToCheckCount return number of triggers ID to check from Redis Set func (connector *DbConnector) GetLocalTriggersToCheckCount() (int64, error) { return connector.getTriggersToCheckCount(localTriggersToCheckKey) @@ -37,6 +45,10 @@ func (connector *DbConnector) GetRemoteTriggersToCheckCount() (int64, error) { return connector.getTriggersToCheckCount(remoteTriggersToCheckKey) } +func (connector *DbConnector) GetPrometheusTriggersToCheckCount() (int64, error) { + return connector.getTriggersToCheckCount(prometheusTriggersToCheckKey) +} + func (connector *DbConnector) addTriggersToCheck(key string, triggerIDs []string) error { ctx := connector.context pipe := (*connector.client).TxPipeline() @@ -81,4 +93,5 @@ func (connector *DbConnector) getTriggersToCheckCount(key string) (int64, error) } var remoteTriggersToCheckKey = "moira-remote-triggers-to-check" +var prometheusTriggersToCheckKey = "moira-prometheus-triggers-to-check" var localTriggersToCheckKey = "moira-triggers-to-check" diff --git a/datatypes.go b/datatypes.go index b725039a3..30ce3c56a 100644 --- a/datatypes.go +++ b/datatypes.go @@ -2,6 +2,7 @@ package moira import ( "bytes" + "encoding/json" "fmt" "math" "sort" @@ -157,14 +158,19 @@ func NotificationEventsToTemplatingEvents(events NotificationEvents) []templatin // TriggerData represents trigger object type TriggerData struct { - ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` - Name string `json:"name" example:"Not enough disk space left"` - Desc string `json:"desc" example:"check the size of /var/log"` - Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` - WarnValue float64 `json:"warn_value" example:"5000"` - ErrorValue float64 `json:"error_value" example:"1000"` - IsRemote bool `json:"is_remote" example:"false"` - Tags []string `json:"__notifier_trigger_tags" example:"server,disk"` + ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` + Name string `json:"name" example:"Not enough disk space left"` + Desc string `json:"desc" example:"check the size of /var/log"` + Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` + WarnValue float64 `json:"warn_value" example:"5000"` + ErrorValue float64 `json:"error_value" example:"1000"` + IsRemote bool `json:"is_remote" example:"false"` + TriggerSource TriggerSource `json:"trigger_source,omitempty" example:"graphite_local"` + Tags []string `json:"__notifier_trigger_tags" example:"server,disk"` +} + +func (trigger TriggerData) GetTriggerSource() TriggerSource { + return trigger.TriggerSource.FillInIfNotSet(trigger.IsRemote) } // GetTriggerURI gets frontUri and returns triggerUrl, returns empty string on selfcheck and test notifications @@ -280,7 +286,7 @@ type Trigger struct { Expression *string `json:"expression,omitempty" example:""` PythonExpression *string `json:"python_expression,omitempty"` Patterns []string `json:"patterns" example:""` - IsRemote bool `json:"is_remote" example:"false"` + TriggerSource TriggerSource `json:"trigger_source,omitempty" example:"graphite_local"` MuteNewMetrics bool `json:"mute_new_metrics" example:"false"` AloneMetrics map[string]bool `json:"alone_metrics" example:"t1:true"` CreatedAt *int64 `json:"created_at" format:"int64"` @@ -289,6 +295,43 @@ type Trigger struct { UpdatedBy string `json:"updated_by"` } +type TriggerSource string + +const ( + TriggerSourceNotSet TriggerSource = "" + GraphiteLocal TriggerSource = "graphite_local" + GraphiteRemote TriggerSource = "graphite_remote" + PrometheusRemote TriggerSource = "prometheus_remote" +) + +func (s *TriggerSource) UnmarshalJSON(data []byte) error { + var v string + if err := json.Unmarshal(data, &v); err != nil { + return err + } + + source := TriggerSource(v) + if source != GraphiteLocal && source != GraphiteRemote && source != PrometheusRemote { + *s = TriggerSourceNotSet + return nil + } + + *s = source + return nil +} + +// Neede for backwards compatibility with moira versions that used oly isRemote flag +func (triggerSource TriggerSource) FillInIfNotSet(isRempte bool) TriggerSource { + if triggerSource == TriggerSourceNotSet { + if isRempte { + return GraphiteRemote + } else { + return GraphiteLocal + } + } + return triggerSource +} + // TriggerCheck represents trigger data with last check data and check timestamp type TriggerCheck struct { Trigger diff --git a/go.mod b/go.mod index 09559cd88..f18b17bd2 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,8 @@ require ( gopkg.in/yaml.v2 v2.4.0 ) +require github.com/prometheus/common v0.37.0 + require ( github.com/mitchellh/mapstructure v1.5.0 github.com/swaggo/http-swagger v1.3.4 @@ -107,6 +109,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.15.14 // indirect @@ -130,6 +133,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/msaf1980/go-stringutils v0.1.4 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/natefinch/atomic v1.0.1 // indirect github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -138,7 +142,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rs/xid v1.4.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect @@ -165,9 +168,11 @@ require ( golang.org/x/exp v0.0.0-20200924195034-c827fd4f18b9 // indirect golang.org/x/image v0.5.0 // indirect golang.org/x/net v0.14.0 // indirect + golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect gonum.org/v1/gonum v0.12.0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index b7930c20f..dcbad53c2 100644 --- a/go.sum +++ b/go.sum @@ -813,6 +813,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -921,6 +922,7 @@ github.com/msaf1980/go-stringutils v0.1.4/go.mod h1:AxmV/6JuQUAtZJg5XmYATB5ZwCWg github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= @@ -1310,6 +1312,7 @@ golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1590,6 +1593,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/helpers_test.go b/helpers_test.go index f2397415b..d2800ee79 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -176,11 +176,11 @@ var triggerVal3 = &Trigger{ } var triggerVal4 = &Trigger{ - ID: "trigger-id-4", - Name: "Super Trigger 4", - IsRemote: true, - TTL: 600, - Tags: []string{"4"}, + ID: "trigger-id-4", + Name: "Super Trigger 4", + TriggerSource: GraphiteRemote, + TTL: 600, + Tags: []string{"4"}, } func TestChunkSlice(t *testing.T) { diff --git a/integration_tests/notifier/notifier_test.go b/integration_tests/notifier/notifier_test.go index 53eb78f22..a77e4dd47 100644 --- a/integration_tests/notifier/notifier_test.go +++ b/integration_tests/notifier/notifier_test.go @@ -62,10 +62,11 @@ var trigger = moira.Trigger{ } var triggerData = moira.TriggerData{ - ID: "triggerID-0000000000001", - Name: "test trigger 1", - Targets: []string{"test.target.1"}, - Tags: []string{"test-tag-1"}, + ID: "triggerID-0000000000001", + Name: "test trigger 1", + Targets: []string{"test.target.1"}, + Tags: []string{"test-tag-1"}, + TriggerSource: moira.GraphiteLocal, } var event = moira.NotificationEvent{ @@ -78,20 +79,35 @@ var event = moira.NotificationEvent{ func TestNotifier(t *testing.T) { mockCtrl = gomock.NewController(t) defer mockCtrl.Finish() + database := redis.NewTestDatabase(logger) - metricsSourceProvider := metricSource.CreateMetricSourceProvider(local.Create(database), nil) database.SaveContact(&contact) //nolint database.SaveSubscription(&subscription) //nolint database.SaveTrigger(trigger.ID, &trigger) //nolint database.PushNotificationEvent(&event, true) //nolint - notifier2 := notifier.NewNotifier(database, logger, notifierConfig, notifierMetrics, metricsSourceProvider, map[string]moira.ImageStore{}) + + metricsSourceProvider := metricSource.CreateMetricSourceProvider(local.Create(database), nil, nil) + + notifierInstance := notifier.NewNotifier( + database, + logger, + notifierConfig, + notifierMetrics, + metricsSourceProvider, + map[string]moira.ImageStore{}, + ) + sender := mock_moira_alert.NewMockSender(mockCtrl) sender.EXPECT().Init(senderSettings, logger, location, dateTimeFormat).Return(nil) - notifier2.RegisterSender(senderSettings, sender) //nolint - sender.EXPECT().SendEvents(gomock.Any(), contact, triggerData, gomock.Any(), false).Return(nil).Do(func(arg0, arg1, arg2, arg3, arg4 interface{}) { - fmt.Print("SendEvents called. End test") - close(shutdown) - }) + sender.EXPECT(). + SendEvents(gomock.Any(), contact, triggerData, gomock.Any(), false). + Return(nil). + Do(func(arg0, arg1, arg2, arg3, arg4 interface{}) { + fmt.Print("SendEvents called. End test") + close(shutdown) + }) + + notifierInstance.RegisterSender(senderSettings, sender) //nolint fetchEventsWorker := events.FetchEventsWorker{ Database: database, @@ -103,7 +119,7 @@ func TestNotifier(t *testing.T) { fetchNotificationsWorker := notifications.FetchNotificationsWorker{ Database: database, Logger: logger, - Notifier: notifier2, + Notifier: notifierInstance, } fetchEventsWorker.Start() diff --git a/interfaces.go b/interfaces.go index 95904cef3..b98064e29 100644 --- a/interfaces.go +++ b/interfaces.go @@ -15,6 +15,7 @@ type Database interface { GetMetricsUpdatesCount() (int64, error) GetChecksUpdatesCount() (int64, error) GetRemoteChecksUpdatesCount() (int64, error) + GetPrometheusChecksUpdatesCount() (int64, error) GetNotifierState() (string, error) SetNotifierState(string) error @@ -26,15 +27,17 @@ type Database interface { // LastCheck storing GetTriggerLastCheck(triggerID string) (CheckData, error) - SetTriggerLastCheck(triggerID string, checkData *CheckData, isRemote bool) error + SetTriggerLastCheck(triggerID string, checkData *CheckData, triggerSource TriggerSource) error RemoveTriggerLastCheck(triggerID string) error SetTriggerCheckMaintenance(triggerID string, metrics map[string]int64, triggerMaintenance *int64, userLogin string, timeCallMaintenance int64) error CleanUpAbandonedTriggerLastCheck() error // Trigger storing - GetLocalTriggerIDs() ([]string, error) GetAllTriggerIDs() ([]string, error) + GetLocalTriggerIDs() ([]string, error) GetRemoteTriggerIDs() ([]string, error) + GetPrometheusTriggerIDs() ([]string, error) + GetTrigger(triggerID string) (Trigger, error) GetTriggers(triggerIDs []string) ([]*Trigger, error) GetTriggerChecks(triggerIDs []string) ([]*TriggerCheck, error) @@ -116,6 +119,10 @@ type Database interface { GetRemoteTriggersToCheck(count int) ([]string, error) GetRemoteTriggersToCheckCount() (int64, error) + AddPrometheusTriggersToCheck(triggerIDs []string) error + GetPrometheusTriggersToCheck(count int) ([]string, error) + GetPrometheusTriggersToCheckCount() (int64, error) + // TriggerCheckLock storing AcquireTriggerCheckLock(triggerID string, maxAttemptsCount int) error DeleteTriggerCheckLock(triggerID string) error diff --git a/local/api.yml b/local/api.yml index ccbb28718..c0b4939db 100644 --- a/local/api.yml +++ b/local/api.yml @@ -18,6 +18,12 @@ remote: check_interval: 60s timeout: 60s metrics_ttl: 168h +prometheus: + url: "http://prometheus:9090" + enabled: true + check_interval: 60s + timeout: 60s + metrics_ttl: 168h api: listen: ":8081" enable_cors: false diff --git a/local/checker.yml b/local/checker.yml index c704183ad..eacd43a5c 100644 --- a/local/checker.yml +++ b/local/checker.yml @@ -17,7 +17,13 @@ remote: url: "http://graphite:80/render" check_interval: 60s timeout: 60s - metrics_ttl: 7d + metrics_ttl: 168h +prometheus: + url: "http://prometheus:9090" + enabled: true + check_interval: 60s + timeout: 60s + metrics_ttl: 168h checker: nodata_check_interval: 60s check_interval: 10s diff --git a/local/notifier.yml b/local/notifier.yml index c9583d4bd..a4a19f8fc 100644 --- a/local/notifier.yml +++ b/local/notifier.yml @@ -17,7 +17,13 @@ remote: url: "http://graphite:80/render" check_interval: 60s timeout: 60s - metrics_ttl: 7d + metrics_ttl: 168h +prometheus: + url: "http://prometheus:9090" + enabled: true + check_interval: 60s + timeout: 60s + metrics_ttl: 168h notifier: sender_timeout: 10s resending_timeout: "1:00" diff --git a/metric_source/local/local.go b/metric_source/local/local.go index 3f932bd17..e964dd55d 100644 --- a/metric_source/local/local.go +++ b/metric_source/local/local.go @@ -33,6 +33,11 @@ func (local *Local) IsConfigured() (bool, error) { return true, nil } +// IsConfigured always returns true. It easy to configure local source =) +func (local *Local) IsAvailable() (bool, error) { + return true, nil +} + // Fetch is analogue of evaluateTarget method in graphite-web, that gets target metrics value from DB and Evaluate it using carbon-api eval package func (local *Local) Fetch(target string, from int64, until int64, allowRealTimeAlerting bool) (metricSource.FetchResult, error) { // Don't fetch intervals larger than metrics TTL to prevent OOM errors diff --git a/metric_source/prometheus/convert.go b/metric_source/prometheus/convert.go new file mode 100644 index 000000000..0792a1375 --- /dev/null +++ b/metric_source/prometheus/convert.go @@ -0,0 +1,89 @@ +package prometheus + +import ( + "sort" + "strings" + + metricSource "github.com/moira-alert/moira/metric_source" + "github.com/prometheus/common/model" +) + +func convertToFetchResult(mat model.Matrix, from, until int64, allowRealTimeAlerting bool) *FetchResult { + result := FetchResult{ + MetricsData: make([]metricSource.MetricData, 0, len(mat)), + } + + for _, res := range mat { + resValues := TrimValuesIfNescesary(res.Values, allowRealTimeAlerting) + + values := make([]float64, 0, len(resValues)) + for _, v := range resValues { + values = append(values, float64(v.Value)) + } + + start, stop := StartStopFromValues(resValues, from, until) + data := metricSource.MetricData{ + Name: targetFromTags(res.Metric), + StartTime: start, + StopTime: stop, + StepTime: StepTimeSeconds, + Values: values, + Wildcard: false, + } + result.MetricsData = append(result.MetricsData, data) + } + + return &result +} + +func StartStopFromValues(values []model.SamplePair, from, until int64) (int64, int64) { + start, stop := from, until + if len(values) != 0 { + start = values[0].Timestamp.Unix() + stop = values[len(values)-1].Timestamp.Unix() + } + return start, stop +} + +func TrimValuesIfNescesary(values []model.SamplePair, allowRealTimeAlerting bool) []model.SamplePair { + if allowRealTimeAlerting || len(values) == 0 { + return values + } + + return values[:len(values)-1] +} + +func targetFromTags(tags model.Metric) string { + target := strings.Builder{} + if name, ok := tags["__name__"]; ok { + target.WriteString(string(name)) + } + + tagsList := make([]struct{ key, value string }, 0) + for key, value := range tags { + tagsList = append(tagsList, struct{ key, value string }{string(key), string(value)}) + } + + sort.Slice(tagsList, func(i, j int) bool { + a, b := tagsList[i], tagsList[j] + if a.key != b.key { + return a.key < b.key + } + + return a.value < b.value + }) + + for _, tag := range tagsList { + if tag.key == "__name__" { + continue + } + if target.Len() != 0 { + target.WriteRune(';') + } + target.WriteString(tag.key) + target.WriteRune('=') + target.WriteString(tag.value) + } + + return target.String() +} diff --git a/metric_source/prometheus/convert_test.go b/metric_source/prometheus/convert_test.go new file mode 100644 index 000000000..565fac521 --- /dev/null +++ b/metric_source/prometheus/convert_test.go @@ -0,0 +1,225 @@ +package prometheus + +import ( + "testing" + "time" + + metricSource "github.com/moira-alert/moira/metric_source" + "github.com/prometheus/common/model" + . "github.com/smartystreets/goconvey/convey" +) + +func MakeSamplePair(time time.Time, value float64) model.SamplePair { + return model.SamplePair{ + Timestamp: model.Time(time.UnixMilli()), + Value: model.SampleValue(value), + } +} + +func TestConvertToFetchResult(t *testing.T) { + metric_1 := model.Metric{ + "__name__": "name", + "label_1": "value_1", + } + metric_2 := model.Metric{ + "__name__": "name", + "label_1": "value_2", + } + + Convey("Given no metrics fetched", t, func() { + now := time.Now() + + mat := model.Matrix{} + + result := convertToFetchResult(mat, now.Unix(), now.Unix(), true) + + expected := &FetchResult{ + MetricsData: make([]metricSource.MetricData, 0), + } + + So(result, ShouldResemble, expected) + }) + + Convey("Given one metric with no values fetched", t, func() { + now := time.Now() + + mat := model.Matrix{&model.SampleStream{ + Metric: metric_1, + Values: []model.SamplePair{}, + }} + + result := convertToFetchResult(mat, now.Unix(), now.Unix(), true) + + expected := &FetchResult{ + MetricsData: []metricSource.MetricData{ + { + Name: "name;label_1=value_1", + StartTime: now.Unix(), + StopTime: now.Unix(), + StepTime: 60, + Values: []float64{}, + Wildcard: false, + }, + }, + } + + So(result, ShouldResemble, expected) + }) + + Convey("Given one metric with one value fetched", t, func() { + now := time.Now() + + mat := model.Matrix{&model.SampleStream{ + Metric: metric_1, + Values: []model.SamplePair{MakeSamplePair(now, 1.0)}, + }} + + result := convertToFetchResult(mat, now.Unix(), now.Unix(), true) + + expected := &FetchResult{ + MetricsData: []metricSource.MetricData{ + { + Name: "name;label_1=value_1", + StartTime: now.Unix(), + StopTime: now.Unix(), + StepTime: 60, + Values: []float64{1.0}, + Wildcard: false, + }, + }, + } + + So(result, ShouldResemble, expected) + }) + + Convey("Given one metric with many values fetched", t, func() { + now := time.Now() + + mat := model.Matrix{&model.SampleStream{ + Metric: metric_1, + Values: []model.SamplePair{ + MakeSamplePair(now, 1.0), + MakeSamplePair(now.Add(60*time.Second), 2.0), + MakeSamplePair(now.Add(120*time.Second), 3.0), + MakeSamplePair(now.Add(180*time.Second), 4.0), + }, + }} + + result := convertToFetchResult(mat, now.Unix(), now.Unix(), true) + + expected := &FetchResult{ + MetricsData: []metricSource.MetricData{ + { + Name: "name;label_1=value_1", + StartTime: now.Unix(), + StopTime: now.Add(180 * time.Second).Unix(), + StepTime: 60, + Values: []float64{1.0, 2.0, 3.0, 4.0}, + Wildcard: false, + }, + }, + } + + So(result, ShouldResemble, expected) + }) + + Convey("Given several metric fetched", t, func() { + now := time.Now() + + mat := model.Matrix{ + &model.SampleStream{ + Metric: metric_1, + Values: []model.SamplePair{ + MakeSamplePair(now, 1.0), + MakeSamplePair(now.Add(60*time.Second), 2.0), + }, + }, + &model.SampleStream{ + Metric: metric_2, + Values: []model.SamplePair{ + MakeSamplePair(now, 3.0), + MakeSamplePair(now.Add(60*time.Second), 4.0), + }, + }, + } + + result := convertToFetchResult(mat, now.Unix(), now.Unix(), true) + + expected := &FetchResult{ + MetricsData: []metricSource.MetricData{ + { + Name: "name;label_1=value_1", + StartTime: now.Unix(), + StopTime: now.Add(60 * time.Second).Unix(), + StepTime: 60, + Values: []float64{1.0, 2.0}, + Wildcard: false, + }, + { + Name: "name;label_1=value_2", + StartTime: now.Unix(), + StopTime: now.Add(60 * time.Second).Unix(), + StepTime: 60, + Values: []float64{3.0, 4.0}, + Wildcard: false, + }, + }, + } + + So(result, ShouldResemble, expected) + }) +} + +func TestTargetFromTags(t *testing.T) { + Convey("Given tags is empty", t, func() { + tags := model.Metric{} + + target := targetFromTags(tags) + + So(target, ShouldEqual, "") + }) + + Convey("Given tags contain only __name__", t, func() { + tags := model.Metric{ + "__name__": "name", + } + + target := targetFromTags(tags) + + So(target, ShouldEqual, "name") + }) + + Convey("Given tags contain one tag", t, func() { + tags := model.Metric{ + "test_1": "value_1", + } + + target := targetFromTags(tags) + + So(target, ShouldEqual, "test_1=value_1") + }) + + Convey("Given tags contain several tags", t, func() { + tags := model.Metric{ + "a": "1", + "ab": "1", + "c": "1", + } + + target := targetFromTags(tags) + + So(target, ShouldEqual, "a=1;ab=1;c=1") + }) + + Convey("Given tags contain __name__ and tags", t, func() { + tags := model.Metric{ + "test_1": "value_1", + "__name__": "name", + "test_2": "value_2", + } + + target := targetFromTags(tags) + + So(target, ShouldEqual, "name;test_1=value_1;test_2=value_2") + }) +} diff --git a/metric_source/prometheus/fetch.go b/metric_source/prometheus/fetch.go new file mode 100644 index 000000000..f2a0b33d7 --- /dev/null +++ b/metric_source/prometheus/fetch.go @@ -0,0 +1,60 @@ +package prometheus + +import ( + "context" + "fmt" + "time" + + metricSource "github.com/moira-alert/moira/metric_source" + + "github.com/moira-alert/moira" + prometheusApi "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +func (prometheus *Prometheus) Fetch(target string, from, until int64, allowRealTimeAlerting bool) (metricSource.FetchResult, error) { + from = moira.MaxInt64(from, until-int64(prometheus.config.MetricsTTL.Seconds())) + + ctx, cancel := context.WithTimeout(context.Background(), prometheus.config.Timeout) + defer cancel() + + val, warns, err := prometheus.api.QueryRange(ctx, target, prometheusApi.Range{ + Start: time.Unix(from, 0), + End: time.Unix(until, 0), + Step: time.Second * time.Duration(StepTimeSeconds), + }) + + if len(warns) != 0 { + prometheus.logger. + Warning(). + Interface("warns", warns). + Msg("Warnings when fetching metrics from remote prometheus") + } + + if err != nil { + return nil, err + } + + mat := val.(model.Matrix) + + return convertToFetchResult(mat, from, until, allowRealTimeAlerting), nil +} + +type FetchResult struct { + MetricsData []metricSource.MetricData +} + +// GetMetricsData return all metrics data from fetch result +func (fetchResult *FetchResult) GetMetricsData() []metricSource.MetricData { + return fetchResult.MetricsData +} + +// GetPatterns always returns error, because we can't fetch target patterns from remote metrics source +func (*FetchResult) GetPatterns() ([]string, error) { + return make([]string, 0), fmt.Errorf("remote fetch result never returns patterns") +} + +// GetPatternMetrics always returns error, because remote fetch doesn't return base pattern metrics +func (*FetchResult) GetPatternMetrics() ([]string, error) { + return make([]string, 0), fmt.Errorf("remote fetch result never returns pattern metrics") +} diff --git a/metric_source/prometheus/prometheus.go b/metric_source/prometheus/prometheus.go new file mode 100644 index 000000000..b9b9e0b78 --- /dev/null +++ b/metric_source/prometheus/prometheus.go @@ -0,0 +1,57 @@ +package prometheus + +import ( + "fmt" + "time" + + "github.com/moira-alert/moira" + metricSource "github.com/moira-alert/moira/metric_source" + + prometheusApi "github.com/prometheus/client_golang/api/prometheus/v1" +) + +const StepTimeSeconds int64 = 60 + +var ErrPrometheusStorageDisabled = fmt.Errorf("remote prometheus storage is not enabled") + +type Config struct { + Enabled bool + CheckInterval time.Duration + MetricsTTL time.Duration + Timeout time.Duration + URL string + User string + Password string +} + +func Create(config *Config, logger moira.Logger) (metricSource.MetricSource, error) { + promApi, err := createPrometheusApi(config) + if err != nil { + return nil, err + } + + return &Prometheus{config: config, api: promApi, logger: logger}, nil +} + +type Prometheus struct { + config *Config + logger moira.Logger + api prometheusApi.API +} + +func (prometheus *Prometheus) GetMetricsTTLSeconds() int64 { + return int64(prometheus.config.MetricsTTL.Seconds()) +} + +func (prometheus *Prometheus) IsConfigured() (bool, error) { + if prometheus.config.Enabled { + return prometheus.config.Enabled, nil + } + return false, ErrPrometheusStorageDisabled +} + +func (prometheus *Prometheus) IsAvailable() (bool, error) { + now := time.Now().Unix() + _, err := prometheus.Fetch("1", now, now, true) + return err == nil, err +} diff --git a/metric_source/prometheus/prometheus_api.go b/metric_source/prometheus/prometheus_api.go new file mode 100644 index 000000000..a979a4a02 --- /dev/null +++ b/metric_source/prometheus/prometheus_api.go @@ -0,0 +1,37 @@ +package prometheus + +import ( + "encoding/base64" + "fmt" + + "github.com/prometheus/client_golang/api" + prometheusApi "github.com/prometheus/client_golang/api/prometheus/v1" + promConfig "github.com/prometheus/common/config" +) + +func createPrometheusApi(config *Config) (prometheusApi.API, error) { + roundTripper := api.DefaultRoundTripper + + if config.User != "" && config.Password != "" { + rawToken := fmt.Sprintf("%s:%s", config.User, config.Password) + token := base64.StdEncoding.EncodeToString([]byte(rawToken)) + + roundTripper = promConfig.NewAuthorizationCredentialsRoundTripper( + "Basic", + promConfig.Secret(token), + roundTripper, + ) + } + + promClientConfig := api.Config{ + Address: config.URL, + RoundTripper: roundTripper, + } + + promCl, err := api.NewClient(promClientConfig) + if err != nil { + return nil, err + } + + return prometheusApi.NewAPI(promCl), nil +} diff --git a/metric_source/prometheus/prometheus_test.go b/metric_source/prometheus/prometheus_test.go new file mode 100644 index 000000000..10744fbe1 --- /dev/null +++ b/metric_source/prometheus/prometheus_test.go @@ -0,0 +1,23 @@ +package prometheus + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestIsConfigured(t *testing.T) { + Convey("Metric source is not configured", t, func() { + source, _ := Create(&Config{URL: "", Enabled: false}, nil) + isConfigured, err := source.IsConfigured() + So(isConfigured, ShouldBeFalse) + So(err, ShouldResemble, ErrPrometheusStorageDisabled) + }) + + Convey("Metric source is configured", t, func() { + source, _ := Create(&Config{URL: "http://host", Enabled: true}, nil) + isConfigured, err := source.IsConfigured() + So(isConfigured, ShouldBeTrue) + So(err, ShouldBeEmpty) + }) +} diff --git a/metric_source/provider.go b/metric_source/provider.go index 7c1745f6f..fed85b087 100644 --- a/metric_source/provider.go +++ b/metric_source/provider.go @@ -11,15 +11,17 @@ var ErrMetricSourceIsNotConfigured = fmt.Errorf("metric source is not configured // SourceProvider is a provider for all known metrics sources type SourceProvider struct { - local MetricSource - remote MetricSource + local MetricSource + remote MetricSource + prometheus MetricSource } // CreateMetricSourceProvider just creates SourceProvider with all known metrics sources -func CreateMetricSourceProvider(local MetricSource, remote MetricSource) *SourceProvider { +func CreateMetricSourceProvider(graphiteLocal, graphiteRemote, prometheusRemote MetricSource) *SourceProvider { return &SourceProvider{ - remote: remote, - local: local, + remote: graphiteRemote, + local: graphiteLocal, + prometheus: prometheusRemote, } } @@ -33,17 +35,30 @@ func (provider *SourceProvider) GetRemote() (MetricSource, error) { return returnSource(provider.remote) } +// GetRemote gets remote metric source. If it not configured returns not empty error +func (provider *SourceProvider) GetPrometheus() (MetricSource, error) { + return returnSource(provider.prometheus) +} + // GetTriggerMetricSource get metrics source by given trigger. If it not configured returns not empty error func (provider *SourceProvider) GetTriggerMetricSource(trigger *moira.Trigger) (MetricSource, error) { - return provider.GetMetricSource(trigger.IsRemote) + return provider.GetMetricSource(trigger.TriggerSource) } // GetMetricSource return metric source depending on trigger flag: is remote trigger or not. GetLocal if not. -func (provider *SourceProvider) GetMetricSource(isRemote bool) (MetricSource, error) { - if isRemote { +func (provider *SourceProvider) GetMetricSource(triggerSource moira.TriggerSource) (MetricSource, error) { + switch triggerSource { + case moira.GraphiteLocal: + return provider.GetLocal() + + case moira.GraphiteRemote: return provider.GetRemote() + + case moira.PrometheusRemote: + return provider.GetPrometheus() } - return provider.GetLocal() + + return nil, fmt.Errorf("unknown metric source") } func returnSource(source MetricSource) (MetricSource, error) { diff --git a/metric_source/remote/remote.go b/metric_source/remote/remote.go index 6be143ecf..4e9aedd17 100644 --- a/metric_source/remote/remote.go +++ b/metric_source/remote/remote.go @@ -82,7 +82,7 @@ func (remote *Remote) IsConfigured() (bool, error) { } // IsRemoteAvailable checks if graphite API is available and returns 200 response -func (remote *Remote) IsRemoteAvailable() (bool, error) { +func (remote *Remote) IsAvailable() (bool, error) { maxRetries := 3 until := time.Now().Unix() from := until - 600 //nolint diff --git a/metric_source/remote/remote_test.go b/metric_source/remote/remote_test.go index 4828ef4ac..001c224e3 100644 --- a/metric_source/remote/remote_test.go +++ b/metric_source/remote/remote_test.go @@ -29,7 +29,7 @@ func TestIsRemoteAvailable(t *testing.T) { Convey("Is available", t, func() { server := createServer([]byte("Some string"), http.StatusOK) remote := Remote{client: server.Client(), config: &Config{URL: server.URL}} - isAvailable, err := remote.IsRemoteAvailable() + isAvailable, err := remote.IsAvailable() So(isAvailable, ShouldBeTrue) So(err, ShouldBeEmpty) }) @@ -37,7 +37,7 @@ func TestIsRemoteAvailable(t *testing.T) { Convey("Not available", t, func() { server := createServer([]byte("Some string"), http.StatusInternalServerError) remote := Remote{client: server.Client(), config: &Config{URL: server.URL}} - isAvailable, err := remote.IsRemoteAvailable() + isAvailable, err := remote.IsAvailable() So(isAvailable, ShouldBeFalse) So(err, ShouldResemble, fmt.Errorf("bad response status %d: %s", http.StatusInternalServerError, "Some string")) }) diff --git a/metric_source/source.go b/metric_source/source.go index b828d619e..620255fe9 100644 --- a/metric_source/source.go +++ b/metric_source/source.go @@ -5,6 +5,7 @@ type MetricSource interface { Fetch(target string, from int64, until int64, allowRealTimeAlerting bool) (FetchResult, error) GetMetricsTTLSeconds() int64 IsConfigured() (bool, error) + IsAvailable() (bool, error) } // FetchResult implements moira metric sources fetching result format diff --git a/metrics/checker.go b/metrics/checker.go index 0d92592c3..a67ca480e 100644 --- a/metrics/checker.go +++ b/metrics/checker.go @@ -6,6 +6,7 @@ import "github.com/moira-alert/moira" type CheckerMetrics struct { LocalMetrics *CheckMetrics RemoteMetrics *CheckMetrics + PrometheusMetrics *CheckMetrics MetricEventsChannelLen Histogram UnusedTriggersCount Histogram MetricEventsHandleTime Timer @@ -13,10 +14,24 @@ type CheckerMetrics struct { // GetCheckMetrics return check metrics dependent on given trigger type func (metrics *CheckerMetrics) GetCheckMetrics(trigger *moira.Trigger) *CheckMetrics { - if trigger.IsRemote { + return metrics.GetCheckMetricsBySource(trigger.TriggerSource) +} + +// GetCheckMetrics return check metrics dependent on given trigger type +func (metrics *CheckerMetrics) GetCheckMetricsBySource(triggerSource moira.TriggerSource) *CheckMetrics { + switch triggerSource { + case moira.GraphiteLocal: + return metrics.LocalMetrics + + case moira.GraphiteRemote: return metrics.RemoteMetrics + + case moira.PrometheusRemote: + return metrics.PrometheusMetrics + + default: + return nil } - return metrics.LocalMetrics } // CheckMetrics is a collection of metrics for trigger checks @@ -28,7 +43,7 @@ type CheckMetrics struct { } // ConfigureCheckerMetrics is checker metrics configurator -func ConfigureCheckerMetrics(registry Registry, remoteEnabled bool) *CheckerMetrics { +func ConfigureCheckerMetrics(registry Registry, remoteEnabled, prometheusEnabled bool) *CheckerMetrics { m := &CheckerMetrics{ LocalMetrics: configureCheckMetrics(registry, "local"), MetricEventsChannelLen: registry.NewHistogram("metricEvents"), @@ -38,6 +53,9 @@ func ConfigureCheckerMetrics(registry Registry, remoteEnabled bool) *CheckerMetr if remoteEnabled { m.RemoteMetrics = configureCheckMetrics(registry, "remote") } + if prometheusEnabled { + m.PrometheusMetrics = configureCheckMetrics(registry, "prometheus") + } return m } diff --git a/mock/metric_source/source.go b/mock/metric_source/source.go index c6d3a3b37..7366920fd 100644 --- a/mock/metric_source/source.go +++ b/mock/metric_source/source.go @@ -63,6 +63,21 @@ func (mr *MockMetricSourceMockRecorder) GetMetricsTTLSeconds() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricsTTLSeconds", reflect.TypeOf((*MockMetricSource)(nil).GetMetricsTTLSeconds)) } +// IsAvailable mocks base method. +func (m *MockMetricSource) IsAvailable() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAvailable") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsAvailable indicates an expected call of IsAvailable. +func (mr *MockMetricSourceMockRecorder) IsAvailable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAvailable", reflect.TypeOf((*MockMetricSource)(nil).IsAvailable)) +} + // IsConfigured mocks base method. func (m *MockMetricSource) IsConfigured() (bool, error) { m.ctrl.T.Helper() diff --git a/mock/moira-alert/database.go b/mock/moira-alert/database.go index 2f2dc189c..8cee64161 100644 --- a/mock/moira-alert/database.go +++ b/mock/moira-alert/database.go @@ -106,6 +106,20 @@ func (mr *MockDatabaseMockRecorder) AddPatternMetric(arg0, arg1 interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPatternMetric", reflect.TypeOf((*MockDatabase)(nil).AddPatternMetric), arg0, arg1) } +// AddPrometheusTriggersToCheck mocks base method. +func (m *MockDatabase) AddPrometheusTriggersToCheck(arg0 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddPrometheusTriggersToCheck", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddPrometheusTriggersToCheck indicates an expected call of AddPrometheusTriggersToCheck. +func (mr *MockDatabaseMockRecorder) AddPrometheusTriggersToCheck(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPrometheusTriggersToCheck", reflect.TypeOf((*MockDatabase)(nil).AddPrometheusTriggersToCheck), arg0) +} + // AddRemoteTriggersToCheck mocks base method. func (m *MockDatabase) AddRemoteTriggersToCheck(arg0 []string) error { m.ctrl.T.Helper() @@ -592,6 +606,66 @@ func (mr *MockDatabaseMockRecorder) GetPatterns() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPatterns", reflect.TypeOf((*MockDatabase)(nil).GetPatterns)) } +// GetPrometheusChecksUpdatesCount mocks base method. +func (m *MockDatabase) GetPrometheusChecksUpdatesCount() (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrometheusChecksUpdatesCount") + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPrometheusChecksUpdatesCount indicates an expected call of GetPrometheusChecksUpdatesCount. +func (mr *MockDatabaseMockRecorder) GetPrometheusChecksUpdatesCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrometheusChecksUpdatesCount", reflect.TypeOf((*MockDatabase)(nil).GetPrometheusChecksUpdatesCount)) +} + +// GetPrometheusTriggerIDs mocks base method. +func (m *MockDatabase) GetPrometheusTriggerIDs() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrometheusTriggerIDs") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPrometheusTriggerIDs indicates an expected call of GetPrometheusTriggerIDs. +func (mr *MockDatabaseMockRecorder) GetPrometheusTriggerIDs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrometheusTriggerIDs", reflect.TypeOf((*MockDatabase)(nil).GetPrometheusTriggerIDs)) +} + +// GetPrometheusTriggersToCheck mocks base method. +func (m *MockDatabase) GetPrometheusTriggersToCheck(arg0 int) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrometheusTriggersToCheck", arg0) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPrometheusTriggersToCheck indicates an expected call of GetPrometheusTriggersToCheck. +func (mr *MockDatabaseMockRecorder) GetPrometheusTriggersToCheck(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrometheusTriggersToCheck", reflect.TypeOf((*MockDatabase)(nil).GetPrometheusTriggersToCheck), arg0) +} + +// GetPrometheusTriggersToCheckCount mocks base method. +func (m *MockDatabase) GetPrometheusTriggersToCheckCount() (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrometheusTriggersToCheckCount") + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPrometheusTriggersToCheckCount indicates an expected call of GetPrometheusTriggersToCheckCount. +func (mr *MockDatabaseMockRecorder) GetPrometheusTriggersToCheckCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrometheusTriggersToCheckCount", reflect.TypeOf((*MockDatabase)(nil).GetPrometheusTriggersToCheckCount)) +} + // GetRemoteChecksUpdatesCount mocks base method. func (m *MockDatabase) GetRemoteChecksUpdatesCount() (int64, error) { m.ctrl.T.Helper() @@ -1497,7 +1571,7 @@ func (mr *MockDatabaseMockRecorder) SetTriggerCheckMaintenance(arg0, arg1, arg2, } // SetTriggerLastCheck mocks base method. -func (m *MockDatabase) SetTriggerLastCheck(arg0 string, arg1 *moira.CheckData, arg2 bool) error { +func (m *MockDatabase) SetTriggerLastCheck(arg0 string, arg1 *moira.CheckData, arg2 moira.TriggerSource) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SetTriggerLastCheck", arg0, arg1, arg2) ret0, _ := ret[0].(error) diff --git a/notifier/events/event.go b/notifier/events/event.go index 48cbdda95..32b72bb84 100644 --- a/notifier/events/event.go +++ b/notifier/events/event.go @@ -100,14 +100,15 @@ func (worker *FetchEventsWorker) processEvent(event moira.NotificationEvent) err } triggerData = moira.TriggerData{ - ID: trigger.ID, - Name: trigger.Name, - Desc: moira.UseString(trigger.Desc), - Targets: trigger.Targets, - WarnValue: moira.UseFloat64(trigger.WarnValue), - ErrorValue: moira.UseFloat64(trigger.ErrorValue), - IsRemote: trigger.IsRemote, - Tags: trigger.Tags, + ID: trigger.ID, + Name: trigger.Name, + Desc: moira.UseString(trigger.Desc), + Targets: trigger.Targets, + WarnValue: moira.UseFloat64(trigger.WarnValue), + ErrorValue: moira.UseFloat64(trigger.ErrorValue), + IsRemote: trigger.TriggerSource == moira.GraphiteRemote, + TriggerSource: trigger.TriggerSource, + Tags: trigger.Tags, } log.Debug(). diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index 514c58556..e3085e3f9 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -211,7 +211,7 @@ func configureNotifier(t *testing.T) { logger, _ = logging.GetLogger("Scheduler") scheduler = mock_scheduler.NewMockScheduler(mockCtrl) sender = mock_moira_alert.NewMockSender(mockCtrl) - metricsSourceProvider := metricSource.CreateMetricSourceProvider(local.Create(dataBase), nil) + metricsSourceProvider := metricSource.CreateMetricSourceProvider(local.Create(dataBase), nil, nil) notif = NewNotifier(dataBase, logger, config, notifierMetrics, metricsSourceProvider, map[string]moira.ImageStore{}) notif.scheduler = scheduler diff --git a/notifier/plotting.go b/notifier/plotting.go index 1f8d78d84..ca95a6a82 100644 --- a/notifier/plotting.go +++ b/notifier/plotting.go @@ -119,7 +119,7 @@ func resolveMetricsWindow(logger moira.Logger, trigger moira.TriggerData, pkg No // resolve remote trigger window. // window is wide: use package window to fetch limited historical data from graphite // window is not wide: use shifted window to fetch extended historical data from graphite - if trigger.IsRemote { + if trigger.GetTriggerSource() == moira.GraphiteRemote { if isWideWindow { return fromTime.Unix(), toTime.Unix() } diff --git a/senders/script/script_test.go b/senders/script/script_test.go index 3a3caf9b9..05564e0a2 100644 --- a/senders/script/script_test.go +++ b/senders/script/script_test.go @@ -82,7 +82,12 @@ func TestBuildCommandData(t *testing.T) { logger, _ := logging.ConfigureLog("stdout", "debug", "test", true) Convey("Test send events", t, func() { sender := Sender{exec: "script.go first second", logger: logger} - scriptFile, args, scriptBody, err := sender.buildCommandData([]moira.NotificationEvent{{Metric: "New metric"}}, moira.ContactData{ID: "ContactID"}, moira.TriggerData{ID: "TriggerID"}, true) + scriptFile, args, scriptBody, err := sender.buildCommandData( + []moira.NotificationEvent{{Metric: "New metric"}}, + moira.ContactData{ID: "ContactID"}, + moira.TriggerData{ID: "TriggerID"}, + true, + ) So(scriptFile, ShouldResemble, "script.go") So(args, ShouldResemble, []string{"first", "second"}) So(err, ShouldBeNil) diff --git a/support/trigger.go b/support/trigger.go index d2851746f..782df7344 100644 --- a/support/trigger.go +++ b/support/trigger.go @@ -129,10 +129,15 @@ func HandlePushTriggerMetrics( return nil } -func HandlePushTriggerLastCheck(logger moira.Logger, database moira.Database, triggerID string, - lastCheck *moira.CheckData, isRemoteTrigger bool) error { +func HandlePushTriggerLastCheck( + logger moira.Logger, + database moira.Database, + triggerID string, + lastCheck *moira.CheckData, + triggerSource moira.TriggerSource, +) error { logger.Info().Msg("Save trigger last check") - if err := database.SetTriggerLastCheck(triggerID, lastCheck, isRemoteTrigger); err != nil { + if err := database.SetTriggerLastCheck(triggerID, lastCheck, triggerSource); err != nil { return fmt.Errorf("cannot set trigger last check: %w", err) } logger.Info().Msg("Trigger last check was saved") From 75f35e1fd9ac1fdcfa41d370f2bee36af3d3d40b Mon Sep 17 00:00:00 2001 From: Xenia N Date: Wed, 30 Aug 2023 16:43:09 +0600 Subject: [PATCH 12/46] Fix/dont mock decode err in api (#902) * fix(api) not mock error_text in api response --- api/middleware/logger.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/middleware/logger.go b/api/middleware/logger.go index dd7166ee7..f10dad4b5 100644 --- a/api/middleware/logger.go +++ b/api/middleware/logger.go @@ -69,6 +69,7 @@ func getErrorResponseIfItHas(writer http.ResponseWriter) *api.ErrorResponse { return &api.ErrorResponse{ HTTPStatusCode: http.StatusInternalServerError, Err: err, + ErrorText: err.Error(), } } @@ -125,6 +126,7 @@ func (entry *apiLoggerEntry) write(status, bytes int, elapsed time.Duration, res errorResponse := getErrorResponseIfItHas(response) if errorResponse != nil { event.Error(errorResponse.Err) + event.String("error_text", errorResponse.ErrorText) } } else { event = entry.logger.Info() From 9d54d2f796f6b66813106424526852cc2f850e7c Mon Sep 17 00:00:00 2001 From: Xenia N Date: Mon, 4 Sep 2023 15:41:24 +0600 Subject: [PATCH 13/46] make not noisy logs (#906) * feat(notifier): make not noisy logs --- cmd/notifier/config.go | 32 ++++++++++++++++++-------------- metrics/checker.go | 2 +- notifier/config.go | 29 +++++++++++++++-------------- notifier/notifier.go | 14 +++++++++++--- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index 1ae79ae29..2af6f6203 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -50,6 +50,8 @@ type notifierConfig struct { DateTimeFormat string `yaml:"date_time_format"` // Amount of messages notifier reads from Redis per iteration. Use notifier.NotificationsLimitUnlimited for unlimited. ReadBatchSize int `yaml:"read_batch_size"` + // Count available mute resend call, if more than set - you see error in logs + MaxFailAttemptToSendAvailable int `yaml:"max_fail_attempt_to_send_available"` // Specify log level by entities SetLogLevel setLogLevelConfig `yaml:"set_log_level"` } @@ -99,9 +101,10 @@ func getDefault() config { LastCheckDelay: "60s", NoticeInterval: "300s", }, - FrontURI: "http://localhost", - Timezone: "UTC", - ReadBatchSize: int(notifier.NotificationsLimitUnlimited), + FrontURI: "http://localhost", + Timezone: "UTC", + ReadBatchSize: int(notifier.NotificationsLimitUnlimited), + MaxFailAttemptToSendAvailable: 3, }, Telemetry: cmd.TelemetryConfig{ Listen: ":8093", @@ -178,17 +181,18 @@ func (config *notifierConfig) getSettings(logger moira.Logger) notifier.Config { Msg("Found dynamic log rules in config for some contacts and subscriptions") return notifier.Config{ - SelfStateEnabled: config.SelfState.Enabled, - SelfStateContacts: config.SelfState.Contacts, - SendingTimeout: to.Duration(config.SenderTimeout), - ResendingTimeout: to.Duration(config.ResendingTimeout), - Senders: config.Senders, - FrontURL: config.FrontURI, - Location: location, - DateTimeFormat: format, - ReadBatchSize: readBatchSize, - LogContactsToLevel: contacts, - LogSubscriptionsToLevel: subscriptions, + SelfStateEnabled: config.SelfState.Enabled, + SelfStateContacts: config.SelfState.Contacts, + SendingTimeout: to.Duration(config.SenderTimeout), + ResendingTimeout: to.Duration(config.ResendingTimeout), + Senders: config.Senders, + FrontURL: config.FrontURI, + Location: location, + DateTimeFormat: format, + ReadBatchSize: readBatchSize, + MaxFailAttemptToSendAvailable: config.MaxFailAttemptToSendAvailable, + LogContactsToLevel: contacts, + LogSubscriptionsToLevel: subscriptions, } } diff --git a/metrics/checker.go b/metrics/checker.go index a67ca480e..7503d882e 100644 --- a/metrics/checker.go +++ b/metrics/checker.go @@ -17,7 +17,7 @@ func (metrics *CheckerMetrics) GetCheckMetrics(trigger *moira.Trigger) *CheckMet return metrics.GetCheckMetricsBySource(trigger.TriggerSource) } -// GetCheckMetrics return check metrics dependent on given trigger type +// GetCheckMetricsBySource return check metrics dependent on given trigger type func (metrics *CheckerMetrics) GetCheckMetricsBySource(triggerSource moira.TriggerSource) *CheckMetrics { switch triggerSource { case moira.GraphiteLocal: diff --git a/notifier/config.go b/notifier/config.go index 5c50b9723..10639101a 100644 --- a/notifier/config.go +++ b/notifier/config.go @@ -8,18 +8,19 @@ const NotificationsLimitUnlimited = int64(-1) // Config is sending settings including log settings type Config struct { - Enabled bool - SelfStateEnabled bool - SelfStateContacts []map[string]string - SendingTimeout time.Duration - ResendingTimeout time.Duration - Senders []map[string]interface{} - LogFile string - LogLevel string - FrontURL string - Location *time.Location - DateTimeFormat string - ReadBatchSize int64 - LogContactsToLevel map[string]string - LogSubscriptionsToLevel map[string]string + Enabled bool + SelfStateEnabled bool + SelfStateContacts []map[string]string + SendingTimeout time.Duration + ResendingTimeout time.Duration + Senders []map[string]interface{} + LogFile string + LogLevel string + FrontURL string + Location *time.Location + DateTimeFormat string + ReadBatchSize int64 + MaxFailAttemptToSendAvailable int + LogContactsToLevel map[string]string + LogSubscriptionsToLevel map[string]string } diff --git a/notifier/notifier.go b/notifier/notifier.go index f3eaf8765..ea945412d 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -219,9 +219,17 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific Msg("Cannot send to broken contact") default: - log.Error(). - Error(err). - Msg("Cannot send notification") + if pkg.FailCount > notifier.config.MaxFailAttemptToSendAvailable { + log.Error(). + Error(err). + Int("fail_count", pkg.FailCount). + Msg("Cannot send notification") + } else { + log.Warning(). + Error(err). + Msg("Cannot send notification") + } + notifier.resend(&pkg, err.Error()) } } From 95f31707a74e1632aaaca20b25b3c207c153e71f Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Mon, 4 Sep 2023 15:47:17 +0300 Subject: [PATCH 14/46] feat: Add metrics for the number of triggers by source (#904) --- cmd/api/main.go | 4 + cmd/api/trigger_stats.go | 66 +++++++++++++++++ cmd/api/trigger_stats_test.go | 49 ++++++++++++ database/redis/trigger.go | 32 ++++++++ filter/metrics_parser_test.go | 2 +- generate_mocks.sh | 3 + interfaces.go | 2 + metrics/prometheus.go | 3 +- metrics/triggers.go | 24 ++++++ mock/moira-alert/database.go | 15 ++++ mock/moira-alert/metrics/meter.go | 60 +++++++++++++++ mock/moira-alert/metrics/registry.go | 107 +++++++++++++++++++++++++++ 12 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 cmd/api/trigger_stats.go create mode 100644 cmd/api/trigger_stats_test.go create mode 100644 metrics/triggers.go create mode 100644 mock/moira-alert/metrics/meter.go create mode 100644 mock/moira-alert/metrics/registry.go diff --git a/cmd/api/main.go b/cmd/api/main.go index db7e47b53..879a99d55 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -99,6 +99,10 @@ func main() { } defer searchIndex.Stop() //nolint + stats := newTriggerStats(logger, database, telemetry.Metrics) + stats.Start() + defer stats.Stop() //nolint + if !searchIndex.IsReady() { logger.Fatal().Msg("Search index is not ready, exit") } diff --git a/cmd/api/trigger_stats.go b/cmd/api/trigger_stats.go new file mode 100644 index 000000000..79de6a3f8 --- /dev/null +++ b/cmd/api/trigger_stats.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" + "gopkg.in/tomb.v2" +) + +type triggerStats struct { + tomb tomb.Tomb + metrics *metrics.TriggersMetrics + database moira.Database + logger moira.Logger +} + +func newTriggerStats( + logger moira.Logger, + database moira.Database, + metricsRegistry metrics.Registry, +) *triggerStats { + return &triggerStats{ + logger: logger, + database: database, + metrics: metrics.NewTriggersMetrics(metricsRegistry), + } +} + +func (stats *triggerStats) Start() { + stats.tomb.Go(stats.startCheckingTriggerCount) +} + +func (stats *triggerStats) startCheckingTriggerCount() error { + checkTicker := time.NewTicker(time.Second * 60) + for { + select { + case <-stats.tomb.Dying(): + return nil + + case <-checkTicker.C: + stats.checkTriggerCount() + } + } +} + +func (stats *triggerStats) Stop() error { + stats.tomb.Kill(nil) + return stats.tomb.Wait() +} + +func (stats *triggerStats) checkTriggerCount() { + triggersCount, err := stats.database.GetTriggerCount() + if err != nil { + stats.logger.Warning(). + Error(err). + Msg("Failed to fetch triggers count") + return + } + + for source, count := range triggersCount { + stats.metrics.Mark(source, count) + stats.logger.Debug().Msg(fmt.Sprintf("source: %s, count: %d", string(source), count)) + } +} diff --git a/cmd/api/trigger_stats_test.go b/cmd/api/trigger_stats_test.go new file mode 100644 index 000000000..2186bd006 --- /dev/null +++ b/cmd/api/trigger_stats_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/logging/zerolog_adapter" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + mock_metrics "github.com/moira-alert/moira/mock/moira-alert/metrics" + . "github.com/smartystreets/goconvey/convey" +) + +func TestTriggerStatsCheckTriggerCount(t *testing.T) { + Convey("Given db returns correct results", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + registry := mock_metrics.NewMockRegistry(mockCtrl) + + graphiteLocalCount := int64(12) + graphiteRemoteCount := int64(24) + promethteusRemoteCount := int64(42) + + graphiteLocalMeter := mock_metrics.NewMockMeter(mockCtrl) + graphiteRemoteMeter := mock_metrics.NewMockMeter(mockCtrl) + promethteusRemoteMeter := mock_metrics.NewMockMeter(mockCtrl) + + registry.EXPECT().NewMeter("triggers", string(moira.GraphiteLocal), "count").Return(graphiteLocalMeter) + registry.EXPECT().NewMeter("triggers", string(moira.GraphiteRemote), "count").Return(graphiteRemoteMeter) + registry.EXPECT().NewMeter("triggers", string(moira.PrometheusRemote), "count").Return(promethteusRemoteMeter) + + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) + dataBase.EXPECT().GetTriggerCount().Return(map[moira.TriggerSource]int64{ + moira.GraphiteLocal: graphiteLocalCount, + moira.GraphiteRemote: graphiteRemoteCount, + moira.PrometheusRemote: promethteusRemoteCount, + }, nil) + + graphiteLocalMeter.EXPECT().Mark(graphiteLocalCount) + graphiteRemoteMeter.EXPECT().Mark(graphiteRemoteCount) + promethteusRemoteMeter.EXPECT().Mark(promethteusRemoteCount) + + logger, _ := zerolog_adapter.GetLogger("Test") + triggerStats := newTriggerStats(logger, dataBase, registry) + + triggerStats.checkTriggerCount() + }) +} diff --git a/database/redis/trigger.go b/database/redis/trigger.go index 34334b205..b8868dbdd 100644 --- a/database/redis/trigger.go +++ b/database/redis/trigger.go @@ -52,6 +52,38 @@ func (connector *DbConnector) GetPrometheusTriggerIDs() ([]string, error) { return triggerIds, nil } +func (connector *DbConnector) GetTriggerCount() (map[moira.TriggerSource]int64, error) { + pipe := (*connector.client).TxPipeline() + + total := pipe.SCard(connector.context, triggersListKey) + remote := pipe.SCard(connector.context, remoteTriggersListKey) + prometheus := pipe.SCard(connector.context, prometheusTriggersListKey) + + _, err := pipe.Exec(connector.context) + if err != nil { + return nil, err + } + + totalCount, err := total.Result() + if err != nil { + return nil, err + } + remoteCount, err := remote.Result() + if err != nil { + return nil, err + } + prometheusCount, err := prometheus.Result() + if err != nil { + return nil, err + } + + return map[moira.TriggerSource]int64{ + moira.GraphiteLocal: totalCount - remoteCount - prometheusCount, + moira.GraphiteRemote: remoteCount, + moira.PrometheusRemote: prometheusCount, + }, nil +} + // GetTrigger gets trigger and trigger tags by given ID and return it in merged object func (connector *DbConnector) GetTrigger(triggerID string) (moira.Trigger, error) { pipe := (*connector.client).TxPipeline() diff --git a/filter/metrics_parser_test.go b/filter/metrics_parser_test.go index 56775fed3..653d3698b 100644 --- a/filter/metrics_parser_test.go +++ b/filter/metrics_parser_test.go @@ -169,7 +169,7 @@ func TestParseMetric(t *testing.T) { // ... // [n=19] One.two.three 123 1234567890.6790847778320312500 - for i := 1; i < 20; i++ { + for i := 1; i < 7; i++ { rawTimestamp := strconv.FormatFloat(float64(testTimestamp)+rand.Float64(), 'f', i, 64) rawMetric := "One.two.three 123 " + rawTimestamp validMetric := ValidMetricCase{rawMetric, "One.two.three", "One.two.three", map[string]string{}, 123, testTimestamp} diff --git a/generate_mocks.sh b/generate_mocks.sh index 4179da531..81428e29c 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -22,4 +22,7 @@ mockgen -destination=mock/heartbeat/heartbeat.go -package=mock_heartbeat github. mockgen -destination=mock/clock/clock.go -package=mock_clock github.com/moira-alert/moira Clock mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost github.com/moira-alert/moira/senders/mattermost Client +mockgen -destination=mock/moira-alert/metrics/registry.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Registry +mockgen -destination=mock/moira-alert/metrics/meter.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Meter + git add mock/* diff --git a/interfaces.go b/interfaces.go index b98064e29..1831018e4 100644 --- a/interfaces.go +++ b/interfaces.go @@ -38,6 +38,8 @@ type Database interface { GetRemoteTriggerIDs() ([]string, error) GetPrometheusTriggerIDs() ([]string, error) + GetTriggerCount() (map[TriggerSource]int64, error) + GetTrigger(triggerID string) (Trigger, error) GetTriggers(triggerIDs []string) ([]*Trigger, error) GetTriggerChecks(triggerIDs []string) ([]*TriggerCheck, error) diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 4e6200b77..979171a41 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -78,8 +78,9 @@ type prometheusMeter struct { summary prometheus.Summary } -func (source *prometheusMeter) Mark(int64) { +func (source *prometheusMeter) Mark(value int64) { atomic.AddInt64(&source.count, 1) + source.summary.Observe(float64(value)) } func (source *prometheusMeter) Count() int64 { diff --git a/metrics/triggers.go b/metrics/triggers.go new file mode 100644 index 000000000..45cd38c21 --- /dev/null +++ b/metrics/triggers.go @@ -0,0 +1,24 @@ +package metrics + +import "github.com/moira-alert/moira" + +// Collection of metrics for trigger count metrics +type TriggersMetrics struct { + countByTriggerSource map[moira.TriggerSource]Meter +} + +// Creates and configurates the instance of TriggersMetrics +func NewTriggersMetrics(registry Registry) *TriggersMetrics { + return &TriggersMetrics{ + countByTriggerSource: map[moira.TriggerSource]Meter{ + moira.GraphiteLocal: registry.NewMeter("triggers", string(moira.GraphiteLocal), "count"), + moira.GraphiteRemote: registry.NewMeter("triggers", string(moira.GraphiteRemote), "count"), + moira.PrometheusRemote: registry.NewMeter("triggers", string(moira.PrometheusRemote), "count"), + }, + } +} + +// Marks the number of trigger for given trigger source +func (metrics *TriggersMetrics) Mark(source moira.TriggerSource, count int64) { + metrics.countByTriggerSource[source].Mark(count) +} diff --git a/mock/moira-alert/database.go b/mock/moira-alert/database.go index 8cee64161..2d14bb7a8 100644 --- a/mock/moira-alert/database.go +++ b/mock/moira-alert/database.go @@ -891,6 +891,21 @@ func (mr *MockDatabaseMockRecorder) GetTriggerChecks(arg0 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTriggerChecks", reflect.TypeOf((*MockDatabase)(nil).GetTriggerChecks), arg0) } +// GetTriggerCount mocks base method. +func (m *MockDatabase) GetTriggerCount() (map[moira.TriggerSource]int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTriggerCount") + ret0, _ := ret[0].(map[moira.TriggerSource]int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTriggerCount indicates an expected call of GetTriggerCount. +func (mr *MockDatabaseMockRecorder) GetTriggerCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTriggerCount", reflect.TypeOf((*MockDatabase)(nil).GetTriggerCount)) +} + // GetTriggerIDsStartWith mocks base method. func (m *MockDatabase) GetTriggerIDsStartWith(arg0 string) ([]string, error) { m.ctrl.T.Helper() diff --git a/mock/moira-alert/metrics/meter.go b/mock/moira-alert/metrics/meter.go new file mode 100644 index 000000000..dcb902c30 --- /dev/null +++ b/mock/moira-alert/metrics/meter.go @@ -0,0 +1,60 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/moira-alert/moira/metrics (interfaces: Meter) + +// Package mock_moira_alert is a generated GoMock package. +package mock_moira_alert + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockMeter is a mock of Meter interface. +type MockMeter struct { + ctrl *gomock.Controller + recorder *MockMeterMockRecorder +} + +// MockMeterMockRecorder is the mock recorder for MockMeter. +type MockMeterMockRecorder struct { + mock *MockMeter +} + +// NewMockMeter creates a new mock instance. +func NewMockMeter(ctrl *gomock.Controller) *MockMeter { + mock := &MockMeter{ctrl: ctrl} + mock.recorder = &MockMeterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMeter) EXPECT() *MockMeterMockRecorder { + return m.recorder +} + +// Count mocks base method. +func (m *MockMeter) Count() int64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count") + ret0, _ := ret[0].(int64) + return ret0 +} + +// Count indicates an expected call of Count. +func (mr *MockMeterMockRecorder) Count() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockMeter)(nil).Count)) +} + +// Mark mocks base method. +func (m *MockMeter) Mark(arg0 int64) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Mark", arg0) +} + +// Mark indicates an expected call of Mark. +func (mr *MockMeterMockRecorder) Mark(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mark", reflect.TypeOf((*MockMeter)(nil).Mark), arg0) +} diff --git a/mock/moira-alert/metrics/registry.go b/mock/moira-alert/metrics/registry.go new file mode 100644 index 000000000..68ccec8be --- /dev/null +++ b/mock/moira-alert/metrics/registry.go @@ -0,0 +1,107 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/moira-alert/moira/metrics (interfaces: Registry) + +// Package mock_moira_alert is a generated GoMock package. +package mock_moira_alert + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + metrics "github.com/moira-alert/moira/metrics" +) + +// MockRegistry is a mock of Registry interface. +type MockRegistry struct { + ctrl *gomock.Controller + recorder *MockRegistryMockRecorder +} + +// MockRegistryMockRecorder is the mock recorder for MockRegistry. +type MockRegistryMockRecorder struct { + mock *MockRegistry +} + +// NewMockRegistry creates a new mock instance. +func NewMockRegistry(ctrl *gomock.Controller) *MockRegistry { + mock := &MockRegistry{ctrl: ctrl} + mock.recorder = &MockRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRegistry) EXPECT() *MockRegistryMockRecorder { + return m.recorder +} + +// NewCounter mocks base method. +func (m *MockRegistry) NewCounter(arg0 ...string) metrics.Counter { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "NewCounter", varargs...) + ret0, _ := ret[0].(metrics.Counter) + return ret0 +} + +// NewCounter indicates an expected call of NewCounter. +func (mr *MockRegistryMockRecorder) NewCounter(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewCounter", reflect.TypeOf((*MockRegistry)(nil).NewCounter), arg0...) +} + +// NewHistogram mocks base method. +func (m *MockRegistry) NewHistogram(arg0 ...string) metrics.Histogram { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "NewHistogram", varargs...) + ret0, _ := ret[0].(metrics.Histogram) + return ret0 +} + +// NewHistogram indicates an expected call of NewHistogram. +func (mr *MockRegistryMockRecorder) NewHistogram(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewHistogram", reflect.TypeOf((*MockRegistry)(nil).NewHistogram), arg0...) +} + +// NewMeter mocks base method. +func (m *MockRegistry) NewMeter(arg0 ...string) metrics.Meter { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "NewMeter", varargs...) + ret0, _ := ret[0].(metrics.Meter) + return ret0 +} + +// NewMeter indicates an expected call of NewMeter. +func (mr *MockRegistryMockRecorder) NewMeter(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewMeter", reflect.TypeOf((*MockRegistry)(nil).NewMeter), arg0...) +} + +// NewTimer mocks base method. +func (m *MockRegistry) NewTimer(arg0 ...string) metrics.Timer { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "NewTimer", varargs...) + ret0, _ := ret[0].(metrics.Timer) + return ret0 +} + +// NewTimer indicates an expected call of NewTimer. +func (mr *MockRegistryMockRecorder) NewTimer(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewTimer", reflect.TypeOf((*MockRegistry)(nil).NewTimer), arg0...) +} From 84c743471f2518c330f635da26628e11c9a3bf32 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Wed, 6 Sep 2023 13:57:28 +0600 Subject: [PATCH 15/46] fix(db|helpers) little fix in contact anf helpers (#905) --- database/redis/contact.go | 4 ++-- helpers.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/database/redis/contact.go b/database/redis/contact.go index ebf5d735b..7521b0680 100644 --- a/database/redis/contact.go +++ b/database/redis/contact.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/go-redis/redis/v8" "github.com/moira-alert/moira" @@ -99,8 +100,7 @@ func (connector *DbConnector) GetAllContacts() ([]*moira.ContactData, error) { contactIDs := make([]string, 0, len(keys)) for _, key := range keys { - key = key[14:] - contactIDs = append(contactIDs, key) + contactIDs = append(contactIDs, strings.TrimPrefix(key, contactKey(""))) } return connector.GetContacts(contactIDs) } diff --git a/helpers.go b/helpers.go index 946ddfc78..9a999fb7a 100644 --- a/helpers.go +++ b/helpers.go @@ -205,16 +205,16 @@ func MaxInt64(a, b int64) int64 { return b } -// ReplaceSubstring removes one substring between the beg and end substrings and replaces it with a rep -func ReplaceSubstring(str, beg, end, rep string) string { +// ReplaceSubstring removes one substring between the beginning and end substrings and replaces it with a replaced +func ReplaceSubstring(str, begin, end, replaced string) string { result := str - startIndex := strings.Index(str, beg) + startIndex := strings.Index(str, begin) if startIndex != -1 { - startIndex += len(beg) + startIndex += len(begin) endIndex := strings.Index(str[startIndex:], end) if endIndex != -1 { endIndex += len(str[:startIndex]) - result = str[:startIndex] + rep + str[endIndex:] + result = str[:startIndex] + replaced + str[endIndex:] } } return result From 6c3310ebb16c41c9eb4f069472bea30ee59be822 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Mon, 11 Sep 2023 16:33:39 +0300 Subject: [PATCH 16/46] feat Add prometheus retries (#909) --- cmd/api/config.go | 6 + cmd/api/config_test.go | 6 + cmd/checker/config.go | 2 + cmd/config.go | 40 ++++-- cmd/notifier/config.go | 6 + generate_mocks.sh | 1 + metric_source/prometheus/fetch.go | 32 ++++- metric_source/prometheus/fetch_test.go | 144 +++++++++++++++++++++ metric_source/prometheus/prometheus.go | 20 +-- metric_source/prometheus/prometheus_api.go | 12 +- mock/moira-alert/prometheus_api.go | 58 +++++++++ 11 files changed, 298 insertions(+), 29 deletions(-) create mode 100644 metric_source/prometheus/fetch_test.go create mode 100644 mock/moira-alert/prometheus_api.go diff --git a/cmd/api/config.go b/cmd/api/config.go index 032d22a73..50ec41623 100644 --- a/cmd/api/config.go +++ b/cmd/api/config.go @@ -146,5 +146,11 @@ func getDefault() config { Timeout: "60s", MetricsTTL: "7d", }, + Prometheus: cmd.PrometheusConfig{ + Timeout: "60s", + MetricsTTL: "7d", + Retries: 1, + RetryTimeout: "10s", + }, } } diff --git a/cmd/api/config_test.go b/cmd/api/config_test.go index 23969361f..75cd644ab 100644 --- a/cmd/api/config_test.go +++ b/cmd/api/config_test.go @@ -92,6 +92,12 @@ func Test_webConfig_getDefault(t *testing.T) { Timeout: "60s", MetricsTTL: "7d", }, + Prometheus: cmd.PrometheusConfig{ + Timeout: "60s", + MetricsTTL: "7d", + Retries: 1, + RetryTimeout: "10s", + }, NotificationHistory: cmd.NotificationHistoryConfig{ NotificationHistoryTTL: "48h", NotificationHistoryQueryLimit: -1, diff --git a/cmd/checker/config.go b/cmd/checker/config.go index 4782b3cc2..149de43a3 100644 --- a/cmd/checker/config.go +++ b/cmd/checker/config.go @@ -109,6 +109,8 @@ func getDefault() config { CheckInterval: "60s", Timeout: "60s", MetricsTTL: "7d", + Retries: 1, + RetryTimeout: "10s", }, } } diff --git a/cmd/config.go b/cmd/config.go index 54504735c..267db3edc 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -157,25 +157,39 @@ func (config *RemoteConfig) GetRemoteSourceSettings() *remoteSource.Config { } type PrometheusConfig struct { - URL string `yaml:"url"` + // Url of prometheus API + URL string `yaml:"url"` + // Min period to perform triggers re-check CheckInterval string `yaml:"check_interval"` - MetricsTTL string `yaml:"metrics_ttl"` - Timeout string `yaml:"timeout"` - User string `yaml:"user"` - Password string `yaml:"password"` - Enabled bool `yaml:"enabled"` + // Moira won't fetch metrics older than this value from prometheus remote storage. + // Large values will lead to OOM problems in checker. + MetricsTTL string `yaml:"metrics_ttl"` + // Timeout for prometheus api requests + Timeout string `yaml:"timeout"` + // Number of retries for prometheus api requests + Retries int `yaml:"retries"` + // Timeout between retries for prometheus api requests + RetryTimeout string `yaml:"retry_timeout"` + // Username for basic auth + User string `yaml:"user"` + // Password for basic auth + Password string `yaml:"password"` + // If true, prometheus remote worker will be enabled. + Enabled bool `yaml:"enabled"` } // GetRemoteSourceSettings returns remote config parsed from moira config files func (config *PrometheusConfig) GetPrometheusSourceSettings() *prometheus.Config { return &prometheus.Config{ - Enabled: config.Enabled, - URL: config.URL, - CheckInterval: to.Duration(config.CheckInterval), - MetricsTTL: to.Duration(config.MetricsTTL), - User: config.User, - Password: config.Password, - Timeout: to.Duration(config.Timeout), + Enabled: config.Enabled, + URL: config.URL, + CheckInterval: to.Duration(config.CheckInterval), + MetricsTTL: to.Duration(config.MetricsTTL), + User: config.User, + Password: config.Password, + RequestTimeout: to.Duration(config.Timeout), + Retries: config.Retries, + RetryTimeout: to.Duration(config.RetryTimeout), } } diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index 2af6f6203..ebdecf65d 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -121,6 +121,12 @@ func getDefault() config { Timeout: "60s", MetricsTTL: "24h", }, + Prometheus: cmd.PrometheusConfig{ + Timeout: "60s", + MetricsTTL: "7d", + Retries: 1, + RetryTimeout: "10s", + }, ImageStores: cmd.ImageStoreConfig{}, } } diff --git a/generate_mocks.sh b/generate_mocks.sh index 81428e29c..43e6fd982 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -24,5 +24,6 @@ mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost mockgen -destination=mock/moira-alert/metrics/registry.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Registry mockgen -destination=mock/moira-alert/metrics/meter.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Meter +mockgen -destination=mock/moira-alert/prometheus_api.go -package=mock_moira_alert github.com/moira-alert/moira/metric_source/prometheus PrometheusApi git add mock/* diff --git a/metric_source/prometheus/fetch.go b/metric_source/prometheus/fetch.go index f2a0b33d7..f1dfceba6 100644 --- a/metric_source/prometheus/fetch.go +++ b/metric_source/prometheus/fetch.go @@ -8,17 +8,43 @@ import ( metricSource "github.com/moira-alert/moira/metric_source" "github.com/moira-alert/moira" - prometheusApi "github.com/prometheus/client_golang/api/prometheus/v1" + promApi "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" ) func (prometheus *Prometheus) Fetch(target string, from, until int64, allowRealTimeAlerting bool) (metricSource.FetchResult, error) { from = moira.MaxInt64(from, until-int64(prometheus.config.MetricsTTL.Seconds())) - ctx, cancel := context.WithTimeout(context.Background(), prometheus.config.Timeout) + var err error + for i := 1; ; i++ { + var res metricSource.FetchResult + res, err = prometheus.fetch(target, from, until, allowRealTimeAlerting) + + if err == nil { + return res, nil + } + + prometheus.logger.Warning(). + Error(err). + Int("retries left", prometheus.config.Retries-i). + String("target", target). + Msg("Failed to fetch prometheus target") + + if i >= prometheus.config.Retries { + break + } + + time.Sleep(prometheus.config.RetryTimeout) + } + + return nil, err +} + +func (prometheus *Prometheus) fetch(target string, from, until int64, allowRealTimeAlerting bool) (metricSource.FetchResult, error) { + ctx, cancel := context.WithTimeout(context.Background(), prometheus.config.RequestTimeout) defer cancel() - val, warns, err := prometheus.api.QueryRange(ctx, target, prometheusApi.Range{ + val, warns, err := prometheus.api.QueryRange(ctx, target, promApi.Range{ Start: time.Unix(from, 0), End: time.Unix(until, 0), Step: time.Second * time.Duration(StepTimeSeconds), diff --git a/metric_source/prometheus/fetch_test.go b/metric_source/prometheus/fetch_test.go new file mode 100644 index 000000000..960c66bdd --- /dev/null +++ b/metric_source/prometheus/fetch_test.go @@ -0,0 +1,144 @@ +package prometheus + +import ( + "fmt" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/moira-alert/moira/logging/zerolog_adapter" + metricsource "github.com/moira-alert/moira/metric_source" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + + "github.com/prometheus/common/model" + . "github.com/smartystreets/goconvey/convey" +) + +func TestPrometheusFetch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + api := mock_moira_alert.NewMockPrometheusApi(ctrl) + + logger, _ := zerolog_adapter.GetLogger("Test") + + now := time.Now() + + prometheus := Prometheus{ + config: &Config{}, + api: api, + logger: logger, + } + + Convey("Given two metric points", t, func() { + fromMilli := now.UnixMilli() + untilMilli := now.Add(time.Second * 60).UnixMilli() + + from := now.Unix() + until := now.Add(time.Second * 60).Unix() + + api.EXPECT().QueryRange(gomock.Any(), "target", gomock.Any()). + Return( + model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{"__name__": "name1"}, + Values: []model.SamplePair{ + {Timestamp: model.Time(fromMilli), Value: 3.14}, + {Timestamp: model.Time(untilMilli), Value: 2.71}, + }, + }, + }, + nil, + nil, + ) + + res, err := prometheus.fetch("target", fromMilli, untilMilli, true) + + So(err, ShouldBeNil) + So(res, ShouldResemble, &FetchResult{ + MetricsData: []metricsource.MetricData{ + { + Name: "name1", + StartTime: from, + StopTime: until, + StepTime: 60, + Values: []float64{3.14, 2.71}, + Wildcard: false, + }, + }, + }) + }) +} + +func TestPrometheusFetchRetries(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + api := mock_moira_alert.NewMockPrometheusApi(ctrl) + + logger, _ := zerolog_adapter.GetLogger("Test") + + now := time.Now() + + prometheus := Prometheus{ + config: &Config{Retries: 3, RetryTimeout: time.Second * 0}, + api: api, + logger: logger, + } + + Convey("Given two metric points and two fails", t, func() { + fromMilli := now.UnixMilli() + untilMilli := now.Add(time.Second * 60).UnixMilli() + + from := now.Unix() + until := now.Add(time.Second * 60).Unix() + + api.EXPECT().QueryRange(gomock.Any(), "target", gomock.Any()). + Return(nil, nil, fmt.Errorf("Error")).Times(2) + api.EXPECT().QueryRange(gomock.Any(), "target", gomock.Any()). + Return( + model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{"__name__": "name1"}, + Values: []model.SamplePair{ + {Timestamp: model.Time(fromMilli), Value: 3.14}, + {Timestamp: model.Time(untilMilli), Value: 2.71}, + }, + }, + }, + nil, + nil, + ) + + res, err := prometheus.Fetch("target", fromMilli, untilMilli, true) + + So(err, ShouldBeNil) + So(res, ShouldResemble, &FetchResult{ + MetricsData: []metricsource.MetricData{ + { + Name: "name1", + StartTime: from, + StopTime: until, + StepTime: 60, + Values: []float64{3.14, 2.71}, + Wildcard: false, + }, + }, + }) + }) + + Convey("Given all requests failed", t, func() { + fromMilli := now.UnixMilli() + untilMilli := now.Add(time.Second * 60).UnixMilli() + + expectedErr := fmt.Errorf("Error") + + api.EXPECT().QueryRange(gomock.Any(), "target", gomock.Any()). + Return(nil, nil, expectedErr).Times(3) + + res, err := prometheus.Fetch("target", fromMilli, untilMilli, true) + + So(res, ShouldBeNil) + So(err, ShouldEqual, expectedErr) + }) +} diff --git a/metric_source/prometheus/prometheus.go b/metric_source/prometheus/prometheus.go index b9b9e0b78..cc80dcebc 100644 --- a/metric_source/prometheus/prometheus.go +++ b/metric_source/prometheus/prometheus.go @@ -6,8 +6,6 @@ import ( "github.com/moira-alert/moira" metricSource "github.com/moira-alert/moira/metric_source" - - prometheusApi "github.com/prometheus/client_golang/api/prometheus/v1" ) const StepTimeSeconds int64 = 60 @@ -15,13 +13,15 @@ const StepTimeSeconds int64 = 60 var ErrPrometheusStorageDisabled = fmt.Errorf("remote prometheus storage is not enabled") type Config struct { - Enabled bool - CheckInterval time.Duration - MetricsTTL time.Duration - Timeout time.Duration - URL string - User string - Password string + Enabled bool + CheckInterval time.Duration + MetricsTTL time.Duration + RequestTimeout time.Duration + Retries int + RetryTimeout time.Duration + URL string + User string + Password string } func Create(config *Config, logger moira.Logger) (metricSource.MetricSource, error) { @@ -36,7 +36,7 @@ func Create(config *Config, logger moira.Logger) (metricSource.MetricSource, err type Prometheus struct { config *Config logger moira.Logger - api prometheusApi.API + api PrometheusApi } func (prometheus *Prometheus) GetMetricsTTLSeconds() int64 { diff --git a/metric_source/prometheus/prometheus_api.go b/metric_source/prometheus/prometheus_api.go index a979a4a02..70bdeeb11 100644 --- a/metric_source/prometheus/prometheus_api.go +++ b/metric_source/prometheus/prometheus_api.go @@ -1,15 +1,21 @@ package prometheus import ( + "context" "encoding/base64" "fmt" "github.com/prometheus/client_golang/api" - prometheusApi "github.com/prometheus/client_golang/api/prometheus/v1" + promApi "github.com/prometheus/client_golang/api/prometheus/v1" promConfig "github.com/prometheus/common/config" + "github.com/prometheus/common/model" ) -func createPrometheusApi(config *Config) (prometheusApi.API, error) { +type PrometheusApi interface { + QueryRange(ctx context.Context, query string, r promApi.Range, opts ...promApi.Option) (model.Value, promApi.Warnings, error) +} + +func createPrometheusApi(config *Config) (promApi.API, error) { roundTripper := api.DefaultRoundTripper if config.User != "" && config.Password != "" { @@ -33,5 +39,5 @@ func createPrometheusApi(config *Config) (prometheusApi.API, error) { return nil, err } - return prometheusApi.NewAPI(promCl), nil + return promApi.NewAPI(promCl), nil } diff --git a/mock/moira-alert/prometheus_api.go b/mock/moira-alert/prometheus_api.go new file mode 100644 index 000000000..5adf72ef0 --- /dev/null +++ b/mock/moira-alert/prometheus_api.go @@ -0,0 +1,58 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/moira-alert/moira/metric_source/prometheus (interfaces: PrometheusApi) + +// Package mock_moira_alert is a generated GoMock package. +package mock_moira_alert + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + model "github.com/prometheus/common/model" +) + +// MockPrometheusApi is a mock of PrometheusApi interface. +type MockPrometheusApi struct { + ctrl *gomock.Controller + recorder *MockPrometheusApiMockRecorder +} + +// MockPrometheusApiMockRecorder is the mock recorder for MockPrometheusApi. +type MockPrometheusApiMockRecorder struct { + mock *MockPrometheusApi +} + +// NewMockPrometheusApi creates a new mock instance. +func NewMockPrometheusApi(ctrl *gomock.Controller) *MockPrometheusApi { + mock := &MockPrometheusApi{ctrl: ctrl} + mock.recorder = &MockPrometheusApiMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPrometheusApi) EXPECT() *MockPrometheusApiMockRecorder { + return m.recorder +} + +// QueryRange mocks base method. +func (m *MockPrometheusApi) QueryRange(arg0 context.Context, arg1 string, arg2 v1.Range, arg3 ...v1.Option) (model.Value, v1.Warnings, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QueryRange", varargs...) + ret0, _ := ret[0].(model.Value) + ret1, _ := ret[1].(v1.Warnings) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// QueryRange indicates an expected call of QueryRange. +func (mr *MockPrometheusApiMockRecorder) QueryRange(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryRange", reflect.TypeOf((*MockPrometheusApi)(nil).QueryRange), varargs...) +} From 8e3367e4866a158a0b4ab833aa90ab9e06a9ad5e Mon Sep 17 00:00:00 2001 From: Xenia N Date: Thu, 14 Sep 2023 14:29:04 +0600 Subject: [PATCH 17/46] add metric in notifier-selfcheck (#910) --- cmd/notifier/config.go | 9 +++++ cmd/notifier/main.go | 15 +++++--- metrics/heartbeat.go | 22 ++++++++++++ notifier/notifications/notifications.go | 2 +- notifier/selfstate/check.go | 36 +++++++++---------- notifier/selfstate/config.go | 3 ++ notifier/selfstate/heartbeat/notifier.go | 25 +++++++++---- notifier/selfstate/heartbeat/notifier_test.go | 5 ++- notifier/selfstate/selfstate.go | 12 +++---- notifier/selfstate/selfstate_test.go | 8 +++-- 10 files changed, 99 insertions(+), 38 deletions(-) create mode 100644 metrics/heartbeat.go diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index ebdecf65d..04d0d9a2f 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -73,6 +73,8 @@ type selfStateConfig struct { Contacts []map[string]string `yaml:"contacts"` // Self state monitor alerting interval NoticeInterval string `yaml:"notice_interval"` + // Self state monitor check interval + CheckInterval string `yaml:"check_interval"` } func getDefault() config { @@ -212,12 +214,19 @@ func checkDateTimeFormat(format string) error { } func (config *selfStateConfig) getSettings() selfstate.Config { + // 10 sec is default check value + checkInterval := 10 * time.Second + if config.CheckInterval != "" { + checkInterval = to.Duration(config.CheckInterval) + } + return selfstate.Config{ Enabled: config.Enabled, RedisDisconnectDelaySeconds: int64(to.Duration(config.RedisDisconnectDelay).Seconds()), LastMetricReceivedDelaySeconds: int64(to.Duration(config.LastMetricReceivedDelay).Seconds()), LastCheckDelaySeconds: int64(to.Duration(config.LastCheckDelay).Seconds()), LastRemoteCheckDelaySeconds: int64(to.Duration(config.LastRemoteCheckDelay).Seconds()), + CheckInterval: checkInterval, Contacts: config.Contacts, NoticeIntervalSeconds: int64(to.Duration(config.NoticeInterval).Seconds()), } diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 4ecf4155d..8a75bb410 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -69,7 +69,7 @@ func main() { } defer logger.Info(). String("moira_version", MoiraVersion). - Msg("Moira Notifier stopped. Version") + Msg("Moira Notifier stopped.") telemetry, err := cmd.ConfigureTelemetry(logger, config.Telemetry, serviceName) if err != nil { @@ -79,7 +79,6 @@ func main() { } defer telemetry.Stop() - notifierMetrics := metrics.ConfigureNotifierMetrics(telemetry.Metrics, serviceName) databaseSettings := config.Redis.GetSettings() notificationHistorySettings := config.NotificationHistory.GetSettings() database := redis.NewDatabase(logger, databaseSettings, notificationHistorySettings, redis.Notifier) @@ -103,7 +102,15 @@ func main() { notifierConfig := config.Notifier.getSettings(logger) - sender := notifier.NewNotifier(database, logger, notifierConfig, notifierMetrics, metricSourceProvider, imageStoreMap) + notifierMetrics := metrics.ConfigureNotifierMetrics(telemetry.Metrics, serviceName) + sender := notifier.NewNotifier( + database, + logger, + notifierConfig, + notifierMetrics, + metricSourceProvider, + imageStoreMap, + ) // Register moira senders if err := sender.RegisterSenders(database); err != nil { @@ -114,7 +121,7 @@ func main() { // Start moira self state checker if config.Notifier.SelfState.getSettings().Enabled { - selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings()) + selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings(), metrics.ConfigureHeartBeatMetrics(telemetry.Metrics)) if err := selfState.Start(); err != nil { logger.Fatal(). Error(err). diff --git a/metrics/heartbeat.go b/metrics/heartbeat.go new file mode 100644 index 000000000..d0eedbbd9 --- /dev/null +++ b/metrics/heartbeat.go @@ -0,0 +1,22 @@ +package metrics + +// HeartBeatMetrics is a collection of metrics used in hearbeats +type HeartBeatMetrics struct { + notifierIsAlive Meter +} + +// ConfigureHeartBeatMetrics is notifier metrics configurator +func ConfigureHeartBeatMetrics(registry Registry) *HeartBeatMetrics { + return &HeartBeatMetrics{ + notifierIsAlive: registry.NewMeter("", "alive"), + } +} + +// MarkNotifierIsAlive marks metric value. +func (hb HeartBeatMetrics) MarkNotifierIsAlive(isAlive bool) { + if isAlive { + hb.notifierIsAlive.Mark(1) + } + + hb.notifierIsAlive.Mark(0) +} diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 8f09b3668..7c573aad1 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -36,7 +36,7 @@ func (worker *FetchNotificationsWorker) Start() { switch err.(type) { case notifierInBadStateError: worker.Logger.Warning(). - String("stop_sending_notofocations_for", sleepAfterNotifierBadState.String()). + String("stop_sending_notifications_for", sleepAfterNotifierBadState.String()). Error(err). Msg("Stop sending notifications for some time. Fix SelfState errors and turn on notifier in /notifications page") <-time.After(sleepAfterNotifierBadState) diff --git a/notifier/selfstate/check.go b/notifier/selfstate/check.go index 489b7f667..9b2ad2fe5 100644 --- a/notifier/selfstate/check.go +++ b/notifier/selfstate/check.go @@ -12,7 +12,7 @@ import ( func (selfCheck *SelfCheckWorker) selfStateChecker(stop <-chan struct{}) error { selfCheck.Logger.Info().Msg("Moira Notifier Self State Monitor started") - checkTicker := time.NewTicker(defaultCheckInterval) + checkTicker := time.NewTicker(selfCheck.Config.CheckInterval) defer checkTicker.Stop() nextSendErrorMessage := time.Now().Unix() @@ -23,33 +23,35 @@ func (selfCheck *SelfCheckWorker) selfStateChecker(stop <-chan struct{}) error { selfCheck.Logger.Info().Msg("Moira Notifier Self State Monitor stopped") return nil case <-checkTicker.C: + selfCheck.Logger.Debug(). + Int64("nextSendErrorMessage", nextSendErrorMessage). + Msg("call check") + nextSendErrorMessage = selfCheck.check(time.Now().Unix(), nextSendErrorMessage) } } } func (selfCheck *SelfCheckWorker) handleCheckServices(nowTS int64) []moira.NotificationEvent { - var events []moira.NotificationEvent //nolint + var events []moira.NotificationEvent for _, heartbeat := range selfCheck.heartbeats { - currentValue, needSend, err := heartbeat.Check(nowTS) + currentValue, hasErrors, err := heartbeat.Check(nowTS) if err != nil { selfCheck.Logger.Error(). Error(err). Msg("Heartbeat failed") } - if !needSend { - continue - } + if hasErrors { + events = append(events, generateNotificationEvent(heartbeat.GetErrorMessage(), currentValue)) + if heartbeat.NeedTurnOffNotifier() { + selfCheck.setNotifierState(moira.SelfStateERROR) + } - events = append(events, generateNotificationEvent(heartbeat.GetErrorMessage(), currentValue)) - if heartbeat.NeedTurnOffNotifier() { - selfCheck.setNotifierState(moira.SelfStateERROR) - } - - if !heartbeat.NeedToCheckOthers() { - break + if !heartbeat.NeedToCheckOthers() { + break + } } } @@ -67,11 +69,9 @@ func (selfCheck *SelfCheckWorker) sendNotification(events []moira.NotificationEv } func (selfCheck *SelfCheckWorker) check(nowTS int64, nextSendErrorMessage int64) int64 { - if nextSendErrorMessage < nowTS { - events := selfCheck.handleCheckServices(nowTS) - if len(events) > 0 { - nextSendErrorMessage = selfCheck.sendNotification(events, nowTS) - } + events := selfCheck.handleCheckServices(nowTS) + if nextSendErrorMessage < nowTS && len(events) > 0 { + nextSendErrorMessage = selfCheck.sendNotification(events, nowTS) } return nextSendErrorMessage diff --git a/notifier/selfstate/config.go b/notifier/selfstate/config.go index 8f1a6f92f..ffd4dbd38 100644 --- a/notifier/selfstate/config.go +++ b/notifier/selfstate/config.go @@ -2,6 +2,7 @@ package selfstate import ( "fmt" + "time" ) // Config is representation of self state worker settings like moira admins contacts and threshold values for checked services @@ -12,6 +13,7 @@ type Config struct { LastCheckDelaySeconds int64 LastRemoteCheckDelaySeconds int64 NoticeIntervalSeconds int64 + CheckInterval time.Duration Contacts []map[string]string } @@ -30,5 +32,6 @@ func (config *Config) checkConfig(senders map[string]bool) error { return fmt.Errorf("value for [%s] must be present", adminContact["type"]) } } + return nil } diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index d63cd7660..e708955cc 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -3,29 +3,42 @@ package heartbeat import ( "fmt" + "github.com/moira-alert/moira/metrics" + "github.com/moira-alert/moira" ) type notifier struct { - db moira.Database - log moira.Logger + db moira.Database + log moira.Logger + metrics *metrics.HeartBeatMetrics } -func GetNotifier(logger moira.Logger, database moira.Database) Heartbeater { +func GetNotifier(logger moira.Logger, database moira.Database, metrics *metrics.HeartBeatMetrics) Heartbeater { return ¬ifier{ - db: database, - log: logger, + db: database, + log: logger, + metrics: metrics, } } func (check notifier) Check(int64) (int64, bool, error) { - if state, _ := check.db.GetNotifierState(); state != moira.SelfStateOK { + state, _ := check.db.GetNotifierState() + if state != moira.SelfStateOK { + check.metrics.MarkNotifierIsAlive(true) + check.log.Error(). String("error", check.GetErrorMessage()). Msg("Notifier is not healthy") return 0, true, nil } + check.metrics.MarkNotifierIsAlive(false) + + check.log.Debug(). + String("state", state). + Msg("Notifier is healthy") + return 0, false, nil } diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 1e865737d..8801957f6 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -4,6 +4,8 @@ import ( "testing" "time" + "github.com/moira-alert/moira/metrics" + "github.com/moira-alert/moira" mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" @@ -45,6 +47,7 @@ func TestNotifierState(t *testing.T) { func createNotifierStateTest(t *testing.T) *notifier { mockCtrl := gomock.NewController(t) logger, _ := logging.GetLogger("MetricDelay") + metric := metrics.ConfigureHeartBeatMetrics(metrics.NewDummyRegistry()) - return GetNotifier(logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*notifier) + return GetNotifier(logger, mock_moira_alert.NewMockDatabase(mockCtrl), metric).(*notifier) } diff --git a/notifier/selfstate/selfstate.go b/notifier/selfstate/selfstate.go index 98af88c66..b44e9f6e4 100644 --- a/notifier/selfstate/selfstate.go +++ b/notifier/selfstate/selfstate.go @@ -3,6 +3,8 @@ package selfstate import ( "time" + "github.com/moira-alert/moira/metrics" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" "gopkg.in/tomb.v2" @@ -12,8 +14,6 @@ import ( w "github.com/moira-alert/moira/worker" ) -var defaultCheckInterval = time.Second * 10 - const selfStateLockName = "moira-self-state-monitor" const selfStateLockTTL = time.Second * 15 @@ -28,8 +28,8 @@ type SelfCheckWorker struct { } // NewSelfCheckWorker creates SelfCheckWorker. -func NewSelfCheckWorker(logger moira.Logger, database moira.Database, notifier notifier.Notifier, config Config) *SelfCheckWorker { - heartbeats := createStandardHeartbeats(logger, database, config) +func NewSelfCheckWorker(logger moira.Logger, database moira.Database, notifier notifier.Notifier, config Config, metrics *metrics.HeartBeatMetrics) *SelfCheckWorker { + heartbeats := createStandardHeartbeats(logger, database, config, metrics) return &SelfCheckWorker{Logger: logger, Database: database, Notifier: notifier, Config: config, heartbeats: heartbeats} } @@ -59,7 +59,7 @@ func (selfCheck *SelfCheckWorker) Stop() error { return selfCheck.tomb.Wait() } -func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf Config) []heartbeat.Heartbeater { +func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf Config, metrics *metrics.HeartBeatMetrics) []heartbeat.Heartbeater { heartbeats := make([]heartbeat.Heartbeater, 0) if hb := heartbeat.GetDatabase(conf.RedisDisconnectDelaySeconds, logger, database); hb != nil { @@ -78,7 +78,7 @@ func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf heartbeats = append(heartbeats, hb) } - if hb := heartbeat.GetNotifier(logger, database); hb != nil { + if hb := heartbeat.GetNotifier(logger, database, metrics); hb != nil { heartbeats = append(heartbeats, hb) } diff --git a/notifier/selfstate/selfstate_test.go b/notifier/selfstate/selfstate_test.go index c3c6ad335..fcee9a219 100644 --- a/notifier/selfstate/selfstate_test.go +++ b/notifier/selfstate/selfstate_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/moira-alert/moira/metrics" + mock_heartbeat "github.com/moira-alert/moira/mock/heartbeat" "github.com/moira-alert/moira/notifier/selfstate/heartbeat" @@ -143,7 +145,6 @@ func configureWorker(t *testing.T, isStart bool) *selfCheckWorkerMock { "type": "admin-mail", "value": "admin@company.com", } - defaultCheckInterval = time.Second * 1 conf := Config{ Enabled: true, Contacts: []map[string]string{ @@ -154,6 +155,7 @@ func configureWorker(t *testing.T, isStart bool) *selfCheckWorkerMock { LastCheckDelaySeconds: 120, NoticeIntervalSeconds: 60, LastRemoteCheckDelaySeconds: 120, + CheckInterval: 1 * time.Second, } mockCtrl := gomock.NewController(t) @@ -172,9 +174,11 @@ func configureWorker(t *testing.T, isStart bool) *selfCheckWorkerMock { database.EXPECT().NewLock(gomock.Any(), gomock.Any()).Return(lock) } + metric := &metrics.HeartBeatMetrics{} + return &selfCheckWorkerMock{ - selfCheckWorker: NewSelfCheckWorker(logger, database, notif, conf), + selfCheckWorker: NewSelfCheckWorker(logger, database, notif, conf, metric), database: database, notif: notif, conf: conf, From 2d493def488180cceb0a6aa0ad78a73dd31d2733 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Thu, 14 Sep 2023 14:52:19 +0600 Subject: [PATCH 18/46] fix(notifier): fix metric in notifier-selfcheck (#914) --- notifier/selfstate/heartbeat/notifier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index e708955cc..7b6877be4 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -25,7 +25,7 @@ func GetNotifier(logger moira.Logger, database moira.Database, metrics *metrics. func (check notifier) Check(int64) (int64, bool, error) { state, _ := check.db.GetNotifierState() if state != moira.SelfStateOK { - check.metrics.MarkNotifierIsAlive(true) + check.metrics.MarkNotifierIsAlive(false) check.log.Error(). String("error", check.GetErrorMessage()). @@ -33,7 +33,7 @@ func (check notifier) Check(int64) (int64, bool, error) { return 0, true, nil } - check.metrics.MarkNotifierIsAlive(false) + check.metrics.MarkNotifierIsAlive(true) check.log.Debug(). String("state", state). From 6b96719ffebf0c58654cdc0fab907a152885b8b7 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Mon, 18 Sep 2023 12:22:03 +0600 Subject: [PATCH 19/46] add api for get unused triggers (#915) --- api/controller/triggers.go | 69 ++++++++++++++++++++++++--------- api/controller/triggers_test.go | 42 ++++++++++++++++++++ api/handler/triggers.go | 24 ++++++++++++ 3 files changed, 117 insertions(+), 18 deletions(-) diff --git a/api/controller/triggers.go b/api/controller/triggers.go index 7f0ff3b80..cd39f7eb3 100644 --- a/api/controller/triggers.go +++ b/api/controller/triggers.go @@ -10,7 +10,7 @@ import ( "github.com/moira-alert/moira" "github.com/moira-alert/moira/api" "github.com/moira-alert/moira/api/dto" - "github.com/moira-alert/moira/database" + db "github.com/moira-alert/moira/database" ) const pageSizeUnlimited int64 = -1 @@ -50,19 +50,16 @@ func GetAllTriggers(database moira.Database) (*dto.TriggersList, *api.ErrorRespo if err != nil { return nil, api.ErrorInternalServer(err) } - triggerChecks, err := database.GetTriggerChecks(triggerIDs) + + triggerChecks, err := getTriggerChecks(database, triggerIDs) if err != nil { return nil, api.ErrorInternalServer(err) } - triggersList := dto.TriggersList{ - List: make([]moira.TriggerCheck, 0), + triggersList := &dto.TriggersList{ + List: triggerChecks, } - for _, triggerCheck := range triggerChecks { - if triggerCheck != nil { - triggersList.List = append(triggersList.List, *triggerCheck) - } - } - return &triggersList, nil + + return triggersList, nil } // SearchTriggers gets trigger page and filter trigger by tags and search request terms @@ -101,7 +98,10 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, page int64 return nil, api.ErrorInternalServer(err) } pagerID = uuid4.String() - database.SaveTriggersSearchResults(pagerID, searchResults) //nolint + err = database.SaveTriggersSearchResults(pagerID, searchResults) + if err != nil { + return nil, api.ErrorInternalServer(err) + } } if createPager { @@ -113,7 +113,7 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, page int64 searchResults = searchResults[from:to] } - var triggerIDs []string //nolint + triggerIDs := make([]string, 0, len(searchResults)) for _, searchResult := range searchResults { triggerIDs = append(triggerIDs, searchResult.ObjectID) } @@ -151,24 +151,57 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, page int64 return &triggersList, nil } -func DeleteTriggersPager(dataBase moira.Database, pagerID string) (dto.TriggersSearchResultDeleteResponse, *api.ErrorResponse) { - exists, err := dataBase.IsTriggersSearchResultsExist(pagerID) +func DeleteTriggersPager(database moira.Database, pagerID string) (dto.TriggersSearchResultDeleteResponse, *api.ErrorResponse) { + exists, err := database.IsTriggersSearchResultsExist(pagerID) if err != nil { return dto.TriggersSearchResultDeleteResponse{}, api.ErrorInternalServer(err) } if !exists { return dto.TriggersSearchResultDeleteResponse{}, api.ErrorNotFound(fmt.Sprintf("pager with id %s not found", pagerID)) } - err = dataBase.DeleteTriggersSearchResults(pagerID) + err = database.DeleteTriggersSearchResults(pagerID) if err != nil { return dto.TriggersSearchResultDeleteResponse{}, api.ErrorInternalServer(err) } return dto.TriggersSearchResultDeleteResponse{PagerID: pagerID}, nil } -func triggerExists(dataBase moira.Database, triggerID string) (bool, error) { - _, err := dataBase.GetTrigger(triggerID) - if err == database.ErrNil { +// GetUnusedTriggerIDs returns unused triggers ids. +func GetUnusedTriggerIDs(database moira.Database) (*dto.TriggersList, *api.ErrorResponse) { + triggerIDs, err := database.GetUnusedTriggerIDs() + if err != nil { + return nil, api.ErrorInternalServer(err) + } + + triggerChecks, err := getTriggerChecks(database, triggerIDs) + if err != nil { + return nil, api.ErrorInternalServer(err) + } + triggersList := &dto.TriggersList{ + List: triggerChecks, + } + + return triggersList, nil +} + +func getTriggerChecks(database moira.Database, triggerIDs []string) ([]moira.TriggerCheck, error) { + triggerChecks, err := database.GetTriggerChecks(triggerIDs) + if err != nil { + return nil, err + } + list := make([]moira.TriggerCheck, 0, len(triggerChecks)) + for _, triggerCheck := range triggerChecks { + if triggerCheck != nil { + list = append(list, *triggerCheck) + } + } + + return list, nil +} + +func triggerExists(database moira.Database, triggerID string) (bool, error) { + _, err := database.GetTrigger(triggerID) + if err == db.ErrNil { return false, nil } if err != nil { diff --git a/api/controller/triggers_test.go b/api/controller/triggers_test.go index 9a1873098..5c503e485 100644 --- a/api/controller/triggers_test.go +++ b/api/controller/triggers_test.go @@ -808,3 +808,45 @@ func TestDeleteTriggersPager(t *testing.T) { }) }) } + +func TestGetUnusedTriggerIDs(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + + Convey("Has triggers", t, func() { + triggerIDs := []string{uuid.Must(uuid.NewV4()).String(), uuid.Must(uuid.NewV4()).String()} + triggers := []*moira.TriggerCheck{{Trigger: moira.Trigger{ID: triggerIDs[0]}}, {Trigger: moira.Trigger{ID: triggerIDs[1]}}} + triggersList := []moira.TriggerCheck{{Trigger: moira.Trigger{ID: triggerIDs[0]}}, {Trigger: moira.Trigger{ID: triggerIDs[1]}}} + mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(triggerIDs, nil) + mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(triggers, nil) + list, err := GetUnusedTriggerIDs(mockDatabase) + So(err, ShouldBeNil) + So(list, ShouldResemble, &dto.TriggersList{List: triggersList}) + }) + + Convey("No triggers", t, func() { + mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(make([]string, 0), nil) + mockDatabase.EXPECT().GetTriggerChecks(make([]string, 0)).Return(make([]*moira.TriggerCheck, 0), nil) + list, err := GetUnusedTriggerIDs(mockDatabase) + So(err, ShouldBeNil) + So(list, ShouldResemble, &dto.TriggersList{List: make([]moira.TriggerCheck, 0)}) + }) + + Convey("GetUnusedTriggerIDs error", t, func() { + expected := fmt.Errorf("getTriggerIDs error") + mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(nil, expected) + list, err := GetUnusedTriggerIDs(mockDatabase) + So(err, ShouldResemble, api.ErrorInternalServer(expected)) + So(list, ShouldBeNil) + }) + + Convey("GetTriggerChecks error", t, func() { + expected := fmt.Errorf("getTriggerChecks error") + mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(make([]string, 0), nil) + mockDatabase.EXPECT().GetTriggerChecks(make([]string, 0)).Return(nil, expected) + list, err := GetUnusedTriggerIDs(mockDatabase) + So(err, ShouldResemble, api.ErrorInternalServer(expected)) + So(list, ShouldBeNil) + }) +} diff --git a/api/handler/triggers.go b/api/handler/triggers.go index 15c8eb095..1bdf6f194 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -28,6 +28,7 @@ func triggers(metricSourceProvider *metricSource.SourceProvider, searcher moira. router.Use(middleware.MetricSourceProvider(metricSourceProvider)) router.Use(middleware.SearchIndexContext(searcher)) router.Get("/", getAllTriggers) + router.Get("/unused", getUnusedTriggers) router.Put("/", createTrigger) router.Put("/check", triggerCheck) router.Route("/{triggerId}", trigger) @@ -61,6 +62,29 @@ func getAllTriggers(writer http.ResponseWriter, request *http.Request) { } } +// nolint: gofmt,goimports +// +// @summary Get unused triggers +// @id get-unused-triggers +// @tags trigger +// @produce json +// @success 200 {object} dto.TriggersList "Fetched unused triggers" +// @failure 422 {object} api.ErrorRenderExample "Render error" +// @failure 500 {object} api.ErrorInternalServerExample "Internal server error" +// @router /trigger [get] +func getUnusedTriggers(writer http.ResponseWriter, request *http.Request) { + triggersList, errorResponse := controller.GetUnusedTriggerIDs(database) + if errorResponse != nil { + render.Render(writer, request, errorResponse) //nolint + return + } + + if err := render.Render(writer, request, triggersList); err != nil { + render.Render(writer, request, api.ErrorRender(err)) //nolint + return + } +} + // nolint: gofmt,goimports // createTrigger handler creates moira.Trigger // From 2203388b897f24f5a12a9e41ec9c438cacf2498f Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Tue, 19 Sep 2023 13:36:46 +0300 Subject: [PATCH 20/46] feature(api): add searching by created by parametr (#908) --- api/controller/triggers.go | 37 ++- api/controller/triggers_test.go | 447 ++++++++++++++++++++------------ api/handler/triggers.go | 33 ++- datatypes.go | 13 + index/bleve/query.go | 34 ++- index/bleve/query_test.go | 87 +++++-- index/bleve/search.go | 31 +-- index/bleve/search_test.go | 292 +++++++++++++++------ index/fixtures/fixtures.go | 169 +++++++----- index/index.go | 2 +- index/mapping/trigger.go | 5 + index/search.go | 4 +- index/search_test.go | 306 ++++++++++++++++------ interfaces.go | 3 +- mock/moira-alert/searcher.go | 8 +- 15 files changed, 986 insertions(+), 485 deletions(-) diff --git a/api/controller/triggers.go b/api/controller/triggers.go index cd39f7eb3..5d4d45c56 100644 --- a/api/controller/triggers.go +++ b/api/controller/triggers.go @@ -63,17 +63,17 @@ func GetAllTriggers(database moira.Database) (*dto.TriggersList, *api.ErrorRespo } // SearchTriggers gets trigger page and filter trigger by tags and search request terms -func SearchTriggers(database moira.Database, searcher moira.Searcher, page int64, size int64, onlyErrors bool, filterTags []string, searchString string, createPager bool, pagerID string) (*dto.TriggersList, *api.ErrorResponse) { //nolint +func SearchTriggers(database moira.Database, searcher moira.Searcher, options moira.SearchOptions) (*dto.TriggersList, *api.ErrorResponse) { //nolint var searchResults []*moira.SearchResult var total int64 - pagerShouldExist := pagerID != "" + pagerShouldExist := options.PagerID != "" - if pagerShouldExist && (searchString != "" || len(filterTags) > 0) { + if pagerShouldExist && (options.SearchString != "" || len(options.Tags) > 0) { return nil, api.ErrorInvalidRequest(fmt.Errorf("cannot handle request with search string or tags and pager ID set")) } if pagerShouldExist { var err error - searchResults, total, err = database.GetTriggersSearchResults(pagerID, page, size) + searchResults, total, err = database.GetTriggersSearchResults(options.PagerID, options.Page, options.Size) if err != nil { return nil, api.ErrorInternalServer(err) } @@ -82,33 +82,32 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, page int64 } } else { var err error - var passSize = size - if createPager { - passSize = pageSizeUnlimited + if options.CreatePager { + options.Size = pageSizeUnlimited } - searchResults, total, err = searcher.SearchTriggers(filterTags, searchString, onlyErrors, page, passSize) + searchResults, total, err = searcher.SearchTriggers(options) if err != nil { return nil, api.ErrorInternalServer(err) } } - if createPager && !pagerShouldExist { + if options.CreatePager && !pagerShouldExist { uuid4, err := uuid.NewV4() if err != nil { return nil, api.ErrorInternalServer(err) } - pagerID = uuid4.String() - err = database.SaveTriggersSearchResults(pagerID, searchResults) + options.PagerID = uuid4.String() + err = database.SaveTriggersSearchResults(options.PagerID, searchResults) if err != nil { return nil, api.ErrorInternalServer(err) } } - if createPager { + if options.CreatePager { var from, to int64 = 0, int64(len(searchResults)) - if size >= 0 { - from = int64(math.Min(float64(page*size), float64(len(searchResults)))) - to = int64(math.Min(float64(from+size), float64(len(searchResults)))) + if options.Size >= 0 { + from = int64(math.Min(float64(options.Page*options.Size), float64(len(searchResults)))) + to = int64(math.Min(float64(from+options.Size), float64(len(searchResults)))) } searchResults = searchResults[from:to] } @@ -124,15 +123,15 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, page int64 } var pagerIDPtr *string - if pagerID != "" { - pagerIDPtr = &pagerID + if options.PagerID != "" { + pagerIDPtr = &options.PagerID } triggersList := dto.TriggersList{ List: make([]moira.TriggerCheck, 0), Total: &total, - Page: &page, - Size: &size, + Page: &options.Page, + Size: &options.Size, Pager: pagerIDPtr, } diff --git a/api/controller/triggers_test.go b/api/controller/triggers_test.go index 5c503e485..6c7f03db2 100644 --- a/api/controller/triggers_test.go +++ b/api/controller/triggers_test.go @@ -152,8 +152,7 @@ func TestSearchTriggers(t *testing.T) { defer mockCtrl.Finish() mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) mockIndex := mock_moira_alert.NewMockSearcher(mockCtrl) - var page int64 - var size int64 = 50 + var exp int64 = 31 testHighlights := make([]moira.SearchHighlight, 0) for field, value := range testHighlightsMap { @@ -178,116 +177,125 @@ func TestSearchTriggers(t *testing.T) { triggerIDs[i] = trigger.ID } - tags := make([]string, 0) - searchString := "" + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + CreatePager: false, + PagerID: "", + } Convey("No tags, no text, onlyErrors = false, ", t, func() { Convey("Page is bigger than triggers number", func() { - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, size).Return(triggerSearchResults, exp, nil) + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults, exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(triggersPointers, nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks, Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) }) Convey("Must return all triggers, when size is -1", func() { - size = -1 - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, size).Return(triggerSearchResults, exp, nil) + searchOptions.Size = -1 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults, exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(triggersPointers, nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks, Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) }) Convey("Page is less than triggers number", func() { - size = 10 - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, size).Return(triggerSearchResults[:10], exp, nil) + searchOptions.Size = 10 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[:10], exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[:10]).Return(triggersPointers[:10], nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks[:10], Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) Convey("Second page", func() { - page = 1 - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, size).Return(triggerSearchResults[10:20], exp, nil) + searchOptions.Page = 1 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[10:20], exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[10:20]).Return(triggersPointers[10:20], nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks[10:20], Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) }) }) }) Convey("Complex search query", t, func() { - size = 10 - page = 0 + searchOptions.Size = 10 + searchOptions.Page = 0 Convey("Only errors", func() { exp = 30 // superTrigger31 is the only trigger without errors - mockIndex.EXPECT().SearchTriggers(tags, searchString, true, page, size).Return(triggerSearchResults[:10], exp, nil) + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[:10], exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[:10]).Return(triggersPointers[:10], nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, true, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks[0:10], Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) Convey("Only errors with tags", func() { - tags = []string{"encounters", "Kobold"} + searchOptions.Tags = []string{"encounters", "Kobold"} exp = 2 - mockIndex.EXPECT().SearchTriggers(tags, searchString, true, page, size).Return(triggerSearchResults[1:3], exp, nil) + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[1:3], exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[1:3]).Return(triggersPointers[1:3], nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, true, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks[1:3], Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) }) Convey("Only errors with text terms", func() { - searchString = "dragonshield medium" + searchOptions.SearchString = "dragonshield medium" exp = 1 - mockIndex.EXPECT().SearchTriggers(tags, searchString, true, page, size).Return(triggerSearchResults[2:3], exp, nil) + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[2:3], exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[2:3]).Return(triggersPointers[2:3], nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, true, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks[2:3], Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) }) Convey("Only errors with tags and text terms", func() { - tags = []string{"traps"} - searchString = "deadly" + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" exp = 4 deadlyTraps := []moira.TriggerCheck{ @@ -314,109 +322,192 @@ func TestSearchTriggers(t *testing.T) { deadlyTrapsTriggerIDs = append(deadlyTrapsTriggerIDs, deadlyTrap.ID) } - mockIndex.EXPECT().SearchTriggers(tags, searchString, true, page, size).Return(deadlyTrapsSearchResults, exp, nil) + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(deadlyTrapsSearchResults, exp, nil) mockDatabase.EXPECT().GetTriggerChecks(deadlyTrapsTriggerIDs).Return(deadlyTrapsPointers, nil) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, true, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: deadlyTraps, Total: &exp, - Page: &page, - Size: &size, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + }) + }) + + Convey("Only errors with createdBy", func() { + searchOptions.CreatedBy = "monster" + searchOptions.NeedSearchByCreatedBy = true + exp = 7 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[9:16], exp, nil) + mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[9:16]).Return(triggersPointers[9:16], nil) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) + So(err, ShouldBeNil) + So(list, ShouldResemble, &dto.TriggersList{ + List: triggerChecks[9:16], + Total: &exp, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + }) + }) + + Convey("Only errors with createdBy and tags", func() { + searchOptions.CreatedBy = "tarasov.da" + searchOptions.NeedSearchByCreatedBy = true + searchOptions.Tags = []string{"Human", "NPCs"} + exp = 2 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[22:24], exp, nil) + mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[22:24]).Return(triggersPointers[22:24], nil) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) + So(err, ShouldBeNil) + So(list, ShouldResemble, &dto.TriggersList{ + List: triggerChecks[22:24], + Total: &exp, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + }) + }) + + Convey("Only errors with createdBy, tags and text terms", func() { + searchOptions.CreatedBy = "internship2023" + searchOptions.NeedSearchByCreatedBy = true + searchOptions.Tags = []string{"Female", "NPCs"} + searchOptions.SearchString = "Music" + exp = 2 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[27:29], exp, nil) + mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[27:29]).Return(triggersPointers[27:29], nil) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) + So(err, ShouldBeNil) + So(list, ShouldResemble, &dto.TriggersList{ + List: triggerChecks[27:29], + Total: &exp, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + }) + }) + + Convey("Only errors with EMPTY createdBy", func() { + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + searchOptions.Tags = []string{} + searchOptions.SearchString = "" + exp = 3 + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults[6:9], exp, nil) + mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[6:9]).Return(triggersPointers[6:9], nil) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) + So(err, ShouldBeNil) + So(list, ShouldResemble, &dto.TriggersList{ + List: triggerChecks[6:9], + Total: &exp, + Page: &searchOptions.Page, + Size: &searchOptions.Size, }) }) }) }) Convey("Find triggers errors", t, func() { + searchOptions = moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } + Convey("Error from searcher", func() { searcherError := fmt.Errorf("very bad request") - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, size).Return(make([]*moira.SearchResult, 0), int64(0), searcherError) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, "") + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(make([]*moira.SearchResult, 0), int64(0), searcherError) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldNotBeNil) So(list, ShouldBeNil) }) Convey("Error from database", func() { - size = 50 + searchOptions.Size = 50 searcherError := fmt.Errorf("very bad request") - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, size).Return(triggerSearchResults, exp, nil) + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults, exp, nil) mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(nil, searcherError) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldNotBeNil) So(list, ShouldBeNil) }) Convey("Error on passed search elements and pagerID", func() { - tags = []string{"test"} - searchString = "test" - pagerID := "test" - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, pagerID) + searchOptions.Tags = []string{"test"} + searchOptions.SearchString = "test" + searchOptions.PagerID = "test" + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldNotBeNil) So(list, ShouldBeNil) }) }) + searchOptions.PagerID = "" + Convey("Search with pager", t, func() { - searchString = "" - tags = []string{} + searchOptions.SearchString = "" + searchOptions.Tags = []string{} Convey("Create pager", func() { - pagerID := "" - page = 0 - size = -1 + searchOptions.Page = 0 + searchOptions.Size = -1 + searchOptions.CreatePager = true exp = 31 gomock.InOrder( - mockIndex.EXPECT().SearchTriggers(tags, searchString, false, page, int64(-1)).Return(triggerSearchResults, exp, nil), + mockIndex.EXPECT().SearchTriggers(searchOptions).Return(triggerSearchResults, exp, nil), mockDatabase.EXPECT().SaveTriggersSearchResults(gomock.Any(), triggerSearchResults).Return(nil).Do(func(pID string, _ interface{}) { - pagerID = pID + searchOptions.PagerID = pID }), mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(triggersPointers, nil), ) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, true, "") + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks, Total: &exp, - Page: &page, - Size: &size, - Pager: &pagerID, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + Pager: &searchOptions.PagerID, }) }) Convey("Use pager", func() { - pagerID := "TestPagerID" - page = 0 - var size int64 = -1 + searchOptions.PagerID = "TestPagerID" + searchOptions.Page = 0 + searchOptions.Size = -1 + searchOptions.CreatePager = false var exp int64 = 31 gomock.InOrder( - mockDatabase.EXPECT().GetTriggersSearchResults(pagerID, page, size).Return(triggerSearchResults, exp, nil), + mockDatabase.EXPECT().GetTriggersSearchResults(searchOptions.PagerID, searchOptions.Page, searchOptions.Size).Return(triggerSearchResults, exp, nil), mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(triggersPointers, nil), ) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, pagerID) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks, Total: &exp, - Page: &page, - Size: &size, - Pager: &pagerID, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + Pager: &searchOptions.PagerID, }) }) Convey("Use pager and page size higher than amount of search results", func() { - pagerID := "TestPagerID" + searchOptions.PagerID = "TestPagerID" var exp int64 = 2 - var size int64 = 10 + searchOptions.Size = 10 + searchOptions.CreatePager = false gomock.InOrder( - mockDatabase.EXPECT().GetTriggersSearchResults(pagerID, page, size).Return(triggerSearchResults[:2], exp, nil), + mockDatabase.EXPECT().GetTriggersSearchResults(searchOptions.PagerID, searchOptions.Page, searchOptions.Size).Return(triggerSearchResults[:2], exp, nil), mockDatabase.EXPECT().GetTriggerChecks(triggerIDs[:2]).Return(triggersPointers[:2], nil), ) - list, err := SearchTriggers(mockDatabase, mockIndex, page, size, false, tags, searchString, false, pagerID) + list, err := SearchTriggers(mockDatabase, mockIndex, searchOptions) So(err, ShouldBeNil) So(list, ShouldResemble, &dto.TriggersList{ List: triggerChecks[:2], Total: &exp, - Page: &page, - Size: &size, - Pager: &pagerID, + Page: &searchOptions.Page, + Size: &searchOptions.Size, + Pager: &searchOptions.PagerID, }) }) }) @@ -425,9 +516,10 @@ func TestSearchTriggers(t *testing.T) { var triggerChecks = []moira.TriggerCheck{ { Trigger: moira.Trigger{ - ID: "SuperTrigger1", - Name: "I used D&D character generator for test triggers: https://donjon.bin.sh", - Tags: []string{"DND-generator", "common"}, + ID: "SuperTrigger1", + Name: "I used D&D character generator for test triggers: https://donjon.bin.sh", + Tags: []string{"DND-generator", "common"}, + CreatedBy: "test", }, LastCheck: moira.CheckData{ Score: 30, @@ -436,9 +528,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger2", - Name: "Kobold Scale Sorcerer (cr 1, vgm 167) and 1 x Kobold (cr 1/8, mm 195); medium, 225 xp", - Tags: []string{"DND-generator", "Kobold", "Sorcerer", "encounters"}, + ID: "SuperTrigger2", + Name: "Kobold Scale Sorcerer (cr 1, vgm 167) and 1 x Kobold (cr 1/8, mm 195); medium, 225 xp", + Tags: []string{"DND-generator", "Kobold", "Sorcerer", "encounters"}, + CreatedBy: "test", }, LastCheck: moira.CheckData{ Score: 29, @@ -447,9 +540,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger3", - Name: "Kobold Dragonshield (cr 1, vgm 165) and 1 x Kobold (cr 1/8, mm 195); medium, 225 xp", - Tags: []string{"DND-generator", "Kobold", "Dragonshield", "encounters"}, + ID: "SuperTrigger3", + Name: "Kobold Dragonshield (cr 1, vgm 165) and 1 x Kobold (cr 1/8, mm 195); medium, 225 xp", + Tags: []string{"DND-generator", "Kobold", "Dragonshield", "encounters"}, + CreatedBy: "test", }, LastCheck: moira.CheckData{ Score: 28, @@ -458,9 +552,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger4", - Name: "Orc Nurtured One of Yurtrus (cr 1/2, vgm 184) and 1 x Orc (cr 1/2, mm 246); hard, 200 xp", - Tags: []string{"DND-generator", "Orc", "encounters"}, + ID: "SuperTrigger4", + Name: "Orc Nurtured One of Yurtrus (cr 1/2, vgm 184) and 1 x Orc (cr 1/2, mm 246); hard, 200 xp", + Tags: []string{"DND-generator", "Orc", "encounters"}, + CreatedBy: "test", }, LastCheck: moira.CheckData{ Score: 27, @@ -469,9 +564,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger5", - Name: "Rust Monster (cr 1/2, mm 262); easy, 100 xp", - Tags: []string{"DND-generator", "Rust-Monster", "encounters"}, + ID: "SuperTrigger5", + Name: "Rust Monster (cr 1/2, mm 262); easy, 100 xp", + Tags: []string{"DND-generator", "Rust-Monster", "encounters"}, + CreatedBy: "test", }, LastCheck: moira.CheckData{ Score: 26, @@ -480,9 +576,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger6", - Name: "Giant Constrictor Snake (cr 2, mm 324); deadly, 450 xp", - Tags: []string{"Giant", "DND-generator", "Snake", "encounters"}, + ID: "SuperTrigger6", + Name: "Giant Constrictor Snake (cr 2, mm 324); deadly, 450 xp", + Tags: []string{"Giant", "DND-generator", "Snake", "encounters"}, + CreatedBy: "test", }, LastCheck: moira.CheckData{ Score: 25, @@ -524,9 +621,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger10", - Name: "Gibbering Mouther (cr 2, mm 157); easy, 450 xp", - Tags: []string{"Gibbering-Mouther", "DND-generator", "encounters"}, + ID: "SuperTrigger10", + Name: "Gibbering Mouther (cr 2, mm 157); easy, 450 xp", + Tags: []string{"Gibbering-Mouther", "DND-generator", "encounters"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 21, @@ -535,9 +633,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger11", - Name: "Scythe Blade: DC 10 to find, DC 10 to disable; +11 to hit against all targets within a 5 ft. arc, 4d10 slashing damage; apprentice tier, deadly", - Tags: []string{"Scythe Blade", "DND-generator", "traps"}, + ID: "SuperTrigger11", + Name: "Scythe Blade: DC 10 to find, DC 10 to disable; +11 to hit against all targets within a 5 ft. arc, 4d10 slashing damage; apprentice tier, deadly", + Tags: []string{"Scythe Blade", "DND-generator", "traps"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 20, @@ -546,9 +645,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger12", - Name: "Falling Block: DC 10 to find, DC 10 to disable; affects all targets within a 10 ft. square area, DC 12 save or take 2d10 damage; apprentice tier, dangerous", - Tags: []string{"Falling-Block", "DND-generator", "traps"}, + ID: "SuperTrigger12", + Name: "Falling Block: DC 10 to find, DC 10 to disable; affects all targets within a 10 ft. square area, DC 12 save or take 2d10 damage; apprentice tier, dangerous", + Tags: []string{"Falling-Block", "DND-generator", "traps"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 19, @@ -557,9 +657,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger13", - Name: "Thunderstone Mine: DC 15 to find, DC 15 to disable; affects all targets within 20 ft., DC 15 save or take 2d10 thunder damage and become deafened for 1d4 rounds; apprentice tier, dangerous", - Tags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, + ID: "SuperTrigger13", + Name: "Thunderstone Mine: DC 15 to find, DC 15 to disable; affects all targets within 20 ft., DC 15 save or take 2d10 thunder damage and become deafened for 1d4 rounds; apprentice tier, dangerous", + Tags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 18, @@ -568,9 +669,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger14", - Name: "Falling Block: DC 10 to find, DC 15 to disable; affects all targets within a 10 ft. square area, DC 12 save or take 2d10 damage; apprentice tier, dangerous", - Tags: []string{"Falling-Block", "DND-generator", "traps"}, + ID: "SuperTrigger14", + Name: "Falling Block: DC 10 to find, DC 15 to disable; affects all targets within a 10 ft. square area, DC 12 save or take 2d10 damage; apprentice tier, dangerous", + Tags: []string{"Falling-Block", "DND-generator", "traps"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 17, @@ -579,9 +681,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger15", - Name: "Chain Flail: DC 15 to find, DC 10 to disable; initiative +3, 1 attack per round, +11 to hit against all targets within 5 ft., 4d10 bludgeoning damage; apprentice tier, deadly", - Tags: []string{"Chain-Flail", "DND-generator", "traps"}, + ID: "SuperTrigger15", + Name: "Chain Flail: DC 15 to find, DC 10 to disable; initiative +3, 1 attack per round, +11 to hit against all targets within 5 ft., 4d10 bludgeoning damage; apprentice tier, deadly", + Tags: []string{"Chain-Flail", "DND-generator", "traps"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 16, @@ -590,9 +693,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger16", - Name: "Falling Block: DC 15 to find, DC 15 to disable; affects all targets within a 10 ft. square area, DC 12 save or take 2d10 damage; apprentice tier, dangerous", - Tags: []string{"Falling-Block", "DND-generator", "traps"}, + ID: "SuperTrigger16", + Name: "Falling Block: DC 15 to find, DC 15 to disable; affects all targets within a 10 ft. square area, DC 12 save or take 2d10 damage; apprentice tier, dangerous", + Tags: []string{"Falling-Block", "DND-generator", "traps"}, + CreatedBy: "monster", }, LastCheck: moira.CheckData{ Score: 15, @@ -601,9 +705,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger17", - Name: "Electrified Floortile: DC 20 to find, DC 15 to disable; affects all targets within a 10 ft. square area, DC 15 save or take 2d10 lightning damage; apprentice tier, dangerous", - Tags: []string{"Electrified-Floortile", "DND-generator", "traps"}, + ID: "SuperTrigger17", + Name: "Electrified Floortile: DC 20 to find, DC 15 to disable; affects all targets within a 10 ft. square area, DC 15 save or take 2d10 lightning damage; apprentice tier, dangerous", + Tags: []string{"Electrified-Floortile", "DND-generator", "traps"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 14, @@ -612,9 +717,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger18", - Name: "Earthmaw Trap: DC 15 to find, DC 10 to disable; +7 to hit against one target, 2d10 piercing damage; apprentice tier, dangerous", - Tags: []string{"Earthmaw-Trap", "DND-generator", "traps"}, + ID: "SuperTrigger18", + Name: "Earthmaw Trap: DC 15 to find, DC 10 to disable; +7 to hit against one target, 2d10 piercing damage; apprentice tier, dangerous", + Tags: []string{"Earthmaw-Trap", "DND-generator", "traps"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 13, @@ -623,9 +729,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger19", - Name: "Thunderstone Mine: DC 15 to find, DC 20 to disable; affects all targets within 20 ft., DC 18 save or take 4d10 thunder damage and become deafened for 1d4 rounds; apprentice tier, deadly", - Tags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, + ID: "SuperTrigger19", + Name: "Thunderstone Mine: DC 15 to find, DC 20 to disable; affects all targets within 20 ft., DC 18 save or take 4d10 thunder damage and become deafened for 1d4 rounds; apprentice tier, deadly", + Tags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 12, @@ -634,9 +741,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger20", - Name: "Scythe Blade: DC 15 to find, DC 10 to disable; +12 to hit against all targets within a 5 ft. arc, 4d10 slashing damage; apprentice tier, deadly", - Tags: []string{"Scythe-Blade", "DND-generator", "traps"}, + ID: "SuperTrigger20", + Name: "Scythe Blade: DC 15 to find, DC 10 to disable; +12 to hit against all targets within a 5 ft. arc, 4d10 slashing damage; apprentice tier, deadly", + Tags: []string{"Scythe-Blade", "DND-generator", "traps"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 11, @@ -645,9 +753,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger21", - Name: "Keelte: Female Elf Monk, LG. Str 12, Dex 14, Con 13, Int 9, Wis 15, Cha 14", - Tags: []string{"Female", "DND-generator", "Elf", "Monk", "NPCs"}, + ID: "SuperTrigger21", + Name: "Keelte: Female Elf Monk, LG. Str 12, Dex 14, Con 13, Int 9, Wis 15, Cha 14", + Tags: []string{"Female", "DND-generator", "Elf", "Monk", "NPCs"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 10, @@ -656,9 +765,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger22", - Name: "Kather Larke: Female Halfling Cleric, CN. Str 8, Dex 8, Con 13, Int 7, Wis 13, Cha 10", - Tags: []string{"Female", "DND-generator", "Halfling", "Cleric", "NPCs"}, + ID: "SuperTrigger22", + Name: "Kather Larke: Female Halfling Cleric, CN. Str 8, Dex 8, Con 13, Int 7, Wis 13, Cha 10", + Tags: []string{"Female", "DND-generator", "Halfling", "Cleric", "NPCs"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 9, @@ -667,9 +777,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger23", - Name: "Cyne: Male Human Soldier, NG. Str 12, Dex 9, Con 8, Int 10, Wis 8, Cha 10", - Tags: []string{"Male", "DND-generator", "Human", "Soldier", "NPCs"}, + ID: "SuperTrigger23", + Name: "Cyne: Male Human Soldier, NG. Str 12, Dex 9, Con 8, Int 10, Wis 8, Cha 10", + Tags: []string{"Male", "DND-generator", "Human", "Soldier", "NPCs"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 8, @@ -678,9 +789,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger24", - Name: "Gytha: Female Human Barbarian, N. Str 16, Dex 13, Con 12, Int 12, Wis 14, Cha 9", - Tags: []string{"Female", "DND-generator", "Human", "Barbarian", "NPCs"}, + ID: "SuperTrigger24", + Name: "Gytha: Female Human Barbarian, N. Str 16, Dex 13, Con 12, Int 12, Wis 14, Cha 9", + Tags: []string{"Female", "DND-generator", "Human", "Barbarian", "NPCs"}, + CreatedBy: "tarasov.da", }, LastCheck: moira.CheckData{ Score: 7, @@ -689,9 +801,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger25", - Name: "Brobern Hawte: Male Half-elf Monk, N. Str 12, Dex 10, Con 8, Int 14, Wis 12, Cha 12", - Tags: []string{"Male", "DND-generator", "Half-elf", "Monk", "NPCs"}, + ID: "SuperTrigger25", + Name: "Brobern Hawte: Male Half-elf Monk, N. Str 12, Dex 10, Con 8, Int 14, Wis 12, Cha 12", + Tags: []string{"Male", "DND-generator", "Half-elf", "Monk", "NPCs"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 6, @@ -700,9 +813,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger26", - Name: "Borneli: Male Elf Servant, LN. Str 12, Dex 12, Con 8, Int 13, Wis 6, Cha 12", - Tags: []string{"Male", "DND-generator", "Elf", "Servant", "NPCs"}, + ID: "SuperTrigger26", + Name: "Borneli: Male Elf Servant, LN. Str 12, Dex 12, Con 8, Int 13, Wis 6, Cha 12", + Tags: []string{"Male", "DND-generator", "Elf", "Servant", "NPCs"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 5, @@ -711,9 +825,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger27", - Name: "Midda: Male Elf Sorcerer, LN. Str 10, Dex 13, Con 11, Int 7, Wis 10, Cha 13", - Tags: []string{"Male", "DND-generator", "Elf", "Sorcerer", "NPCs"}, + ID: "SuperTrigger27", + Name: "Midda: Male Elf Sorcerer, LN. Str 10, Dex 13, Con 11, Int 7, Wis 10, Cha 13", + Tags: []string{"Male", "DND-generator", "Elf", "Sorcerer", "NPCs"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 4, @@ -722,9 +837,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger28", - Name: "Burgwe: Female Human Bard, CN. Str 13, Dex 11, Con 10, Int 13, Wis 12, Cha 17.", - Tags: []string{"Female", "DND-generator", "Human", "Bard", "NPCs"}, + ID: "SuperTrigger28", + Name: "Burgwe: Female Human Bard, CN. Str 13, Dex 11, Con 10, Int 13, Wis 12, Cha 17. Music!", + Tags: []string{"Female", "DND-generator", "Human", "Bard", "NPCs"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 3, @@ -733,9 +849,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger29", - Name: "Carel: Female Gnome Druid, Neutral. Str 11, Dex 12, Con 7, Int 10, Wis 17, Cha 10", - Tags: []string{"Female", "DND-generator", "Gnome", "Druid", "NPCs"}, + ID: "SuperTrigger29", + Name: "Carel: Female Gnome Druid, Neutral. Str 11, Dex 12, Con 7, Int 10, Wis 17, Cha 10. Music!", + Tags: []string{"Female", "DND-generator", "Gnome", "Druid", "NPCs"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 2, @@ -744,9 +861,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger30", - Name: "Suse Salte: Female Human Aristocrat, N. Str 10, Dex 7, Con 10, Int 9, Wis 7, Cha 13", - Tags: []string{"Female", "DND-generator", "Human", "Aristocrat", "NPCs"}, + ID: "SuperTrigger30", + Name: "Suse Salte: Female Human Aristocrat, N. Str 10, Dex 7, Con 10, Int 9, Wis 7, Cha 13", + Tags: []string{"Female", "DND-generator", "Human", "Aristocrat", "NPCs"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 1, @@ -755,9 +873,10 @@ var triggerChecks = []moira.TriggerCheck{ }, { Trigger: moira.Trigger{ - ID: "SuperTrigger31", - Name: "Surprise!", - Tags: []string{"Something-extremely-new"}, + ID: "SuperTrigger31", + Name: "Surprise!", + Tags: []string{"Something-extremely-new"}, + CreatedBy: "internship2023", }, LastCheck: moira.CheckData{ Score: 0, diff --git a/api/handler/triggers.go b/api/handler/triggers.go index 1bdf6f194..22dbb4636 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -250,6 +250,7 @@ func triggerCheck(writer http.ResponseWriter, request *http.Request) { // @param size query integer false "Page size" default(10) // @param createPager query boolean false "Create pager" default(false) // @param pagerID query string false "Pager ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param createdBy query string false "Created By" default(moira.team) // @success 200 {object} dto.TriggersList "Successfully fetched matching triggers" // @failure 400 {object} api.ErrorInvalidRequestExample "Bad request from client" // @failure 404 {object} api.ErrorNotFoundExample "Resource not found" @@ -258,17 +259,21 @@ func triggerCheck(writer http.ResponseWriter, request *http.Request) { // @router /trigger/search [get] func searchTriggers(writer http.ResponseWriter, request *http.Request) { request.ParseForm() //nolint - onlyErrors := getOnlyProblemsFlag(request) - filterTags := getRequestTags(request) - searchString := getSearchRequestString(request) - page := middleware.GetPage(request) - size := middleware.GetSize(request) - - createPager := middleware.GetCreatePager(request) - pagerID := middleware.GetPagerID(request) + createdBy, ok := getTriggerCreatedBy(request) + searchOptions := moira.SearchOptions{ + Page: middleware.GetPage(request), + Size: middleware.GetSize(request), + OnlyProblems: getOnlyProblemsFlag(request), + Tags: getRequestTags(request), + SearchString: getSearchRequestString(request), + CreatedBy: createdBy, + NeedSearchByCreatedBy: ok, + CreatePager: middleware.GetCreatePager(request), + PagerID: middleware.GetPagerID(request), + } - triggersList, errorResponse := controller.SearchTriggers(database, searchIndex, page, size, onlyErrors, filterTags, searchString, createPager, pagerID) + triggersList, errorResponse := controller.SearchTriggers(database, searchIndex, searchOptions) if errorResponse != nil { render.Render(writer, request, errorResponse) //nolint return @@ -329,6 +334,16 @@ func getOnlyProblemsFlag(request *http.Request) bool { return false } +// Checks if the createdBy field has been set: +// if the field has been set, searches for triggers with a specific author createdBy +// if the field has not been set, searches for triggers with any author +func getTriggerCreatedBy(request *http.Request) (string, bool) { + if createdBy, ok := request.Form["createdBy"]; ok { + return createdBy[0], true + } + return "", false +} + func getSearchRequestString(request *http.Request) string { searchText := request.FormValue("text") searchText = strings.ToLower(searchText) diff --git a/datatypes.go b/datatypes.go index 30ce3c56a..32fe29bc8 100644 --- a/datatypes.go +++ b/datatypes.go @@ -340,6 +340,19 @@ type TriggerCheck struct { Highlights map[string]string `json:"highlights"` } +// SearchOptions represents the options that can be selected when searching triggers +type SearchOptions struct { + Page int64 + Size int64 + OnlyProblems bool + SearchString string + Tags []string + CreatedBy string + NeedSearchByCreatedBy bool + CreatePager bool + PagerID string +} + // MaintenanceCheck set maintenance user, time type MaintenanceCheck interface { SetMaintenance(maintenanceInfo *MaintenanceInfo, maintenance int64) diff --git a/index/bleve/query.go b/index/bleve/query.go index cdeb6999a..f07993d19 100644 --- a/index/bleve/query.go +++ b/index/bleve/query.go @@ -1,21 +1,27 @@ package bleve import ( + "regexp" + "strings" + "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/search/query" + "github.com/moira-alert/moira" "github.com/moira-alert/moira/index/mapping" ) -func buildSearchQuery(filterTags, searchTerms []string, onlyErrors bool) query.Query { - if !onlyErrors && len(filterTags) == 0 && len(searchTerms) == 0 { +func buildSearchQuery(options moira.SearchOptions) query.Query { + searchTerms := splitStringToTerms(options.SearchString) + if !options.OnlyProblems && len(options.Tags) == 0 && len(searchTerms) == 0 && !options.NeedSearchByCreatedBy { return bleve.NewMatchAllQuery() } searchQueries := make([]query.Query, 0) - searchQueries = append(searchQueries, buildQueryForTags(filterTags)...) + searchQueries = append(searchQueries, buildQueryForTags(options.Tags)...) searchQueries = append(searchQueries, buildQueryForTerms(searchTerms)...) - searchQueries = append(searchQueries, buildQueryForOnlyErrors(onlyErrors)...) + searchQueries = append(searchQueries, buildQueryForOnlyErrors(options.OnlyProblems)...) + searchQueries = append(searchQueries, buildQueryForCreatedBy(options.CreatedBy, options.NeedSearchByCreatedBy)...) return bleve.NewConjunctionQuery(searchQueries...) } @@ -29,6 +35,16 @@ func buildQueryForTags(filterTags []string) (searchQueries []query.Query) { return } +func buildQueryForCreatedBy(createdBy string, needSearchByCreatedBy bool) (searchQueries []query.Query) { + if !needSearchByCreatedBy { + return + } + qr := bleve.NewTermQuery(createdBy) + qr.FieldVal = mapping.TriggerCreatedBy.GetName() + searchQueries = append(searchQueries, qr) + return +} + func buildQueryForTerms(searchTerms []string) (searchQueries []query.Query) { for _, term := range searchTerms { nameQuery, nameField := bleve.NewFuzzyQuery(term), mapping.TriggerName @@ -51,3 +67,13 @@ func buildQueryForOnlyErrors(onlyErrors bool) (searchQueries []query.Query) { qr.FieldVal = mapping.TriggerLastCheckScore.GetName() return append(searchQueries, qr) } + +func splitStringToTerms(searchString string) (searchTerms []string) { + searchString = escapeString(searchString) + + return strings.Fields(searchString) +} + +func escapeString(original string) (escaped string) { + return regexp.MustCompile(`[|+\-=&<>!(){}\[\]^"'~*?\\/.,:;_@]`).ReplaceAllString(original, " ") +} diff --git a/index/bleve/query_test.go b/index/bleve/query_test.go index 266c2b022..8545301e0 100644 --- a/index/bleve/query_test.go +++ b/index/bleve/query_test.go @@ -5,61 +5,104 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/search/query" + "github.com/moira-alert/moira" . "github.com/smartystreets/goconvey/convey" ) +const defaultSearchString = "123 456" + func TestIndex_BuildSearchQuery(t *testing.T) { - tags := make([]string, 0) - searchTerms := make([]string, 0) - onlyErrors := false + searchOptions := moira.SearchOptions{ + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } Convey("Test build search query", t, func() { Convey("Empty query", func() { expected := bleve.NewMatchAllQuery() - actual := buildSearchQuery(tags, searchTerms, onlyErrors) + actual := buildSearchQuery(searchOptions) So(actual, ShouldResemble, expected) }) Convey("Complex query", func() { Convey("Only errors = true", func() { - onlyErrors = true - qr := buildQueryForOnlyErrors(onlyErrors) + searchOptions.OnlyProblems = true + qr := buildQueryForOnlyErrors(searchOptions.OnlyProblems) expected := bleve.NewConjunctionQuery(qr...) - actual := buildSearchQuery(tags, searchTerms, onlyErrors) + actual := buildSearchQuery(searchOptions) So(actual, ShouldResemble, expected) }) Convey("Only errors = false, several tags", func() { - onlyErrors = false - tags = []string{"123", "456"} - qr := buildQueryForTags(tags) + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{"123", "456"} + qr := buildQueryForTags(searchOptions.Tags) expected := bleve.NewConjunctionQuery(qr...) - actual := buildSearchQuery(tags, searchTerms, onlyErrors) + actual := buildSearchQuery(searchOptions) So(actual, ShouldResemble, expected) }) Convey("Only errors = false, no tags, several terms", func() { - onlyErrors = false - tags = make([]string, 0) - searchTerms = []string{"123", "456"} + searchOptions.OnlyProblems = false + searchOptions.Tags = make([]string, 0) + searchOptions.SearchString = defaultSearchString + searchTerms := []string{"123", "456"} + qr := buildQueryForTerms(searchTerms) expected := bleve.NewConjunctionQuery(qr...) - actual := buildSearchQuery(tags, searchTerms, onlyErrors) + actual := buildSearchQuery(searchOptions) + So(actual, ShouldResemble, expected) + }) + + Convey("Only errors = false, several tags, several terms", func() { + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{"123", "456"} + searchOptions.SearchString = defaultSearchString + searchTerms := []string{"123", "456"} + + searchQueries := make([]query.Query, 0) + + searchQueries = append(searchQueries, buildQueryForTags(searchOptions.Tags)...) + searchQueries = append(searchQueries, buildQueryForTerms(searchTerms)...) + searchQueries = append(searchQueries, buildQueryForOnlyErrors(searchOptions.OnlyProblems)...) + expected := bleve.NewConjunctionQuery(searchQueries...) + + actual := buildSearchQuery(searchOptions) + So(actual, ShouldResemble, expected) + }) + + Convey("Only errors = false, no tags, without terms, with created by", func() { + searchOptions.OnlyProblems = false + searchOptions.Tags = make([]string, 0) + searchOptions.SearchString = "" + searchOptions.CreatedBy = "test" + searchOptions.NeedSearchByCreatedBy = true + + qr := buildQueryForCreatedBy(searchOptions.CreatedBy, searchOptions.NeedSearchByCreatedBy) + expected := bleve.NewConjunctionQuery(qr...) + + actual := buildSearchQuery(searchOptions) So(actual, ShouldResemble, expected) }) - Convey("Only errors = true, several tags, several terms", func() { - onlyErrors = false - tags = []string{"123", "456"} - searchTerms = []string{"123", "456"} + Convey("Only errors = true, several tags, several terms, with created by", func() { + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"123", "456"} + searchOptions.SearchString = defaultSearchString + searchTerms := []string{"123", "456"} + searchQueries := make([]query.Query, 0) - searchQueries = append(searchQueries, buildQueryForTags(tags)...) + searchQueries = append(searchQueries, buildQueryForTags(searchOptions.Tags)...) searchQueries = append(searchQueries, buildQueryForTerms(searchTerms)...) - searchQueries = append(searchQueries, buildQueryForOnlyErrors(onlyErrors)...) + searchQueries = append(searchQueries, buildQueryForOnlyErrors(searchOptions.OnlyProblems)...) + searchQueries = append(searchQueries, buildQueryForCreatedBy(searchOptions.CreatedBy, searchOptions.NeedSearchByCreatedBy)...) expected := bleve.NewConjunctionQuery(searchQueries...) - actual := buildSearchQuery(tags, searchTerms, onlyErrors) + actual := buildSearchQuery(searchOptions) So(actual, ShouldResemble, expected) }) }) diff --git a/index/bleve/search.go b/index/bleve/search.go index 843f89d14..be5eef2dd 100644 --- a/index/bleve/search.go +++ b/index/bleve/search.go @@ -2,8 +2,6 @@ package bleve import ( "fmt" - "regexp" - "strings" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/search" @@ -16,14 +14,14 @@ import ( // TriggerCheck.Score (desc) // Relevance (asc) // Trigger.Name (asc) -func (index *TriggerIndex) Search(filterTags []string, searchString string, onlyErrors bool, page int64, size int64) (searchResults []*moira.SearchResult, total int64, err error) { - if size < 0 { - page = 0 +func (index *TriggerIndex) Search(options moira.SearchOptions) (searchResults []*moira.SearchResult, total int64, err error) { + if options.Size < 0 { + options.Page = 0 docs, _ := index.index.DocCount() - size = int64(docs) + options.Size = int64(docs) } - req := buildSearchRequest(filterTags, searchString, onlyErrors, int(page), int(size)) + req := buildSearchRequest(options) searchResult, err := index.index.Search(req) if err != nil { @@ -62,12 +60,11 @@ func getHighlights(fragmentsMap search.FieldFragmentMap, triggerFields ...mappin return highlights } -func buildSearchRequest(filterTags []string, searchString string, onlyErrors bool, page, size int) *bleve.SearchRequest { - searchTerms := splitStringToTerms(searchString) - searchQuery := buildSearchQuery(filterTags, searchTerms, onlyErrors) +func buildSearchRequest(options moira.SearchOptions) *bleve.SearchRequest { + searchQuery := buildSearchQuery(options) - from := page * size - req := bleve.NewSearchRequestOptions(searchQuery, size, from, false) + from := options.Page * options.Size + req := bleve.NewSearchRequestOptions(searchQuery, int(options.Size), int(from), false) // sorting order: // TriggerCheck.Score (desc) // Relevance (asc) @@ -77,13 +74,3 @@ func buildSearchRequest(filterTags []string, searchString string, onlyErrors boo return req } - -func splitStringToTerms(searchString string) (searchTerms []string) { - searchString = escapeString(searchString) - - return strings.Fields(searchString) -} - -func escapeString(original string) (escaped string) { - return regexp.MustCompile(`[|+\-=&<>!(){}\[\]^"'~*?\\/.,:;_@]`).ReplaceAllString(original, " ") -} diff --git a/index/bleve/search_test.go b/index/bleve/search_test.go index e1560c20e..ecdcabae5 100644 --- a/index/bleve/search_test.go +++ b/index/bleve/search_test.go @@ -39,212 +39,340 @@ func TestTriggerIndex_Search(t *testing.T) { }) Convey("Search for triggers without pagination", t, func() { - page := int64(0) - size := int64(50) - tags := make([]string, 0) - searchString := "" - onlyErrors := false - - Convey("No tags, no searchString, onlyErrors = false", func() { - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)) + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } + + Convey("No tags, no searchString, onlyErrors = false, without createdBy", func() { + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)) So(count, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("No tags, no searchString, onlyErrors = false, size = -1 (must return all triggers)", func() { - size = -1 - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)) + searchOptions.Size = -1 + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)) So(count, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("OnlyErrors = true", func() { - size = 50 - onlyErrors = true - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[:30]) + searchOptions.Size = 50 + searchOptions.OnlyProblems = true + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[:30]) So(count, ShouldEqual, 30) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags", func() { - onlyErrors = true - tags = []string{"encounters", "Kobold"} - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[1:3]) + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"encounters", "Kobold"} + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[1:3]) So(count, ShouldEqual, 2) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, several tags", func() { - onlyErrors = false - tags = []string{"Something-extremely-new"} - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[30:]) + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{"Something-extremely-new"} + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[30:]) So(count, ShouldEqual, 2) So(err, ShouldBeNil) }) Convey("Empty list should be", func() { - onlyErrors = true - tags = []string{"Something-extremely-new"} - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"Something-extremely-new"} + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldBeEmpty) So(count, ShouldBeZeroValue) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, no tags, several text terms", func() { - onlyErrors = true - tags = make([]string, 0) - searchString = "dragonshield medium" - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[2:3]) + searchOptions.OnlyProblems = true + searchOptions.Tags = make([]string, 0) + searchOptions.SearchString = "dragonshield medium" + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[2:3]) So(count, ShouldEqual, 1) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags, several text terms", func() { - onlyErrors = true - tags = []string{"traps"} - searchString = "deadly" //nolint + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" //nolint deadlyTraps := []int{10, 14, 18, 19} deadlyTrapsSearchResults := make([]*moira.SearchResult, 0) for _, ind := range deadlyTraps { - deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldResemble, deadlyTrapsSearchResults) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) + + Convey("OnlyErrors = false, no tags, no terms, with createdBy", func() { + searchOptions.OnlyProblems = false + searchOptions.Tags = make([]string, 0) + searchOptions.SearchString = "" + searchOptions.CreatedBy = "test" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[0:4]) + So(count, ShouldEqual, 4) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, one tag, no terms, with createdBy", func() { + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"shadows"} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "tarasov.da" + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[14:16]) + So(count, ShouldEqual, 2) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, several tags, one text term, with createdBy", func() { + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"Coldness", "Dark"} + searchOptions.SearchString = "deadly" + searchOptions.CreatedBy = "tarasov.da" + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[18:19]) + So(count, ShouldEqual, 1) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, several tags, no text, with EMPTY createdBy", func() { + searchOptions.OnlyProblems = true + searchOptions.SearchString = "" + searchOptions.Tags = []string{"Darkness", "DND-generator"} + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[5:7]) + So(count, ShouldEqual, 2) + So(err, ShouldBeNil) + }) }) Convey("Search for triggers with pagination", t, func() { - page := int64(0) - size := int64(10) - tags := make([]string, 0) - searchString := "" - onlyErrors := false + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 10, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } Convey("No tags, no searchString, onlyErrors = false, page -> 0, size -> 10", func() { - searchResults, total, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[:10]) + searchResults, total, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[:10]) So(total, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("No tags, no searchString, onlyErrors = false, page -> 1, size -> 10", func() { - page = 1 - searchResults, total, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[10:20]) + searchOptions.Page = 1 + searchResults, total, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[10:20]) So(total, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("No tags, no searchString, onlyErrors = false, page -> 1, size -> 20", func() { - page = 1 - size = 20 - searchResults, total, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[20:]) + searchOptions.Page = 1 + searchOptions.Size = 20 + searchResults, total, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[20:]) So(total, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags, several text terms, page -> 0, size 2", func() { - page = 0 - size = 2 - onlyErrors = true - tags = []string{"traps"} - searchString = "deadly" + searchOptions.Page = 0 + searchOptions.Size = 2 + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" deadlyTraps := []int{10, 14, 18, 19} deadlyTrapsSearchResults := make([]*moira.SearchResult, 0) for _, ind := range deadlyTraps { - deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldResemble, deadlyTrapsSearchResults[:2]) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags, several text terms, page -> 1, size 10", func() { - page = 1 - size = 10 - onlyErrors = true - tags = []string{"traps"} - searchString = "deadly" + searchOptions.Page = 1 + searchOptions.Size = 10 + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldBeEmpty) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) + + Convey("OnlyErrors = true, several tags, no terms, with createdBy, page -> 0, size 2", func() { + searchOptions.Page = 0 + searchOptions.Size = 2 + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"Human", "NPCs"} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "internship2023" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[22:24]) + So(count, ShouldEqual, 4) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = false, one tags, no terms, with createdBy, page -> 0, size 5", func() { + searchOptions.Page = 0 + searchOptions.Size = 5 + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{"Something-extremely-new"} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "internship2023" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[30:32]) + So(count, ShouldEqual, 2) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = false, no tags, no terms, with EMPTY createdBy, page -> 0, size 3", func() { + searchOptions.Page = 0 + searchOptions.Size = 3 + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[4:7]) + So(count, ShouldEqual, 3) + So(err, ShouldBeNil) + }) }) Convey("Search for triggers by description", t, func() { - page := int64(0) - size := int64(50) - tags := make([]string, 0) - searchString := "" - onlyErrors := false + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } Convey("OnlyErrors = false, search by name and description, 0 results", func() { - searchString = "life female druid" - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchOptions.SearchString = "life female druid" + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldBeEmpty) So(count, ShouldEqual, 0) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, search by name and description, 3 results", func() { - searchString = "easy" + searchOptions.SearchString = "easy" easy := []int{4, 9, 30, 31} easySearchResults := make([]*moira.SearchResult, 0) for _, ind := range easy { - easySearchResults = append(easySearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + easySearchResults = append(easySearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchString = "easy" - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldResemble, easySearchResults) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, search by name and description, 1 result", func() { - searchString = "little monster" - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[4:5]) + searchOptions.SearchString = "little monster" + searchResults, count, err := newIndex.Search(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[4:5]) So(count, ShouldEqual, 1) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, search by description and tags, 2 results", func() { - searchString = "mama" - tags := []string{"traps"} + searchOptions.SearchString = "mama" + searchOptions.Tags = []string{"traps"} mamaTraps := []int{11, 19} mamaTrapsSearchResults := make([]*moira.SearchResult, 0) for _, ind := range mamaTraps { - mamaTrapsSearchResults = append(mamaTrapsSearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + mamaTrapsSearchResults = append(mamaTrapsSearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := newIndex.Search(tags, searchString, onlyErrors, page, size) + searchResults, count, err := newIndex.Search(searchOptions) So(searchResults, ShouldResemble, mamaTrapsSearchResults) So(count, ShouldEqual, 2) So(err, ShouldBeNil) }) + + Convey("OnlyErrors = false, search by description, no tags, with createdBy, 3 results", func() { + searchOptions.SearchString = "mama" + searchOptions.Tags = make([]string, 0) + searchOptions.CreatedBy = "monster" + searchOptions.NeedSearchByCreatedBy = true + + _, count, err := newIndex.Search(searchOptions) + So(count, ShouldEqual, 3) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = false, search by description, no tags, with EMPTY createdBy, 1 result", func() { + searchOptions.SearchString = "little monster" + searchOptions.Tags = make([]string, 0) + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + + _, count, err := newIndex.Search(searchOptions) + So(count, ShouldEqual, 1) + So(err, ShouldBeNil) + }) }) } diff --git a/index/fixtures/fixtures.go b/index/fixtures/fixtures.go index 83a20a32b..e9413e286 100644 --- a/index/fixtures/fixtures.go +++ b/index/fixtures/fixtures.go @@ -12,11 +12,12 @@ type fixtureIndexedTriggers struct { } type fixtureIndexedTrigger struct { - triggerID string - triggerName fixtureIndexedField - triggerDesc fixtureIndexedField - triggerTags []string - triggerScore int64 + triggerID string + triggerName fixtureIndexedField + triggerDesc fixtureIndexedField + triggerTags []string + triggerCreatedBy string + triggerScore int64 } func (it *fixtureIndexedTrigger) GetHighLights(searchString string) []moira.SearchHighlight { @@ -35,10 +36,11 @@ func (its *fixtureIndexedTriggers) ToTriggerChecks() []*moira.TriggerCheck { for _, indexedTrigger := range its.list { triggerCheck := moira.TriggerCheck{ Trigger: moira.Trigger{ - ID: indexedTrigger.triggerID, - Name: indexedTrigger.triggerName.content, - Tags: indexedTrigger.triggerTags, - Desc: new(string), + ID: indexedTrigger.triggerID, + Name: indexedTrigger.triggerName.content, + Tags: indexedTrigger.triggerTags, + CreatedBy: indexedTrigger.triggerCreatedBy, + Desc: new(string), }, LastCheck: moira.CheckData{ Score: indexedTrigger.triggerScore, @@ -81,8 +83,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "0: Is this the real life? Is this just fantasy?", }, - triggerTags: []string{"DND-generator", "common"}, - triggerScore: 30, //nolint + triggerTags: []string{"DND-generator", "common"}, + triggerCreatedBy: "test", + triggerScore: 30, //nolint }, { triggerID: "SuperTrigger2", @@ -92,8 +95,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "1: Caught in a landslide, no escape from reality", }, - triggerTags: []string{"DND-generator", "Kobold", "Sorcerer", "encounters"}, - triggerScore: 29, //nolint + triggerTags: []string{"DND-generator", "Kobold", "Sorcerer", "encounters"}, + triggerCreatedBy: "test", + triggerScore: 29, //nolint }, { triggerID: "SuperTrigger3", @@ -111,8 +115,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "2: Open your eyes, look up to the skies and see", }, - triggerTags: []string{"DND-generator", "Kobold", "Dragonshield", "encounters"}, - triggerScore: 28, //nolint + triggerTags: []string{"DND-generator", "Kobold", "Dragonshield", "encounters"}, + triggerCreatedBy: "test", + triggerScore: 28, //nolint }, { triggerID: "SuperTrigger4", @@ -122,8 +127,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "3: I'm just a poor boy, I need no sympathy", }, - triggerTags: []string{"DND-generator", "Orc", "encounters"}, - triggerScore: 27, //nolint + triggerTags: []string{"DND-generator", "Orc", "encounters"}, + triggerCreatedBy: "test", + triggerScore: 27, //nolint }, { triggerID: "SuperTrigger5", @@ -172,7 +178,7 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "5: Any way the wind blows doesn't really matter to me, to me", }, - triggerTags: []string{"Giant", "DND-generator", "Snake", "encounters"}, + triggerTags: []string{"Giant", "DND-generator", "Snake", "encounters", "Darkness"}, triggerScore: 25, //nolint }, { @@ -183,7 +189,7 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "6", }, - triggerTags: []string{"Darkling", "DND-generator", "encounters"}, + triggerTags: []string{"Darkling", "DND-generator", "encounters", "Darkness"}, triggerScore: 24, //nolint }, { @@ -194,8 +200,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "7: Mama, just killed a man", }, - triggerTags: []string{"Ghost", "DND-generator", "encounters"}, - triggerScore: 23, //nolint + triggerTags: []string{"Ghost", "DND-generator", "encounters"}, + triggerCreatedBy: "monster", + triggerScore: 23, //nolint }, { triggerID: "SuperTrigger9", @@ -205,8 +212,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "8: Put a gun against his head, pulled my trigger, now he's dead", }, - triggerTags: []string{"Spectator", "DND-generator", "encounters"}, - triggerScore: 22, //nolint + triggerTags: []string{"Spectator", "DND-generator", "encounters"}, + triggerCreatedBy: "monster", + triggerScore: 22, //nolint }, { triggerID: "SuperTrigger10", @@ -224,8 +232,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "9: Mama, life had just begun", }, - triggerTags: []string{"Gibbering-Mouther", "DND-generator", "encounters"}, - triggerScore: 21, //nolint + triggerTags: []string{"Gibbering-Mouther", "DND-generator", "encounters"}, + triggerCreatedBy: "monster", + triggerScore: 21, //nolint }, { triggerID: "SuperTrigger11", @@ -243,8 +252,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "10: But now I've gone and thrown it all away", }, - triggerTags: []string{"Scythe Blade", "DND-generator", "traps"}, - triggerScore: 20, //nolint + triggerTags: []string{"Scythe Blade", "DND-generator", "traps"}, + triggerCreatedBy: "monster", + triggerScore: 20, //nolint }, { triggerID: "SuperTrigger12", @@ -262,8 +272,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ }, }, }, - triggerTags: []string{"Falling-Block", "DND-generator", "traps"}, - triggerScore: 19, //nolint + triggerTags: []string{"Falling-Block", "DND-generator", "traps"}, + triggerCreatedBy: "monster", + triggerScore: 19, //nolint }, { triggerID: "SuperTrigger13", @@ -273,8 +284,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "12: If I'm not back again this time tomorrow", }, - triggerTags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, - triggerScore: 18, //nolint + triggerTags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, + triggerCreatedBy: "monster", + triggerScore: 18, //nolint }, { triggerID: "SuperTrigger14", @@ -284,8 +296,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "13: Carry on, carry on as if nothing really matters", }, - triggerTags: []string{"Falling-Block", "DND-generator", "traps"}, - triggerScore: 17, //nolint + triggerTags: []string{"Falling-Block", "DND-generator", "traps"}, + triggerCreatedBy: "monster", + triggerScore: 17, //nolint }, { triggerID: "SuperTrigger15", @@ -303,8 +316,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "14", }, - triggerTags: []string{"Chain-Flail", "DND-generator", "traps"}, - triggerScore: 16, //nolint + triggerTags: []string{"Chain-Flail", "DND-generator", "traps", "shadows"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 16, //nolint }, { triggerID: "SuperTrigger16", @@ -314,8 +328,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "15: Too late, my time has come", }, - triggerTags: []string{"Falling-Block", "DND-generator", "traps"}, - triggerScore: 15, //nolint + triggerTags: []string{"Falling-Block", "DND-generator", "traps", "shadows"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 15, //nolint }, { triggerID: "SuperTrigger17", @@ -325,8 +340,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "16: Sends shivers down my spine, body's aching all the time", }, - triggerTags: []string{"Electrified-Floortile", "DND-generator", "traps"}, - triggerScore: 14, //nolint + triggerTags: []string{"Electrified-Floortile", "DND-generator", "traps", "Coldness", "Dark"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 14, //nolint }, { triggerID: "SuperTrigger18", @@ -336,8 +352,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "17: Goodbye, everybody, I've got to go", }, - triggerTags: []string{"Earthmaw-Trap", "DND-generator", "traps"}, - triggerScore: 13, //nolint + triggerTags: []string{"Earthmaw-Trap", "DND-generator", "traps", "Coldness", "Dark"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 13, //nolint }, { triggerID: "SuperTrigger19", @@ -355,8 +372,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "18: Gotta leave you all behind and face the truth", }, - triggerTags: []string{"Thunderstone-Mine", "DND-generator", "traps"}, - triggerScore: 12, //nolint + triggerTags: []string{"Thunderstone-Mine", "DND-generator", "traps", "Coldness", "Dark"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 12, //nolint }, { triggerID: "SuperTrigger20", @@ -382,8 +400,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ }, }, }, - triggerTags: []string{"Scythe-Blade", "DND-generator", "traps"}, - triggerScore: 11, //nolint + triggerTags: []string{"Scythe-Blade", "DND-generator", "traps"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 11, //nolint }, { triggerID: "SuperTrigger21", @@ -393,8 +412,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "20: I don't wanna die", }, - triggerTags: []string{"Female", "DND-generator", "Elf", "Monk", "NPCs"}, - triggerScore: 10, //nolint + triggerTags: []string{"Female", "DND-generator", "Elf", "Monk", "NPCs"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 10, //nolint }, { triggerID: "SuperTrigger22", @@ -404,8 +424,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "21: I sometimes wish I'd never been born at all", }, - triggerTags: []string{"Female", "DND-generator", "Halfling", "Cleric", "NPCs"}, - triggerScore: 9, //nolint + triggerTags: []string{"Female", "DND-generator", "Halfling", "Cleric", "NPCs"}, + triggerCreatedBy: "tarasov.da", + triggerScore: 9, //nolint }, { triggerID: "SuperTrigger23", @@ -415,8 +436,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "22", }, - triggerTags: []string{"Male", "DND-generator", "Human", "Soldier", "NPCs"}, - triggerScore: 8, //nolint + triggerTags: []string{"Male", "DND-generator", "Human", "Soldier", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 8, //nolint }, { triggerID: "SuperTrigger24", @@ -426,8 +448,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "23: I see a little silhouetto of a man", }, - triggerTags: []string{"Female", "DND-generator", "Human", "Barbarian", "NPCs"}, - triggerScore: 7, //nolint + triggerTags: []string{"Female", "DND-generator", "Human", "Barbarian", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 7, //nolint }, { triggerID: "SuperTrigger25", @@ -437,8 +460,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "24: Scaramouche, Scaramouche, will you do the Fandango?", }, - triggerTags: []string{"Male", "DND-generator", "Half-elf", "Monk", "NPCs"}, - triggerScore: 6, //nolint + triggerTags: []string{"Male", "DND-generator", "Half-elf", "Monk", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 6, //nolint }, { triggerID: "SuperTrigger26", @@ -448,8 +472,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "25: Thunderbolt and lightning, very, very fright'ning me", }, - triggerTags: []string{"Male", "DND-generator", "Elf", "Servant", "NPCs"}, - triggerScore: 5, //nolint + triggerTags: []string{"Male", "DND-generator", "Elf", "Servant", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 5, //nolint }, { triggerID: "SuperTrigger27", @@ -459,8 +484,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "26: (Galileo) Galileo, (Galileo) Galileo, Galileo Figaro magnifico", }, - triggerTags: []string{"Male", "DND-generator", "Elf", "Sorcerer", "NPCs"}, - triggerScore: 4, //nolint + triggerTags: []string{"Male", "DND-generator", "Elf", "Sorcerer", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 4, //nolint }, { triggerID: "SuperTrigger28", @@ -470,8 +496,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "27: I'm just a poor boy, nobody loves me", }, - triggerTags: []string{"Female", "DND-generator", "Human", "Bard", "NPCs"}, - triggerScore: 3, //nolint + triggerTags: []string{"Female", "DND-generator", "Human", "Bard", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 3, //nolint }, { triggerID: "SuperTrigger29", @@ -481,8 +508,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "28: He's just a poor boy from a poor family", }, - triggerTags: []string{"Female", "DND-generator", "Gnome", "Druid", "NPCs"}, - triggerScore: 2, //nolint + triggerTags: []string{"Female", "DND-generator", "Gnome", "Druid", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 2, //nolint }, { triggerID: "SuperTrigger30", @@ -492,8 +520,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ triggerDesc: fixtureIndexedField{ content: "29: Spare him his life from this monstrosity", }, - triggerTags: []string{"Female", "DND-generator", "Human", "Aristocrat", "NPCs"}, - triggerScore: 1, + triggerTags: []string{"Female", "DND-generator", "Human", "Aristocrat", "NPCs"}, + triggerCreatedBy: "internship2023", + triggerScore: 1, }, { triggerID: "SuperTrigger31", @@ -528,8 +557,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ }, }, }, - triggerTags: []string{"Something-extremely-new"}, - triggerScore: 0, + triggerTags: []string{"Something-extremely-new"}, + triggerCreatedBy: "internship2023", + triggerScore: 0, }, { triggerID: "SuperTrigger32", @@ -556,8 +586,9 @@ var IndexedTriggerTestCases = fixtureIndexedTriggers{ }, }, }, - triggerTags: []string{"Something-extremely-new"}, - triggerScore: 0, + triggerTags: []string{"Something-extremely-new"}, + triggerCreatedBy: "internship2023", + triggerScore: 0, }, }, } diff --git a/index/index.go b/index/index.go index f96b8201c..c96bd9033 100644 --- a/index/index.go +++ b/index/index.go @@ -12,7 +12,7 @@ const defaultIndexBatchSize = 1000 // TriggerIndex is index for moira.TriggerChecks type type TriggerIndex interface { - Search(filterTags []string, searchString string, onlyErrors bool, page int64, size int64) (searchResults []*moira.SearchResult, total int64, err error) + Search(options moira.SearchOptions) (searchResults []*moira.SearchResult, total int64, err error) Write(checks []*moira.TriggerCheck) error Delete(triggerIDs []string) error GetCount() (int64, error) diff --git a/index/mapping/trigger.go b/index/mapping/trigger.go index 6b058a948..875cda696 100644 --- a/index/mapping/trigger.go +++ b/index/mapping/trigger.go @@ -15,6 +15,8 @@ var ( TriggerDesc = FieldData{"Desc", "desc", 1} // TriggerTags represents field data for moira.Trigger.Tags TriggerTags = FieldData{"Tags", "tags", 0} + // TriggerCreatedBy represents field data for moira.Trigger.CreatedBy + TriggerCreatedBy = FieldData{"CreatedBy", "created_by", 0} // TriggerLastCheckScore represents field data for moira.CheckData score TriggerLastCheckScore = FieldData{"LastCheckScore", "", 0} ) @@ -25,6 +27,7 @@ type Trigger struct { Name string Desc string Tags []string + CreatedBy string LastCheckScore int64 } @@ -40,6 +43,7 @@ func (Trigger) GetDocumentMapping() *mapping.DocumentMapping { triggerMapping.AddFieldMappingsAt(TriggerName.GetName(), getStandardMapping()) triggerMapping.AddFieldMappingsAt(TriggerTags.GetName(), getKeywordMapping()) triggerMapping.AddFieldMappingsAt(TriggerDesc.GetName(), getStandardMapping()) + triggerMapping.AddFieldMappingsAt(TriggerCreatedBy.GetName(), getKeywordMapping()) triggerMapping.AddFieldMappingsAt(TriggerLastCheckScore.GetName(), getNumericMapping()) return triggerMapping @@ -52,6 +56,7 @@ func CreateIndexedTrigger(triggerCheck *moira.TriggerCheck) Trigger { Name: triggerCheck.Name, Desc: moira.UseString(triggerCheck.Desc), Tags: triggerCheck.Tags, + CreatedBy: triggerCheck.CreatedBy, LastCheckScore: triggerCheck.LastCheck.Score, } } diff --git a/index/search.go b/index/search.go index a6f8ca134..9b9a5b0d7 100644 --- a/index/search.go +++ b/index/search.go @@ -12,11 +12,11 @@ const ( ) // SearchTriggers search for triggers in index and returns slice of trigger IDs -func (index *Index) SearchTriggers(filterTags []string, searchString string, onlyErrors bool, page int64, size int64) (searchResults []*moira.SearchResult, total int64, err error) { +func (index *Index) SearchTriggers(options moira.SearchOptions) (searchResults []*moira.SearchResult, total int64, err error) { if !index.checkIfIndexIsReady() { return make([]*moira.SearchResult, 0), 0, fmt.Errorf("index is not ready, please try later") } - return index.triggerIndex.Search(filterTags, searchString, onlyErrors, page, size) + return index.triggerIndex.Search(options) } func (index *Index) checkIfIndexIsReady() bool { diff --git a/index/search_test.go b/index/search_test.go index ed38a726f..131c18348 100644 --- a/index/search_test.go +++ b/index/search_test.go @@ -39,211 +39,342 @@ func TestIndex_SearchTriggers(t *testing.T) { }) Convey("Search for triggers without pagination", t, func() { - page := int64(0) - size := int64(50) - tags := make([]string, 0) - searchString := "" - onlyErrors := false + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } Convey("No tags, no searchString, onlyErrors = false", func() { - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)) + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)) So(count, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("No tags, no searchString, onlyErrors = false, size = -1 (must return all triggers)", func() { - size = -1 - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)) + searchOptions.Size = -1 + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)) So(count, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("OnlyErrors = true", func() { - size = 50 - onlyErrors = true - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[:30]) + searchOptions.Size = 50 + searchOptions.OnlyProblems = true + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[:30]) So(count, ShouldEqual, 30) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags", func() { - onlyErrors = true - tags = []string{"encounters", "Kobold"} - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[1:3]) + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"encounters", "Kobold"} + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[1:3]) So(count, ShouldEqual, 2) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, several tags", func() { - onlyErrors = false - tags = []string{"Something-extremely-new"} - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[30:]) + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{"Something-extremely-new"} + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[30:]) So(count, ShouldEqual, 2) So(err, ShouldBeNil) }) Convey("Empty list should be", func() { - onlyErrors = true - tags = []string{"Something-extremely-new"} - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"Something-extremely-new"} + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldBeEmpty) So(count, ShouldBeZeroValue) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, no tags, several text terms", func() { - onlyErrors = true - tags = make([]string, 0) - searchString = "dragonshield medium" - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[2:3]) + searchOptions.OnlyProblems = true + searchOptions.Tags = make([]string, 0) + searchOptions.SearchString = "dragonshield medium" + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[2:3]) So(count, ShouldEqual, 1) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags, several text terms", func() { - onlyErrors = true - tags = []string{"traps"} - searchString = "deadly" //nolint + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" //nolint deadlyTraps := []int{10, 14, 18, 19} deadlyTrapsSearchResults := make([]*moira.SearchResult, 0) for _, ind := range deadlyTraps { - deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldResemble, deadlyTrapsSearchResults) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) + + Convey("OnlyErrors = false, no tags, no terms, with createdBy", func() { + searchOptions.OnlyProblems = false + searchOptions.Tags = make([]string, 0) + searchOptions.SearchString = "" + searchOptions.CreatedBy = "test" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[0:4]) + So(count, ShouldEqual, 4) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, one tag, no terms, with createdBy", func() { + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"shadows"} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "tarasov.da" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[14:16]) + So(count, ShouldEqual, 2) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, several tags, one text term, with createdBy", func() { + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"Coldness", "Dark"} + searchOptions.SearchString = "deadly" + searchOptions.CreatedBy = "tarasov.da" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[18:19]) + So(count, ShouldEqual, 1) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, several tags, no text, with EMPTY createdBy", func() { + searchOptions.OnlyProblems = true + searchOptions.SearchString = "" + searchOptions.Tags = []string{"Darkness", "DND-generator"} + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[5:7]) + So(count, ShouldEqual, 2) + So(err, ShouldBeNil) + }) }) Convey("Search for triggers with pagination", t, func() { - page := int64(0) - size := int64(10) - tags := make([]string, 0) - searchString := "" - onlyErrors := false + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 10, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } Convey("No tags, no searchString, onlyErrors = false, page -> 0, size -> 10", func() { - searchResults, total, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[:10]) + searchResults, total, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[:10]) So(total, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("No tags, no searchString, onlyErrors = false, page -> 1, size -> 10", func() { - page = 1 - searchResults, total, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[10:20]) + searchOptions.Page = 1 + searchResults, total, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[10:20]) So(total, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("No tags, no searchString, onlyErrors = false, page -> 1, size -> 20", func() { - page = 1 - size = 20 - searchResults, total, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[20:]) + searchOptions.Page = 1 + searchOptions.Size = 20 + searchResults, total, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[20:]) So(total, ShouldEqual, 32) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags, several text terms, page -> 0, size 2", func() { - page = 0 - size = 2 - onlyErrors = true - tags = []string{"traps"} - searchString = "deadly" + searchOptions.Page = 0 + searchOptions.Size = 2 + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" deadlyTraps := []int{10, 14, 18, 19} deadlyTrapsSearchResults := make([]*moira.SearchResult, 0) for _, ind := range deadlyTraps { - deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + deadlyTrapsSearchResults = append(deadlyTrapsSearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldResemble, deadlyTrapsSearchResults[:2]) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) Convey("OnlyErrors = true, several tags, several text terms, page -> 1, size 10", func() { - page = 1 - size = 10 - onlyErrors = true - tags = []string{"traps"} - searchString = "deadly" + searchOptions.Page = 1 + searchOptions.Size = 10 + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"traps"} + searchOptions.SearchString = "deadly" - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldBeEmpty) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) + + Convey("OnlyErrors = true, several tags, no terms, with createdBy, page -> 0, size 2", func() { + searchOptions.Page = 0 + searchOptions.Size = 2 + searchOptions.OnlyProblems = true + searchOptions.Tags = []string{"Human", "NPCs"} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "internship2023" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[22:24]) + So(count, ShouldEqual, 4) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = true, one tags, no terms, with createdBy, page -> 0, size 5", func() { + searchOptions.Page = 0 + searchOptions.Size = 5 + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{"Something-extremely-new"} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "internship2023" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[30:32]) + So(count, ShouldEqual, 2) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = false, no tags, no terms, with EMPTY createdBy, page -> 0, size 3", func() { + searchOptions.Page = 0 + searchOptions.Size = 3 + searchOptions.OnlyProblems = false + searchOptions.Tags = []string{} + searchOptions.SearchString = "" + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[4:7]) + So(count, ShouldEqual, 3) + So(err, ShouldBeNil) + }) }) Convey("Search for triggers by description", t, func() { - page := int64(0) - size := int64(50) - tags := make([]string, 0) - searchString := "" - onlyErrors := false + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } Convey("OnlyErrors = false, search by name and description, 0 results", func() { - searchString = "life female druid" - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchOptions.SearchString = "life female druid" + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldBeEmpty) So(count, ShouldEqual, 0) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, search by name and description, 3 results", func() { - searchString = "easy" + searchOptions.SearchString = "easy" easy := []int{4, 9, 30, 31} easySearchResults := make([]*moira.SearchResult, 0) for _, ind := range easy { - easySearchResults = append(easySearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + easySearchResults = append(easySearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldResemble, easySearchResults) So(count, ShouldEqual, 4) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, search by name and description, 1 result", func() { - searchString = "little monster" - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) - So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchString)[4:5]) + searchOptions.SearchString = "little monster" + searchResults, count, err := index.SearchTriggers(searchOptions) + So(searchResults, ShouldResemble, triggerTestCases.ToSearchResults(searchOptions.SearchString)[4:5]) So(count, ShouldEqual, 1) So(err, ShouldBeNil) }) Convey("OnlyErrors = false, search by description and tags, 2 results", func() { - searchString = "mama" - tags := []string{"traps"} + searchOptions.SearchString = "mama" + searchOptions.Tags = []string{"traps"} mamaTraps := []int{11, 19} mamaTrapsSearchResults := make([]*moira.SearchResult, 0) for _, ind := range mamaTraps { - mamaTrapsSearchResults = append(mamaTrapsSearchResults, triggerTestCases.ToSearchResults(searchString)[ind]) + mamaTrapsSearchResults = append(mamaTrapsSearchResults, triggerTestCases.ToSearchResults(searchOptions.SearchString)[ind]) } - searchResults, count, err := index.SearchTriggers(tags, searchString, onlyErrors, page, size) + searchResults, count, err := index.SearchTriggers(searchOptions) So(searchResults, ShouldResemble, mamaTrapsSearchResults) So(count, ShouldEqual, 2) So(err, ShouldBeNil) }) + + Convey("OnlyErrors = true, search by description, no tags, with createdBy, 3 results", func() { + searchOptions.SearchString = "mama" + searchOptions.Tags = make([]string, 0) + searchOptions.CreatedBy = "monster" + searchOptions.NeedSearchByCreatedBy = true + + _, count, err := index.SearchTriggers(searchOptions) + So(count, ShouldEqual, 3) + So(err, ShouldBeNil) + }) + + Convey("OnlyErrors = false, search by description, no tags, with EMPTY createdBy, 1 result", func() { + searchOptions.SearchString = "little monster" + searchOptions.Tags = make([]string, 0) + searchOptions.CreatedBy = "" + searchOptions.NeedSearchByCreatedBy = true + + _, count, err := index.SearchTriggers(searchOptions) + So(count, ShouldEqual, 1) + So(err, ShouldBeNil) + }) }) } @@ -274,12 +405,17 @@ func TestIndex_SearchErrors(t *testing.T) { index.indexed = false Convey("Test search on non-ready index", t, func() { - page := int64(0) - size := int64(50) - tags := make([]string, 0) - searchString := "" - - actualTriggerIDs, total, err := index.SearchTriggers(tags, searchString, false, page, size) + searchOptions := moira.SearchOptions{ + Page: 0, + Size: 50, + OnlyProblems: false, + Tags: make([]string, 0), + SearchString: "", + CreatedBy: "", + NeedSearchByCreatedBy: false, + } + + actualTriggerIDs, total, err := index.SearchTriggers(searchOptions) So(actualTriggerIDs, ShouldBeEmpty) So(total, ShouldBeZeroValue) So(err, ShouldNotBeNil) diff --git a/interfaces.go b/interfaces.go index 1831018e4..8b5ffbd1f 100644 --- a/interfaces.go +++ b/interfaces.go @@ -215,8 +215,7 @@ type Searcher interface { Start() error Stop() error IsReady() bool - SearchTriggers(filterTags []string, searchString string, onlyErrors bool, - page int64, size int64) (searchResults []*SearchResult, total int64, err error) + SearchTriggers(options SearchOptions) (searchResults []*SearchResult, total int64, err error) } // PlotTheme is an interface to access plot theme styles diff --git a/mock/moira-alert/searcher.go b/mock/moira-alert/searcher.go index 2f4e78521..f6fc5adf8 100644 --- a/mock/moira-alert/searcher.go +++ b/mock/moira-alert/searcher.go @@ -49,9 +49,9 @@ func (mr *MockSearcherMockRecorder) IsReady() *gomock.Call { } // SearchTriggers mocks base method. -func (m *MockSearcher) SearchTriggers(arg0 []string, arg1 string, arg2 bool, arg3, arg4 int64) ([]*moira.SearchResult, int64, error) { +func (m *MockSearcher) SearchTriggers(arg0 moira.SearchOptions) ([]*moira.SearchResult, int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SearchTriggers", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "SearchTriggers", arg0) ret0, _ := ret[0].([]*moira.SearchResult) ret1, _ := ret[1].(int64) ret2, _ := ret[2].(error) @@ -59,9 +59,9 @@ func (m *MockSearcher) SearchTriggers(arg0 []string, arg1 string, arg2 bool, arg } // SearchTriggers indicates an expected call of SearchTriggers. -func (mr *MockSearcherMockRecorder) SearchTriggers(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { +func (mr *MockSearcherMockRecorder) SearchTriggers(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchTriggers", reflect.TypeOf((*MockSearcher)(nil).SearchTriggers), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchTriggers", reflect.TypeOf((*MockSearcher)(nil).SearchTriggers), arg0) } // Start mocks base method. From 5d16cf1a43d082ff2457adfd839a44554604402d Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:49:52 +0300 Subject: [PATCH 21/46] fix(senders): fix mapstructure types (#911) --- senders/discord/init.go | 14 ++++---- senders/mail/mail.go | 30 ++++++++-------- senders/mattermost/sender.go | 28 +++++++-------- senders/mattermost/sender_internal_test.go | 4 +-- senders/mattermost/sender_test.go | 6 ++-- senders/msteams/msteams.go | 18 ++++------ senders/msteams/msteams_test.go | 8 ++--- senders/opsgenie/init.go | 10 +++--- senders/pagerduty/init.go | 8 ++--- senders/pushover/pushover.go | 11 +++--- senders/script/script.go | 13 +++---- senders/slack/slack.go | 19 +++++----- senders/slack/slack_test.go | 9 +++-- senders/telegram/init.go | 15 ++++---- senders/twilio/twilio.go | 41 ++++++++++------------ senders/twilio/twilio_test.go | 6 ++-- senders/victorops/init.go | 13 +++---- senders/webhook/webhook.go | 27 +++++++------- 18 files changed, 130 insertions(+), 150 deletions(-) diff --git a/senders/discord/init.go b/senders/discord/init.go index 2d9a40ccf..6461c1761 100644 --- a/senders/discord/init.go +++ b/senders/discord/init.go @@ -18,7 +18,7 @@ const ( ) // Structure that represents the Discord configuration in the YAML file -type discord struct { +type config struct { Token string `mapstructure:"token"` FrontURI string `mapstructure:"front_uri"` } @@ -35,21 +35,21 @@ type Sender struct { // Init reads the yaml config func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var ds discord - err := mapstructure.Decode(senderSettings, &ds) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to discord config: %w", err) } - token := ds.Token - if token == "" { + + if cfg.Token == "" { return fmt.Errorf("cannot read the discord token from the config") } - sender.session, err = discordgo.New("Bot " + token) + sender.session, err = discordgo.New("Bot " + cfg.Token) if err != nil { return fmt.Errorf("error creating discord session: %s", err) } sender.logger = logger - sender.frontURI = ds.FrontURI + sender.frontURI = cfg.FrontURI sender.location = location handleMsg := func(s *discordgo.Session, m *discordgo.MessageCreate) { diff --git a/senders/mail/mail.go b/senders/mail/mail.go index 626a8e793..478eff39e 100644 --- a/senders/mail/mail.go +++ b/senders/mail/mail.go @@ -6,7 +6,6 @@ import ( "html/template" "net/smtp" "path/filepath" - "strconv" "time" "github.com/mitchellh/mapstructure" @@ -14,12 +13,12 @@ import ( ) // Structure that represents the Mail configuration in the YAML file -type mail struct { +type config struct { MailFrom string `mapstructure:"mail_from"` SMTPHello string `mapstructure:"smtp_hello"` SMTPHost string `mapstructure:"smtp_host"` - SMTPPort string `mapstructure:"smtp_port"` - InsecureTLS string `mapstructure:"insecure_tls"` + SMTPPort int64 `mapstructure:"smtp_port"` + InsecureTLS bool `mapstructure:"insecure_tls"` FrontURI string `mapstructure:"front_uri"` SMTPPass string `mapstructure:"smtp_pass"` SMTPUser string `mapstructure:"smtp_user"` @@ -59,21 +58,22 @@ func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, loca } func (sender *Sender) fillSettings(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var m mail - err := mapstructure.Decode(senderSettings, &m) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to mail config: %w", err) } + sender.logger = logger - sender.From = m.MailFrom - sender.SMTPHello = m.SMTPHello - sender.SMTPHost = m.SMTPHost - sender.SMTPPort, _ = strconv.ParseInt(m.SMTPPort, 10, 64) - sender.InsecureTLS, _ = strconv.ParseBool(m.InsecureTLS) - sender.FrontURI = m.FrontURI - sender.Password = m.SMTPPass - sender.Username = m.SMTPUser - sender.TemplateFile = m.TemplateFile + sender.From = cfg.MailFrom + sender.SMTPHello = cfg.SMTPHello + sender.SMTPHost = cfg.SMTPHost + sender.SMTPPort = cfg.SMTPPort + sender.InsecureTLS = cfg.InsecureTLS + sender.FrontURI = cfg.FrontURI + sender.Password = cfg.SMTPPass + sender.Username = cfg.SMTPUser + sender.TemplateFile = cfg.TemplateFile sender.location = location sender.dateTimeFormat = dateTimeFormat if sender.Username == "" { diff --git a/senders/mattermost/sender.go b/senders/mattermost/sender.go index 52ec2e25e..7cd1bc5be 100644 --- a/senders/mattermost/sender.go +++ b/senders/mattermost/sender.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "fmt" "net/http" - "strconv" "strings" "time" @@ -16,9 +15,9 @@ import ( ) // Structure that represents the Mattermost configuration in the YAML file -type mattermost struct { +type config struct { Url string `mapstructure:"url"` - InsecureTLS string `mapstructure:"insecure_tls"` + InsecureTLS bool `mapstructure:"insecure_tls"` APIToken string `mapstructure:"api_token"` FrontURI string `mapstructure:"front_uri"` } @@ -35,41 +34,38 @@ type Sender struct { // Init configures Sender. func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, _ string) error { - var mm mattermost - err := mapstructure.Decode(senderSettings, &mm) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to mattermost config: %w", err) } - url := mm.Url - if url == "" { + + if cfg.Url == "" { return fmt.Errorf("can not read Mattermost url from config") } - client := model.NewAPIv4Client(url) + client := model.NewAPIv4Client(cfg.Url) - insecureTLS, err := strconv.ParseBool(mm.InsecureTLS) if err != nil { return fmt.Errorf("can not parse insecure_tls: %v", err) } client.HTTPClient = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ - InsecureSkipVerify: insecureTLS, + InsecureSkipVerify: cfg.InsecureTLS, }, }, } sender.client = client - token := mm.APIToken - if token == "" { + if cfg.APIToken == "" { return fmt.Errorf("can not read Mattermost api_token from config") } - sender.client.SetToken(token) + sender.client.SetToken(cfg.APIToken) - frontURI := mm.FrontURI - if frontURI == "" { + if cfg.FrontURI == "" { return fmt.Errorf("can not read Mattermost front_uri from config") } - sender.frontURI = frontURI + sender.frontURI = cfg.FrontURI sender.location = location sender.logger = logger diff --git a/senders/mattermost/sender_internal_test.go b/senders/mattermost/sender_internal_test.go index 18b5950df..104a9c776 100644 --- a/senders/mattermost/sender_internal_test.go +++ b/senders/mattermost/sender_internal_test.go @@ -25,7 +25,7 @@ func TestSendEvents(t *testing.T) { "url": "qwerty", "api_token": "qwerty", "front_uri": "qwerty", - "insecure_tls": "true", + "insecure_tls": true, } err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldBeNil) @@ -80,7 +80,7 @@ func TestBuildMessage(t *testing.T) { senderSettings := map[string]interface{}{ "url": "qwerty", "api_token": "qwerty", // redundant, but necessary config "front_uri": "http://moira.url", - "insecure_tls": "true", + "insecure_tls": true, } location, _ := time.LoadLocation("UTC") err := sender.Init(senderSettings, logger, location, "") diff --git a/senders/mattermost/sender_test.go b/senders/mattermost/sender_test.go index 493183895..75e12c909 100644 --- a/senders/mattermost/sender_test.go +++ b/senders/mattermost/sender_test.go @@ -18,7 +18,7 @@ func TestInit(t *testing.T) { senderSettings := map[string]interface{}{ "api_token": "qwerty", "front_uri": "qwerty", - "insecure_tls": "true", + "insecure_tls": true, } err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldNotBeNil) @@ -29,7 +29,7 @@ func TestInit(t *testing.T) { "url": "", "api_token": "qwerty", "front_uri": "qwerty", - "insecure_tls": "true", + "insecure_tls": true, } err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldNotBeNil) @@ -64,7 +64,7 @@ func TestInit(t *testing.T) { "url": "qwerty", "api_token": "qwerty", "front_uri": "qwerty", - "insecure_tls": "true", + "insecure_tls": true, } err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldBeNil) diff --git a/senders/msteams/msteams.go b/senders/msteams/msteams.go index 0102dfe19..bf91c2f1b 100644 --- a/senders/msteams/msteams.go +++ b/senders/msteams/msteams.go @@ -7,7 +7,6 @@ import ( "io" "net/http" "net/url" - "strconv" "strings" "time" @@ -37,9 +36,9 @@ var headers = map[string]string{ } // Structure that represents the MSTeams configuration in the YAML file -type msTeams struct { +type config struct { FrontURI string `mapstructure:"front_uri"` - MaxEvents string `mapstructure:"max_events"` + MaxEvents int `mapstructure:"max_events"` } // Sender implements moira sender interface via MS Teams @@ -53,19 +52,16 @@ type Sender struct { // Init initialises settings required for full functionality func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var msteams msTeams - err := mapstructure.Decode(senderSettings, &msteams) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to msteams config: %w", err) } + sender.logger = logger sender.location = location - sender.frontURI = msteams.FrontURI - maxEvents, err := strconv.Atoi(msteams.MaxEvents) - if err != nil { - return fmt.Errorf("max_events should be an integer: %w", err) - } - sender.maxEvents = maxEvents + sender.frontURI = cfg.FrontURI + sender.maxEvents = cfg.MaxEvents sender.client = &http.Client{ Timeout: time.Duration(30) * time.Second, //nolint } diff --git a/senders/msteams/msteams_test.go b/senders/msteams/msteams_test.go index 2b796a73e..faf612731 100644 --- a/senders/msteams/msteams_test.go +++ b/senders/msteams/msteams_test.go @@ -16,12 +16,8 @@ func TestInit(t *testing.T) { Convey("Init tests", t, func() { sender := Sender{} senderSettings := map[string]interface{}{ - "max_events": "-1", + "max_events": -1, } - Convey("Empty map should fail", func() { - err := sender.Init(map[string]interface{}{}, logger, nil, "") - So(err, ShouldNotResemble, nil) - }) Convey("Minimal settings", func() { err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldResemble, nil) @@ -36,7 +32,7 @@ func TestMSTeamsHttpResponse(t *testing.T) { logger, _ := logging.ConfigureLog("stdout", "info", "test", true) location, _ := time.LoadLocation("UTC") _ = sender.Init(map[string]interface{}{ - "max_events": "-1", + "max_events": -1, }, logger, location, "") event := moira.NotificationEvent{ TriggerID: "TriggerID", diff --git a/senders/opsgenie/init.go b/senders/opsgenie/init.go index ab04e82dc..5b627c6f1 100644 --- a/senders/opsgenie/init.go +++ b/senders/opsgenie/init.go @@ -12,7 +12,7 @@ import ( ) // Structure that represents the OpsGenie configuration in the YAML file -type opsGenie struct { +type config struct { APIKey string `mapstructure:"api_key"` FrontURI string `mapstructure:"front_uri"` } @@ -32,14 +32,14 @@ type Sender struct { // Init initializes the opsgenie sender func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var opsgenie opsGenie + var cfg config - err := mapstructure.Decode(senderSettings, &opsgenie) + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to opsgenie config: %w", err) } - sender.apiKey = opsgenie.APIKey + sender.apiKey = cfg.APIKey if sender.apiKey == "" { return fmt.Errorf("cannot read the api_key from the sender settings") } @@ -54,7 +54,7 @@ func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, loca return fmt.Errorf("error while creating opsgenie client: %s", err) } - sender.frontURI = opsgenie.FrontURI + sender.frontURI = cfg.FrontURI sender.logger = logger sender.location = location return nil diff --git a/senders/pagerduty/init.go b/senders/pagerduty/init.go index 188cf2ce5..0679511df 100644 --- a/senders/pagerduty/init.go +++ b/senders/pagerduty/init.go @@ -10,7 +10,7 @@ import ( ) // Structure that represents the PagerDuty configuration in the YAML file -type pagerDuty struct { +type config struct { FrontURI string `mapstructure:"front_uri"` } @@ -27,13 +27,13 @@ type Sender struct { // Init loads yaml config, configures the pagerduty client func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var pagerduty pagerDuty - err := mapstructure.Decode(senderSettings, &pagerduty) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to pagerduty config: %w", err) } - sender.frontURI = pagerduty.FrontURI + sender.frontURI = cfg.FrontURI sender.imageStoreID, sender.imageStore, sender.imageStoreConfigured = senders.ReadImageStoreConfig(senderSettings, sender.ImageStores, logger) diff --git a/senders/pushover/pushover.go b/senders/pushover/pushover.go index c796f9e7f..8e87c1e15 100644 --- a/senders/pushover/pushover.go +++ b/senders/pushover/pushover.go @@ -16,7 +16,7 @@ const titleLimit = 250 const urlLimit = 512 // Structure that represents the Pushover configuration in the YAML file -type pushover struct { +type config struct { APIToken string `mapstructure:"api_token"` FrontURI string `mapstructure:"front_uri"` } @@ -33,18 +33,19 @@ type Sender struct { // Init read yaml config func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var p pushover - err := mapstructure.Decode(senderSettings, &p) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to pushover config: %w", err) } - sender.apiToken = p.APIToken + + sender.apiToken = cfg.APIToken if sender.apiToken == "" { return fmt.Errorf("can not read pushover api_token from config") } sender.client = pushover_client.New(sender.apiToken) sender.logger = logger - sender.frontURI = p.FrontURI + sender.frontURI = cfg.FrontURI sender.location = location return nil } diff --git a/senders/script/script.go b/senders/script/script.go index d34f3bfec..6d55e09ad 100644 --- a/senders/script/script.go +++ b/senders/script/script.go @@ -14,7 +14,7 @@ import ( ) // Structure that represents the Script configuration in the YAML file -type script struct { +type config struct { Name string `mapstructure:"name"` Exec string `mapstructure:"exec"` } @@ -35,19 +35,20 @@ type scriptNotification struct { // Init read yaml config func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var s script - err := mapstructure.Decode(senderSettings, &s) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to script config: %w", err) } - if s.Name == "" { + + if cfg.Name == "" { return fmt.Errorf("required name for sender type script") } - _, _, err = parseExec(s.Exec) + _, _, err = parseExec(cfg.Exec) if err != nil { return err } - sender.exec = s.Exec + sender.exec = cfg.Exec sender.logger = logger return nil } diff --git a/senders/slack/slack.go b/senders/slack/slack.go index 84a01d1b5..bdb43f899 100644 --- a/senders/slack/slack.go +++ b/senders/slack/slack.go @@ -3,7 +3,6 @@ package slack import ( "bytes" "fmt" - "strconv" "strings" "time" @@ -42,9 +41,9 @@ var stateEmoji = map[moira.State]string{ } // Structure that represents the Slack configuration in the YAML file -type slack struct { +type config struct { APIToken string `mapstructure:"api_token"` - UseEmoji string `mapstructure:"use_emoji"` + UseEmoji bool `mapstructure:"use_emoji"` FrontURI string `mapstructure:"front_uri"` } @@ -59,20 +58,20 @@ type Sender struct { // Init read yaml config func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var s slack - err := mapstructure.Decode(senderSettings, &s) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to slack config: %w", err) } - apiToken := s.APIToken - if apiToken == "" { + + if cfg.APIToken == "" { return fmt.Errorf("can not read slack api_token from config") } - sender.useEmoji, _ = strconv.ParseBool(s.UseEmoji) + sender.useEmoji = cfg.UseEmoji sender.logger = logger - sender.frontURI = s.FrontURI + sender.frontURI = cfg.FrontURI sender.location = location - sender.client = slack_client.New(apiToken) + sender.client = slack_client.New(cfg.APIToken) return nil } diff --git a/senders/slack/slack_test.go b/senders/slack/slack_test.go index 9adac7976..c368d178e 100644 --- a/senders/slack/slack_test.go +++ b/senders/slack/slack_test.go @@ -34,24 +34,23 @@ func TestInit(t *testing.T) { }) Convey("use_emoji set to false", func() { - senderSettings["use_emoji"] = "false" + senderSettings["use_emoji"] = false err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldBeNil) So(sender, ShouldResemble, Sender{logger: logger, client: client}) }) Convey("use_emoji set to true", func() { - senderSettings["use_emoji"] = "true" + senderSettings["use_emoji"] = true err := sender.Init(senderSettings, logger, nil, "") So(err, ShouldBeNil) So(sender, ShouldResemble, Sender{logger: logger, useEmoji: true, client: client}) }) Convey("use_emoji set to something wrong", func() { - senderSettings["use_emoji"] = "123" + senderSettings["use_emoji"] = 123 err := sender.Init(senderSettings, logger, nil, "") - So(err, ShouldBeNil) - So(sender, ShouldResemble, Sender{logger: logger, useEmoji: false, client: client}) + So(err, ShouldNotBeNil) }) }) }) diff --git a/senders/telegram/init.go b/senders/telegram/init.go index f8cf9215c..e477f3e4d 100644 --- a/senders/telegram/init.go +++ b/senders/telegram/init.go @@ -32,7 +32,7 @@ var ( ) // Structure that represents the Telegram configuration in the YAML file -type telegram struct { +type config struct { APIToken string `mapstructure:"api_token"` FrontURI string `mapstructure:"front_uri"` } @@ -60,18 +60,17 @@ func removeTokenFromError(err error, bot *telebot.Bot) error { // Init loads yaml config, configures and starts telegram bot func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var tg telegram - err := mapstructure.Decode(senderSettings, &tg) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to telegram config: %w", err) } - apiToken := tg.APIToken - if apiToken == "" { + + if cfg.APIToken == "" { return fmt.Errorf("can not read telegram api_token from config") } - - sender.apiToken = apiToken - sender.frontURI = tg.FrontURI + sender.apiToken = cfg.APIToken + sender.frontURI = cfg.FrontURI sender.logger = logger sender.location = location sender.bot, err = telebot.NewBot(telebot.Settings{ diff --git a/senders/twilio/twilio.go b/senders/twilio/twilio.go index 717396528..ef51f0970 100644 --- a/senders/twilio/twilio.go +++ b/senders/twilio/twilio.go @@ -10,14 +10,14 @@ import ( ) // Structure that represents the Twilio configuration in the YAML file -type twilio struct { +type config struct { Type string `mapstructure:"type"` APIAsid string `mapstructure:"api_asid"` APIAuthToken string `mapstructure:"api_authtoken"` APIFromPhone string `mapstructure:"api_fromphone"` VoiceURL string `mapstructure:"voiceurl"` - TwimletsEcho string `mapstructure:"twimlets_echo"` - AppendMessage string `mapstructure:"append_message"` + TwimletsEcho bool `mapstructure:"twimlets_echo"` + AppendMessage bool `mapstructure:"append_message"` } // Sender implements moira sender interface via twilio @@ -38,53 +38,48 @@ type twilioSender struct { // Init read yaml config func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var t twilio - err := mapstructure.Decode(senderSettings, &t) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to twilio config: %w", err) } - apiType := t.Type + apiType := cfg.Type - apiASID := t.APIAsid - if apiASID == "" { + if cfg.APIAsid == "" { return fmt.Errorf("can not read [%s] api_sid param from config", apiType) } - apiAuthToken := t.APIAuthToken - if apiAuthToken == "" { + if cfg.APIAuthToken == "" { return fmt.Errorf("can not read [%s] api_authtoken param from config", apiType) } - apiFromPhone := t.APIFromPhone - if apiFromPhone == "" { + if cfg.APIFromPhone == "" { return fmt.Errorf("can not read [%s] api_fromphone param from config", apiType) } - twilioClient := twilio_client.NewClient(apiASID, apiAuthToken) + twilioClient := twilio_client.NewClient(cfg.APIAsid, cfg.APIAuthToken) - twilioSender1 := twilioSender{ + tSender := twilioSender{ client: twilioClient, - APIFromPhone: apiFromPhone, + APIFromPhone: cfg.APIFromPhone, logger: logger, location: location, } switch apiType { case "twilio sms": - sender.sender = &twilioSenderSms{twilioSender1} + sender.sender = &twilioSenderSms{tSender} case "twilio voice": - twimletsEcho := t.TwimletsEcho == "true" //nolint - appendMessage := (t.AppendMessage == "true") || (twimletsEcho) + appendMessage := cfg.AppendMessage || cfg.TwimletsEcho - voiceURL := t.VoiceURL - if voiceURL == "" && !twimletsEcho { + if cfg.VoiceURL == "" && !cfg.TwimletsEcho { return fmt.Errorf("can not read [%s] voiceurl param from config", apiType) } sender.sender = &twilioSenderVoice{ - twilioSender: twilioSender1, - voiceURL: voiceURL, - twimletsEcho: twimletsEcho, + twilioSender: tSender, + voiceURL: cfg.VoiceURL, + twimletsEcho: cfg.TwimletsEcho, appendMessage: appendMessage, } diff --git a/senders/twilio/twilio_test.go b/senders/twilio/twilio_test.go index aca9d2d24..98fb01689 100644 --- a/senders/twilio/twilio_test.go +++ b/senders/twilio/twilio_test.go @@ -71,7 +71,7 @@ func TestInit(t *testing.T) { Convey("has voice url", func() { settings["voiceurl"] = "url here" Convey("append_message == true", func() { - settings["append_message"] = "true" + settings["append_message"] = true err := sender.Init(settings, logger, location, "15:04") So(err, ShouldBeNil) So(sender, ShouldResemble, Sender{sender: &twilioSenderVoice{ @@ -86,8 +86,8 @@ func TestInit(t *testing.T) { }}) }) - Convey("append_message is something another string", func() { - settings["append_message"] = "something another string" + Convey("append_message is false", func() { + settings["append_message"] = false err := sender.Init(settings, logger, location, "15:04") So(err, ShouldBeNil) So(sender, ShouldResemble, Sender{sender: &twilioSenderVoice{ diff --git a/senders/victorops/init.go b/senders/victorops/init.go index a50a5a0d8..f9462f91c 100644 --- a/senders/victorops/init.go +++ b/senders/victorops/init.go @@ -11,7 +11,7 @@ import ( ) // Structure that represents the VictorOps configuration in the YAML file -type victorOps struct { +type config struct { RoutingURL string `mapstructure:"routing_url"` ImageStore string `mapstructure:"image_store"` FrontURI string `mapstructure:"front_uri"` @@ -34,17 +34,18 @@ type Sender struct { // Init loads yaml config, configures the victorops sender func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var victorops victorOps - err := mapstructure.Decode(senderSettings, &victorops) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to victorops config: %w", err) } - sender.routingURL = victorops.RoutingURL + + sender.routingURL = cfg.RoutingURL if sender.routingURL == "" { return fmt.Errorf("cannot read the routing url from the yaml config") } - sender.imageStoreID = victorops.ImageStore + sender.imageStoreID = cfg.ImageStore if sender.imageStoreID == "" { logger.Warning().Msg("Cannot read image_store from the config, will not be able to attach plot images to events") } else { @@ -61,7 +62,7 @@ func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, loca sender.client = api.NewClient(sender.routingURL, nil) - sender.frontURI = victorops.FrontURI + sender.frontURI = cfg.FrontURI sender.logger = logger sender.location = location diff --git a/senders/webhook/webhook.go b/senders/webhook/webhook.go index 4ca1bf512..7bbfe44f0 100644 --- a/senders/webhook/webhook.go +++ b/senders/webhook/webhook.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "net/http" - "strconv" "time" "github.com/mitchellh/mapstructure" @@ -12,12 +11,12 @@ import ( ) // Structure that represents the Webhook configuration in the YAML file -type webHook struct { +type config struct { Name string `mapstructure:"name"` URL string `mapstructure:"url"` User string `mapstructure:"user"` Password string `mapstructure:"password"` - Timeout string `mapstructure:"timeout"` + Timeout int `mapstructure:"timeout"` } // Sender implements moira sender interface via webhook @@ -32,35 +31,33 @@ type Sender struct { // Init read yaml config func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { - var webhook webHook - err := mapstructure.Decode(senderSettings, &webhook) + var cfg config + err := mapstructure.Decode(senderSettings, &cfg) if err != nil { return fmt.Errorf("failed to decode senderSettings to webhook config: %w", err) } - if webhook.Name == "" { + if cfg.Name == "" { return fmt.Errorf("required name for sender type webhook") } - sender.url = webhook.URL + sender.url = cfg.URL if sender.url == "" { return fmt.Errorf("can not read url from config") } - sender.user, sender.password = webhook.User, webhook.Password + sender.user, sender.password = cfg.User, cfg.Password sender.headers = map[string]string{ "User-Agent": "Moira", "Content-Type": "application/json", } - timeout := 30 - timeoutRaw := webhook.Timeout - if timeoutRaw != "" { - timeout, err = strconv.Atoi(timeoutRaw) - if err != nil { - return fmt.Errorf("can not read timeout from config: %s", err.Error()) - } + var timeout int + if cfg.Timeout != 0 { + timeout = cfg.Timeout + } else { + timeout = 30 } sender.log = logger From f6fb1c597d23ffcab0bc44eda2a56fd994f404e9 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Thu, 21 Sep 2023 18:30:59 +0600 Subject: [PATCH 22/46] fix(cli): fix validation when upgrade 2.7->2.8 (#918) --- cmd/cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 01ead7ed6..5fda18983 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -23,7 +23,7 @@ var ( GoVersion = "unknown" ) -var moiraValidVersions = []string{"2.3", "2.6"} +var moiraValidVersions = []string{"2.3", "2.6", "2.7"} var ( configFileName = flag.String("config", "/etc/moira/cli.yml", "Path to configuration file") From 001fa8c3cbf2b89d4c63bf66d6ce73cc6a728d16 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Thu, 21 Sep 2023 23:02:54 +0600 Subject: [PATCH 23/46] fix(cli): fix renaming func (#919) --- cmd/cli/cluster.go | 15 ++++-- cmd/cli/cluster_test.go | 107 +++++++++++++++++++++++++++++++++++++ cmd/cli/from_2.7_to_2.8.go | 14 +++-- 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/cmd/cli/cluster.go b/cmd/cli/cluster.go index 747e53b3f..abd713c01 100644 --- a/cmd/cli/cluster.go +++ b/cmd/cli/cluster.go @@ -19,14 +19,19 @@ var tagSubscriptionsKeyPrefixNew = "{moira-tag-subscriptions}:" var tagTriggersKeyKeyPrefixOld = "moira-tag-triggers:" var tagTriggersKeyKeyPrefixNew = "{moira-tag-triggers}:" -func renameKey(database moira.Database, oldKey string, newKey string) error { +func renameKey(database moira.Database, oldValue, newValue string) error { switch d := database.(type) { case *redis.DbConnector: - err := d.Client().Rename(d.Context(), oldKey, newKey).Err() + pipe := d.Client().TxPipeline() + iter := d.Client().Scan(d.Context(), 0, oldValue, 0).Iterator() + for iter.Next(d.Context()) { + oldKey := iter.Val() + newKey := strings.Replace(iter.Val(), oldValue, newValue, 1) + pipe.Rename(d.Context(), oldKey, newKey) + } + _, err := pipe.Exec(d.Context()) if err != nil { - if err.Error() != noSuchKeyError { - return err - } + return err } } diff --git a/cmd/cli/cluster_test.go b/cmd/cli/cluster_test.go index c4bc3fadf..0592eef22 100644 --- a/cmd/cli/cluster_test.go +++ b/cmd/cli/cluster_test.go @@ -8,6 +8,7 @@ import ( "github.com/moira-alert/moira" + rds "github.com/go-redis/redis/v8" . "github.com/smartystreets/goconvey/convey" ) @@ -295,3 +296,109 @@ func createDataWithNewKeys(database moira.Database) { client.SAdd(ctx, "{moira-tag-triggers}:tag3", "triggerID-0000000000003") } } + +func Test_renameKey(t *testing.T) { + logger, _ := logging.GetLogger("Test Worker") + oldKey := "my_test_key" + newKey := "my_new_test_key" + + Convey("Something was renamed", t, func() { + database := redis.NewTestDatabase(logger) + database.Flush() + defer database.Flush() + err := database.Client().Set(database.Context(), oldKey, "123", 0).Err() + So(err, ShouldBeNil) + + err = renameKey(database, oldKey, newKey) + So(err, ShouldBeNil) + + res, err := database.Client().Get(database.Context(), newKey).Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "123") + err = database.Client().Get(database.Context(), oldKey).Err() + So(err, ShouldEqual, rds.Nil) + }) + + Convey("Nothing was renamed", t, func() { + database := redis.NewTestDatabase(logger) + database.Flush() + defer database.Flush() + err := database.Client().Set(database.Context(), oldKey, "123", 0).Err() + So(err, ShouldBeNil) + + err = renameKey(database, "no_exist_key", newKey) + So(err, ShouldBeNil) + + err = database.Client().Get(database.Context(), newKey).Err() + So(err, ShouldEqual, rds.Nil) + + res, err := database.Client().Get(database.Context(), oldKey).Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "123") + }) +} + +func Test_changeKeysPrefix(t *testing.T) { + logger, _ := logging.GetLogger("Test Worker") + oldKey := "my_test_key" + newKey := "my_new_test_key" + + Convey("Something was renamed", t, func() { + database := redis.NewTestDatabase(logger) + database.Flush() + defer database.Flush() + err := database.Client().Set(database.Context(), oldKey+"1", "1", 0).Err() + So(err, ShouldBeNil) + err = database.Client().Set(database.Context(), oldKey+"2", "2", 0).Err() + So(err, ShouldBeNil) + err = database.Client().Set(database.Context(), oldKey+"3", "3", 0).Err() + So(err, ShouldBeNil) + + err = changeKeysPrefix(database, oldKey, newKey) + So(err, ShouldBeNil) + + res, err := database.Client().Get(database.Context(), newKey+"1").Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "1") + res, err = database.Client().Get(database.Context(), newKey+"2").Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "2") + res, err = database.Client().Get(database.Context(), newKey+"3").Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "3") + err = database.Client().Get(database.Context(), oldKey+"1").Err() + So(err, ShouldEqual, rds.Nil) + err = database.Client().Get(database.Context(), oldKey+"2").Err() + So(err, ShouldEqual, rds.Nil) + err = database.Client().Get(database.Context(), oldKey+"3").Err() + So(err, ShouldEqual, rds.Nil) + }) + + Convey("Nothing was renamed", t, func() { + database := redis.NewTestDatabase(logger) + database.Flush() + defer database.Flush() + err := database.Client().Set(database.Context(), oldKey+"1", "1", 0).Err() + So(err, ShouldBeNil) + err = database.Client().Set(database.Context(), oldKey+"2", "2", 0).Err() + So(err, ShouldBeNil) + err = database.Client().Set(database.Context(), oldKey+"3", "3", 0).Err() + So(err, ShouldBeNil) + + err = renameKey(database, "no_exist_key", newKey) + So(err, ShouldBeNil) + + err = database.Client().Get(database.Context(), newKey).Err() + So(err, ShouldEqual, rds.Nil) + + res, err := database.Client().Get(database.Context(), oldKey+"1").Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "1") + res, err = database.Client().Get(database.Context(), oldKey+"2").Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "2") + res, err = database.Client().Get(database.Context(), oldKey+"3").Result() + So(err, ShouldBeNil) + So(res, ShouldResemble, "3") + }) +} diff --git a/cmd/cli/from_2.7_to_2.8.go b/cmd/cli/from_2.7_to_2.8.go index 7a70f3f4a..61cc02fdb 100644 --- a/cmd/cli/from_2.7_to_2.8.go +++ b/cmd/cli/from_2.7_to_2.8.go @@ -1,17 +1,21 @@ package main -import "github.com/moira-alert/moira" +import ( + "fmt" + + "github.com/moira-alert/moira" +) func updateFrom27(logger moira.Logger, dataBase moira.Database) error { logger.Info().Msg("Update 2.7 -> 2.8 was started") logger.Info().Msg("Rename keys was started") if err := updateSubscriptionKeyForAnonymous(logger, dataBase); err != nil { - return err + return fmt.Errorf("failed updateSubscriptionKeyForAnonymous, has error %w", err) } if err := updateContactKeyForAnonymous(logger, dataBase); err != nil { - return err + return fmt.Errorf("failed updateContactKeyForAnonymous, has error %w", err) } logger.Info().Msg("Update 2.7 -> 2.8 was finished") @@ -42,7 +46,7 @@ var ( ) func updateSubscriptionKeyForAnonymous(logger moira.Logger, database moira.Database) error { - err := changeKeysPrefix(database, subscriptionKeyForAnonymousOld, subscriptionKeyForAnonymousNew) + err := renameKey(database, subscriptionKeyForAnonymousOld, subscriptionKeyForAnonymousNew) if err != nil { return err } @@ -53,7 +57,7 @@ func updateSubscriptionKeyForAnonymous(logger moira.Logger, database moira.Datab } func updateContactKeyForAnonymous(logger moira.Logger, database moira.Database) error { - err := changeKeysPrefix(database, contactKeyForAnonymousOld, contactKeyForAnonymousNew) + err := renameKey(database, contactKeyForAnonymousOld, contactKeyForAnonymousNew) if err != nil { return err } From f919618588b24c619d5a881abe995b13e774412f Mon Sep 17 00:00:00 2001 From: Xenia N Date: Thu, 21 Sep 2023 23:07:07 +0600 Subject: [PATCH 24/46] dont mark release as latest (#920) --- .github/workflows/docker-feature.yml | 6 +++--- .github/workflows/docker-nightly.yml | 6 +++--- .github/workflows/docker-release.yml | 7 +++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-feature.yml b/.github/workflows/docker-feature.yml index b183b726e..addc281f5 100644 --- a/.github/workflows/docker-feature.yml +++ b/.github/workflows/docker-feature.yml @@ -11,7 +11,7 @@ jobs: pull-requests: write strategy: matrix: - servis: [api, checker, cli, notifier, filter] + services: [api, checker, cli, notifier, filter] runs-on: ubuntu-22.04 if: ${{github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/build') && github.event.comment.author_association == 'MEMBER'}} steps: @@ -64,12 +64,12 @@ jobs: uses: docker/build-push-action@v4 with: context: "https://github.com/${{fromJSON(steps.get-pr.outputs.result).head.repo.full_name}}.git#${{fromJSON(steps.get-pr.outputs.result).head.ref}}" - file: ./Dockerfile.${{matrix.servis}} + file: ./Dockerfile.${{matrix.services}} build-args: | MoiraVersion=${{ env.DOCKER_TAG }} GIT_COMMIT=${{ fromJSON(steps.get-pr.outputs.result).head.sha }} push: true - tags: moira/${{matrix.servis}}-unstable:${{env.DOCKER_TAG}} + tags: moira/${{matrix.services}}-unstable:${{env.DOCKER_TAG}} - name: Comment PR with build tag uses: mshick/add-pr-comment@v2 diff --git a/.github/workflows/docker-nightly.yml b/.github/workflows/docker-nightly.yml index 66e1973cb..6b1db3485 100644 --- a/.github/workflows/docker-nightly.yml +++ b/.github/workflows/docker-nightly.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - servis: [api, checker, cli, notifier, filter] + services: [api, checker, cli, notifier, filter] steps: - name: Set up Docker Buildx @@ -28,12 +28,12 @@ jobs: - name: Build and push uses: docker/build-push-action@v4 with: - file: ./Dockerfile.${{matrix.servis}} + file: ./Dockerfile.${{matrix.services}} build-args: | MoiraVersion=${{env.DOCKER_TAG}} GIT_COMMIT=${{github.sha}} push: true - tags: moira/${{matrix.servis}}-nightly:${{env.DOCKER_TAG}} + tags: moira/${{matrix.services}}-nightly:${{env.DOCKER_TAG}} - name: Comment PR with build tag uses: mshick/add-pr-comment@v2 diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index fe6d5a36e..0ea580136 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - servis: [api, checker, cli, notifier, filter] + services: [api, checker, cli, notifier, filter] steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -27,10 +27,9 @@ jobs: - name: Build and push uses: docker/build-push-action@v4 with: - file: ./Dockerfile.${{matrix.servis}} + file: ./Dockerfile.${{matrix.services}} build-args: | MoiraVersion=${{env.DOCKER_TAG}} GIT_COMMIT=${{github.sha}} push: true - tags: moira/${{matrix.servis}}:${{env.DOCKER_TAG}},moira/${{matrix.servis}}:latest - \ No newline at end of file + tags: moira/${{matrix.services}}:${{env.DOCKER_TAG}} From c2b5c9828205c2f35eb3296c2111b1f4495a382b Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Tue, 3 Oct 2023 09:41:29 +0300 Subject: [PATCH 25/46] fix(checker,api): fix switch to maintenance at set del (#916) --- api/controller/trigger.go | 16 ++++ api/controller/trigger_test.go | 38 ++++++++++ checker/check.go | 21 +++++- checker/check_test.go | 130 ++++++++++++++++++++++++++++++++- cmd/cli/cluster.go | 1 - datatypes.go | 3 + 6 files changed, 202 insertions(+), 7 deletions(-) diff --git a/api/controller/trigger.go b/api/controller/trigger.go index 9d0206d1c..fa0c7d1a7 100644 --- a/api/controller/trigger.go +++ b/api/controller/trigger.go @@ -112,6 +112,18 @@ func GetTriggerThrottling(database moira.Database, triggerID string) (*dto.Throt return &dto.ThrottlingResponse{Throttling: throttlingUnix}, nil } +// Need to not show the user metrics that should have been deleted due to ttlState = Del, +// but remained in the database because their Maintenance did not expire +func getAliveMetrics(metrics map[string]moira.MetricState) map[string]moira.MetricState { + aliveMetrics := make(map[string]moira.MetricState, len(metrics)) + for metricName, metricState := range metrics { + if !metricState.DeletedButKept { + aliveMetrics[metricName] = metricState + } + } + return aliveMetrics +} + // GetTriggerLastCheck gets trigger last check data func GetTriggerLastCheck(dataBase moira.Database, triggerID string) (*dto.TriggerCheck, *api.ErrorResponse) { lastCheck := &moira.CheckData{} @@ -125,6 +137,10 @@ func GetTriggerLastCheck(dataBase moira.Database, triggerID string) (*dto.Trigge lastCheck = nil } + if lastCheck != nil && len(lastCheck.Metrics) != 0 { + lastCheck.Metrics = getAliveMetrics(lastCheck.Metrics) + } + triggerCheck := dto.TriggerCheck{ CheckData: lastCheck, TriggerID: triggerID, diff --git a/api/controller/trigger_test.go b/api/controller/trigger_test.go index 1c5769e85..9208e4064 100644 --- a/api/controller/trigger_test.go +++ b/api/controller/trigger_test.go @@ -372,6 +372,44 @@ func TestGetTriggerLastCheck(t *testing.T) { }) }) + Convey("Returns all metrics, because their DeletedButKept is false", t, func() { + lastCheck = moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "metric": {}, + "metric2": {}, + }, + } + dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(lastCheck, nil) + check, err := GetTriggerLastCheck(dataBase, triggerID) + So(err, ShouldBeNil) + So(check, ShouldResemble, &dto.TriggerCheck{ + TriggerID: triggerID, + CheckData: &lastCheck, + }) + }) + + Convey("Does not return all metrics, as some DeletedButKept is true", t, func() { + lastCheck = moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "metric": { + DeletedButKept: true, + }, + "metric2": {}, + }, + } + dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(lastCheck, nil) + check, err := GetTriggerLastCheck(dataBase, triggerID) + So(err, ShouldBeNil) + So(check, ShouldResemble, &dto.TriggerCheck{ + TriggerID: triggerID, + CheckData: &moira.CheckData{ + Metrics: map[string]moira.MetricState{ + "metric2": {}, + }, + }, + }) + }) + Convey("Error", t, func() { expected := fmt.Errorf("oooops! Error get") dataBase.EXPECT().GetTriggerLastCheck(triggerID).Return(moira.CheckData{}, expected) diff --git a/checker/check.go b/checker/check.go index f2493729b..973f00857 100644 --- a/checker/check.go +++ b/checker/check.go @@ -285,6 +285,12 @@ func (triggerChecker *TriggerChecker) preparePatternMetrics(fetchedMetrics conve return result, duplicates } +// Checks if the metric has changed since the previous check +func isMetricChanged(metrics map[string]moira.MetricState, metricName string, metricState moira.MetricState) bool { + prevMetricState := metrics[metricName] + return prevMetricState.Timestamp != metricState.Timestamp +} + // check is the function that handles check on prepared metrics. func (triggerChecker *TriggerChecker) check( metrics map[string]map[string]metricSource.MetricData, @@ -305,16 +311,19 @@ func (triggerChecker *TriggerChecker) check( log := logger.Clone(). String(moira.LogFieldNameMetricName, metricName) - log.Debug().Msg("Checking metrics") - targets = conversion.Merge(targets, aloneMetrics) metricState, needToDeleteMetric, err := triggerChecker.checkTargets(metricName, targets, log) + if needToDeleteMetric { - log.Info().Msg("Remove metric") + log.Debug().String("metric_name", metricName).Msg("Remove metric") checkData.RemoveMetricState(metricName) err = triggerChecker.database.RemovePatternsMetrics(triggerChecker.trigger.Patterns) } else { + // Starting to show user the updated metric, which has been hidden as its Maintenance time is not over + if metricState.DeletedButKept && isMetricChanged(checkData.Metrics, metricName, metricState) { + metricState.DeletedButKept = false + } checkData.Metrics[metricName] = metricState } @@ -325,7 +334,7 @@ func (triggerChecker *TriggerChecker) check( return checkData, nil } -// checkTargets is a Function that takes a +// checkTargets is a function which determines the last state of the metric and information about whether it should be deleted func (triggerChecker *TriggerChecker) checkTargets( metricName string, metrics map[string]metricSource.MetricData, @@ -381,6 +390,10 @@ func (triggerChecker *TriggerChecker) checkForNoData( Msg("Metric TTL expired for state") if triggerChecker.ttlState == moira.TTLStateDEL && metricLastState.EventTimestamp != 0 { + if metricLastState.Maintenance != 0 && lastCheckTimeStamp <= metricLastState.Maintenance { + metricLastState.DeletedButKept = true + return false, &metricLastState + } return true, nil } diff --git a/checker/check_test.go b/checker/check_test.go index 75de07472..dc90dcf97 100644 --- a/checker/check_test.go +++ b/checker/check_test.go @@ -563,12 +563,34 @@ func TestCheckForNODATA(t *testing.T) { metricLastState.Timestamp = 399 triggerChecker.ttlState = moira.TTLStateDEL - Convey("TTLState is DEL and has EventTimeStamp", t, func() { + Convey("TTLState is DEL, has EventTimeStamp, Maintenance metric has expired and will be deleted", t, func() { + metricLastState.Maintenance = 111 needToDeleteMetric, currentState := triggerChecker.checkForNoData(metricLastState, logger) So(needToDeleteMetric, ShouldBeTrue) So(currentState, ShouldBeNil) }) + Convey("TTLState is DEL, has EventTimeStamp, the metric doesn't have Maintenance and will be deleted", t, func() { + metricLastState.Maintenance = 0 + needToDeleteMetric, currentState := triggerChecker.checkForNoData(metricLastState, logger) + So(needToDeleteMetric, ShouldBeTrue) + So(currentState, ShouldBeNil) + }) + + Convey("TTLState is DEL, has EventTimeStamp, but the metric is on Maintenance, so it's not deleted and DeletedButKept = true", t, func() { + metricLastState.Maintenance = 11111 + needToDeleteMetric, currentState := triggerChecker.checkForNoData(metricLastState, logger) + So(needToDeleteMetric, ShouldBeFalse) + So(currentState, ShouldNotBeNil) + So(*currentState, ShouldResemble, moira.MetricState{ + Timestamp: metricLastState.Timestamp, + EventTimestamp: metricLastState.EventTimestamp, + Maintenance: metricLastState.Maintenance, + Suppressed: metricLastState.Suppressed, + DeletedButKept: true, + }) + }) + Convey("Has new metricState", t, func() { Convey("TTLState is DEL, but no EventTimestamp", func() { metricLastState.EventTimestamp = 0 @@ -1157,7 +1179,111 @@ func TestHandleTrigger(t *testing.T) { }) }) - Convey("No data too long and ttlState is delete", t, func() { + Convey("No data too long and ttlState is delete, the metric is not on Maintenance, so it will be removed", t, func() { + triggerChecker.from = 4217 + triggerChecker.until = 4267 + triggerChecker.ttlState = moira.TTLStateDEL + lastCheck.Timestamp = 4267 + + dataBase.EXPECT().RemovePatternsMetrics(triggerChecker.trigger.Patterns).Return(nil) + + aloneMetrics := map[string]metricSource.MetricData{"t1": *metricSource.MakeMetricData(metric, []float64{}, retention, triggerChecker.from)} + lastCheck.MetricsToTargetRelation = conversion.GetRelations(aloneMetrics, triggerChecker.trigger.AloneMetrics) + checkData := newCheckData(&lastCheck, triggerChecker.until) + metricsToCheck := map[string]map[string]metricSource.MetricData{} + + checkData, err := triggerChecker.check(metricsToCheck, aloneMetrics, checkData, logger) + + So(err, ShouldBeNil) + So(checkData, ShouldResemble, moira.CheckData{ + Metrics: make(map[string]moira.MetricState), + Timestamp: triggerChecker.until, + State: moira.StateOK, + Score: 0, + LastSuccessfulCheckTimestamp: 0, + MetricsToTargetRelation: map[string]string{}, + }) + }) + + metricState := lastCheck.Metrics[metric] + metricState.Maintenance = 5000 + lastCheck.Metrics[metric] = metricState + + Convey("No data too long and ttlState is delete, but the metric is on maintenance and DeletedButKept is false, so it won't be deleted", t, func() { + triggerChecker.from = 4217 + triggerChecker.until = 4267 + triggerChecker.ttlState = moira.TTLStateDEL + lastCheck.Timestamp = 4267 + + aloneMetrics := map[string]metricSource.MetricData{"t1": *metricSource.MakeMetricData(metric, []float64{}, retention, triggerChecker.from)} + lastCheck.MetricsToTargetRelation = conversion.GetRelations(aloneMetrics, triggerChecker.trigger.AloneMetrics) + checkData := newCheckData(&lastCheck, triggerChecker.until) + metricsToCheck := map[string]map[string]metricSource.MetricData{} + oldMetricState := lastCheck.Metrics[metric] + + checkData, err := triggerChecker.check(metricsToCheck, aloneMetrics, checkData, logger) + + So(err, ShouldBeNil) + So(checkData, ShouldResemble, moira.CheckData{ + Metrics: map[string]moira.MetricState{ + metric: { + Timestamp: oldMetricState.Timestamp, + EventTimestamp: oldMetricState.EventTimestamp, + State: oldMetricState.State, + Values: oldMetricState.Values, + Maintenance: oldMetricState.Maintenance, + DeletedButKept: true, + }, + }, + MetricsToTargetRelation: map[string]string{}, + Timestamp: triggerChecker.until, + State: moira.StateOK, + Score: 0, + }) + }) + + metricState = lastCheck.Metrics[metric] + metricState.DeletedButKept = true + lastCheck.Metrics[metric] = metricState + + Convey("Metric on maintenance, DeletedButKept is true, ttlState is delete, but a new metric comes in and DeletedButKept becomes false", t, func() { + triggerChecker.from = 4217 + triggerChecker.until = 4267 + triggerChecker.ttlState = moira.TTLStateDEL + lastCheck.Timestamp = 4227 + + aloneMetrics := map[string]metricSource.MetricData{"t1": *metricSource.MakeMetricData(metric, []float64{5}, retention, triggerChecker.from)} + lastCheck.MetricsToTargetRelation = conversion.GetRelations(aloneMetrics, triggerChecker.trigger.AloneMetrics) + checkData := newCheckData(&lastCheck, triggerChecker.until) + metricsToCheck := map[string]map[string]metricSource.MetricData{} + oldMetricState := lastCheck.Metrics[metric] + + checkData, err := triggerChecker.check(metricsToCheck, aloneMetrics, checkData, logger) + + So(err, ShouldBeNil) + So(checkData, ShouldResemble, moira.CheckData{ + Metrics: map[string]moira.MetricState{ + metric: { + Timestamp: triggerChecker.from, + EventTimestamp: oldMetricState.EventTimestamp, + State: oldMetricState.State, + Values: map[string]float64{"t1": 5}, + Maintenance: oldMetricState.Maintenance, + DeletedButKept: false, + }, + }, + MetricsToTargetRelation: map[string]string{}, + Timestamp: triggerChecker.until, + State: moira.StateOK, + Score: 0, + }) + }) + + metricState = lastCheck.Metrics[metric] + metricState.Maintenance = 4000 + lastCheck.Metrics[metric] = metricState + + Convey("No data too long and ttlState is delete, the time for Maintenance of metric is over, so it will be deleted", t, func() { triggerChecker.from = 4217 triggerChecker.until = 4267 triggerChecker.ttlState = moira.TTLStateDEL diff --git a/cmd/cli/cluster.go b/cmd/cli/cluster.go index abd713c01..15f7a4bdb 100644 --- a/cmd/cli/cluster.go +++ b/cmd/cli/cluster.go @@ -7,7 +7,6 @@ import ( "github.com/moira-alert/moira/database/redis" ) -var noSuchKeyError = "ERR no such key" var anyTagsSubscriptionsKeyOld = "moira-any-tags-subscriptions" var anyTagsSubscriptionsKeyNew = "{moira-tag-subscriptions}:moira-any-tags-subscriptions" var triggersListKeyOld = "moira-triggers-list" diff --git a/datatypes.go b/datatypes.go index 32fe29bc8..fcfb1096e 100644 --- a/datatypes.go +++ b/datatypes.go @@ -399,6 +399,9 @@ type MetricState struct { Values map[string]float64 `json:"values,omitempty"` Maintenance int64 `json:"maintenance,omitempty" example:"0" format:"int64"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` + // DeletedButKept controls whether the metric is shown to the user if the trigger has ttlState = Del + // and the metric is in Maintenance. The metric remains in the database + DeletedButKept bool `json:"deleted_but_kept,omitempty" example:"false"` // AloneMetrics map[string]string `json:"alone_metrics"` // represents a relation between name of alone metrics and their targets } From 247719408995d09914ef50c58bf6f95a3e0847c8 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Tue, 3 Oct 2023 17:47:52 +0600 Subject: [PATCH 26/46] fix(build): dont mark release as latest (#927) --- .github/workflows/docker-release.yml | 2 +- .github/workflows/publish-packages.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 0ea580136..67705c48e 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -32,4 +32,4 @@ jobs: MoiraVersion=${{env.DOCKER_TAG}} GIT_COMMIT=${{github.sha}} push: true - tags: moira/${{matrix.services}}:${{env.DOCKER_TAG}} + tags: moira/${{matrix.services}}:${{env.DOCKER_TAG}},moira/${{matrix.services}}:latest diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index a3a1781e1..8b5f534bf 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -27,8 +27,8 @@ jobs: uses: docker://antonyurchenko/git-release:latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DRAFT_RELEASE: "false" - PRE_RELEASE: "false" + DRAFT_RELEASE: "true" + PRE_RELEASE: "true" CHANGELOG_FILE: "none" ALLOW_EMPTY_CHANGELOG: "true" with: From e15004eb25275c3e74acd553cf10dbf83215df05 Mon Sep 17 00:00:00 2001 From: Iurii Pliner Date: Thu, 5 Oct 2023 08:19:31 +0100 Subject: [PATCH 27/46] Switch to a maintained version of carbon-c-relay (#925) The latest version of openmetric/carbon-c-relay was build almost 4 years ago. Obviously, it doesn't support arm, so runs under emulation locally on m1/m2 macbooks. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ff4fd78f3..f8b54a9f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,14 +101,14 @@ services: restart: always relay: - image: openmetric/carbon-c-relay + image: openitcockpit/carbon-c-relay ports: - "2003:2003" depends_on: - graphite volumes: - - ./local/relay.conf:/openmetric/conf/relay.conf - command: /usr/bin/relay -E -s -f /openmetric/conf/relay.conf + - ./local/relay.conf:/opt/carbon-c-relay/relay.conf + command: /opt/carbon-c-relay/bin/relay -E -s -f /opt/carbon-c-relay/relay.conf restart: always networks: balancer: From 35dafcd3895007f8e7010325670cf18d426da20d Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Thu, 5 Oct 2023 11:36:47 +0300 Subject: [PATCH 28/46] feat(filter): Support wildcards (#923) by @lordvidex --- filter/series_by_tag.go | 14 ++++++---- filter/series_by_tag_pattern_index_test.go | 32 +++++++++++++++++----- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/filter/series_by_tag.go b/filter/series_by_tag.go index 1e8e0c5bc..48762dddb 100644 --- a/filter/series_by_tag.go +++ b/filter/series_by_tag.go @@ -42,7 +42,12 @@ type TagSpec struct { func transformWildcardToRegexpInSeriesByTag(input string) (string, bool) { var isTransformed = false - result := input + result := strings.ReplaceAll(input, ".", "\\.") + + if strings.Contains(result, "*") { + result = strings.ReplaceAll(result, "*", ".*") + isTransformed = true + } for { matchedWildcardIndexes := wildcardExprRegex.FindStringSubmatchIndex(result) @@ -62,11 +67,10 @@ func transformWildcardToRegexpInSeriesByTag(input string) (string, bool) { isTransformed = true } - if isTransformed { - result += "$" + if !isTransformed { + return input, false } - - return result, isTransformed + return "^" + result + "$", true } // ParseSeriesByTag parses seriesByTag pattern and returns tags specs diff --git a/filter/series_by_tag_pattern_index_test.go b/filter/series_by_tag_pattern_index_test.go index 440ebfd89..8114a9439 100644 --- a/filter/series_by_tag_pattern_index_test.go +++ b/filter/series_by_tag_pattern_index_test.go @@ -17,17 +17,27 @@ func TestTransformTaggedWildCardToMatchOperator(t *testing.T) { }{ { `{405,406,407,411,413,414,415}`, - `(405|406|407|411|413|414|415)$`, + `^(405|406|407|411|413|414|415)$`, true, }, { `aaa.{405,406,407,411,413,414,415}.bbb`, - `aaa.(405|406|407|411|413|414|415).bbb$`, + `^aaa\.(405|406|407|411|413|414|415)\.bbb$`, true, }, { `aaa.{405,406}.bbb.{301,302}`, - `aaa.(405|406).bbb.(301|302)$`, + `^aaa\.(405|406)\.bbb\.(301|302)$`, + true, + }, + { + `aaa.bbb*`, + `^aaa\.bbb.*$`, + true, + }, + { + `aaa.bbb.*`, + `^aaa\.bbb\..*$`, true, }, { @@ -35,6 +45,11 @@ func TestTransformTaggedWildCardToMatchOperator(t *testing.T) { `a(b|c|d)e`, false, }, + { + `a.e`, + `a.e`, + false, + }, } for _, testCase := range testCases { @@ -62,12 +77,15 @@ func TestParseSeriesByTag(t *testing.T) { {"seriesByTag(\"a=\")", []TagSpec{{"a", EqualOperator, ""}}}, {`seriesByTag("a=b","a=c")`, []TagSpec{{"a", EqualOperator, "b"}, {"a", EqualOperator, "c"}}}, {`seriesByTag("a=b","b=c","c=d")`, []TagSpec{{"a", EqualOperator, "b"}, {"b", EqualOperator, "c"}, {"c", EqualOperator, "d"}}}, - {`seriesByTag("a={b,c,d}")`, []TagSpec{{"a", MatchOperator, "(b|c|d)$"}}}, + {`seriesByTag("a={b,c,d}")`, []TagSpec{{"a", MatchOperator, "^(b|c|d)$"}}}, {`seriesByTag("a=~aa.(b|c|d)$")`, []TagSpec{{"a", MatchOperator, "aa.(b|c|d)$"}}}, {`seriesByTag("respCode=~^(4|5)\d{2}")`, []TagSpec{{"respCode", MatchOperator, "^(4|5)\\d{2}"}}}, - {`seriesByTag("a={b,c,d}", "e=f")`, []TagSpec{{"a", MatchOperator, "(b|c|d)$"}, {"e", EqualOperator, "f"}}}, - {`seriesByTag("a!={b,c,d}", "e=f")`, []TagSpec{{"a", NotMatchOperator, "(b|c|d)$"}, {"e", EqualOperator, "f"}}}, - {`seriesByTag('a!={b,c,d}', 'e=f')`, []TagSpec{{"a", NotMatchOperator, "(b|c|d)$"}, {"e", EqualOperator, "f"}}}, + {`seriesByTag("a={b,c,d}", "e=f")`, []TagSpec{{"a", MatchOperator, "^(b|c|d)$"}, {"e", EqualOperator, "f"}}}, + {`seriesByTag("a!={b,c,d}", "e=f")`, []TagSpec{{"a", NotMatchOperator, "^(b|c|d)$"}, {"e", EqualOperator, "f"}}}, + {`seriesByTag('a!={b,c,d}', 'e=f')`, []TagSpec{{"a", NotMatchOperator, "^(b|c|d)$"}, {"e", EqualOperator, "f"}}}, + {`seriesByTag('a=b*', 'e=f')`, []TagSpec{{"a", MatchOperator, "^b.*$"}, {"e", EqualOperator, "f"}}}, + {`seriesByTag('a=b.*')`, []TagSpec{{"a", MatchOperator, "^b\\..*$"}}}, + {`seriesByTag('a=b.c')`, []TagSpec{{"a", EqualOperator, "b.c"}}}, } for _, validCase := range validSeriesByTagCases { From eeeaa6bfd929343f07dda960f9c46368382fecc6 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Thu, 5 Oct 2023 13:25:31 +0300 Subject: [PATCH 29/46] fix(checker): nil ptr panic (#928) --- checker/worker/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checker/worker/worker.go b/checker/worker/worker.go index 24336cf73..c072de4b5 100644 --- a/checker/worker/worker.go +++ b/checker/worker/worker.go @@ -39,12 +39,12 @@ type Checker struct { func (check *Checker) Start() error { var err error - err = check.startLocalMetricEvents() + err = check.startLazyTriggers() if err != nil { return err } - err = check.startLazyTriggers() + err = check.startLocalMetricEvents() if err != nil { return err } From 488b89f04cd95537bb3e0287e876a38e868adf37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D1=80=D0=B8=D0=B3=D0=BE=D1=80=D0=B8=D0=B9?= <89540685+oxoxoekb@users.noreply.github.com> Date: Fri, 6 Oct 2023 15:36:45 +0500 Subject: [PATCH 30/46] fix(build): add tag 'latest' to nightly images (#931) --- .github/workflows/docker-nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-nightly.yml b/.github/workflows/docker-nightly.yml index 6b1db3485..fe3ad20a0 100644 --- a/.github/workflows/docker-nightly.yml +++ b/.github/workflows/docker-nightly.yml @@ -33,8 +33,8 @@ jobs: MoiraVersion=${{env.DOCKER_TAG}} GIT_COMMIT=${{github.sha}} push: true - tags: moira/${{matrix.services}}-nightly:${{env.DOCKER_TAG}} - + tags: moira/${{matrix.services}}-nightly:${{env.DOCKER_TAG}},moira/${{matrix.services}}-nightly:latest + - name: Comment PR with build tag uses: mshick/add-pr-comment@v2 if: always() From 015eb5590715e1356deb9f734aa6a3e58bdc6fe7 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Tue, 10 Oct 2023 11:04:46 +0300 Subject: [PATCH 31/46] feat(filter): loose tag regex and compatibility (#924) by @lordvidex --- cmd/filter/compatibility.go | 25 ++ cmd/filter/config.go | 6 + cmd/filter/main.go | 4 +- filter/compatibility.go | 7 + filter/pattern_index.go | 7 +- filter/pattern_index_test.go | 2 +- filter/patterns_storage.go | 21 +- filter/patterns_storage_test.go | 9 +- filter/series_by_tag.go | 35 ++- filter/series_by_tag_pattern_index.go | 20 +- filter/series_by_tag_pattern_index_test.go | 270 ++++++++++++++++-- .../filter/performance_test_utils.go | 3 +- 12 files changed, 362 insertions(+), 47 deletions(-) create mode 100644 cmd/filter/compatibility.go create mode 100644 filter/compatibility.go diff --git a/cmd/filter/compatibility.go b/cmd/filter/compatibility.go new file mode 100644 index 000000000..a010d6bbc --- /dev/null +++ b/cmd/filter/compatibility.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/moira-alert/moira/filter" +) + +// Compatibility struct contains feature-flags that give user control over +// features supported by different versions of graphit compatible with moira +type compatibility struct { + // Controls how regices in tag matching are treated + // If false (default value), regex will match start of the string strictly. 'tag~=foo' is equivalent to 'tag~=^foo.*' + // If true, regex will match start of the string loosely. 'tag~=foo' is equivalent to 'tag~=.*foo.*' + AllowRegexLooseStartMatch bool `yaml:"allow_regex_loose_start_match"` + // Controls how absent tags are treated + // If true (default value), empty tags in regices will be matched + // If false, empty tags will be discarded + AllowRegexMatchEmpty bool `yaml:"allow_regex_match_empty"` +} + +func (compatibility *compatibility) toFilterCompatibility() filter.Compatibility { + return filter.Compatibility{ + AllowRegexLooseStartMatch: compatibility.AllowRegexLooseStartMatch, + AllowRegexMatchEmpty: compatibility.AllowRegexMatchEmpty, + } +} diff --git a/cmd/filter/config.go b/cmd/filter/config.go index c59525f8a..ab99965a1 100644 --- a/cmd/filter/config.go +++ b/cmd/filter/config.go @@ -28,6 +28,8 @@ type filterConfig struct { PatternsUpdatePeriod string `yaml:"patterns_update_period"` // DropMetricsTTL this is time window how older metric we can get from now. DropMetricsTTL string `yaml:"drop_metrics_ttl"` + // Flags for compatibility with different graphite behaviours + Compatibility compatibility `yaml:"graphite_compatibility"` } func getDefault() config { @@ -49,6 +51,10 @@ func getDefault() config { MaxParallelMatches: 0, PatternsUpdatePeriod: "1s", DropMetricsTTL: "1h", + Compatibility: compatibility{ + AllowRegexLooseStartMatch: false, + AllowRegexMatchEmpty: true, + }, }, Telemetry: cmd.TelemetryConfig{ Listen: ":8094", diff --git a/cmd/filter/main.go b/cmd/filter/main.go index a4d3773f3..ced22bb6e 100644 --- a/cmd/filter/main.go +++ b/cmd/filter/main.go @@ -69,6 +69,8 @@ func main() { String("moira_version", MoiraVersion). Msg("Moira Filter stopped. Version") + compatibility := config.Filter.Compatibility.toFilterCompatibility() + telemetry, err := cmd.ConfigureTelemetry(logger, config.Telemetry, serviceName) if err != nil { logger.Fatal(). @@ -103,7 +105,7 @@ func main() { Msg("Failed to initialize cache storage with given config") } - patternStorage, err := filter.NewPatternStorage(database, filterMetrics, logger) + patternStorage, err := filter.NewPatternStorage(database, filterMetrics, logger, compatibility) if err != nil { logger.Fatal(). Error(err). diff --git a/filter/compatibility.go b/filter/compatibility.go new file mode 100644 index 000000000..d295a1a3f --- /dev/null +++ b/filter/compatibility.go @@ -0,0 +1,7 @@ +package filter + +// See cmd/filter/compatibility for usage examples +type Compatibility struct { + AllowRegexLooseStartMatch bool + AllowRegexMatchEmpty bool +} diff --git a/filter/pattern_index.go b/filter/pattern_index.go index d622d5a96..c171f0a94 100644 --- a/filter/pattern_index.go +++ b/filter/pattern_index.go @@ -6,17 +6,18 @@ import ( // PatternIndex helps to index patterns and allows to match them by metric type PatternIndex struct { - Tree *PrefixTree + Tree *PrefixTree + compatibility Compatibility } // NewPatternIndex creates new PatternIndex using patterns -func NewPatternIndex(logger moira.Logger, patterns []string) *PatternIndex { +func NewPatternIndex(logger moira.Logger, patterns []string, compatibility Compatibility) *PatternIndex { prefixTree := &PrefixTree{Logger: logger, Root: &PatternNode{}} for _, pattern := range patterns { prefixTree.Add(pattern) } - return &PatternIndex{Tree: prefixTree} + return &PatternIndex{Tree: prefixTree, compatibility: compatibility} } // MatchPatterns allows matching pattern by metric diff --git a/filter/pattern_index_test.go b/filter/pattern_index_test.go index cba30528a..0d527a6c7 100644 --- a/filter/pattern_index_test.go +++ b/filter/pattern_index_test.go @@ -26,7 +26,7 @@ func TestPatternIndex(t *testing.T) { "Question.at_the_end?", } - index := NewPatternIndex(logger, patterns) + index := NewPatternIndex(logger, patterns, Compatibility{AllowRegexLooseStartMatch: true}) testCases := []struct { Metric string MatchedPatterns []string diff --git a/filter/patterns_storage.go b/filter/patterns_storage.go index 5e1002ac1..a23e5e511 100644 --- a/filter/patterns_storage.go +++ b/filter/patterns_storage.go @@ -19,15 +19,22 @@ type PatternStorage struct { logger moira.Logger PatternIndex atomic.Value SeriesByTagPatternIndex atomic.Value + compatibility Compatibility } // NewPatternStorage creates new PatternStorage struct -func NewPatternStorage(database moira.Database, metrics *metrics.FilterMetrics, logger moira.Logger) (*PatternStorage, error) { +func NewPatternStorage( + database moira.Database, + metrics *metrics.FilterMetrics, + logger moira.Logger, + compatibility Compatibility, +) (*PatternStorage, error) { storage := &PatternStorage{ - database: database, - metrics: metrics, - logger: logger, - clock: clock.NewSystemClock(), + database: database, + metrics: metrics, + logger: logger, + clock: clock.NewSystemClock(), + compatibility: compatibility, } err := storage.Refresh() return storage, err @@ -51,8 +58,8 @@ func (storage *PatternStorage) Refresh() error { } } - storage.PatternIndex.Store(NewPatternIndex(storage.logger, patterns)) - storage.SeriesByTagPatternIndex.Store(NewSeriesByTagPatternIndex(storage.logger, seriesByTagPatterns)) + storage.PatternIndex.Store(NewPatternIndex(storage.logger, patterns, storage.compatibility)) + storage.SeriesByTagPatternIndex.Store(NewSeriesByTagPatternIndex(storage.logger, seriesByTagPatterns, storage.compatibility)) return nil } diff --git a/filter/patterns_storage_test.go b/filter/patterns_storage_test.go index c42891350..913096174 100644 --- a/filter/patterns_storage_test.go +++ b/filter/patterns_storage_test.go @@ -30,12 +30,17 @@ func TestProcessIncomingMetric(t *testing.T) { Convey("Create new pattern storage, GetPatterns returns error, should error", t, func() { database.EXPECT().GetPatterns().Return(nil, fmt.Errorf("some error here")) filterMetrics := metrics.ConfigureFilterMetrics(metrics.NewDummyRegistry()) - _, err := NewPatternStorage(database, filterMetrics, logger) + _, err := NewPatternStorage(database, filterMetrics, logger, Compatibility{AllowRegexLooseStartMatch: true}) So(err, ShouldBeError, fmt.Errorf("some error here")) }) database.EXPECT().GetPatterns().Return(testPatterns, nil) - patternsStorage, err := NewPatternStorage(database, metrics.ConfigureFilterMetrics(metrics.NewDummyRegistry()), logger) + patternsStorage, err := NewPatternStorage( + database, + metrics.ConfigureFilterMetrics(metrics.NewDummyRegistry()), + logger, + Compatibility{AllowRegexLooseStartMatch: true}, + ) systemClock := mock_clock.NewMockClock(mockCtrl) systemClock.EXPECT().Now().Return(time.Date(2009, 2, 13, 23, 31, 30, 0, time.UTC)).AnyTimes() patternsStorage.clock = systemClock diff --git a/filter/series_by_tag.go b/filter/series_by_tag.go index 48762dddb..7ede4da1c 100644 --- a/filter/series_by_tag.go +++ b/filter/series_by_tag.go @@ -129,7 +129,7 @@ func ParseSeriesByTag(input string) ([]TagSpec, error) { type MatchingHandler func(string, map[string]string) bool // CreateMatchingHandlerForPattern creates function for matching by tag list -func CreateMatchingHandlerForPattern(tagSpecs []TagSpec) (string, MatchingHandler) { +func CreateMatchingHandlerForPattern(tagSpecs []TagSpec, compatibility *Compatibility) (string, MatchingHandler) { matchingHandlers := make([]MatchingHandler, 0) var nameTagValue string @@ -137,7 +137,8 @@ func CreateMatchingHandlerForPattern(tagSpecs []TagSpec) (string, MatchingHandle if tagSpec.Name == "name" && tagSpec.Operator == EqualOperator { nameTagValue = tagSpec.Value } else { - matchingHandlers = append(matchingHandlers, createMatchingHandlerForOneTag(tagSpec)) + handler := createMatchingHandlerForOneTag(tagSpec, compatibility) + matchingHandlers = append(matchingHandlers, handler) } } @@ -153,9 +154,10 @@ func CreateMatchingHandlerForPattern(tagSpecs []TagSpec) (string, MatchingHandle return nameTagValue, matchingHandler } -func createMatchingHandlerForOneTag(spec TagSpec) MatchingHandler { +func createMatchingHandlerForOneTag(spec TagSpec, compatibility *Compatibility) MatchingHandler { var matchingHandlerCondition func(string) bool allowMatchEmpty := false + switch spec.Operator { case EqualOperator: allowMatchEmpty = true @@ -167,13 +169,18 @@ func createMatchingHandlerForOneTag(spec TagSpec) MatchingHandler { return value != spec.Value } case MatchOperator: - allowMatchEmpty = true - matchRegex := regexp.MustCompile("^" + spec.Value) + allowMatchEmpty = compatibility.AllowRegexMatchEmpty + + matchRegex := newMatchRegex(spec.Value, compatibility) + matchingHandlerCondition = func(value string) bool { return matchRegex.MatchString(value) } case NotMatchOperator: - matchRegex := regexp.MustCompile("^" + spec.Value) + allowMatchEmpty = compatibility.AllowRegexMatchEmpty + + matchRegex := newMatchRegex(spec.Value, compatibility) + matchingHandlerCondition = func(value string) bool { return !matchRegex.MatchString(value) } @@ -194,3 +201,19 @@ func createMatchingHandlerForOneTag(spec TagSpec) MatchingHandler { return allowMatchEmpty && matchEmpty } } + +func newMatchRegex(value string, compatibility *Compatibility) *regexp.Regexp { + var matchRegex *regexp.Regexp + + if value == "*" { + value = ".*" + } + + if compatibility.AllowRegexLooseStartMatch { + matchRegex = regexp.MustCompile(value) + } else { + matchRegex = regexp.MustCompile("^" + value) + } + + return matchRegex +} diff --git a/filter/series_by_tag_pattern_index.go b/filter/series_by_tag_pattern_index.go index e19d4319e..f51047c97 100644 --- a/filter/series_by_tag_pattern_index.go +++ b/filter/series_by_tag_pattern_index.go @@ -1,6 +1,8 @@ package filter -import "github.com/moira-alert/moira" +import ( + "github.com/moira-alert/moira" +) // SeriesByTagPatternIndex helps to index the seriesByTag patterns and allows to match them by metric type SeriesByTagPatternIndex struct { @@ -8,15 +10,21 @@ type SeriesByTagPatternIndex struct { namesPrefixTree *PrefixTree // withoutStrictNameTagPatternMatchers stores MatchingHandler's for patterns that have no name tag withoutStrictNameTagPatternMatchers map[string]MatchingHandler + // Flags for compatibility with different graphite behaviours + compatibility Compatibility } // NewSeriesByTagPatternIndex creates new SeriesByTagPatternIndex using seriesByTag patterns and parsed specs comes from ParseSeriesByTag -func NewSeriesByTagPatternIndex(logger moira.Logger, tagSpecsByPattern map[string][]TagSpec) *SeriesByTagPatternIndex { +func NewSeriesByTagPatternIndex( + logger moira.Logger, + tagSpecsByPattern map[string][]TagSpec, + compatibility Compatibility, +) *SeriesByTagPatternIndex { namesPrefixTree := &PrefixTree{Logger: logger, Root: &PatternNode{}} withoutStrictNameTagPatternMatchers := make(map[string]MatchingHandler) for pattern, tagSpecs := range tagSpecsByPattern { - nameTagValue, matchingHandler := CreateMatchingHandlerForPattern(tagSpecs) + nameTagValue, matchingHandler := CreateMatchingHandlerForPattern(tagSpecs, &compatibility) if nameTagValue == "" { withoutStrictNameTagPatternMatchers[pattern] = matchingHandler @@ -26,8 +34,10 @@ func NewSeriesByTagPatternIndex(logger moira.Logger, tagSpecsByPattern map[strin } return &SeriesByTagPatternIndex{ + compatibility: compatibility, namesPrefixTree: namesPrefixTree, - withoutStrictNameTagPatternMatchers: withoutStrictNameTagPatternMatchers} + withoutStrictNameTagPatternMatchers: withoutStrictNameTagPatternMatchers, + } } // MatchPatterns allows to match patterns by metric name and its labels @@ -42,7 +52,7 @@ func (index *SeriesByTagPatternIndex) MatchPatterns(metricName string, labels ma } for pattern, matchingHandler := range index.withoutStrictNameTagPatternMatchers { - if (matchingHandler)(metricName, labels) { + if matchingHandler(metricName, labels) { matchedPatterns = append(matchedPatterns, pattern) } } diff --git a/filter/series_by_tag_pattern_index_test.go b/filter/series_by_tag_pattern_index_test.go index 8114a9439..47abe4612 100644 --- a/filter/series_by_tag_pattern_index_test.go +++ b/filter/series_by_tag_pattern_index_test.go @@ -111,7 +111,10 @@ func TestParseSeriesByTag(t *testing.T) { func TestSeriesByTagPatternIndex(t *testing.T) { var logger, _ = logging.GetLogger("SeriesByTag") Convey("Given empty patterns with tagspecs, should build index and match patterns", t, func(c C) { - index := NewSeriesByTagPatternIndex(logger, map[string][]TagSpec{}) + compatibility := Compatibility{ + AllowRegexLooseStartMatch: true, + } + index := NewSeriesByTagPatternIndex(logger, map[string][]TagSpec{}, compatibility) c.So(index.MatchPatterns("", nil), ShouldResemble, []string{}) }) @@ -146,16 +149,20 @@ func TestSeriesByTagPatternIndex(t *testing.T) { Labels map[string]string MatchedPatterns []string }{ - {"cpu1", map[string]string{}, []string{"name=cpu1", "name~=cpu", "name~=cpu;dc=", "name~=cpu;dc~="}}, - {"cpu2", map[string]string{}, []string{"name!=cpu1", "name~=cpu", "name~=cpu;dc=", "name~=cpu;dc~="}}, + {"cpu1", map[string]string{}, []string{"name=cpu1", "name~=cpu", "name~=cpu;dc="}}, + {"cpu2", map[string]string{}, []string{"name!=cpu1", "name~=cpu", "name~=cpu;dc="}}, {"disk", map[string]string{}, []string{"name!=cpu1", "name!~=cpu"}}, {"cpu1", map[string]string{"dc": "ru1"}, []string{"dc=ru1", "dc~=ru", "name=cpu1", "name=cpu1;dc=ru1", "name~=cpu", "name~=cpu;dc!=", "name~=cpu;dc~="}}, {"cpu1", map[string]string{"dc": "ru2"}, []string{"dc!=ru1", "dc~=ru", "name=cpu1", "name=cpu1;dc=ru2", "name~=cpu", "name~=cpu;dc!=", "name~=cpu;dc~="}}, {"cpu1", map[string]string{"dc": "us"}, []string{"dc!=ru1", "dc!~=ru", "name=cpu1", "name=cpu1;dc=us", "name~=cpu", "name~=cpu;dc!=", "name~=cpu;dc~="}}, - {"cpu1", map[string]string{"machine": "machine"}, []string{"name=cpu1", "name~=cpu", "name~=cpu;dc=", "name~=cpu;dc~="}}, + {"cpu1", map[string]string{"machine": "machine"}, []string{"name=cpu1", "name~=cpu", "name~=cpu;dc="}}, } - index := NewSeriesByTagPatternIndex(logger, tagSpecsByPattern) + compatibility := Compatibility{ + AllowRegexMatchEmpty: false, + AllowRegexLooseStartMatch: true, + } + index := NewSeriesByTagPatternIndex(logger, tagSpecsByPattern, compatibility) for _, testCase := range testCases { patterns := index.MatchPatterns(testCase.Name, testCase.Labels) sort.Strings(patterns) @@ -172,25 +179,42 @@ func TestSeriesByTagPatternIndex(t *testing.T) { "name=cpu.*.test2;tag1=val1": { {"name", EqualOperator, "cpu.*.test2"}, - {"tag1", EqualOperator, "val1"}}, + {"tag1", EqualOperator, "val1"}, + }, "name=cpu.*.test2;tag2=val2": { {"name", EqualOperator, "cpu.*.test2"}, - {"tag2", EqualOperator, "val2"}}, + {"tag2", EqualOperator, "val2"}, + }, "name=cpu.*.test2;tag1=val1;tag2=val2": { {"name", EqualOperator, "cpu.*.test2"}, {"tag1", EqualOperator, "val1"}, - {"tag2", EqualOperator, "val2"}}, - + {"tag2", EqualOperator, "val2"}, + }, "name!=cpu.test1.test2;tag1=val1;tag2=val2": { {"name", NotEqualOperator, "cpu.test1.test2"}, {"tag1", EqualOperator, "val1"}, - {"tag2", EqualOperator, "val2"}}, + {"tag2", EqualOperator, "val2"}, + }, "name=~cpu;tag1=val1": { {"name", MatchOperator, "cpu"}, - {"tag1", EqualOperator, "val1"}}, + {"tag1", EqualOperator, "val1"}, + }, + "name=~test1": { + {"name", MatchOperator, "test1"}, + }, + "tag1=~al1": { + {"tag1", MatchOperator, "al1"}, + }, + "tag2=~*": { + {"tag2", MatchOperator, "*"}, + }, + "tag2=~.*": { + {"tag2", MatchOperator, ".*"}, + }, "tag1=val1;tag2=val2": { {"tag1", EqualOperator, "val1"}, - {"tag2", EqualOperator, "val2"}}, + {"tag2", EqualOperator, "val2"}, + }, } testCases := []struct { @@ -200,37 +224,235 @@ func TestSeriesByTagPatternIndex(t *testing.T) { }{ {"cpu.test1.test2", map[string]string{}, - []string{"name=cpu.*.*", "name=cpu.*.test2", "name=cpu.test1.*", "name=cpu.test1.test2"}}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~test1", + }}, {"cpu.test1.test2", map[string]string{"tag": "val"}, - []string{"name=cpu.*.*", "name=cpu.*.test2", "name=cpu.test1.*", "name=cpu.test1.test2"}}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~test1", + }}, + {"cpu.test1.test2", + map[string]string{"tag1": "val1"}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.*.test2;tag1=val1", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~cpu;tag1=val1", + "name=~test1", + "tag1=~al1", + }}, + {"cpu.test1.test2", + map[string]string{"tag1": "val2"}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~test1", + }}, + {"cpu.test1.test2", + map[string]string{"tag1": "val1", "tag2": "val1"}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.*.test2;tag1=val1", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~cpu;tag1=val1", + "name=~test1", + "tag1=~al1", + "tag2=~*", + "tag2=~.*", + }}, + {"cpu.test1.test2", + map[string]string{"tag2": "val2"}, + []string{"name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.*.test2;tag2=val2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~test1", + "tag2=~*", + "tag2=~.*", + }}, + {"cpu.test3.test2", + map[string]string{"tag2": "val2"}, + []string{"name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.*.test2;tag2=val2", + "tag2=~*", + "tag2=~.*", + }}, + {"cpu.test1.test2", + map[string]string{"tag1": "val1", "tag2": "val2"}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.*.test2;tag1=val1", + "name=cpu.*.test2;tag1=val1;tag2=val2", + "name=cpu.*.test2;tag2=val2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "name=~cpu;tag1=val1", + "name=~test1", + "tag1=val1;tag2=val2", + "tag1=~al1", + "tag2=~*", + "tag2=~.*", + }}, + } + compatibility := Compatibility{ + AllowRegexLooseStartMatch: true, + AllowRegexMatchEmpty: false, + } + index := NewSeriesByTagPatternIndex(logger, tagSpecsByPattern, compatibility) + for _, testCase := range testCases { + patterns := index.MatchPatterns(testCase.Name, testCase.Labels) + sort.Strings(patterns) + c.So(patterns, ShouldResemble, testCase.MatchedPatterns) + } + }) +} + +func TestSeriesByTagPatternIndexCabonCompatibility(t *testing.T) { + var logger, _ = logging.GetLogger("SeriesByTag") + + Convey("Given related patterns with tagspecs, should build index and match patterns", t, func(c C) { + tagSpecsByPattern := map[string][]TagSpec{ + "name=cpu.test1.test2": {{"name", EqualOperator, "cpu.test1.test2"}}, + "name=cpu.*.test2": {{"name", EqualOperator, "cpu.*.test2"}}, + "name=cpu.test1.*": {{"name", EqualOperator, "cpu.test1.*"}}, + "name=cpu.*.*": {{"name", EqualOperator, "cpu.*.*"}}, + + "name=cpu.*.test2;tag1=val1": { + {"name", EqualOperator, "cpu.*.test2"}, + {"tag1", EqualOperator, "val1"}, + }, + "name=cpu.*.test2;tag2=val2": { + {"name", EqualOperator, "cpu.*.test2"}, + {"tag2", EqualOperator, "val2"}, + }, + "name=cpu.*.test2;tag1=val1;tag2=val2": { + {"name", EqualOperator, "cpu.*.test2"}, + {"tag1", EqualOperator, "val1"}, + {"tag2", EqualOperator, "val2"}, + }, + "name!=cpu.test1.test2;tag1=val1;tag2=val2": { + {"name", NotEqualOperator, "cpu.test1.test2"}, + {"tag1", EqualOperator, "val1"}, + {"tag2", EqualOperator, "val2"}, + }, + "name=~cpu;tag1=val1": { + {"name", MatchOperator, "cpu"}, + {"tag1", EqualOperator, "val1"}, + }, + "name=~test1": { + {"name", MatchOperator, "test1"}, + }, + "tag1=~al1": { + {"tag1", MatchOperator, "al1"}, + }, + "tag2=~*": { + {"tag2", MatchOperator, "*"}, + }, + "tag2=~.*": { + {"tag2", MatchOperator, ".*"}, + }, + "tag1=val1;tag2=val2": { + {"tag1", EqualOperator, "val1"}, + {"tag2", EqualOperator, "val2"}, + }, + } + + testCases := []struct { + Name string + Labels map[string]string + MatchedPatterns []string + }{ + {"cpu.test1.test2", + map[string]string{}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "tag2=~*", + "tag2=~.*", + }}, + {"cpu.test1.test2", + map[string]string{"tag": "val"}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "tag2=~*", + "tag2=~.*", + }}, {"cpu.test1.test2", map[string]string{"tag1": "val1"}, []string{ - "name=cpu.*.*", "name=cpu.*.test2", + "name=cpu.*.*", + "name=cpu.*.test2", "name=cpu.*.test2;tag1=val1", "name=cpu.test1.*", "name=cpu.test1.test2", - "name=~cpu;tag1=val1"}}, + "name=~cpu;tag1=val1", + "tag2=~*", + "tag2=~.*", + }}, {"cpu.test1.test2", map[string]string{"tag1": "val2"}, - []string{"name=cpu.*.*", "name=cpu.*.test2", "name=cpu.test1.*", "name=cpu.test1.test2"}}, + []string{ + "name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.test1.*", + "name=cpu.test1.test2", + "tag2=~*", + "tag2=~.*", + }}, {"cpu.test1.test2", map[string]string{"tag1": "val1", "tag2": "val1"}, []string{ - "name=cpu.*.*", "name=cpu.*.test2", + "name=cpu.*.*", + "name=cpu.*.test2", "name=cpu.*.test2;tag1=val1", "name=cpu.test1.*", "name=cpu.test1.test2", - "name=~cpu;tag1=val1"}}, + "name=~cpu;tag1=val1", + "tag2=~*", + "tag2=~.*", + }}, {"cpu.test1.test2", map[string]string{"tag2": "val2"}, []string{"name=cpu.*.*", "name=cpu.*.test2", "name=cpu.*.test2;tag2=val2", "name=cpu.test1.*", - "name=cpu.test1.test2"}}, + "name=cpu.test1.test2", + "tag2=~*", + "tag2=~.*", + }}, + {"cpu.test3.test2", + map[string]string{"tag2": "val2"}, + []string{"name=cpu.*.*", + "name=cpu.*.test2", + "name=cpu.*.test2;tag2=val2", + "tag2=~*", + "tag2=~.*", + }}, {"cpu.test1.test2", map[string]string{"tag1": "val1", "tag2": "val2"}, []string{ @@ -243,10 +465,16 @@ func TestSeriesByTagPatternIndex(t *testing.T) { "name=cpu.test1.test2", "name=~cpu;tag1=val1", "tag1=val1;tag2=val2", + "tag2=~*", + "tag2=~.*", }}, } - index := NewSeriesByTagPatternIndex(logger, tagSpecsByPattern) + compatibility := Compatibility{ + AllowRegexLooseStartMatch: false, + AllowRegexMatchEmpty: true, + } + index := NewSeriesByTagPatternIndex(logger, tagSpecsByPattern, compatibility) for _, testCase := range testCases { patterns := index.MatchPatterns(testCase.Name, testCase.Labels) sort.Strings(patterns) diff --git a/perfomance_tests/filter/performance_test_utils.go b/perfomance_tests/filter/performance_test_utils.go index fd6eb7447..055730a84 100644 --- a/perfomance_tests/filter/performance_test_utils.go +++ b/perfomance_tests/filter/performance_test_utils.go @@ -42,7 +42,8 @@ func createPatternsStorage(patterns *[]string, b *testing.B) (*filter.PatternSto filterMetrics := metrics.ConfigureFilterMetrics(metrics.NewDummyRegistry()) logger, _ := logging.GetLogger("Benchmark") - patternsStorage, err := filter.NewPatternStorage(database, filterMetrics, logger) + compatibility := filter.Compatibility{AllowRegexLooseStartMatch: true} + patternsStorage, err := filter.NewPatternStorage(database, filterMetrics, logger, compatibility) if err != nil { return nil, err } From d9b2ab4c5bbef5b37e85aef324ef7bb9a41d947d Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Wed, 11 Oct 2023 11:07:04 +0200 Subject: [PATCH 32/46] feat(filter): Add error handling for regex compilation (#933) --- filter/series_by_tag.go | 38 ++++++++++++++++----------- filter/series_by_tag_pattern_index.go | 10 ++++++- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/filter/series_by_tag.go b/filter/series_by_tag.go index 7ede4da1c..619e05753 100644 --- a/filter/series_by_tag.go +++ b/filter/series_by_tag.go @@ -129,7 +129,7 @@ func ParseSeriesByTag(input string) ([]TagSpec, error) { type MatchingHandler func(string, map[string]string) bool // CreateMatchingHandlerForPattern creates function for matching by tag list -func CreateMatchingHandlerForPattern(tagSpecs []TagSpec, compatibility *Compatibility) (string, MatchingHandler) { +func CreateMatchingHandlerForPattern(tagSpecs []TagSpec, compatibility *Compatibility) (string, MatchingHandler, error) { matchingHandlers := make([]MatchingHandler, 0) var nameTagValue string @@ -137,7 +137,11 @@ func CreateMatchingHandlerForPattern(tagSpecs []TagSpec, compatibility *Compatib if tagSpec.Name == "name" && tagSpec.Operator == EqualOperator { nameTagValue = tagSpec.Value } else { - handler := createMatchingHandlerForOneTag(tagSpec, compatibility) + handler, err := createMatchingHandlerForOneTag(tagSpec, compatibility) + if err != nil { + return "", nil, err + } + matchingHandlers = append(matchingHandlers, handler) } } @@ -151,10 +155,10 @@ func CreateMatchingHandlerForPattern(tagSpecs []TagSpec, compatibility *Compatib return true } - return nameTagValue, matchingHandler + return nameTagValue, matchingHandler, nil } -func createMatchingHandlerForOneTag(spec TagSpec, compatibility *Compatibility) MatchingHandler { +func createMatchingHandlerForOneTag(spec TagSpec, compatibility *Compatibility) (MatchingHandler, error) { var matchingHandlerCondition func(string) bool allowMatchEmpty := false @@ -171,7 +175,10 @@ func createMatchingHandlerForOneTag(spec TagSpec, compatibility *Compatibility) case MatchOperator: allowMatchEmpty = compatibility.AllowRegexMatchEmpty - matchRegex := newMatchRegex(spec.Value, compatibility) + matchRegex, err := newMatchRegex(spec.Value, compatibility) + if err != nil { + return nil, err + } matchingHandlerCondition = func(value string) bool { return matchRegex.MatchString(value) @@ -179,7 +186,10 @@ func createMatchingHandlerForOneTag(spec TagSpec, compatibility *Compatibility) case NotMatchOperator: allowMatchEmpty = compatibility.AllowRegexMatchEmpty - matchRegex := newMatchRegex(spec.Value, compatibility) + matchRegex, err := newMatchRegex(spec.Value, compatibility) + if err != nil { + return nil, err + } matchingHandlerCondition = func(value string) bool { return !matchRegex.MatchString(value) @@ -199,21 +209,19 @@ func createMatchingHandlerForOneTag(spec TagSpec, compatibility *Compatibility) return matchingHandlerCondition(value) } return allowMatchEmpty && matchEmpty - } + }, nil } -func newMatchRegex(value string, compatibility *Compatibility) *regexp.Regexp { - var matchRegex *regexp.Regexp - +func newMatchRegex(value string, compatibility *Compatibility) (*regexp.Regexp, error) { if value == "*" { value = ".*" } - if compatibility.AllowRegexLooseStartMatch { - matchRegex = regexp.MustCompile(value) - } else { - matchRegex = regexp.MustCompile("^" + value) + if !compatibility.AllowRegexLooseStartMatch { + value = "^" + value } - return matchRegex + matchRegex, err := regexp.Compile(value) + + return matchRegex, err } diff --git a/filter/series_by_tag_pattern_index.go b/filter/series_by_tag_pattern_index.go index f51047c97..0a0966c7c 100644 --- a/filter/series_by_tag_pattern_index.go +++ b/filter/series_by_tag_pattern_index.go @@ -24,7 +24,15 @@ func NewSeriesByTagPatternIndex( withoutStrictNameTagPatternMatchers := make(map[string]MatchingHandler) for pattern, tagSpecs := range tagSpecsByPattern { - nameTagValue, matchingHandler := CreateMatchingHandlerForPattern(tagSpecs, &compatibility) + nameTagValue, matchingHandler, err := CreateMatchingHandlerForPattern(tagSpecs, &compatibility) + + if err != nil { + logger.Info(). + Error(err). + String("pattern", pattern). + Msg("Failed to create MatchingHandler for pattern") + continue + } if nameTagValue == "" { withoutStrictNameTagPatternMatchers[pattern] = matchingHandler From 723c0199d2ed1cf1f4fa30802a8aaeb186c6ce39 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Wed, 11 Oct 2023 11:45:47 +0200 Subject: [PATCH 33/46] fix(checker): Closed channel write panic (#929) --- checker/worker/handler.go | 3 +-- database/redis/metric.go | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/checker/worker/handler.go b/checker/worker/handler.go index 24ef5c947..dddb81c6d 100644 --- a/checker/worker/handler.go +++ b/checker/worker/handler.go @@ -34,8 +34,7 @@ func (check *Checker) startTriggerHandler(triggerIDsToCheck <-chan string, metri } } -func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) error { - var err error +func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) (err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: '%s' stack: %s", r, debug.Stack()) diff --git a/database/redis/metric.go b/database/redis/metric.go index cdf3e6e15..f7b9433ac 100644 --- a/database/redis/metric.go +++ b/database/redis/metric.go @@ -7,6 +7,7 @@ import ( "math/rand" "strconv" "strings" + "sync/atomic" "time" "github.com/go-redis/redis/v8" @@ -162,11 +163,6 @@ func (connector *DbConnector) SubscribeMetricEvents(tomb *tomb.Tomb, params *moi return nil, err } - go func() { - <-tomb.Dying() - close(responseChannel) - }() - go func() { for { response, ok := <-responseChannel @@ -187,7 +183,10 @@ func (connector *DbConnector) SubscribeMetricEvents(tomb *tomb.Tomb, params *moi } }() - for channelIdx := 0; channelIdx < len(metricEventsChannels); channelIdx++ { + totalEventsChannels := len(metricEventsChannels) + closedEventChannels := int32(0) + + for channelIdx := 0; channelIdx < totalEventsChannels; channelIdx++ { metricEventsChannel := metricEventsChannels[channelIdx] go func() { var popDelay time.Duration @@ -195,6 +194,10 @@ func (connector *DbConnector) SubscribeMetricEvents(tomb *tomb.Tomb, params *moi startPop := time.After(popDelay) select { case <-tomb.Dying(): + if atomic.AddInt32(&closedEventChannels, 1) == int32(totalEventsChannels) { + close(responseChannel) + } + return case <-startPop: data, err := c.SPopN(ctx, metricEventsChannel, params.BatchSize).Result() From 583f34b99518479569a39d44e90063403574825c Mon Sep 17 00:00:00 2001 From: Xenia N Date: Wed, 11 Oct 2023 18:35:48 +0600 Subject: [PATCH 34/46] fix(build): upgrade linter version (#934) --- .github/workflows/lint.yml | 2 +- .golangci.yml | 7 +++++++ cmd/notifier/config.go | 2 +- database/redis/metric.go | 6 +++--- metrics/graphite.go | 4 ++-- notifier/plotting.go | 2 +- perfomance_tests/filter/performance_test_utils.go | 13 ++++++++++--- 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d0e0119ba..3905b2673 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,6 +17,6 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.52.2 + version: v1.54.2 only-new-issues: true diff --git a/.golangci.yml b/.golangci.yml index a894168df..cff12d45a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,7 +24,11 @@ linters: # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint disable-all: true enable: + - asasalint + - asciicheck + - bidichk - bodyclose + - dogsled - errcheck - goconst - gocyclo @@ -34,8 +38,11 @@ linters: - gosimple - govet - ineffassign + - makezero - misspell + - nilerr - prealloc + - promlinter - staticcheck - typecheck - unconvert diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index 04d0d9a2f..e3679ade9 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -208,7 +208,7 @@ func checkDateTimeFormat(format string) error { fallbackTime := time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) parsedTime, err := time.Parse(format, time.Now().Format(format)) if err != nil || parsedTime == fallbackTime { - return fmt.Errorf("could not parse date time format '%v', result: '%v', error: '%v'", format, parsedTime, err) + return fmt.Errorf("could not parse date time format '%v', result: '%v', error: '%w'", format, parsedTime, err) } return nil } diff --git a/database/redis/metric.go b/database/redis/metric.go index f7b9433ac..a0e349d8a 100644 --- a/database/redis/metric.go +++ b/database/redis/metric.go @@ -23,7 +23,7 @@ func (connector *DbConnector) GetPatterns() ([]string, error) { c := *connector.client patterns, err := c.SMembers(connector.context, patternsListKey).Result() if err != nil { - return nil, fmt.Errorf("failed to get moira patterns, error: %v", err) + return nil, fmt.Errorf("failed to get moira patterns, error: %w", err) } return patterns, nil } @@ -86,11 +86,11 @@ func (connector *DbConnector) getMetricRetention(metric string) (int64, error) { if err == redis.Nil { return 60, database.ErrNil //nolint } - return 0, fmt.Errorf("failed GET metric retention:%s, error: %v", metric, err) + return 0, fmt.Errorf("failed GET metric retention:%s, error: %w", metric, err) } retention, err := strconv.ParseInt(retentionStr, 10, 64) if err != nil { - return 0, fmt.Errorf("failed GET metric retention:%s, error: %v", metric, err) + return 0, fmt.Errorf("failed GET metric retention:%s, error: %w", metric, err) } return retention, nil } diff --git a/metrics/graphite.go b/metrics/graphite.go index af3c3c12f..37bc4fbaf 100644 --- a/metrics/graphite.go +++ b/metrics/graphite.go @@ -30,11 +30,11 @@ func NewGraphiteRegistry(config GraphiteRegistryConfig, serviceName string) (*Gr if config.Enabled { address, err := net.ResolveTCPAddr("tcp", config.URI) if err != nil { - return nil, fmt.Errorf("can't resolve graphiteURI %s: %s", config.URI, err) + return nil, fmt.Errorf("can't resolve graphiteURI %s: %w", config.URI, err) } prefix, err := initPrefix(config.Prefix) if err != nil { - return nil, fmt.Errorf("can't get OS hostname %s: %s", config.Prefix, err) + return nil, fmt.Errorf("can't get OS hostname %s: %w", config.Prefix, err) } if config.RuntimeStats { goMetrics.RegisterRuntimeMemStats(registry) diff --git a/notifier/plotting.go b/notifier/plotting.go index ca95a6a82..e917b1276 100644 --- a/notifier/plotting.go +++ b/notifier/plotting.go @@ -165,7 +165,7 @@ func (notifier *StandardNotifier) evaluateTriggerMetrics(from, to int64, trigger func fetchAvailableSeries(metricsSource metricSource.MetricSource, target string, from, to int64) ([]metricSource.MetricData, error) { realtimeFetchResult, realtimeErr := metricsSource.Fetch(target, from, to, true) if realtimeErr == nil { - return realtimeFetchResult.GetMetricsData(), realtimeErr + return realtimeFetchResult.GetMetricsData(), nil } if errFailedWithPanic, ok := realtimeErr.(local.ErrEvaluateTargetFailedWithPanic); ok { fetchResult, err := metricsSource.Fetch(target, from, to, false) diff --git a/perfomance_tests/filter/performance_test_utils.go b/perfomance_tests/filter/performance_test_utils.go index 055730a84..75e0d324a 100644 --- a/perfomance_tests/filter/performance_test_utils.go +++ b/perfomance_tests/filter/performance_test_utils.go @@ -3,6 +3,7 @@ package filter import ( "bufio" "fmt" + "io" "math/rand" "os" "strings" @@ -23,12 +24,18 @@ func loadPatterns(filename string) (*[]string, error) { if err != nil { return nil, err } + + defer patternsFile.Close() + patterns := make([]string, 0) patternsReader := bufio.NewReader(patternsFile) for { - pattern, err1 := patternsReader.ReadString('\n') - if err1 != nil { - break + pattern, parseErr := patternsReader.ReadString('\n') + if len(pattern) == 0 && parseErr != nil { + if parseErr == io.EOF { + break + } + return &patterns, parseErr } patterns = append(patterns, pattern[:len(pattern)-1]) } From 3b1bf554d942dab9fe458d2ce81166cba1472739 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Thu, 12 Oct 2023 11:20:27 +0200 Subject: [PATCH 35/46] fix(checker): Revert panic logging (#936) --- checker/worker/handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/checker/worker/handler.go b/checker/worker/handler.go index dddb81c6d..24ef5c947 100644 --- a/checker/worker/handler.go +++ b/checker/worker/handler.go @@ -34,7 +34,8 @@ func (check *Checker) startTriggerHandler(triggerIDsToCheck <-chan string, metri } } -func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) (err error) { +func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) error { + var err error defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: '%s' stack: %s", r, debug.Stack()) From 0b379cbb21ec41a6ebc90aa81d9a6ff3f1c4087d Mon Sep 17 00:00:00 2001 From: Xenia N Date: Mon, 16 Oct 2023 15:28:03 +0600 Subject: [PATCH 36/46] upgrade(build): upgrade mattermost & x/net version (#937) --- go.mod | 82 +++++----- go.sum | 174 ++++++++++----------- mock/notifier/mattermost/client.go | 19 +-- senders/mattermost/client.go | 10 +- senders/mattermost/sender.go | 17 +- senders/mattermost/sender_internal_test.go | 11 +- 6 files changed, 155 insertions(+), 158 deletions(-) diff --git a/go.mod b/go.mod index f18b17bd2..dfdd7baf2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/PagerDuty/go-pagerduty v1.5.1 github.com/ansel1/merry v1.6.2 - github.com/aws/aws-sdk-go v1.44.219 + github.com/aws/aws-sdk-go v1.44.293 github.com/blevesearch/bleve/v2 v2.3.8 github.com/bwmarrin/discordgo v0.25.0 github.com/carlosdp/twiliogo v0.0.0-20161027183705-b26045ebb9d1 @@ -26,13 +26,12 @@ require ( github.com/gotokatsuya/ipare v0.0.0-20161202043954-fd52c5b6c44b github.com/gregdel/pushover v1.1.0 github.com/karriereat/blackfriday-slack v0.1.0 - github.com/mattermost/mattermost-server/v6 v6.0.0-20230405170428-2a75f997ee6c // it is last commit of 7.9.2 (https://github.com/mattermost/mattermost-server/commits/v7.9.2). Can't use v7, issue https://github.com/mattermost/mattermost-server/issues/20817 github.com/moira-alert/go-chart v0.0.0-20230220064910-812fb2829b9b github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.13 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.14.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 - github.com/rs/cors v1.8.3 + github.com/rs/cors v1.9.0 github.com/rs/zerolog v1.29.0 github.com/russross/blackfriday/v2 v2.1.0 github.com/slack-go/slack v0.12.1 @@ -50,6 +49,7 @@ require ( require github.com/prometheus/common v0.37.0 require ( + github.com/mattermost/mattermost/server/public v0.0.9 github.com/mitchellh/mapstructure v1.5.0 github.com/swaggo/http-swagger v1.3.4 ) @@ -58,43 +58,41 @@ require ( bitbucket.org/tebeka/strftime v0.0.0-20140926081919-2194253a23c0 // indirect github.com/JaderDias/movingmedian v0.0.0-20220813210630-d8c6b6de8835 // indirect github.com/Masterminds/sprig/v3 v3.2.3 - github.com/RoaringBitmap/roaring v1.2.1 // indirect + github.com/RoaringBitmap/roaring v1.3.0 // indirect github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect github.com/ansel1/merry/v2 v2.1.1 // indirect github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.4.0 // indirect - github.com/blang/semver v3.5.1+incompatible // indirect + github.com/bits-and-blooms/bitset v1.8.0 // indirect github.com/blend/go-sdk v2.0.0+incompatible // indirect github.com/blevesearch/bleve_index_api v1.0.5 // indirect github.com/blevesearch/geo v0.1.17 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.1.5 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect github.com/blevesearch/vellum v1.0.9 // indirect - github.com/blevesearch/zapx/v11 v11.3.7 // indirect - github.com/blevesearch/zapx/v12 v12.3.7 // indirect - github.com/blevesearch/zapx/v13 v13.3.7 // indirect - github.com/blevesearch/zapx/v14 v14.3.7 // indirect - github.com/blevesearch/zapx/v15 v15.3.10 // indirect + github.com/blevesearch/zapx/v11 v11.3.8 // indirect + github.com/blevesearch/zapx/v12 v12.3.8 // indirect + github.com/blevesearch/zapx/v13 v13.3.8 // indirect + github.com/blevesearch/zapx/v14 v14.3.8 // indirect + github.com/blevesearch/zapx/v15 v15.3.11 // indirect github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-expirecache v0.0.0-20170314133854-743ef98b2adb // indirect github.com/dgryski/go-onlinestats v0.0.0-20170612111826-1c7d19468768 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/disintegration/imaging v1.6.2 // indirect - github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect github.com/evmar/gocairo v0.0.0-20160222165215-ddd30f837497 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomodule/redigo v1.8.9 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -112,22 +110,17 @@ require ( github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.15.14 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect - github.com/lib/pq v1.10.7 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 // indirect github.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/maruel/natural v1.1.0 // indirect github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d // indirect - github.com/mattermost/logr/v2 v2.0.16 // indirect + github.com/mattermost/logr/v2 v2.0.18 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.45 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -137,54 +130,54 @@ require ( github.com/natefinch/atomic v1.0.1 // indirect github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/rs/xid v1.4.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/smartystreets/assertions v1.2.0 // indirect - github.com/spf13/afero v1.9.3 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.14.0 // indirect + github.com/spf13/viper v1.16.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/stretchr/testify v1.8.4 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/tinylib/msgp v1.1.8 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f // indirect - github.com/wiggin77/merror v1.0.4 // indirect + github.com/wiggin77/merror v1.0.5 // indirect github.com/wiggin77/srslog v1.0.1 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.uber.org/atomic v1.10.0 // indirect + go.etcd.io/bbolt v1.3.7 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20200924195034-c827fd4f18b9 // indirect - golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/image v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gonum.org/v1/gonum v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( - github.com/BurntSushi/toml v1.3.2 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.9 // indirect @@ -195,10 +188,11 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/swaggo/files v1.0.1 // indirect github.com/swaggo/swag v1.8.12 // indirect - golang.org/x/tools v0.11.1 // indirect + golang.org/x/tools v0.12.0 // indirect ) // Have to exclude version that is incorectly retracted by authors diff --git a/go.sum b/go.sum index dcbad53c2..f73e0f1bf 100644 --- a/go.sum +++ b/go.sum @@ -400,8 +400,6 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JaderDias/movingmedian v0.0.0-20220813210630-d8c6b6de8835 h1:mbxQnovjDz5SvlatpxkbiMvybHH1hsSEu6OhPDLlfU8= github.com/JaderDias/movingmedian v0.0.0-20220813210630-d8c6b6de8835/go.mod h1:zsfWLaDctbM7aV1TsQAwkVswuKQ0k7PK4rjC1VZqpbI= @@ -411,15 +409,16 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PagerDuty/go-pagerduty v1.5.1 h1:zpMQ8WwWlUahipB2q+ERVIA9D0/ti8kvsQUSagCK86g= github.com/PagerDuty/go-pagerduty v1.5.1/go.mod h1:txr8VbObXdk2RkqF+C2an4qWssdGY99fK26XYUDjh+4= -github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= -github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= +github.com/RoaringBitmap/roaring v1.3.0 h1:aQmu9zQxDU0uhwR8SXOH/OrqEf+X8A0LQmwW3JX8Lcg= +github.com/RoaringBitmap/roaring v1.3.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= @@ -442,8 +441,8 @@ github.com/ansel1/merry/v2 v2.1.1/go.mod h1:4p/FFyQbCgqlDbseWOVQaL5USpgkE9sr5xh4 github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.44.219 h1:YOFxTUQZvdRzgwb6XqLFRwNHxoUdKBuunITC7IFhvbc= -github.com/aws/aws-sdk-go v1.44.219/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.293 h1:oBPrQqsyMYe61Sl/xKVvQFflXjPwYH11aKi8QR3Nhts= +github.com/aws/aws-sdk-go v1.44.293/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -453,10 +452,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= -github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= +github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blend/go-sdk v2.0.0+incompatible h1:FL9X/of4ZYO5D2JJNI4vHrbXPfuSDbUa7h8JP9+E92w= github.com/blend/go-sdk v2.0.0+incompatible/go.mod h1:3GUb0YsHFNTJ6hsJTpzdmCUl05o8HisKjx5OAlzYKdw= github.com/blevesearch/bleve/v2 v2.3.8 h1:IqFyMJ73n4gY8AmVqM8Sa6EtAZ5beE8yramVqCvs2kQ= @@ -471,8 +470,8 @@ github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZG github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk= -github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0= +github.com/blevesearch/scorch_segment_api/v2 v2.1.5 h1:1g713kpCQZ8u4a3stRGBfrwVOuGRnmxOVU5MQkUPrHU= +github.com/blevesearch/scorch_segment_api/v2 v2.1.5/go.mod h1:f2nOkKS1HcjgIWZgDAErgBdxmr2eyt0Kn7IY+FU1Xe4= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -481,16 +480,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ= github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.7 h1:Y6yIAF/DVPiqZUA/jNgSLXmqewfzwHzuwfKyfdG+Xaw= -github.com/blevesearch/zapx/v11 v11.3.7/go.mod h1:Xk9Z69AoAWIOvWudNDMlxJDqSYGf90LS0EfnaAIvXCA= -github.com/blevesearch/zapx/v12 v12.3.7 h1:DfQ6rsmZfEK4PzzJJRXjiM6AObG02+HWvprlXQ1Y7eI= -github.com/blevesearch/zapx/v12 v12.3.7/go.mod h1:SgEtYIBGvM0mgIBn2/tQE/5SdrPXaJUaT/kVqpAPxm0= -github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ubM4d2l8= -github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s= -github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20= -github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w= -github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ= -github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= +github.com/blevesearch/zapx/v11 v11.3.8 h1:bhJyuzkHjVCXegv9/nwXEWJf/NGvApL/N2aBM7jlZrQ= +github.com/blevesearch/zapx/v11 v11.3.8/go.mod h1:W/dwGRt6/Fik7aEdSymBHknA+CLhj/VFHDfOnbfT0xE= +github.com/blevesearch/zapx/v12 v12.3.8 h1:2iOBdzwlrvYx2CHh0hRtYni/uyBjwRj+5rgUQ0vHXho= +github.com/blevesearch/zapx/v12 v12.3.8/go.mod h1:04059x924iuaWsuLFNvKx/JV76aRAgTqDuygzKKRBfU= +github.com/blevesearch/zapx/v13 v13.3.8 h1:xqsFb+EZajPRMiuk4vItuSaIOtux0p7MWTtxvsG0EUo= +github.com/blevesearch/zapx/v13 v13.3.8/go.mod h1:t4lMpGlRiUmC5pLKKgICF7sJcVtD7fF7zhXeBVNFFBs= +github.com/blevesearch/zapx/v14 v14.3.8 h1:HXGGdVZQGG9qOfvj5Ihcm0JJIvUdkI3it2OT46Lx8qo= +github.com/blevesearch/zapx/v14 v14.3.8/go.mod h1:vS6exLagv0vXmgpUbNRZC6UuEV0xwTfCmgaWgjLmf/U= +github.com/blevesearch/zapx/v15 v15.3.11 h1:dstyZki9s10FNLsW4LpEvPQ+fmM3nX15h4wKfcBwnEg= +github.com/blevesearch/zapx/v15 v15.3.11/go.mod h1:hiYbBDf5/Ud/Eji0faUmMTOyeOjcl8q1vWGgRe7+bIQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 h1:hjXJeBcAMS1WGENGqDpzvmgS43oECTx8UXq31UBu0Jw= github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= @@ -564,13 +563,13 @@ github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0+ github.com/evmar/gocairo v0.0.0-20160222165215-ddd30f837497 h1:DIQ8EvZ8OjuPNfcV4NgsyBeZho7WsTD0JEkDM5napMI= github.com/evmar/gocairo v0.0.0-20160222165215-ddd30f837497/go.mod h1:YXKUYPSqs+jDG8mvexHN2uTik4PKwg2B0WK9itQ0VrE= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -641,8 +640,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= -github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= +github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -675,8 +674,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -779,7 +779,7 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -833,12 +833,6 @@ github.com/karriereat/blackfriday-slack v0.1.0/go.mod h1:iM9iGxIpITGTuxl7benyJWi github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= -github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -851,8 +845,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 h1:KVyDGUXjVOdHQt24wIgY4ZdGFXHtQHLWw0L/MAK3Kb0= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80/go.mod h1:T7SQVaLtK7mcQIEVzveZVJzsDQpAtzTs2YoezrIBdvI= github.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463 h1:SN/0TEkyYpp8tit79JPUnecebCGZsXiYYPxN8i3I6Rk= @@ -874,10 +868,10 @@ github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 h1:Khvh6waxG github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34= github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d h1:/RJ/UV7M5c7L2TQ0KNm4yZxxFvC1nvRz/gY/Daa35aI= github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d/go.mod h1:HLbgMEI5K131jpxGazJ97AxfPDt31osq36YS1oxFQPQ= -github.com/mattermost/logr/v2 v2.0.16 h1:jnePX4cPskC3WDFvUardh/xZfxNdsFXbEERJQ1kUEDE= -github.com/mattermost/logr/v2 v2.0.16/go.mod h1:1dm/YhTpozsqANXxo5Pi5zYLBsal2xY0pX+JZNbzYJY= -github.com/mattermost/mattermost-server/v6 v6.0.0-20230405170428-2a75f997ee6c h1:sBnkgaSMddx5t5+tjUp/9tVC152anV8oG1QfI8G3GvY= -github.com/mattermost/mattermost-server/v6 v6.0.0-20230405170428-2a75f997ee6c/go.mod h1:o61MGMh7We01wGr1ydGDA5mmNpjTzaBVWUAlezsgx50= +github.com/mattermost/logr/v2 v2.0.18 h1:qiznuwwKckZJoGtBYc4Y9FAY97/oQwV1Pq9oO5qP5nk= +github.com/mattermost/logr/v2 v2.0.18/go.mod h1:1dm/YhTpozsqANXxo5Pi5zYLBsal2xY0pX+JZNbzYJY= +github.com/mattermost/mattermost/server/public v0.0.9 h1:Qsktgxx5dc8xVAUHP5MbSLi6Cf82iB/83r6S9bluHto= +github.com/mattermost/mattermost/server/public v0.0.9/go.mod h1:sgXQrYzs+IJy51mB8E8OBljagk2u3YwQRoYlBH5goiw= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -886,18 +880,12 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.45 h1:g4IeM9M9pW/Lo8AGGNOjBZYlvmtlE1N5TQEYWXRWzIs= -github.com/minio/minio-go/v7 v7.0.45/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -955,8 +943,8 @@ github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1003,10 +991,10 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= -github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= +github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= @@ -1043,8 +1031,8 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slack-go/slack v0.12.1 h1:X97b9g2hnITDtNsNe5GkGx6O2/Sz/uC20ejRZN6QxOw= github.com/slack-go/slack v0.12.1/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -1057,17 +1045,17 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -1082,12 +1070,13 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= @@ -1105,8 +1094,8 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f h1:9DDCDwOyEy/gId+IEMrFHLuQ5R/WV0KNxWLler8X2OY= github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f/go.mod h1:8sdOQnirw1PrcnTJYkmW1iOHtUmblMmGdUOHyWYycLI= -github.com/wiggin77/merror v1.0.4 h1:XxFLEevmQQfgJW2AxhapuMG7C1fQqfbim/XyUmYv/ZM= -github.com/wiggin77/merror v1.0.4/go.mod h1:H2ETSu7/bPE0Ymf4bEwdUoo73OOEkdClnoRisfw0Nm0= +github.com/wiggin77/merror v1.0.5 h1:P+lzicsn4vPMycAf2mFf7Zk6G9eco5N+jB1qJ2XW3ME= +github.com/wiggin77/merror v1.0.5/go.mod h1:H2ETSu7/bPE0Ymf4bEwdUoo73OOEkdClnoRisfw0Nm0= github.com/wiggin77/srslog v1.0.1 h1:gA2XjSMy3DrRdX9UqLuDtuVAAshb8bE1NhX1YK0Qe+8= github.com/wiggin77/srslog v1.0.1/go.mod h1:fehkyYDq1QfuYn60TDPu9YdY2bB85VUW2mvN1WynEls= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= @@ -1115,7 +1104,6 @@ github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49 github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiam/to v0.0.0-20200126224905-d60d31e03561 h1:SVoNK97S6JlaYlHcaC+79tg3JUlQABcc0dH2VQ4Y+9s= github.com/xiam/to v0.0.0-20200126224905-d60d31e03561/go.mod h1:cqbG7phSzrbdg3aj+Kn63bpVruzwDZi58CpxlZkjwzw= -github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1123,8 +1111,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -1139,8 +1127,8 @@ go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5f go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= @@ -1165,9 +1153,10 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1183,8 +1172,8 @@ golang.org/x/exp v0.0.0-20200924195034-c827fd4f18b9/go.mod h1:1phAWC201xIgDyaFpm golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.8.0 h1:agUcRXV/+w6L9ryntYYsF2x9fQTMd4T8fiiYXAVW6Jg= +golang.org/x/image v0.8.0/go.mod h1:PwLxp3opCYg4WR2WO9P0L6ESnsD6bLTWcw8zanLMVFM= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -1213,6 +1202,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1263,6 +1253,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1282,8 +1273,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1312,8 +1303,9 @@ golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1370,7 +1362,6 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1413,7 +1404,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1424,8 +1414,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1447,8 +1438,9 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1516,8 +1508,9 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc= -golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1772,8 +1765,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1792,8 +1786,8 @@ gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaD gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= diff --git a/mock/notifier/mattermost/client.go b/mock/notifier/mattermost/client.go index 3407b00bd..f3ddb4903 100644 --- a/mock/notifier/mattermost/client.go +++ b/mock/notifier/mattermost/client.go @@ -5,10 +5,11 @@ package mock_mattermost import ( + context "context" reflect "reflect" gomock "github.com/golang/mock/gomock" - model "github.com/mattermost/mattermost-server/v6/model" + model "github.com/mattermost/mattermost/server/public/model" ) // MockClient is a mock of Client interface. @@ -35,9 +36,9 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // CreatePost mocks base method. -func (m *MockClient) CreatePost(arg0 *model.Post) (*model.Post, *model.Response, error) { +func (m *MockClient) CreatePost(arg0 context.Context, arg1 *model.Post) (*model.Post, *model.Response, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreatePost", arg0) + ret := m.ctrl.Call(m, "CreatePost", arg0, arg1) ret0, _ := ret[0].(*model.Post) ret1, _ := ret[1].(*model.Response) ret2, _ := ret[2].(error) @@ -45,9 +46,9 @@ func (m *MockClient) CreatePost(arg0 *model.Post) (*model.Post, *model.Response, } // CreatePost indicates an expected call of CreatePost. -func (mr *MockClientMockRecorder) CreatePost(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) CreatePost(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePost", reflect.TypeOf((*MockClient)(nil).CreatePost), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePost", reflect.TypeOf((*MockClient)(nil).CreatePost), arg0, arg1) } // SetToken mocks base method. @@ -63,9 +64,9 @@ func (mr *MockClientMockRecorder) SetToken(arg0 interface{}) *gomock.Call { } // UploadFile mocks base method. -func (m *MockClient) UploadFile(arg0 []byte, arg1, arg2 string) (*model.FileUploadResponse, *model.Response, error) { +func (m *MockClient) UploadFile(arg0 context.Context, arg1 []byte, arg2, arg3 string) (*model.FileUploadResponse, *model.Response, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UploadFile", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "UploadFile", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*model.FileUploadResponse) ret1, _ := ret[1].(*model.Response) ret2, _ := ret[2].(error) @@ -73,7 +74,7 @@ func (m *MockClient) UploadFile(arg0 []byte, arg1, arg2 string) (*model.FileUplo } // UploadFile indicates an expected call of UploadFile. -func (mr *MockClientMockRecorder) UploadFile(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) UploadFile(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadFile", reflect.TypeOf((*MockClient)(nil).UploadFile), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadFile", reflect.TypeOf((*MockClient)(nil).UploadFile), arg0, arg1, arg2, arg3) } diff --git a/senders/mattermost/client.go b/senders/mattermost/client.go index b91163b15..bf16985ac 100644 --- a/senders/mattermost/client.go +++ b/senders/mattermost/client.go @@ -1,10 +1,14 @@ package mattermost -import "github.com/mattermost/mattermost-server/v6/model" +import ( + "context" + + "github.com/mattermost/mattermost/server/public/model" +) // Client is abstraction over model.Client4. type Client interface { SetToken(token string) - CreatePost(post *model.Post) (*model.Post, *model.Response, error) - UploadFile(data []byte, channelId string, filename string) (*model.FileUploadResponse, *model.Response, error) + CreatePost(ctx context.Context, post *model.Post) (*model.Post, *model.Response, error) + UploadFile(ctx context.Context, data []byte, channelId string, filename string) (*model.FileUploadResponse, *model.Response, error) } diff --git a/senders/mattermost/sender.go b/senders/mattermost/sender.go index 7cd1bc5be..0b514069f 100644 --- a/senders/mattermost/sender.go +++ b/senders/mattermost/sender.go @@ -1,6 +1,7 @@ package mattermost import ( + "context" "crypto/tls" "fmt" "net/http" @@ -10,7 +11,7 @@ import ( "github.com/moira-alert/moira" "github.com/moira-alert/moira/senders" - "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost/server/public/model" "github.com/mitchellh/mapstructure" ) @@ -75,12 +76,13 @@ func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, loca // SendEvents implements moira.Sender interface. func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.ContactData, trigger moira.TriggerData, plots [][]byte, throttled bool) error { message := sender.buildMessage(events, trigger, throttled) - post, err := sender.sendMessage(message, contact.Value, trigger.ID) + ctx := context.Background() + post, err := sender.sendMessage(ctx, message, contact.Value, trigger.ID) if err != nil { return err } if len(plots) > 0 { - err = sender.sendPlots(plots, contact.Value, post.Id, trigger.ID) + err = sender.sendPlots(ctx, plots, contact.Value, post.Id, trigger.ID) if err != nil { sender.logger.Warning(). String("trigger_id", trigger.ID). @@ -195,13 +197,13 @@ func (sender *Sender) buildEventsString(events moira.NotificationEvents, charsFo return eventsString } -func (sender *Sender) sendMessage(message string, contact string, triggerID string) (*model.Post, error) { +func (sender *Sender) sendMessage(ctx context.Context, message string, contact string, triggerID string) (*model.Post, error) { post := model.Post{ ChannelId: contact, Message: message, } - sentPost, _, err := sender.client.CreatePost(&post) + sentPost, _, err := sender.client.CreatePost(ctx, &post) if err != nil { return nil, fmt.Errorf("failed to send %s event message to Mattermost [%s]: %s", triggerID, contact, err) } @@ -209,12 +211,12 @@ func (sender *Sender) sendMessage(message string, contact string, triggerID stri return sentPost, nil } -func (sender *Sender) sendPlots(plots [][]byte, channelID, postID, triggerID string) error { +func (sender *Sender) sendPlots(ctx context.Context, plots [][]byte, channelID, postID, triggerID string) error { var filesID []string filename := fmt.Sprintf("%s.png", triggerID) for _, plot := range plots { - file, _, err := sender.client.UploadFile(plot, channelID, filename) + file, _, err := sender.client.UploadFile(ctx, plot, channelID, filename) if err != nil { return err } @@ -225,6 +227,7 @@ func (sender *Sender) sendPlots(plots [][]byte, channelID, postID, triggerID str if len(filesID) > 0 { _, _, err := sender.client.CreatePost( + ctx, &model.Post{ ChannelId: channelID, RootId: postID, diff --git a/senders/mattermost/sender_internal_test.go b/senders/mattermost/sender_internal_test.go index 104a9c776..ffc0d4299 100644 --- a/senders/mattermost/sender_internal_test.go +++ b/senders/mattermost/sender_internal_test.go @@ -1,12 +1,13 @@ package mattermost import ( + "context" "errors" "strings" "testing" "time" - "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost/server/public/model" "github.com/moira-alert/moira" @@ -33,7 +34,7 @@ func TestSendEvents(t *testing.T) { Convey("When client return error, SendEvents should return error", func() { ctrl := gomock.NewController(t) client := mock.NewMockClient(ctrl) - client.EXPECT().CreatePost(gomock.Any()).Return(nil, nil, errors.New("")) + client.EXPECT().CreatePost(context.Background(), gomock.Any()).Return(nil, nil, errors.New("")) sender.client = client events, contact, trigger, plots, throttled := moira.NotificationEvents{}, moira.ContactData{}, moira.TriggerData{}, make([][]byte, 0), false @@ -44,7 +45,7 @@ func TestSendEvents(t *testing.T) { Convey("When client CreatePost is success and no plots, SendEvents should not return error", func() { ctrl := gomock.NewController(t) client := mock.NewMockClient(ctrl) - client.EXPECT().CreatePost(gomock.Any()).Return(&model.Post{Id: "postID"}, nil, nil) + client.EXPECT().CreatePost(context.Background(), gomock.Any()).Return(&model.Post{Id: "postID"}, nil, nil) sender.client = client events, contact, trigger, plots, throttled := moira.NotificationEvents{}, moira.ContactData{}, moira.TriggerData{}, make([][]byte, 0), false @@ -55,8 +56,8 @@ func TestSendEvents(t *testing.T) { Convey("When client CreatePost is success and have succeeded sent plots, SendEvents should not return error", func() { ctrl := gomock.NewController(t) client := mock.NewMockClient(ctrl) - client.EXPECT().CreatePost(gomock.Any()).Return(&model.Post{Id: "postID"}, nil, nil).Times(2) - client.EXPECT().UploadFile(gomock.Any(), "contactDataID", "triggerID.png"). + client.EXPECT().CreatePost(context.Background(), gomock.Any()).Return(&model.Post{Id: "postID"}, nil, nil).Times(2) + client.EXPECT().UploadFile(context.Background(), gomock.Any(), "contactDataID", "triggerID.png"). Return( &model.FileUploadResponse{ FileInfos: []*model.FileInfo{{Id: "fileID"}}, From 586cc22a76feb853bba21da734334e58df1b695a Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:06:29 +0300 Subject: [PATCH 37/46] fix(api): add x-nullable annotation and fix documentaion in config handler (#903) --- api/dto/target.go | 2 +- api/dto/triggers.go | 24 +++++++++++------------ api/handler/config.go | 7 +++++-- api/handler/triggers.go | 7 ++++--- datatypes.go | 42 ++++++++++++++++++++--------------------- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/api/dto/target.go b/api/dto/target.go index 0654b5c3e..39044d3ac 100644 --- a/api/dto/target.go +++ b/api/dto/target.go @@ -125,7 +125,7 @@ func (p *ProblemOfTarget) hasError() bool { type TreeOfProblems struct { SyntaxOk bool `json:"syntax_ok" example:"true"` - TreeOfProblems *ProblemOfTarget `json:"tree_of_problems,omitempty"` + TreeOfProblems *ProblemOfTarget `json:"tree_of_problems,omitempty" extensions:"x-nullable"` } // TargetVerification validates trigger targets. diff --git a/api/dto/triggers.go b/api/dto/triggers.go index 30c372b56..78755b53f 100644 --- a/api/dto/triggers.go +++ b/api/dto/triggers.go @@ -23,10 +23,10 @@ var targetNameRegex = regexp.MustCompile("t(\\d+)") var asteriskPattern = "*" type TriggersList struct { - Page *int64 `json:"page,omitempty" format:"int64"` - Size *int64 `json:"size,omitempty" format:"int64"` - Total *int64 `json:"total,omitempty" format:"int64"` - Pager *string `json:"pager,omitempty"` + Page *int64 `json:"page,omitempty" format:"int64" extensions:"x-nullable"` + Size *int64 `json:"size,omitempty" format:"int64" extensions:"x-nullable"` + Total *int64 `json:"total,omitempty" format:"int64" extensions:"x-nullable"` + Pager *string `json:"pager,omitempty" extensions:"x-nullable"` List []moira.TriggerCheck `json:"list"` } @@ -46,23 +46,23 @@ type TriggerModel struct { // Trigger name Name string `json:"name" example:"Not enough disk space left"` // Description string - Desc *string `json:"desc,omitempty" example:"check the size of /var/log"` + Desc *string `json:"desc,omitempty" example:"check the size of /var/log" extensions:"x-nullable"` // Graphite-like targets: t1, t2, ... Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` // WARN threshold - WarnValue *float64 `json:"warn_value" example:"500"` + WarnValue *float64 `json:"warn_value" example:"500" extensions:"x-nullable"` // ERROR threshold - ErrorValue *float64 `json:"error_value" example:"1000"` + ErrorValue *float64 `json:"error_value" example:"1000" extensions:"x-nullable"` // Could be: rising, falling, expression TriggerType string `json:"trigger_type" example:"rising"` // Set of tags to manipulate subscriptions Tags []string `json:"tags" example:"server,disk"` // When there are no metrics for trigger, Moira will switch metric to TTLState state after TTL seconds - TTLState *moira.TTLState `json:"ttl_state,omitempty" example:"NODATA"` + TTLState *moira.TTLState `json:"ttl_state,omitempty" example:"NODATA" extensions:"x-nullable"` // When there are no metrics for trigger, Moira will switch metric to TTLState state after TTL seconds TTL int64 `json:"ttl,omitempty" example:"600" format:"int64"` // Determines when Moira should monitor trigger - Schedule *moira.ScheduleData `json:"sched,omitempty"` + Schedule *moira.ScheduleData `json:"sched,omitempty" extensions:"x-nullable"` // Used if you need more complex logic than provided by WARN/ERROR values Expression string `json:"expression" example:""` // Graphite patterns for trigger @@ -78,9 +78,9 @@ type TriggerModel struct { // A list of targets that have only alone metrics AloneMetrics map[string]bool `json:"alone_metrics" example:"t1:true"` // Datetime when the trigger was created - CreatedAt *time.Time `json:"created_at"` + CreatedAt *time.Time `json:"created_at" extensions:"x-nullable"` // Datetime when the trigger was updated - UpdatedAt *time.Time `json:"updated_at"` + UpdatedAt *time.Time `json:"updated_at" extensions:"x-nullable"` // Username who created trigger CreatedBy string `json:"created_by"` // Username who updated trigger @@ -396,7 +396,7 @@ func (*MetricsMaintenance) Bind(*http.Request) error { } type TriggerMaintenance struct { - Trigger *int64 `json:"trigger" example:"1594225165" format:"int64"` + Trigger *int64 `json:"trigger" example:"1594225165" format:"int64" extensions:"x-nullable"` Metrics map[string]int64 `json:"metrics"` } diff --git a/api/handler/config.go b/api/handler/config.go index ea62edc2b..3cd075195 100644 --- a/api/handler/config.go +++ b/api/handler/config.go @@ -3,8 +3,11 @@ package handler import "net/http" type ContactExample struct { - Type string `json:"type" example:"telegram"` - Label string `json:"label" example:"Telegram"` + Type string `json:"type" example:"webhook kontur"` + Label string `json:"label" example:"Webhook Kontur"` + Validation string `json:"validation" example:"^(http|https):\\/\\/.*(testkontur.ru|kontur.host|skbkontur.ru)(:[0-9]{2,5})?\\/"` + Placeholder string `json:"placeholder" example:"https://service.testkontur.ru/webhooks/moira"` + Help string `json:"help" example:"### Domains whitelist:\n - skbkontur.ru\n - testkontur.ru\n - kontur.host"` } type ConfigurationResponse struct { diff --git a/api/handler/triggers.go b/api/handler/triggers.go index 22dbb4636..c16f56bdd 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -71,7 +71,7 @@ func getAllTriggers(writer http.ResponseWriter, request *http.Request) { // @success 200 {object} dto.TriggersList "Fetched unused triggers" // @failure 422 {object} api.ErrorRenderExample "Render error" // @failure 500 {object} api.ErrorInternalServerExample "Internal server error" -// @router /trigger [get] +// @router /trigger/unused [get] func getUnusedTriggers(writer http.ResponseWriter, request *http.Request) { triggersList, errorResponse := controller.GetUnusedTriggerIDs(database) if errorResponse != nil { @@ -288,14 +288,15 @@ func searchTriggers(writer http.ResponseWriter, request *http.Request) { // nolint: gofmt,goimports // // @summary Delete triggers pager +// @id delete-pager // @tags trigger // @produce json -// @param pagerID path string true "Pager ID to delete" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) +// @param pagerID query string false "Pager ID" default(bcba82f5-48cf-44c0-b7d6-e1d32c64a88c) // @success 200 {object} dto.TriggersSearchResultDeleteResponse "Successfully deleted pager" // @failure 404 {object} api.ErrorNotFoundExample "Resource not found" // @failure 422 {object} api.ErrorRenderExample "Render error" // @failure 500 {object} api.ErrorInternalServerExample "Internal server error" -// @router /trigger/pagers/{pagerID} [delete] +// @router /trigger/search/pager [delete] func deletePager(writer http.ResponseWriter, request *http.Request) { pagerID := middleware.GetPagerID(request) diff --git a/datatypes.go b/datatypes.go index fcfb1096e..d30a20db8 100644 --- a/datatypes.go +++ b/datatypes.go @@ -46,15 +46,15 @@ type NotificationEvent struct { IsTriggerEvent bool `json:"trigger_event,omitempty" example:"true"` Timestamp int64 `json:"timestamp" example:"1590741878" format:"int64"` Metric string `json:"metric" example:"carbon.agents.*.metricsReceived"` - Value *float64 `json:"value,omitempty" example:"70"` + Value *float64 `json:"value,omitempty" example:"70" extensions:"x-nullable"` Values map[string]float64 `json:"values,omitempty"` State State `json:"state" example:"OK"` TriggerID string `json:"trigger_id" example:"5ff37996-8927-4cab-8987-970e80d8e0a8"` - SubscriptionID *string `json:"sub_id,omitempty"` + SubscriptionID *string `json:"sub_id,omitempty" extensions:"x-nullable"` ContactID string `json:"contact_id,omitempty"` OldState State `json:"old_state" example:"ERROR"` - Message *string `json:"msg,omitempty"` - MessageEventInfo *EventInfo `json:"event_message"` + Message *string `json:"msg,omitempty" extensions:"x-nullable"` + MessageEventInfo *EventInfo `json:"event_message" extensions:"x-nullable"` } // NotificationEventHistoryItem is in use to store notifications history of channel @@ -70,8 +70,8 @@ type NotificationEventHistoryItem struct { // EventInfo - a base for creating messages. type EventInfo struct { - Maintenance *MaintenanceInfo `json:"maintenance,omitempty"` - Interval *int64 `json:"interval,omitempty" example:"0" format:"int64"` + Maintenance *MaintenanceInfo `json:"maintenance,omitempty" extensions:"x-nullable"` + Interval *int64 `json:"interval,omitempty" example:"0" format:"int64" extensions:"x-nullable"` } // CreateMessage - creates a message based on EventInfo. @@ -274,30 +274,30 @@ const ( type Trigger struct { ID string `json:"id" example:"292516ed-4924-4154-a62c-ebe312431fce"` Name string `json:"name" example:"Not enough disk space left"` - Desc *string `json:"desc,omitempty" example:"check the size of /var/log"` + Desc *string `json:"desc,omitempty" example:"check the size of /var/log" extensions:"x-nullable"` Targets []string `json:"targets" example:"devOps.my_server.hdd.freespace_mbytes"` - WarnValue *float64 `json:"warn_value" example:"5000"` - ErrorValue *float64 `json:"error_value" example:"1000"` + WarnValue *float64 `json:"warn_value" example:"5000" extensions:"x-nullable"` + ErrorValue *float64 `json:"error_value" example:"1000" extensions:"x-nullable"` TriggerType string `json:"trigger_type" example:"rising"` Tags []string `json:"tags" example:"server,disk"` - TTLState *TTLState `json:"ttl_state,omitempty" example:"NODATA"` + TTLState *TTLState `json:"ttl_state,omitempty" example:"NODATA" extensions:"x-nullable"` TTL int64 `json:"ttl,omitempty" example:"600" format:"int64"` - Schedule *ScheduleData `json:"sched,omitempty"` - Expression *string `json:"expression,omitempty" example:""` - PythonExpression *string `json:"python_expression,omitempty"` + Schedule *ScheduleData `json:"sched,omitempty" extensions:"x-nullable"` + Expression *string `json:"expression,omitempty" example:"" extensions:"x-nullable"` + PythonExpression *string `json:"python_expression,omitempty" extensions:"x-nullable"` Patterns []string `json:"patterns" example:""` TriggerSource TriggerSource `json:"trigger_source,omitempty" example:"graphite_local"` MuteNewMetrics bool `json:"mute_new_metrics" example:"false"` AloneMetrics map[string]bool `json:"alone_metrics" example:"t1:true"` - CreatedAt *int64 `json:"created_at" format:"int64"` - UpdatedAt *int64 `json:"updated_at" format:"int64"` + CreatedAt *int64 `json:"created_at" format:"int64" extensions:"x-nullable"` + UpdatedAt *int64 `json:"updated_at" format:"int64" extensions:"x-nullable"` CreatedBy string `json:"created_by"` UpdatedBy string `json:"updated_by"` } type TriggerSource string -const ( +var ( TriggerSourceNotSet TriggerSource = "" GraphiteLocal TriggerSource = "graphite_local" GraphiteRemote TriggerSource = "graphite_remote" @@ -395,7 +395,7 @@ type MetricState struct { Suppressed bool `json:"suppressed" example:"false"` SuppressedState State `json:"suppressed_state,omitempty"` Timestamp int64 `json:"timestamp" example:"1590741878" format:"int64"` - Value *float64 `json:"value,omitempty" example:"70"` + Value *float64 `json:"value,omitempty" example:"70" extensions:"x-nullable"` Values map[string]float64 `json:"values,omitempty"` Maintenance int64 `json:"maintenance,omitempty" example:"0" format:"int64"` MaintenanceInfo MaintenanceInfo `json:"maintenance_info"` @@ -418,10 +418,10 @@ func (metricState *MetricState) GetMaintenance() (MaintenanceInfo, int64) { // MaintenanceInfo represents user and time set/unset maintenance type MaintenanceInfo struct { - StartUser *string `json:"setup_user"` - StartTime *int64 `json:"setup_time" example:"0" format:"int64"` - StopUser *string `json:"remove_user"` - StopTime *int64 `json:"remove_time" example:"0" format:"int64"` + StartUser *string `json:"setup_user" extensions:"x-nullable"` + StartTime *int64 `json:"setup_time" example:"0" format:"int64" extensions:"x-nullable"` + StopUser *string `json:"remove_user" extensions:"x-nullable"` + StopTime *int64 `json:"remove_time" example:"0" format:"int64" extensions:"x-nullable"` } // Set maintanace start and stop users and times From fee498968558c401702ee07ab108e0edfa2d7708 Mon Sep 17 00:00:00 2001 From: Xenia N Date: Wed, 18 Oct 2023 11:57:41 +0600 Subject: [PATCH 38/46] build(go): update go to 1.19 (#938) --- Dockerfile.api | 2 +- Dockerfile.checker | 2 +- Dockerfile.cli | 2 +- Dockerfile.filter | 2 +- Dockerfile.notifier | 2 +- go.mod | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile.api b/Dockerfile.api index f34cbc4e8..1a1e228ae 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.19 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.checker b/Dockerfile.checker index b7d1b7b9e..d6628c867 100644 --- a/Dockerfile.checker +++ b/Dockerfile.checker @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.19 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.cli b/Dockerfile.cli index 0ef5decc9..44bb49c5c 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.19 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.filter b/Dockerfile.filter index 1b7c81cce..3b61a26d8 100644 --- a/Dockerfile.filter +++ b/Dockerfile.filter @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.19 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.notifier b/Dockerfile.notifier index 13431c7af..3c6dd5063 100644 --- a/Dockerfile.notifier +++ b/Dockerfile.notifier @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.19 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/go.mod b/go.mod index dfdd7baf2..7ceaad8d9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/moira-alert/moira -go 1.18 +go 1.19 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible From cac64eaecacfd1f9051e9b664d0f06280d4d4c55 Mon Sep 17 00:00:00 2001 From: Tetrergeru Date: Wed, 18 Oct 2023 14:36:24 +0200 Subject: [PATCH 39/46] fix: trigger check panics (#939) --- api/controller/trigger_metrics.go | 2 +- checker/check.go | 32 +++-- checker/check_test.go | 76 +++++++++-- checker/metrics/conversion/fetched_metrics.go | 6 +- checker/metrics/conversion/set_helper.go | 57 ++++++--- checker/metrics/conversion/set_helper_test.go | 68 +++++----- checker/metrics/conversion/trigger_metrics.go | 63 ++++----- .../conversion/trigger_metrics_test.go | 120 +++++++++++------- checker/worker/handler.go | 3 +- helpers.go | 12 +- helpers_test.go | 8 +- metric_source/metric_data.go | 2 + plotting/curve.go | 4 +- plotting/limits.go | 2 +- 14 files changed, 287 insertions(+), 168 deletions(-) diff --git a/api/controller/trigger_metrics.go b/api/controller/trigger_metrics.go index 96bd19d18..bb74c88d9 100644 --- a/api/controller/trigger_metrics.go +++ b/api/controller/trigger_metrics.go @@ -65,7 +65,7 @@ func GetTriggerMetrics(dataBase moira.Database, metricSourceProvider *metricSour for i, l := 0, len(timeSeries.Values); i < l; i++ { timestamp := timeSeries.StartTime + int64(i)*timeSeries.StepTime value := timeSeries.GetTimestampValue(timestamp) - if moira.IsValidFloat64(value) { + if moira.IsFiniteNumber(value) { values = append(values, moira.MetricValue{Value: value, Timestamp: timestamp}) } } diff --git a/checker/check.go b/checker/check.go index 973f00857..b1493a82b 100644 --- a/checker/check.go +++ b/checker/check.go @@ -300,6 +300,9 @@ func (triggerChecker *TriggerChecker) check( ) (moira.CheckData, error) { // Case when trigger have only alone metrics if len(metrics) == 0 { + if len(aloneMetrics) == 0 { + return checkData, nil + } if metrics == nil { metrics = make(map[string]map[string]metricSource.MetricData, 1) } @@ -446,7 +449,7 @@ func (triggerChecker *TriggerChecker) getMetricStepsStates( valueTimestamp := startTime + stepTime*stepsDifference endTimestamp := triggerChecker.until + stepTime for ; valueTimestamp < endTimestamp; valueTimestamp += stepTime { - metricNewState, err := triggerChecker.getMetricDataState(&metrics, &previousState, &valueTimestamp, &checkPoint, logger) + metricNewState, err := triggerChecker.getMetricDataState(metrics, &previousState, &valueTimestamp, &checkPoint, logger) if err != nil { return last, current, err } @@ -459,11 +462,16 @@ func (triggerChecker *TriggerChecker) getMetricStepsStates( return last, current, nil } -func (triggerChecker *TriggerChecker) getMetricDataState(metrics *map[string]metricSource.MetricData, - lastState *moira.MetricState, valueTimestamp, checkPoint *int64, logger moira.Logger) (*moira.MetricState, error) { +func (triggerChecker *TriggerChecker) getMetricDataState( + metrics map[string]metricSource.MetricData, + lastState *moira.MetricState, + valueTimestamp, checkPoint *int64, + logger moira.Logger, +) (*moira.MetricState, error) { if *valueTimestamp <= *checkPoint { return nil, nil } + triggerExpression, values, noEmptyValues := getExpressionValues(metrics, valueTimestamp) if !noEmptyValues { return nil, nil @@ -493,18 +501,24 @@ func (triggerChecker *TriggerChecker) getMetricDataState(metrics *map[string]met ), nil } -func getExpressionValues(metrics *map[string]metricSource.MetricData, valueTimestamp *int64) (*expression.TriggerExpression, map[string]float64, bool) { +func getExpressionValues(metrics map[string]metricSource.MetricData, valueTimestamp *int64) ( + triggerExpression *expression.TriggerExpression, + values map[string]float64, + noEmptyValues bool, +) { expression := &expression.TriggerExpression{ - AdditionalTargetsValues: make(map[string]float64, len(*metrics)-1), + AdditionalTargetsValues: make(map[string]float64, len(metrics)-1), } - values := make(map[string]float64, len(*metrics)) + values = make(map[string]float64, len(metrics)) - for i := 0; i < len(*metrics); i++ { + for i := 0; i < len(metrics); i++ { targetName := fmt.Sprintf("t%d", i+1) - metric := (*metrics)[targetName] + metric := metrics[targetName] + value := metric.GetTimestampValue(*valueTimestamp) values[targetName] = value - if !moira.IsValidFloat64(value) { + + if !moira.IsFiniteNumber(value) { return expression, values, false } if i == 0 { diff --git a/checker/check_test.go b/checker/check_test.go index dc90dcf97..b75e335c9 100644 --- a/checker/check_test.go +++ b/checker/check_test.go @@ -62,7 +62,7 @@ func TestGetMetricDataState(t *testing.T) { var valueTimestamp int64 = 37 var checkPoint int64 = 47 Convey("Checkpoint more than valueTimestamp", t, func() { - metricState, err := triggerChecker.getMetricDataState(&metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) + metricState, err := triggerChecker.getMetricDataState(metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) So(err, ShouldBeNil) So(metricState, ShouldBeNil) }) @@ -71,7 +71,7 @@ func TestGetMetricDataState(t *testing.T) { Convey("Has all value by eventTimestamp step", func() { var valueTimestamp int64 = 42 var checkPoint int64 = 27 - metricState, err := triggerChecker.getMetricDataState(&metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) + metricState, err := triggerChecker.getMetricDataState(metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) So(err, ShouldBeNil) So(metricState, ShouldResemble, &moira.MetricState{ State: moira.StateOK, @@ -86,7 +86,7 @@ func TestGetMetricDataState(t *testing.T) { Convey("No value in main metric data by eventTimestamp step", func() { var valueTimestamp int64 = 66 var checkPoint int64 = 11 - metricState, err := triggerChecker.getMetricDataState(&metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) + metricState, err := triggerChecker.getMetricDataState(metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) So(err, ShouldBeNil) So(metricState, ShouldBeNil) }) @@ -94,7 +94,7 @@ func TestGetMetricDataState(t *testing.T) { Convey("IsAbsent in main metric data by eventTimestamp step", func() { var valueTimestamp int64 = 29 var checkPoint int64 = 11 - metricState, err := triggerChecker.getMetricDataState(&metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) + metricState, err := triggerChecker.getMetricDataState(metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) So(err, ShouldBeNil) So(metricState, ShouldBeNil) }) @@ -102,7 +102,7 @@ func TestGetMetricDataState(t *testing.T) { Convey("No value in additional metric data by eventTimestamp step", func() { var valueTimestamp int64 = 26 var checkPoint int64 = 11 - metricState, err := triggerChecker.getMetricDataState(&metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) + metricState, err := triggerChecker.getMetricDataState(metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) So(err, ShouldBeNil) So(metricState, ShouldBeNil) }) @@ -113,7 +113,7 @@ func TestGetMetricDataState(t *testing.T) { triggerChecker.trigger.ErrorValue = nil var valueTimestamp int64 = 42 var checkPoint int64 = 27 - metricState, err := triggerChecker.getMetricDataState(&metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) + metricState, err := triggerChecker.getMetricDataState(metrics, &metricLastState, &valueTimestamp, &checkPoint, logger) So(err.Error(), ShouldResemble, "error value and warning value can not be empty") So(metricState, ShouldBeNil) }) @@ -974,6 +974,54 @@ func TestCheck(t *testing.T) { }) } +func TestCheckWithNoMetrics(t *testing.T) { + logger, _ := logging.GetLogger("Test") + metricsToCheck := map[string]map[string]metricSource.MetricData{} + + Convey("given triggerChecker.check is called with empty metric map", t, func() { + warnValue := float64(10) + errValue := float64(20) + pattern := "super.puper.pattern" + ttl := int64(600) + + lastCheck := moira.CheckData{ + Metrics: make(map[string]moira.MetricState), + State: moira.StateNODATA, + Timestamp: 66, + } + + triggerChecker := TriggerChecker{ + triggerID: "SuperId", + logger: logger, + config: &Config{}, + from: 3617, + until: 3667, + ttl: ttl, + ttlState: moira.TTLStateNODATA, + trigger: &moira.Trigger{ + ErrorValue: &errValue, + WarnValue: &warnValue, + TriggerType: moira.RisingTrigger, + Targets: []string{pattern}, + Patterns: []string{pattern}, + }, + lastCheck: &lastCheck, + } + aloneMetrics := map[string]metricSource.MetricData{} + checkData := newCheckData(&lastCheck, triggerChecker.until) + newCheckData, err := triggerChecker.check(metricsToCheck, aloneMetrics, checkData, logger) + + So(err, ShouldBeNil) + So(newCheckData, ShouldResemble, moira.CheckData{ + Metrics: map[string]moira.MetricState{}, + MetricsToTargetRelation: map[string]string{}, + Timestamp: triggerChecker.until, + State: moira.StateNODATA, + Score: 0, + }) + }) +} + func TestIgnoreNodataToOk(t *testing.T) { mockCtrl := gomock.NewController(t) logger, _ := logging.GetLogger("Test") @@ -1501,25 +1549,25 @@ func TestGetExpressionValues(t *testing.T) { expectedValues := map[string]float64{"t1": 0} var valueTimestamp int64 = 17 - expression, values, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + expression, values, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeTrue) So(expression, ShouldResemble, expectedExpression) So(values, ShouldResemble, expectedValues) }) Convey("last value is empty", func() { var valueTimestamp int64 = 67 - _, _, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + _, _, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeFalse) }) Convey("value before first value", func() { var valueTimestamp int64 = 11 - _, _, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + _, _, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeFalse) }) Convey("value in the middle is empty ", func() { var valueTimestamp int64 = 44 - _, _, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + _, _, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeFalse) }) @@ -1531,7 +1579,7 @@ func TestGetExpressionValues(t *testing.T) { expectedValues := map[string]float64{"t1": 3} var valueTimestamp int64 = 53 - expression, values, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + expression, values, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeTrue) So(expression, ShouldResemble, expectedExpression) So(values, ShouldResemble, expectedValues) @@ -1560,13 +1608,13 @@ func TestGetExpressionValues(t *testing.T) { Convey("t1 value in the middle is empty ", func() { var valueTimestamp int64 = 29 - _, _, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + _, _, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeFalse) }) Convey("t1 and t2 values in the middle is empty ", func() { var valueTimestamp int64 = 42 - _, _, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + _, _, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeFalse) }) @@ -1574,7 +1622,7 @@ func TestGetExpressionValues(t *testing.T) { expectedValues := map[string]float64{"t1": 0, "t2": 4} var valueTimestamp int64 = 17 - expression, values, noEmptyValues := getExpressionValues(&metrics, &valueTimestamp) + expression, values, noEmptyValues := getExpressionValues(metrics, &valueTimestamp) So(noEmptyValues, ShouldBeTrue) So(expression.MainTargetValue, ShouldBeIn, []float64{0, 4}) So(values, ShouldResemble, expectedValues) diff --git a/checker/metrics/conversion/fetched_metrics.go b/checker/metrics/conversion/fetched_metrics.go index 51cf196c2..3261946c4 100644 --- a/checker/metrics/conversion/fetched_metrics.go +++ b/checker/metrics/conversion/fetched_metrics.go @@ -38,15 +38,15 @@ func (m FetchedTargetMetrics) CleanWildcards() FetchedTargetMetrics { // the same name and returns new FetchedPatternMetrics without duplicates and slice of duplicated metrics names. func (m FetchedTargetMetrics) Deduplicate() (FetchedTargetMetrics, []string) { deduplicated := NewFetchedTargetMetricsWithCapacity(len(m)) - collectedNames := make(setHelper, len(m)) + collectedNames := make(set[string], len(m)) var duplicates []string for _, metric := range m { - if collectedNames[metric.Name] { + if collectedNames.contains(metric.Name) { duplicates = append(duplicates, metric.Name) } else { deduplicated = append(deduplicated, metric) } - collectedNames[metric.Name] = true + collectedNames[metric.Name] = void } return deduplicated, duplicates } diff --git a/checker/metrics/conversion/set_helper.go b/checker/metrics/conversion/set_helper.go index c386500e4..bbe98e5ca 100644 --- a/checker/metrics/conversion/set_helper.go +++ b/checker/metrics/conversion/set_helper.go @@ -1,37 +1,64 @@ package conversion -// setHelper is a map that represents a set of strings with corresponding methods. -type setHelper map[string]bool +var void struct{} = struct{}{} -// newSetHelperFromTriggerTargetMetrics is a constructor function for setHelper. -func newSetHelperFromTriggerTargetMetrics(metrics TriggerTargetMetrics) setHelper { - result := make(setHelper, len(metrics)) +// set[string] is a map that represents a set of strings with corresponding methods. +type set[K comparable] map[K]struct{} + +func (set set[K]) contains(key K) bool { + _, ok := set[key] + return ok +} + +func (set set[K]) insert(key K) { + set[key] = void +} + +func newSet[K comparable](value map[K]bool) set[K] { + res := make(set[K], len(value)) + + for k, v := range value { + if v { + res.insert(k) + } + } + + return res +} + +// newSetFromTriggerTargetMetrics is a constructor function for setHelper. +func newSetFromTriggerTargetMetrics(metrics TriggerTargetMetrics) set[string] { + result := make(set[string], len(metrics)) for metricName := range metrics { - result[metricName] = true + result.insert(metricName) } return result } // diff is a set relative complement operation that returns a new set with elements // that appear only in second set. -func (h setHelper) diff(other setHelper) setHelper { - result := make(setHelper, len(h)) +func (self set[string]) diff(other set[string]) set[string] { + result := make(set[string], len(self)) + for metricName := range other { - if _, ok := h[metricName]; !ok { - result[metricName] = true + if !self.contains(metricName) { + result.insert(metricName) } } + return result } // union is a sets union operation that return a new set with elements from both sets. -func (h setHelper) union(other setHelper) setHelper { - result := make(setHelper, len(h)+len(other)) - for metricName := range h { - result[metricName] = true +func (self set[string]) union(other set[string]) set[string] { + result := make(set[string], len(self)+len(other)) + + for metricName := range self { + result.insert(metricName) } for metricName := range other { - result[metricName] = true + result.insert(metricName) } + return result } diff --git a/checker/metrics/conversion/set_helper_test.go b/checker/metrics/conversion/set_helper_test.go index ab53e26c3..93c477beb 100644 --- a/checker/metrics/conversion/set_helper_test.go +++ b/checker/metrics/conversion/set_helper_test.go @@ -13,14 +13,14 @@ func Test_newSetHelperFromTriggerTargetMetrics(t *testing.T) { tests := []struct { name string args args - want setHelper + want set[string] }{ { name: "is empty", args: args{ metrics: TriggerTargetMetrics{}, }, - want: setHelper{}, + want: set[string]{}, }, { name: "is not empty", @@ -29,14 +29,14 @@ func Test_newSetHelperFromTriggerTargetMetrics(t *testing.T) { "metric.test.1": {Name: "metric.name.1"}, }, }, - want: setHelper{"metric.test.1": true}, + want: set[string]{"metric.test.1": void}, }, } Convey("TriggerPatterMetrics", t, func() { for _, tt := range tests { Convey(tt.name, func() { - actual := newSetHelperFromTriggerTargetMetrics(tt.args.metrics) + actual := newSetFromTriggerTargetMetrics(tt.args.metrics) So(actual, ShouldResemble, tt.want) }) } @@ -45,53 +45,53 @@ func Test_newSetHelperFromTriggerTargetMetrics(t *testing.T) { func Test_setHelper_union(t *testing.T) { type args struct { - other setHelper + other set[string] } tests := []struct { name string - h setHelper + h set[string] args args - want setHelper + want set[string] }{ { name: "Both empty", - h: setHelper{}, + h: set[string]{}, args: args{ - other: setHelper{}, + other: set[string]{}, }, - want: setHelper{}, + want: set[string]{}, }, { name: "Target is empty, other is not empty", - h: setHelper{}, + h: set[string]{}, args: args{ - other: setHelper{"metric.test.1": true}, + other: set[string]{"metric.test.1": void}, }, - want: setHelper{"metric.test.1": true}, + want: set[string]{"metric.test.1": void}, }, { name: "Target is not empty, other is empty", - h: setHelper{"metric.test.1": true}, + h: set[string]{"metric.test.1": void}, args: args{ - other: setHelper{}, + other: set[string]{}, }, - want: setHelper{"metric.test.1": true}, + want: set[string]{"metric.test.1": void}, }, { name: "Both are not empty", - h: setHelper{"metric.test.1": true}, + h: set[string]{"metric.test.1": void}, args: args{ - other: setHelper{"metric.test.2": true}, + other: set[string]{"metric.test.2": void}, }, - want: setHelper{"metric.test.1": true, "metric.test.2": true}, + want: set[string]{"metric.test.1": void, "metric.test.2": void}, }, { name: "Both are not empty and have same names", - h: setHelper{"metric.test.1": true, "metric.test.2": true}, + h: set[string]{"metric.test.1": void, "metric.test.2": void}, args: args{ - other: setHelper{"metric.test.2": true, "metric.test.3": true}, + other: set[string]{"metric.test.2": void, "metric.test.3": void}, }, - want: setHelper{"metric.test.1": true, "metric.test.2": true, "metric.test.3": true}, + want: set[string]{"metric.test.1": void, "metric.test.2": void, "metric.test.3": void}, }, } Convey("union", t, func() { @@ -106,37 +106,37 @@ func Test_setHelper_union(t *testing.T) { func Test_setHelper_diff(t *testing.T) { type args struct { - other setHelper + other set[string] } tests := []struct { name string - h setHelper + h set[string] args args - want setHelper + want set[string] }{ { name: "both have same elements", - h: setHelper{"t1": true, "t2": true}, + h: set[string]{"t1": void, "t2": void}, args: args{ - other: setHelper{"t1": true, "t2": true}, + other: set[string]{"t1": void, "t2": void}, }, - want: setHelper{}, + want: set[string]{}, }, { name: "other have additional values", - h: setHelper{"t1": true, "t2": true}, + h: set[string]{"t1": void, "t2": void}, args: args{ - other: setHelper{"t1": true, "t2": true, "t3": true}, + other: set[string]{"t1": void, "t2": void, "t3": void}, }, - want: setHelper{"t3": true}, + want: set[string]{"t3": void}, }, { name: "origin have additional values", - h: setHelper{"t1": true, "t2": true, "t3": true}, + h: set[string]{"t1": void, "t2": void, "t3": void}, args: args{ - other: setHelper{"t1": true, "t2": true}, + other: set[string]{"t1": void, "t2": void}, }, - want: setHelper{}, + want: set[string]{}, }, } Convey("diff", t, func() { diff --git a/checker/metrics/conversion/trigger_metrics.go b/checker/metrics/conversion/trigger_metrics.go index ea58fabea..deeaef8f9 100644 --- a/checker/metrics/conversion/trigger_metrics.go +++ b/checker/metrics/conversion/trigger_metrics.go @@ -27,7 +27,7 @@ func NewTriggerTargetMetrics(source FetchedTargetMetrics) TriggerTargetMetrics { // Populate is a function that takes the list of metric names that first appeared and // adds metrics with this names and empty values. -func (m TriggerTargetMetrics) Populate(lastMetrics map[string]bool, from, to int64) TriggerTargetMetrics { +func (m TriggerTargetMetrics) Populate(lastMetrics set[string], from, to int64) TriggerTargetMetrics { result := newTriggerTargetMetricsWithCapacity(len(m)) var firstMetric metricSource.MetricData @@ -62,35 +62,37 @@ func NewTriggerMetricsWithCapacity(capacity int) TriggerMetrics { // Populate is a function that takes TriggerMetrics and populate targets // that is missing metrics that appear in another targets except the targets that have // only alone metrics. -func (m TriggerMetrics) Populate(lastMetrics map[string]moira.MetricState, declaredAloneMetrics map[string]bool, from int64, to int64) TriggerMetrics { +func (triggerMetrics TriggerMetrics) Populate(lastMetrics map[string]moira.MetricState, declaredAloneMetrics map[string]bool, from int64, to int64) TriggerMetrics { // This one have all metrics that should be in final TriggerMetrics. // This structure filled with metrics from last check, // current received metrics alone metrics from last check. - allMetrics := make(map[string]map[string]bool, len(m)) + allMetrics := make(map[string]set[string], len(triggerMetrics)) for metricName, metricState := range lastMetrics { for targetName := range metricState.Values { if _, ok := allMetrics[targetName]; !ok { - allMetrics[targetName] = make(map[string]bool) + allMetrics[targetName] = make(set[string]) } - allMetrics[targetName][metricName] = true + + allMetrics[targetName].insert(metricName) } } - for targetName, metrics := range m { + for targetName, metrics := range triggerMetrics { + if _, ok := allMetrics[targetName]; !ok { + allMetrics[targetName] = make(set[string]) + } + for metricName := range metrics { - if _, ok := allMetrics[targetName]; !ok { - allMetrics[targetName] = make(map[string]bool) - } - allMetrics[targetName][metricName] = true + allMetrics[targetName].insert(metricName) } } - diff := m.Diff(declaredAloneMetrics) + diff := triggerMetrics.FindMissingMetrics(newSet(declaredAloneMetrics)) for targetName, metrics := range diff { for metricName := range metrics { - allMetrics[targetName][metricName] = true + allMetrics[targetName].insert(metricName) } } @@ -100,7 +102,7 @@ func (m TriggerMetrics) Populate(lastMetrics map[string]moira.MetricState, decla // if declaredAloneMetrics[targetName] { // continue // } - targetMetrics, ok := m[targetName] + targetMetrics, ok := triggerMetrics[targetName] if !ok { targetMetrics = newTriggerTargetMetricsWithCapacity(len(metrics)) } @@ -138,20 +140,21 @@ func (m TriggerMetrics) Populate(lastMetrics map[string]moira.MetricState, decla // { // "t3": {metrics}, // } -func (m TriggerMetrics) FilterAloneMetrics(declaredAloneMetrics map[string]bool) (TriggerMetrics, AloneMetrics, error) { +func (triggerMetrics TriggerMetrics) FilterAloneMetrics(declaredAloneMetrics map[string]bool) (TriggerMetrics, AloneMetrics, error) { if len(declaredAloneMetrics) == 0 { - return m, NewAloneMetricsWithCapacity(0), nil + return triggerMetrics, NewAloneMetricsWithCapacity(0), nil } - result := NewTriggerMetricsWithCapacity(len(m)) - aloneMetrics := NewAloneMetricsWithCapacity(len(m)) // Just use len of m for optimization + metricCountUpperBound := len(triggerMetrics) + result := NewTriggerMetricsWithCapacity(metricCountUpperBound) + aloneMetrics := NewAloneMetricsWithCapacity(metricCountUpperBound) errorBuilder := newErrUnexpectedAloneMetricBuilder() errorBuilder.setDeclared(declaredAloneMetrics) - for targetName, targetMetrics := range m { + for targetName, targetMetrics := range triggerMetrics { if !declaredAloneMetrics[targetName] { - result[targetName] = m[targetName] + result[targetName] = triggerMetrics[targetName] continue } @@ -171,28 +174,28 @@ func (m TriggerMetrics) FilterAloneMetrics(declaredAloneMetrics map[string]bool) return result, aloneMetrics, nil } -// Diff is a function that returns a map of target names with metric names that are absent in +// FindMissingMetrics is a function that returns a map of target names with metric names that are absent in // current target but appear in another targets. -func (m TriggerMetrics) Diff(declaredAloneMetrics map[string]bool) map[string]map[string]bool { - result := make(map[string]map[string]bool) +func (triggerMetrics TriggerMetrics) FindMissingMetrics(declaredAloneMetrics set[string]) map[string]set[string] { + result := make(map[string]set[string]) - if len(m) == 0 { + if len(triggerMetrics) == 0 { return result } - fullMetrics := make(setHelper) + fullMetrics := make(set[string]) - for targetName, targetMetrics := range m { - if declaredAloneMetrics[targetName] { + for targetName, targetMetrics := range triggerMetrics { + if declaredAloneMetrics.contains(targetName) { continue } - currentMetrics := newSetHelperFromTriggerTargetMetrics(targetMetrics) + currentMetrics := newSetFromTriggerTargetMetrics(targetMetrics) fullMetrics = fullMetrics.union(currentMetrics) } - for targetName, targetMetrics := range m { - metricsSet := newSetHelperFromTriggerTargetMetrics(targetMetrics) - if declaredAloneMetrics[targetName] { + for targetName, targetMetrics := range triggerMetrics { + metricsSet := newSetFromTriggerTargetMetrics(targetMetrics) + if declaredAloneMetrics.contains(targetName) { continue } diff := metricsSet.diff(fullMetrics) diff --git a/checker/metrics/conversion/trigger_metrics_test.go b/checker/metrics/conversion/trigger_metrics_test.go index 94d0fd961..e6baf929d 100644 --- a/checker/metrics/conversion/trigger_metrics_test.go +++ b/checker/metrics/conversion/trigger_metrics_test.go @@ -35,7 +35,7 @@ func TestNewTriggerTargetMetrics(t *testing.T) { func TestTriggerTargetMetrics_Populate(t *testing.T) { type args struct { - lastMetrics map[string]bool + lastMetrics set[string] from int64 to int64 } @@ -52,9 +52,9 @@ func TestTriggerTargetMetrics_Populate(t *testing.T) { "metric.test.2": {Name: "metric.test.2", StartTime: 17, StopTime: 67, StepTime: 60, Values: []float64{0}}, }, args: args{ - lastMetrics: map[string]bool{ - "metric.test.1": true, - "metric.test.2": true, + lastMetrics: set[string]{ + "metric.test.1": void, + "metric.test.2": void, }, from: 17, to: 67, @@ -70,9 +70,9 @@ func TestTriggerTargetMetrics_Populate(t *testing.T) { "metric.test.1": {Name: "metric.test.1", StartTime: 17, StopTime: 67, StepTime: 60, Values: []float64{0}}, }, args: args{ - lastMetrics: map[string]bool{ - "metric.test.1": true, - "metric.test.2": true, + lastMetrics: set[string]{ + "metric.test.1": void, + "metric.test.2": void, }, from: 17, to: 67, @@ -117,14 +117,14 @@ func TestTriggerMetrics_Populate(t *testing.T) { to int64 } tests := []struct { - name string - m TriggerMetrics - args args - want TriggerMetrics + name string + triggerMetrics TriggerMetrics + args args + want TriggerMetrics }{ { name: "origin do not have missing metrics", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, "metric.test.2": {Name: "metric.test.2"}, @@ -148,7 +148,7 @@ func TestTriggerMetrics_Populate(t *testing.T) { }, { name: "origin have missing metrics", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, }, @@ -169,14 +169,46 @@ func TestTriggerMetrics_Populate(t *testing.T) { }, }, }, + { + name: "no trigger metrics for t2, empty last check (used to panic before PR #939)", + triggerMetrics: TriggerMetrics{ + "t1": TriggerTargetMetrics{ + "metric.test.1": {Name: "metric.test.1"}, + "metric.test.2": {Name: "metric.test.2"}, + }, + "t2": TriggerTargetMetrics{}, + }, + args: args{ + lastCheck: map[string]moira.MetricState{}, + declaredAloneMetrics: map[string]bool{}, + from: 17, + to: 67, + }, + want: TriggerMetrics{ + "t1": TriggerTargetMetrics{ + "metric.test.1": {Name: "metric.test.1", StopTime: 0}, + "metric.test.2": {Name: "metric.test.2", StopTime: 0}, + }, + "t2": TriggerTargetMetrics{ + "metric.test.1": {Name: "metric.test.1", StartTime: 17, StopTime: 67, StepTime: 60, Values: []float64{math.NaN()}}, + "metric.test.2": {Name: "metric.test.2", StartTime: 17, StopTime: 67, StepTime: 60, Values: []float64{math.NaN()}}, + }, + }, + }, } Convey("Populate", t, func() { - for _, tt := range tests { - Convey(tt.name, func() { - actual := tt.m.Populate(tt.args.lastCheck, tt.args.declaredAloneMetrics, tt.args.from, tt.args.to) - So(actual, ShouldHaveLength, len(tt.want)) + for _, testCase := range tests { + Convey(testCase.name, func() { + actual := testCase.triggerMetrics.Populate( + testCase.args.lastCheck, + testCase.args.declaredAloneMetrics, + testCase.args.from, + testCase.args.to, + ) + + So(actual, ShouldHaveLength, len(testCase.want)) for targetName, metrics := range actual { - wantMetrics, ok := tt.want[targetName] + wantMetrics, ok := testCase.want[targetName] So(metrics, ShouldHaveLength, len(wantMetrics)) So(ok, ShouldBeTrue) for metricName, actualMetric := range metrics { @@ -305,13 +337,13 @@ func TestTriggerMetrics_FilterAloneMetrics(t *testing.T) { func TestTriggerMetrics_Diff(t *testing.T) { tests := []struct { name string - m TriggerMetrics + triggerMetrics TriggerMetrics declaredAloneMetrics map[string]bool - want map[string]map[string]bool + want map[string]set[string] }{ { name: "all targets have same metrics", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, "metric.test.2": {Name: "metric.test.2"}, @@ -324,11 +356,11 @@ func TestTriggerMetrics_Diff(t *testing.T) { }, }, declaredAloneMetrics: map[string]bool{}, - want: map[string]map[string]bool{}, + want: map[string]set[string]{}, }, { name: "one target have missed metric", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, "metric.test.2": {Name: "metric.test.2"}, @@ -340,11 +372,11 @@ func TestTriggerMetrics_Diff(t *testing.T) { }, }, declaredAloneMetrics: map[string]bool{}, - want: map[string]map[string]bool{"t2": {"metric.test.3": true}}, + want: map[string]set[string]{"t2": {"metric.test.3": void}}, }, { name: "one target is alone metric", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, "metric.test.2": {Name: "metric.test.2"}, @@ -355,11 +387,11 @@ func TestTriggerMetrics_Diff(t *testing.T) { }, }, declaredAloneMetrics: map[string]bool{"t2": true}, - want: map[string]map[string]bool{}, + want: map[string]set[string]{}, }, { name: "another target have missed metric", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, "metric.test.2": {Name: "metric.test.2"}, @@ -373,11 +405,11 @@ func TestTriggerMetrics_Diff(t *testing.T) { }, }, declaredAloneMetrics: map[string]bool{}, - want: map[string]map[string]bool{"t1": {"metric.test.4": true}}, + want: map[string]set[string]{"t1": {"metric.test.4": void}}, }, { name: "one target is empty", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{}, "t2": TriggerTargetMetrics{ "metric.test.1": {Name: "metric.test.1"}, @@ -387,16 +419,16 @@ func TestTriggerMetrics_Diff(t *testing.T) { }, }, declaredAloneMetrics: map[string]bool{}, - want: map[string]map[string]bool{"t1": { - "metric.test.1": true, - "metric.test.2": true, - "metric.test.3": true, - "metric.test.4": true, + want: map[string]set[string]{"t1": { + "metric.test.1": void, + "metric.test.2": void, + "metric.test.3": void, + "metric.test.4": void, }}, }, { name: "Multiple targets with different metrics", - m: TriggerMetrics{ + triggerMetrics: TriggerMetrics{ "t1": TriggerTargetMetrics{ "metric.test.2": {Name: "metric.test.2"}, "metric.test.3": {Name: "metric.test.3"}, @@ -419,27 +451,27 @@ func TestTriggerMetrics_Diff(t *testing.T) { }, }, declaredAloneMetrics: map[string]bool{}, - want: map[string]map[string]bool{ + want: map[string]set[string]{ "t1": { - "metric.test.1": true, + "metric.test.1": void, }, "t2": { - "metric.test.2": true, + "metric.test.2": void, }, "t3": { - "metric.test.3": true, + "metric.test.3": void, }, "t4": { - "metric.test.4": true, + "metric.test.4": void, }, }, }, } Convey("Diff", t, func() { - for _, tt := range tests { - Convey(tt.name, func() { - actual := tt.m.Diff(tt.declaredAloneMetrics) - So(actual, ShouldResemble, tt.want) + for _, testCase := range tests { + Convey(testCase.name, func() { + actual := testCase.triggerMetrics.FindMissingMetrics(newSet(testCase.declaredAloneMetrics)) + So(actual, ShouldResemble, testCase.want) }) } }) diff --git a/checker/worker/handler.go b/checker/worker/handler.go index 24ef5c947..dddb81c6d 100644 --- a/checker/worker/handler.go +++ b/checker/worker/handler.go @@ -34,8 +34,7 @@ func (check *Checker) startTriggerHandler(triggerIDsToCheck <-chan string, metri } } -func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) error { - var err error +func (check *Checker) handleTrigger(triggerID string, metrics *metrics.CheckMetrics) (err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: '%s' stack: %s", r, debug.Stack()) diff --git a/helpers.go b/helpers.go index 9a999fb7a..1c4a367a5 100644 --- a/helpers.go +++ b/helpers.go @@ -76,15 +76,9 @@ func UseFloat64(f *float64) float64 { return *f } -// IsValidFloat64 checks float64 for Inf and NaN. If it is then float64 is not valid -func IsValidFloat64(val float64) bool { - if math.IsNaN(val) { - return false - } - if math.IsInf(val, 0) { - return false - } - return true +// IsFiniteNumber checks float64 for Inf and NaN. If it is then float64 is not valid +func IsFiniteNumber(val float64) bool { + return !(math.IsNaN(val) || math.IsInf(val, 0)) } // Subset return whether first is a subset of second diff --git a/helpers_test.go b/helpers_test.go index d2800ee79..803376070 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -203,10 +203,10 @@ func TestChunkSlice(t *testing.T) { func TestIsValidFloat64(t *testing.T) { Convey("values +Inf -Inf and NaN is invalid", t, func() { - So(IsValidFloat64(math.NaN()), ShouldBeFalse) - So(IsValidFloat64(math.Inf(-1)), ShouldBeFalse) - So(IsValidFloat64(math.Inf(1)), ShouldBeFalse) - So(IsValidFloat64(3.14), ShouldBeTrue) + So(IsFiniteNumber(math.NaN()), ShouldBeFalse) + So(IsFiniteNumber(math.Inf(-1)), ShouldBeFalse) + So(IsFiniteNumber(math.Inf(1)), ShouldBeFalse) + So(IsFiniteNumber(3.14), ShouldBeTrue) }) } diff --git a/metric_source/metric_data.go b/metric_source/metric_data.go index f3f020535..a70b49804 100644 --- a/metric_source/metric_data.go +++ b/metric_source/metric_data.go @@ -47,7 +47,9 @@ func (metricData *MetricData) GetTimestampValue(valueTimestamp int64) float64 { if valueTimestamp < metricData.StartTime { return math.NaN() } + valueIndex := int((valueTimestamp - metricData.StartTime) / metricData.StepTime) + if len(metricData.Values) <= valueIndex { return math.NaN() } diff --git a/plotting/curve.go b/plotting/curve.go index 186e35eed..c86531708 100644 --- a/plotting/curve.go +++ b/plotting/curve.go @@ -60,7 +60,7 @@ func describePlotCurves(metricData metricSource.MetricData) []plotCurve { for valInd := start; valInd < len(metricData.Values); valInd++ { pointValue := metricData.Values[valInd] - if moira.IsValidFloat64(pointValue) { + if moira.IsFiniteNumber(pointValue) { timeStampValue := moira.Int64ToTime(timeStamp) curves[curvesInd].timeStamps = append(curves[curvesInd].timeStamps, timeStampValue) curves[curvesInd].values = append(curves[curvesInd].values, pointValue) @@ -80,7 +80,7 @@ func resolveFirstPoint(metricData metricSource.MetricData) (int, int64) { start := 0 startTime := metricData.StartTime for _, metricVal := range metricData.Values { - if !moira.IsValidFloat64(metricVal) { + if !moira.IsFiniteNumber(metricVal) { start++ startTime += metricData.StepTime } else { diff --git a/plotting/limits.go b/plotting/limits.go index 5cc91f243..c1f975f25 100644 --- a/plotting/limits.go +++ b/plotting/limits.go @@ -33,7 +33,7 @@ func resolveLimits(metricsData []metricSource.MetricData) plotLimits { allTimes := make([]time.Time, 0) for _, metricData := range metricsData { for _, metricValue := range metricData.Values { - if moira.IsValidFloat64(metricValue) { + if moira.IsFiniteNumber(metricValue) { allValues = append(allValues, metricValue) } } From 27662e7519888065a63eb6b13073b9b2a701eb66 Mon Sep 17 00:00:00 2001 From: Iurii Pliner Date: Wed, 25 Oct 2023 07:59:33 +0100 Subject: [PATCH 40/46] fix: calculate RemoteAllowed based on remote/prom configs (#945) Currently, RemoteAllowed controls if triggers of both "remote" types (remote graphite and prometheus) could be created, but it is calculated only based on remote graphite config. In our case we only use prometheus triggers, so it is a little odd to fill in remote graphite config section to allow to create prometheus triggers. --- cmd/api/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 879a99d55..bbf76fffd 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -137,7 +137,7 @@ func main() { prometheusSource, ) - webConfigContent, err := applicationConfig.Web.getSettings(remoteConfig.Enabled) + webConfigContent, err := applicationConfig.Web.getSettings(remoteConfig.Enabled || prometheusConfig.Enabled) if err != nil { logger.Fatal(). Error(err). From 3faa4fc2fb1f428c37774bfa8b7d9b58862cd82a Mon Sep 17 00:00:00 2001 From: Xenia N Date: Wed, 25 Oct 2023 17:59:08 +0600 Subject: [PATCH 41/46] feat(notifier): add metric to dropped notifications count (#942) --- metrics/notifier.go | 20 +++++++++++++++++-- notifier/notifier.go | 43 ++++++++++++++++++++--------------------- notifier/registrator.go | 2 ++ 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/metrics/notifier.go b/metrics/notifier.go index a2b5889e9..bcf61daa3 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -1,6 +1,6 @@ package metrics -// NotifierMetrics is a collection of metrics used in notifier +// NotifierMetrics is a collection of metrics used in notifier. type NotifierMetrics struct { SubsMalformed Meter EventsReceived Meter @@ -10,11 +10,12 @@ type NotifierMetrics struct { SendingFailed Meter SendersOkMetrics MetersCollection SendersFailedMetrics MetersCollection + SendersDroppedNotifications MetersCollection PlotsBuildDurationMs Histogram PlotsEvaluateTriggerDurationMs Histogram } -// ConfigureNotifierMetrics is notifier metrics configurator +// ConfigureNotifierMetrics is notifier metrics configurator. func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics { return &NotifierMetrics{ SubsMalformed: registry.NewMeter("subs", "malformed"), @@ -25,7 +26,22 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics SendingFailed: registry.NewMeter("sending", "failed"), SendersOkMetrics: NewMetersCollection(registry), SendersFailedMetrics: NewMetersCollection(registry), + SendersDroppedNotifications: NewMetersCollection(registry), PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), } } + +// MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications. +func (metrics *NotifierMetrics) MarkSendersDroppedNotifications(contactType string) { + if metric, found := metrics.SendersDroppedNotifications.GetRegisteredMeter(contactType); found { + metric.Mark(1) + } +} + +// MarkSendersOkMetrics marks metrics as 1 by contactType when notifications were successfully sent. +func (metrics *NotifierMetrics) MarkSendersOkMetrics(contactType string) { + if metric, found := metrics.SendersOkMetrics.GetRegisteredMeter(contactType); found { + metric.Mark(1) + } +} diff --git a/notifier/notifier.go b/notifier/notifier.go index ea945412d..fd0752c6d 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -208,30 +208,29 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific err = sender.SendEvents(pkg.Events, pkg.Contact, pkg.Trigger, plots, pkg.Throttled) if err == nil { - if metric, found := notifier.metrics.SendersOkMetrics.GetRegisteredMeter(pkg.Contact.Type); found { - metric.Mark(1) - } - } else { - switch e := err.(type) { - case moira.SenderBrokenContactError: + notifier.metrics.MarkSendersOkMetrics(pkg.Contact.Type) + continue + } + switch e := err.(type) { + case moira.SenderBrokenContactError: + log.Warning(). + Error(e). + Msg("Cannot send to broken contact") + notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type) + default: + if pkg.FailCount > notifier.config.MaxFailAttemptToSendAvailable { + log.Error(). + Error(err). + Int("fail_count", pkg.FailCount). + Msg("Cannot send notification") + } else { log.Warning(). - Error(e). - Msg("Cannot send to broken contact") - - default: - if pkg.FailCount > notifier.config.MaxFailAttemptToSendAvailable { - log.Error(). - Error(err). - Int("fail_count", pkg.FailCount). - Msg("Cannot send notification") - } else { - log.Warning(). - Error(err). - Msg("Cannot send notification") - } - - notifier.resend(&pkg, err.Error()) + Error(err). + Msg("Cannot send notification") + notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type) } + + notifier.resend(&pkg, err.Error()) } } } diff --git a/notifier/registrator.go b/notifier/registrator.go index 841e064da..e31f6e7b1 100644 --- a/notifier/registrator.go +++ b/notifier/registrator.go @@ -112,6 +112,7 @@ func (notifier *StandardNotifier) RegisterSender(senderSettings map[string]inter default: senderIdent = senderType } + err := sender.Init(senderSettings, notifier.logger, notifier.config.Location, notifier.config.DateTimeFormat) if err != nil { return fmt.Errorf("failed to initialize sender [%s], err [%s]", senderIdent, err.Error()) @@ -120,6 +121,7 @@ func (notifier *StandardNotifier) RegisterSender(senderSettings map[string]inter notifier.senders[senderIdent] = eventsChannel notifier.metrics.SendersOkMetrics.RegisterMeter(senderIdent, getGraphiteSenderIdent(senderIdent), "sends_ok") notifier.metrics.SendersFailedMetrics.RegisterMeter(senderIdent, getGraphiteSenderIdent(senderIdent), "sends_failed") + notifier.metrics.SendersDroppedNotifications.RegisterMeter(senderIdent, getGraphiteSenderIdent(senderIdent), "notifications_dropped") notifier.runSenders(sender, eventsChannel) notifier.logger.Info(). String("sender_id", senderIdent). From 8a9a727e51eeffc3adc2e3c98b13c20a551020a4 Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:52:53 +0300 Subject: [PATCH 42/46] fix(api): fix remove contact in teams (#940) --- api/handler/contact.go | 4 +- api/handler/contact_test.go | 802 +++++++++++++++++++++++++++++++++ database/redis/reply/metric.go | 4 +- 3 files changed, 806 insertions(+), 4 deletions(-) create mode 100644 api/handler/contact_test.go diff --git a/api/handler/contact.go b/api/handler/contact.go index 9c4f338b5..084c5f3f5 100644 --- a/api/handler/contact.go +++ b/api/handler/contact.go @@ -100,7 +100,7 @@ func createNewContact(writer http.ResponseWriter, request *http.Request) { } userLogin := middleware.GetLogin(request) - if err := controller.CreateContact(database, contact, userLogin, ""); err != nil { + if err := controller.CreateContact(database, contact, userLogin, contact.TeamID); err != nil { render.Render(writer, request, err) //nolint return } @@ -176,7 +176,7 @@ func updateContact(writer http.ResponseWriter, request *http.Request) { // @router /contact/{contactID} [delete] func removeContact(writer http.ResponseWriter, request *http.Request) { contactData := request.Context().Value(contactKey).(moira.ContactData) - err := controller.RemoveContact(database, contactData.ID, contactData.User, "") + err := controller.RemoveContact(database, contactData.ID, contactData.User, contactData.Team) if err != nil { render.Render(writer, request, err) //nolint } diff --git a/api/handler/contact_test.go b/api/handler/contact_test.go new file mode 100644 index 000000000..03aae2e77 --- /dev/null +++ b/api/handler/contact_test.go @@ -0,0 +1,802 @@ +package handler + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/api" + "github.com/moira-alert/moira/api/dto" + "github.com/moira-alert/moira/api/middleware" + db "github.com/moira-alert/moira/database" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" +) + +const ( + ContactIDKey = "contactID" + ContactKey = "contact" + LoginKey = "login" + defaultContact = "testContact" + defaultLogin = "testLogin" +) + +func TestGetAllContacts(t *testing.T) { + Convey("Test get all contacts", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + testErr := errors.New("test error") + + Convey("Correctly returns all contacts", func() { + mockDb.EXPECT().GetAllContacts().Return([]*moira.ContactData{ + { + ID: defaultContact, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "moira", + Team: "", + }, + }, nil).Times(1) + database = mockDb + + expected := &dto.ContactList{ + List: []*moira.ContactData{ + { + ID: defaultContact, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "moira", + Team: "", + }, + }, + } + + testRequest := httptest.NewRequest(http.MethodGet, "/contact", nil) + + getAllContacts(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + actual := &dto.ContactList{} + err := json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Internal server error when trying to get all contacts", func() { + mockDb.EXPECT().GetAllContacts().Return(nil, testErr).Times(1) + database = mockDb + + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: testErr.Error(), + } + + testRequest := httptest.NewRequest(http.MethodGet, "/contact", nil) + + getAllContacts(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, _ := io.ReadAll(response.Body) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err := json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + }) +} + +func TestGetContactById(t *testing.T) { + Convey("Test get contact by id", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + testErr := errors.New("test error") + + Convey("Correctly returns contact by id", func() { + contactID := defaultContact + mockDb.EXPECT().GetContact(contactID).Return(moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "test", + Team: "", + }, nil).Times(1) + database = mockDb + + expected := &dto.Contact{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "test", + TeamID: "", + } + + testRequest := httptest.NewRequest(http.MethodGet, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactIDKey, contactID)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ID: contactID})) + + getContactById(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &dto.Contact{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Internal server error when trying to get a contact by id", func() { + contactID := defaultContact + mockDb.EXPECT().GetContact(contactID).Return(moira.ContactData{}, testErr).Times(1) + database = mockDb + + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: testErr.Error(), + } + + testRequest := httptest.NewRequest(http.MethodGet, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactIDKey, contactID)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ID: contactID})) + + getContactById(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + }) +} + +func TestCreateNewContact(t *testing.T) { + Convey("Test create new contact", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + login := defaultLogin + testErr := errors.New("test error") + + newContactDto := &dto.Contact{ + ID: defaultContact, + Type: "mail", + Value: "moira@skbkontur.ru", + User: login, + TeamID: "", + } + + Convey("Correctly create new contact with the given id", func() { + jsonContact, err := json.Marshal(newContactDto) + So(err, ShouldBeNil) + + mockDb.EXPECT().GetContact(defaultContact).Return(moira.ContactData{}, db.ErrNil).Times(1) + mockDb.EXPECT().SaveContact(&moira.ContactData{ + ID: newContactDto.ID, + Type: newContactDto.Type, + Value: newContactDto.Value, + User: newContactDto.User, + Team: newContactDto.TeamID, + }).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPut, "/contact", bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), LoginKey, login)) + testRequest.Header.Add("content-type", "application/json") + + createNewContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &dto.Contact{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, newContactDto) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Correctly create new contact without given id", func() { + newContactDto.ID = "" + defer func() { + newContactDto.ID = defaultContact + }() + + jsonContact, err := json.Marshal(newContactDto) + So(err, ShouldBeNil) + + mockDb.EXPECT().SaveContact(gomock.Any()).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPut, "/contact", bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), LoginKey, login)) + testRequest.Header.Add("content-type", "application/json") + + createNewContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &dto.Contact{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual.TeamID, ShouldEqual, newContactDto.TeamID) + So(actual.Type, ShouldEqual, newContactDto.Type) + So(actual.User, ShouldEqual, newContactDto.User) + So(actual.Value, ShouldEqual, newContactDto.Value) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Trying to create a new contact with the id of an existing contact", func() { + expected := &api.ErrorResponse{ + StatusText: "Invalid request", + ErrorText: "contact with this ID already exists", + } + jsonContact, err := json.Marshal(newContactDto) + So(err, ShouldBeNil) + + mockDb.EXPECT().GetContact(newContactDto.ID).Return(moira.ContactData{ + ID: newContactDto.ID, + Type: newContactDto.Type, + Value: newContactDto.Value, + User: newContactDto.User, + Team: newContactDto.TeamID, + }, nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPut, "/contact", bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), LoginKey, login)) + testRequest.Header.Add("content-type", "application/json") + + createNewContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("Internal error when trying to create a new contact with id", func() { + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: testErr.Error(), + } + jsonContact, err := json.Marshal(newContactDto) + So(err, ShouldBeNil) + + mockDb.EXPECT().GetContact(newContactDto.ID).Return(moira.ContactData{}, testErr).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPut, "/contact", bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), LoginKey, login)) + testRequest.Header.Add("content-type", "application/json") + + createNewContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + + Convey("Trying to create a contact when both userLogin and teamID specified", func() { + newContactDto.TeamID = "test" + defer func() { + newContactDto.TeamID = "" + }() + + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: "CreateContact: cannot create contact when both userLogin and teamID specified", + } + jsonContact, err := json.Marshal(newContactDto) + So(err, ShouldBeNil) + + testRequest := httptest.NewRequest(http.MethodPut, "/contact", bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), LoginKey, login)) + testRequest.Header.Add("content-type", "application/json") + + createNewContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + }) +} + +func TestUpdateContact(t *testing.T) { + Convey("Test update contact", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + testErr := errors.New("test error") + contactID := defaultContact + updatedContactDto := &dto.Contact{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "test", + TeamID: "", + } + + Convey("Successful contact updated", func() { + jsonContact, err := json.Marshal(updatedContactDto) + So(err, ShouldBeNil) + + mockDb.EXPECT().SaveContact(&moira.ContactData{ + ID: updatedContactDto.ID, + Type: updatedContactDto.Type, + Value: updatedContactDto.Value, + User: updatedContactDto.User, + }).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPut, "/contact/"+contactID, bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: updatedContactDto.Type, + Value: updatedContactDto.Value, + User: updatedContactDto.User, + })) + testRequest.Header.Add("content-type", "application/json") + + updateContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &dto.Contact{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, updatedContactDto) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Internal error when trying to update contact", func() { + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: testErr.Error(), + } + jsonContact, err := json.Marshal(updatedContactDto) + So(err, ShouldBeNil) + + mockDb.EXPECT().SaveContact(&moira.ContactData{ + ID: updatedContactDto.ID, + Type: updatedContactDto.Type, + Value: updatedContactDto.Value, + User: updatedContactDto.User, + }).Return(testErr).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPut, "/contact/"+contactID, bytes.NewBuffer(jsonContact)) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: updatedContactDto.Type, + Value: updatedContactDto.Value, + User: updatedContactDto.User, + })) + testRequest.Header.Add("content-type", "application/json") + + updateContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + }) +} + +func TestRemoveContact(t *testing.T) { + Convey("Test remove contact", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + testErr := errors.New("test error") + contactID := defaultContact + + Convey("Successful deletion of a contact without user, team id and subscriptions", func() { + mockDb.EXPECT().GetSubscriptions([]string{}).Return([]*moira.SubscriptionData{}, nil).Times(1) + mockDb.EXPECT().RemoveContact(contactID).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + actual := contentBytes + + So(actual, ShouldResemble, []byte{}) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Successful deletion of a contact without team id and subscriptions", func() { + mockDb.EXPECT().GetUserSubscriptionIDs("test").Return([]string{}, nil).Times(1) + mockDb.EXPECT().GetSubscriptions([]string{}).Return([]*moira.SubscriptionData{}, nil).Times(1) + mockDb.EXPECT().RemoveContact(contactID).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "test", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + actual := contentBytes + + So(actual, ShouldResemble, []byte{}) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Successful deletion of a contact without user id and subscriptions", func() { + mockDb.EXPECT().GetTeamSubscriptionIDs("test").Return([]string{}, nil).Times(1) + mockDb.EXPECT().GetSubscriptions([]string{}).Return([]*moira.SubscriptionData{}, nil).Times(1) + mockDb.EXPECT().RemoveContact(contactID).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + Team: "test", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + actual := contentBytes + + So(actual, ShouldResemble, []byte{}) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Successful deletion of a contact without subscriptions", func() { + mockDb.EXPECT().GetUserSubscriptionIDs("test").Return([]string{}, nil).Times(1) + mockDb.EXPECT().GetTeamSubscriptionIDs("test").Return([]string{}, nil).Times(1) + mockDb.EXPECT().GetSubscriptions([]string{}).Return([]*moira.SubscriptionData{}, nil).Times(1) + mockDb.EXPECT().RemoveContact(contactID).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "test", + Team: "test", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + actual := contentBytes + + So(actual, ShouldResemble, []byte{}) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Error when deleting a contact, the user has existing subscriptions", func() { + expected := &api.ErrorResponse{ + StatusText: "Invalid request", + ErrorText: "this contact is being used in following subscriptions: (tags: test)", + } + + mockDb.EXPECT().GetUserSubscriptionIDs("test").Return([]string{"test"}, nil).Times(1) + mockDb.EXPECT().GetSubscriptions([]string{"test"}).Return([]*moira.SubscriptionData{ + { + Contacts: []string{"testContact"}, + Tags: []string{"test"}, + }, + }, nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + User: "test", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("Error when deleting a contact, the team has existing subscriptions", func() { + expected := &api.ErrorResponse{ + StatusText: "Invalid request", + ErrorText: "this contact is being used in following subscriptions: (tags: test)", + } + + mockDb.EXPECT().GetTeamSubscriptionIDs("test").Return([]string{"test"}, nil).Times(1) + mockDb.EXPECT().GetSubscriptions([]string{"test"}).Return([]*moira.SubscriptionData{ + { + Contacts: []string{"testContact"}, + Tags: []string{"test"}, + }, + }, nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + Team: "test", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("Error when deleting a contact, the user and team has existing subscriptions", func() { + expected := &api.ErrorResponse{ + StatusText: "Invalid request", + ErrorText: "this contact is being used in following subscriptions: (tags: test1), (tags: test2)", + } + + mockDb.EXPECT().GetUserSubscriptionIDs("test1").Return([]string{"test1"}, nil).Times(1) + mockDb.EXPECT().GetTeamSubscriptionIDs("test2").Return([]string{"test2"}, nil).Times(1) + mockDb.EXPECT().GetSubscriptions([]string{"test1", "test2"}).Return([]*moira.SubscriptionData{ + { + Contacts: []string{"testContact"}, + Tags: []string{"test1"}, + }, + { + Contacts: []string{"testContact"}, + Tags: []string{"test2"}, + }, + }, nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + Team: "test2", + User: "test1", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("Internal server error when deleting of a contact without user, team id and subscriptions", func() { + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: testErr.Error(), + } + + mockDb.EXPECT().GetSubscriptions([]string{}).Return([]*moira.SubscriptionData{}, nil).Times(1) + mockDb.EXPECT().RemoveContact(contactID).Return(testErr).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodDelete, "/contact/"+contactID, nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactKey, moira.ContactData{ + ID: contactID, + Type: "mail", + Value: "moira@skbkontur.ru", + })) + testRequest.Header.Add("content-type", "application/json") + + removeContact(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + }) +} + +func TestSendTestContactNotification(t *testing.T) { + Convey("Test send test contact notification", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + responseWriter := httptest.NewRecorder() + mockDb := mock_moira_alert.NewMockDatabase(mockCtrl) + + testErr := errors.New("test error") + contactID := defaultContact + + Convey("Successful send test contact notification", func() { + mockDb.EXPECT().PushNotificationEvent(gomock.Any(), false).Return(nil).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPost, "/contact/"+contactID+"/test", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactIDKey, contactID)) + + sendTestContactNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + actual := contentBytes + + So(actual, ShouldResemble, []byte{}) + So(response.StatusCode, ShouldEqual, http.StatusOK) + }) + + Convey("Internal server error when sendin test contact notification", func() { + expected := &api.ErrorResponse{ + StatusText: "Internal Server Error", + ErrorText: testErr.Error(), + } + + mockDb.EXPECT().PushNotificationEvent(gomock.Any(), false).Return(testErr).Times(1) + database = mockDb + + testRequest := httptest.NewRequest(http.MethodPost, "/contact/"+contactID+"/test", nil) + testRequest = testRequest.WithContext(middleware.SetContextValueForTest(testRequest.Context(), ContactIDKey, contactID)) + + sendTestContactNotification(responseWriter, testRequest) + + response := responseWriter.Result() + defer response.Body.Close() + contentBytes, err := io.ReadAll(response.Body) + So(err, ShouldBeNil) + contents := string(contentBytes) + actual := &api.ErrorResponse{} + err = json.Unmarshal([]byte(contents), actual) + So(err, ShouldBeNil) + + So(actual, ShouldResemble, expected) + So(response.StatusCode, ShouldEqual, http.StatusInternalServerError) + }) + }) +} diff --git a/database/redis/reply/metric.go b/database/redis/reply/metric.go index af3a80c49..4a9896864 100644 --- a/database/redis/reply/metric.go +++ b/database/redis/reply/metric.go @@ -18,11 +18,11 @@ func MetricValues(values *redis.ZSliceCmd) ([]*moira.MetricValue, error) { } return nil, fmt.Errorf("failed to read metricValues: %s", err.Error()) } - metricsValues := make([]*moira.MetricValue, 0, len(resultByMetricArr)) //nolint + metricsValues := make([]*moira.MetricValue, 0, len(resultByMetricArr)) for i := 0; i < len(resultByMetricArr); i++ { val := resultByMetricArr[i].Member.(string) valuesArr := strings.Split(val, " ") - if len(valuesArr) != 2 { //nolint + if len(valuesArr) != 2 { return nil, fmt.Errorf("value format is not valid: %s", val) } timestamp, err := strconv.ParseInt(valuesArr[0], 10, 64) From 18568f02abc97ef15535fdb55c1e9104255d0553 Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Fri, 27 Oct 2023 10:51:05 +0300 Subject: [PATCH 43/46] fix config example (#948) --- api/handler/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/handler/config.go b/api/handler/config.go index 3cd075195..bfa6a84f1 100644 --- a/api/handler/config.go +++ b/api/handler/config.go @@ -5,9 +5,9 @@ import "net/http" type ContactExample struct { Type string `json:"type" example:"webhook kontur"` Label string `json:"label" example:"Webhook Kontur"` - Validation string `json:"validation" example:"^(http|https):\\/\\/.*(testkontur.ru|kontur.host|skbkontur.ru)(:[0-9]{2,5})?\\/"` - Placeholder string `json:"placeholder" example:"https://service.testkontur.ru/webhooks/moira"` - Help string `json:"help" example:"### Domains whitelist:\n - skbkontur.ru\n - testkontur.ru\n - kontur.host"` + Validation string `json:"validation" example:"^(http|https):\\/\\/.*(moira.ru)(:[0-9]{2,5})?\\/"` + Placeholder string `json:"placeholder" example:"https://moira.ru/webhooks/moira"` + Help string `json:"help" example:"### Domains whitelist:\n - moira.ru\n"` } type ConfigurationResponse struct { From 1a59ff3a6141da7f25ce5675ef6c636bf46c0fde Mon Sep 17 00:00:00 2001 From: Xenia N Date: Mon, 30 Oct 2023 16:48:58 +0600 Subject: [PATCH 44/46] feat(readme): add go-client link into README.md (#949) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 063be0f6e..1c298c460 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Moira 2.0 ![Build Status](https://github.com/moira-alert/moira/actions/workflows/test.yml/badge.svg?branch=master) [![codecov](https://codecov.io/gh/moira-alert/moira/branch/master/graph/badge.svg)](https://codecov.io/gh/moira-alert/moira) [![Documentation Status](https://readthedocs.org/projects/moira/badge/?version=latest)](http://moira.readthedocs.io/en/latest/?badge=latest) [![Telegram](https://img.shields.io/badge/telegram-join%20chat-3796cd.svg)](https://t.me/moira_alert) [![Go Report Card](https://goreportcard.com/badge/github.com/moira-alert/moira)](https://goreportcard.com/report/github.com/moira-alert/moira) -Moira is a real-time alerting tool, based on [Graphite](https://graphite.readthedocs.io) data. +Moira is a real-time alerting tool, based on [Graphite](https://graphite.readthedocs.io) or [Prometheus](https://prometheus.io/)/[VictoriaMetrics](https://victoriametrics.com/) metrics. ## Installation @@ -41,6 +41,7 @@ Code in this repository is the backend part of Moira monitoring application. * [doc](https://github.com/moira-alert/doc) is the documentation (hosted on [Read the Docs](https://moira.readthedocs.io)). * [moira-trigger-role](https://github.com/moira-alert/moira-trigger-role) is the Ansible role you can use to manage triggers. * [python-moira-client](https://github.com/moira-alert/python-moira-client) is the Python API client. +* [go-client](https://github.com/moira-alert/client-go) is the GO API client. ## Contact us From d7c2a231a73d0a3b0e3348a39405726d3d503401 Mon Sep 17 00:00:00 2001 From: almostinf <87192879+almostinf@users.noreply.github.com> Date: Mon, 30 Oct 2023 14:16:36 +0300 Subject: [PATCH 45/46] feature(metrics): add fetch notifications metric (#941) --- cmd/notifier/main.go | 1 + integration_tests/notifier/notifier_test.go | 1 + metrics/notifier.go | 11 ++++++++++- notifier/notifications/notifications.go | 17 ++++++++++++++++- notifier/notifications/notifications_test.go | 5 +++++ 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 8a75bb410..4408deadb 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -137,6 +137,7 @@ func main() { Logger: logger, Database: database, Notifier: sender, + Metrics: notifierMetrics, } fetchNotificationsWorker.Start() defer stopNotificationsFetcher(fetchNotificationsWorker) diff --git a/integration_tests/notifier/notifier_test.go b/integration_tests/notifier/notifier_test.go index a77e4dd47..51bc894a9 100644 --- a/integration_tests/notifier/notifier_test.go +++ b/integration_tests/notifier/notifier_test.go @@ -119,6 +119,7 @@ func TestNotifier(t *testing.T) { fetchNotificationsWorker := notifications.FetchNotificationsWorker{ Database: database, Logger: logger, + Metrics: notifierMetrics, Notifier: notifierInstance, } diff --git a/metrics/notifier.go b/metrics/notifier.go index bcf61daa3..9149eee2b 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -1,6 +1,8 @@ package metrics -// NotifierMetrics is a collection of metrics used in notifier. +import "time" + +// NotifierMetrics is a collection of metrics used in notifier type NotifierMetrics struct { SubsMalformed Meter EventsReceived Meter @@ -13,6 +15,7 @@ type NotifierMetrics struct { SendersDroppedNotifications MetersCollection PlotsBuildDurationMs Histogram PlotsEvaluateTriggerDurationMs Histogram + fetchNotificationsDurationMs Histogram } // ConfigureNotifierMetrics is notifier metrics configurator. @@ -29,9 +32,15 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics SendersDroppedNotifications: NewMetersCollection(registry), PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), + fetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), } } +// UpdateFetchNotificationsDurationMs - counts how much time has passed since fetchNotificationsStartTime in ms and updates the metric +func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime time.Time) { + metrics.fetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) +} + // MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications. func (metrics *NotifierMetrics) MarkSendersDroppedNotifications(contactType string) { if metric, found := metrics.SendersDroppedNotifications.GetRegisteredMeter(contactType); found { diff --git a/notifier/notifications/notifications.go b/notifier/notifications/notifications.go index 7c573aad1..e7cd5be9d 100644 --- a/notifier/notifications/notifications.go +++ b/notifier/notifications/notifications.go @@ -8,6 +8,7 @@ import ( "gopkg.in/tomb.v2" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira/notifier" ) @@ -18,9 +19,19 @@ type FetchNotificationsWorker struct { Logger moira.Logger Database moira.Database Notifier notifier.Notifier + Metrics *metrics.NotifierMetrics tomb tomb.Tomb } +func (worker *FetchNotificationsWorker) updateFetchNotificationsMetric(fetchNotificationsStartTime time.Time) { + if worker.Metrics == nil { + worker.Logger.Warning().Msg("Cannot update fetch notifications metric because Metrics is nil") + return + } + + worker.Metrics.UpdateFetchNotificationsDurationMs(fetchNotificationsStartTime) +} + // Start is a cycle that fetches scheduled notifications from database func (worker *FetchNotificationsWorker) Start() { worker.tomb.Go(func() error { @@ -63,14 +74,18 @@ func (worker *FetchNotificationsWorker) processScheduledNotifications() error { if err != nil { return notifierInBadStateError("can't get current notifier state") } + if state != moira.SelfStateOK { return notifierInBadStateError(fmt.Sprintf("notifier in a bad state: %v", state)) } - notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) + fetchNotificationsStartTime := time.Now() + notifications, err := worker.Database.FetchNotifications(time.Now().Unix(), worker.Notifier.GetReadBatchSize()) if err != nil { return err } + worker.updateFetchNotificationsMetric(fetchNotificationsStartTime) + notificationPackages := make(map[string]*notifier.NotificationPackage) for _, notification := range notifications { packageKey := fmt.Sprintf("%s:%s:%s", notification.Contact.Type, notification.Contact.Value, notification.Event.TriggerID) diff --git a/notifier/notifications/notifications_test.go b/notifier/notifications/notifications_test.go index d97aaa890..66f488001 100644 --- a/notifier/notifications/notifications_test.go +++ b/notifier/notifications/notifications_test.go @@ -6,6 +6,7 @@ import ( "github.com/golang/mock/gomock" logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" "github.com/moira-alert/moira" @@ -14,6 +15,8 @@ import ( notifier2 "github.com/moira-alert/moira/notifier" ) +var notifierMetrics = metrics.ConfigureNotifierMetrics(metrics.NewDummyRegistry(), "notifier") + func TestProcessScheduledEvent(t *testing.T) { subID2 := "subscriptionID-00000000000002" subID5 := "subscriptionID-00000000000005" @@ -60,6 +63,7 @@ func TestProcessScheduledEvent(t *testing.T) { Database: dataBase, Logger: logger, Notifier: notifier, + Metrics: notifierMetrics, } Convey("Two different notifications, should send two packages", t, func() { @@ -159,6 +163,7 @@ func TestGoRoutine(t *testing.T) { Database: dataBase, Logger: logger, Notifier: notifier, + Metrics: notifierMetrics, } shutdown := make(chan struct{}) From d0cc902f3d564b73e4ce449aa6950a48de8bdc1b Mon Sep 17 00:00:00 2001 From: Xenia N Date: Wed, 1 Nov 2023 13:47:48 +0600 Subject: [PATCH 46/46] feat(build): upgrade go to 1.20 (#951) --- Dockerfile.api | 2 +- Dockerfile.checker | 2 +- Dockerfile.cli | 2 +- Dockerfile.filter | 2 +- Dockerfile.notifier | 2 +- go.mod | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile.api b/Dockerfile.api index 1a1e228ae..aa443b958 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -1,4 +1,4 @@ -FROM golang:1.19 as builder +FROM golang:1.20 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.checker b/Dockerfile.checker index d6628c867..f1d31bd92 100644 --- a/Dockerfile.checker +++ b/Dockerfile.checker @@ -1,4 +1,4 @@ -FROM golang:1.19 as builder +FROM golang:1.20 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.cli b/Dockerfile.cli index 44bb49c5c..c501de6b1 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.19 as builder +FROM golang:1.20 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.filter b/Dockerfile.filter index 3b61a26d8..54282d4f4 100644 --- a/Dockerfile.filter +++ b/Dockerfile.filter @@ -1,4 +1,4 @@ -FROM golang:1.19 as builder +FROM golang:1.20 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/Dockerfile.notifier b/Dockerfile.notifier index 3c6dd5063..84f1e720b 100644 --- a/Dockerfile.notifier +++ b/Dockerfile.notifier @@ -1,4 +1,4 @@ -FROM golang:1.19 as builder +FROM golang:1.20 as builder COPY go.mod go.sum /go/src/github.com/moira-alert/moira/ WORKDIR /go/src/github.com/moira-alert/moira diff --git a/go.mod b/go.mod index 7ceaad8d9..37386c962 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/moira-alert/moira -go 1.19 +go 1.20 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible