fix webhook sender retries

senders/webhook/webhook.go

@@ -83,12 +83,23 @@ func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.
 		} else {
 			serverResponse = string(responseBody)
 		}
-		return fmt.Errorf("invalid status code: %d, server response: %s", response.StatusCode, serverResponse)
+
+		statusCode := response.StatusCode
+		err = fmt.Errorf("invalid status code: %d, server response: %s", statusCode, serverResponse)
+		if isRetryableResponseCode(statusCode) {
+			return err
+		}
+		return moira.NewSenderBrokenContactError(err)
 	}
 
 	return nil
 }
 
+func isRetryableResponseCode(statusCode int) bool {
+	return statusCode == http.StatusServiceUnavailable ||
+		statusCode == http.StatusTooManyRequests
+}
+
 func isAllowedResponseCode(responseCode int) bool {
 	return (responseCode >= 200) && (responseCode <= 299)
 }

senders/webhook/webhook_test.go

@@ -33,16 +33,16 @@ func TestSender_SendEvents(t *testing.T) {
 					status, err := testRequestURL(r)
 					if err != nil {
 						w.WriteHeader(status)
-						w.Write([]byte(err.Error())) //nolint
+						_, _ = w.Write([]byte(err.Error())) //nolint
 					}
 					status, err = testRequestHeaders(r)
 					if err != nil {
 						w.WriteHeader(status)
-						w.Write([]byte(err.Error())) //nolint
+						_, _ = w.Write([]byte(err.Error())) //nolint
 					}
 					status, err = testRequestBody(r)
 					if err != nil {
-						w.Write([]byte(err.Error())) //nolint
+						_, _ = w.Write([]byte(err.Error())) //nolint
 						w.WriteHeader(status)
 					}
 					w.WriteHeader(status)
@@ -130,3 +130,57 @@ func getLastLine(longString string) (string, error) {
 	}
 	return lastLine, nil
 }
+
+func TestSender_SendsReturnError(t *testing.T) {
+	var statusCode int
+	ts := httptest.NewServer(
+		http.HandlerFunc(
+			func(w http.ResponseWriter, _ *http.Request) {
+				code := &statusCode
+				w.WriteHeader(*code)
+			},
+		),
+	)
+	defer ts.Close()
+
+	senderSettings := map[string]string{
+		"name":     "testWebhook",
+		"url":      fmt.Sprintf("%s/%s", ts.URL, moira.VariableTriggerID),
+		"user":     testUser,
+		"password": testPass,
+	}
+	sender := Sender{}
+	err := sender.Init(senderSettings, logger, time.UTC, "")
+
+	Convey("Sending returns error about broken contact", t, func() {
+		statusCode = http.StatusNotFound
+		err = sender.SendEvents(testEvents, testContact, testTrigger, testPlot, false)
+		So(err, ShouldBeError)
+		_, ok := err.(moira.SenderBrokenContactError)
+		So(ok, ShouldBeTrue)
+	})
+
+	Convey("Sending returns error about broken contact 2", t, func() {
+		statusCode = http.StatusBadGateway
+		err = sender.SendEvents(testEvents, testContact, testTrigger, testPlot, false)
+		So(err, ShouldBeError)
+		_, ok := err.(moira.SenderBrokenContactError)
+		So(ok, ShouldBeTrue)
+	})
+
+	Convey("Sending return simple error", t, func() {
+		statusCode = http.StatusServiceUnavailable
+		err = sender.SendEvents(testEvents, testContact, testTrigger, testPlot, false)
+		So(err, ShouldBeError)
+		_, ok := err.(moira.SenderBrokenContactError)
+		So(ok, ShouldBeFalse)
+	})
+
+	Convey("Sending return simple error 2", t, func() {
+		statusCode = http.StatusTooManyRequests
+		err = sender.SendEvents(testEvents, testContact, testTrigger, testPlot, false)
+		So(err, ShouldBeError)
+		_, ok := err.(moira.SenderBrokenContactError)
+		So(ok, ShouldBeFalse)
+	})
+}