Feature/detect broken contacts (#615)

* Added notifier error to describe broken contact

* Catch broken contact for telegram sender

* fix

* Catch broken contacts for slack sender

* Added more telegram cases to broken contact

* lint fix

* review fixes

* fix telegram sender detecting broken contacts

* Added test: no send repeats for broken contacts

* review fixes

Co-authored-by: Andrey Kolkov <[email protected]>

errors.go

@@ -0,0 +1,17 @@
+package moira
+
+//SenderBrokenContactError means than sender has no way to send message to contact.
+// Maybe receive contact was deleted, blocked or archived.
+type SenderBrokenContactError struct {
+	SenderError error
+}
+
+func NewSenderBrokenContactError(senderError error) SenderBrokenContactError {
+	return SenderBrokenContactError{
+		SenderError: senderError,
+	}
+}
+
+func (e SenderBrokenContactError) Error() string {
+	return e.SenderError.Error()
+}

notifier/log_common.go

@@ -6,9 +6,13 @@ func getLogWithPackageContext(log *moira.Logger, pkg *NotificationPackage, confi
 	logger := (*log).Clone().
 		String(moira.LogFieldNameContactID, pkg.Contact.ID).
 		String(moira.LogFieldNameContactType, pkg.Contact.Type).
-		String(moira.LogFieldNameContactValue, pkg.Contact.Value).
-		String(moira.LogFieldNameTriggerID, pkg.Trigger.ID).
-		String(moira.LogFieldNameTriggerName, pkg.Trigger.Name)
+		String(moira.LogFieldNameContactValue, pkg.Contact.Value)
+	if pkg.Trigger.ID != "" { // note: test notification without trigger info
+		logger.
+			String(moira.LogFieldNameTriggerID, pkg.Trigger.ID).
+			String(moira.LogFieldNameTriggerName, pkg.Trigger.Name)
+	}
+
 	SetLogLevelByConfig(config.LogContactsToLevel, pkg.Contact.ID, &logger)
 	return logger
 }

notifier/notifier.go

@@ -192,8 +192,14 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific
 				metric.Mark(1)
 			}
 		} else {
-			log.Errorf("Cannot send notification: %s", err.Error())
-			notifier.resend(&pkg, err.Error())
+			switch e := err.(type) {
+			case moira.SenderBrokenContactError:
+				log.Errorf("Cannot send to broken contact: %s", e.Error())
+
+			default:
+				log.Errorf("Cannot send notification: %s", err.Error())
+				notifier.resend(&pkg, err.Error())
+			}
 		}
 	}
 }

notifier/notifier_test.go

@@ -117,6 +117,31 @@ func TestFailSendEvent(t *testing.T) {
 	time.Sleep(time.Second * 2)
 }
 
+func TestNoResendForSendToBrokenContact(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	configureNotifier(t)
+	defer afterTest()
+
+	var eventsData moira.NotificationEvents = []moira.NotificationEvent{event}
+
+	pkg := NotificationPackage{
+		Events: eventsData,
+		Contact: moira.ContactData{
+			Type: "test",
+		},
+	}
+	sender.EXPECT().SendEvents(eventsData, pkg.Contact, pkg.Trigger, plots, pkg.Throttled).
+		Return(moira.NewSenderBrokenContactError(fmt.Errorf("some sender reason")))
+
+	var wg sync.WaitGroup
+	notif.Send(&pkg, &wg)
+	wg.Wait()
+	time.Sleep(time.Second * 2)
+}
+
 func TestTimeout(t *testing.T) {
 	configureNotifier(t)
 	var wg sync.WaitGroup

senders/slack/slack.go

@@ -24,6 +24,11 @@ const (
 	testEmoji      = ":moira-state-test:"
 
 	messageMaxCharacters = 4000
+
+	//see errors https://api.slack.com/methods/chat.postMessage
+	ErrorTextChannelArchived = "is_archived"
+	ErrorTextChannelNotFound = "channel_not_found"
+	ErrorTextNotInChannel    = "not_in_channel"
 )
 
 var stateEmoji = map[moira.State]string{
@@ -185,7 +190,13 @@ func (sender *Sender) sendMessage(message string, contact string, triggerID stri
 	sender.logger.Debugf("Calling slack with message body %s", message)
 	channelID, threadTimestamp, err := sender.client.PostMessage(contact, slack.MsgOptionText(message, false), slack.MsgOptionPostMessageParameters(params))
 	if err != nil {
-		return channelID, threadTimestamp, fmt.Errorf("failed to send %s event message to slack [%s]: %s", triggerID, contact, err.Error())
+		errorText := err.Error()
+		if errorText == ErrorTextChannelArchived || errorText == ErrorTextNotInChannel ||
+			errorText == ErrorTextChannelNotFound {
+			return channelID, threadTimestamp, moira.NewSenderBrokenContactError(err)
+		}
+		return channelID, threadTimestamp, fmt.Errorf("failed to send %s event message to slack [%s]: %s",
+			triggerID, contact, errorText)
 	}
 	return channelID, threadTimestamp, nil
 }

senders/telegram/send.go

@@ -37,10 +37,10 @@ func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.
 	sender.logger.Debugf("Calling telegram api with chat_id %s and message body %s", contact.Value, message)
 	chat, err := sender.getChat(contact.Value)
 	if err != nil {
-		return err
+		return checkBrokenContactError(sender.logger, err)
 	}
 	if err := sender.talk(chat, message, plots, msgType); err != nil {
-		return fmt.Errorf("failed to send message to telegram contact %s: %s. ", contact.Value, err)
+		return checkBrokenContactError(sender.logger, err)
 	}
 	return nil
 }
@@ -116,17 +116,37 @@ func (sender *Sender) getChat(username string) (*telebot.Chat, error) {
 // talk processes one talk
 func (sender *Sender) talk(chat *telebot.Chat, message string, plots [][]byte, messageType messageType) error {
 	if messageType == Album {
+		sender.logger.Debug("talk as album")
 		return sender.sendAsAlbum(chat, plots, message)
 	}
+	sender.logger.Debug("talk as send message")
 	return sender.sendAsMessage(chat, message)
 }
 
 func (sender *Sender) sendAsMessage(chat *telebot.Chat, message string) error {
 	_, err := sender.bot.Send(chat, message)
 	if err != nil {
-		return fmt.Errorf("can't send event message [%s] to %v: %s", message, chat.ID, err.Error())
+		sender.logger.Debugf("can't send event message [%s] to %v: %s", message, chat.ID, err.Error())
 	}
-	return nil
+	return err
+}
+
+func checkBrokenContactError(logger moira.Logger, err error) error {
+	logger.Debug("Check broken contact")
+	if err == nil {
+		return nil
+	}
+	if e, ok := err.(*telebot.APIError); ok {
+		logger.Debug("It's telebot.APIError from talk(): code = %d, msg = %s, desc = %s", e.Code, e.Message, e.Description)
+		if e.Code == telebot.ErrUnauthorized.Code { // all forbid errors
+			return moira.NewSenderBrokenContactError(err)
+		}
+	}
+	if strings.HasPrefix(err.Error(), "failed to get username uuid") {
+		logger.Debug("It's error from getChat(): ", err)
+		return moira.NewSenderBrokenContactError(err)
+	}
+	return err
 }
 
 func prepareAlbum(plots [][]byte, caption string) telebot.Album {
@@ -144,9 +164,9 @@ func (sender *Sender) sendAsAlbum(chat *telebot.Chat, plots [][]byte, caption st
 
 	_, err := sender.bot.SendAlbum(chat, album)
 	if err != nil {
-		return fmt.Errorf("can't send event plots to %v: %s", chat.ID, err.Error())
+		sender.logger.Debugf("can't send event plots to %v: %s", chat.ID, err.Error())
 	}
-	return nil
+	return err
 }
 
 func getMessageType(plots [][]byte) messageType {