From 2ab94934c3ccb7a113309f282ff3b4243d4ece7e Mon Sep 17 00:00:00 2001 From: tantra35 Date: Thu, 16 Jan 2020 16:21:59 +0300 Subject: [PATCH 1/7] Adding ability to automatically close incident in PagerDuty, when moira decided that alert is resolved(send OK status). This change conform incident lifecycle, that described in https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 --- senders/pagerduty/send.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/senders/pagerduty/send.go b/senders/pagerduty/send.go index f2ad0025c..50b0f847d 100644 --- a/senders/pagerduty/send.go +++ b/senders/pagerduty/send.go @@ -17,10 +17,18 @@ const summaryMaxChars = 1024 // SendEvents implements Sender interface Send func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.ContactData, trigger moira.TriggerData, plot []byte, throttled bool) error { event := sender.buildEvent(events, contact, trigger, plot, throttled) - _, err := pagerduty.ManageEvent(event) + event.Action = "trigger" + if events.GetSubjectState() == moira.StateOK { + event.Action = "resolve" + } + event.DedupKey = trigger.ID + sender.logger.Infof("to pagerDuty sended folow message: %s", spew.Sdump(event)) + + res, err := pagerduty.ManageEvent(event) if err != nil { return fmt.Errorf("failed to post the event to the pagerduty contact %s : %s. ", contact.Value, err) } + return nil } @@ -63,7 +71,6 @@ func (sender *Sender) buildEvent(events moira.NotificationEvents, contact moira. event := pagerduty.V2Event{ RoutingKey: contact.Value, - Action: "trigger", Payload: payload, } From 0a944420c9875e80837a613f56590f1c1a69ebc5 Mon Sep 17 00:00:00 2001 From: tantra35 Date: Thu, 16 Jan 2020 16:26:43 +0300 Subject: [PATCH 2/7] remove debuging code --- senders/pagerduty/send.go | 1 - 1 file changed, 1 deletion(-) diff --git a/senders/pagerduty/send.go b/senders/pagerduty/send.go index 50b0f847d..4e56c4580 100644 --- a/senders/pagerduty/send.go +++ b/senders/pagerduty/send.go @@ -22,7 +22,6 @@ func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira. event.Action = "resolve" } event.DedupKey = trigger.ID - sender.logger.Infof("to pagerDuty sended folow message: %s", spew.Sdump(event)) res, err := pagerduty.ManageEvent(event) if err != nil { From c1beff26875bd8cd2409ad1efdca0e3e11aa9db4 Mon Sep 17 00:00:00 2001 From: tantra35 Date: Fri, 17 Jan 2020 12:01:10 +0300 Subject: [PATCH 3/7] remove unused variable res --- senders/pagerduty/send.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/senders/pagerduty/send.go b/senders/pagerduty/send.go index 4e56c4580..53a14b365 100644 --- a/senders/pagerduty/send.go +++ b/senders/pagerduty/send.go @@ -23,7 +23,7 @@ func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira. } event.DedupKey = trigger.ID - res, err := pagerduty.ManageEvent(event) + _, err := pagerduty.ManageEvent(event) if err != nil { return fmt.Errorf("failed to post the event to the pagerduty contact %s : %s. ", contact.Value, err) } From 3a539dd6ce9091c7735e61d5a599c0f13780047c Mon Sep 17 00:00:00 2001 From: tantra35 Date: Fri, 17 Jan 2020 12:31:51 +0300 Subject: [PATCH 4/7] Modify code according comments in PR --- senders/pagerduty/send.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/senders/pagerduty/send.go b/senders/pagerduty/send.go index 53a14b365..8498d9fde 100644 --- a/senders/pagerduty/send.go +++ b/senders/pagerduty/send.go @@ -17,11 +17,6 @@ const summaryMaxChars = 1024 // SendEvents implements Sender interface Send func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.ContactData, trigger moira.TriggerData, plot []byte, throttled bool) error { event := sender.buildEvent(events, contact, trigger, plot, throttled) - event.Action = "trigger" - if events.GetSubjectState() == moira.StateOK { - event.Action = "resolve" - } - event.DedupKey = trigger.ID _, err := pagerduty.ManageEvent(event) if err != nil { @@ -68,9 +63,16 @@ func (sender *Sender) buildEvent(events moira.NotificationEvents, contact moira. Details: details, } + lAction := "trigger" + if events.GetSubjectState() == moira.StateOK { + lAction = "resolve" + } + event := pagerduty.V2Event{ RoutingKey: contact.Value, + Action: lAction, Payload: payload, + DedupKey: trigger.ID, } if len(plot) > 0 && sender.imageStoreConfigured { From 7c230dfc698d59d35ac5c19fbb9112e284e44148 Mon Sep 17 00:00:00 2001 From: tantra35 Date: Fri, 17 Jan 2020 12:45:42 +0300 Subject: [PATCH 5/7] remove unnecessary empty line --- senders/pagerduty/send.go | 1 - 1 file changed, 1 deletion(-) diff --git a/senders/pagerduty/send.go b/senders/pagerduty/send.go index 8498d9fde..d038c0d88 100644 --- a/senders/pagerduty/send.go +++ b/senders/pagerduty/send.go @@ -17,7 +17,6 @@ const summaryMaxChars = 1024 // SendEvents implements Sender interface Send func (sender *Sender) SendEvents(events moira.NotificationEvents, contact moira.ContactData, trigger moira.TriggerData, plot []byte, throttled bool) error { event := sender.buildEvent(events, contact, trigger, plot, throttled) - _, err := pagerduty.ManageEvent(event) if err != nil { return fmt.Errorf("failed to post the event to the pagerduty contact %s : %s. ", contact.Value, err) From a6bba5556b57680cf114214178aa4a3f7b88852f Mon Sep 17 00:00:00 2001 From: tantra35 Date: Thu, 23 Jan 2020 14:08:19 +0300 Subject: [PATCH 6/7] rename variable name lAction to action, so it possible to pass CI --- senders/pagerduty/send.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/senders/pagerduty/send.go b/senders/pagerduty/send.go index d038c0d88..9567541c5 100644 --- a/senders/pagerduty/send.go +++ b/senders/pagerduty/send.go @@ -62,14 +62,14 @@ func (sender *Sender) buildEvent(events moira.NotificationEvents, contact moira. Details: details, } - lAction := "trigger" + action := "trigger" if events.GetSubjectState() == moira.StateOK { - lAction = "resolve" + action = "resolve" } event := pagerduty.V2Event{ RoutingKey: contact.Value, - Action: lAction, + Action: action, Payload: payload, DedupKey: trigger.ID, } From 086e3888ca969b86311406ab9d70e653d471825a Mon Sep 17 00:00:00 2001 From: tantra35 Date: Tue, 18 Feb 2020 18:34:29 +0300 Subject: [PATCH 7/7] modify test for pagerduty sender --- senders/pagerduty/send_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/senders/pagerduty/send_test.go b/senders/pagerduty/send_test.go index 126fc3e16..594d5ef65 100644 --- a/senders/pagerduty/send_test.go +++ b/senders/pagerduty/send_test.go @@ -43,6 +43,7 @@ func TestBuildEvent(t *testing.T) { baseExpected := pagerduty.V2Event{ RoutingKey: contact.Value, Action: "trigger", + DedupKey: "TriggerID", Payload: &pagerduty.V2Payload{ Summary: "NODATA Trigger Name [tag1][tag2]", Severity: "warning",