Fix pagerduty2 integration

CHANGELOG.md

@@ -25,6 +25,7 @@ alert will auto-recover.
 - [#1795](https://github.com/influxdata/kapacitor/pull/1795): Support PagerDuty API v2
 - [#1842](https://github.com/influxdata/kapacitor/pull/1842): Add alert inhibitors that allow an alert to supress events from other matching alerts.
 - [#1776](https://github.com/influxdata/kapacitor/issues/1776): Fix bug where you could not delete a topic handler with the same name as its topic.
+- [#1905](https://github.com/influxdata/kapacitor/pull/1905): Adjust PagerDuty v2 service-test names and capture detailed error messages.
 
 ## v1.4.1 [2018-03-13]
 

alert.go

@@ -209,7 +209,7 @@ func newAlertNode(et *ExecutingTask, n *pipeline.AlertNode, d NodeDiagnostic) (a
 
 	for _, pd := range n.PagerDuty2Handlers {
 		c := pagerduty2.HandlerConfig{
-			ServiceKey: pd.ServiceKey,
+			RoutingKey: pd.ServiceKey,
 		}
 		h := et.tm.PagerDuty2Service.Handler(c, ctx...)
 		an.handlers = append(an.handlers, h)

integrations/streamer_test.go

@@ -9295,7 +9295,7 @@ stream
 		c := pagerduty2.NewConfig()
 		c.Enabled = true
 		c.URL = ts.URL
-		c.ServiceKey = "service_key"
+		c.RoutingKey = "service_key"
 		pd := pagerduty2.NewService(c, diagService.NewPagerDuty2Handler())
 		pd.HTTPDService = tm.HTTPDService
 		tm.PagerDuty2Service = pd

server/server_test.go

@@ -7442,7 +7442,7 @@ func TestServer_UpdateConfig(t *testing.T) {
 		{
 			section: "pagerduty2",
 			setDefaults: func(c *server.Config) {
-				c.PagerDuty2.ServiceKey = "secret"
+				c.PagerDuty2.RoutingKey = "secret"
 			},
 			expDefaultSection: client.ConfigSection{
 				Link: client.Link{Relation: client.Self, Href: "/kapacitor/v1/config/pagerduty2"},
@@ -7451,11 +7451,11 @@ func TestServer_UpdateConfig(t *testing.T) {
 					Options: map[string]interface{}{
 						"enabled":     false,
 						"global":      false,
-						"service-key": true,
+						"routing-key": true,
 						"url":         pagerduty2.DefaultPagerDuty2APIURL,
 					},
 					Redacted: []string{
-						"service-key",
+						"routing-key",
 					},
 				}},
 			},
@@ -7464,18 +7464,18 @@ func TestServer_UpdateConfig(t *testing.T) {
 				Options: map[string]interface{}{
 					"enabled":     false,
 					"global":      false,
-					"service-key": true,
+					"routing-key": true,
 					"url":         pagerduty2.DefaultPagerDuty2APIURL,
 				},
 				Redacted: []string{
-					"service-key",
+					"routing-key",
 				},
 			},
 			updates: []updateAction{
 				{
 					updateAction: client.ConfigUpdateAction{
 						Set: map[string]interface{}{
-							"service-key": "",
+							"routing-key": "",
 							"enabled":     true,
 						},
 					},
@@ -7486,11 +7486,11 @@ func TestServer_UpdateConfig(t *testing.T) {
 							Options: map[string]interface{}{
 								"enabled":     true,
 								"global":      false,
-								"service-key": false,
+								"routing-key": false,
 								"url":         pagerduty2.DefaultPagerDuty2APIURL,
 							},
 							Redacted: []string{
-								"service-key",
+								"routing-key",
 							},
 						}},
 					},
@@ -7499,11 +7499,11 @@ func TestServer_UpdateConfig(t *testing.T) {
 						Options: map[string]interface{}{
 							"enabled":     true,
 							"global":      false,
-							"service-key": false,
+							"routing-key": false,
 							"url":         pagerduty2.DefaultPagerDuty2APIURL,
 						},
 						Redacted: []string{
-							"service-key",
+							"routing-key",
 						},
 					},
 				},
@@ -8779,9 +8779,9 @@ func TestServer_ListServiceTests(t *testing.T) {
 				Link: client.Link{Relation: client.Self, Href: "/kapacitor/v1/service-tests/pagerduty2"},
 				Name: "pagerduty2",
 				Options: client.ServiceTestOptions{
-					"incident-key": "testIncidentKey",
-					"description":  "test pagerduty2 message",
-					"level":        "CRITICAL",
+					"alert_id":    "testAlertID",
+					"description": "test pagerduty2 message",
+					"level":       "CRITICAL",
 					"event_data": map[string]interface{}{
 						"Fields": map[string]interface{}{},
 						"Result": map[string]interface{}{
@@ -9855,7 +9855,7 @@ func TestServer_AlertHandlers(t *testing.T) {
 			handler: client.TopicHandler{
 				Kind: "pagerduty2",
 				Options: map[string]interface{}{
-					"service-key": "service_key",
+					"routing-key": "rkey",
 				},
 			},
 			setup: func(c *server.Config, ha *client.TopicHandler) (context.Context, error) {
@@ -9898,7 +9898,7 @@ func TestServer_AlertHandlers(t *testing.T) {
 							},
 							Timestamp: "1970-01-01T00:00:00.000000000Z",
 						},
-						RoutingKey: "service_key",
+						RoutingKey: "rkey",
 					},
 				}}
 

services/pagerduty2/config.go

@@ -15,8 +15,8 @@ type Config struct {
 	Enabled bool `toml:"enabled" override:"enabled"`
 	// The PagerDuty API URL, should not need to be changed.
 	URL string `toml:"url" override:"url"`
-	// The PagerDuty service key.
-	ServiceKey string `toml:"service-key" override:"service-key,redact"`
+	// The PagerDuty routing key, this is associated with an Event v2 API integration service.
+	RoutingKey string `toml:"routing-key" override:"routing-key,redact"`
 	// Whether every alert should automatically go to PagerDuty
 	Global bool `toml:"global" override:"global"`
 }

services/pagerduty2/service.go

@@ -155,12 +155,11 @@ func (s *Service) Global() bool {
 }
 
 type testOptions struct {
-	IncidentKey string `json:"incident-key"`
-	Description string `json:"description"`
-	//	Details     string          `json:"details"`
-	Level     alert.Level     `json:"level"`
-	Data      alert.EventData `json:"event_data"`
-	Timestamp time.Time       `json:"timestamp"`
+	AlertID     string          `json:"alert_id"`
+	Description string          `json:"description"`
+	Level       alert.Level     `json:"level"`
+	Data        alert.EventData `json:"event_data"`
+	Timestamp   time.Time       `json:"timestamp"`
 }
 
 // TestOptions returns optional values for the test harness
@@ -170,11 +169,10 @@ func (s *Service) TestOptions() interface{} {
 	t, _ := time.Parse(layout, str)
 
 	return &testOptions{
-		IncidentKey: "testIncidentKey",
+		AlertID:     "testAlertID",
 		Description: "test pagerduty2 message",
 		Level:       alert.Critical,
 		Timestamp:   t,
-		//		Details:     html.EscapeString(`{"Test": "test_value"}`),
 		Data: alert.EventData{
 			Name:   "testPagerDuty2",
 			Tags:   make(map[string]string),
@@ -192,8 +190,8 @@ func (s *Service) Test(options interface{}) error {
 	}
 	c := s.config()
 	return s.Alert(
-		c.ServiceKey,
-		o.IncidentKey,
+		c.RoutingKey,
+		o.AlertID,
 		o.Description,
 		o.Level,
 		o.Timestamp,
@@ -205,8 +203,8 @@ func (s *Service) Test(options interface{}) error {
 //
 // The req headers are now required with the API v2:
 // https://v2.developer.pagerduty.com/docs/migrating-to-api-v2
-func (s *Service) Alert(serviceKey, incidentKey, desc string, level alert.Level, timestamp time.Time, data alert.EventData) error {
-	url, post, err := s.preparePost(serviceKey, incidentKey, desc, level, timestamp, data)
+func (s *Service) Alert(serviceKey, alertID, desc string, level alert.Level, timestamp time.Time, data alert.EventData) error {
+	url, post, err := s.preparePost(serviceKey, alertID, desc, level, timestamp, data)
 	if err != nil {
 		return err
 	}
@@ -231,18 +229,20 @@ func (s *Service) Alert(serviceKey, incidentKey, desc string, level alert.Level,
 			return err
 		}
 		type response struct {
-			Message string `json:"message"`
+			Status  string   `json:"status"`
+			Message string   `json:"message"`
+			Errors  []string `json:"errors"`
 		}
 		r := &response{Message: fmt.Sprintf("failed to understand PagerDuty2 response. code: %d content: %s", resp.StatusCode, string(body))}
 		b := bytes.NewReader(body)
 		dec := json.NewDecoder(b)
 		dec.Decode(r)
-		return errors.New(r.Message)
+		return fmt.Errorf("Status: %s, Message: %s Errors: %v", r.Status, r.Message, r.Errors)
 	}
 	return nil
 }
 
-func (s *Service) sendResolve(c Config, serviceKey, incidentKey string) (string, io.Reader, error) {
+func (s *Service) sendResolve(c Config, serviceKey, alertID string) (string, io.Reader, error) {
 	// create a new AlertPayload for us to fire off
 	type Resolve struct {
 		RoutingKey  string `json:"routing_key"`
@@ -253,12 +253,12 @@ func (s *Service) sendResolve(c Config, serviceKey, incidentKey string) (string,
 	ap := Resolve{}
 
 	if serviceKey == "" {
-		ap.RoutingKey = c.ServiceKey
+		ap.RoutingKey = c.RoutingKey
 	} else {
 		ap.RoutingKey = serviceKey
 	}
 
-	ap.DedupKey = incidentKey
+	ap.DedupKey = alertID
 	ap.EventAction = "resolve"
 
 	var post bytes.Buffer
@@ -272,7 +272,7 @@ func (s *Service) sendResolve(c Config, serviceKey, incidentKey string) (string,
 }
 
 // preparePost is a helper method that sets up the payload for transmission to PagerDuty
-func (s *Service) preparePost(serviceKey, incidentKey, desc string, level alert.Level, timestamp time.Time, data alert.EventData) (string, io.Reader, error) {
+func (s *Service) preparePost(serviceKey, alertID, desc string, level alert.Level, timestamp time.Time, data alert.EventData) (string, io.Reader, error) {
 	c := s.config()
 	if !c.Enabled {
 		return "", nil, errors.New("service is not enabled")
@@ -290,7 +290,7 @@ func (s *Service) preparePost(serviceKey, incidentKey, desc string, level alert.
 		severity = "info"
 	default:
 		// default is a 'resolve' function
-		return s.sendResolve(c, serviceKey, incidentKey)
+		return s.sendResolve(c, serviceKey, alertID)
 	}
 
 	// create a new AlertPayload for us to fire off
@@ -299,33 +299,24 @@ func (s *Service) preparePost(serviceKey, incidentKey, desc string, level alert.
 	}
 
 	if serviceKey == "" {
-		ap.RoutingKey = c.ServiceKey
+		ap.RoutingKey = c.RoutingKey
 	} else {
 		ap.RoutingKey = serviceKey
 	}
 
 	ap.Client = "kapacitor"
 	ap.ClientURL = s.HTTPDService.URL()
-	ap.DedupKey = incidentKey
+	ap.DedupKey = alertID
 	ap.EventAction = eventType
 
 	ap.Payload.CustomDetails = make(map[string]interface{})
 	ap.Payload.CustomDetails["result"] = data.Result
 
-	// The API doesn't explicitly mention a requirement for nanosecond resolution but payloads seem to
-	// fail if we don't include it (even zeroes). This hack is not graceful, but adds a negligible
-	// nanosecond resolution to our timestamp
-	m := timestamp.Format("2006-01-02T15:04:05.000000000Z07:00")
-	//	m := timestamp.Format(time.RFC3339Nano)
-	//	if timestamp.Nanosecond() == 0 {
-	//		m = time.Unix(timestamp.Unix(), 1).In(timestamp.Location()).Format(time.RFC3339Nano)
-	//	}
-
 	ap.Payload.Class = data.TaskName
 	ap.Payload.Severity = severity
 	ap.Payload.Source = "unknown"
 	ap.Payload.Summary = desc
-	ap.Payload.Timestamp = m
+	ap.Payload.Timestamp = timestamp.Format("2006-01-02T15:04:05.000000000Z07:00")
 
 	if _, ok := data.Tags["host"]; ok {
 		ap.Payload.Source = data.Tags["host"]
@@ -344,9 +335,9 @@ func (s *Service) preparePost(serviceKey, incidentKey, desc string, level alert.
 
 // HandlerConfig defines the high-level struct required to connect to PagerDuty
 type HandlerConfig struct {
-	// The service key to use for the alert.
+	// The routing key to use for the alert.
 	// Defaults to the value in the configuration if empty.
-	ServiceKey string `mapstructure:"service-key"`
+	RoutingKey string `mapstructure:"routing-key"`
 }
 
 type handler struct {
@@ -367,7 +358,7 @@ func (s *Service) Handler(c HandlerConfig, ctx ...keyvalue.T) alert.Handler {
 // Handle is a bound method to the handler that processes a given alert
 func (h *handler) Handle(event alert.Event) {
 	if err := h.s.Alert(
-		h.c.ServiceKey,
+		h.c.RoutingKey,
 		event.State.ID,
 		event.State.Message,
 		event.State.Level,