From dec73bd056551701207991d2da8e743c6e6dbc46 Mon Sep 17 00:00:00 2001 From: Nathaniel Cook Date: Thu, 21 Apr 2016 17:09:35 -0600 Subject: [PATCH] add duration to alert data --- CHANGELOG.md | 1 + alert.go | 91 +++++-- integrations/batcher_test.go | 108 ++++++-- .../data/TestStream_AlertDuration.srpl | 36 +++ integrations/streamer_test.go | 240 +++++++++++++++--- pipeline/alert.go | 5 + 6 files changed, 397 insertions(+), 84 deletions(-) create mode 100644 integrations/data/TestStream_AlertDuration.srpl diff --git a/CHANGELOG.md b/CHANGELOG.md index e46ab2563..de20ba813 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ For example, let's say we want to store all data that triggered an alert in Infl - [#384](https://github.com/influxdata/kapacitor/issues/384): Add `elapsed` function to compute the time difference between subsequent points. - [#230](https://github.com/influxdata/kapacitor/issues/230): Alert.StateChangesOnly now accepts optional duration arg. An alert will be triggered for every interval even if the state has not changed. - [#426](https://github.com/influxdata/kapacitor/issues/426): Add `skip-format` query parameter to the `GET /task` endpoint so that returned TICKscript content is left unmodified from the user input. +- [#388](https://github.com/influxdata/kapacitor/issues/388): The duration of an alert is now tracked and exposed as part of the alert data as well as can be set as a field via `.durationField('duration')`. ### Bugfixes diff --git a/alert.go b/alert.go index dbe7121a3..4cf828245 100644 --- a/alert.go +++ b/alert.go @@ -80,12 +80,13 @@ func (l *AlertLevel) UnmarshalText(text []byte) error { } type AlertData struct { - ID string `json:"id"` - Message string `json:"message"` - Details string `json:"details"` - Time time.Time `json:"time"` - Level AlertLevel `json:"level"` - Data influxql.Result `json:"data"` + ID string `json:"id"` + Message string `json:"message"` + Details string `json:"details"` + Time time.Time `json:"time"` + Duration time.Duration `json:"duration"` + Level AlertLevel `json:"level"` + Data influxql.Result `json:"data"` // Info for custom templates info detailsInfo @@ -313,11 +314,12 @@ func (a *AlertNode) runAlert([]byte) error { Tags: p.Tags, Points: []models.BatchPoint{models.BatchPointFromPoint(p)}, } - ad, err := a.alertData(p.Name, p.Group, p.Tags, p.Fields, l, p.Time, batch) + state.triggered(p.Time) + duration := state.duration() + ad, err := a.alertData(p.Name, p.Group, p.Tags, p.Fields, l, p.Time, duration, batch) if err != nil { return err } - state.lastAlert = p.Time a.handleAlert(ad) if a.a.LevelTag != "" || a.a.IdTag != "" { p.Tags = p.Tags.Copy() @@ -328,7 +330,7 @@ func (a *AlertNode) runAlert([]byte) error { p.Tags[a.a.IdTag] = ad.ID } } - if a.a.LevelField != "" || a.a.IdField != "" { + if a.a.LevelField != "" || a.a.IdField != "" || a.a.DurationField != "" { p.Fields = p.Fields.Copy() if a.a.LevelField != "" { p.Fields[a.a.LevelField] = l.String() @@ -336,6 +338,9 @@ func (a *AlertNode) runAlert([]byte) error { if a.a.IdField != "" { p.Fields[a.a.IdField] = ad.ID } + if a.a.DurationField != "" { + p.Fields[a.a.DurationField] = int64(duration) + } } a.timer.Pause() for _, child := range a.outs { @@ -390,17 +395,19 @@ func (a *AlertNode) runAlert([]byte) error { (l != OKAlert && !((a.a.UseFlapping && state.flapping) || (a.a.IsStateChangesOnly && !state.changed && !state.expired))) { - ad, err := a.alertData(b.Name, b.Group, b.Tags, highestPoint.Fields, l, t, b) + state.triggered(t) + duration := state.duration() + ad, err := a.alertData(b.Name, b.Group, b.Tags, highestPoint.Fields, l, t, duration, b) if err != nil { return err } - state.lastAlert = t a.handleAlert(ad) // Update tags or fields for Level property if a.a.LevelTag != "" || a.a.LevelField != "" || a.a.IdTag != "" || - a.a.IdField != "" { + a.a.IdField != "" || + a.a.DurationField != "" { for i := range b.Points { if a.a.LevelTag != "" || a.a.IdTag != "" { b.Points[i].Tags = b.Points[i].Tags.Copy() @@ -411,7 +418,7 @@ func (a *AlertNode) runAlert([]byte) error { b.Points[i].Tags[a.a.IdTag] = ad.ID } } - if a.a.LevelField != "" || a.a.IdField != "" { + if a.a.LevelField != "" || a.a.IdField != "" || a.a.DurationField != "" { b.Points[i].Fields = b.Points[i].Fields.Copy() if a.a.LevelField != "" { b.Points[i].Fields[a.a.LevelField] = l.String() @@ -419,6 +426,9 @@ func (a *AlertNode) runAlert([]byte) error { if a.a.IdField != "" { b.Points[i].Fields[a.a.IdField] = ad.ID } + if a.a.DurationField != "" { + b.Points[i].Fields[a.a.DurationField] = int64(duration) + } } } if a.a.LevelTag != "" || a.a.IdTag != "" { @@ -485,6 +495,7 @@ func (a *AlertNode) alertData( fields models.Fields, level AlertLevel, t time.Time, + d time.Duration, b models.Batch, ) (*AlertData, error) { id, err := a.renderID(name, group, tags) @@ -496,32 +507,58 @@ func (a *AlertNode) alertData( return nil, err } ad := &AlertData{ - ID: id, - Message: msg, - Details: details, - Time: t, - Level: level, - Data: a.batchToResult(b), - info: info, + ID: id, + Message: msg, + Details: details, + Time: t, + Duration: d, + Level: level, + Data: a.batchToResult(b), + info: info, } return ad, nil } type alertState struct { - history []AlertLevel - idx int - flapping bool - changed bool - lastAlert time.Time - expired bool + history []AlertLevel + idx int + flapping bool + changed bool + // Time when first alert was triggered + firstTriggered time.Time + // Time when last alert was triggered. + // Note: Alerts are not triggered for every event. + lastTriggered time.Time + expired bool +} + +// Return the duration of the current alert state. +func (a *alertState) duration() time.Duration { + return a.lastTriggered.Sub(a.firstTriggered) +} + +// Record that the alert was triggered at time t. +func (a *alertState) triggered(t time.Time) { + a.lastTriggered = t + // Check if we are being triggered for first time since an OKAlert + // If so reset firstTriggered time + p := a.idx - 1 + if p == -1 { + p = len(a.history) - 1 + } + if a.history[p] == OKAlert { + a.firstTriggered = t + } } +// Record an event in the alert history. func (a *alertState) addEvent(level AlertLevel) { a.changed = a.history[a.idx] != level a.idx = (a.idx + 1) % len(a.history) a.history[a.idx] = level } +// Compute the percentage change in the alert history. func (a *alertState) percentChange() float64 { l := len(a.history) changes := 0.0 @@ -564,7 +601,7 @@ func (a *AlertNode) updateState(t time.Time, level AlertLevel, group models.Grou state.flapping = true } } - state.expired = !state.changed && a.a.StateChangesOnlyDuration != 0 && t.Sub(state.lastAlert) >= a.a.StateChangesOnlyDuration + state.expired = !state.changed && a.a.StateChangesOnlyDuration != 0 && t.Sub(state.lastTriggered) >= a.a.StateChangesOnlyDuration return state } diff --git a/integrations/batcher_test.go b/integrations/batcher_test.go index d0727276f..38ecf7f77 100644 --- a/integrations/batcher_test.go +++ b/integrations/batcher_test.go @@ -519,6 +519,64 @@ batch testBatcherWithOutput(t, "TestBatch_SimpleMR", script, 30*time.Second, er) } +func TestBatch_AlertDuration(t *testing.T) { + + var script = ` +batch + |query(''' + SELECT mean("value") + FROM "telegraf"."default".cpu_usage_idle + WHERE "host" = 'serverA' AND "cpu" != 'cpu-total' +''') + .period(10s) + .every(10s) + .groupBy(time(2s), 'cpu') + |alert() + .crit(lambda:"mean" > 95) + .durationField('duration') + |httpOut('TestBatch_SimpleMR') +` + + er := kapacitor.Result{ + Series: imodels.Rows{ + { + Name: "cpu_usage_idle", + Tags: map[string]string{"cpu": "cpu1"}, + Columns: []string{"time", "duration", "mean"}, + Values: [][]interface{}{ + { + time.Date(1971, 1, 1, 0, 0, 20, 0, time.UTC), + float64(14 * time.Second), + 96.49999999996908, + }, + { + time.Date(1971, 1, 1, 0, 0, 22, 0, time.UTC), + float64(14 * time.Second), + 93.46464646468584, + }, + { + time.Date(1971, 1, 1, 0, 0, 24, 0, time.UTC), + float64(14 * time.Second), + 95.00950095007724, + }, + { + time.Date(1971, 1, 1, 0, 0, 26, 0, time.UTC), + float64(14 * time.Second), + 92.99999999998636, + }, + { + time.Date(1971, 1, 1, 0, 0, 28, 0, time.UTC), + float64(14 * time.Second), + 90.99999999998545, + }, + }, + }, + }, + } + + testBatcherWithOutput(t, "TestBatch_SimpleMR", script, 30*time.Second, er) +} + func TestBatch_AlertStateChangesOnly(t *testing.T) { requestCount := int32(0) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -542,14 +600,15 @@ func TestBatch_AlertStateChangesOnly(t *testing.T) { } } else { expAd := kapacitor.AlertData{ - ID: "cpu_usage_idle:cpu=cpu-total,", - Message: "cpu_usage_idle:cpu=cpu-total, is OK", - Time: time.Date(1971, 1, 1, 0, 0, 38, 0, time.UTC), - Level: kapacitor.OKAlert, + ID: "cpu_usage_idle:cpu=cpu-total,", + Message: "cpu_usage_idle:cpu=cpu-total, is OK", + Time: time.Date(1971, 1, 1, 0, 0, 38, 0, time.UTC), + Duration: 38 * time.Second, + Level: kapacitor.OKAlert, } ad.Data = influxql.Result{} if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) + t.Errorf("unexpected alert data for request: %d %s", rc, msg) } } })) @@ -592,30 +651,31 @@ func TestBatch_AlertStateChangesOnlyExpired(t *testing.T) { if err != nil { t.Fatal(err) } + // We don't care about the data for this test + ad.Data = influxql.Result{} + var expAd kapacitor.AlertData atomic.AddInt32(&requestCount, 1) - if rc := atomic.LoadInt32(&requestCount); rc < 3 { - expAd := kapacitor.AlertData{ - ID: "cpu_usage_idle:cpu=cpu-total,", - Message: "cpu_usage_idle:cpu=cpu-total, is CRITICAL", - Time: time.Date(1971, 1, 1, 0, 0, int(rc-1)*20, 0, time.UTC), - Level: kapacitor.CritAlert, - } - ad.Data = influxql.Result{} - if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) + rc := atomic.LoadInt32(&requestCount) + if rc < 3 { + expAd = kapacitor.AlertData{ + ID: "cpu_usage_idle:cpu=cpu-total,", + Message: "cpu_usage_idle:cpu=cpu-total, is CRITICAL", + Time: time.Date(1971, 1, 1, 0, 0, int(rc-1)*20, 0, time.UTC), + Duration: time.Duration(rc-1) * 20 * time.Second, + Level: kapacitor.CritAlert, } } else { - expAd := kapacitor.AlertData{ - ID: "cpu_usage_idle:cpu=cpu-total,", - Message: "cpu_usage_idle:cpu=cpu-total, is OK", - Time: time.Date(1971, 1, 1, 0, 0, 38, 0, time.UTC), - Level: kapacitor.OKAlert, - } - ad.Data = influxql.Result{} - if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) + expAd = kapacitor.AlertData{ + ID: "cpu_usage_idle:cpu=cpu-total,", + Message: "cpu_usage_idle:cpu=cpu-total, is OK", + Time: time.Date(1971, 1, 1, 0, 0, 38, 0, time.UTC), + Duration: 38 * time.Second, + Level: kapacitor.OKAlert, } } + if eq, msg := compareAlertData(expAd, ad); !eq { + t.Errorf("unexpected alert data for request: %d %s", rc, msg) + } })) defer ts.Close() var script = ` diff --git a/integrations/data/TestStream_AlertDuration.srpl b/integrations/data/TestStream_AlertDuration.srpl new file mode 100644 index 000000000..1c28cd180 --- /dev/null +++ b/integrations/data/TestStream_AlertDuration.srpl @@ -0,0 +1,36 @@ +dbname +rpname +cpu,type=idle,host=serverA value=9 0000000001 +dbname +rpname +cpu,type=idle,host=serverA value=9 0000000002 +dbname +rpname +cpu,type=idle,host=serverA value=8 0000000003 +dbname +rpname +cpu,type=idle,host=serverA value=8 0000000004 +dbname +rpname +cpu,type=idle,host=serverA value=6 0000000005 +dbname +rpname +cpu,type=idle,host=serverA value=8 0000000006 +dbname +rpname +cpu,type=idle,host=serverA value=8 0000000007 +dbname +rpname +cpu,type=idle,host=serverA value=8 0000000008 +dbname +rpname +cpu,type=idle,host=serverA value=3 0000000009 +dbname +rpname +cpu,type=idle,host=serverA value=5 0000000010 +dbname +rpname +cpu,type=idle,host=serverA value=7 0000000011 +dbname +rpname +cpu,type=idle,host=serverA value=7 0000000012 diff --git a/integrations/streamer_test.go b/integrations/streamer_test.go index 41d23104e..fd62f612d 100644 --- a/integrations/streamer_test.go +++ b/integrations/streamer_test.go @@ -2171,6 +2171,7 @@ func TestStream_Alert(t *testing.T) { t.Fatal(err) } atomic.AddInt32(&requestCount, 1) + rc := atomic.LoadInt32(&requestCount) expAd := kapacitor.AlertData{ ID: "kapacitor/cpu/serverA", Message: "kapacitor/cpu/serverA is CRITICAL", @@ -2192,7 +2193,7 @@ func TestStream_Alert(t *testing.T) { }, } if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) + t.Errorf("unexpected alert data for request: %d %s", rc, msg) } })) defer ts.Close() @@ -2249,6 +2250,178 @@ stream } } +func TestStream_AlertDuration(t *testing.T) { + requestCount := int32(0) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ad := kapacitor.AlertData{} + dec := json.NewDecoder(r.Body) + err := dec.Decode(&ad) + if err != nil { + t.Fatal(err) + } + atomic.AddInt32(&requestCount, 1) + var expAd kapacitor.AlertData + rc := atomic.LoadInt32(&requestCount) + switch rc { + case 1: + expAd = kapacitor.AlertData{ + ID: "kapacitor/cpu/serverA", + Message: "kapacitor/cpu/serverA is CRITICAL", + Details: "details", + Time: time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), + Duration: 0, + Level: kapacitor.CritAlert, + Data: influxql.Result{ + Series: imodels.Rows{ + { + Name: "cpu", + Tags: map[string]string{"host": "serverA", "type": "idle"}, + Columns: []string{"time", "value"}, + Values: [][]interface{}{[]interface{}{ + time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), + 9.0, + }}, + }, + }, + }, + } + case 2: + expAd = kapacitor.AlertData{ + ID: "kapacitor/cpu/serverA", + Message: "kapacitor/cpu/serverA is WARNING", + Details: "details", + Time: time.Date(1971, 1, 1, 0, 0, 2, 0, time.UTC), + Duration: 2 * time.Second, + Level: kapacitor.WarnAlert, + Data: influxql.Result{ + Series: imodels.Rows{ + { + Name: "cpu", + Tags: map[string]string{"host": "serverA", "type": "idle"}, + Columns: []string{"time", "value"}, + Values: [][]interface{}{[]interface{}{ + time.Date(1971, 1, 1, 0, 0, 2, 0, time.UTC), + 8.0, + }}, + }, + }, + }, + } + case 3: + expAd = kapacitor.AlertData{ + ID: "kapacitor/cpu/serverA", + Message: "kapacitor/cpu/serverA is OK", + Details: "details", + Time: time.Date(1971, 1, 1, 0, 0, 4, 0, time.UTC), + Duration: 4 * time.Second, + Level: kapacitor.OKAlert, + Data: influxql.Result{ + Series: imodels.Rows{ + { + Name: "cpu", + Tags: map[string]string{"host": "serverA", "type": "idle"}, + Columns: []string{"time", "value"}, + Values: [][]interface{}{[]interface{}{ + time.Date(1971, 1, 1, 0, 0, 4, 0, time.UTC), + 6.0, + }}, + }, + }, + }, + } + case 4: + expAd = kapacitor.AlertData{ + ID: "kapacitor/cpu/serverA", + Message: "kapacitor/cpu/serverA is WARNING", + Details: "details", + Time: time.Date(1971, 1, 1, 0, 0, 5, 0, time.UTC), + Duration: 0, + Level: kapacitor.WarnAlert, + Data: influxql.Result{ + Series: imodels.Rows{ + { + Name: "cpu", + Tags: map[string]string{"host": "serverA", "type": "idle"}, + Columns: []string{"time", "value"}, + Values: [][]interface{}{[]interface{}{ + time.Date(1971, 1, 1, 0, 0, 5, 0, time.UTC), + 8.0, + }}, + }, + }, + }, + } + case 5: + expAd = kapacitor.AlertData{ + ID: "kapacitor/cpu/serverA", + Message: "kapacitor/cpu/serverA is OK", + Details: "details", + Time: time.Date(1971, 1, 1, 0, 0, 8, 0, time.UTC), + Duration: 3 * time.Second, + Level: kapacitor.OKAlert, + Data: influxql.Result{ + Series: imodels.Rows{ + { + Name: "cpu", + Tags: map[string]string{"host": "serverA", "type": "idle"}, + Columns: []string{"time", "value"}, + Values: [][]interface{}{[]interface{}{ + time.Date(1971, 1, 1, 0, 0, 8, 0, time.UTC), + 3.0, + }}, + }, + }, + }, + } + } + if eq, msg := compareAlertData(expAd, ad); !eq { + t.Errorf("unexpected alert data for request: %d %s", rc, msg) + } + })) + defer ts.Close() + + var script = ` +var warnThreshold = 7.0 +var critThreshold = 8.0 + +stream + |from() + .measurement('cpu') + .where(lambda: "host" == 'serverA') + .groupBy('host') + |alert() + .id('kapacitor/{{ .Name }}/{{ index .Tags "host" }}') + .details('details') + .durationField('duration') + .warn(lambda: "value" > warnThreshold) + .crit(lambda: "value" > critThreshold) + .stateChangesOnly() + .post('` + ts.URL + `') + |httpOut('TestStream_AlertDuration') +` + + er := kapacitor.Result{ + Series: imodels.Rows{ + { + Name: "cpu", + Tags: map[string]string{"host": "serverA", "type": "idle"}, + Columns: []string{"time", "duration", "value"}, + Values: [][]interface{}{[]interface{}{ + time.Date(1971, 1, 1, 0, 0, 8, 0, time.UTC), + float64(3 * time.Second), + 3.0, + }}, + }, + }, + } + + testStreamerWithOutput(t, "TestStream_AlertDuration", script, 13*time.Second, er, nil, false) + + if exp, rc := 5, int(atomic.LoadInt32(&requestCount)); rc != exp { + t.Errorf("got %v exp %v", rc, exp) + } +} + func TestStream_AlertSensu(t *testing.T) { requestCount := int32(0) addr, err := net.ResolveTCPAddr("tcp", "localhost:0") @@ -2982,9 +3155,11 @@ func TestStream_AlertSigma(t *testing.T) { if err != nil { t.Fatal(err) } + var expAd kapacitor.AlertData atomic.AddInt32(&requestCount, 1) + rc := atomic.LoadInt32(&requestCount) if rc := atomic.LoadInt32(&requestCount); rc == 1 { - expAd := kapacitor.AlertData{ + expAd = kapacitor.AlertData{ ID: "cpu:nil", Message: "cpu:nil is INFO", Details: "cpu:nil is INFO", @@ -3005,16 +3180,14 @@ func TestStream_AlertSigma(t *testing.T) { }, }, } - if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) - } } else { - expAd := kapacitor.AlertData{ - ID: "cpu:nil", - Message: "cpu:nil is OK", - Details: "cpu:nil is OK", - Time: time.Date(1971, 1, 1, 0, 0, 8, 0, time.UTC), - Level: kapacitor.OKAlert, + expAd = kapacitor.AlertData{ + ID: "cpu:nil", + Message: "cpu:nil is OK", + Details: "cpu:nil is OK", + Time: time.Date(1971, 1, 1, 0, 0, 8, 0, time.UTC), + Duration: time.Second, + Level: kapacitor.OKAlert, Data: influxql.Result{ Series: imodels.Rows{ { @@ -3030,9 +3203,9 @@ func TestStream_AlertSigma(t *testing.T) { }, }, } - if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) - } + } + if eq, msg := compareAlertData(expAd, ad); !eq { + t.Errorf("unexpected alert data for request: %d %s", rc, msg) } })) defer ts.Close() @@ -3149,30 +3322,31 @@ func TestStream_AlertStateChangesOnlyExpired(t *testing.T) { if err != nil { t.Fatal(err) } + //We don't care about the data for this test + ad.Data = influxql.Result{} + var expAd kapacitor.AlertData atomic.AddInt32(&requestCount, 1) - if rc := atomic.LoadInt32(&requestCount); rc < 6 { - expAd := kapacitor.AlertData{ - ID: "cpu:nil", - Message: "cpu:nil is CRITICAL", - Time: time.Date(1971, 1, 1, 0, 0, int(rc)*2-1, 0, time.UTC), - Level: kapacitor.CritAlert, - } - ad.Data = influxql.Result{} - if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) + rc := atomic.LoadInt32(&requestCount) + if rc < 6 { + expAd = kapacitor.AlertData{ + ID: "cpu:nil", + Message: "cpu:nil is CRITICAL", + Time: time.Date(1971, 1, 1, 0, 0, int(rc)*2-1, 0, time.UTC), + Duration: time.Duration(rc-1) * 2 * time.Second, + Level: kapacitor.CritAlert, } } else { - expAd := kapacitor.AlertData{ - ID: "cpu:nil", - Message: "cpu:nil is OK", - Time: time.Date(1971, 1, 1, 0, 0, 10, 0, time.UTC), - Level: kapacitor.OKAlert, - } - ad.Data = influxql.Result{} - if eq, msg := compareAlertData(expAd, ad); !eq { - t.Error(msg) + expAd = kapacitor.AlertData{ + ID: "cpu:nil", + Message: "cpu:nil is OK", + Time: time.Date(1971, 1, 1, 0, 0, 10, 0, time.UTC), + Duration: 9 * time.Second, + Level: kapacitor.OKAlert, } } + if eq, msg := compareAlertData(expAd, ad); !eq { + t.Errorf("unexpected alert data for request: %d %s", rc, msg) + } })) defer ts.Close() var script = ` diff --git a/pipeline/alert.go b/pipeline/alert.go index 568e2f079..8e2c9cf72 100644 --- a/pipeline/alert.go +++ b/pipeline/alert.go @@ -54,6 +54,7 @@ const defaultLogFileMode = 0600 // * Message -- the alert message, user defined. // * Details -- the alert details, user defined HTML content. // * Time -- the time the alert occurred. +// * Duration -- the duration of the alert in nanoseconds. // * Level -- one of OK, INFO, WARNING or CRITICAL. // * Data -- influxql.Result containing the data that triggered the alert. // @@ -215,6 +216,10 @@ type AlertNode struct { // Optional field key to add to the data, containing the alert level as a string. LevelField string + // Optional field key to add the alert duration to the data. + // The duration is always in units of nanoseconds. + DurationField string + // Optional tag key to use when tagging the data with the alert ID. IdTag string // Optional field key to add to the data, containing the alert ID as a string.