Cherry-pick pull #11105 to 7.0

CHANGELOG.next.asciidoc

@@ -62,6 +62,8 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di
 
 - Fix errors in filebeat Zeek dashboard and README files. Add notice.log support. {pull}10916[10916]
 - Fix a bug when converting NetFlow fields to snake_case. {pull}10950[10950]
+- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test 
+  case. {issue}11004[11004] {pull}11105[11105]
 
 *Heartbeat*
 

x-pack/filebeat/module/zeek/connection/ingest/pipeline.json

@@ -10,7 +10,7 @@
     {
       "script": {
         "lang": "painless",
-        "source": "ctx.event.duration = (long)ctx.temp.duration * params.scale",
+        "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)",
         "params": {
           "scale": 1000000000
         },
@@ -74,5 +74,11 @@
         "target_field": "source.geo"
       }
     }
-  ]
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
 }

x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json

@@ -8,7 +8,7 @@
         "destination.port": 53,
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "zeek.connection",
-        "event.duration": 0.0,
+        "event.duration": 76967000,
         "event.id": "CAcJw21BbVedgFnYH3",
         "event.module": "zeek",
         "fileset.name": "connection",
@@ -48,7 +48,7 @@
         "destination.port": 53,
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "zeek.connection",
-        "event.duration": 0.0,
+        "event.duration": 76967000,
         "event.id": "CAcJw21BbVedgFnYH4",
         "event.module": "zeek",
         "fileset.name": "connection",
@@ -87,7 +87,7 @@
         "destination.port": 53,
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "zeek.connection",
-        "event.duration": 0.0,
+        "event.duration": 76967000,
         "event.id": "CAcJw21BbVedgFnYH5",
         "event.module": "zeek",
         "fileset.name": "connection",

x-pack/filebeat/module/zeek/dns/ingest/pipeline.json

@@ -38,5 +38,11 @@
         "target_field": "source.geo"
       }
     }
-  ]
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
 }

x-pack/filebeat/module/zeek/files/ingest/pipeline.json

@@ -36,5 +36,11 @@
         "if": "ctx.zeek.session_id != null"
       }
     }
-  ]
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
 }

x-pack/filebeat/module/zeek/http/ingest/pipeline.json

@@ -50,5 +50,11 @@
         "ignore_missing": true
       }
     }
-  ]
-}
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
+}

x-pack/filebeat/module/zeek/notice/config/notice.yml

@@ -19,7 +19,7 @@ processors:
         - from: "zeek.notice.src"
           to: "source.address"
 
-        - from: "zeek.notice.dest"
+        - from: "zeek.notice.dst"
           to: "destination.address"
 
         - from: "zeek.notice.uid"

x-pack/filebeat/module/zeek/notice/ingest/pipeline.json

@@ -42,5 +42,11 @@
         "ignore_missing": true
       }
     }
-  ]
-}
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
+}

x-pack/filebeat/module/zeek/notice/test/notice-json.log

@@ -1 +1,2 @@
 {"ts":1320435875.879278,"note":"SSH::Password_Guessing","msg":"172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).","sub":"Sampled servers:  172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136","src":"172.16.238.1","peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false}
+{"ts":1551393388.426472,"note":"Scan::Port_Scan","msg":"8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s","sub":"remote","src":"8.42.77.171","dst":"207.154.238.205","peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false}

x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json

@@ -19,5 +19,42 @@
         "zeek.notice.peer_descr": "bro",
         "zeek.notice.sub": "Sampled servers:  172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136",
         "zeek.notice.suppress_for": 3600
+    },
+    {
+        "@timestamp": 1551393388000,
+        "destination.address": "207.154.238.205",
+        "destination.geo.city_name": "New York",
+        "destination.geo.continent_name": "North America",
+        "destination.geo.country_iso_code": "US",
+        "destination.geo.location.lat": 40.7214,
+        "destination.geo.location.lon": -74.0052,
+        "destination.geo.region_iso_code": "US-NY",
+        "destination.geo.region_name": "New York",
+        "destination.ip": "207.154.238.205",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "zeek.notice",
+        "event.module": "zeek",
+        "fileset.name": "notice",
+        "input.type": "log",
+        "log.offset": 357,
+        "service.type": "zeek",
+        "source.address": "8.42.77.171",
+        "source.geo.city_name": "Longmont",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.location.lat": 40.1791,
+        "source.geo.location.lon": -105.1986,
+        "source.geo.region_iso_code": "US-CO",
+        "source.geo.region_name": "Colorado",
+        "source.ip": "8.42.77.171",
+        "tags": [
+            "zeek.notice"
+        ],
+        "zeek.notice.dropped": false,
+        "zeek.notice.msg": "8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s",
+        "zeek.notice.note": "Scan::Port_Scan",
+        "zeek.notice.peer_descr": "bro",
+        "zeek.notice.sub": "remote",
+        "zeek.notice.suppress_for": 3600
     }
 ]

x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json

@@ -38,5 +38,11 @@
         "target_field": "source.geo"
       }
     }
-  ]
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
 }