Add ISO8601 as supported timestamp type (#25564) (#26238)

* Add ISO8601 as supported timestamp type Co-authored-by: Lee E. Hinman <[email protected]> (cherry picked from commit 7edb457) Co-authored-by: weslambert <[email protected]>
elastic · Jun 10, 2021 · 5854d35 · 5854d35
1 parent e9a3f18
commit 5854d35
Show file tree

Hide file tree

Showing 42 changed files with 115 additions and 0 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -581,6 +581,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `filestream` input GA. {pull}26127[26127]
 - http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
 - Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
+- Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.capture_loss.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.capture_loss.ts
 - set:

diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.connection.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.connection.ts
 - set:

diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log b/x-pack/filebeat/module/zeek/connection/test/connection-json.log
@@ -2,3 +2,4 @@
 {"ts":1547188416.857497,"uid":"CAcJw21BbVedgFnYH4","id.orig_h":"192.168.86.167","id.orig_p":38340,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]}
 {"ts":1547188417.857497,"uid":"CAcJw21BbVedgFnYH5","id.orig_h":"4.4.2.2","id.orig_p":38341,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":false,"local_resp":false,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]}
 {"ts":1551399000.57855,"uid":"Cc6NJ3GRlfjE44I3h","id.orig_h":"192.0.2.205","id.orig_p":3,"id.resp_h":"198.51.100.249","id.resp_p":3,"proto":"icmp","conn_state":"OTH","local_orig":false,"local_resp":false,"missed_bytes":0,"orig_pkts":1,"orig_ip_bytes":107,"resp_pkts":0,"resp_ip_bytes":0,"tunnel_parents":[]}
+{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.217.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json
@@ -218,5 +218,60 @@
         "zeek.connection.state": "OTH",
         "zeek.connection.state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).",
         "zeek.session_id": "Cc6NJ3GRlfjE44I3h"
+    },
+    {
+        "@timestamp": "2021-06-09T20:55:13.160Z",
+        "destination.address": "172.217.9.68",
+        "destination.as.number": 15169,
+        "destination.as.organization.name": "Google LLC",
+        "destination.bytes": 0,
+        "destination.geo.continent_name": "North America",
+        "destination.geo.country_iso_code": "US",
+        "destination.geo.country_name": "United States",
+        "destination.geo.location.lat": 37.751,
+        "destination.geo.location.lon": -97.822,
+        "destination.ip": "172.217.9.68",
+        "destination.packets": 0,
+        "destination.port": 80,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "zeek.connection",
+        "event.id": "C2KP1V3alRLoxl4JB9",
+        "event.kind": "event",
+        "event.module": "zeek",
+        "event.type": [
+            "connection",
+            "info"
+        ],
+        "fileset.name": "connection",
+        "input.type": "log",
+        "log.offset": 1488,
+        "network.bytes": 0,
+        "network.community_id": "1:DzqI9CYXjMSYV8VoSAHtMNfMIeU=",
+        "network.direction": "outbound",
+        "network.packets": 0,
+        "network.transport": "tcp",
+        "related.ip": [
+            "10.0.2.15",
+            "172.217.9.68"
+        ],
+        "service.type": "zeek",
+        "source.address": "10.0.2.15",
+        "source.bytes": 0,
+        "source.ip": "10.0.2.15",
+        "source.packets": 0,
+        "source.port": 46408,
+        "tags": [
+            "zeek.connection",
+            "local_orig"
+        ],
+        "zeek.connection.history": "C",
+        "zeek.connection.local_orig": true,
+        "zeek.connection.local_resp": false,
+        "zeek.connection.missed_bytes": 0,
+        "zeek.connection.state": "OTH",
+        "zeek.connection.state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).",
+        "zeek.session_id": "C2KP1V3alRLoxl4JB9"
     }
 ]
diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.dce_rpc.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.dce_rpc.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.dhcp.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.dhcp.ts
 - set:

diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.dnp3.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.dnp3.ts
 - set:

diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml
@@ -12,6 +12,7 @@ processors:
       field: zeek.dns.ts
       formats:
         - UNIX
+        - ISO8601
   - remove:
       field: zeek.dns.ts
 

diff --git a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.dpd.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.dpd.ts
 - geoip:

diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.files.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.files.ts
 - script:

diff --git a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.ftp.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.ftp.ts
 - dot_expander:

diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.http.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.http.ts
 - geoip:

diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml
@@ -11,6 +11,7 @@ processors:
       field: zeek.intel.ts
       formats:
         - UNIX
+        - ISO8601
   - remove:
       field: zeek.intel.ts
   # IP Geolocation Lookup

diff --git a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.irc.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.irc.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.kerberos.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.kerberos.ts
 - script:
@@ -20,12 +21,14 @@ processors:
     target_field: zeek.kerberos.valid.until
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.kerberos.valid?.until != null
 - date:
     field: zeek.kerberos.valid.from
     target_field: zeek.kerberos.valid.from
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.kerberos.valid?.from != null
 - set:
     field: event.outcome

diff --git a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.modbus.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.modbus.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.mysql.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.mysql.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.notice.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.notice.ts
 - geoip:

diff --git a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.ntlm.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.ntlm.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/ntp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ntp/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
       field: zeek.ntp.ts
       formats:
       - UNIX
+      - ISO8601
   - remove:
       field: zeek.ntp.ts
   # IP Geolocation Lookup
@@ -85,21 +86,25 @@ processors:
       target_field: zeek.ntp.ref_time
       formats:
       - UNIX
+      - ISO8601
   - date:
       field: zeek.ntp.org_time
       target_field: zeek.ntp.org_time
       formats:
       - UNIX
+      - ISO8601
   - date:
       field: zeek.ntp.rec_time
       target_field: zeek.ntp.rec_time
       formats:
       - UNIX
+      - ISO8601
   - date:
       field: zeek.ntp.xmt_time
       target_field: zeek.ntp.xmt_time
       formats:
       - UNIX
+      - ISO8601
   - convert:
       ignore_missing: true
       field: zeek.ntp.version

diff --git a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml
@@ -10,25 +10,29 @@ processors:
     field: zeek.ocsp.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.ocsp.ts
 - date:
     field: zeek.ocsp.revoke.date
     target_field: zeek.ocsp.revoke.date
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.ocsp.revoke?.date != null
 - date:
     field: zeek.ocsp.update.this
     target_field: zeek.ocsp.update.this
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.ocsp.update?.this != null
 - date:
     field: zeek.ocsp.update.next
     target_field: zeek.ocsp.update.next
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.ocsp.update?.next != null
 - append:
     field: related.hash

diff --git a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml
@@ -10,13 +10,15 @@ processors:
     field: zeek.pe.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.pe.ts
 - date:
     field: zeek.pe.compile_time
     target_field: zeek.pe.compile_time
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.pe.compile_time != null
 on_failure:
 - set:

diff --git a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.radius.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.radius.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.rdp.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.rdp.ts
 - convert:

diff --git a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.rfb.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.rfb.ts
 - append:

diff --git a/x-pack/filebeat/module/zeek/signature/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/signature/ingest/pipeline.yml
@@ -11,6 +11,7 @@ processors:
       field: zeek.signature.ts
       formats:
         - UNIX
+        - ISO8601
   - remove:
       field: zeek.signature.ts
   # IP Geolocation Lookup

diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.sip.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.sip.ts
 - grok:

diff --git a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.smb_cmd.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.smb_cmd.ts
 - remove:

diff --git a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml
@@ -10,6 +10,7 @@ processors:
     field: zeek.smb_files.ts
     formats:
     - UNIX
+    - ISO8601
 - remove:
     field: zeek.smb_files.ts
 - dot_expander:
@@ -29,6 +30,7 @@ processors:
     target_field: zeek.smb_files.times.accessed
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.smb_files.times?.accessed != null
 - set:
     field: file.accessed
@@ -39,6 +41,7 @@ processors:
     target_field: zeek.smb_files.times.changed
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.smb_files.times?.accessed != null
 - set:
     field: file.ctime
@@ -49,6 +52,7 @@ processors:
     target_field: zeek.smb_files.times.created
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.smb_files.times?.accessed != null
 - set:
     field: file.created
@@ -59,6 +63,7 @@ processors:
     target_field: zeek.smb_files.times.modified
     formats:
     - UNIX
+    - ISO8601
     if: ctx.zeek.smb_files.times?.accessed != null
 - set:
     field: file.mtime
-Original file line number
+Diff line change
@@ Expand Up / @@ -10,6 +10,7 @@ processors: @@
         field: zeek.capture_loss.ts
         formats:
         - UNIX
+        - ISO8601
     - remove:
         field: zeek.capture_loss.ts
     - set:
@@ Expand Down @@