[filebeat][auditd] Sync auditd module with changes done in package

CHANGELOG.next.asciidoc

@@ -548,6 +548,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add file integrity module ECS categorization fields. {pull}18012[18012]
 - Add `file.mime_type`, `file.extension`, and `file.drive_letter` for file integrity module. {pull}18012[18012]
 - Add ECS categorization info for auditd module {pull}18596[18596]
+- Add several improvements for auditd module for improved ECS field mapping {pull}22647[22647]
 
 *Filebeat*
 

filebeat/module/auditd/log/ingest/pipeline.yml

@@ -11,11 +11,12 @@ processors:
       AUDIT_NODE: "node=%{IPORHOST:auditd.log.node} "
       AUDIT_PREFIX: "^(?:%{AUDIT_NODE})?%{AUDIT_TYPE} msg=audit\\(%{NUMBER:auditd.log.epoch}:%{NUMBER:auditd.log.sequence}\\):(%{DATA})?"
       AUDIT_KEY_VALUES: "%{WORD}=%{GREEDYDATA}"
+      ANY: ".*"
     patterns:
     - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} old auid=%{NUMBER:auditd.log.old_auid}
       new auid=%{NUMBER:auditd.log.new_auid} old ses=%{NUMBER:auditd.log.old_ses}
       new ses=%{NUMBER:auditd.log.new_ses}"
-    - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=['\"]%{AUDIT_KEY_VALUES:auditd.log.sub_kv}['\"]"
+    - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=['\"]([^=]*\\s)?%{ANY:auditd.log.sub_kv}['\"]"
     - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv}"
     - "%{AUDIT_PREFIX}"
     - "%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:auditd.log.kv}"
@@ -48,10 +49,14 @@ processors:
 - remove:
     field: auditd.log.epoch
     ignore_failure: true
-- convert:
-    field: auditd.log.sequence
-    type: integer
-    ignore_missing: true
+- rename:
+    ignore_failure: true
+    field: auditd.log.old-auid
+    target_field: auditd.log.old_auid
+- rename:
+    ignore_failure: true
+    field: auditd.log.old-ses
+    target_field: auditd.log.old_ses
 - script:
     lang: painless
     source: |
@@ -136,6 +141,38 @@ processors:
     params:
       single_quote: "'"
       double_quote: "\""
+- convert:
+    field: auditd.log.sequence
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.lport
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.rport
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.entries
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.dst_prefixlen
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.ksize
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.size
+    type: long
+    ignore_missing: true
+- convert:
+    field: auditd.log.src_prefixlen
+    type: long
+    ignore_missing: true
 - set:
     field: event.kind
     value: event
@@ -221,6 +258,10 @@ processors:
     ignore_failure: true
     field: auditd.log.acct
     target_field: user.name
+- rename:
+    ignore_failure: true
+    field: auditd.log.user
+    target_field: user.name
 - rename:
     ignore_failure: true
     field: auditd.log.uid
@@ -301,13 +342,21 @@ processors:
     ignore_failure: true
     field: process.args
     separator: "\\s+"
+- rename:
+    ignore_failure: true
+    field: auditd.log.argc
+    target_field: process.args_count
 - script:
     if: "ctx?.process?.args != null"
     lang: painless
     source: >-
       if (ctx.process.args instanceof List) {
         ctx.process.args_count = ctx.process.args.length;
       }
+- convert:
+    ignore_missing: true
+    field: process.args_count
+    type: long
 - rename:
     ignore_failure: true
     field: auditd.log.exit
@@ -343,6 +392,11 @@ processors:
     ignore_failure: true
     field: auditd.log.src
     target_field: source.address
+- rename:
+    ignore_failure: true
+    field: auditd.log.addr
+    target_field: source.address
+    if: ctx?.source?.address == null
 - rename:
     ignore_failure: true
     field: auditd.log.dst

filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json

@@ -98,7 +98,6 @@
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 862,
-        "message": "",
         "process.executable": "/usr/lib/systemd/systemd-update-utmp",
         "process.name": "systemd-update-utmp",
         "process.pid": 1667,

filebeat/module/auditd/log/test/audit-rhel6.log-expected.json

@@ -105,11 +105,11 @@
     },
     {
         "@timestamp": "2017-03-14T19:23:02.529Z",
-        "auditd.log.dst_prefixlen": "22",
+        "auditd.log.dst_prefixlen": 22,
         "auditd.log.op": "SPD-add",
         "auditd.log.sequence": 19600354,
         "auditd.log.ses": "4294967295",
-        "auditd.log.src_prefixlen": "16",
+        "auditd.log.src_prefixlen": 16,
         "destination.address": "10.100.4.0",
         "event.action": "mac_ipsec_event",
         "event.dataset": "auditd.log",
@@ -180,13 +180,12 @@
     },
     {
         "@timestamp": "2017-03-16T04:02:40.070Z",
-        "auditd.log.addr": "96.241.146.97",
         "auditd.log.direction": "both",
         "auditd.log.kind": "session",
         "auditd.log.laddr": "107.170.139.210",
-        "auditd.log.lport": "50022",
+        "auditd.log.lport": 50022,
         "auditd.log.op": "destroy",
-        "auditd.log.rport": "58994",
+        "auditd.log.rport": 58994,
         "auditd.log.sequence": 19623788,
         "auditd.log.ses": "6793",
         "auditd.log.spid": "28282",
@@ -201,13 +200,24 @@
         "process.executable": "/usr/sbin/sshd",
         "process.pid": 28281,
         "service.type": "auditd",
+        "source.address": "96.241.146.97",
+        "source.as.number": 701,
+        "source.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business",
+        "source.geo.city_name": "Aldie",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.country_name": "United States",
+        "source.geo.location.lat": 38.9637,
+        "source.geo.location.lon": -77.6099,
+        "source.geo.region_iso_code": "US-VA",
+        "source.geo.region_name": "Virginia",
+        "source.ip": "96.241.146.97",
         "user.audit.id": "700",
         "user.id": "0",
         "user.saved.id": "74"
     },
     {
         "@timestamp": "2017-03-16T04:02:40.072Z",
-        "auditd.log.addr": "96.241.146.97",
         "auditd.log.op": "success",
         "auditd.log.sequence": 19623789,
         "auditd.log.ses": "6793",
@@ -224,6 +234,18 @@
         "process.executable": "/usr/sbin/sshd",
         "process.pid": 28281,
         "service.type": "auditd",
+        "source.address": "96.241.146.97",
+        "source.as.number": 701,
+        "source.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business",
+        "source.geo.city_name": "Aldie",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.country_name": "United States",
+        "source.geo.location.lat": 38.9637,
+        "source.geo.location.lon": -77.6099,
+        "source.geo.region_iso_code": "US-VA",
+        "source.geo.region_name": "Virginia",
+        "source.ip": "96.241.146.97",
         "user.audit.id": "700",
         "user.id": "0",
         "user.name": "admin",