diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index aeaffb86999f..8b11f9a9bbfb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -400,6 +400,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Log to stderr when running using reference kubernetes manifests. {pull}17443[174443] - Fix syscall kprobe arguments for 32-bit systems in socket module. {pull}17500[17500] - Add ECS categorization info for auditd module {pull}18596[18596] +- Add several improvements for auditd module for improved ECS field mapping {pull}22647[22647] *Filebeat* diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml index 4f6d8ba2d0df..caec9d979f23 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.yml +++ b/filebeat/module/auditd/log/ingest/pipeline.yml @@ -11,11 +11,12 @@ processors: AUDIT_NODE: "node=%{IPORHOST:auditd.log.node} " AUDIT_PREFIX: "^(?:%{AUDIT_NODE})?%{AUDIT_TYPE} msg=audit\\(%{NUMBER:auditd.log.epoch}:%{NUMBER:auditd.log.sequence}\\):(%{DATA})?" AUDIT_KEY_VALUES: "%{WORD}=%{GREEDYDATA}" + ANY: ".*" patterns: - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} old auid=%{NUMBER:auditd.log.old_auid} new auid=%{NUMBER:auditd.log.new_auid} old ses=%{NUMBER:auditd.log.old_ses} new ses=%{NUMBER:auditd.log.new_ses}" - - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=['\"]%{AUDIT_KEY_VALUES:auditd.log.sub_kv}['\"]" + - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=['\"]([^=]*\\s)?%{ANY:auditd.log.sub_kv}['\"]" - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv}" - "%{AUDIT_PREFIX}" - "%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:auditd.log.kv}" @@ -48,10 +49,14 @@ processors: - remove: field: auditd.log.epoch ignore_failure: true -- convert: - field: auditd.log.sequence - type: integer - ignore_missing: true +- rename: + ignore_failure: true + field: auditd.log.old-auid + target_field: auditd.log.old_auid +- rename: + ignore_failure: true + field: auditd.log.old-ses + target_field: auditd.log.old_ses - script: lang: painless source: | @@ -136,6 +141,38 @@ processors: params: single_quote: "'" double_quote: "\"" +- convert: + field: auditd.log.sequence + type: long + ignore_missing: true +- convert: + field: auditd.log.lport + type: long + ignore_missing: true +- convert: + field: auditd.log.rport + type: long + ignore_missing: true +- convert: + field: auditd.log.entries + type: long + ignore_missing: true +- convert: + field: auditd.log.dst_prefixlen + type: long + ignore_missing: true +- convert: + field: auditd.log.ksize + type: long + ignore_missing: true +- convert: + field: auditd.log.size + type: long + ignore_missing: true +- convert: + field: auditd.log.src_prefixlen + type: long + ignore_missing: true - set: field: event.kind value: event @@ -221,6 +258,10 @@ processors: ignore_failure: true field: auditd.log.acct target_field: user.name +- rename: + ignore_failure: true + field: auditd.log.user + target_field: user.name - rename: ignore_failure: true field: auditd.log.uid @@ -301,6 +342,10 @@ processors: ignore_failure: true field: process.args separator: "\\s+" +- rename: + ignore_failure: true + field: auditd.log.argc + target_field: process.args_count - script: if: "ctx?.process?.args != null" lang: painless @@ -308,6 +353,10 @@ processors: if (ctx.process.args instanceof List) { ctx.process.args_count = ctx.process.args.length; } +- convert: + ignore_missing: true + field: process.args_count + type: long - rename: ignore_failure: true field: auditd.log.exit @@ -343,6 +392,11 @@ processors: ignore_failure: true field: auditd.log.src target_field: source.address +- rename: + ignore_failure: true + field: auditd.log.addr + target_field: source.address + if: ctx?.source?.address == null - rename: ignore_failure: true field: auditd.log.dst diff --git a/filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json b/filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json index 6001a762f9f3..b47e9806d529 100644 --- a/filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json +++ b/filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json @@ -98,7 +98,6 @@ "fileset.name": "log", "input.type": "log", "log.offset": 862, - "message": "", "process.executable": "/usr/lib/systemd/systemd-update-utmp", "process.name": "systemd-update-utmp", "process.pid": 1667, diff --git a/filebeat/module/auditd/log/test/audit-rhel6.log-expected.json b/filebeat/module/auditd/log/test/audit-rhel6.log-expected.json index b2532651d2b0..09140e1ea567 100644 --- a/filebeat/module/auditd/log/test/audit-rhel6.log-expected.json +++ b/filebeat/module/auditd/log/test/audit-rhel6.log-expected.json @@ -105,11 +105,11 @@ }, { "@timestamp": "2017-03-14T19:23:02.529Z", - "auditd.log.dst_prefixlen": "22", + "auditd.log.dst_prefixlen": 22, "auditd.log.op": "SPD-add", "auditd.log.sequence": 19600354, "auditd.log.ses": "4294967295", - "auditd.log.src_prefixlen": "16", + "auditd.log.src_prefixlen": 16, "destination.address": "10.100.4.0", "event.action": "mac_ipsec_event", "event.dataset": "auditd.log", @@ -180,13 +180,12 @@ }, { "@timestamp": "2017-03-16T04:02:40.070Z", - "auditd.log.addr": "96.241.146.97", "auditd.log.direction": "both", "auditd.log.kind": "session", "auditd.log.laddr": "107.170.139.210", - "auditd.log.lport": "50022", + "auditd.log.lport": 50022, "auditd.log.op": "destroy", - "auditd.log.rport": "58994", + "auditd.log.rport": 58994, "auditd.log.sequence": 19623788, "auditd.log.ses": "6793", "auditd.log.spid": "28282", @@ -201,13 +200,24 @@ "process.executable": "/usr/sbin/sshd", "process.pid": 28281, "service.type": "auditd", + "source.address": "96.241.146.97", + "source.as.number": 701, + "source.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business", + "source.geo.city_name": "Aldie", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 38.9637, + "source.geo.location.lon": -77.6099, + "source.geo.region_iso_code": "US-VA", + "source.geo.region_name": "Virginia", + "source.ip": "96.241.146.97", "user.audit.id": "700", "user.id": "0", "user.saved.id": "74" }, { "@timestamp": "2017-03-16T04:02:40.072Z", - "auditd.log.addr": "96.241.146.97", "auditd.log.op": "success", "auditd.log.sequence": 19623789, "auditd.log.ses": "6793", @@ -224,6 +234,18 @@ "process.executable": "/usr/sbin/sshd", "process.pid": 28281, "service.type": "auditd", + "source.address": "96.241.146.97", + "source.as.number": 701, + "source.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business", + "source.geo.city_name": "Aldie", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 38.9637, + "source.geo.location.lon": -77.6099, + "source.geo.region_iso_code": "US-VA", + "source.geo.region_name": "Virginia", + "source.ip": "96.241.146.97", "user.audit.id": "700", "user.id": "0", "user.name": "admin", diff --git a/filebeat/module/auditd/log/test/audit-rhel7.log-expected.json b/filebeat/module/auditd/log/test/audit-rhel7.log-expected.json index b25dde0881ba..311c49b661a9 100644 --- a/filebeat/module/auditd/log/test/audit-rhel7.log-expected.json +++ b/filebeat/module/auditd/log/test/audit-rhel7.log-expected.json @@ -54,7 +54,6 @@ "fileset.name": "log", "input.type": "log", "log.offset": 419, - "message": "", "process.executable": "/usr/lib/systemd/systemd-update-utmp", "process.name": "systemd-update-utmp", "process.pid": 273, @@ -253,7 +252,7 @@ }, { "@timestamp": "2016-12-07T02:16:24.827Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "2", "auditd.log.sequence": 17, "auditd.log.table": "filter", @@ -305,7 +304,7 @@ }, { "@timestamp": "2016-12-07T02:16:24.858Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "2", "auditd.log.sequence": 18, "auditd.log.table": "raw", @@ -357,7 +356,7 @@ }, { "@timestamp": "2016-12-07T02:16:24.870Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "2", "auditd.log.sequence": 19, "auditd.log.table": "security", @@ -409,7 +408,7 @@ }, { "@timestamp": "2016-12-07T02:16:24.877Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "2", "auditd.log.sequence": 20, "auditd.log.table": "mangle", @@ -461,7 +460,7 @@ }, { "@timestamp": "2016-12-07T02:16:24.931Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "2", "auditd.log.sequence": 21, "auditd.log.table": "nat", @@ -618,7 +617,7 @@ }, { "@timestamp": "2016-12-07T02:16:24.982Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "10", "auditd.log.sequence": 27, "auditd.log.table": "filter", @@ -754,7 +753,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.069Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "10", "auditd.log.sequence": 32, "auditd.log.table": "raw", @@ -827,7 +826,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.099Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "10", "auditd.log.sequence": 34, "auditd.log.table": "security", @@ -879,7 +878,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.128Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "10", "auditd.log.sequence": 35, "auditd.log.table": "mangle", @@ -1099,7 +1098,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.191Z", - "auditd.log.entries": "0", + "auditd.log.entries": 0, "auditd.log.family": "10", "auditd.log.sequence": 44, "auditd.log.table": "nat", @@ -1172,7 +1171,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.528Z", - "auditd.log.entries": "5", + "auditd.log.entries": 5, "auditd.log.family": "2", "auditd.log.sequence": 46, "auditd.log.table": "nat", @@ -1224,7 +1223,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.532Z", - "auditd.log.entries": "5", + "auditd.log.entries": 5, "auditd.log.family": "2", "auditd.log.sequence": 47, "auditd.log.table": "nat", @@ -1276,7 +1275,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.534Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 48, "auditd.log.table": "mangle", @@ -1328,7 +1327,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.537Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 49, "auditd.log.table": "mangle", @@ -1380,7 +1379,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.538Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "2", "auditd.log.sequence": 50, "auditd.log.table": "security", @@ -1432,7 +1431,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.542Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "2", "auditd.log.sequence": 51, "auditd.log.table": "security", @@ -1484,7 +1483,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.543Z", - "auditd.log.entries": "3", + "auditd.log.entries": 3, "auditd.log.family": "2", "auditd.log.sequence": 52, "auditd.log.table": "raw", @@ -1536,7 +1535,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.546Z", - "auditd.log.entries": "3", + "auditd.log.entries": 3, "auditd.log.family": "2", "auditd.log.sequence": 53, "auditd.log.table": "raw", @@ -1588,7 +1587,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.548Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "2", "auditd.log.sequence": 54, "auditd.log.table": "filter", @@ -1640,7 +1639,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.552Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "2", "auditd.log.sequence": 55, "auditd.log.table": "filter", @@ -1692,7 +1691,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.553Z", - "auditd.log.entries": "5", + "auditd.log.entries": 5, "auditd.log.family": "10", "auditd.log.sequence": 56, "auditd.log.table": "nat", @@ -1744,7 +1743,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.556Z", - "auditd.log.entries": "5", + "auditd.log.entries": 5, "auditd.log.family": "10", "auditd.log.sequence": 57, "auditd.log.table": "nat", @@ -1796,7 +1795,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.557Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "10", "auditd.log.sequence": 58, "auditd.log.table": "mangle", @@ -1848,7 +1847,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.560Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "10", "auditd.log.sequence": 59, "auditd.log.table": "mangle", @@ -1900,7 +1899,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.562Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "10", "auditd.log.sequence": 60, "auditd.log.table": "security", @@ -1952,7 +1951,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.566Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "10", "auditd.log.sequence": 61, "auditd.log.table": "security", @@ -2004,7 +2003,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.569Z", - "auditd.log.entries": "3", + "auditd.log.entries": 3, "auditd.log.family": "10", "auditd.log.sequence": 62, "auditd.log.table": "raw", @@ -2056,7 +2055,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.573Z", - "auditd.log.entries": "3", + "auditd.log.entries": 3, "auditd.log.family": "10", "auditd.log.sequence": 63, "auditd.log.table": "raw", @@ -2108,7 +2107,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.575Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "10", "auditd.log.sequence": 64, "auditd.log.table": "filter", @@ -2160,7 +2159,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.578Z", - "auditd.log.entries": "4", + "auditd.log.entries": 4, "auditd.log.family": "10", "auditd.log.sequence": 65, "auditd.log.table": "filter", @@ -2212,7 +2211,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.580Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 66, "auditd.log.table": "mangle", @@ -2264,7 +2263,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.582Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 67, "auditd.log.table": "mangle", @@ -2316,7 +2315,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.583Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 68, "auditd.log.table": "mangle", @@ -2368,7 +2367,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.585Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 69, "auditd.log.table": "mangle", @@ -2420,7 +2419,7 @@ }, { "@timestamp": "2016-12-07T02:16:25.587Z", - "auditd.log.entries": "6", + "auditd.log.entries": 6, "auditd.log.family": "2", "auditd.log.sequence": 70, "auditd.log.table": "mangle", diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json index f122becadda6..58ff1fee37d4 100644 --- a/filebeat/module/auditd/log/test/test.log-expected.json +++ b/filebeat/module/auditd/log/test/test.log-expected.json @@ -1,11 +1,11 @@ [ { "@timestamp": "2017-01-31T20:17:14.891Z", - "auditd.log.dst_prefixlen": "16", + "auditd.log.dst_prefixlen": 16, "auditd.log.op": "SPD-delete", "auditd.log.sequence": 18877201, "auditd.log.ses": "4294967295", - "auditd.log.src_prefixlen": "24", + "auditd.log.src_prefixlen": 24, "destination.address": "192.168.0.0", "event.action": "mac_ipsec_event", "event.dataset": "auditd.log", @@ -82,15 +82,14 @@ }, { "@timestamp": "2016-12-07T02:17:21.515Z", - "auditd.log.addr": "96.241.146.97", "auditd.log.cipher": "chacha20-poly1305@openssh.com", "auditd.log.direction": "from-server", - "auditd.log.ksize": "512", + "auditd.log.ksize": 512, "auditd.log.laddr": "10.142.0.2", - "auditd.log.lport": "22", + "auditd.log.lport": 22, "auditd.log.op": "start", "auditd.log.pfs": "curve25519-sha256@libssh.org", - "auditd.log.rport": "63927", + "auditd.log.rport": 63927, "auditd.log.sequence": 406, "auditd.log.ses": "4294967295", "auditd.log.spid": "1299", @@ -106,6 +105,18 @@ "process.executable": "/usr/sbin/sshd", "process.pid": 1298, "service.type": "auditd", + "source.address": "96.241.146.97", + "source.as.number": 701, + "source.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business", + "source.geo.city_name": "Aldie", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 38.9637, + "source.geo.location.lon": -77.6099, + "source.geo.region_iso_code": "US-VA", + "source.geo.region_name": "Virginia", + "source.ip": "96.241.146.97", "user.audit.id": "4294967295", "user.id": "0", "user.saved.id": "74" @@ -198,7 +209,6 @@ "fileset.name": "log", "input.type": "log", "log.offset": 2196, - "message": "", "process.executable": "/usr/lib/systemd/systemd-update-utmp", "process.name": "systemd-update-utmp", "process.pid": 678, @@ -221,7 +231,6 @@ "fileset.name": "log", "input.type": "log", "log.offset": 2438, - "message": "", "process.executable": "/usr/lib/systemd/systemd-update-utmp", "process.name": "systemd-update-utmp", "process.pid": 4440, @@ -232,7 +241,6 @@ { "@timestamp": "2020-02-10T21:59:44.206Z", "auditd.log.a0": "top", - "auditd.log.argc": "1", "auditd.log.sequence": 579393, "event.action": "execve", "event.dataset": "auditd.log", @@ -241,6 +249,7 @@ "fileset.name": "log", "input.type": "log", "log.offset": 2688, + "process.args_count": 1, "service.type": "auditd" }, { @@ -304,7 +313,6 @@ "auditd.log.sequence": 145, "auditd.log.ses": "3", "auditd.log.subj": "system_u:system_r:container_runtime_t:s0", - "auditd.log.user": "root", "event.action": "virt_control", "event.category": "host", "event.dataset": "auditd.log", @@ -319,7 +327,8 @@ "process.pid": 1431, "service.type": "auditd", "user.audit.id": "100", - "user.id": "0" + "user.id": "0", + "user.name": "root" }, { "@timestamp": "2016-12-16T15:45:43.572Z",