diff --git a/packages/rapid7_insightvm/changelog.yml b/packages/rapid7_insightvm/changelog.yml index 4bfc5cd9d6b..fdfb46691c2 100644 --- a/packages/rapid7_insightvm/changelog.yml +++ b/packages/rapid7_insightvm/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.0" + changes: + - description: Canonicalize `host.name` to lower case and map subdomain to `host.hostname`. + type: enhancement + link: https://github.com/elastic/integrations/pull/9665 - version: "1.9.0" changes: - description: Set sensitive values as secret. diff --git a/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log b/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log index aaf992de3a8..f0dd0a77b5c 100644 --- a/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log +++ b/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log @@ -1,3 +1,3 @@ {"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":0,"exploits":0,"id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-199","ip":"10.1.0.128","last_assessed_for_vulnerabilities":"2020-03-20T19:19:42.611Z","last_scan_end":"2020-03-20T19:19:42.611Z","last_scan_start":"2020-03-20T19:18:13.611Z","malware_kits":0,"moderate_vulnerabilities":2,"os_architecture":"x86_64","os_description":"CentOS Linux 2.6.18","os_family":"Linux","os_name":"Linux","os_system_name":"CentOS Linux","os_type":"General","os_vendor":"CentOS","os_version":"2.6.18","risk_score":0,"severe_vulnerabilities":0,"tags":[{"name":"lab","type":"SITE"}],"total_vulnerabilities":2,"new":[],"remediated":[]} -{"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":1,"exploits":9,"host_name":"host.domain.com","id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198","ip":"10.4.24.164","last_scan_end":"2020-03-20T19:12:39.766Z","last_scan_start":"2020-03-20T19:05:06.766Z","malware_kits":0,"moderate_vulnerabilities":11,"os_architecture":"","os_description":"Ubuntu Linux 12.04","os_family":"Linux","os_name":"Linux","os_system_name":"Ubuntu Linux","os_type":"","os_vendor":"Ubuntu","os_version":"12.04","risk_score":12251.76171875,"severe_vulnerabilities":16,"tags":[{"name":"all_assets2","type":"CUSTOM"},{"name":"all_assets","type":"CUSTOM"},{"name":"Linux","type":"CUSTOM"},{"name":"docker hosts","type":"SITE"},{"name":"lab","type":"SITE"}],"total_vulnerabilities":28,"new":[],"remediated":[],"unique_identifiers":{"id":"4421d73dfe04f594df731e6bcd8156a","source":"R7 Agent"}} +{"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":1,"exploits":9,"host_name":"HOST.domain.com","id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198","ip":"10.4.24.164","last_scan_end":"2020-03-20T19:12:39.766Z","last_scan_start":"2020-03-20T19:05:06.766Z","malware_kits":0,"moderate_vulnerabilities":11,"os_architecture":"","os_description":"Ubuntu Linux 12.04","os_family":"Linux","os_name":"Linux","os_system_name":"Ubuntu Linux","os_type":"","os_vendor":"Ubuntu","os_version":"12.04","risk_score":12251.76171875,"severe_vulnerabilities":16,"tags":[{"name":"all_assets2","type":"CUSTOM"},{"name":"all_assets","type":"CUSTOM"},{"name":"Linux","type":"CUSTOM"},{"name":"docker hosts","type":"SITE"},{"name":"lab","type":"SITE"}],"total_vulnerabilities":28,"new":[],"remediated":[],"unique_identifiers":{"id":"4421d73dfe04f594df731e6bcd8156a","source":"R7 Agent"}} {"data":[],"metadata":{"number":0,"size":0,"totalResources":2195,"totalPages":2195,"cursor":null},"links":[{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2","rel":"first"},{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2","rel":"self"},{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=1097&size=2","rel":"last"}]} diff --git a/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json b/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json index f638edce24b..72495111434 100644 --- a/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json +++ b/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json @@ -85,12 +85,13 @@ "host" ], "kind": "state", - "original": "{\"assessed_for_policies\":false,\"assessed_for_vulnerabilities\":true,\"critical_vulnerabilities\":1,\"exploits\":9,\"host_name\":\"host.domain.com\",\"id\":\"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198\",\"ip\":\"10.4.24.164\",\"last_scan_end\":\"2020-03-20T19:12:39.766Z\",\"last_scan_start\":\"2020-03-20T19:05:06.766Z\",\"malware_kits\":0,\"moderate_vulnerabilities\":11,\"os_architecture\":\"\",\"os_description\":\"Ubuntu Linux 12.04\",\"os_family\":\"Linux\",\"os_name\":\"Linux\",\"os_system_name\":\"Ubuntu Linux\",\"os_type\":\"\",\"os_vendor\":\"Ubuntu\",\"os_version\":\"12.04\",\"risk_score\":12251.76171875,\"severe_vulnerabilities\":16,\"tags\":[{\"name\":\"all_assets2\",\"type\":\"CUSTOM\"},{\"name\":\"all_assets\",\"type\":\"CUSTOM\"},{\"name\":\"Linux\",\"type\":\"CUSTOM\"},{\"name\":\"docker hosts\",\"type\":\"SITE\"},{\"name\":\"lab\",\"type\":\"SITE\"}],\"total_vulnerabilities\":28,\"new\":[],\"remediated\":[],\"unique_identifiers\":{\"id\":\"4421d73dfe04f594df731e6bcd8156a\",\"source\":\"R7 Agent\"}}", + "original": "{\"assessed_for_policies\":false,\"assessed_for_vulnerabilities\":true,\"critical_vulnerabilities\":1,\"exploits\":9,\"host_name\":\"HOST.domain.com\",\"id\":\"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198\",\"ip\":\"10.4.24.164\",\"last_scan_end\":\"2020-03-20T19:12:39.766Z\",\"last_scan_start\":\"2020-03-20T19:05:06.766Z\",\"malware_kits\":0,\"moderate_vulnerabilities\":11,\"os_architecture\":\"\",\"os_description\":\"Ubuntu Linux 12.04\",\"os_family\":\"Linux\",\"os_name\":\"Linux\",\"os_system_name\":\"Ubuntu Linux\",\"os_type\":\"\",\"os_vendor\":\"Ubuntu\",\"os_version\":\"12.04\",\"risk_score\":12251.76171875,\"severe_vulnerabilities\":16,\"tags\":[{\"name\":\"all_assets2\",\"type\":\"CUSTOM\"},{\"name\":\"all_assets\",\"type\":\"CUSTOM\"},{\"name\":\"Linux\",\"type\":\"CUSTOM\"},{\"name\":\"docker hosts\",\"type\":\"SITE\"},{\"name\":\"lab\",\"type\":\"SITE\"}],\"total_vulnerabilities\":28,\"new\":[],\"remediated\":[],\"unique_identifiers\":{\"id\":\"4421d73dfe04f594df731e6bcd8156a\",\"source\":\"R7 Agent\"}}", "type": [ "info" ] }, "host": { + "hostname": "host", "id": "452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198", "ip": [ "10.4.24.164" @@ -113,7 +114,7 @@ "assessed_for_vulnerabilities": true, "critical_vulnerabilities": 1, "exploits": 9, - "host_name": "host.domain.com", + "host_name": "HOST.domain.com", "id": "452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198", "ip": "10.4.24.164", "last_scan_end": "2020-03-20T19:12:39.766Z", @@ -162,7 +163,8 @@ }, "related": { "hosts": [ - "host.domain.com" + "host.domain.com", + "host" ], "ip": [ "10.4.24.164" diff --git a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index a441d3f2147..3b0295cc967 100644 --- a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -115,10 +115,28 @@ processors: field: host.name copy_from: rapid7.insightvm.asset.host_name ignore_empty_value: true + - lowercase: + field: host.name + ignore_missing: true + - append: + field: related.hosts + value: '{{{host.name}}}' + if: ctx.host?.name != null + allow_duplicates: false + - script: + lang: painless + if: ctx.host?.name != null + source: | + int idx = ctx.host.name.indexOf("."); + if (idx == -1) { + ctx.host.hostname = ctx.host.name; + } else { + ctx.host.hostname = ctx.host.name.substring(0, idx); + } - append: field: related.hosts - value: '{{{rapid7.insightvm.asset.host_name}}}' - if: ctx.rapid7?.insightvm?.asset?.host_name != null + value: '{{{host.hostname}}}' + if: ctx.host?.hostname != null allow_duplicates: false - rename: field: json.id diff --git a/packages/rapid7_insightvm/manifest.yml b/packages/rapid7_insightvm/manifest.yml index 18678963c25..8a24711bc0b 100644 --- a/packages/rapid7_insightvm/manifest.yml +++ b/packages/rapid7_insightvm/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: rapid7_insightvm title: Rapid7 InsightVM -version: "1.9.0" +version: "1.10.0" source: license: "Elastic-2.0" description: Collect logs from Rapid7 InsightVM with Elastic Agent.