diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 013397a4bdbb..94fe8b3db3a4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -513,6 +513,7 @@ from being added to events by default. {pull}18159[18159] - Copy tag names from MISP data into events. {pull}21664[21664] - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291] - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696] +- Add platform logs in the azure filebeat module. {pull}22371[22371] - Added `event.ingested` field to data from the Netflow module. {pull}22412[22412] - Improve panw ECS url fields mapping. {pull}22481[22481] - Improve Nats filebeat dashboard. {pull}22726[22726] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 102c83f9e0c6..86380c30962b 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3055,6 +3055,153 @@ type: keyword -- +[float] +=== platformlogs + +Fields for Azure platform logs. + + + +*`azure.platformlogs.operation_name`*:: ++ +-- +Operation name + + +type: keyword + +-- + +*`azure.platformlogs.result_type`*:: ++ +-- +Result type + + +type: keyword + +-- + +*`azure.platformlogs.result_signature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.platformlogs.category`*:: ++ +-- +Category + + +type: keyword + +-- + +*`azure.platformlogs.event_category`*:: ++ +-- +Event Category + + +type: keyword + +-- + +*`azure.platformlogs.status`*:: ++ +-- +Status + + +type: keyword + +-- + +*`azure.platformlogs.ccpNamespace`*:: ++ +-- +ccpNamespace + + +type: keyword + +-- + +*`azure.platformlogs.Cloud`*:: ++ +-- +Cloud + + +type: keyword + +-- + +*`azure.platformlogs.Environment`*:: ++ +-- +Environment + + +type: keyword + +-- + +*`azure.platformlogs.EventTimeString`*:: ++ +-- +EventTimeString + + +type: keyword + +-- + +*`azure.platformlogs.Caller`*:: ++ +-- +Caller + + +type: keyword + +-- + +*`azure.platformlogs.ScaleUnit`*:: ++ +-- +ScaleUnit + + +type: keyword + +-- + +*`azure.platformlogs.ActivityId`*:: ++ +-- +ActivityId + + +type: keyword + +-- + +*`azure.platformlogs.properties.*`*:: ++ +-- +Properties + + +type: object + +-- + [float] === signinlogs diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 45010618214a..af8c24131856 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -24,6 +24,9 @@ The module contains the following filesets: `activitylogs` :: Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. +`platformlogs` :: +Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on. + `signinlogs` :: Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. @@ -46,6 +49,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi storage_account_key: "" resource_manager_endpoint: "" + platformlogs: + enabled: false + var: + eventhub: "" + consumer_group: "$Default" + connection_string: "" + storage_account: "" + storage_account_key: "" + resource_manager_endpoint: "" + auditlogs: enabled: false var: diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 88d55a69f3bb..dca6a906750f 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -365,6 +365,16 @@ filebeat.modules: # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index ab7f477b8bb7..fdea9b1f2526 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -14,6 +14,16 @@ # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 485d6ddcab34..ee7c5961f85a 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -19,6 +19,9 @@ The module contains the following filesets: `activitylogs` :: Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. +`platformlogs` :: +Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on. + `signinlogs` :: Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. @@ -41,6 +44,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi storage_account_key: "" resource_manager_endpoint: "" + platformlogs: + enabled: false + var: + eventhub: "" + consumer_group: "$Default" + connection_string: "" + storage_account: "" + storage_account_key: "" + resource_manager_endpoint: "" + auditlogs: enabled: false var: diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index d358caa2edd2..f37b4bf9ee8c 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW01z2zYQvftX7PiYady7Dp1RnaTjmSbOOM6ZA5MrGjVEsAAoj/LrO+D3BwBCFkil0+rUmsq+R2B3sfsWeg8veNwA+VEIvAJQVDHcwPVW///1FYBAhkTiBp5QkSuABGUsaK4ozzbw2xUAQPld+MyTgmkTO4oskZvy0XvIyB478/qjjjluIBW8yOu/GGwOzfRNyeKp/XZEk/Z5Y/gFj69c9P9uNF99Kup9k3D3YQIZcyGQkSCIt50tE5TCjGTqbJTKjAlAoOSFiHFiv78hM9YfpjbGu9WHHLyM64VmYPvQ/VfrY43fIhTc1G6DmAt+oAmKBUC1jV81iszJYLU7dPPTVeHDIX8ZW2vTR6GeuaA/qhAUVZ4JBLrt24aB7RY+VvRA1ZHxVM7GzThhdjR2pGAqKiNlAzvCJPqF26cytmDHRZ2vGj6gCd14RyFmiqqjceVMYTOzbndmeyYKfRoxI3QvI5pRRYnCJHo6RoWchI+bmgc9/bktsaDFgqcjWLBstMHt7EOqZv87gTCYgmBMI6UHzNbh8ocTqjuRxTp0vjmAGjK7grF12HxyIbVrEz/jnqywNGacYdTdvLNGGX/6C2NleFw9iOao9r4W7Ume0yyt/831u+tzotf6SoMzYYnssZ0B8EkZMub5GoFhhekfYca3CMxka8dpqKAul7JJxTIkY9u3E6h8dOO4tq9PV3CGEZGSptkeMxW5thQ8F/OEt9CfB84QOgpOrxoRT3BXHn/jzmUt1h3+uGK3UV6f5MkucJGV7O2/x0rmgmYxzQlbnezXBvk0mprJpYhOsBt6PEdR9RxhG537xu60puz16bpbMCzLea1dwZT9fWtQ7WhEdYpNOGSz5bZKIQpTLswtyptgb00Wu4NIB3N40I/arhs6F9q1FMVxffPmduyrzeJcQyZRHGiMkcC/C5SW5DYXfh5h963CgYcKB+6mdlpKiqhCRjFPTCkhBJcSAAYA066/SKjyavlP6t211UnjPiMOuPr68P77+IytVeA7UM/YZcIbgNtCCMwUO/4C2/JtqKy+k7EjyCLPudDN9oGwAm9WzauPfab23NrBH1DIab0ajIHJvJce8yZsoyLjEpbPfdeJyAw/z1kSPsPqrW2j125/Lt9W67JMXvuTpzb7Yy0zSqjMGTmaIjAQm20jU9ZQZgFt5C8CiXT082cReihtl8lY5ys2v1rWyU9AVpaJ0JgK42layaX1ib2YC6WVUmqDmWZRS+0egE1XKBsxrBOmQPCO7ejiiChU1BFD+gtnBpC2LxXZTxWRmSog0DpYrbenCxEp6uCtRkcupdGm7CygID6WrNqR51RL9BPxZjIlhFXQPriSJcw5fWAylha+3XZ70x6QxKMNpV2JPCJJIlCadjgwGaA5bB1gDadCoog6eWMd3/kuUXSayrwL7XlCdxSTqKtmjJELXrqs56gA3hDsJ6zB5/qd7BVa8/FVgDN8jcpmxu4US+hTX/AV3LAnJKiFODbJyo8nZ8klFvKeJRaCbQLpTaNPPrh86ohsx8W+vmGAKREJzdKyBq2R+Vtn0iS3RWOwOcrWCuEbQHUl2Qqu1hF3n3lQF2jEJ4/cCNPAWp/vbBEAQx8wiGmL8tvmuY+8P974tWlOt32moLHcDoGQ8fTdjuEbUJrmBaPJt8yAf10orT0s84ghmrvKzcWYGcpcn/9oI5+mGc2Cq+fa7HuahdTP/9ei/3vicA3fMxWcgM32Ree7K88cVp7nLiP5V5nsPM3/AvpkLLBsaIh91PB2YfK2Ml5aAKP4OZBBlh81lBWJ15jhJ1L0/SSiUKszU68NSC21MCURl6ie54thW1qWPvQKI7E893NTp6QZInPkYLPf3RmmmKlIL0whcalQKUH06oMRpAvbLCmvERIWkThGKaPqlshSEdzCQQUHFri2jhM0pZrd0vdo7msgEM1FGrtTUxnRTKEoh1ULufSdBBdGW2LyF8wiKmWBYsEAe9QwUMG4I2xAaLkJ5oCQc4iZC64drZTC6R4jw933htGOceO57sHna4tSHtxAM9hTxqhEHWJ2/xZUvkQJKkLZMgv1QOULWAAGJBgekEUkTQWmugpZkE4JBQ4oA7GkEHoLq7JxcW4VWlmk3k3RBvR0AlvIyUtCZvvj352ucMi2P2D0OmlnjpEzVP9vNss+wj4KwYXtCiSEHRt+1FhgxGrlOywvic4kgEstVs1ujYn4hxJqTkeuZRedB45S4X55XvcNIjgQG3pPgr/OKd1BWP3uAPrJr1coUUijPrMAqUeN5a4OmmvaMz/sCHY5un109U8AAAD//4AUZiI=" + return "eJzsXM9v2zoSvuevGORYbLP3HBbwpukiwLYpkvQsMNJY4Qst6pGUA/evfyBl/bJIio4pOQ+vPrWR830fh5zhcIbKZ3jF3TWQX5XACwBFFcNruFzp/19eAAhkSCRewzMqcgGQoUwFLRXlxTX85wIAwHwXvvGsYhpiTZFl8to8+gwF2WAHrz9qV+I15IJX5f4nFswhTB9KVs/ttxOatc8b4FfcvXHR/7kVvv7U0vuQcPdlRJlyIZCRKIw3HZaNSmFBCnUySw1jIxAoeSVSHOH3J2QC/WGMcThbfcrBYHwDmqDtU/eH1uc6HEUsujFuw1gKvqUZihlINca/NYssycDaHbv96aL08Zi/H6K14aNSL1zQX7ULijrORCJd9bFhgN3Sp4puqdoxnstJvzkMmJ2MNamYSoynXMOaMIlh7vbV+BasudjHq0YPaEFXwV6IhaJqZ7WczW0m7HZnx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyH380gLk6c+N4YKWC5534OByyQb/Yh9Kta+/IwSDzQkOZeR0i8UyWv7npep2ZLGMnEcPUSNmXTG2jJqvPqbWNukLbsgCprHzDL3u6pPTy/jzH5gqy+P6QTIltfe1ZEPKkhb5/ncuP12e4r3OIQ32hDmix2qCICRkyJSXSziGk6a/hVlHEVnJys3TSEGdLhWjjGUoxjVvR0i59fP4pq8vV3CGCZGS5sUGC5X4phQCjXnEKPTngTOEToJ3VR0Iz3Bttr/Dk8tSqjv+w4zdJXl5kUcvgbNYsjf/AZYsBS1SWhK2uNgfDfNxMrWScwkdcTfyeImiPnPEPejcN7jjnLJ3TtenBYtZTjvaVUy5x7sn1QuNqK5iE4/ZjtxmKURhzoX9iPIu2hsbYrcRaWeOT3qrcf3UpdBLS1E8zG/efRz74UKcOpBJFFuaYiLwzwqlI7hNuV+A2z3WPPBQ88DdGKeVpIiqZJLyzBYSYmgxBDAgGJ/6q4yqoCP/UWd3jTo6uE8UB3zn+vjr9+kFW1Tga1Av2EXCK4CbSggsFNv9C1ZmNFTW3ynYDmRVllzow/aWsAqvFo2rT32l7tja0W9RyHG+Gk2BDT6oHvMubmtFxldYPnWsoyIzfJy9JH6E1VPbeq8bfyre1naZJ679n+cu/MNaZpJRWTKys3lgJDWrpky5p7IX0A7Wi0AiPef5kwQ9GGwTjHW8YtPWcnZ+IqpydIQOpTCe53W5dL9jz7aE8rpS6qIZR1FH7h5BTZcoWzmcHaZI9J7p6PyIKFTU40P6Cyc6kMaXimzGFZGJLCCSHZzo7e5CRI7aeevWka/S6KrszFBBfDKq2pbnuJYYVsSbiJQQt4L2xRcsYWrRRxbjOMK30+4+tEcU8eRiaS1RJiTLBErbDEcWA7SElYes0VRJFElX3lhm7fyUKLqayvQS2vCMrilmSZfNWD0Xguqyga0CeIezH2GDb/sxuTO05hNaAS7wLTGHGfeimKM+9R3fwE97RICaSWMTrMJ0cpadw5D3LHMIbANIrxt99MYVkkcUay42+xsGmBOR0SI3Oeiemb+3J01KlzdG66OsnBShDrTPJNuCq7PF3VcedQk0xaeA2Ahjx1pe72QSAMM1YCmmzapvVZYh5f3DiV9a5njaJxIax+0QiOlPP90coQ6lZZ7Rm0LTDPjbudLSzbIAH6KlL92cTZklzQ35R1t+Y0TpPe+jXJlr9Bx1Ze53+y8y8+/2X7/BFY/ycYzXGjUtv8e/I+xEbWhvGK8ithrGcA3RbbGlgpt7ERGn0AHakuopfqIbfFSCFnnkteMAbk1LGIt53dyC11A9poThz4JGNK0dsiFsKp6jRPGUC952zHGjaFT68FyNDKh1HFPjOL65P9pxdWylRfR+tYb9TIuYHevf3d9/Xjt2T9+Dii7AhX3WlGrhLv/CKdQ8TfY6kp3WZT9DRzAVaEqIxN3cf38r8KYGNwhgbTcOGg/zN/dNDSCosf+BeuhhTZlY1pmokAxEzWUYI8TXxi7L2bgdRcI+9QKXUMoybJl6m4gxIkcJLvzuLR2qj6TaMJXEuVzFkGjrg5Wkc9siMxf3CUtImqKUifXYGktWRwc1nf2U3MvjBM2pVjf3zdX7PRGI5uqqe1FTmdBCoTDXQ2Za0ncSfBxtislfsUiolBWKGR3sSdNATeP3sIGg+e4MDQR5rw2VguuFZg5mdIOJ5W2zRtGaceu+HqDnR8tiNm6gBWwoY1SidjH3+hZUviYZKkLZPIZ6oPIVHAQDEQy3yBKS5wJznYXMKMdQgYfKIiyrhJ7COm2cXVvNZpLUuzHbQJ4OYDMtciPIjn/4lx4W2GTbPxkQtNNObCMn9NktddD6E9JKRyG4cL10AHEv6txqLrBytQ0zNK9lTASAcxlrr26JO2hfDNVU53ZfdtFxYCcVbubXdd8wgoexkfcs+NtUbzmKqv96iD74hUYlKmmtz8wg6klz+bOD5sWoiVcpo72O1D66+CsAAP//n/8+qg==" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml new file mode 100644 index 000000000000..ac03e0004f5b --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -0,0 +1,66 @@ +- name: platformlogs + type: group + release: beta + default_field: false + description: > + Fields for Azure platform logs. + fields: + - name: operation_name + type: keyword + description: > + Operation name + - name: result_type + type: keyword + description: > + Result type + - name: result_signature + type: keyword + description: > + Result signature + - name: category + type: keyword + description: > + Category + - name: event_category + type: keyword + description: > + Event Category + - name: status + type: keyword + description: > + Status + - name: ccpNamespace + type: keyword + description: > + ccpNamespace + - name: Cloud + type: keyword + description: > + Cloud + - name: Environment + type: keyword + description: > + Environment + - name: EventTimeString + type: keyword + description: > + EventTimeString + - name: Caller + type: keyword + description: > + Caller + - name: ScaleUnit + type: keyword + description: > + ScaleUnit + - name: ActivityId + type: keyword + description: > + ActivityId + - name: properties.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Properties + diff --git a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml new file mode 100644 index 000000000000..496480aa1d0c --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml @@ -0,0 +1,16 @@ +type: azure-eventhub +connection_string: {{ .connection_string }} +eventhub: {{ .eventhub }} +consumer_group: {{ .consumer_group }} +storage_account: {{ .storage_account }} +storage_account_key: {{ .storage_account_key }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} +storage_account_container: filebeat-platformlogs-{{ .eventhub }} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/platformlogs/config/file.yml b/x-pack/filebeat/module/azure/platformlogs/config/file.yml new file mode 100644 index 000000000000..e9470671e071 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/config/file.yml @@ -0,0 +1,14 @@ +type: log +paths: + {{ range $i, $path := .paths }} +- {{$path}} + {{ end }} +exclude_files: [".gz$"] +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml new file mode 100644 index 000000000000..8493ef886fe2 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -0,0 +1,195 @@ +description: Pipeline for parsing azure platform logs. +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: azure + target_field: azure-eventhub + ignore_missing: true +- script: + source: ctx.message = ctx.message.replace(params.empty_field_name, '') + params: + empty_field_name: '"":"",' + ignore_failure: true +- json: + field: message + target_field: azure.platformlogs +- date: + field: azure.platformlogs.time + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 +- date: + field: azure.platformlogs.EventTimeString + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 + - "M/d/yyyy h:mm:ss a XXX" +- remove: + field: + - message + - azure.platformlogs.time + ignore_missing: true +- rename: + field: azure.platformlogs.resourceId + target_field: azure.resource_id + ignore_missing: true +- rename: + field: azure.platformlogs.Region + target_field: cloud.region + ignore_missing: true +- json: + field: azure.platformlogs.EventProperties + target_field: azure.platformlogs.properties + ignore_failure: true +- remove: + if: ctx.azure.platformlogs.properties != null + field: + - azure.platformlogs.EventProperties + ignore_missing: true +- rename: + field: azure.platformlogs.EventName + target_field: event.action + ignore_missing: true +- rename: + field: azure.platformlogs.properties.log + target_field: message + ignore_missing: true +- rename: + field: azure.platformlogs.callerIpAddress + target_field: source.ip + ignore_missing: true +- rename: + field: azure.platformlogs.level + target_field: log.level + ignore_missing: true +- rename: + field: azure.platformlogs.durationMs + target_field: event.duration + ignore_missing: true +- script: + lang: painless + source: if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration + * params.param_nano;} + params: + param_nano: 1000000 + ignore_failure: true +- rename: + field: azure.platformlogs.location + target_field: geo.name + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx?.azure?.platformlogs?.properties?.eventCategory != null) { + ctx.azure.platformlogs.event_category = ctx.azure.platformlogs.properties.eventCategory; + } + else if (ctx?.azure?.platformlogs?.properties?.policies != null) { + ctx.azure.platformlogs.event_category = 'Policy'; + } + else { + ctx.azure.platformlogs.event_category = 'Administrative'; + } + ignore_failure: true +- rename: + field: azure.platformlogs.resultType + target_field: azure.platformlogs.result_type + ignore_missing: true +- convert: + field: azure.platformlogs.result_type + target_field: event.outcome + type: string + if: "ctx?.azure?.platformlogs?.result_type != null && ctx.azure.platformlogs.result_type instanceof String && (ctx.azure.platformlogs.result_type.toLowerCase() == 'success' || ctx.azure.platformlogs.result_type.toLowerCase() == 'failure')" +- convert: + field: azure.platformlogs.properties.result + target_field: event.outcome + type: string + if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)" +- convert: + field: azure.platformlogs.Status + target_field: event.outcome + type: string + if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.Status != null && ctx?.azure?.platformlogs?.Status instanceof String && ['success', 'failure', 'unknown', 'Succeeded', 'Failed'].contains(ctx.azure?.platformlogs?.Status)" +- rename: + field: azure.platformlogs.operationName + target_field: azure.platformlogs.operation_name + ignore_missing: true +- convert: + field: azure.platformlogs.operation_name + target_field: event.action + type: string + ignore_missing: true +- rename: + field: azure.platformlogs.resultSignature + target_field: azure.platformlogs.result_signature + ignore_missing: true +- rename: + field: azure.platformlogs.correlationId + target_field: azure.correlation_id + ignore_missing: true +- rename: + field: azure.platformlogs.properties.statusCode + target_field: azure.platformlogs.properties.status_code + ignore_missing: true +- rename: + field: azure.platformlogs.Status + target_field: azure.platformlogs.status + ignore_missing: true +- geoip: + field: source.ip + target_field: geo + ignore_missing: true +- script: + lang: painless + ignore_failure: true + params: + "write": + type: + - change + "read": + type: + - access + "delete": + type: + - deletion + "action": + type: + - change + source: >- + if (ctx?.azure?.platformlogs?.category == null) { + return; + } + def hm = new HashMap(params.get(ctx.azure.platformlogs.category.toLowerCase())); + hm.forEach((k, v) -> ctx.event[k] = v); +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: event.kind + value: event +- pipeline: + name: '{< IngestPipeline "azure-shared-pipeline" >}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/platformlogs/manifest.yml b/x-pack/filebeat/module/azure/platformlogs/manifest.yml new file mode 100644 index 000000000000..a67dc604dd24 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: input + default: azure-eventhub + - name: eventhub + - name: consumer_group + default: "$Default" + - name: connection_string + - name: storage_account + - name: storage_account_key + - name: resource_manager_endpoint + - name: tags + default: [forwarded] + +ingest_pipeline: + - ingest/pipeline.yml + - ../azure-shared-pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log new file mode 100644 index 000000000000..13f18cfe2c2f --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log @@ -0,0 +1 @@ +{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json new file mode 100644 index 000000000000..ca2c95be8242 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -0,0 +1,36 @@ +[ + { + "@timestamp": "2020-11-03T09:06:42.000Z", + "azure.platformlogs.ActivityId": "30ed877c-a36b-491a-bd4d-ddd847fe55b8", + "azure.platformlogs.Caller": "Portal", + "azure.platformlogs.Environment": "PROD", + "azure.platformlogs.EventTimeString": "11/3/2020 9:06:42 AM +00:00", + "azure.platformlogs.ScaleUnit": "PROD-AM3-AZ501", + "azure.platformlogs.category": "OperationalLogs", + "azure.platformlogs.event_category": "Administrative", + "azure.platformlogs.properties.Namespace": "obstesteventhubs", + "azure.platformlogs.properties.SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "azure.platformlogs.properties.TrackingId": "30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2", + "azure.platformlogs.properties.Via": "sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04&$skip=0&$top=100", + "azure.platformlogs.status": "Succeeded", + "azure.resource.group": "OBS-TEST", + "azure.resource.id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", + "azure.resource.name": "OBSTESTEVENTHUBS", + "azure.resource.provider": "MICROSOFT.EVENTHUB/NAMESPACES", + "azure.subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", + "cloud.provider": "azure", + "cloud.region": "West Europe", + "event.action": "Retreive ConsumerGroup", + "event.dataset": "azure.platformlogs", + "event.kind": "event", + "event.module": "azure", + "event.outcome": "succeeded", + "fileset.name": "platformlogs", + "input.type": "log", + "log.offset": 0, + "service.type": "azure", + "tags": [ + "forwarded" + ] + } +] diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log new file mode 100644 index 000000000000..7b8930fb3416 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log @@ -0,0 +1 @@ +{"Cloud":"AzureCloud","Environment":"prod","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json new file mode 100644 index 000000000000..fb95fe0ba809 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json @@ -0,0 +1,31 @@ +[ + { + "@timestamp": "2020-11-09T10:57:31.000Z", + "azure.platformlogs.Cloud": "AzureCloud", + "azure.platformlogs.Environment": "prod", + "azure.platformlogs.category": "kube-audit", + "azure.platformlogs.ccpNamespace": "5e4bf4baee195b00017cdbfa", + "azure.platformlogs.event_category": "Administrative", + "azure.platformlogs.operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "azure.platformlogs.properties.pod": "kube-apiserver-666bd4b459-hjgdc", + "azure.platformlogs.properties.stream": "stdout", + "azure.resource.group": "OBS-INFRASTRUCTURE", + "azure.resource.id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "azure.resource.name": "OBSKUBE", + "azure.resource.provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "azure.subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "cloud.provider": "azure", + "event.action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "event.dataset": "azure.platformlogs", + "event.kind": "event", + "event.module": "azure", + "fileset.name": "platformlogs", + "input.type": "log", + "log.offset": 0, + "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", + "service.type": "azure", + "tags": [ + "forwarded" + ] + } +] diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index d6e57c7de262..3b2bc1ecf106 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -17,6 +17,16 @@ # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: