From 5ff223350266fcf4740ab1696d1f28e417af8488 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 30 Apr 2020 12:59:56 -0400 Subject: [PATCH] Skip add_host_metadata for forwarded event logs Update config examples to use the "forwarded" tag to skip adding host metadata. Also disable host.name being added by libbeat. This field was overwritten by the winlog.computer_name so it didn't serve any purpose to have libbeat set it. Relates #13920 --- CHANGELOG.next.asciidoc | 1 + dev-tools/mage/config.go | 1 + libbeat/_meta/config.yml.tmpl | 3 ++- winlogbeat/_meta/beat.yml.tmpl | 17 +++++------------ winlogbeat/_meta/common.yml.tmpl | 6 ++++++ winlogbeat/cmd/root.go | 7 ++++++- winlogbeat/scripts/mage/config.go | 3 ++- winlogbeat/winlogbeat.reference.yml | 11 ++--------- winlogbeat/winlogbeat.yml | 8 ++++---- x-pack/winlogbeat/_meta/beat.yml.tmpl | 14 ++++++++++++++ x-pack/winlogbeat/winlogbeat.reference.yml | 14 ++++++++++++++ x-pack/winlogbeat/winlogbeat.yml | 20 ++++++++++++++++---- 12 files changed, 73 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 78a1385e589d..3fb367f063af 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -212,6 +212,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Kerberos support to Elasticsearch output. {pull}17927[17927] - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191] - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000] +- Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153] *Auditbeat* diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 668c71e10740..6cabac5c9b6c 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -116,6 +116,7 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error "UseDockerMetadataProcessor": true, "UseKubernetesMetadataProcessor": false, "ExcludeDashboards": false, + "UseProcessorsTemplate": false, } for k, v := range args.ExtraVars { params[k] = v diff --git a/libbeat/_meta/config.yml.tmpl b/libbeat/_meta/config.yml.tmpl index 2d5e510e33f9..3cf0e3e00af2 100644 --- a/libbeat/_meta/config.yml.tmpl +++ b/libbeat/_meta/config.yml.tmpl @@ -90,6 +90,7 @@ output.elasticsearch: #ssl.key: "/etc/pki/client/cert.key" {{end}} #================================ Processors ===================================== +{{if .UseProcessorsTemplate}}{{template "processors.yml.tmpl" .}}{{else -}} {{if not .UseObserverProcessor}} # Configure processors to enhance or manipulate events generated by the beat. @@ -112,7 +113,7 @@ processors: #name: us-east-1a # Lat, Lon " #location: "37.926868, -78.024902" -{{end}} +{{end}}{{end}} #================================ Logging ===================================== # Sets log level. The default log level is info. diff --git a/winlogbeat/_meta/beat.yml.tmpl b/winlogbeat/_meta/beat.yml.tmpl index 24b27d366480..093c6f69c042 100644 --- a/winlogbeat/_meta/beat.yml.tmpl +++ b/winlogbeat/_meta/beat.yml.tmpl @@ -2,18 +2,11 @@ winlogbeat.event_logs: - name: Application ignore_older: 72h -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + - name: System -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + - name: Security -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + + - name: ForwardedEvents + tags: [forwarded] {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/winlogbeat/_meta/common.yml.tmpl b/winlogbeat/_meta/common.yml.tmpl index 7a7feaddf5a0..63aa30fa0b5e 100644 --- a/winlogbeat/_meta/common.yml.tmpl +++ b/winlogbeat/_meta/common.yml.tmpl @@ -34,3 +34,9 @@ setup.template.settings: #index.codec: best_compression #_source.enabled: false {{end -}} +{{define "processors.yml.tmpl"}} +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ +{{end -}} diff --git a/winlogbeat/cmd/root.go b/winlogbeat/cmd/root.go index 7075a51aeb00..ecc3aa8e38f0 100644 --- a/winlogbeat/cmd/root.go +++ b/winlogbeat/cmd/root.go @@ -20,6 +20,7 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" + "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/winlogbeat/beater" // Register fields. @@ -35,4 +36,8 @@ import ( var Name = "winlogbeat" // RootCmd to handle beats cli -var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, HasDashboards: true}) +var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{ + Name: Name, + HasDashboards: true, + Processing: processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()), +}) diff --git a/winlogbeat/scripts/mage/config.go b/winlogbeat/scripts/mage/config.go index 1dae96510dda..70cc8cb43a6c 100644 --- a/winlogbeat/scripts/mage/config.go +++ b/winlogbeat/scripts/mage/config.go @@ -54,7 +54,8 @@ func configFileParams() devtools.ConfigFileParams { devtools.LibbeatDir("_meta/config.docker.yml"), }, ExtraVars: map[string]interface{}{ - "GOOS": "windows", + "GOOS": "windows", + "UseProcessorsTemplate": true, }, } } diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 471b6c4e7fc2..39e22d25e349 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -26,19 +26,12 @@ winlogbeat.event_logs: - name: Application ignore_older: 72h - # Set to true to publish fields with null values in events. - #keep_null: false - - name: System - # Set to true to publish fields with null values in events. - #keep_null: false - - name: Security - # Set to true to publish fields with null values in events. - #keep_null: false - + - name: ForwardedEvents + tags: [forwarded] #================================ General ====================================== diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 8887e8d75c7b..d816327de992 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -25,6 +25,8 @@ winlogbeat.event_logs: - name: Security + - name: ForwardedEvents + tags: [forwarded] #==================== Elasticsearch template settings ========================== setup.template.settings: @@ -125,12 +127,10 @@ output.elasticsearch: #================================ Processors ===================================== -# Configure processors to enhance or manipulate events generated by the beat. - processors: - - add_host_metadata: ~ + - add_host_metadata: + when.not.contains.tags: forwarded - add_cloud_metadata: ~ - - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/winlogbeat/_meta/beat.yml.tmpl b/x-pack/winlogbeat/_meta/beat.yml.tmpl index f2660df68bd9..1ea8cdcc879c 100644 --- a/x-pack/winlogbeat/_meta/beat.yml.tmpl +++ b/x-pack/winlogbeat/_meta/beat.yml.tmpl @@ -19,4 +19,18 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index c8643d904abd..3fd2ffcba4ff 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -42,6 +42,20 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + #================================ General ====================================== diff --git a/x-pack/winlogbeat/winlogbeat.yml b/x-pack/winlogbeat/winlogbeat.yml index bc5dbc294d24..e718fb91d413 100644 --- a/x-pack/winlogbeat/winlogbeat.yml +++ b/x-pack/winlogbeat/winlogbeat.yml @@ -37,6 +37,20 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + #==================== Elasticsearch template settings ========================== setup.template.settings: @@ -137,12 +151,10 @@ output.elasticsearch: #================================ Processors ===================================== -# Configure processors to enhance or manipulate events generated by the beat. - processors: - - add_host_metadata: ~ + - add_host_metadata: + when.not.contains.tags: forwarded - add_cloud_metadata: ~ - - add_docker_metadata: ~ #================================ Logging =====================================