diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d0073444094..2e7c57ca634 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -289,6 +289,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918] - Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738] - Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108] +- Added an input option `publisher_pipeline.disable_host` to disable `host.name` + from being added to events by default. {pull}18159[18159] - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065] *Heartbeat* diff --git a/filebeat/_meta/common.reference.inputs.yml b/filebeat/_meta/common.reference.inputs.yml index b7de598be0b..db105f17ef0 100644 --- a/filebeat/_meta/common.reference.inputs.yml +++ b/filebeat/_meta/common.reference.inputs.yml @@ -65,6 +65,11 @@ filebeat.inputs: # Set to true to publish fields with null values in events. #keep_null: false + # By default, all events contain `host.name`. This option can be set to true + # to disable the addition of this field to all events. The default value is + # false. + #publisher_pipeline.disable_host: false + # Ignore files which were modified more then the defined timespan in the past. # ignore_older is disabled by default, so no files are ignored by setting it to 0. # Time strings like 2h (2 hours), 5m (5 minutes) can be used. diff --git a/filebeat/channel/runner.go b/filebeat/channel/runner.go index 51a17e17939..941e1cc8161 100644 --- a/filebeat/channel/runner.go +++ b/filebeat/channel/runner.go @@ -42,6 +42,10 @@ type commonInputConfig struct { Processors processors.PluginConfig `config:"processors"` KeepNull bool `config:"keep_null"` + PublisherPipeline struct { + DisableHost bool `config:"disable_host"` // Disable addition of host.name. + } `config:"publisher_pipeline"` + // implicit event fields Type string `config:"type"` // input.type ServiceType string `config:"service.type"` // service.type @@ -184,6 +188,7 @@ func newCommonConfigEditor( clientCfg.Processing.Fields = fields clientCfg.Processing.Processor = procs clientCfg.Processing.KeepNull = config.KeepNull + clientCfg.Processing.DisableHost = config.PublisherPipeline.DisableHost return clientCfg, nil }, nil diff --git a/filebeat/docs/inputs/input-common-options.asciidoc b/filebeat/docs/inputs/input-common-options.asciidoc index 8e08a8074e4..de22a519846 100644 --- a/filebeat/docs/inputs/input-common-options.asciidoc +++ b/filebeat/docs/inputs/input-common-options.asciidoc @@ -101,3 +101,9 @@ version and the event timestamp; for access to dynamic fields, use Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might expand to `"filebeat-myindex-2019.11.01"`. + +[float] +===== `publisher_pipeline.disable_host` + +By default, all events contain `host.name`. This option can be set to `true` to +disable the addition of this field to all events. The default value is `false`. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index dfa8f631360..ea2ad11e672 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -451,6 +451,11 @@ filebeat.inputs: # Set to true to publish fields with null values in events. #keep_null: false + # By default, all events contain `host.name`. This option can be set to true + # to disable the addition of this field to all events. The default value is + # false. + #publisher_pipeline.disable_host: false + # Ignore files which were modified more then the defined timespan in the past. # ignore_older is disabled by default, so no files are ignored by setting it to 0. # Time strings like 2h (2 hours), 5m (5 minutes) can be used. diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 0c48b34ebc2..2275d761bca 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -52,6 +52,11 @@ filebeat.{{input_config | default("inputs")}}: {{k}}: {{v}} {% endfor %} {% endif %} + {%- if publisher_pipeline %} + {%- for name, value in publisher_pipeline.items() %} + publisher_pipeline.{{name}}: {{value | tojson}} + {%- endfor %} + {% endif %} fields_under_root: {{"true" if fieldsUnderRoot else "false"}} diff --git a/filebeat/tests/system/test_input.py b/filebeat/tests/system/test_input.py index 0075329205b..684f4f852af 100644 --- a/filebeat/tests/system/test_input.py +++ b/filebeat/tests/system/test_input.py @@ -662,3 +662,23 @@ def test_disable_recursive_glob(self): "recursive glob disabled"), max_timeout=10) filebeat.check_kill_and_wait() + + def test_input_processing_pipeline_disable_host(self): + """ + Check processing_pipeline.disable_host in input config. + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/test.log", + publisher_pipeline={ + "disable_host": True, + }, + ) + with open(self.working_dir + "/test.log", "w") as f: + f.write("test message\n") + + filebeat = self.start_beat() + self.wait_until(lambda: self.output_has(lines=1)) + filebeat.check_kill_and_wait() + + output = self.read_output() + assert "host.name" not in output[0] diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go index c4f4665af1b..9d6fbf04e86 100644 --- a/libbeat/beat/pipeline.go +++ b/libbeat/beat/pipeline.go @@ -110,6 +110,9 @@ type ProcessingConfig struct { // KeepNull determines whether published events will keep null values or omit them. KeepNull bool + // Disables the addition of host.name if it was enabled for the publisher. + DisableHost bool + // Private contains additional information to be passed to the processing // pipeline builder. Private interface{} diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go index f020423b733..cf99e03d4d3 100644 --- a/libbeat/publisher/processing/default.go +++ b/libbeat/publisher/processing/default.go @@ -269,6 +269,12 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, needsCopy := b.alwaysCopy || localProcessors != nil || b.processors != nil builtin := b.builtinMeta + if cfg.DisableHost { + tmp := builtin.Clone() + tmp.Delete("host") + builtin = tmp + } + var clientFields common.MapStr for _, mod := range b.modifiers { m := mod.ClientFields(b.info, cfg) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9ac62c5e34f..115d8678fd7 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1114,6 +1114,11 @@ filebeat.inputs: # Set to true to publish fields with null values in events. #keep_null: false + # By default, all events contain `host.name`. This option can be set to true + # to disable the addition of this field to all events. The default value is + # false. + #publisher_pipeline.disable_host: false + # Ignore files which were modified more then the defined timespan in the past. # ignore_older is disabled by default, so no files are ignored by setting it to 0. # Time strings like 2h (2 hours), 5m (5 minutes) can be used.