From 43914f746e27964f04bd92821f2c465858e48e4f Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 5 Jan 2016 21:44:16 +0100 Subject: [PATCH 1/4] Update default spooler size and logstash max bulk size - set default spooler size in filebeat to 2048, in order to increase number of lines to be combined into one bulk request - set default bulk max size to 2048, to guarantee not to split bigger bulk requests into too many small requests. Tested different spooler and bulk sizes using NASA-HTTP logs: spooler bulk | cpu(%) mem-res(MB) throughput(1k lines/s) --------------------------------------------------------------------- 1024 200 | 26 20 2.1 1024 1024 | 55 20 15.7 2048 2048 | 66 25 18.2 3072 3072 | 66 28 18.5 4096 4096 | 70 33 18.7 --- filebeat/config/config.go | 2 +- libbeat/publisher/async.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/config/config.go b/filebeat/config/config.go index af53d00abb9d..76c5a89dfa20 100644 --- a/filebeat/config/config.go +++ b/filebeat/config/config.go @@ -17,7 +17,7 @@ const ( DefaultRegistryFile = ".filebeat" DefaultIgnoreOlderDuration time.Duration = 24 * time.Hour DefaultScanFrequency time.Duration = 10 * time.Second - DefaultSpoolSize uint64 = 1024 + DefaultSpoolSize uint64 = 2048 DefaultIdleTimeout time.Duration = 5 * time.Second DefaultHarvesterBufferSize int = 16 << 10 // 16384 DefaultInputType = "log" diff --git a/libbeat/publisher/async.go b/libbeat/publisher/async.go index efbaa321656f..5f1a89bb1ff1 100644 --- a/libbeat/publisher/async.go +++ b/libbeat/publisher/async.go @@ -17,7 +17,7 @@ type asyncPublisher struct { const ( defaultFlushInterval = 1000 * time.Millisecond // 1s - defaultBulkSize = 200 + defaultBulkSize = 2048 ) func newAsyncPublisher(pub *PublisherType, hwm, bulkHWM int) *asyncPublisher { From 7ef39f639900f6cdda884e59bf3d07361b4b2b09 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 5 Jan 2016 23:24:15 +0100 Subject: [PATCH 2/4] Update changelog - add changelog entries about spool_size and bulk_max_size defaults --- CHANGELOG.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 736160355568..6ae7128a61a0 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -29,6 +29,7 @@ https://github.com/elastic/beats/compare/1.0.0...master[Check the HEAD diff] - Fix logging issue with file based output where newlines could be misplaced during concurrent logging {pull}650[650] - Reduce memory usage by separate queue sizes for single events and bulk events. {pull}649[649] {issue}516[516] +- Set default default bulk_max_size value to 2048 {pull}628[628] *Packetbeat* - Fix setting direction to out and use its value to decide when dropping events if ignore_outgoing is enabled {pull}557[557] @@ -37,6 +38,7 @@ https://github.com/elastic/beats/compare/1.0.0...master[Check the HEAD diff] *Filebeat* - Add exclude_files configuration option {pull}563[563] +- Set spool_size default value to 2048 {pull}628[628] *Winlogbeat* From 97142b47e77ea836ff20a43ad972281468a20cf7 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 5 Jan 2016 23:16:28 +0100 Subject: [PATCH 3/4] Update spool_size default in docs and sample configs --- filebeat/docs/configuration.asciidoc | 2 +- filebeat/etc/beat.yml | 2 +- filebeat/etc/filebeat.yml | 2 +- filebeat/tests/files/config.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filebeat/docs/configuration.asciidoc b/filebeat/docs/configuration.asciidoc index 683cd4267799..768902d5f48c 100644 --- a/filebeat/docs/configuration.asciidoc +++ b/filebeat/docs/configuration.asciidoc @@ -245,7 +245,7 @@ value is exceeded. [source,yaml] ------------------------------------------------------------------------------------- filebeat: - spool_size: 1024 + spool_size: 2048 ------------------------------------------------------------------------------------- diff --git a/filebeat/etc/beat.yml b/filebeat/etc/beat.yml index 17d8bfb269e2..329b4d5cca21 100644 --- a/filebeat/etc/beat.yml +++ b/filebeat/etc/beat.yml @@ -143,7 +143,7 @@ filebeat: # General filebeat configuration options # # Event count spool threshold - forces network flush if exceeded - #spool_size: 1024 + #spool_size: 2048 # Defines how often the spooler is flushed. After idle_timeout the spooler is # Flush even though spool_size is not reached. diff --git a/filebeat/etc/filebeat.yml b/filebeat/etc/filebeat.yml index e72593875f0d..727456612357 100644 --- a/filebeat/etc/filebeat.yml +++ b/filebeat/etc/filebeat.yml @@ -143,7 +143,7 @@ filebeat: # General filebeat configuration options # # Event count spool threshold - forces network flush if exceeded - #spool_size: 1024 + #spool_size: 2048 # Defines how often the spooler is flushed. After idle_timeout the spooler is # Flush even though spool_size is not reached. diff --git a/filebeat/tests/files/config.yml b/filebeat/tests/files/config.yml index cba48cd1d721..2afa45871735 100644 --- a/filebeat/tests/files/config.yml +++ b/filebeat/tests/files/config.yml @@ -25,7 +25,7 @@ filebeat: fields: input: stdin # Paths is not required - spool_size: 1024 + spool_size: 2048 idle_timeout: 5s config_dir: "/prospectorConfigs/" From cb6eea154c56f77959fd05e60f0843b986c9b4bb Mon Sep 17 00:00:00 2001 From: urso Date: Wed, 6 Jan 2016 19:42:19 +0100 Subject: [PATCH 4/4] Fix config test --- filebeat/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/config/config_test.go b/filebeat/config/config_test.go index 4fe2a1014005..c2cbb4b15bb2 100644 --- a/filebeat/config/config_test.go +++ b/filebeat/config/config_test.go @@ -20,7 +20,7 @@ func TestReadConfig(t *testing.T) { assert.Nil(t, err) - assert.Equal(t, uint64(1024), config.Filebeat.SpoolSize) + assert.Equal(t, uint64(DefaultSpoolSize), config.Filebeat.SpoolSize) assert.Equal(t, "/prospectorConfigs/", config.Filebeat.ConfigDir) prospectors := config.Filebeat.Prospectors