Minor tweaks to logstash workers and pipeline size to be a little mor…

…e conservative to avoid logstash crashes due to heap issues
cisagov · Mar 17, 2022 · fd8adb9 · fd8adb9
1 parent 1678272
commit fd8adb9
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 7 deletions.
diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
@@ -98,8 +98,8 @@ x-dashboards-helper-variables: &dashboards-helper-variables
   ISM_SNAPSHOT_REPO : 'logs'
 
 x-logstash-variables: &logstash-variables
-  pipeline.workers : 4
-  pipeline.batch.size : 100
+  pipeline.workers : 3
+  pipeline.batch.size : 75
   pipeline.batch.delay : 50
   LOGSTASH_OUI_LOOKUP       : 'true'
   LOGSTASH_SEVERITY_SCORING : 'true'

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -98,8 +98,8 @@ x-dashboards-helper-variables: &dashboards-helper-variables
   ISM_SNAPSHOT_REPO : 'logs'
 
 x-logstash-variables: &logstash-variables
-  pipeline.workers : 4
-  pipeline.batch.size : 100
+  pipeline.workers : 3
+  pipeline.batch.size : 75
   pipeline.batch.delay : 50
   LOGSTASH_OUI_LOOKUP       : 'true'
   LOGSTASH_SEVERITY_SCORING : 'true'

diff --git a/scripts/install.py b/scripts/install.py
@@ -303,11 +303,11 @@ def tweak_malcolm_runtime(
         # - https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
         # we don't want it too high, as in Malcolm Logstash also competes with OpenSearch, etc. for resources
         if self.totalCores > 16:
-            lsWorkers = 12
+            lsWorkers = 10
         elif self.totalCores >= 12:
-            lsWorkers = 8
+            lsWorkers = 6
         else:
-            lsWorkers = 4
+            lsWorkers = 3
 
         while not InstallerYesOrNo(
             f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?', default=True