From 9e6ad1f70e0eb92a8717ae2d717df31adce3ee77 Mon Sep 17 00:00:00 2001 From: Alejandro Santoyo Date: Tue, 6 Feb 2024 15:38:33 +0100 Subject: [PATCH] Fixing filebeat holding onto old file descriptors Under certain scenariosFilebeat may hold onto old log file descriptors. There are various settings on Filebeat to prevent this from happening. This patch adds to the charm three key settings that help prevent this issue: - ignore_older - clean_inactive Fixing: [#87](https://github.com/canonical/layer-filebeat/issues/87) --- config.yaml | 17 +++++++++++++ reactive/filebeat.py | 52 +++++++++++++++++++++++++++++++++++++++- templates/filebeat-5.yml | 4 ++++ templates/filebeat-6.yml | 4 ++++ templates/filebeat-7.yml | 2 ++ 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index a27003b..0d82f9e 100644 --- a/config.yaml +++ b/config.yaml @@ -49,3 +49,20 @@ options: default: "" description: | A YAML list which will be injected to define additional prospectors/inputs. + ignore_older: + type: string + default: "0" + description: | + If this option is enabled, Filebeat ignores any files that were modified before the specified timespan. + Configuring ignore_older can be especially useful if you keep log files for a long time. The passed + values should be integers, e.g., "90", "3600", "600". The values are assumed to be in seconds + https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-ignore-older" + clean_inactive: + type: string + default: "0" + description: | + When this option is enabled, Filebeat removes the state of a file after the specified period of inactivity has + elapsed. Bear in mind that if clean_inactive is enabled, it must be greater than ignore_older + scan_frequency to + make sure that no states are removed while a file is still being harvested. The passed values should be integers, + e.g., "90", "3600", "600". The values are assumed to be in seconds: + https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-clean-inactive" diff --git a/reactive/filebeat.py b/reactive/filebeat.py index fd01f6a..fedb79e 100644 --- a/reactive/filebeat.py +++ b/reactive/filebeat.py @@ -193,7 +193,57 @@ def check_filebeat_repo(): remove_state("filebeat.repo.changed") -@hook("stop") +@when('apt.installed.filebeat') +@when('config.changed') +def sanity_check(): + config_items = { + 'ignore_older': config().get('ignore_older'), + 'clean_inactive': config().get('clean_inactive') + } + + log("Executing sanity checks on config values.") + + # sanity check config strings + for cfg, value in config_items.items(): + if is_valid_time_string(value) is None: + log("Invalid config, the value for {} is not an integer." + "Passed value: {}.".format(cfg, value)) + #status.blocked("Invalid config. Please check the logs for details.") + status_set("blocked", "Invalid config. Please check the logs for details.") + return + + # sanity check value compliance (e.g., clean_inactive must be greater + # than ignore_older + scan_frequency). scan_frequency is no longer a + # charm config option but it defaults to 10s and it's set to that in + # the charm templates so using that below. + scan_frequency = 10 + ignore_older = int(config_items['ignore_older']) + clean_inactive = int(config_items['clean_inactive']) + + if clean_inactive and \ + int(clean_inactive) <= int(ignore_older) + int(scan_frequency): + log("Invalid config, make sure that the value for 'clean_inactive' " + "is greater than 'ignore_older + scan_frequency'.") + #status.blocked("Invalid config. Please check the logs for details.") + status_set("blocked", "Invalid config. Please check the logs for details.") + return + + log("Charm config sanity checks passed. Restarting Filebeat") + service("restart", "filebeat") + + +def is_valid_time_string(str_in): + """ + Returns str_in as integer if possible, logs an error and returns + None otherwise. + """ + try: + return int(str_in) + except (ValueError, TypeError): + return None + + +@hook('stop') def remove_filebeat(): """Stop, purge, and remove all traces of filebeat.""" status.maint("Removing filebeat.") diff --git a/templates/filebeat-5.yml b/templates/filebeat-5.yml index 3542d6c..4d94139 100644 --- a/templates/filebeat-5.yml +++ b/templates/filebeat-5.yml @@ -16,6 +16,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields: juju_model_name: {{ juju_model_name }} juju_model_uuid: {{ juju_model_uuid }} @@ -34,6 +36,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: true symlinks: true # NB: disable json decoding; setting any of these would result in the diff --git a/templates/filebeat-6.yml b/templates/filebeat-6.yml index 34f48b1..09b3390 100644 --- a/templates/filebeat-6.yml +++ b/templates/filebeat-6.yml @@ -17,6 +17,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: false fields: type: logpath-logs @@ -39,6 +41,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: false fields: type: container-logs diff --git a/templates/filebeat-7.yml b/templates/filebeat-7.yml index 8031f82..912b89b 100644 --- a/templates/filebeat-7.yml +++ b/templates/filebeat-7.yml @@ -16,6 +16,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: false fields: type: logpath-logs