From 08e12d163ff3e8993ae03e1371afa2f2144f7308 Mon Sep 17 00:00:00 2001 From: Alejandro Santoyo Date: Tue, 6 Feb 2024 15:38:33 +0100 Subject: [PATCH] Fixing filebeat holding onto old file descriptors Under certain scenariosFilebeat may hold onto old log file descriptors. There are various settings on Filebeat to prevent this from happening. This patch adds to the charm three key settings that help prevent this issue: - ignore_older - clean_inactive Fixing: [#87](https://github.com/canonical/layer-filebeat/issues/87) --- config.yaml | 17 +++++++++++++++++ reactive/filebeat.py | 34 +++++++++++++++++++++++++++++++++- templates/filebeat-5.yml | 4 ++++ templates/filebeat-6.yml | 4 ++++ templates/filebeat-7.yml | 2 ++ 5 files changed, 60 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index a27003b..fdc33bc 100644 --- a/config.yaml +++ b/config.yaml @@ -49,3 +49,20 @@ options: default: "" description: | A YAML list which will be injected to define additional prospectors/inputs. + ignore_older: + type: int + default: 0 + description: | + If this option is enabled, Filebeat ignores any files that were modified before the specified timespan. + Configuring ignore_older can be especially useful if you keep log files for a long time. The passed + values should be integers, e.g., "90", "3600", "600". The values are assumed to be in seconds + https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-ignore-older" + clean_inactive: + type: int + default: 0 + description: | + When this option is enabled, Filebeat removes the state of a file after the specified period of inactivity has + elapsed. Bear in mind that if clean_inactive is enabled, it must be greater than ignore_older + scan_frequency to + make sure that no states are removed while a file is still being harvested. The passed values should be integers, + e.g., "90", "3600", "600". The values are assumed to be in seconds: + https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-clean-inactive" diff --git a/reactive/filebeat.py b/reactive/filebeat.py index fd01f6a..4e6c534 100644 --- a/reactive/filebeat.py +++ b/reactive/filebeat.py @@ -76,6 +76,10 @@ def render_filebeat_template(): ) log(msg) + # run the sanity checks, abort the rendering if config is not valid + if not is_config_valid(): + return + # The v5 template is compatible with all versions < 6 major = charms.apt.get_package_version("filebeat")[0] version = major if major.isdigit() and int(major) > 5 else "5" @@ -193,7 +197,35 @@ def check_filebeat_repo(): remove_state("filebeat.repo.changed") -@hook("stop") +@when("apt.installed.filebeat") +@when('config.changed') +def is_config_valid() -> bool: + """Sanity check config options. + + :return: True if all checks pass, False otherwise + """ + log("Executing sanity checks on config values.") + + # sanity check value compliance (e.g., clean_inactive must be greater + # than ignore_older + scan_frequency). scan_frequency is no longer a + # charm config option but it defaults to 10s and it's set to that in + # the charm templates so using that below. + scan_frequency = 10 + ignore_older = config().get('ignore_older') + clean_inactive = config().get('clean_inactive') + + if clean_inactive and clean_inactive <= ignore_older + scan_frequency: + log("Invalid config, make sure that the value for 'clean_inactive' " + "is greater than 'ignore_older + scan_frequency'.") + status.blocked("Invalid config. Please check the logs for details.") + return False + + log("Charm config sanity checks passed. Restarting Filebeat") + service("restart", "filebeat") + return True + + +@hook('stop') def remove_filebeat(): """Stop, purge, and remove all traces of filebeat.""" status.maint("Removing filebeat.") diff --git a/templates/filebeat-5.yml b/templates/filebeat-5.yml index 3542d6c..4d94139 100644 --- a/templates/filebeat-5.yml +++ b/templates/filebeat-5.yml @@ -16,6 +16,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields: juju_model_name: {{ juju_model_name }} juju_model_uuid: {{ juju_model_uuid }} @@ -34,6 +36,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: true symlinks: true # NB: disable json decoding; setting any of these would result in the diff --git a/templates/filebeat-6.yml b/templates/filebeat-6.yml index 34f48b1..09b3390 100644 --- a/templates/filebeat-6.yml +++ b/templates/filebeat-6.yml @@ -17,6 +17,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: false fields: type: logpath-logs @@ -39,6 +41,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: false fields: type: container-logs diff --git a/templates/filebeat-7.yml b/templates/filebeat-7.yml index 8031f82..912b89b 100644 --- a/templates/filebeat-7.yml +++ b/templates/filebeat-7.yml @@ -16,6 +16,8 @@ filebeat: scan_frequency: 10s harvester_buffer_size: {{ harvester_buffer_size }} max_bytes: {{ max_bytes }} + ignore_older: {{ ignore_older }} + clean_inactive: {{ clean_inactive }} fields_under_root: false fields: type: logpath-logs