Combined updates:

* fix to rsyslog restart
* logrotate duplication in AlmaLinux
* GeoIP MindDB update
* GeoIP rework for Nginx
* Elasticsearch cluster-health prometheus rule
* Sensu status log fix and SELinux fix

roles/base/handlers/main.yml

@@ -20,9 +20,15 @@
     state: restarted
 
 
+- name: "Restart logrotate"
+  service:
+    name:  "rsyslog"
+    state: restarted
+
+
 - name: "Restart rsyslog"
   service:
-    name:  "sshd"
+    name:  "rsyslog"
     state: restarted
 
 

roles/base/tasks/main.yml

@@ -65,7 +65,7 @@
     -  logging is defined
 
 
-- name: "Syslog rotate"
+- name: "Log rotation"
   tags:
     - cfg
     - logging
@@ -76,6 +76,7 @@
     owner: root
     group: root
   when: logrotate is defined
+  notify: "Restart logrotate"
 
 
 - name: "Journald volatile"

roles/base/templates/logrotate.conf.j2

@@ -14,22 +14,6 @@ dateext
 # uncomment this if you want your log files compressed
 {{ (logrotate.compress|default(false)) | ternary("compress","#compress") }}
 
-# RPM packages drop log rotation information into this directory
+# system-specific logs may be also be configured here.
 include /etc/logrotate.d
 
-# no packages own wtmp and btmp -- we'll rotate them here
-/var/log/wtmp {
-    monthly
-    create 0664 root utmp
-	minsize 1M
-    rotate 1
-}
-
-/var/log/btmp {
-    missingok
-    monthly
-    create 0600 root utmp
-    rotate 1
-}
-
-# system-specific logs may be also be configured here.

roles/goaccess/defaults/main.yml

@@ -6,5 +6,5 @@ goaccess:
   url: "https://epi-repository.s3.eu-west-1.amazonaws.com/release/goaccess/goaccess-1.9.3.tar.gz"
 
 GeoLite2:
-  url: "https://epi-repository.s3.eu-west-1.amazonaws.com/release/GeoLite2/GeoLite2-City_20240531.tar.gz"
-  version: "City_20240531"
+  url: "https://epi-repository.s3.eu-west-1.amazonaws.com/release/GeoLite2/GeoLite2-City_20250121.tar.gz"
+  version: "City_20250121"

roles/goaccess/tasks/AlmaLinux.yml

@@ -1,4 +1,4 @@
-- name: "Goaccess requirements: MaxMinDDB"
+- name: "Goaccess requirements: MaxMindDB"
   tags:
     - goaccess
     - pkg

roles/goaccess/tasks/geolite2.yml

@@ -0,0 +1,22 @@
+---
+- name: "GeoLite2 archive"
+  tags:
+    - geoip
+  unarchive:
+    src: "{{ item.url }}"
+    dest: "{{ location }}"
+    remote_src: yes
+  loop_control:
+    label: "{{ item.name }}"
+  loop:
+    - { name: "GeoLite2-City", url: "{{ GeoLite2.url }}" }
+
+
+- name: "GeoLite2 DB link"
+  tags:
+    - geoip
+    - goaccess
+  file:
+    src:  "GeoLite2-{{ GeoLite2.version }}"
+    path: "{{ location }}/GeoLite2"
+    state: link

roles/goaccess/tasks/main.yml

@@ -2,32 +2,11 @@
 - name: "{{ ansible_distribution }}"
   include_tasks: "{{ ansible_distribution }}.yml"
 
-- name: "Goaccess archive [{{ goaccess.url }}]"
-  tags:
-    - goaccess
-  unarchive:
-    src: "{{ goaccess.url }}"
-    dest: /usr/local
-    remote_src: yes
-
-
-- name: "GeoLite2 archive [{{ GeoLite2.url }}]"
-  tags:
-    - geoip
-  unarchive:
-    src: "{{ GeoLite2.url }}"
-    dest: /usr/local/share
-    remote_src: yes
-
 
-- name: "GeoLite2 DB [{{ GeoLite2.version }}]"
-  tags:
-    - geoip
-    - goaccess
-  file:
-    src:  "GeoLite2-{{ GeoLite2.version }}"
-    path: "/usr/local/share/GeoLite2"
-    state: link
+- name: "GeoLite2"
+  vars:
+  - location: "/usr/local/share"
+  include_tasks: "geolite2.yml"
 
 
 - name: "Access DNS"

roles/metrics/templates/prometheus-rules/Readme.md

@@ -18,6 +18,35 @@ No new records indexed for 10 minutes.
       annotations:
         message: "Logging Down"
 
+### Elasticsearh Critical
+
+Elasticsearch cluster is reporting critial error
+
+#### Details
+
+  - alert: Elasticsearh Critical
+    expr: elasticsearch_cluster_health_status{cluster="docker-cluster", color="red", job="elasticsearch"} > 0
+    for: 10m
+    labels:
+      severity: critical
+    annotations:
+      runbook_url: "https://github.com/epimorphics/master-ansible-deployment/blob/main/roles/metrics/templates/prometheus-rules/Readme.md#elasticsearch-critical"
+
+
+### Elasticsearh Warning
+
+Elasticsearch cluster is reporting a problem
+
+#### Details
+
+    - alert: Elasticsearh Warning
+      expr: elasticsearch_cluster_health_status{cluster="docker-cluster", color="yellow", job="elasticsearch"} > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        runbook_url: "https://github.com/epimorphics/master-ansible-deployment/blob/main/roles/metrics/templates/prometheus-rules/Readme.md#elasticsearch-warning"
+
 ### Target Down
 
 #### Details

roles/metrics/templates/prometheus-rules/logging.yml.j2

@@ -12,3 +12,23 @@ groups:
       message: "Logging Down"
 {% endraw %}
       runbook_url: "https://github.com/epimorphics/master-ansible-deployment/blob/main/roles/metrics/templates/prometheus-rules/Readme.md#logging-down"
+  - alert: Elasticsearh Critical
+    expr: elasticsearch_cluster_health_status{cluster="docker-cluster", color="red", job="elasticsearch"} > 0
+    for: 10m
+    labels:
+      severity: critical
+    annotations:
+{%- raw %}
+      message: "Elasticsearch Critical"
+{% endraw %}
+      runbook_url: "https://github.com/epimorphics/master-ansible-deployment/blob/main/roles/metrics/templates/prometheus-rules/Readme.md#elasticsearch-critical"
+  - alert: Elasticsearh Warning
+    expr: elasticsearch_cluster_health_status{cluster="docker-cluster", color="yellow", job="elasticsearch"} > 0
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+{%- raw %}
+      message: "Elasticsearch Warning"
+{% endraw %}
+      runbook_url: "https://github.com/epimorphics/master-ansible-deployment/blob/main/roles/metrics/templates/prometheus-rules/Readme.md#elasticsearch-warning"

roles/sensu/defaults/main.yml

@@ -2,7 +2,6 @@
 tag: "6.11.0"
 
 sensu_backend:
-  config: "/etc/sensu/backend.yml"
   directory: "{{ docker.directory }}/{{ sensu_container.name }}"
   image: "{{ registry.common }}/sensu:{{ tag }}"
   ports:
@@ -16,7 +15,7 @@ sensu_backend:
 sensu_am:
   image: "{{ registry.common }}/sensu-alertmanager"
   tag: "v2.1.3"
-  debug: 1
+  debug: 0
 
 services:
   - "{{ sensu_gui }}"

roles/sensu/handlers/main.yml

@@ -6,3 +6,9 @@
   docker_container:
     name: sensu
     state: absent
+
+
+- name: "Restart logrotate"
+  systemd:
+    name:  "logrotate"
+    state: "restarted"

roles/sensu/tasks/main.yml

@@ -17,6 +17,18 @@
     mode:  "{{ item.mode  | default('0755') }}"
   with_items:
     - { path: "{{ sensu_backend.directory }}" }
+    - { path: "{{ sensu_backend.directory }}/logs" }
+
+
+- name: "Sensu SELinux"
+  tags:
+    - cfg
+  sefcontext:
+    target: "{{ item.path }}"
+    setype: "var_log_t"
+  with_items:
+    - { path: "{{ sensu_backend.directory }}/logs" }
+    - { path: "{{ sensu_backend.directory }}/logs/stats.log" }
 
 
 - name: "Sensu container ({{ tag }})"
@@ -36,6 +48,8 @@
       SENSU_BACKEND_STATE_DIR: "{{ sensu_backend.state }}/sensu-backend"
       SENSU_BACKEND_ETCD_HEARTBEAT_INTERVAL: "1000"
       SENSU_BACKEND_ETCD_ELECTION_TIMEOUT: "5000"
+      SENSU_BACKEND_DISABLE_PLATFORM_METRICS: "true"
+      SENSU_BACKEND_PLATFORM_METRICS_LOG_FILE: "{{ sensu_backend.state }}/logs/stats.log"
     log_driver: "{{ docker.logging.driver }}"
     log_options: "{{ docker.logging.options }}"
     network_mode: "{{ docker.network.mode }}"
@@ -74,6 +88,7 @@
   template:
     src: "logrotate.j2"
     dest: "/etc/logrotate.d/sensu"
+  notify: "Restart logrotate"
 
 
 - name: "Sensu api operational"
@@ -97,11 +112,3 @@
     namespace: "sensu-system"
     name: "{{ sensu_instance.container.Config.Hostname }}"
     state: "absent"
-
-
-- name: "Remove sensu_system namespace"
-  delegate_to: "{{ sensu_server }}"
-  sensu.sensu_go.namespace:
-    auth: "{{ sensu.auth }}"
-    name: "sensu-system"
-    state: "absent"

roles/sensu/templates/logrotate.j2

@@ -1,9 +1,8 @@
-{{ sensu_backend.directory }}/backend-stats.log
+{{ sensu_backend.directory }}/logs/stats.log
 {
-    rotate 7
-    daily
-    missingok
-    dateext
-    copytruncate
-    compress
-}    
+  copytruncate
+  rotate 3
+  hourly
+  missingok
+  compress
+}