Add support for client addresses with port in Apache error logs (#12695)

Client addresses can contain the port in the default log format, add support for this.
elastic · Jul 1, 2019 · fa8c467 · fa8c467
1 parent 66caea4
commit fa8c467
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 3 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -210,6 +210,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add MSSQL module {pull}12079[12079]
 - Add ISO8601 date parsing support for system module. {pull}12568[12568] {pull}12578[12579]
 - Use correct OS path separator in `add_kubernetes_metadata` to support Windows nodes. {pull}9205[9205]
+- Add support for client addresses with port in Apache error logs {pull}12695[12695]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json
@@ -5,8 +5,8 @@
       "grok": {
         "field": "message",
         "patterns": [
-          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}\\])? %{GREEDYDATA:message}",
-          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}\\])? %{GREEDYDATA:message}"
+          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}",
+          "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"
         ],
         "pattern_definitions": {
           "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"

diff --git a/filebeat/module/apache/error/test/test.log b/filebeat/module/apache/error/test/test.log
@@ -1,3 +1,4 @@
 [Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico
 [Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'
 [Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico
+[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html
diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json
@@ -47,5 +47,28 @@
         "source.geo.region_iso_code": "US-GA",
         "source.geo.region_name": "Georgia",
         "source.ip": "72.15.99.187"
+    },
+    {
+        "@timestamp": "2019-06-27T06:58:09.169Z",
+        "apache.error.module": "include",
+        "event.dataset": "apache.error",
+        "event.module": "apache",
+        "fileset.name": "error",
+        "input.type": "log",
+        "log.level": "warn",
+        "log.offset": 384,
+        "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html",
+        "process.pid": 15934,
+        "service.type": "apache",
+        "source.address": "123.123.123.123",
+        "source.geo.city_name": "Beijing",
+        "source.geo.continent_name": "Asia",
+        "source.geo.country_iso_code": "CN",
+        "source.geo.location.lat": 39.9288,
+        "source.geo.location.lon": 116.3889,
+        "source.geo.region_iso_code": "CN-BJ",
+        "source.geo.region_name": "Beijing",
+        "source.ip": "123.123.123.123",
+        "source.port": "12345"
     }
-]
+]