From 06c4c76f4d3265fe92e94e1a388124433975e61f Mon Sep 17 00:00:00 2001
From: ruflin <spam@ruflin.com>
Date: Wed, 30 May 2018 15:20:04 +0200
Subject: [PATCH] Add dissect to traefik/access metricset for first stage of
 processing

This should provide an example use case on how grok and dissect can be combined. The resulting outcome should be identical. The main difference is that now some of the fields can be used on the Beats side for filtering out events.
---
 CHANGELOG.asciidoc                            |   1 +
 .../traefik/access/config/traefik-access.yml  |   9 ++
 .../traefik/access/ingest/pipeline.json       | 112 ++++++++++--------
 3 files changed, 75 insertions(+), 47 deletions(-)

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 973b0aa4db30..7da6fc0f29e6 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -177,6 +177,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Support MySQL 5.7.19 by mysql/slowlog {pull}6969[6969]
 - Correctly join partial log lines when using `docker` input. {pull}6967[6967]
 - Add support for TLS with client authentication to the TCP input {pull}7056[7056]
+- Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
 
 *Heartbeat*
 
diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml
index 0afd17317d4f..2df383f52fd7 100644
--- a/filebeat/module/traefik/access/config/traefik-access.yml
+++ b/filebeat/module/traefik/access/config/traefik-access.yml
@@ -4,3 +4,12 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+processors:
+- dissect:
+    tokenizer: '%{traefik.access.remote_ip} - %{traefik.access.user_name} [%{traefik.access.time}]
+    "%{traefik.access.method} %{traefik.access.url} HTTP/%{traefik.access.http_version}"
+    %{traefik.access.response_code} %{traefik.access.message}'
+
+    field: "message"
+    target_prefix: ""
diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json
index ec9209abe5c8..8e95385c14bb 100644
--- a/filebeat/module/traefik/access/ingest/pipeline.json
+++ b/filebeat/module/traefik/access/ingest/pipeline.json
@@ -1,52 +1,70 @@
 {
     "description": "Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins.",
-    "processors": [{
-        "grok": {
-            "field": "message",
-            "patterns":[
-                "%{IPORHOST:traefik.access.remote_ip} - %{DATA:traefik.access.user_name} \\[%{HTTPDATE:traefik.access.time}\\] \"%{WORD:traefik.access.method} %{DATA:traefik.access.url} HTTP/%{NUMBER:traefik.access.http_version}\" %{NUMBER:traefik.access.response_code} (?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?"
-            ],
-            "ignore_missing": true
+    "processors": [
+        {
+            "grok": {
+                "field": "traefik.access.message",
+                "patterns": [
+                    "(?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?"
+                ],
+                "ignore_missing": true
+            }
+        },
+        {
+            "remove": {
+                "field": "message"
+            }
+        },
+        {
+            "remove": {
+                "field": "traefik.access.message"
+            }
+        },
+        {
+            "rename": {
+                "field": "@timestamp",
+                "target_field": "read_timestamp"
+            }
+        },
+        {
+            "date": {
+                "field": "traefik.access.time",
+                "target_field": "@timestamp",
+                "formats": [
+                    "dd/MMM/YYYY:H:m:s Z"
+                ]
+            }
+        },
+        {
+            "remove": {
+                "field": "traefik.access.time"
+            }
+        },
+        {
+            "user_agent": {
+                "field": "traefik.access.agent",
+                "target_field": "traefik.access.user_agent",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "traefik.access.agent"
+            }
+        },
+        {
+            "geoip": {
+                "field": "traefik.access.remote_ip",
+                "target_field": "traefik.access.geoip"
+            }
         }
-    },{
-        "remove":{
-            "field": "message"
+    ],
+    "on_failure": [
+        {
+            "set": {
+                "field": "error.message",
+                "value": "{{ _ingest.on_failure_message }}"
+            }
         }
-    }, {
-        "rename": {
-            "field": "@timestamp",
-            "target_field": "read_timestamp"
-        }
-    }, {
-        "date": {
-            "field": "traefik.access.time",
-            "target_field": "@timestamp",
-            "formats": ["dd/MMM/YYYY:H:m:s Z"]
-        }
-    }, {
-        "remove": {
-            "field": "traefik.access.time"
-        }
-    }, {
-        "user_agent": {
-            "field": "traefik.access.agent",
-            "target_field": "traefik.access.user_agent",
-            "ignore_failure": true
-        }
-    }, {
-        "remove": {
-            "field": "traefik.access.agent"
-        }
-    }, {
-        "geoip": {
-            "field": "traefik.access.remote_ip",
-            "target_field": "traefik.access.geoip"
-        }
-    }],
-    "on_failure" : [{
-        "set" : {
-            "field" : "error.message",
-            "value" : "{{ _ingest.on_failure_message }}"
-        }
-    }]
+    ]
 }