From 56684dec5abf1cb24deaeef47c2f3d4c9dffeeca Mon Sep 17 00:00:00 2001
From: yaron-idan <yaronidan@gmail.com>
Date: Wed, 28 Mar 2018 09:20:30 +0300
Subject: [PATCH 1/3] added http-x-forwarded-for field to nginx parser

---
 lib/fluent/plugin/parser_nginx.rb | 2 +-
 test/plugin/test_parser_nginx.rb  | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/fluent/plugin/parser_nginx.rb b/lib/fluent/plugin/parser_nginx.rb
index 8cad4ed322..9ed97b9e7f 100644
--- a/lib/fluent/plugin/parser_nginx.rb
+++ b/lib/fluent/plugin/parser_nginx.rb
@@ -21,7 +21,7 @@ module Plugin
     class NginxParser < RegexpParser
       Plugin.register_parser("nginx", self)
 
-      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<http_x_forwarded_for>[^ ]*))?$/}
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end
diff --git a/test/plugin/test_parser_nginx.rb b/test/plugin/test_parser_nginx.rb
index d2707fe5c7..5d9c7f65f8 100644
--- a/test/plugin/test_parser_nginx.rb
+++ b/test/plugin/test_parser_nginx.rb
@@ -14,7 +14,8 @@ def setup
       'code'    => '200',
       'size'    => '777',
       'referer' => '-',
-      'agent'   => 'Opera/12.0'
+      'agent'   => 'Opera/12.0',
+      'http_x_forwarded_for' => '-'
     }
   end
 
@@ -24,7 +25,7 @@ def create_driver
 
   def test_parse
     d = create_driver
-    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }
@@ -32,7 +33,7 @@ def test_parse
 
   def test_parse_with_empty_included_path
     d = create_driver
-    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected.merge('path' => '/a[ ]b'), record)
     }
@@ -40,7 +41,7 @@ def test_parse_with_empty_included_path
 
   def test_parse_without_http_version
     d = create_driver
-    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0" -') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }

From ea5e2e6e01f6108f184f7e035fa43f4d135849ea Mon Sep 17 00:00:00 2001
From: yaron-idan <yaronidan@gmail.com>
Date: Wed, 4 Apr 2018 21:24:20 +0300
Subject: [PATCH 2/3] make http-x-forwarded-for optional and add test

---
 lib/fluent/plugin/parser_nginx.rb |  2 +-
 test/plugin/test_parser_nginx.rb  | 27 +++++++++++++++++++++++----
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/lib/fluent/plugin/parser_nginx.rb b/lib/fluent/plugin/parser_nginx.rb
index 9ed97b9e7f..3c779f2b9b 100644
--- a/lib/fluent/plugin/parser_nginx.rb
+++ b/lib/fluent/plugin/parser_nginx.rb
@@ -21,7 +21,7 @@ module Plugin
     class NginxParser < RegexpParser
       Plugin.register_parser("nginx", self)
 
-      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<http_x_forwarded_for>[^ ]*))?$/}
+      config_set_default :expression, %q{/^(?<remote>\S*) (?<host>\S*) (?<user>\S*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?:\s+\S*)?)?" (?<code>\S*) (?<size>\S*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>\S+))?)?$/}
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end
diff --git a/test/plugin/test_parser_nginx.rb b/test/plugin/test_parser_nginx.rb
index 5d9c7f65f8..fc06485da4 100644
--- a/test/plugin/test_parser_nginx.rb
+++ b/test/plugin/test_parser_nginx.rb
@@ -14,7 +14,18 @@ def setup
       'code'    => '200',
       'size'    => '777',
       'referer' => '-',
-      'agent'   => 'Opera/12.0',
+      'agent'   => 'Opera/12.0'
+    }
+    @expected_extended = {
+      'remote'               => '127.0.0.1',
+      'host'                 => '192.168.0.1',
+      'user'                 => '-',
+      'method'               => 'GET',
+      'path'                 => '/',
+      'code'                 => '200',
+      'size'                 => '777',
+      'referer'              => '-',
+      'agent'                => 'Opera/12.0',
       'http_x_forwarded_for' => '-'
     }
   end
@@ -25,7 +36,7 @@ def create_driver
 
   def test_parse
     d = create_driver
-    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }
@@ -33,7 +44,7 @@ def test_parse
 
   def test_parse_with_empty_included_path
     d = create_driver
-    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected.merge('path' => '/a[ ]b'), record)
     }
@@ -41,9 +52,17 @@ def test_parse_with_empty_included_path
 
   def test_parse_without_http_version
     d = create_driver
-    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0" -') { |time, record|
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }
   end
+
+  def test_parse_with_http_x_forwarded_for
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected_extended, record)
+    }
+  end
 end

From aa1aa065ad6e8dc114c5cc4199f9e073a51eee0a Mon Sep 17 00:00:00 2001
From: yaron-idan <yaronidan@gmail.com>
Date: Fri, 6 Apr 2018 15:03:07 +0300
Subject: [PATCH 3/3] change back to [^ ]

Signed-off-by: yaron-idan <yaronidan@gmail.com>
---
 lib/fluent/plugin/parser_nginx.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_nginx.rb b/lib/fluent/plugin/parser_nginx.rb
index 3c779f2b9b..c9cc67f80a 100644
--- a/lib/fluent/plugin/parser_nginx.rb
+++ b/lib/fluent/plugin/parser_nginx.rb
@@ -21,7 +21,7 @@ module Plugin
     class NginxParser < RegexpParser
       Plugin.register_parser("nginx", self)
 
-      config_set_default :expression, %q{/^(?<remote>\S*) (?<host>\S*) (?<user>\S*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?:\s+\S*)?)?" (?<code>\S*) (?<size>\S*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>\S+))?)?$/}
+      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>[^ ]+))?)?$/}
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end