From 38830b7a0caad2ea10601a57ca7dfd65f16f9708 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Tue, 7 Mar 2017 14:50:21 +0900
Subject: [PATCH 01/11] parser_syslog: Add rfc5424 regexp

---
 lib/fluent/plugin/parser_syslog.rb |  9 +++++++-
 test/plugin/test_parser_syslog.rb  | 35 ++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index ace75a97b9..d9accf3210 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -27,9 +27,11 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$\z/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424], default: :rfc3164
 
       def initialize
         super
@@ -39,7 +41,12 @@ def initialize
       def configure(conf)
         super
 
-        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+        @regexp = case @message_format
+                  when :rfc3164
+                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                  when :rfc5424
+                    REGEXP_RFC5424
+                  end
         @time_parser = time_parser_create
       end
 
diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index a5b06c1543..c1242ff701 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -63,4 +63,39 @@ def test_parse_with_keep_time_key
       assert_equal "Feb 28 00:00:12", record['time']
     end
   end
+
+  class TestRFC5424Regexp < self
+    def test_parse_with_rfc5424_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'keep_time_key'=>'true',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_structured_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'keep_time_key'=>'true',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+  end
 end

From 9f9f747614c1aa9946ff7ef5aa61350de7efc8af Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Tue, 7 Mar 2017 16:11:45 +0900
Subject: [PATCH 02/11] Add native implementation for detecting rfc3164 and
 rfc5424 format

---
 lib/fluent/plugin/parser_syslog.rb | 12 +++++-
 test/plugin/test_parser_syslog.rb  | 61 ++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index d9accf3210..34cde349ea 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -31,7 +31,7 @@ class SyslogParser < Parser
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
-      config_param :message_format, :enum, list: [:rfc3164, :rfc5424], default: :rfc3164
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
 
       def initialize
         super
@@ -46,6 +46,8 @@ def configure(conf)
                     @with_priority ? REGEXP_WITH_PRI : REGEXP
                   when :rfc5424
                     REGEXP_RFC5424
+                  when :auto
+                    nil
                   end
         @time_parser = time_parser_create
       end
@@ -55,6 +57,14 @@ def patterns
       end
 
       def parse(text)
+        if @regexp.nil? and (@message_format == :auto)
+          if REGEXP_RFC5424.match(text)
+            @regexp = REGEXP_RFC5424
+          else
+            @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          end
+        end
+
         m = @regexp.match(text)
         unless m
           yield nil, nil
diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index c1242ff701..94d22c75dc 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -98,4 +98,65 @@ def test_parse_with_rfc5424_structured_message
       end
     end
   end
+
+  class TestAutoRegexp < self
+    def test_auto_with_legacy_syslog_message
+      @parser.configure(
+                        'time_format' => '%b %d %M:%S:%H',
+                        'keep_time_key'=>'true',
+                        'mseeage_format' => 'auto',
+                        )
+      text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(@expected.merge('time' => 'Feb 28 00:00:12'), record)
+      end
+    end
+
+    def test_auto_with_legacy_syslog_priority_message
+      @parser.configure(
+                        'time_format' => '%b %d %M:%S:%H',
+                        'keep_time_key'=>'true',
+                        'with_priority' => true,
+                        'mseeage_format' => 'auto',
+                        )
+      text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal "Feb 28 12:00:00", record['time']
+        assert_equal(@expected.merge('time' => 'Feb 28 12:00:00', 'pri' => 6), record)
+      end
+    end
+
+    def test_parse_with_rfc5424_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'keep_time_key'=>'true',
+                        'message_format' => 'auto',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_structured_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'keep_time_key'=>'true',
+                        'message_format' => 'auto',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+  end
 end

From 84ea64c1fea3aa37bacd96f5f855023b9c1d73ca Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Tue, 7 Mar 2017 17:23:23 +0900
Subject: [PATCH 03/11] Use parsed time in assertions instead of kept time
 record

---
 test/plugin/test_parser_syslog.rb | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index 94d22c75dc..9be449665d 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -68,12 +68,11 @@ class TestRFC5424Regexp < self
     def test_parse_with_rfc5424_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-                        'keep_time_key'=>'true',
                         'message_format' => 'rfc5424',
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
-        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
         assert_equal "-", record["pid"]
         assert_equal "-", record["msgid"]
         assert_equal "-", record["extradata"]
@@ -84,13 +83,12 @@ def test_parse_with_rfc5424_message
     def test_parse_with_rfc5424_structured_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-                        'keep_time_key'=>'true',
                         'message_format' => 'rfc5424',
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
-        assert_equal "2017-02-06T13:14:15.003Z", record['time']
-        assert_equal "11111", record["pid"]
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
                      record["extradata"]
@@ -103,38 +101,36 @@ class TestAutoRegexp < self
     def test_auto_with_legacy_syslog_message
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
-                        'keep_time_key'=>'true',
                         'mseeage_format' => 'auto',
                         )
       text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
-        assert_equal(@expected.merge('time' => 'Feb 28 00:00:12'), record)
+        assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected, record)
       end
     end
 
     def test_auto_with_legacy_syslog_priority_message
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
-                        'keep_time_key'=>'true',
                         'with_priority' => true,
                         'mseeage_format' => 'auto',
                         )
       text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
-        assert_equal "Feb 28 12:00:00", record['time']
-        assert_equal(@expected.merge('time' => 'Feb 28 12:00:00', 'pri' => 6), record)
+        assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected.merge('pri' => 6), record)
       end
     end
 
     def test_parse_with_rfc5424_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-                        'keep_time_key'=>'true',
                         'message_format' => 'auto',
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
-        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
         assert_equal "-", record["pid"]
         assert_equal "-", record["msgid"]
         assert_equal "-", record["extradata"]
@@ -145,12 +141,11 @@ def test_parse_with_rfc5424_message
     def test_parse_with_rfc5424_structured_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-                        'keep_time_key'=>'true',
                         'message_format' => 'auto',
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
-        assert_equal "2017-02-06T13:14:15.003Z", record['time']
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
         assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",

From 0918deb6157bb8def111831c14528a8bbcc076b7 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Tue, 7 Mar 2017 17:54:04 +0900
Subject: [PATCH 04/11] Use more strict regex for rfc5424

rfc5424 says:

```log
"<" PRI ">" VERSION ...

PRI             = "<" PRIVAL ">"
PRIVAL          = 1*3DIGIT ; range 0 .. 191
VERSION = NONZERO-DIGIT 0*2DIGIT
<snip>
```

So, we shouldn't supporse like <192>1.
---
 lib/fluent/plugin/parser_syslog.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 34cde349ea..fe78181436 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -27,7 +27,7 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$\z/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$\z/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false

From f36a662e49a011577a79bcd9657de722c765019c Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Tue, 7 Mar 2017 18:13:50 +0900
Subject: [PATCH 05/11] Loosen regexp in extradata part

---
 lib/fluent/plugin/parser_syslog.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index fe78181436..9c2dde7531 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -27,7 +27,7 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$\z/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false

From b0e9b73708dbf73f2b2130d93e8c47e4ee88a535 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Tue, 7 Mar 2017 19:26:29 +0900
Subject: [PATCH 06/11] Implement more flexible syslog format detection

---
 lib/fluent/plugin/parser_syslog.rb |  5 +++--
 test/plugin/test_parser_syslog.rb  | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 9c2dde7531..519e73ef5d 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -28,6 +28,7 @@ class SyslogParser < Parser
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
@@ -57,8 +58,8 @@ def patterns
       end
 
       def parse(text)
-        if @regexp.nil? and (@message_format == :auto)
-          if REGEXP_RFC5424.match(text)
+        if @message_format == :auto
+          if REGEXP_DETECT_RFC5424.match(text)
             @regexp = REGEXP_RFC5424
           else
             @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index 9be449665d..0fe11c880c 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -153,5 +153,25 @@ def test_parse_with_rfc5424_structured_message
         assert_equal "Hi, from Fluentd!", record["message"]
       end
     end
+
+    def test_parse_with_both_message_type
+      @parser.configure(
+                        'time_format' => nil,
+                        'with_priority' => true,
+                        'message_format' => 'auto',
+                        )
+      text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(@expected.merge('pri' => 6), record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
   end
 end

From b33d74393b1b5eafe26ac40d1c09c376c4ab54e6 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Wed, 8 Mar 2017 10:14:23 +0900
Subject: [PATCH 07/11] Re-create time_parser when auto choosed

---
 lib/fluent/plugin/parser_syslog.rb | 3 +++
 test/plugin/test_parser_syslog.rb  | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 519e73ef5d..f1050d2846 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -33,6 +33,7 @@ class SyslogParser < Parser
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
       config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize
         super
@@ -61,8 +62,10 @@ def parse(text)
         if @message_format == :auto
           if REGEXP_DETECT_RFC5424.match(text)
             @regexp = REGEXP_RFC5424
+            @time_parser = time_parser_create(format: @rfc5424_time_format)
           else
             @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+            @time_parser = time_parser_create(format: @time_format)
           end
         end
 
diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index 0fe11c880c..350b43263d 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -156,16 +156,19 @@ def test_parse_with_rfc5424_structured_message
 
     def test_parse_with_both_message_type
       @parser.configure(
-                        'time_format' => nil,
+                        'time_format' => '%b %d %M:%S:%H',
+                        'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'with_priority' => true,
                         'message_format' => 'auto',
                         )
       text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       end
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
         assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",

From ac1496dba97447a36e17127a986ee0b35fc058a5 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Wed, 8 Mar 2017 11:32:45 +0900
Subject: [PATCH 08/11] Use more complex test case for auto format detection
 test

---
 test/plugin/test_parser_syslog.rb | 48 +++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index 350b43263d..e031990b05 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -155,6 +155,41 @@ def test_parse_with_rfc5424_structured_message
     end
 
     def test_parse_with_both_message_type
+      @parser.configure(
+        'time_format' => '%b %d %M:%S:%H',
+        'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+        'message_format' => 'auto',
+      )
+      text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected, record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+      text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected, record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_both_message_type_and_priority
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
                         'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
@@ -175,6 +210,19 @@ def test_parse_with_both_message_type
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected.merge('pri' => 16), record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
     end
   end
 end

From d71c39b7183acc0174362aec77a2b21ca3f22a1b Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Wed, 8 Mar 2017 11:47:42 +0900
Subject: [PATCH 09/11] Forward appropriate methods

---
 lib/fluent/plugin/parser_syslog.rb | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index f1050d2846..0bf8870d45 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -45,10 +45,19 @@ def configure(conf)
 
         @regexp = case @message_format
                   when :rfc3164
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
                     @with_priority ? REGEXP_WITH_PRI : REGEXP
                   when :rfc5424
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
                     REGEXP_RFC5424
                   when :auto
+                    class << self
+                      alias_method :parse, :parse_auto
+                    end
                     nil
                   end
         @time_parser = time_parser_create
@@ -58,7 +67,7 @@ def patterns
         {'format' => @regexp, 'time_format' => @time_format}
       end
 
-      def parse(text)
+      def parse_auto(text, &block)
         if @message_format == :auto
           if REGEXP_DETECT_RFC5424.match(text)
             @regexp = REGEXP_RFC5424
@@ -68,7 +77,10 @@ def parse(text)
             @time_parser = time_parser_create(format: @time_format)
           end
         end
+        parse_plain(text, &block)
+      end
 
+      def parse_plain(text, &block)
         m = @regexp.match(text)
         unless m
           yield nil, nil

From 5ba24adb94d38c603ad7fee88a943c3ad2f732c3 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Wed, 8 Mar 2017 12:57:55 +0900
Subject: [PATCH 10/11] Create time parsers in #configure phase

---
 lib/fluent/plugin/parser_syslog.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 0bf8870d45..083c25d0dc 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -43,6 +43,7 @@ def initialize
       def configure(conf)
         super
 
+        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
         @regexp = case @message_format
                   when :rfc3164
                     class << self
@@ -58,6 +59,8 @@ class << self
                     class << self
                       alias_method :parse, :parse_auto
                     end
+                    @time_parser_rfc3164 = time_parser_create(format: @time_format)
+                    @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
                     nil
                   end
         @time_parser = time_parser_create
@@ -71,10 +74,10 @@ def parse_auto(text, &block)
         if @message_format == :auto
           if REGEXP_DETECT_RFC5424.match(text)
             @regexp = REGEXP_RFC5424
-            @time_parser = time_parser_create(format: @rfc5424_time_format)
+            @time_parser = @time_parser_rfc5424
           else
             @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
-            @time_parser = time_parser_create(format: @time_format)
+            @time_parser = @time_parser_rfc3164
           end
         end
         parse_plain(text, &block)

From 866b2caf0891b805fca54efff6c28e10cd617f35 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Wed, 8 Mar 2017 12:58:51 +0900
Subject: [PATCH 11/11] Keep #parse for method check safety

---
 lib/fluent/plugin/parser_syslog.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 083c25d0dc..8a984888ea 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -70,6 +70,10 @@ def patterns
         {'format' => @regexp, 'time_format' => @time_format}
       end
 
+      def parse(text)
+        # This is overwritten in configure
+      end
+
       def parse_auto(text, &block)
         if @message_format == :auto
           if REGEXP_DETECT_RFC5424.match(text)