From cc1200b2c20f62f403295fa5fff44a012e4e11ba Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 11 Jul 2016 15:06:06 +0900
Subject: [PATCH 01/10] Extract Fluent::Plugin::RegexpParser

We can use Fluent::Plugin::RegexpParser as following:

```
<parse>
  @type regexp
  expression /pattern.+/
</parse>
```
---
 lib/fluent/compat/parser.rb              | 17 ++++++
 lib/fluent/plugin/parser.rb              | 69 ------------------------
 lib/fluent/plugin/parser_apache.rb       |  2 +-
 lib/fluent/plugin/parser_apache_error.rb |  2 +-
 lib/fluent/plugin/parser_multiline.rb    |  6 +--
 lib/fluent/plugin/parser_nginx.rb        |  2 +-
 lib/fluent/plugin/parser_regexp.rb       | 68 +++++++++++++++++++++++
 test/plugin/test_parser_regexp.rb        | 16 +++---
 8 files changed, 99 insertions(+), 83 deletions(-)
 create mode 100644 lib/fluent/plugin/parser_regexp.rb

diff --git a/lib/fluent/compat/parser.rb b/lib/fluent/compat/parser.rb
index af9bd61736..836ee1eeb9 100644
--- a/lib/fluent/compat/parser.rb
+++ b/lib/fluent/compat/parser.rb
@@ -18,6 +18,7 @@
 require 'fluent/plugin/parser'
 require 'fluent/mixin'
 
+require 'fluent/plugin/parser_regexp'
 require 'fluent/plugin/parser_json'
 require 'fluent/plugin/parser_tsv'
 require 'fluent/plugin/parser_ltsv'
@@ -121,6 +122,22 @@ class TimeParser < Fluent::Plugin::Parser::TimeParser
 
       class RegexpParser < Fluent::Plugin::RegexpParser
         # TODO: warn when deprecated
+        def initialize(regexp, conf = {})
+          super()
+
+          unless conf.empty?
+            unless conf.is_a?(Config::Element)
+              conf = Config::Element.new('default_regexp_conf', '', conf, [])
+            end
+            configure(conf)
+          end
+
+          @regexp = regexp
+        end
+
+        def patterns
+          {'format' => @regexp, 'time_format' => @time_format}
+        end
       end
 
       class ValuesParser < Fluent::Plugin::ValuesParser
diff --git a/lib/fluent/plugin/parser.rb b/lib/fluent/plugin/parser.rb
index b3f085adc0..8faf713869 100644
--- a/lib/fluent/plugin/parser.rb
+++ b/lib/fluent/plugin/parser.rb
@@ -97,75 +97,6 @@ def parse(value)
       end
     end
 
-    class RegexpParser < Parser
-      include Fluent::TypeConverter
-
-      config_param :time_key, :string, default: 'time'
-      config_param :time_format, :string, default: nil
-
-      def initialize(regexp, conf={})
-        super()
-
-        unless conf.empty?
-          unless conf.is_a?(Config::Element)
-            conf = Config::Element.new('default_regexp_conf', '', conf, [])
-          end
-          configure(conf)
-        end
-
-        @regexp = regexp
-        @time_parser = TimeParser.new(@time_format)
-        @mutex = Mutex.new
-      end
-
-      def configure(conf)
-        super
-        @time_parser = TimeParser.new(@time_format)
-      end
-
-      def patterns
-        {'format' => @regexp, 'time_format' => @time_format}
-      end
-
-      def parse(text)
-        m = @regexp.match(text)
-        unless m
-          yield nil, nil
-          return
-        end
-
-        time = nil
-        record = {}
-
-        m.names.each do |name|
-          if value = m[name]
-            if name == @time_key
-              time = @mutex.synchronize { @time_parser.parse(value) }
-              if @keep_time_key
-                record[name] = if @type_converters.nil?
-                                 value
-                               else
-                                 convert_type(name, value)
-                               end
-              end
-            else
-              record[name] = if @type_converters.nil?
-                               value
-                             else
-                               convert_type(name, value)
-                             end
-            end
-          end
-        end
-
-        if @estimate_current_event
-          time ||= Fluent::EventTime.now
-        end
-
-        yield time, record
-      end
-    end
-
     class ValuesParser < Parser
       include Fluent::TypeConverter
 
diff --git a/lib/fluent/plugin/parser_apache.rb b/lib/fluent/plugin/parser_apache.rb
index 607933a784..e33b55659a 100644
--- a/lib/fluent/plugin/parser_apache.rb
+++ b/lib/fluent/plugin/parser_apache.rb
@@ -17,7 +17,7 @@
 require 'fluent/plugin/parser'
 
 Fluent::Plugin.register_parser('apache', Proc.new{
-  Fluent::Plugin::RegexpParser.new(
+  Fluent::Compat::TextParser::RegexpParser.new(
     /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,
     {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}
   )
diff --git a/lib/fluent/plugin/parser_apache_error.rb b/lib/fluent/plugin/parser_apache_error.rb
index 85f0586a5b..6168c5f3a0 100644
--- a/lib/fluent/plugin/parser_apache_error.rb
+++ b/lib/fluent/plugin/parser_apache_error.rb
@@ -17,5 +17,5 @@
 require 'fluent/plugin/parser'
 
 Fluent::Plugin.register_parser('apache_error', Proc.new {
-  Fluent::Plugin::RegexpParser.new(/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/)
+  Fluent::Compat::TextParser::RegexpParser.new(/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/)
 })
diff --git a/lib/fluent/plugin/parser_multiline.rb b/lib/fluent/plugin/parser_multiline.rb
index 240281e94c..675d695890 100644
--- a/lib/fluent/plugin/parser_multiline.rb
+++ b/lib/fluent/plugin/parser_multiline.rb
@@ -30,11 +30,11 @@ def configure(conf)
 
         formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
         begin
-          @regex = Regexp.new(formats, Regexp::MULTILINE)
-          if @regex.named_captures.empty?
+          regex = Regexp.new(formats, Regexp::MULTILINE)
+          if regex.named_captures.empty?
             raise "No named captures"
           end
-          @parser = RegexpParser.new(@regex, conf)
+          @parser = Fluent::Compat::TextParser::RegexpParser.new(regex, conf)
         rescue => e
           raise ConfigError, "Invalid regexp '#{formats}': #{e}"
         end
diff --git a/lib/fluent/plugin/parser_nginx.rb b/lib/fluent/plugin/parser_nginx.rb
index a33b81498a..581f8a2775 100644
--- a/lib/fluent/plugin/parser_nginx.rb
+++ b/lib/fluent/plugin/parser_nginx.rb
@@ -17,7 +17,7 @@
 require 'fluent/plugin/parser'
 
 Fluent::Plugin.register_parser('nginx', Proc.new {
-  Fluent::Plugin::RegexpParser.new(
+  Fluent::Compat::TextParser::RegexpParser.new(
     /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,
     {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}
   )
diff --git a/lib/fluent/plugin/parser_regexp.rb b/lib/fluent/plugin/parser_regexp.rb
new file mode 100644
index 0000000000..2b433f28a0
--- /dev/null
+++ b/lib/fluent/plugin/parser_regexp.rb
@@ -0,0 +1,68 @@
+module Fluent
+  module Plugin
+    class RegexpParser < Parser
+      include Fluent::Compat::TypeConverter
+
+      Plugin.register_parser("regexp", self)
+
+      config_param :expression, :string, default: ""
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+        @time_parser = TimeParser.new(@time_format)
+        unless @expression.empty?
+          if @expression[0] == "/" && @expression[-1] == "/"
+            @regexp = Regexp.new(@expression[1..-2])
+          else
+            raise Fluent::ConfigError, "expression must start with `/` and end with `/`: #{@expression}"
+          end
+        end
+      end
+
+      def parse(text)
+        m = @regexp.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        time = nil
+        record = {}
+
+        m.names.each do |name|
+          if value = m[name]
+            if name == @time_key
+              time = @mutex.synchronize { @time_parser.parse(value) }
+              if @keep_time_key
+                record[name] = if @type_converters.nil?
+                                 value
+                               else
+                                 convert_type(name, value)
+                               end
+              end
+            else
+              record[name] = if @type_converters.nil?
+                               value
+                             else
+                               convert_type(name, value)
+                             end
+            end
+          end
+        end
+
+        if @estimate_current_event
+          time ||= Fluent::EventTime.now
+        end
+
+        yield time, record
+      end
+    end
+  end
+end
diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index 5837e21671..af56ed4b1c 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -25,13 +25,13 @@ def internal_test_case(parser)
 
   def test_parse_with_typed
     # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
-    internal_test_case(Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
+    internal_test_case(Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
   end
 
   def test_parse_with_configure
     # Specify conf by configure method instaed of intializer
     regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(regexp))
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp))
     parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
     internal_test_case(parser.instance)
     assert_equal(regexp, parser.instance.patterns['format'])
@@ -39,11 +39,11 @@ def test_parse_with_configure
   end
 
   def test_parse_with_typed_and_name_separator
-    internal_test_case(Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|')).instance)
+    internal_test_case(Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|')).instance)
   end
 
   def test_parse_with_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(/(?<logtime>[^\]]*)/))
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(/(?<logtime>[^\]]*)/))
     parser.instance.configure(
                               'time_format'=>"%Y-%m-%d %H:%M:%S %z",
                               'time_key'=>'logtime',
@@ -58,7 +58,7 @@ def test_parse_without_time
     time_at_start = Time.now.to_i
     text = "tagomori_satoshi tagomoris 34\n"
 
-    parser = Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
+    parser = Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
     parser.configure('types'=>'name:string,user:string,age:integer')
 
     parser.parse(text) { |time, record|
@@ -68,7 +68,7 @@ def test_parse_without_time
       assert_equal 34, record["age"]
     }
 
-    parser2 = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)))
+    parser2 = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)))
     parser2.configure('types'=>'name:string,user:string,age:integer')
     parser2.instance.estimate_current_event = false
 
@@ -82,7 +82,7 @@ def test_parse_without_time
   end
 
   def test_parse_with_keep_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(
                                                 Regexp.new(%q!(?<time>.*)!),
                                                 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
                                                 'keep_time_key'=>'true',
@@ -95,7 +95,7 @@ def test_parse_with_keep_time_key
   end
 
   def test_parse_with_keep_time_key_with_typecast
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(
                                                 Regexp.new(%q!(?<time>.*)!),
                                                 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
                                                 'keep_time_key'=>'true',

From 165c769f6966f320a7358ad0d07aeb5c58e3c12b Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 11 Jul 2016 15:22:00 +0900
Subject: [PATCH 02/10] Extract `create_driver` and use it

---
 test/plugin/test_parser_regexp.rb | 100 +++++++++++++++++++-----------
 1 file changed, 64 insertions(+), 36 deletions(-)

diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index af56ed4b1c..0a2029fcb2 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -7,6 +7,10 @@ def setup
     Fluent::Test.setup
   end
 
+  def create_driver(regexp, conf = {})
+    Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+  end
+
   def internal_test_case(parser)
     text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
     parser.parse(text) { |time, record|
@@ -25,31 +29,47 @@ def internal_test_case(parser)
 
   def test_parse_with_typed
     # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
-    internal_test_case(Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
+    regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+    }
+    d = create_driver(regexp, conf)
+    internal_test_case(d.instance)
   end
 
   def test_parse_with_configure
     # Specify conf by configure method instaed of intializer
     regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp))
-    parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
-    internal_test_case(parser.instance)
-    assert_equal(regexp, parser.instance.patterns['format'])
-    assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.instance.patterns['time_format'])
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+    }
+    d = create_driver(regexp, conf)
+    internal_test_case(d.instance)
+    assert_equal(regexp, d.instance.patterns['format'])
+    assert_equal("%d/%b/%Y:%H:%M:%S %z", d.instance.patterns['time_format'])
   end
 
   def test_parse_with_typed_and_name_separator
-    internal_test_case(Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|')).instance)
+    regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
+      'types_label_delimiter' => '|'
+    }
+    d = create_driver(regexp, conf)
+    internal_test_case(d.instance)
   end
 
   def test_parse_with_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(/(?<logtime>[^\]]*)/))
-    parser.instance.configure(
-                              'time_format'=>"%Y-%m-%d %H:%M:%S %z",
-                              'time_key'=>'logtime',
-                              )
+    conf = {
+      'time_format' => "%Y-%m-%d %H:%M:%S %z",
+      'time_key' => 'logtime'
+    }
+    d = create_driver(/(?<logtime>[^\]]*)/, conf)
     text = '2013-02-28 12:00:00 +0900'
-    parser.instance.parse(text) do |time, record|
+    d.instance.parse(text) do |time, _record|
       assert_equal Fluent::EventTime.parse(text), time
     end
   end
@@ -58,21 +78,29 @@ def test_parse_without_time
     time_at_start = Time.now.to_i
     text = "tagomori_satoshi tagomoris 34\n"
 
-    parser = Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
-    parser.configure('types'=>'name:string,user:string,age:integer')
+    regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
+    conf = {
+      'types' => 'name:string,user:string,age:integer'
+    }
+    d = create_driver(regexp, conf)
 
-    parser.parse(text) { |time, record|
+    d.instance.parse(text) { |time, record|
       assert time && time >= time_at_start, "parser puts current time without time input"
       assert_equal "tagomori_satoshi", record["name"]
       assert_equal "tagomoris", record["user"]
       assert_equal 34, record["age"]
     }
+  end
 
-    parser2 = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)))
-    parser2.configure('types'=>'name:string,user:string,age:integer')
-    parser2.instance.estimate_current_event = false
-
-    parser2.instance.parse(text) { |time, record|
+  def test_parse_without_time_estimate_curent_event_false
+    text = "tagomori_satoshi tagomoris 34\n"
+    regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
+    conf = {
+      'types' => 'name:string,user:string,age:integer'
+    }
+    d = create_driver(regexp, conf)
+    d.instance.estimate_current_event = false
+    d.instance.parse(text) { |time, record|
       assert_equal "tagomori_satoshi", record["name"]
       assert_equal "tagomoris", record["user"]
       assert_equal 34, record["age"]
@@ -82,28 +110,28 @@ def test_parse_without_time
   end
 
   def test_parse_with_keep_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(
-                                                Regexp.new(%q!(?<time>.*)!),
-                                                'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-                                                'keep_time_key'=>'true',
-                                                )
-                                              )
+    regexp = Regexp.new(%q!(?<time>.*)!)
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'keep_time_key' => 'true',
+    }
+    d = create_driver(regexp, conf)
     text = '28/Feb/2013:12:00:00 +0900'
-    parser.instance.parse(text) do |time, record|
+    d.instance.parse(text) do |_time, record|
       assert_equal text, record['time']
     end
   end
 
   def test_parse_with_keep_time_key_with_typecast
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(
-                                                Regexp.new(%q!(?<time>.*)!),
-                                                'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-                                                'keep_time_key'=>'true',
-                                                'types'=>'time:time:%d/%b/%Y:%H:%M:%S %z',
-                                                )
-                                              )
+    regexp = Regexp.new(%q!(?<time>.*)!)
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'keep_time_key' => 'true',
+      'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
+    }
+    d = create_driver(regexp, conf)
     text = '28/Feb/2013:12:00:00 +0900'
-    parser.instance.parse(text) do |time, record|
+    d.instance.parse(text) do |_time, record|
       assert_equal 1362020400, record['time']
     end
   end

From 5ff908c09a568838fbaf47b9264924184f4c39f6 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 11 Jul 2016 15:42:19 +0900
Subject: [PATCH 03/10] Add test case when `#initialize` takes 2 arguments

---
 test/plugin/test_parser_regexp.rb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index 0a2029fcb2..24fe1c4e51 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -7,8 +7,12 @@ def setup
     Fluent::Test.setup
   end
 
-  def create_driver(regexp, conf = {})
-    Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+  def create_driver(regexp, conf = {}, initialize_conf: false)
+    if initialize_conf
+      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
+    else
+      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+    end
   end
 
   def internal_test_case(parser)
@@ -34,7 +38,7 @@ def test_parse_with_typed
       'time_format' => "%d/%b/%Y:%H:%M:%S %z",
       'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
     }
-    d = create_driver(regexp, conf)
+    d = create_driver(regexp, conf, initialize_conf: true)
     internal_test_case(d.instance)
   end
 

From a195c84bfa79e8312236f034ce334a93ba47c748 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 11 Jul 2016 15:43:03 +0900
Subject: [PATCH 04/10] Add test for `Fluent::Plugin::RegexpParser`

---
 test/plugin/test_parser_regexp.rb | 102 ++++++++++++++++++++++++++++++
 1 file changed, 102 insertions(+)

diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index 24fe1c4e51..81b99aeec4 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -139,4 +139,106 @@ def test_parse_with_keep_time_key_with_typecast
       assert_equal 1362020400, record['time']
     end
   end
+
+  sub_test_case "Fluent::Plugin::RegexpParser" do
+    def create_driver(conf)
+      Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new).configure(conf)
+    end
+
+    def test_parse_with_typed
+      conf = {
+        'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+      }
+      d = create_driver(conf)
+      internal_test_case(d.instance)
+    end
+
+    def test_parse_with_typed_and_name_separator
+      conf = {
+        'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
+        'types_label_delimiter' => '|'
+      }
+      d = create_driver(conf)
+      internal_test_case(d.instance)
+    end
+
+    def test_parse_with_time_key
+      conf = {
+        'expression' => %q!/(?<logtime>[^\]]*)/!,
+        'time_format' => "%Y-%m-%d %H:%M:%S %z",
+        'time_key' => 'logtime'
+      }
+      d = create_driver(conf)
+      text = '2013-02-28 12:00:00 +0900'
+      d.instance.parse(text) do |time, _record|
+        assert_equal Fluent::EventTime.parse(text), time
+      end
+    end
+
+    def test_parse_without_time
+      time_at_start = Time.now.to_i
+      text = "tagomori_satoshi tagomoris 34\n"
+
+      conf = {
+        'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
+        'types' => 'name:string,user:string,age:integer'
+      }
+      d = create_driver(conf)
+
+      d.instance.parse(text) { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal "tagomori_satoshi", record["name"]
+        assert_equal "tagomoris", record["user"]
+        assert_equal 34, record["age"]
+      }
+    end
+
+    def test_parse_without_time_estimate_curent_event_false
+      text = "tagomori_satoshi tagomoris 34\n"
+      conf = {
+        'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
+        'types' => 'name:string,user:string,age:integer'
+      }
+      d = create_driver(conf)
+      d.instance.estimate_current_event = false
+      d.instance.parse(text) { |time, record|
+        assert_equal "tagomori_satoshi", record["name"]
+        assert_equal "tagomoris", record["user"]
+        assert_equal 34, record["age"]
+
+        assert_nil time, "parser returns nil if configured so"
+      }
+    end
+
+    def test_parse_with_keep_time_key
+      conf = {
+        'expression' => %q!/(?<time>.*)/!,
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'true',
+      }
+      d = create_driver(conf)
+      text = '28/Feb/2013:12:00:00 +0900'
+      d.instance.parse(text) do |_time, record|
+        assert_equal text, record['time']
+      end
+    end
+
+    def test_parse_with_keep_time_key_with_typecast
+      conf = {
+        'expression' => %q!/(?<time>.*)/!,
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'true',
+        'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
+      }
+      d = create_driver(conf)
+      text = '28/Feb/2013:12:00:00 +0900'
+      d.instance.parse(text) do |_time, record|
+        assert_equal 1362020400, record['time']
+      end
+    end
+  end
 end

From 8d175c054022a9ad7f5fef8c4f255f84f59084bd Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 11 Jul 2016 15:51:25 +0900
Subject: [PATCH 05/10] Add test group for
 Fluent::Compat::TextParser::RegexpParser

---
 test/plugin/test_parser_regexp.rb | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index 81b99aeec4..152aa4f38c 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -7,14 +7,6 @@ def setup
     Fluent::Test.setup
   end
 
-  def create_driver(regexp, conf = {}, initialize_conf: false)
-    if initialize_conf
-      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
-    else
-      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
-    end
-  end
-
   def internal_test_case(parser)
     text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
     parser.parse(text) { |time, record|
@@ -31,6 +23,15 @@ def internal_test_case(parser)
     }
   end
 
+  sub_test_case "Fluent::Compat::TextParser::RegexpParser" do
+  def create_driver(regexp, conf = {}, initialize_conf: false)
+    if initialize_conf
+      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
+    else
+      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+    end
+  end
+
   def test_parse_with_typed
     # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
     regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
@@ -139,6 +140,7 @@ def test_parse_with_keep_time_key_with_typecast
       assert_equal 1362020400, record['time']
     end
   end
+  end
 
   sub_test_case "Fluent::Plugin::RegexpParser" do
     def create_driver(conf)

From 58e8bce09a1dc478eaa345da5e49adf2696a8282 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 11 Jul 2016 15:51:55 +0900
Subject: [PATCH 06/10] Indent

---
 test/plugin/test_parser_regexp.rb | 204 +++++++++++++++---------------
 1 file changed, 102 insertions(+), 102 deletions(-)

diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index 152aa4f38c..0dfa081429 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -24,122 +24,122 @@ def internal_test_case(parser)
   end
 
   sub_test_case "Fluent::Compat::TextParser::RegexpParser" do
-  def create_driver(regexp, conf = {}, initialize_conf: false)
-    if initialize_conf
-      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
-    else
-      Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+    def create_driver(regexp, conf = {}, initialize_conf: false)
+      if initialize_conf
+        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
+      else
+        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+      end
     end
-  end
 
-  def test_parse_with_typed
-    # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
-    regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-    conf = {
-      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-      'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
-    }
-    d = create_driver(regexp, conf, initialize_conf: true)
-    internal_test_case(d.instance)
-  end
+    def test_parse_with_typed
+      # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
+      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
+      conf = {
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+      }
+      d = create_driver(regexp, conf, initialize_conf: true)
+      internal_test_case(d.instance)
+    end
 
-  def test_parse_with_configure
-    # Specify conf by configure method instaed of intializer
-    regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-    conf = {
-      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-      'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
-    }
-    d = create_driver(regexp, conf)
-    internal_test_case(d.instance)
-    assert_equal(regexp, d.instance.patterns['format'])
-    assert_equal("%d/%b/%Y:%H:%M:%S %z", d.instance.patterns['time_format'])
-  end
+    def test_parse_with_configure
+      # Specify conf by configure method instaed of intializer
+      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
+      conf = {
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+      }
+      d = create_driver(regexp, conf)
+      internal_test_case(d.instance)
+      assert_equal(regexp, d.instance.patterns['format'])
+      assert_equal("%d/%b/%Y:%H:%M:%S %z", d.instance.patterns['time_format'])
+    end
 
-  def test_parse_with_typed_and_name_separator
-    regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-    conf = {
-      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-      'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
-      'types_label_delimiter' => '|'
-    }
-    d = create_driver(regexp, conf)
-    internal_test_case(d.instance)
-  end
+    def test_parse_with_typed_and_name_separator
+      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
+      conf = {
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
+        'types_label_delimiter' => '|'
+      }
+      d = create_driver(regexp, conf)
+      internal_test_case(d.instance)
+    end
 
-  def test_parse_with_time_key
-    conf = {
-      'time_format' => "%Y-%m-%d %H:%M:%S %z",
-      'time_key' => 'logtime'
-    }
-    d = create_driver(/(?<logtime>[^\]]*)/, conf)
-    text = '2013-02-28 12:00:00 +0900'
-    d.instance.parse(text) do |time, _record|
-      assert_equal Fluent::EventTime.parse(text), time
+    def test_parse_with_time_key
+      conf = {
+        'time_format' => "%Y-%m-%d %H:%M:%S %z",
+        'time_key' => 'logtime'
+      }
+      d = create_driver(/(?<logtime>[^\]]*)/, conf)
+      text = '2013-02-28 12:00:00 +0900'
+      d.instance.parse(text) do |time, _record|
+        assert_equal Fluent::EventTime.parse(text), time
+      end
     end
-  end
 
-  def test_parse_without_time
-    time_at_start = Time.now.to_i
-    text = "tagomori_satoshi tagomoris 34\n"
+    def test_parse_without_time
+      time_at_start = Time.now.to_i
+      text = "tagomori_satoshi tagomoris 34\n"
 
-    regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
-    conf = {
-      'types' => 'name:string,user:string,age:integer'
-    }
-    d = create_driver(regexp, conf)
+      regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
+      conf = {
+        'types' => 'name:string,user:string,age:integer'
+      }
+      d = create_driver(regexp, conf)
 
-    d.instance.parse(text) { |time, record|
-      assert time && time >= time_at_start, "parser puts current time without time input"
-      assert_equal "tagomori_satoshi", record["name"]
-      assert_equal "tagomoris", record["user"]
-      assert_equal 34, record["age"]
-    }
-  end
+      d.instance.parse(text) { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal "tagomori_satoshi", record["name"]
+        assert_equal "tagomoris", record["user"]
+        assert_equal 34, record["age"]
+      }
+    end
 
-  def test_parse_without_time_estimate_curent_event_false
-    text = "tagomori_satoshi tagomoris 34\n"
-    regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
-    conf = {
-      'types' => 'name:string,user:string,age:integer'
-    }
-    d = create_driver(regexp, conf)
-    d.instance.estimate_current_event = false
-    d.instance.parse(text) { |time, record|
-      assert_equal "tagomori_satoshi", record["name"]
-      assert_equal "tagomoris", record["user"]
-      assert_equal 34, record["age"]
-
-      assert_nil time, "parser returns nil if configured so"
-    }
-  end
+    def test_parse_without_time_estimate_curent_event_false
+      text = "tagomori_satoshi tagomoris 34\n"
+      regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
+      conf = {
+        'types' => 'name:string,user:string,age:integer'
+      }
+      d = create_driver(regexp, conf)
+      d.instance.estimate_current_event = false
+      d.instance.parse(text) { |time, record|
+        assert_equal "tagomori_satoshi", record["name"]
+        assert_equal "tagomoris", record["user"]
+        assert_equal 34, record["age"]
 
-  def test_parse_with_keep_time_key
-    regexp = Regexp.new(%q!(?<time>.*)!)
-    conf = {
-      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-      'keep_time_key' => 'true',
-    }
-    d = create_driver(regexp, conf)
-    text = '28/Feb/2013:12:00:00 +0900'
-    d.instance.parse(text) do |_time, record|
-      assert_equal text, record['time']
+        assert_nil time, "parser returns nil if configured so"
+      }
     end
-  end
 
-  def test_parse_with_keep_time_key_with_typecast
-    regexp = Regexp.new(%q!(?<time>.*)!)
-    conf = {
-      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-      'keep_time_key' => 'true',
-      'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
-    }
-    d = create_driver(regexp, conf)
-    text = '28/Feb/2013:12:00:00 +0900'
-    d.instance.parse(text) do |_time, record|
-      assert_equal 1362020400, record['time']
+    def test_parse_with_keep_time_key
+      regexp = Regexp.new(%q!(?<time>.*)!)
+      conf = {
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'true',
+      }
+      d = create_driver(regexp, conf)
+      text = '28/Feb/2013:12:00:00 +0900'
+      d.instance.parse(text) do |_time, record|
+        assert_equal text, record['time']
+      end
+    end
+
+    def test_parse_with_keep_time_key_with_typecast
+      regexp = Regexp.new(%q!(?<time>.*)!)
+      conf = {
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'true',
+        'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
+      }
+      d = create_driver(regexp, conf)
+      text = '28/Feb/2013:12:00:00 +0900'
+      d.instance.parse(text) do |_time, record|
+        assert_equal 1362020400, record['time']
+      end
     end
-  end
   end
 
   sub_test_case "Fluent::Plugin::RegexpParser" do

From 22f8548d912edd4eb32f2ea4f0126907ca03037a Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jul 2016 12:42:46 +0900
Subject: [PATCH 07/10] Use new RegexpParser instead of
 Fluent::Compat::TextParser::RegexpParser

---
 lib/fluent/plugin/parser_apache.rb       | 16 ++++++++++------
 lib/fluent/plugin/parser_apache_error.rb | 11 ++++++++---
 lib/fluent/plugin/parser_nginx.rb        | 16 ++++++++++------
 test/plugin/test_parser_apache.rb        | 22 +++++++++++++---------
 test/plugin/test_parser_apache_error.rb  |  4 ++--
 test/plugin/test_parser_nginx.rb         |  5 +++--
 6 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/lib/fluent/plugin/parser_apache.rb b/lib/fluent/plugin/parser_apache.rb
index e33b55659a..1973f9c653 100644
--- a/lib/fluent/plugin/parser_apache.rb
+++ b/lib/fluent/plugin/parser_apache.rb
@@ -16,9 +16,13 @@
 
 require 'fluent/plugin/parser'
 
-Fluent::Plugin.register_parser('apache', Proc.new{
-  Fluent::Compat::TextParser::RegexpParser.new(
-    /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,
-    {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}
-  )
-})
+module Fluent
+  module Plugin
+    class ApacheParser < RegexpParser
+      Plugin.register_parser("apache", self)
+
+      config_set_default :expression, %q{/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end
diff --git a/lib/fluent/plugin/parser_apache_error.rb b/lib/fluent/plugin/parser_apache_error.rb
index 6168c5f3a0..deb68e715c 100644
--- a/lib/fluent/plugin/parser_apache_error.rb
+++ b/lib/fluent/plugin/parser_apache_error.rb
@@ -16,6 +16,11 @@
 
 require 'fluent/plugin/parser'
 
-Fluent::Plugin.register_parser('apache_error', Proc.new {
-  Fluent::Compat::TextParser::RegexpParser.new(/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/)
-})
+module Fluent
+  module Plugin
+    class ApacheErrorParser < RegexpParser
+      Plugin.register_parser("apache_error", self)
+      config_set_default :expression, %q{/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/}
+    end
+  end
+end
diff --git a/lib/fluent/plugin/parser_nginx.rb b/lib/fluent/plugin/parser_nginx.rb
index 581f8a2775..9dc7151add 100644
--- a/lib/fluent/plugin/parser_nginx.rb
+++ b/lib/fluent/plugin/parser_nginx.rb
@@ -16,9 +16,13 @@
 
 require 'fluent/plugin/parser'
 
-Fluent::Plugin.register_parser('nginx', Proc.new {
-  Fluent::Compat::TextParser::RegexpParser.new(
-    /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,
-    {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}
-  )
-})
+module Fluent
+  module Plugin
+    class NginxParser < RegexpParser
+      Plugin.register_parser("nginx", self)
+
+      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end
diff --git a/test/plugin/test_parser_apache.rb b/test/plugin/test_parser_apache.rb
index df4e223b3b..e31f9faea3 100644
--- a/test/plugin/test_parser_apache.rb
+++ b/test/plugin/test_parser_apache.rb
@@ -1,16 +1,20 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_apache'
 
 class ApacheParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache'))
+  end
+
+  def create_driver(conf = {})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheParser.new).configure(conf)
   end
 
   data('parse' => :parse, 'call' => :call)
   def test_call(method_name)
-    m = @parser.instance.method(method_name)
+    d = create_driver
+    m = d.instance.method(method_name)
     m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal({
@@ -25,13 +29,13 @@ def test_call(method_name)
   end
 
   def test_parse_with_keep_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache'))
-    parser.instance.configure(
-                              'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-                              'keep_time_key'=>'true',
-                              )
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'keep_time_key' => 'true',
+    }
+    d = create_driver(conf)
     text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
-    parser.instance.parse(text) do |time, record|
+    d.instance.parse(text) do |_time, record|
       assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
     end
   end
diff --git a/test/plugin/test_parser_apache_error.rb b/test/plugin/test_parser_apache_error.rb
index 5a2795bf6a..e6a77e9dfb 100644
--- a/test/plugin/test_parser_apache_error.rb
+++ b/test/plugin/test_parser_apache_error.rb
@@ -1,11 +1,11 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_apache_error'
 
 class ApacheErrorParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache_error'))
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new)
     @parser.configure({})
     @expected = {
       'level' => 'error',
diff --git a/test/plugin/test_parser_nginx.rb b/test/plugin/test_parser_nginx.rb
index ce14a11eb0..23e49892f0 100644
--- a/test/plugin/test_parser_nginx.rb
+++ b/test/plugin/test_parser_nginx.rb
@@ -1,11 +1,12 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_nginx'
 
 class NginxParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('nginx'))
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::NginxParser.new)
+    @parser.configure({})
     @expected = {
       'remote'  => '127.0.0.1',
       'host'    => '192.168.0.1',

From 6591be449945e6317a9772486e82245d164010dc Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 13 Jul 2016 10:06:34 +0900
Subject: [PATCH 08/10] Add new options ignorecase and multiline to
 RegexpParser

---
 lib/fluent/plugin/parser_regexp.rb |  7 ++++++-
 test/plugin/test_parser_regexp.rb  | 31 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/parser_regexp.rb b/lib/fluent/plugin/parser_regexp.rb
index 2b433f28a0..94b0c3e760 100644
--- a/lib/fluent/plugin/parser_regexp.rb
+++ b/lib/fluent/plugin/parser_regexp.rb
@@ -6,6 +6,8 @@ class RegexpParser < Parser
       Plugin.register_parser("regexp", self)
 
       config_param :expression, :string, default: ""
+      config_param :ignorecase, :bool, default: false
+      config_param :multiline, :bool, default: false
       config_param :time_key, :string, default: 'time'
       config_param :time_format, :string, default: nil
 
@@ -19,7 +21,10 @@ def configure(conf)
         @time_parser = TimeParser.new(@time_format)
         unless @expression.empty?
           if @expression[0] == "/" && @expression[-1] == "/"
-            @regexp = Regexp.new(@expression[1..-2])
+            regexp_option = 0
+            regexp_option |= Regexp::IGNORECASE if @ignorecase
+            regexp_option |= Regexp::MULTILINE if @multiline
+            @regexp = Regexp.new(@expression[1..-2], regexp_option)
           else
             raise Fluent::ConfigError, "expression must start with `/` and end with `/`: #{@expression}"
           end
diff --git a/test/plugin/test_parser_regexp.rb b/test/plugin/test_parser_regexp.rb
index 0dfa081429..fa80f211ad 100644
--- a/test/plugin/test_parser_regexp.rb
+++ b/test/plugin/test_parser_regexp.rb
@@ -147,6 +147,37 @@ def create_driver(conf)
       Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new).configure(conf)
     end
 
+    sub_test_case "configure" do
+      def test_default_options
+        conf = {
+          'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
+          'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+          'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+        }
+        d = create_driver(conf)
+        regexp = d.instance.instance_variable_get(:@regexp)
+        assert_equal(0, regexp.options)
+      end
+
+      data(
+        ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
+        multiline: [{ "multiline" => true }, Regexp::MULTILINE],
+        ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
+      )
+      def test_options(data)
+        regexp_option, expected = data
+        conf = {
+          'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
+          'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+          'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
+        }
+        conf = conf.merge(regexp_option)
+        d = create_driver(conf)
+        regexp = d.instance.instance_variable_get(:@regexp)
+        assert_equal(expected, regexp.options)
+      end
+    end
+
     def test_parse_with_typed
       conf = {
         'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,

From c0f1e83c2a36a293157e589fe494a22b090bb320 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 13 Jul 2016 10:07:04 +0900
Subject: [PATCH 09/10] Use Fluent::Plugin::RegexpParser instead of
 Fluent::Compat::TextParser::RegexpParser

Set options via Fluent::Config::Element.
---
 lib/fluent/plugin/parser_multiline.rb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/fluent/plugin/parser_multiline.rb b/lib/fluent/plugin/parser_multiline.rb
index 675d695890..206093603b 100644
--- a/lib/fluent/plugin/parser_multiline.rb
+++ b/lib/fluent/plugin/parser_multiline.rb
@@ -30,11 +30,13 @@ def configure(conf)
 
         formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
         begin
-          regex = Regexp.new(formats, Regexp::MULTILINE)
-          if regex.named_captures.empty?
+          regexp = Regexp.new(formats, Regexp::MULTILINE)
+          if regexp.named_captures.empty?
             raise "No named captures"
           end
-          @parser = Fluent::Compat::TextParser::RegexpParser.new(regex, conf)
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/", "multiline" => true }, [])
+          @parser = Fluent::Plugin::RegexpParser.new
+          @parser.configure(conf + regexp_conf)
         rescue => e
           raise ConfigError, "Invalid regexp '#{formats}': #{e}"
         end

From 2e464e57de89651bd498bba3993296c513391f32 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 19 Jul 2016 15:30:03 +0900
Subject: [PATCH 10/10] `Fluent::Test::Driver::Parser.new` creates driver not
 parser

---
 test/plugin/test_parser_apache_error.rb | 15 ++++++++++-----
 test/plugin/test_parser_nginx.rb        | 15 ++++++++++-----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/test/plugin/test_parser_apache_error.rb b/test/plugin/test_parser_apache_error.rb
index e6a77e9dfb..1b805793ff 100644
--- a/test/plugin/test_parser_apache_error.rb
+++ b/test/plugin/test_parser_apache_error.rb
@@ -5,8 +5,6 @@
 class ApacheErrorParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new)
-    @parser.configure({})
     @expected = {
       'level' => 'error',
       'client' => '127.0.0.1',
@@ -14,22 +12,29 @@ def setup
     }
   end
 
+  def create_driver
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new).configure({})
+  end
+
   def test_parse
-    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
       assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
       assert_equal(@expected, record)
     }
   end
 
   def test_parse_with_pid
-    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
       assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
       assert_equal(@expected.merge('pid' => '1000'), record)
     }
   end
 
   def test_parse_without_client
-    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
       assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
       assert_equal({
                      'level' => 'notice',
diff --git a/test/plugin/test_parser_nginx.rb b/test/plugin/test_parser_nginx.rb
index 23e49892f0..d2707fe5c7 100644
--- a/test/plugin/test_parser_nginx.rb
+++ b/test/plugin/test_parser_nginx.rb
@@ -5,8 +5,6 @@
 class NginxParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::NginxParser.new)
-    @parser.configure({})
     @expected = {
       'remote'  => '127.0.0.1',
       'host'    => '192.168.0.1',
@@ -20,22 +18,29 @@ def setup
     }
   end
 
+  def create_driver
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::NginxParser.new).configure({})
+  end
+
   def test_parse
-    @parser.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }
   end
 
   def test_parse_with_empty_included_path
-    @parser.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected.merge('path' => '/a[ ]b'), record)
     }
   end
 
   def test_parse_without_http_version
-    @parser.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }