Use config_section for filter_grep

lib/fluent/plugin/filter_grep.rb

@@ -24,43 +24,70 @@ class GrepFilter < Filter
 
     REGEXP_MAX_NUM = 20
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil, deprecated: "Use <exclude> section" }
+
+    config_section :regexp, param_name: :regexps, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern do |value|
+        Regexp.compile(value)
+      end
+    end
+
+    config_section :exclude, param_name: :excludes, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern do |value|
+        Regexp.compile(value)
+      end
+    end
 
     # for test
-    attr_reader :regexps
-    attr_reader :excludes
+    attr_reader :_regexps
+    attr_reader :_excludes
 
     def configure(conf)
       super
 
-      @regexps = {}
+      @_regexps = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
-        @regexps[key] = Regexp.compile(regexp)
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @_regexps[key]
+        @_regexps[key] = Regexp.compile(regexp)
       end
 
-      @excludes = {}
+      @_excludes = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
-        @excludes[key] = Regexp.compile(exclude)
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @_excludes[key]
+        @_excludes[key] = Regexp.compile(exclude)
+      end
+
+      @regexps.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_regexps.key?(e.key)
+        @_regexps[e.key] = e.pattern
+      end
+      @excludes.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_excludes.key?(e.key)
+        @_excludes[e.key] = e.pattern
       end
     end
 
     def filter(tag, time, record)
       result = nil
       begin
         catch(:break_loop) do
-          @regexps.each do |key, regexp|
+          @_regexps.each do |key, regexp|
             throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
           end
-          @excludes.each do |key, exclude|
+          @_excludes.each do |key, exclude|
             throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
           end
           result = record

test/plugin/test_filter_grep.rb

@@ -23,12 +23,51 @@ def create_driver(conf = '')
 
     test "regexpN can contain a space" do
       d = create_driver(%[regexp1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance.regexps['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance._regexps['message'])
     end
 
     test "excludeN can contain a space" do
       d = create_driver(%[exclude1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance.excludes['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance._excludes['message'])
+    end
+
+    sub_test_case "duplicate key" do
+      test "flat" do
+        conf = %[
+          regexp1 message test
+          regexp2 message test2
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+      test "section" do
+        conf = %[
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+          <regexp>
+            key message
+            pattern test2
+          </regexp>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+      test "mix" do
+        conf = %[
+          regexp1 message test
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
     end
   end
 
@@ -77,6 +116,38 @@ def filter(config, msgs)
       end
     end
 
+    test 'regexps' do
+      conf = %[
+        <regexp>
+          key message
+          pattern WARN
+        </regexp>
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(3, filtered_records.size)
+      assert_block('only WARN logs') do
+        filtered_records.all? { |r|
+          !r['message'].include?('INFO')
+        }
+      end
+    end
+
+    test 'excludes' do
+      conf = %[
+        <exclude>
+          key message
+          pattern favicon
+        </exclude>
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(3, filtered_records.size)
+      assert_block('remove favicon logs') do
+        filtered_records.all? { |r|
+          !r['message'].include?('favicon')
+        }
+      end
+    end
+
     sub_test_case 'with invalid sequence' do
       def messages
         [