From 53459ad4d1466487d8d945b7d2405c71e589a3b8 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 16 Oct 2015 15:41:02 +0900
Subject: [PATCH 001/193] Add `config_set_desc`

We can write description of plugin parameter as following:

```
class Fluent::SomeInput < Fluent::Input
  config_set_desc :tag, "This value is the tag assigned to the generated events."
  config_param :tag, :string
end
```
---
 lib/fluent/configurable.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
index 24b1fbff60..a42b9e0470 100644
--- a/lib/fluent/configurable.rb
+++ b/lib/fluent/configurable.rb
@@ -105,6 +105,10 @@ def config_set_default(name, defval)
         configure_proxy(self.name).config_set_default(name, defval)
       end
 
+      def config_set_desc(name, desc)
+        configure_proxy(self.name).config_set_desc(name, desc)
+      end
+
       def config_section(name, *args, &block)
         configure_proxy(self.name).config_section(name, *args, &block)
         attr_accessor configure_proxy(self.name).sections[name].param_name

From 59225ed66931b9bc2a2abfdeaad0e5aeea62a2f7 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 16 Oct 2015 16:06:12 +0900
Subject: [PATCH 002/193] Add `desc` method

We can write description of plugin parameter as following:

```
class Fluent::SomeInput < Fluent::Input
  desc "This value is the tag assigned to the generated events."
  config_param :tag, :string
end
```
---
 lib/fluent/config/configure_proxy.rb |  9 ++++++
 lib/fluent/configurable.rb           |  4 +++
 test/config/test_configure_proxy.rb  | 42 ++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+)

diff --git a/lib/fluent/config/configure_proxy.rb b/lib/fluent/config/configure_proxy.rb
index c91b87b967..1e9b1d744f 100644
--- a/lib/fluent/config/configure_proxy.rb
+++ b/lib/fluent/config/configure_proxy.rb
@@ -178,6 +178,11 @@ def config_argument(name, *args, &block)
       def config_param(name, *args, &block)
         name, block, opts = parameter_configuration(name, *args, &block)
 
+        if @current_description
+          config_set_desc(name, @current_description)
+          @current_description = nil
+        end
+
         @sections.delete(name)
         @params[name] = [block, opts]
         name
@@ -205,6 +210,10 @@ def config_set_desc(name, description)
         nil
       end
 
+      def desc(description)
+        @current_description = description
+      end
+
       def config_section(name, *args, &block)
         unless block_given?
           raise ArgumentError, "#{self.name}: config_section requires block parameter"
diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
index a42b9e0470..e7f3f95284 100644
--- a/lib/fluent/configurable.rb
+++ b/lib/fluent/configurable.rb
@@ -114,6 +114,10 @@ def config_section(name, *args, &block)
         attr_accessor configure_proxy(self.name).sections[name].param_name
       end
 
+      def desc(description)
+        configure_proxy(self.name).desc(description)
+      end
+
       def merged_configure_proxy
         configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
 
diff --git a/test/config/test_configure_proxy.rb b/test/config/test_configure_proxy.rb
index 18ff36911c..27b5236c2e 100644
--- a/test/config/test_configure_proxy.rb
+++ b/test/config/test_configure_proxy.rb
@@ -112,6 +112,26 @@ class TestConfigureProxy < ::Test::Unit::TestCase
         end
       end
 
+      sub_test_case '#desc' do
+        setup do
+          @proxy = Fluent::Config::ConfigureProxy.new(:section)
+        end
+
+        test 'permit to specify description twice' do
+          @proxy.desc("description1")
+          @proxy.desc("description2")
+          @proxy.config_param(:name, :string)
+          assert_equal("description2", @proxy.descriptions[:name])
+        end
+
+        test 'does not permit description specification twice' do
+          @proxy.desc("description1")
+          assert_raise(ArgumentError) do
+            @proxy.config_param(:name, :string, :desc => "description2")
+          end
+        end
+      end
+
       sub_test_case '#dump' do
         setup do
           @proxy = Fluent::Config::ConfigureProxy.new(:section)
@@ -206,6 +226,28 @@ class TestConfigureProxy < ::Test::Unit::TestCase
             end
             assert_equal(<<CONFIG, @proxy.dump)
 
+sub
+ name1: string: <"name1"> # desc1
+ name2: string: <"name2"> # desc2
+ sub2
+  name3: string: <"name3">
+  name4: string: <"name4"> # desc4
+CONFIG
+          end
+
+          test 'sub proxy w/ desc method' do
+            @proxy.config_section(:sub) do
+              desc("desc1")
+              config_param(:name1, :string, default: "name1")
+              config_param(:name2, :string, default: "name2", desc: "desc2")
+              config_section(:sub2) do
+                config_param(:name3, :string, default: "name3")
+                desc("desc4")
+                config_param(:name4, :string, default: "name4")
+              end
+            end
+            assert_equal(<<CONFIG, @proxy.dump)
+
 sub
  name1: string: <"name1"> # desc1
  name2: string: <"name2"> # desc2

From 76d5292ad643765273bba7036c4c6a107d926a20 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 16 Oct 2015 16:08:10 +0900
Subject: [PATCH 003/193] Add descriptions to in_dummy plugin parameters

---
 lib/fluent/plugin/in_dummy.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/fluent/plugin/in_dummy.rb b/lib/fluent/plugin/in_dummy.rb
index 5f2c0c9d17..00492ce006 100644
--- a/lib/fluent/plugin/in_dummy.rb
+++ b/lib/fluent/plugin/in_dummy.rb
@@ -20,9 +20,13 @@ class DummyInput < Input
 
     BIN_NUM = 10
 
+    desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
+    desc "It configures how many events to generate per second."
     config_param :rate, :integer, :default => 1
+    desc "If specified, each generated event has an auto-incremented key field."
     config_param :auto_increment_key, :string, :default => nil
+    desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
       begin
         parsed = JSON.parse(val)

From 99e30ec93f36d06df05b32853066c48dbbce4659 Mon Sep 17 00:00:00 2001
From: Eduardo Silva <eduardo@treasure-data.com>
Date: Wed, 21 Oct 2015 11:18:49 -0600
Subject: [PATCH 004/193] Doc: update README with Slack Team info

Signed-off-by: Eduardo Silva <eduardo@treasure-data.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 39735ded80..a853d77a1a 100644
--- a/README.md
+++ b/README.md
@@ -38,6 +38,7 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 - Documentation: http://docs.fluentd.org/
 - Source repository: http://github.com/fluent
 - Discussion: http://groups.google.com/group/fluentd
+- Slack / Community: http://slack.fluentd.org
 - Newsletters: http://get.treasuredata.com/Fluentd_education
 - Author: Sadayuki Furuhashi
 - Copyright: (c) 2011 FURUHASHI Sadayuki
@@ -48,4 +49,3 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
 
 [<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)
-

From 95e0151c1394818a8ea579966c998945541ca064 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 26 Oct 2015 16:00:50 +0900
Subject: [PATCH 005/193] Don't use popular name to prevent field overwrite
 when enable_ruby is true

---
 lib/fluent/plugin/filter_record_transformer.rb | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index f61d4f3598..8129587ba9 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -230,18 +230,17 @@ def prepare_placeholders(time, record, opts)
         @placeholders = struct
       end
 
-      def expand(str, force_stringify=false)
+      def expand(_str_for_eval_, force_stringify=false)
         if @auto_typecast and !force_stringify
-          single_placeholder_matched = str.match(/\A\${([^}]+)}\z/)
-          if single_placeholder_matched
-            code = single_placeholder_matched[1]
-            return eval code, @placeholders.instance_eval { binding }
+          _single_placeholder_matched_ = _str_for_eval_.match(/\A\${([^}]+)}\z/)
+          if _single_placeholder_matched_
+            return eval _single_placeholder_matched_[1], @placeholders.instance_eval { binding }
           end
         end
-        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-        eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
+        _interpolated_for_eval_ = _str_for_eval_.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+        eval "\"#{_interpolated_for_eval_}\"", @placeholders.instance_eval { binding }
       rescue => e
-        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
+        log.warn "failed to expand `#{_str_for_eval_}`", :error_class => e.class, :error => e.message
         log.warn_backtrace
         nil
       end

From e90135a0fe47a0cf7b9cc867899be095dd9de6d9 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 4 Sep 2015 21:31:21 +0900
Subject: [PATCH 006/193] in_monitor_agent emits plugins info to use exisiting
 plugins. fix #667

---
 lib/fluent/plugin/in_monitor_agent.rb | 64 ++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index 6f609a12d3..18b3cd4d82 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -27,6 +27,8 @@ def initialize
 
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :port, :integer, :default => 24220
+    config_param :tag, :string, :default => nil
+    config_param :emit_interval, :time, :default => 60
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -199,6 +201,36 @@ def process(req, res)
       end
     end
 
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, log, &callback)
+        @callback = callback
+        @log = log
+
+        # Avoid long shutdown time
+        @num_call = 0
+        if interval >= 10
+          min_interval = 10
+          @call_interval = interval / 10
+        else
+          min_interval = interval
+          @call_interval = 0
+        end
+
+        super(min_interval, true)
+      end
+
+      def on_timer
+        @num_call += 1
+        if @num_call >= @call_interval
+          @num_call = 0
+          @callback.call
+        end
+      rescue => e
+        @log.error e.to_s
+        @log.error_backtrace
+      end
+    end
+
     def start
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
       @srv = WEBrick::HTTPServer.new({
@@ -214,6 +246,29 @@ def start
       @thread = Thread.new {
         @srv.start
       }
+      if @tag
+        log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
+
+        @loop = Coolio::Loop.new
+        opts = {:with_config => false}
+        timer = TimerWatcher.new(@emit_interval, log) {
+          es = MultiEventStream.new
+          now = Engine.now
+          plugins_info_all(opts).each { |record|
+            es.add(now, record)
+          }
+          router.emit_stream(@tag, es)
+        }
+        @loop.attach(timer)
+        @thread_for_emit = Thread.new(&method(:run))
+      end
+    end
+
+    def run
+      @loop.run
+    rescue => e
+      log.error "unexpected error", :error => e.to_s
+      log.error_backtrace
     end
 
     def shutdown
@@ -225,6 +280,13 @@ def shutdown
         @thread.join
         @thread = nil
       end
+      if @tag
+        @loop.watchers.each { |w| w.detach }
+        @loop.stop
+        @loop = nil
+        @thread_for_emit.join
+        @thread_for_emit = nil
+      end
     end
 
     MONITOR_INFO = {
@@ -320,7 +382,7 @@ def get_monitor_info(pe, opts={})
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
       obj['type'] = pe.config['@type'] || pe.config['type']
-      obj['config'] = pe.config
+      obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|

From 64b613f02dcb07c278211d6697af7526b195e1e7 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 27 Oct 2015 18:41:42 +0900
Subject: [PATCH 007/193] Log shutdown plugin info to check shutdown sequence

---
 lib/fluent/agent.rb      | 2 ++
 lib/fluent/root_agent.rb | 1 +
 2 files changed, 3 insertions(+)

diff --git a/lib/fluent/agent.rb b/lib/fluent/agent.rb
index b2cc379ad1..5080f2377c 100644
--- a/lib/fluent/agent.rb
+++ b/lib/fluent/agent.rb
@@ -78,6 +78,7 @@ def shutdown
       @started_filters.map { |f|
         Thread.new do
           begin
+            log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
             f.shutdown
           rescue => e
             log.warn "unexpected error while shutting down filter plugins", :plugin => f.class, :plugin_id => f.plugin_id, :error_class => e.class, :error => e
@@ -91,6 +92,7 @@ def shutdown
       @started_outputs.map { |o|
         Thread.new do
           begin
+            log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
             o.shutdown
           rescue => e
             log.warn "unexpected error while shutting down output plugins", :plugin => o.class, :plugin_id => o.plugin_id, :error_class => e.class, :error => e
diff --git a/lib/fluent/root_agent.rb b/lib/fluent/root_agent.rb
index 2368e1274c..a0851c78b7 100644
--- a/lib/fluent/root_agent.rb
+++ b/lib/fluent/root_agent.rb
@@ -118,6 +118,7 @@ def shutdown
       @started_inputs.map { |i|
         Thread.new do
           begin
+            log.info "shutting down input", type: Plugin.lookup_name_from_class(i.class), plugin_id: i.plugin_id
             i.shutdown
           rescue => e
             log.warn "unexpected error while shutting down input plugin", :plugin => i.class, :plugin_id => i.plugin_id, :error_class => e.class, :error => e

From fce745f992461dde8ea37b7a26989dd8d35f8a28 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 28 Oct 2015 17:26:22 +0900
Subject: [PATCH 008/193] Tempfile should be binary mode

---
 lib/fluent/plugin/out_exec.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/plugin/out_exec.rb b/lib/fluent/plugin/out_exec.rb
index 28adca6adf..baf7f6e276 100644
--- a/lib/fluent/plugin/out_exec.rb
+++ b/lib/fluent/plugin/out_exec.rb
@@ -81,6 +81,7 @@ def write(chunk)
         prog = "#{@command} #{chunk.path}"
       else
         tmpfile = Tempfile.new("fluent-plugin-exec-")
+        tmpfile.binmode
         chunk.write_to(tmpfile)
         tmpfile.close
         prog = "#{@command} #{tmpfile.path}"

From 988dbe867def28f38b629ab8305347fa7f8194d5 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 29 Oct 2015 16:19:18 +0900
Subject: [PATCH 009/193] Forgot to add v0.12 test in travis-ci

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index dcfa243458..9752cc3e68 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ branches:
   only:
     - master
     - v0.10
+    - v0.12
     - v0.14
 
 gemfile:

From aac0b72f00b8804f4a35f4a4659d0350ca861814 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 29 Oct 2015 16:13:34 +0900
Subject: [PATCH 010/193] Add Engine#msgpack_factory for v0.14 compatibility

---
 lib/fluent/buffer.rb            |  2 +-
 lib/fluent/command/cat.rb       |  2 +-
 lib/fluent/engine.rb            | 13 +++++++++++++
 lib/fluent/event.rb             |  4 ++--
 lib/fluent/plugin/buf_memory.rb |  2 +-
 lib/fluent/plugin/exec_util.rb  |  2 +-
 lib/fluent/plugin/in_forward.rb |  2 +-
 lib/fluent/plugin/in_http.rb    |  2 +-
 lib/fluent/plugin/in_stream.rb  |  2 +-
 lib/fluent/plugin/out_stream.rb |  2 +-
 lib/fluent/process.rb           |  2 +-
 test/plugin/test_in_forward.rb  | 14 +++++++-------
 test/plugin/test_in_stream.rb   | 10 +++++-----
 test/plugin/test_out_copy.rb    |  2 +-
 14 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/lib/fluent/buffer.rb b/lib/fluent/buffer.rb
index 71d0443b2a..4b06de4bf3 100644
--- a/lib/fluent/buffer.rb
+++ b/lib/fluent/buffer.rb
@@ -112,7 +112,7 @@ def write_to(io)
 
     def msgpack_each(&block)
       open {|io|
-        u = MessagePack::Unpacker.new(io)
+        u = Fluent::Engine.msgpack_factory.unpacker(io)
         begin
           u.each(&block)
         rescue EOFError
diff --git a/lib/fluent/command/cat.rb b/lib/fluent/command/cat.rb
index db7c0ed80f..49f4aaa469 100644
--- a/lib/fluent/command/cat.rb
+++ b/lib/fluent/command/cat.rb
@@ -289,7 +289,7 @@ def abort_message(time, record)
 
 when 'msgpack'
   begin
-    u = MessagePack::Unpacker.new($stdin)
+    u = Fluent::Engine.msgpack_factory.unpacker($stdin)
     u.each {|record|
       w.write(record)
     }
diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index c347e7a4eb..49e3be2229 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -19,6 +19,16 @@ module Fluent
   require 'fluent/root_agent'
 
   class EngineClass
+    class DummyMessagePackFactory
+      def packer(io = nil)
+        MessagePack::Packer.new(io)
+      end
+
+      def unpacker(io = nil)
+        MessagePack::Unpacker.new(io)
+      end
+    end
+
     def initialize
       @root_agent = nil
       @event_router = nil
@@ -30,6 +40,8 @@ def initialize
       @log_event_queue = []
 
       @suppress_config_dump = false
+
+      @msgpack_factory = DummyMessagePackFactory.new
     end
 
     MATCH_CACHE_SIZE = 1024
@@ -37,6 +49,7 @@ def initialize
 
     attr_reader :root_agent
     attr_reader :matches, :sources
+    attr_reader :msgpack_factory
 
     def init(opts = {})
       BasicSocket.do_not_reverse_lookup = true
diff --git a/lib/fluent/event.rb b/lib/fluent/event.rb
index 252fe92f72..f07bfd1a1e 100644
--- a/lib/fluent/event.rb
+++ b/lib/fluent/event.rb
@@ -27,7 +27,7 @@ def each(&block)
     end
 
     def to_msgpack_stream
-      out = MessagePack::Packer.new # MessagePack::Packer is fastest way to serialize events
+      out = Fluent::Engine.msgpack_factory.packer
       each {|time,record|
         out.write([time,record])
       }
@@ -143,7 +143,7 @@ def repeatable?
 
     def each(&block)
       # TODO format check
-      unpacker = MessagePack::Unpacker.new
+      unpacker = Fluent::Engine.msgpack_factory.unpacker
       unpacker.feed_each(@data, &block)
       nil
     end
diff --git a/lib/fluent/plugin/buf_memory.rb b/lib/fluent/plugin/buf_memory.rb
index 4773d1707e..f725d503d5 100644
--- a/lib/fluent/plugin/buf_memory.rb
+++ b/lib/fluent/plugin/buf_memory.rb
@@ -57,7 +57,7 @@ def write_to(io)
 
     # optimize
     def msgpack_each(&block)
-      u = MessagePack::Unpacker.new
+      u = Fluent::Engine.msgpack_factory.unpacker
       u.feed_each(@data, &block)
     end
   end
diff --git a/lib/fluent/plugin/exec_util.rb b/lib/fluent/plugin/exec_util.rb
index 35fd05853a..7fe262ecf6 100644
--- a/lib/fluent/plugin/exec_util.rb
+++ b/lib/fluent/plugin/exec_util.rb
@@ -58,7 +58,7 @@ def call(io)
 
     class MessagePackParser < Parser
       def call(io)
-        @u = MessagePack::Unpacker.new(io)
+        @u = Fluent::Engine.msgpack_factory.unpacker(io)
         begin
           @u.each(&@on_message)
         rescue EOFError
diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index 2bfcee690d..0a937cbd61 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -219,7 +219,7 @@ def on_read(data)
         else
           m = method(:on_read_msgpack)
           @serializer = :to_msgpack.to_proc
-          @u = MessagePack::Unpacker.new
+          @u = Fluent::Engine.msgpack_factory.unpacker
         end
 
         (class << self; self; end).module_eval do
diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index 2285100501..c91c16b47a 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -191,7 +191,7 @@ def on_request(path_info, params)
 
     def parse_params_default(params)
       record = if msgpack = params['msgpack']
-                 MessagePack.unpack(msgpack)
+                 Engine.msgpack_factory.unpacker.feed(msgpack).read
                elsif js = params['json']
                  JSON.parse(js)
                else
diff --git a/lib/fluent/plugin/in_stream.rb b/lib/fluent/plugin/in_stream.rb
index ce2f72e636..e254531138 100644
--- a/lib/fluent/plugin/in_stream.rb
+++ b/lib/fluent/plugin/in_stream.rb
@@ -130,7 +130,7 @@ def on_read(data)
           @y.on_parse_complete = @on_message
         else
           m = method(:on_read_msgpack)
-          @u = MessagePack::Unpacker.new
+          @u = Fluent::Engine.msgpack_factory.unpacker
         end
 
         (class << self; self; end).module_eval do
diff --git a/lib/fluent/plugin/out_stream.rb b/lib/fluent/plugin/out_stream.rb
index f5d51bd79c..41db049c6d 100644
--- a/lib/fluent/plugin/out_stream.rb
+++ b/lib/fluent/plugin/out_stream.rb
@@ -65,7 +65,7 @@ def write(chunk)
         chain = NullOutputChain.instance
         chunk.open {|io|
           # TODO use MessagePackIoEventStream
-          u = MessagePack::Unpacker.new(io)
+          u = Fluent::Engine.msgpack_factory.unpacker(io)
           begin
             u.each {|(tag,entries)|
               es = MultiEventStream.new
diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index 19a31c7e8c..f2b04e9058 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -176,7 +176,7 @@ def input_forward_main(ipr, pid)
     end
 
     def read_event_stream(r, &block)
-      u = MessagePack::Unpacker.new(r)
+      u = Fluent::Engine.msgpack_factory.unpacker(r)
       begin
         #buf = ''
         #map = {}
diff --git a/test/plugin/test_in_forward.rb b/test/plugin/test_in_forward.rb
index 329725f19e..a012ba7214 100644
--- a/test/plugin/test_in_forward.rb
+++ b/test/plugin/test_in_forward.rb
@@ -59,7 +59,7 @@ def test_message
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        send_data [tag, time, record].to_msgpack
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
       }
     end
   end
@@ -77,7 +77,7 @@ def test_forward
       d.expected_emits.each {|tag,time,record|
         entries << [time, record]
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
@@ -92,9 +92,9 @@ def test_packed_forward
     d.run do
       entries = ''
       d.expected_emits.each {|tag,time,record|
-        [time, record].to_msgpack(entries)
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
@@ -128,7 +128,7 @@ def test_send_large_chunk_warning
     assert chunk.size < (32 * 1024 * 1024)
 
     d.run do
-      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
         d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
@@ -157,7 +157,7 @@ def test_send_large_chunk_only_warning
     chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
 
     d.run do
-      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
         d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
@@ -184,7 +184,7 @@ def test_send_large_chunk_limit
 
     # d.run => send_data
     d.run do
-      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
         d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
diff --git a/test/plugin/test_in_stream.rb b/test/plugin/test_in_stream.rb
index 0c89c806a7..5eb13c5b8a 100644
--- a/test/plugin/test_in_stream.rb
+++ b/test/plugin/test_in_stream.rb
@@ -17,7 +17,7 @@ def test_time
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        send_data [tag, 0, record].to_msgpack
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
       }
     end
   end
@@ -32,7 +32,7 @@ def test_message
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        send_data [tag, time, record].to_msgpack
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
       }
     end
   end
@@ -50,7 +50,7 @@ def test_forward
       d.expected_emits.each {|tag,time,record|
         entries << [time, record]
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
@@ -65,9 +65,9 @@ def test_packed_forward
     d.run do
       entries = ''
       d.expected_emits.each {|tag,time,record|
-        [time, record].to_msgpack(entries)
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
diff --git a/test/plugin/test_out_copy.rb b/test/plugin/test_out_copy.rb
index 4ff5523019..04725d198c 100644
--- a/test/plugin/test_out_copy.rb
+++ b/test/plugin/test_out_copy.rb
@@ -86,7 +86,7 @@ def test_msgpack_es_emit_bug
 
     es = if defined?(MessagePack::Packer)
            time = Time.parse("2013-05-26 06:37:22 UTC").to_i
-           packer = MessagePack::Packer.new
+           packer = Fluent::Engine.msgpack_factory.packer
            packer.pack([time, {"a" => 1}])
            packer.pack([time, {"a" => 2}])
            Fluent::MessagePackEventStream.new(packer.to_s)

From c3185fc6e07aaab52a64ba1937287860f17fe540 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 30 Oct 2015 15:25:52 +0900
Subject: [PATCH 011/193] Use *args to forward all method arguments

---
 lib/fluent/engine.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index 49e3be2229..046a2d98af 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -20,12 +20,12 @@ module Fluent
 
   class EngineClass
     class DummyMessagePackFactory
-      def packer(io = nil)
-        MessagePack::Packer.new(io)
+      def packer(*args)
+        MessagePack::Packer.new(*args)
       end
 
-      def unpacker(io = nil)
-        MessagePack::Unpacker.new(io)
+      def unpacker(*args)
+        MessagePack::Unpacker.new(*args)
       end
     end
 

From d6711233dd90f176760cf6adfa7d2202d3da0a4d Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 4 Nov 2015 20:02:59 +0900
Subject: [PATCH 012/193] Update test-unit gem to 3.1.4

---
 fluentd.gemspec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index 3f3f08800d..dc9f6a3db2 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -35,6 +35,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("simplecov", ["~> 0.6.4"])
   gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.0.2"])
+  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
 end

From f03ee90254be76befd471ac1f4749012a5b7e941 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 4 Nov 2015 23:02:50 +0900
Subject: [PATCH 013/193] v0.12.17

---
 ChangeLog             | 21 +++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 7737f8588a..8f37e68cff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,26 @@
 # v0.12
 
+## Release 0.12.17 - 2015/11/04
+
+### New features / Enhancement
+
+* Engine: Add Engine#msgpack_factory for v0.14 compatibility
+  https://github.com/fluent/fluentd/pull/693
+* Log shutdown plugin info to check shutdown sequence
+  https://github.com/fluent/fluentd/pull/689
+* in_monitor_agent: Emit plugin info to use existing plugins
+  https://github.com/fluent/fluentd/pull/670
+* config: Improve describing plugin parameters
+  https://github.com/fluent/fluentd/pull/683
+
+### Bug fixes
+
+* Tempfile should be binary mode
+  https://github.com/fluent/fluentd/pull/691
+* filter_record_transformer: Don't use popular name to prevent field overwrite when enable_ruby is true
+  https://github.com/fluent/fluentd/pull/687
+
+
 ## Release 0.12.16 - 2015/09/30
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 77dcd1534a..0a399e8918 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.16'
+  VERSION = '0.12.17'
 
 end

From 694af8baa967e91f1f7992f47ef4bb7d136b95e6 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 13 Nov 2015 19:50:02 +0900
Subject: [PATCH 014/193] Merge pull request #701 from
 cosmo0920/add-descriptions-out_file

Add descriptions to out_file plugin parameters
---
 lib/fluent/plugin/out_file.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/fluent/plugin/out_file.rb b/lib/fluent/plugin/out_file.rb
index 37b5718ecd..d5ba98ee14 100644
--- a/lib/fluent/plugin/out_file.rb
+++ b/lib/fluent/plugin/out_file.rb
@@ -23,9 +23,13 @@ class FileOutput < TimeSlicedOutput
       'gzip' => :gz,
     }
 
+    desc "The Path of the file."
     config_param :path, :string
+    desc "The format of the file content. The default is out_file."
     config_param :format, :string, :default => 'out_file'
+    desc "The flushed chunk is appended to existence file or not."
     config_param :append, :bool, :default => false
+    desc "Compress flushed file."
     config_param :compress, :default => nil do |val|
       c = SUPPORTED_COMPRESS[val]
       unless c
@@ -33,6 +37,7 @@ class FileOutput < TimeSlicedOutput
       end
       c
     end
+    desc "Create symlink to temporary buffered file when buffer_type is file."
     config_param :symlink_path, :string, :default => nil
 
     def initialize

From a9131dbcda08781f4b3d686ec2ca7bc50882d30f Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 24 Nov 2015 15:09:18 +0900
Subject: [PATCH 015/193] Merge pull request #707 from
 fluent/forward-reject-empty

out_forward raises ZeroDivisionError when no <server> is available
---
 lib/fluent/plugin/out_forward.rb | 4 ++++
 test/plugin/test_out_forward.rb  | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index b5e3976684..58a97d5599 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -129,6 +129,10 @@ def configure(conf)
         end
         log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight, :plugin_id=>plugin_id
       }
+
+      if @nodes.empty?
+        raise ConfigError, "forward output plugin requires at least one <server> is required"
+      end
     end
 
     def start
diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb
index c3a4dc2076..f2db81895e 100644
--- a/test/plugin/test_out_forward.rb
+++ b/test/plugin/test_out_forward.rb
@@ -78,6 +78,12 @@ def test_configure_dns_round_robin
     assert_equal true, d.instance.nodes.first.conf.dns_round_robin
   end
 
+  def test_configure_no_server
+    assert_raise(Fluent::ConfigError, 'forward output plugin requires at least one <server> is required') do
+      create_driver('')
+    end
+  end
+
   def test_phi_failure_detector
     d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
     node = d.instance.nodes.first

From 677278417bceb588480d10c96e328c000c54ea72 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 26 Nov 2015 01:17:22 +0900
Subject: [PATCH 016/193] Merge pull request #708 from
 fluent/support-process-name-configuration

Add process_name parameter in system to change fluentd's process name
---
 lib/fluent/supervisor.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index 66588f8274..fcc915484e 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -111,6 +111,7 @@ def initialize(opt)
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
       @rpc_server = nil
+      @process_name = nil
 
       @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
@@ -306,6 +307,7 @@ def stop_rpc_server
     def supervise(&block)
       start_time = Time.now
 
+      Process.setproctitle("supervisor:#{@process_name}") if @process_name
       $log.info "starting fluentd-#{Fluent::VERSION}"
       @main_pid = fork do
         main_process(&block)
@@ -332,6 +334,8 @@ def supervise(&block)
     end
 
     def main_process(&block)
+      Process.setproctitle("worker:#{@process_name}") if @process_name
+
       begin
         block.call
 
@@ -462,6 +466,7 @@ class SystemConfig
       config_param :without_source, :bool, :default => nil
       config_param :rpc_endpoint, :string, :default => nil
       config_param :enable_get_dump, :bool, :default => nil
+      config_param :process_name, :default => nil
 
       def initialize(conf)
         super()
@@ -478,6 +483,7 @@ def apply(supervisor)
           @without_source = system.without_source unless system.without_source.nil?
           @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
           @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
+          @process_name = system.process_name unless system.process_name.nil?
         }
       end
     end

From 4e7e8019a542972e6de50a7137dfe27848f7ddf2 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 26 Nov 2015 01:17:50 +0900
Subject: [PATCH 017/193] Merge pull request #702 from
 cosmo0920/add-parser-test-driver

---
 lib/fluent/test.rb             |  1 +
 lib/fluent/test/parser_test.rb | 66 ++++++++++++++++++++++++++++++++++
 test/test_parser.rb            | 62 ++++++++++++++++++++++----------
 3 files changed, 110 insertions(+), 19 deletions(-)
 create mode 100644 lib/fluent/test/parser_test.rb

diff --git a/lib/fluent/test.rb b/lib/fluent/test.rb
index 150230e1c1..a6edd9395a 100644
--- a/lib/fluent/test.rb
+++ b/lib/fluent/test.rb
@@ -20,5 +20,6 @@
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
 require 'fluent/test/filter_test'
+require 'fluent/test/parser_test'
 
 $log ||= Fluent::Log.new(Fluent::Test::DummyLogDevice.new)
diff --git a/lib/fluent/test/parser_test.rb b/lib/fluent/test/parser_test.rb
new file mode 100644
index 0000000000..949ab833fa
--- /dev/null
+++ b/lib/fluent/test/parser_test.rb
@@ -0,0 +1,66 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Test
+    class ParserTestDriver
+      def initialize(klass_or_str, format=nil, conf={}, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.module_eval(&block)
+          end
+          case klass_or_str.instance_method(:initialize).arity
+          when 0
+            @instance = klass_or_str.new
+          when -2
+            # for RegexpParser
+            @instance = klass_or_str.new(format, conf)
+          end
+        elsif klass_or_str.is_a?(String)
+          @instance = TextParser::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          io = StringIO.new(conf)
+          @config = Config::Parser.parse(io, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def parse(text, &block)
+        @instance.parse(text, &block)
+      end
+    end
+  end
+end
diff --git a/test/test_parser.rb b/test/test_parser.rb
index ead7d095fc..b18181456d 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -39,6 +39,31 @@ def test_call
     end
   end
 
+  class BaseParserTestWithTestDriver < ::Test::Unit::TestCase
+    include ParserTest
+
+    def create_driver(conf={})
+      Fluent::Test::ParserTestDriver.new(Fluent::Parser).configure(conf)
+    end
+
+    def test_init
+      d = create_driver
+      assert_true d.instance.estimate_current_event
+    end
+
+    def test_configure_against_string_literal
+      d = create_driver('keep_time_key true')
+      assert_true d.instance.keep_time_key
+    end
+
+    def test_parse
+      d = create_driver
+      assert_raise NotImplementedError do
+        d.parse('')
+      end
+    end
+  end
+
   class TimeParserTest < ::Test::Unit::TestCase
     include ParserTest
 
@@ -94,19 +119,19 @@ def test_parse_with_typed
     def test_parse_with_configure
       # Specify conf by configure method instaed of intializer
       regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-      parser = TextParser::RegexpParser.new(regexp)
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, regexp)
       parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
       internal_test_case(parser)
-      assert_equal(regexp, parser.patterns['format'])
-      assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.patterns['time_format'])
+      assert_equal(regexp, parser.instance.patterns['format'])
+      assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_typed_and_name_separator
-      internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
+      internal_test_case(Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
     end
 
     def test_parse_with_time_key
-      parser = TextParser::RegexpParser.new(/(?<logtime>[^\]]*)/)
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, /(?<logtime>[^\]]*)/)
       parser.configure(
         'time_format'=>"%Y-%m-%d %H:%M:%S %z",
         'time_key'=>'logtime',
@@ -131,10 +156,9 @@ def test_parse_without_time
         assert_equal 34, record["age"]
       }
 
-      parser2 = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
-      parser2.configure('types'=>'name:string,user:string,age:bool')
-      parser2.time_default_current = false
-
+      parser2 = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
+      parser2.configure('types'=>'name:string,user:string,age:integer')
+      parser2.instance.estimate_current_event = false
       [parser2.parse(text), parser2.parse(text) { |time, record| return time, record}].each { |time, record|
         assert_equal "tagomori_satoshi", record["name"]
         assert_equal "tagomoris", record["user"]
@@ -145,7 +169,7 @@ def test_parse_without_time
     end
 
     def test_parse_with_keep_time_key
-      parser = TextParser::RegexpParser.new(
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser,
         Regexp.new(%q!(?<time>.*)!),
         'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
         'keep_time_key'=>'true',
@@ -157,7 +181,7 @@ def test_parse_with_keep_time_key
     end
 
     def test_parse_with_keep_time_key_with_typecast
-      parser = TextParser::RegexpParser.new(
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser,
         Regexp.new(%q!(?<time>.*)!),
         'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
         'keep_time_key'=>'true',
@@ -281,7 +305,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
     include ParserTest
 
     def setup
-      @parser = TextParser::TEMPLATE_REGISTRY.lookup('syslog').call
+      @parser = Fluent::Test::ParserTestDriver.new('syslog')
       @expected = {
         'host'    => '192.168.0.1',
         'ident'   => 'fluentd',
@@ -296,8 +320,8 @@ def test_parse
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
-      assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
-      assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
+      assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
+      assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_time_format
@@ -306,7 +330,7 @@ def test_parse_with_time_format
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
-      assert_equal('%b %d %M:%S:%H', @parser.patterns['time_format'])
+      assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_priority
@@ -315,8 +339,8 @@ def test_parse_with_priority
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       }
-      assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.patterns['format'])
-      assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
+      assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+      assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
     end
 
     def test_parse_without_colon
@@ -325,8 +349,8 @@ def test_parse_without_colon
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
-      assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
-      assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
+      assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
+      assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_keep_time_key

From aa80092cc55030d2304fd3be17233bd74932b369 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 26 Nov 2015 01:21:37 +0900
Subject: [PATCH 018/193] Merge pull request #710 from
 tagomoris/safe-detach-process

fix bug not to protect in-memory-buffer for multi thread emitting / run loop
---
 lib/fluent/process.rb | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index f2b04e9058..d49a41973f 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -232,26 +232,36 @@ def initialize(w, interval)
         @w = w
         @interval = interval
         @buffer = {}
+        @mutex = Mutex.new
         Thread.new(&method(:run))
       end
 
       def emit(tag, es)
-        if ms = @buffer[tag]
-          ms << es.to_msgpack_stream
-        else
-          @buffer[tag] = es.to_msgpack_stream
+        stream = es.to_msgpack_stream
+        @mutex.synchronize do
+          if @buffer[tag]
+            @buffer[tag] << stream
+          else
+            @buffer[tag] = stream
+          end
         end
       end
 
       def run
         while true
           sleep @interval
-          @buffer.keys.each {|tag|
-            if ms = @buffer.delete(tag)
-              [tag, ms].to_msgpack(@w)
-              #@w.write [tag, ms].to_msgpack
+
+          pairs = []
+          @mutex.synchronize do
+            @buffer.keys.each do |tag|
+              if msg = @buffer.delete(tag)
+                pairs << [tag, ms]
+              end
             end
-          }
+          end
+          pairs.each do |pair|
+            pair.to_msgpack(@w)
+          end
         end
       rescue
         $log.error "error on forwerder thread", :error=>$!.to_s

From 1080a53677608d4067cfdb9a4abbc6d3aadd2080 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Fri, 27 Nov 2015 13:36:55 +0900
Subject: [PATCH 019/193] Merge pull request #713 from
 okkez/add-descriptions-filter-record-transformer

Add descriptions to filter_record_transformer
---
 lib/fluent/plugin/filter_record_transformer.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 8129587ba9..66c8a86818 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -25,11 +25,17 @@ def initialize
       super
     end
 
+    desc 'A comma-delimited list of keys to delete.'
     config_param :remove_keys, :string, :default => nil
+    desc 'A comma-delimited list of keys to keep.'
     config_param :keep_keys, :string, :default => nil
+    desc 'Create new Hash to transform incoming data'
     config_param :renew_record, :bool, :default => false
+    desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
     config_param :renew_time_key, :string, :default => nil
+    desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
     config_param :enable_ruby, :bool, :default => false
+    desc 'Use original value type.'
     config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
 
     def configure(conf)

From 82147269348841f562e116d50f631657fbdafc31 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:00:41 +0900
Subject: [PATCH 020/193] Merge pull request #714 from
 okkez/add-descriptions-filter-stdout

Add description to filter-stdout
---
 lib/fluent/plugin/filter_stdout.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/plugin/filter_stdout.rb b/lib/fluent/plugin/filter_stdout.rb
index 7b45cd007e..9e49d3d10c 100644
--- a/lib/fluent/plugin/filter_stdout.rb
+++ b/lib/fluent/plugin/filter_stdout.rb
@@ -21,6 +21,7 @@ class StdoutFilter < Filter
     # for tests
     attr_reader :formatter
 
+    desc 'The format of the output.'
     config_param :format, :string, :default => 'stdout'
     # config_param :output_type, :string, :default => 'json' (StdoutFormatter defines this)
 

From f8b9f0c20464d86186db12ac3b316ba1d8d483a9 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:05:26 +0900
Subject: [PATCH 021/193] Merge pull request #715 from
 okkez/add-description-to-log-level

Add description to log level
---
 lib/fluent/log.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/log.rb b/lib/fluent/log.rb
index 7bffd33975..f14f27c32f 100644
--- a/lib/fluent/log.rb
+++ b/lib/fluent/log.rb
@@ -343,6 +343,7 @@ def enable_color(b = true)
   module PluginLoggerMixin
     def self.included(klass)
       klass.instance_eval {
+        desc 'Allows the user to set different levels of logging for each plugin.'
         config_param :log_level, :string, :default => nil, :alias => :@log_level
       }
     end

From 199cd75be805f3e2d9a0a601cedac786fe4d26ec Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:06:25 +0900
Subject: [PATCH 022/193] Merge pull request #716 from
 okkez/add-descriptions-in-exec

Add descriptions to in-exec
---
 lib/fluent/plugin/in_exec.rb | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/fluent/plugin/in_exec.rb b/lib/fluent/plugin/in_exec.rb
index fe31908dec..538e55eb8d 100644
--- a/lib/fluent/plugin/in_exec.rb
+++ b/lib/fluent/plugin/in_exec.rb
@@ -30,19 +30,27 @@ def initialize
       'msgpack' => :msgpack,
     }
 
+    desc 'The command (program) to execute.'
     config_param :command, :string
+    desc 'The format used to map the program output to the incoming event.(tsv,json,msgpack)'
     config_param :format, :default => :tsv do |val|
       f = SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f
       f
     end
+    desc 'Specify the comma-separated keys when using the tsv format.'
     config_param :keys, :default => [] do |val|
       val.split(',')
     end
+    desc 'Tag of the output events.'
     config_param :tag, :string, :default => nil
+    desc 'The key to use as the event tag instead of the value in the event record. '
     config_param :tag_key, :string, :default => nil
+    desc 'The key to use as the event time instead of the value in the event record.'
     config_param :time_key, :string, :default => nil
+    desc 'The format of the event time used for the time_key parameter.'
     config_param :time_format, :string, :default => nil
+    desc 'The interval time between periodic program runs.'
     config_param :run_interval, :time, :default => nil
 
     def configure(conf)

From e5c19b4b19b3b6f16bb4b3eb9173d029bb4d1f40 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:07:13 +0900
Subject: [PATCH 023/193] Merge pull request #717 from
 okkez/add-descriptions-in-forward

Add descriptions to in_forward
---
 lib/fluent/plugin/in_forward.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index 0a937cbd61..48581f5029 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -23,15 +23,20 @@ def initialize
       require 'fluent/plugin/socket_util'
     end
 
+    desc 'The port to listen to.'
     config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+    desc 'The bind address to listen to.'
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :backlog, :integer, :default => nil
     # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
+    desc 'The timeout time used to set linger option.'
     config_param :linger_timeout, :integer, :default => 0
     # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
     config_param :blocking_timeout, :time, :default => 0.5
 
+    desc 'Log warning if received chunk size is larger than this value.'
     config_param :chunk_size_warn_limit, :size, :default => nil
+    desc 'Received chunk is dropped if it is larger than this value.'
     config_param :chunk_size_limit, :size, :default => nil
 
     def configure(conf)

From 85013f9ea19727448b46f0611c825943b39aa9d3 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:17:49 +0900
Subject: [PATCH 024/193] Merge pull request #720 from
 okkez/add-descriptions-in-tcp

Add descriptions to in_tcp
---
 lib/fluent/plugin/in_tcp.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb
index 24623efe63..181d4bbc6d 100644
--- a/lib/fluent/plugin/in_tcp.rb
+++ b/lib/fluent/plugin/in_tcp.rb
@@ -21,6 +21,7 @@ class TcpInput < SocketUtil::BaseInput
     Plugin.register_input('tcp', self)
 
     config_set_default :port, 5170
+    desc 'The payload is read up to this character.'
     config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
     def listen(callback)

From fffddc0082a799ad9bc0175d89d6035d66b63464 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:18:34 +0900
Subject: [PATCH 025/193] Merge pull request #723 from
 okkez/add-descriptions-out-copy

Add description to out_copy plugin
---
 lib/fluent/plugin/out_copy.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/plugin/out_copy.rb b/lib/fluent/plugin/out_copy.rb
index 38d3624de0..6a0ec95086 100644
--- a/lib/fluent/plugin/out_copy.rb
+++ b/lib/fluent/plugin/out_copy.rb
@@ -18,6 +18,7 @@ module Fluent
   class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
 
+    desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, :default => false
 
     def initialize

From 98cedf2f6366840e9be6a490de54ad79b1e6b961 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:20:50 +0900
Subject: [PATCH 026/193] Merge pull request #722 from
 okkez/add-descriptions-buffer

Add descriptions buffer
---
 lib/fluent/buffer.rb            | 2 ++
 lib/fluent/plugin/buf_file.rb   | 2 ++
 lib/fluent/plugin/buf_memory.rb | 1 +
 3 files changed, 5 insertions(+)

diff --git a/lib/fluent/buffer.rb b/lib/fluent/buffer.rb
index 4b06de4bf3..30dda85c54 100644
--- a/lib/fluent/buffer.rb
+++ b/lib/fluent/buffer.rb
@@ -139,7 +139,9 @@ def enable_parallel(b=true)
     # This configuration assumes plugins to send records to a remote server.
     # Local file based plugins which should provide more reliability and efficiency
     # should override buffer_chunk_limit with a larger size.
+    desc 'The size of each buffer chunk.'
     config_param :buffer_chunk_limit, :size, :default => 8*1024*1024
+    desc 'The length limit of the chunk queue.'
     config_param :buffer_queue_limit, :integer, :default => 256
 
     alias chunk_limit buffer_chunk_limit
diff --git a/lib/fluent/plugin/buf_file.rb b/lib/fluent/plugin/buf_file.rb
index 868fa1060d..15f161b2f0 100644
--- a/lib/fluent/plugin/buf_file.rb
+++ b/lib/fluent/plugin/buf_file.rb
@@ -82,7 +82,9 @@ def initialize
       @uri_parser = URI::Parser.new
     end
 
+    desc 'The path where buffer chunks are stored.'
     config_param :buffer_path, :string
+    desc 'If true, queued chunks are flushed at shutdown process.'
     config_param :flush_at_shutdown, :bool, :default => false
 
     # 'symlink_path' is currently only for out_file.
diff --git a/lib/fluent/plugin/buf_memory.rb b/lib/fluent/plugin/buf_memory.rb
index f725d503d5..81d69256d8 100644
--- a/lib/fluent/plugin/buf_memory.rb
+++ b/lib/fluent/plugin/buf_memory.rb
@@ -70,6 +70,7 @@ def initialize
       super
     end
 
+    desc 'If true, queued chunks are flushed at shutdown process. Otherwise queued chunks are discarded'
     config_param :flush_at_shutdown, :bool, :default => true
     # Overwrite default BasicBuffer#buffer_queue_limit
     # to limit total memory usage upto 512MB.

From 2bf623525d3e61cd1bbc00d6c3ddb3c138c6721e Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Sat, 28 Nov 2015 01:21:29 +0900
Subject: [PATCH 027/193] Merge pull request #721 from
 okkez/add-descriptions-base-input

Add descriptions to Fluent::SocketUtil::BaseInput
---
 lib/fluent/plugin/socket_util.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/fluent/plugin/socket_util.rb b/lib/fluent/plugin/socket_util.rb
index 4b076a8ae2..0f1e1496a7 100644
--- a/lib/fluent/plugin/socket_util.rb
+++ b/lib/fluent/plugin/socket_util.rb
@@ -94,10 +94,15 @@ def initialize
         require 'fluent/parser'
       end
 
+      desc 'Tag of output events.'
       config_param :tag, :string
+      desc 'The format of the payload.'
       config_param :format, :string
+      desc 'The port to listen to.'
       config_param :port, :integer, :default => 5150
+      desc 'The bind address to listen to.'
       config_param :bind, :string, :default => '0.0.0.0'
+      desc "The field name of the client's hostname."
       config_param :source_host_key, :string, :default => nil
       config_param :blocking_timeout, :time, :default => 0.5
 

From ed1701b1d71a204f5c2462358825ffd6f6697cb9 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sun, 29 Nov 2015 01:16:53 +0900
Subject: [PATCH 028/193] Merge pull request #724 from
 fluent/fix_delayed_forwarder

Fix DelayedForwarder#run and add unit test
---
 lib/fluent/process.rb |  2 +-
 test/test_process.rb  | 47 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 test/test_process.rb

diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index d49a41973f..517a5e4b8f 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -254,7 +254,7 @@ def run
           pairs = []
           @mutex.synchronize do
             @buffer.keys.each do |tag|
-              if msg = @buffer.delete(tag)
+              if ms = @buffer.delete(tag)
                 pairs << [tag, ms]
               end
             end
diff --git a/test/test_process.rb b/test/test_process.rb
new file mode 100644
index 0000000000..ae846c3529
--- /dev/null
+++ b/test/test_process.rb
@@ -0,0 +1,47 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/event'
+require 'stringio'
+require 'msgpack'
+
+module FluentProcessTest
+  class DelayedForwarderTest < Test::Unit::TestCase
+    include Fluent
+
+    test 'run and emit' do
+      io = StringIO.new
+      fwd = Fluent::DetachProcessManager::DelayedForwarder.new(io, 0.001)
+
+      num_events_per_tag = 5000
+      num_tags = 20
+
+      now = Time.now.to_i
+      record = {'key' => 'value'}
+      (num_tags * num_events_per_tag).times do |i|
+        tag = "foo.bar#{i % num_tags}"
+        fwd.emit(tag, OneEventStream.new(now, record))
+      end
+      sleep 0.5
+
+      io.pos = 0
+
+      tags = {}
+      MessagePack::Unpacker.new(io).each do |tag_and_msgpacks|
+        tag, ms = *tag_and_msgpacks
+        tags[tag] ||= ''
+        tags[tag] << ms
+      end
+
+      assert_equal(num_tags, tags.size)
+      num_tags.times do |i|
+        tag = "foo.bar#{i % num_tags}"
+        ms = tags[tag]
+        count = 0
+        MessagePack::Unpacker.new(StringIO.new(ms)).each do |x|
+          count += 1
+        end
+        assert_equal(num_events_per_tag, count)
+      end
+    end
+  end
+end

From 552d101a4550474de7305f7503055191679a07e5 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:06:34 +0900
Subject: [PATCH 029/193] Merge pull request #718 from
 okkez/add-descriptions-in-http

Add descriptions to in_http
---
 lib/fluent/plugin/in_http.rb | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index c91c16b47a..a44ce0b3fd 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -30,16 +30,25 @@ def initialize
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
+    desc 'The port to listen to.'
     config_param :port, :integer, :default => 9880
+    desc 'The bind address to listen to.'
     config_param :bind, :string, :default => '0.0.0.0'
+    desc 'The size limit of the POSTed element. Default is 32MB.'
     config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
+    desc 'The timeout limit for keeping the connection alive.'
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
+    desc 'Add HTTP_ prefix headers to the record.'
     config_param :add_http_headers, :bool, :default => false
+    desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, :default => false
+    desc 'The format of the HTTP body.'
     config_param :format, :string, :default => 'default'
     config_param :blocking_timeout, :time, :default => 0.5
+    desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, :default => nil
+    desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, :default => false
 
     def configure(conf)

From c1dd17e9e69be3c0097196bbebecde6cf7e528a7 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:07:09 +0900
Subject: [PATCH 030/193] Merge pull request #727 from
 okkez/add-descriptions-in-unix

Add descriptions to in_unix
---
 lib/fluent/plugin/in_stream.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/fluent/plugin/in_stream.rb b/lib/fluent/plugin/in_stream.rb
index e254531138..0328d30d33 100644
--- a/lib/fluent/plugin/in_stream.rb
+++ b/lib/fluent/plugin/in_stream.rb
@@ -164,7 +164,9 @@ def on_close
   class UnixInput < StreamInput
     Plugin.register_input('unix', self)
 
+    desc 'The path to your Unix Domain Socket.'
     config_param :path, :string, :default => DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
     config_param :backlog, :integer, :default => nil
 
     def configure(conf)

From 45249709167e5dd3eaef684c267b9a3fb12a5deb Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:07:32 +0900
Subject: [PATCH 031/193] Merge pull request #728 from
 okkez/add-descriptions-out-exec-filter

Add descriptions to out_exec_filter plugin
---
 lib/fluent/plugin/out_exec_filter.rb | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/fluent/plugin/out_exec_filter.rb b/lib/fluent/plugin/out_exec_filter.rb
index 971090c9aa..261a7571c1 100644
--- a/lib/fluent/plugin/out_exec_filter.rb
+++ b/lib/fluent/plugin/out_exec_filter.rb
@@ -25,33 +25,44 @@ def initialize
       require 'fluent/timezone'
     end
 
+    desc 'The command (program) to execute.'
     config_param :command, :string
 
     config_param :remove_prefix, :string, :default => nil
     config_param :add_prefix, :string, :default => nil
 
+    desc "The format used to map the incoming event to the program input.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :in_format, :default => :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported in_format '#{val}'" unless f
       f
     end
+    desc 'Specify comma-separated values for tsv format.'
     config_param :in_keys, :default => [] do |val|
       val.split(',')
     end
+    desc 'The name of the key to use as the event tag.'
     config_param :in_tag_key, :default => nil
+    desc 'The name of the key to use as the event time.'
     config_param :in_time_key, :default => nil
+    desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
     config_param :in_time_format, :default => nil
 
+    desc "The format used to process the program output.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :out_format, :default => :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported out_format '#{val}'" unless f
       f
     end
+    desc 'Specify comma-separated values for tsv format.'
     config_param :out_keys, :default => [] do |val|  # for tsv format
       val.split(',')
     end
+    desc 'The name of the key to use as the event tag.'
     config_param :out_tag_key, :default => nil
+    desc 'The name of the key to use as the event time.'
     config_param :out_time_key, :default => nil
+    desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
     config_param :out_time_format, :default => nil
 
     config_param :tag, :string, :default => nil
@@ -59,10 +70,14 @@ def initialize
     config_param :time_key, :string, :default => nil
     config_param :time_format, :string, :default => nil
 
+    desc 'If true, use localtime with in_time_format.'
     config_param :localtime, :bool, :default => true
+    desc 'If true, use timezone with in_time_format.'
     config_param :timezone, :string, :default => nil
+    desc 'The number of spawned process for command.'
     config_param :num_children, :integer, :default => 1
 
+    desc 'Respawn command when command exit.'
     # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
     config_param :child_respawn, :string, :default => nil
 

From 6f4fe9cb5119abfab39cf644fd0a08119d098006 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:07:53 +0900
Subject: [PATCH 032/193] Merge pull request #729 from
 okkez/add-descriptions-out-forward

Add descriptions to out_forward plugin
---
 lib/fluent/plugin/out_forward.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 58a97d5599..9e85b84cce 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -36,7 +36,9 @@ def initialize
       @nodes = []  #=> [Node]
     end
 
+    desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, :default => 60
+    desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
     config_param :heartbeat_type, :default => :udp do |val|
       case val.downcase
       when 'tcp'
@@ -49,19 +51,28 @@ def initialize
         raise ConfigError, "forward output heartbeat type should be 'tcp', 'udp', or 'none'"
       end
     end
+    desc 'The interval of the heartbeat packer.'
     config_param :heartbeat_interval, :time, :default => 1
+    desc 'The wait time before accepting a server fault recovery.'
     config_param :recover_wait, :time, :default => 10
+    desc 'The hard timeout used to detect server failure.'
     config_param :hard_timeout, :time, :default => 60
+    desc 'Set TTL to expire DNS cache in seconds.'
     config_param :expire_dns_cache, :time, :default => nil  # 0 means disable cache
+    desc 'The threshold parameter used to detect server faults.'
     config_param :phi_threshold, :integer, :default => 16
+    desc 'Use the "Phi accrual failure detector" to detect server failure.'
     config_param :phi_failure_detector, :bool, :default => true
 
     # if any options added that requires extended forward api, fix @extend_internal_protocol
 
+    desc 'Change the protocol to at-least-once.'
     config_param :require_ack_response, :bool, :default => false  # require in_forward to respond with ack
+    desc 'This option is used when require_ack_response is true.'
     config_param :ack_response_timeout, :time, :default => 190  # 0 means do not wait for ack responses
     # Linux default tcp_syn_retries is 5 (in many environment)
     # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
+    desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, :default => false # heartbeat_type 'udp' is not available for this
 
     attr_reader :nodes

From cdaddc41f7e79ee9e99d33f905f4882c6ec1b747 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:08:07 +0900
Subject: [PATCH 033/193] Merge pull request #730 from
 okkez/add-descriptions-out-stdout

Add descriptions to out_stdout plugin
---
 lib/fluent/plugin/out_stdout.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/plugin/out_stdout.rb b/lib/fluent/plugin/out_stdout.rb
index ba45d710fb..eb289217c2 100644
--- a/lib/fluent/plugin/out_stdout.rb
+++ b/lib/fluent/plugin/out_stdout.rb
@@ -18,6 +18,7 @@ module Fluent
   class StdoutOutput < Output
     Plugin.register_output('stdout', self)
 
+    desc 'Output format.(json,hash)'
     config_param :output_type, :default => 'json'
 
     def configure(conf)

From 5391dd19a4a204be5137859bf7f43f6d851979a9 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:09:12 +0900
Subject: [PATCH 034/193] Merge pull request #732 from
 okkez/add-descriptions-in-syslog

Add descriptions to in_syslog
---
 lib/fluent/plugin/in_syslog.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/fluent/plugin/in_syslog.rb b/lib/fluent/plugin/in_syslog.rb
index aa767c5009..ae5476f5e9 100644
--- a/lib/fluent/plugin/in_syslog.rb
+++ b/lib/fluent/plugin/in_syslog.rb
@@ -64,9 +64,13 @@ def initialize
       require 'fluent/plugin/socket_util'
     end
 
+    desc 'The port to listen to.'
     config_param :port, :integer, :default => 5140
+    desc 'The bind address to listen to.'
     config_param :bind, :string, :default => '0.0.0.0'
+    desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
+    desc 'The transport protocol used to receive logs.(udp, tcp)'
     config_param :protocol_type, :default => :udp do |val|
       case val.downcase
       when 'tcp'
@@ -77,7 +81,9 @@ def initialize
         raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
       end
     end
+    desc 'If true, add source host to event record.'
     config_param :include_source_host, :bool, :default => false
+    desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, :default => 'source_host'.freeze
     config_param :blocking_timeout, :time, :default => 0.5
 

From a6904e31d27e7927007aa87012fd21bc844503ca Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:09:31 +0900
Subject: [PATCH 035/193] Merge pull request #712 from
 cosmo0920/add-formatter-test-driver

Add FormatterTestDriver
---
 lib/fluent/test.rb                |  1 +
 lib/fluent/test/formatter_test.rb | 60 +++++++++++++++++++++++++++++++
 test/test_formatter.rb            | 37 +++++++++++++++++--
 3 files changed, 96 insertions(+), 2 deletions(-)
 create mode 100644 lib/fluent/test/formatter_test.rb

diff --git a/lib/fluent/test.rb b/lib/fluent/test.rb
index a6edd9395a..f552f7a20d 100644
--- a/lib/fluent/test.rb
+++ b/lib/fluent/test.rb
@@ -21,5 +21,6 @@
 require 'fluent/test/output_test'
 require 'fluent/test/filter_test'
 require 'fluent/test/parser_test'
+require 'fluent/test/formatter_test'
 
 $log ||= Fluent::Log.new(Fluent::Test::DummyLogDevice.new)
diff --git a/lib/fluent/test/formatter_test.rb b/lib/fluent/test/formatter_test.rb
new file mode 100644
index 0000000000..fedce36f33
--- /dev/null
+++ b/lib/fluent/test/formatter_test.rb
@@ -0,0 +1,60 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Test
+    class FormatterTestDriver
+      def initialize(klass_or_str, proc=nil, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.module_eval(&block)
+          end
+          @instance = klass_or_str.new
+        elsif klass_or_str.is_a?(String)
+          @instance = TextFormatter::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          io = StringIO.new(conf)
+          @config = Config::Parser.parse(io, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def format(tag, time, record)
+        @instance.format(tag, time, record)
+      end
+    end
+  end
+end
diff --git a/test/test_formatter.rb b/test/test_formatter.rb
index 38f26c6a92..ca94de9113 100644
--- a/test/test_formatter.rb
+++ b/test/test_formatter.rb
@@ -48,11 +48,33 @@ def test_call
     end
   end
 
+  class BaseFormatterTestWithTestDriver < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def create_driver(conf={})
+      formatter = Fluent::Test::FormatterTestDriver.new(Formatter).configure(conf)
+    end
+
+    def test_call
+      d = create_driver
+      assert_raise NotImplementedError do
+        d.format('tag', Engine.now, {})
+      end
+    end
+
+    def test_call_with_string_literal_configure
+      d = create_driver('')
+      assert_raise NotImplementedError do
+        d.format('tag', Engine.now, {})
+      end
+    end
+  end
+
   class OutFileFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::TEMPLATE_REGISTRY.lookup('out_file').call
+      @formatter = Fluent::Test::FormatterTestDriver.new('out_file')
       @time = Engine.now
     end
 
@@ -87,13 +109,24 @@ def test_format_without_time_and_tag
 
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
+
+    def test_format_without_time_and_tag_against_string_literal_configure
+      @formatter.configure(%[
+        utc         true
+        output_tag  false
+        output_time false
+      ])
+      formatted = @formatter.format('tag', @time, record)
+
+      assert_equal("#{Yajl.dump(record)}\n", formatted)
+    end
   end
 
   class JsonFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::JSONFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::JSONFormatter)
       @time = Engine.now
     end
 

From 2b20c3f07fc9659e65550585c1ca5f3adeb98593 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Dec 2015 18:09:59 +0900
Subject: [PATCH 036/193] Merge pull request #706 from
 okkez/show-plugin-config-ancestors

Show plugin config ancestors
---
 lib/fluent/config/configure_proxy.rb |  4 ++--
 lib/fluent/configurable.rb           |  4 ++--
 lib/fluent/supervisor.rb             | 14 +++++++++++++-
 test/config/test_configure_proxy.rb  | 11 +----------
 4 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/lib/fluent/config/configure_proxy.rb b/lib/fluent/config/configure_proxy.rb
index 1e9b1d744f..5083da0b4a 100644
--- a/lib/fluent/config/configure_proxy.rb
+++ b/lib/fluent/config/configure_proxy.rb
@@ -235,7 +235,7 @@ def config_section(name, *args, &block)
       end
 
       def dump(level = 0)
-        dumped_config = "\n"
+        dumped_config = ""
         indent = " " * level
         @params.each do |name, config|
           dumped_config << "#{indent}#{name}: #{config[1][:type]}: <#{@defaults[name].inspect}>"
@@ -243,7 +243,7 @@ def dump(level = 0)
           dumped_config << "\n"
         end
         @sections.each do |section_name, sub_proxy|
-          dumped_config << "#{indent}#{section_name}#{sub_proxy.dump(level + 1)}"
+          dumped_config << "#{indent}#{section_name}\n#{sub_proxy.dump(level + 1)}"
         end
         dumped_config
       end
diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
index e7f3f95284..a17008ed2a 100644
--- a/lib/fluent/configurable.rb
+++ b/lib/fluent/configurable.rb
@@ -130,8 +130,8 @@ def merged_configure_proxy
         configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
       end
 
-      def dump
-        configure_proxy_map[self.to_s].dump
+      def dump(level = 0)
+        configure_proxy_map[self.to_s].dump(level)
       end
     end
   end
diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index fcc915484e..10299ba3ea 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -194,7 +194,19 @@ def show_plugin_config
       $log.info "Show config for #{@show_plugin_config}"
       name, type = @show_plugin_config.split(":")
       plugin = Plugin.__send__("new_#{name}", type)
-      $log.info plugin.class.dump
+      dumped_config = "\n"
+      level = 0
+      plugin.class.ancestors.reverse_each do |plugin_class|
+        if plugin_class.respond_to?(:dump)
+          $log.on_debug do
+            dumped_config << plugin_class.name
+            dumped_config << "\n"
+            level = 1
+          end
+          dumped_config << plugin_class.dump(level)
+        end
+      end
+      $log.info dumped_config
       exit 0
     rescue => e
       $log.error "show config failed: #{e}"
diff --git a/test/config/test_configure_proxy.rb b/test/config/test_configure_proxy.rb
index 27b5236c2e..b46d85a710 100644
--- a/test/config/test_configure_proxy.rb
+++ b/test/config/test_configure_proxy.rb
@@ -138,13 +138,12 @@ class TestConfigureProxy < ::Test::Unit::TestCase
         end
 
         test 'empty proxy' do
-          assert_equal("\n", @proxy.dump)
+          assert_equal("", @proxy.dump)
         end
 
         test 'plain proxy w/o default value' do
           @proxy.config_param(:name, :string)
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <nil>
 CONFIG
         end
@@ -152,7 +151,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
         test 'plain proxy w/ default value' do
           @proxy.config_param(:name, :string, default: "name1")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <"name1">
 CONFIG
         end
@@ -161,7 +159,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
           @proxy.config_param(:name, :string)
           @proxy.config_set_default(:name, "name1")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <"name1">
 CONFIG
         end
@@ -171,7 +168,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
             config_param(:name, :string, default: "name1")
           end
           assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name: string: <"name1">
 CONFIG
@@ -187,7 +183,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
             end
           end
           assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name1: string: <"name1">
  name2: string: <"name2">
@@ -201,7 +196,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
           test 'single proxy' do
             @proxy.config_param(:name, :string, desc: "description for name")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <nil> # description for name
 CONFIG
           end
@@ -210,7 +204,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
             @proxy.config_param(:name, :string)
             @proxy.config_set_desc(:name, "description for name")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <nil> # description for name
 CONFIG
           end
@@ -225,7 +218,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
               end
             end
             assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name1: string: <"name1"> # desc1
  name2: string: <"name2"> # desc2
@@ -247,7 +239,6 @@ class TestConfigureProxy < ::Test::Unit::TestCase
               end
             end
             assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name1: string: <"name1"> # desc1
  name2: string: <"name2"> # desc2

From 945627046ace86152532ad96728d2c1f172e35da Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 4 Dec 2015 11:01:08 +0900
Subject: [PATCH 037/193] Merge pull request #719 from
 okkez/add-descriptions-in-tail

Add descriptions to in_tail
---
 lib/fluent/plugin/in_tail.rb | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 2270103519..e3aff5fa40 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -24,13 +24,21 @@ def initialize
       @tails = {}
     end
 
+    desc 'The paths to read. Multiple paths can be specified, separated by comma.'
     config_param :path, :string
+    desc 'The tag of the event.'
     config_param :tag, :string
+    desc 'The paths to exclude the files from watcher list.'
     config_param :exclude_path, :array, :default => []
+    desc 'Specify interval to keep reference to old file when rotate a file.'
     config_param :rotate_wait, :time, :default => 5
+    desc 'Fluentd will record the position it last read into this file.'
     config_param :pos_file, :string, :default => nil
+    desc 'Start to read the logs from the head of file, not bottom.'
     config_param :read_from_head, :bool, :default => false
+    desc 'The interval of refreshing the list of watch file.'
     config_param :refresh_interval, :time, :default => 60
+    desc 'The number of reading lines at each IO.'
     config_param :read_lines_limit, :integer, :default => 1000
 
     attr_reader :paths

From aace26c7c6a28d8f9f46f8f8a37bcef14108d308 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 4 Dec 2015 14:37:54 +0900
Subject: [PATCH 038/193] Merge pull request #704 from
 fluent/support-text-parser-in-exec

Support built-in text parsers in in_exec. fix #698
---
 lib/fluent/plugin/exec_util.rb | 21 +++++++++++++++++
 lib/fluent/plugin/in_exec.rb   | 42 +++++++++++++++++-----------------
 test/plugin/test_in_exec.rb    | 34 ++++++++++++++++++++++++---
 test/scripts/exec_script.rb    |  6 +++++
 4 files changed, 79 insertions(+), 24 deletions(-)

diff --git a/lib/fluent/plugin/exec_util.rb b/lib/fluent/plugin/exec_util.rb
index 7fe262ecf6..9ed3d9edff 100644
--- a/lib/fluent/plugin/exec_util.rb
+++ b/lib/fluent/plugin/exec_util.rb
@@ -16,6 +16,8 @@
 
 module Fluent
   module ExecUtil
+    require 'fluent/parser'
+
     SUPPORTED_FORMAT = {
       'tsv' => :tsv,
       'json' => :json,
@@ -28,6 +30,25 @@ def initialize(on_message)
       end
     end
 
+    class TextParserWrapperParser < Parser
+      def initialize(conf, on_message)
+        @parser = Plugin.new_parser(conf['format'])
+        @parser.configure(conf)
+        super(on_message)
+      end
+
+      def call(io)
+        io.each_line(&method(:each_line))
+      end
+
+      def each_line(line)
+        line.chomp!
+        @parser.parse(line) { |time, record|
+          @on_message.call(record, time)
+        }
+      end
+    end
+
     class TSVParser < Parser
       def initialize(keys, on_message)
         @keys = keys
diff --git a/lib/fluent/plugin/in_exec.rb b/lib/fluent/plugin/in_exec.rb
index 538e55eb8d..50efe5f76c 100644
--- a/lib/fluent/plugin/in_exec.rb
+++ b/lib/fluent/plugin/in_exec.rb
@@ -24,20 +24,10 @@ def initialize
       require 'fluent/timezone'
     end
 
-    SUPPORTED_FORMAT = {
-      'tsv' => :tsv,
-      'json' => :json,
-      'msgpack' => :msgpack,
-    }
-
     desc 'The command (program) to execute.'
     config_param :command, :string
     desc 'The format used to map the program output to the incoming event.(tsv,json,msgpack)'
-    config_param :format, :default => :tsv do |val|
-      f = SUPPORTED_FORMAT[val]
-      raise ConfigError, "Unsupported format '#{val}'" unless f
-      f
-    end
+    config_param :format, :string, :default => 'tsv'
     desc 'Specify the comma-separated keys when using the tsv format.'
     config_param :keys, :default => [] do |val|
       val.split(',')
@@ -80,16 +70,22 @@ def configure(conf)
         end
       end
 
+      @parser = setup_parser(conf)
+    end
+
+    def setup_parser(conf)
       case @format
-      when :tsv
+      when 'tsv'
         if @keys.empty?
           raise ConfigError, "keys option is required on exec input for tsv format"
         end
-        @parser = ExecUtil::TSVParser.new(@keys, method(:on_message))
-      when :json
-        @parser = ExecUtil::JSONParser.new(method(:on_message))
-      when :msgpack
-        @parser = ExecUtil::MessagePackParser.new(method(:on_message))
+        ExecUtil::TSVParser.new(@keys, method(:on_message))
+      when 'json'
+        ExecUtil::JSONParser.new(method(:on_message))
+      when 'msgpack'
+        ExecUtil::MessagePackParser.new(method(:on_message))
+      else
+        ExecUtil::TextParserWrapperParser.new(conf, method(:on_message))
       end
     end
 
@@ -145,17 +141,21 @@ def run_periodic
 
     private
 
-    def on_message(record)
+    def on_message(record, parsed_time = nil)
       if val = record.delete(@tag_key)
         tag = val
       else
         tag = @tag
       end
 
-      if val = record.delete(@time_key)
-        time = @time_parse_proc.call(val)
+      if parsed_time
+        time = parsed_time
       else
-        time = Engine.now
+        if val = record.delete(@time_key)
+          time = @time_parse_proc.call(val)
+        else
+          time = Engine.now
+        end
       end
 
       router.emit(tag, time, record)
diff --git a/test/plugin/test_in_exec.rb b/test/plugin/test_in_exec.rb
index 5d5911d474..4738ced94b 100644
--- a/test/plugin/test_in_exec.rb
+++ b/test/plugin/test_in_exec.rb
@@ -44,9 +44,18 @@ def msgpack_config
     ]
   end
 
+  def regexp_config
+    %[
+      command ruby #{@script} "2011-01-02 13:14:15" 3
+      format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
+      tag regex_tag
+      run_interval 1s
+    ]
+  end
+
   def test_configure
     d = create_driver
-    assert_equal :tsv, d.instance.format
+    assert_equal 'tsv', d.instance.format
     assert_equal ["time","tag","k1"], d.instance.keys
     assert_equal "tag", d.instance.tag_key
     assert_equal "time", d.instance.time_key
@@ -55,16 +64,22 @@ def test_configure
 
   def test_configure_with_json
     d = create_driver json_config
-    assert_equal :json, d.instance.format
+    assert_equal 'json', d.instance.format
     assert_equal [], d.instance.keys
   end
 
   def test_configure_with_msgpack
     d = create_driver msgpack_config
-    assert_equal :msgpack, d.instance.format
+    assert_equal 'msgpack', d.instance.format
     assert_equal [], d.instance.keys
   end
 
+  def test_configure_with_regexp
+    d = create_driver regexp_config
+    assert_equal '/(?<time>[^\]]*) (?<message>[^ ]*)/', d.instance.format
+    assert_equal 'regex_tag', d.instance.tag
+  end
+
   # TODO: Merge following tests into one case with parameters
 
   def test_emit
@@ -102,4 +117,17 @@ def test_emit_msgpack
     assert_equal true, emits.length > 0
     assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
   end
+
+  def test_emit_regexp
+    d = create_driver regexp_config
+
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    assert_equal true, emits.length > 0
+    assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
+    assert_equal_event_time(@test_time, emits[0][1])
+  end
 end
diff --git a/test/scripts/exec_script.rb b/test/scripts/exec_script.rb
index b50c09b1ae..0e41ab0fe3 100644
--- a/test/scripts/exec_script.rb
+++ b/test/scripts/exec_script.rb
@@ -13,6 +13,10 @@ def gen_msgpack(time)
   {'tagger' => 'tag1', 'datetime' => time, 'k1' => 'ok'}.to_msgpack
 end
 
+def gen_raw_string(time)
+  "#{time} hello"
+end
+
 time = ARGV.first
 time = Integer(time) rescue time
 
@@ -23,4 +27,6 @@ def gen_msgpack(time)
   puts gen_json(time)
 when 2
   print gen_msgpack(time)
+when 3
+  print gen_raw_string(time)
 end

From 0d3f85695ce0a9bb55ca76c4d0ecff2e9cd06bbd Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 4 Dec 2015 14:45:15 +0900
Subject: [PATCH 039/193] Merge pull request #733 from cosmo0920/desc-out-exec

Add descriptions to out_exec
---
 lib/fluent/plugin/out_exec.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/fluent/plugin/out_exec.rb b/lib/fluent/plugin/out_exec.rb
index baf7f6e276..cdf4ee496a 100644
--- a/lib/fluent/plugin/out_exec.rb
+++ b/lib/fluent/plugin/out_exec.rb
@@ -26,13 +26,19 @@ def initialize
       @localtime = false
     end
 
+    desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argumen'
     config_param :command, :string
+    desc 'Specify the comma-separated keys when using the tsv format.'
     config_param :keys, :default => [] do |val|
       val.split(',')
     end
+    desc 'The name of the key to use as the event tag. This replaces the value in the event record.'
     config_param :tag_key, :string, :default => nil
+    desc 'The name of the key to use as the event time. This replaces the the value in the event record.'
     config_param :time_key, :string, :default => nil
+    desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
     config_param :time_format, :string, :default => nil
+    desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :format, :default => :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f

From 24ce7696d8a332e4c1c40dfb2ca3c8c1432404c5 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 8 Dec 2015 07:05:00 +0900
Subject: [PATCH 040/193] Merge pull request #734 from
 icyfork/in_tail-exclude_all_unreadable_files

in_tail: expand_paths exclude unreadable files
---
 lib/fluent/plugin/in_tail.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index e3aff5fa40..6c78c6c0e5 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -114,7 +114,14 @@ def expand_paths
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
-          paths += Dir.glob(path)
+          paths += Dir.glob(path).select { |p|
+            if File.readable?(p)
+              true
+            else
+              log.warn "#{p} unreadable. It is excluded and would be examined next time."
+              false
+            end
+          }
         else
           # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
           paths << path

From d288762e21c696bbf7189b6d42d341b027bd9825 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 8 Dec 2015 17:03:23 +0900
Subject: [PATCH 041/193] test should not use v0.14 assert

---
 test/plugin/test_in_exec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/plugin/test_in_exec.rb b/test/plugin/test_in_exec.rb
index 4738ced94b..83c47166bb 100644
--- a/test/plugin/test_in_exec.rb
+++ b/test/plugin/test_in_exec.rb
@@ -128,6 +128,6 @@ def test_emit_regexp
     emits = d.emits
     assert_equal true, emits.length > 0
     assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
-    assert_equal_event_time(@test_time, emits[0][1])
+    assert_equal @test_time, emits[0][1]
   end
 end

From f2b92e70c4790a5e0c5dc80e24a21e8f859393f8 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 8 Dec 2015 17:47:07 +0900
Subject: [PATCH 042/193] Update travis to follow master branch

---
 .travis.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9752cc3e68..cf8a942d3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,7 +4,7 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1
-  - 2.2
+  - 2.2.3
   - ruby-head
   - rbx-2
 
@@ -30,4 +30,3 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx-2
-    - rvm: 2.2

From 4f6ecffc02fe7e7ce9cb97f27d3f21c78216682a Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 9 Dec 2015 03:03:50 +0900
Subject: [PATCH 043/193] Merge pull request #526 from
 sabottenda/fix-in_exec-shutdown

Stop in_exec immediately at shutdown
---
 lib/fluent/plugin/in_exec.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_exec.rb b/lib/fluent/plugin/in_exec.rb
index 50efe5f76c..bc7bd82178 100644
--- a/lib/fluent/plugin/in_exec.rb
+++ b/lib/fluent/plugin/in_exec.rb
@@ -103,6 +103,8 @@ def start
     def shutdown
       if @run_interval
         @finished = true
+        # call Thread#run which interupts sleep in order to stop run_periodic thread immediately.
+        @thread.run
         @thread.join
       else
         begin
@@ -126,12 +128,13 @@ def run
     end
 
     def run_periodic
+      sleep @run_interval
       until @finished
         begin
-          sleep @run_interval
           io = IO.popen(@command, "r")
           @parser.call(io)
           Process.waitpid(io.pid)
+          sleep @run_interval
         rescue
           log.error "exec failed to run or shutdown child process", :error => $!.to_s, :error_class => $!.class.to_s
           log.warn_backtrace $!.backtrace

From 1e46058caa6f443890a85d196b8216abad577a43 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 9 Dec 2015 15:17:51 +0900
Subject: [PATCH 044/193] v0.12.18

---
 ChangeLog             | 25 +++++++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 8f37e68cff..43871534a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,30 @@
 # v0.12
 
+## Release 0.12.18 - 2015/12/09
+
+### New features / Enhancement
+
+* in_exec: Stop in_exec immediately at shutdown
+  https://github.com/fluent/fluentd/pull/526
+* in_exec: Support built-in text parsers
+  https://github.com/fluent/fluentd/pull/704
+* out_forward: raises ZeroDivisionError when no <server> is available
+  https://github.com/fluent/fluentd/pull/707
+* Add process_name parameter in system to change fluentd's process name
+  https://github.com/fluent/fluentd/pull/708
+* test: Add ParserTestDriver
+  https://github.com/fluent/fluentd/pull/702
+* test: Add FormatterTestDriver
+  https://github.com/fluent/fluentd/pull/712
+* Add parameter description to built-in plugins
+
+### Bug fixes
+
+* fix bug not to protect in-memory-buffer for multi thread emitting / run loop
+  https://github.com/fluent/fluentd/pull/710
+* in_tail: expand_paths exclude unreadable files
+  https://github.com/fluent/fluentd/pull/734
+
 ## Release 0.12.17 - 2015/11/04
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 0a399e8918..613d562206 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.17'
+  VERSION = '0.12.18'
 
 end

From 4ca4cade195ccf83d2e44b6f59856069d1f3fa6c Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 10 Dec 2015 15:53:14 +0900
Subject: [PATCH 045/193] Merge pull request #743 from
 skippy/standardize-input-logging

standardize logging level for source configs
---
 lib/fluent/plugin/in_stream.rb | 2 +-
 lib/fluent/plugin/in_syslog.rb | 2 +-
 lib/fluent/plugin/in_tcp.rb    | 2 +-
 lib/fluent/plugin/in_udp.rb    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/fluent/plugin/in_stream.rb b/lib/fluent/plugin/in_stream.rb
index 0328d30d33..a14b76efe1 100644
--- a/lib/fluent/plugin/in_stream.rb
+++ b/lib/fluent/plugin/in_stream.rb
@@ -179,7 +179,7 @@ def listen
         File.unlink(@path)
       end
       FileUtils.mkdir_p File.dirname(@path)
-      log.debug "listening fluent socket on #{@path}"
+      log.info "listening fluent socket on #{@path}"
       s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s
diff --git a/lib/fluent/plugin/in_syslog.rb b/lib/fluent/plugin/in_syslog.rb
index ae5476f5e9..0ec6148562 100644
--- a/lib/fluent/plugin/in_syslog.rb
+++ b/lib/fluent/plugin/in_syslog.rb
@@ -173,7 +173,7 @@ def receive_data(data, addr)
     private
 
     def listen(callback)
-      log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb
index 181d4bbc6d..5993d95052 100644
--- a/lib/fluent/plugin/in_tcp.rb
+++ b/lib/fluent/plugin/in_tcp.rb
@@ -25,7 +25,7 @@ class TcpInput < SocketUtil::BaseInput
     config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
     def listen(callback)
-      log.debug "listening tcp socket on #{@bind}:#{@port}"
+      log.info "listening tcp socket on #{@bind}:#{@port}"
       Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback)
     end
   end
diff --git a/lib/fluent/plugin/in_udp.rb b/lib/fluent/plugin/in_udp.rb
index 56f70f9d16..d07ef6cc86 100644
--- a/lib/fluent/plugin/in_udp.rb
+++ b/lib/fluent/plugin/in_udp.rb
@@ -24,7 +24,7 @@ class UdpInput < SocketUtil::BaseInput
     config_param :body_size_limit, :size, :default => 4096
 
     def listen(callback)
-      log.debug "listening udp socket on #{@bind}:#{@port}"
+      log.info "listening udp socket on #{@bind}:#{@port}"
       @usock = SocketUtil.create_udp_socket(@bind)
       @usock.bind(@bind, @port)
       SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)

From b96be61be205f14fb1b7702d84f52e94bc16fa6f Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 16 Dec 2015 16:55:19 +0900
Subject: [PATCH 046/193] Merge pull request #747 from fluent/fix-fluent-cat

Fix fluent-cat with --msgpack option
---
 lib/fluent/command/cat.rb | 2 ++
 lib/fluent/engine.rb      | 1 +
 2 files changed, 3 insertions(+)

diff --git a/lib/fluent/command/cat.rb b/lib/fluent/command/cat.rb
index 49f4aaa469..8ec8a5e2c1 100644
--- a/lib/fluent/command/cat.rb
+++ b/lib/fluent/command/cat.rb
@@ -288,6 +288,8 @@ def abort_message(time, record)
   end
 
 when 'msgpack'
+  require 'fluent/engine'
+
   begin
     u = Fluent::Engine.msgpack_factory.unpacker($stdin)
     u.each {|record|
diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index 046a2d98af..48acb7c4df 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -17,6 +17,7 @@
 module Fluent
   require 'fluent/event_router'
   require 'fluent/root_agent'
+  require 'fluent/time'
 
   class EngineClass
     class DummyMessagePackFactory

From cd01f11f2b9948a3629ff962f63837ce5da878c0 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 21 Dec 2015 18:46:32 +0900
Subject: [PATCH 047/193] Add fluent/time.rb placeholder for v0.14

---
 lib/fluent/time.rb | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 lib/fluent/time.rb

diff --git a/lib/fluent/time.rb b/lib/fluent/time.rb
new file mode 100644
index 0000000000..e96aa188ee
--- /dev/null
+++ b/lib/fluent/time.rb
@@ -0,0 +1 @@
+# placeholder for v0.14 time module

From d179ee957f67f45f727999f383506781f01bc4ca Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 18 Dec 2015 20:50:11 +0900
Subject: [PATCH 048/193] Merge pull request #748 from
 fluent/support-oj-in-json-parser

Use 'oj' for performance improvement when oj is installed
---
 fluentd.gemspec                   |  1 +
 lib/fluent/formatter.rb           | 17 ++++++++++++++++-
 lib/fluent/parser.rb              | 15 +++++++++++++--
 lib/fluent/test/formatter_test.rb |  5 +++--
 lib/fluent/test/parser_test.rb    |  5 +++--
 test/plugin/test_out_stdout.rb    | 14 ++++++++++----
 test/test_formatter.rb            | 20 ++++++++++++--------
 test/test_parser.rb               | 23 ++++++++++++++++-------
 8 files changed, 74 insertions(+), 26 deletions(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index dc9f6a3db2..ec7eb7a678 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -37,4 +37,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("timecop", [">= 0.3.0"])
   gem.add_development_dependency("test-unit", ["~> 3.1.4"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
+  gem.add_development_dependency("oj", ["~> 2.14"])
 end
diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index ccc1b6a7f3..5b14892f3a 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -131,8 +131,23 @@ class JSONFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
+      config_param :json_parser, :string, :default => 'oj'
+
+      def configure(conf)
+        super
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          Oj.default_options = {:mode => :compat}
+          @dump_proc = Oj.method(:dump)
+        rescue LoadError
+          @dump_proc = Yajl.method(:dump)
+        end
+      end
+
       def format_record(record)
-        "#{Yajl.dump(record)}\n"
+        "#{@dump_proc.call(record)}\n"
       end
     end
 
diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 7f06bca353..7f6b4e7375 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -237,6 +237,7 @@ def parse(text)
     class JSONParser < Parser
       config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
+      config_param :json_parser, :string, :default => 'oj'
 
       def configure(conf)
         super
@@ -245,10 +246,20 @@ def configure(conf)
           @time_parser = TimeParser.new(@time_format)
           @mutex = Mutex.new
         end
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          @load_proc = Oj.method(:load)
+          @error_class = Oj::ParseError
+        rescue LoadError
+          @load_proc = Yajl.method(:load)
+          @error_class = Yajl::ParseError
+        end
       end
 
       def parse(text)
-        record = Yajl.load(text)
+        record = @load_proc.call(text)
 
         value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
         if value
@@ -274,7 +285,7 @@ def parse(text)
         else
           return time, record
         end
-      rescue Yajl::ParseError
+      rescue @error_class
         if block_given?
           yield nil, nil
         else
diff --git a/lib/fluent/test/formatter_test.rb b/lib/fluent/test/formatter_test.rb
index fedce36f33..f840cebf52 100644
--- a/lib/fluent/test/formatter_test.rb
+++ b/lib/fluent/test/formatter_test.rb
@@ -16,6 +16,8 @@
 
 module Fluent
   module Test
+    require 'fluent/config'
+
     class FormatterTestDriver
       def initialize(klass_or_str, proc=nil, &block)
         if klass_or_str.is_a?(Class)
@@ -41,8 +43,7 @@ def configure(conf)
         when Fluent::Config::Element
           @config = conf
         when String
-          io = StringIO.new(conf)
-          @config = Config::Parser.parse(io, 'fluent.conf')
+          @config = Config.parse(conf, 'fluent.conf')
         when Hash
           @config = Config::Element.new('ROOT', '', conf, [])
         else
diff --git a/lib/fluent/test/parser_test.rb b/lib/fluent/test/parser_test.rb
index 949ab833fa..7c6f6ddd6b 100644
--- a/lib/fluent/test/parser_test.rb
+++ b/lib/fluent/test/parser_test.rb
@@ -16,6 +16,8 @@
 
 module Fluent
   module Test
+    require 'fluent/config'
+
     class ParserTestDriver
       def initialize(klass_or_str, format=nil, conf={}, &block)
         if klass_or_str.is_a?(Class)
@@ -47,8 +49,7 @@ def configure(conf)
         when Fluent::Config::Element
           @config = conf
         when String
-          io = StringIO.new(conf)
-          @config = Config::Parser.parse(io, 'fluent.conf')
+          @config = Config.parse(conf, 'fluent.conf')
         when Hash
           @config = Config::Element.new('ROOT', '', conf, [])
         else
diff --git a/test/plugin/test_out_stdout.rb b/test/plugin/test_out_stdout.rb
index 9b94793f69..a774b48d07 100644
--- a/test/plugin/test_out_stdout.rb
+++ b/test/plugin/test_out_stdout.rb
@@ -30,14 +30,20 @@ def test_configure_output_type
     end
   end
 
-  def test_emit_json
-    d = create_driver(CONFIG + "\noutput_type json")
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_emit_json(data)
+    d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
     time = Time.now
     out = capture_log { d.emit({'test' => 'test'}, time) }
     assert_equal "#{time.localtime} test: {\"test\":\"test\"}\n", out
 
-    # NOTE: Float::NAN is not jsonable
-    assert_raise(Yajl::EncodeError) { d.emit({'test' => Float::NAN}, time) }
+    if data == 'yajl'
+      # NOTE: Float::NAN is not jsonable
+      assert_raise(Yajl::EncodeError) { d.emit({'test' => Float::NAN}, time) }
+    else
+      out = capture_log { d.emit({'test' => Float::NAN}, Fluent::EventTime.from_time(time)) }
+      assert_equal "#{time.localtime} test: {\"test\":NaN}\n", out
+    end
   end
 
   def test_emit_hash
diff --git a/test/test_formatter.rb b/test/test_formatter.rb
index ca94de9113..416b92917b 100644
--- a/test/test_formatter.rb
+++ b/test/test_formatter.rb
@@ -130,15 +130,17 @@ def setup
       @time = Engine.now
     end
 
-    def test_format
-      @formatter.configure({})
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format(data)
+      @formatter.configure('json_parser' => data)
       formatted = @formatter.format(tag, @time, record)
 
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
 
-    def test_format_with_include_tag
-      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo')
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_include_tag(data)
+      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo', 'json_parser' => data)
       formatted = @formatter.format(tag, @time, record.dup)
 
       r = record
@@ -146,8 +148,9 @@ def test_format_with_include_tag
       assert_equal("#{Yajl.dump(r)}\n", formatted)
     end
 
-    def test_format_with_include_time
-      @formatter.configure('include_time_key' => 'true', 'localtime' => '')
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_include_time(data)
+      @formatter.configure('include_time_key' => 'true', 'localtime' => '', 'json_parser' => data)
       formatted = @formatter.format(tag, @time, record.dup)
 
       r = record
@@ -155,8 +158,9 @@ def test_format_with_include_time
       assert_equal("#{Yajl.dump(r)}\n", formatted)
     end
 
-    def test_format_with_include_time_as_number
-      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch')
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_include_time_as_number(data)
+      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch', 'json_parser' => data)
       formatted = @formatter.format(tag, @time, record.dup)
 
       r = record
diff --git a/test/test_parser.rb b/test/test_parser.rb
index b18181456d..ba96e319f9 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/parser'
+require 'fluent/plugin/string_util'
 
 module ParserTest
   include Fluent
@@ -372,7 +373,9 @@ def setup
       @parser = TextParser::JSONParser.new
     end
 
-    def test_parse
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse(data)
+      @parser.configure('json_parser' => data)
       @parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
         assert_equal(str2time('2013-02-28 12:00:00 +0900').to_i, time)
         assert_equal({
@@ -383,9 +386,11 @@ def test_parse
       }
     end
 
-    def test_parse_without_time
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_without_time(data)
       time_at_start = Time.now.to_i
 
+      @parser.configure('json_parser' => data)
       @parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
         assert time && time >= time_at_start, "parser puts current time without time input"
         assert_equal({
@@ -397,7 +402,7 @@ def test_parse_without_time
 
       parser = TextParser::JSONParser.new
       parser.estimate_current_event = false
-      parser.configure({})
+      parser.configure('json_parser' => data)
       parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
         assert_equal({
           'host'   => '192.168.0.1',
@@ -408,17 +413,21 @@ def test_parse_without_time
       }
     end
 
-    def test_parse_with_invalid_time
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_invalid_time(data)
+      @parser.configure('json_parser' => data)
       assert_raise Fluent::ParserError do
         @parser.parse('{"time":[],"k":"v"}') { |time, record| }
       end
     end
 
-    def test_parse_with_keep_time_key
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_keep_time_key(data)
       parser = TextParser::JSONParser.new
       parser.configure(
-        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-        'keep_time_key'=>'true',
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'true',
+        'json_parser' => data
       )
       text = "28/Feb/2013:12:00:00 +0900"
       parser.parse("{\"time\":\"#{text}\"}") do |time, record|

From 316d3de179037d29728f8b1fa2ab00caa120e2f1 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 21 Dec 2015 04:54:10 +0900
Subject: [PATCH 049/193] Merge pull request #745 from
 fluent/add-block-action-to-buffer

Add 'buffer_queue_full_action'  parameter to control buffer actions
---
 lib/fluent/buffer.rb | 34 +++++++++++++++-----
 test/test_buffer.rb  | 76 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+), 8 deletions(-)

diff --git a/lib/fluent/buffer.rb b/lib/fluent/buffer.rb
index 30dda85c54..97f10d2ad0 100644
--- a/lib/fluent/buffer.rb
+++ b/lib/fluent/buffer.rb
@@ -143,6 +143,8 @@ def enable_parallel(b=true)
     config_param :buffer_chunk_limit, :size, :default => 8*1024*1024
     desc 'The length limit of the chunk queue.'
     config_param :buffer_queue_limit, :integer, :default => 256
+    desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
+    config_param :buffer_queue_full_action, :enum, :list => [:exception, :block], :default => :exception
 
     alias chunk_limit buffer_chunk_limit
     alias chunk_limit= buffer_chunk_limit=
@@ -151,6 +153,10 @@ def enable_parallel(b=true)
 
     def configure(conf)
       super
+
+      if @buffer_queue_full_action == :block
+        $log.warn "'block' action stops input process until the buffer full is resolved. Check your pipeline this action is fit or not"
+      end
     end
 
     def start
@@ -179,16 +185,28 @@ def emit(key, data, chain)
       key = key.to_s
 
       synchronize do
-        # chunk unique id is generated in #new_chunk
-        chunk = (@map[key] ||= new_chunk(key))
+        begin
+          # chunk unique id is generated in #new_chunk
+          chunk = (@map[key] ||= new_chunk(key))
 
-        if storable?(chunk, data)
-          chain.next
-          chunk << data
-          return false
+          if storable?(chunk, data)
+            chain.next
+            chunk << data
+            return false
 
-        elsif @queue.size >= @buffer_queue_limit
-          raise BufferQueueLimitError, "queue size exceeds limit"
+          elsif @queue.size >= @buffer_queue_limit
+            raise BufferQueueLimitError, "queue size exceeds limit"
+          end
+        rescue BufferQueueLimitError => e
+          case @buffer_queue_full_action
+          when :exception
+            raise e
+          when :block
+            # This is rough implementation. New Buffer API should improve this routine by using wait/signal
+            $log.debug "buffer queue is full. Wait 1 second to re-emit events"
+            sleep 1
+            retry
+          end
         end
 
         if data.bytesize > @buffer_chunk_limit
diff --git a/test/test_buffer.rb b/test/test_buffer.rb
index 5ab7081b06..1350b273f8 100644
--- a/test/test_buffer.rb
+++ b/test/test_buffer.rb
@@ -4,6 +4,7 @@
 
 require 'stringio'
 require 'msgpack'
+require 'timeout'
 
 module FluentBufferTest
   class BufferTest < Test::Unit::TestCase
@@ -526,5 +527,80 @@ def test_clear!
 
       assert chunks.reduce(true){|a,b| a && b.purged }
     end
+
+    sub_test_case 'buffer_queue_full_action' do
+      def dummy_buffer(action)
+        db = DummyBuffer.new
+        db.configure(
+          "buffer_chunk_limit" => 1024,
+          "buffer_queue_limit" => 1,
+          "buffer_queue_full_action" => action
+        )
+        db.start
+        db
+      end
+
+      def data
+        @data ||= "a" * 1024
+      end
+
+      def chain
+        @chain ||= DummyChain.new
+      end
+
+      def test_emit_with_exception
+        db = dummy_buffer(:exception)
+
+        assert !db.emit('key', data, chain)
+        assert db.emit('key', data, chain)
+
+        assert_raise(Fluent::BufferQueueLimitError) {
+          assert db.emit('key', data, chain)
+        }
+        assert db.queue.size == 1
+
+        assert !pop_chunk(db)
+        assert db.queue.size == 0
+
+        # queue is now empty so can emit data again
+        assert db.emit('key', data, chain)
+      end
+
+      def test_emit_with_block
+        db = dummy_buffer(:block)
+
+        assert !db.emit('key', data, chain)
+        assert db.emit('key', data, chain)
+
+        begin
+          # with block, emit events to full queue causes sleep loop
+          timeout(1) {
+            assert db.emit('key', data, chain)
+          }
+          flunk("timeout must happen")
+        rescue Timeout::Error => e
+        end
+        assert db.queue.size == 1
+
+        assert !pop_chunk(db)
+        assert db.queue.size == 0
+
+        # queue is now empty so can emit data again
+        assert db.emit('key', data, chain)
+      end
+
+      def pop_chunk(db)
+        out = DummyOutput.new
+        c1 = DummyChunk.new('k1', 1)
+
+        pop_return_value = nil
+        c1.synchronize do
+          pop_return_value = Thread.new {
+            db.pop(out)
+          }.value
+        end
+        pop_return_value
+      end
+    end
   end
 end

From 3e26b979b52ca182bfaeaffcbe87dca47c9949f3 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 21 Dec 2015 19:16:28 +0900
Subject: [PATCH 050/193] 0.12 doesn't have EventTime

---
 test/plugin/test_out_stdout.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/plugin/test_out_stdout.rb b/test/plugin/test_out_stdout.rb
index a774b48d07..a169f8bca8 100644
--- a/test/plugin/test_out_stdout.rb
+++ b/test/plugin/test_out_stdout.rb
@@ -41,7 +41,7 @@ def test_emit_json(data)
       # NOTE: Float::NAN is not jsonable
       assert_raise(Yajl::EncodeError) { d.emit({'test' => Float::NAN}, time) }
     else
-      out = capture_log { d.emit({'test' => Float::NAN}, Fluent::EventTime.from_time(time)) }
+      out = capture_log { d.emit({'test' => Float::NAN}, time) }
       assert_equal "#{time.localtime} test: {\"test\":NaN}\n", out
     end
   end

From 920645ad6aad360e151165eb4c448822149f970b Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 21 Dec 2015 20:43:26 +0900
Subject: [PATCH 051/193] v0.12.19

---
 ChangeLog             | 16 ++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 43871534a6..d773eb25ef 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,21 @@
 # v0.12
 
+## Release 0.12.19 - 2015/12/21
+
+### New features / Enhancement
+
+* input: Standardize logging level for source configs
+  https://github.com/fluent/fluentd/pull/743
+* buffer: Add buffer_queue_full_action parameter
+  https://github.com/fluent/fluentd/pull/745
+* parser/formatter: Use 'oj' when oj is installed in json parser/formatter
+  https://github.com/fluent/fluentd/pull/748
+
+### Bug fixes
+
+* fluent-cat: fix the regression of --msgpack option
+  https://github.com/fluent/fluentd/pull/747
+
 ## Release 0.12.18 - 2015/12/09
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 613d562206..3ad7d0359a 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.18'
+  VERSION = '0.12.19'
 
 end

From 232c38a2c21187d3edefa2babe38b343a74bbdd9 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Fri, 8 Jan 2016 17:22:57 +0900
Subject: [PATCH 052/193] Merge pull request #766 from
 sonots/fast_record_transformer

record_transformer: faster implementation of ruby placeholder expander
---
 .../plugin/filter_record_transformer.rb       | 157 +++++++++++++-----
 test/plugin/test_filter_record_transformer.rb |  72 +++++++-
 2 files changed, 183 insertions(+), 46 deletions(-)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 66c8a86818..319917da35 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -41,12 +41,12 @@ def initialize
     def configure(conf)
       super
 
-      @map = {}
+      map = {}
       # <record></record> directive
       conf.elements.select { |element| element.name == 'record' }.each do |element|
         element.each_pair do |k, v|
           element.has_key?(k) # to suppress unread configuration warning
-          @map[k] = parse_value(v)
+          map[k] = parse_value(v)
         end
       end
 
@@ -73,6 +73,7 @@ def configure(conf)
         else
           PlaceholderExpander.new(placeholder_expander_params)
         end
+      @map = @placeholder_expander.preprocess_map(map)
 
       @hostname = Socket.gethostname
     end
@@ -82,17 +83,21 @@ def filter_stream(tag, es)
       tag_parts = tag.split('.')
       tag_prefix = tag_prefix(tag_parts)
       tag_suffix = tag_suffix(tag_parts)
-      placeholders = {
-        'tag' => tag,
-        'tag_parts' => tag_parts,
+      placeholder_values = {
+        'tag'        => tag,
+        'tag_parts'  => tag_parts,
         'tag_prefix' => tag_prefix,
         'tag_suffix' => tag_suffix,
-        'hostname' => @hostname,
+        'hostname'   => @hostname,
       }
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        new_record = reform(time, record, placeholders)
+        placeholder_values.merge!({
+          'time'     => @placeholder_expander.time_value(time),
+          'record'   => record,
+        })
+        new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = new_record[@renew_time_key].to_i
         end
@@ -102,7 +107,7 @@ def filter_stream(tag, es)
     rescue => e
       log.warn "failed to reform records", :error_class => e.class, :error => e.message
       log.warn_backtrace
-      log.debug "map:#{@map} record:#{last_record} placeholders:#{placeholders}"
+      log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
 
     private
@@ -118,8 +123,8 @@ def parse_value(value_str)
       value_str # emit as string
     end
 
-    def reform(time, record, opts)
-      @placeholder_expander.prepare_placeholders(time, record, opts)
+    def reform(record, placeholder_values)
+      @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
@@ -175,17 +180,29 @@ def initialize(params)
         @auto_typecast = params[:auto_typecast]
       end
 
-      def prepare_placeholders(time, record, opts)
-        placeholders = { '${time}' => Time.at(time).to_s }
-        record.each {|key, value| placeholders.store("${#{key}}", value) }
+      def time_value(time)
+        Time.at(time).to_s
+      end
+
+      def preprocess_map(value, force_stringify = false)
+        value
+      end
 
-        opts.each do |key, value|
+      def prepare_placeholders(placeholder_values)
+        placeholders = {}
+
+        placeholder_values.each do |key, value|
           if value.kind_of?(Array) # tag_parts, etc
             size = value.size
-            value.each_with_index { |v, idx|
+            value.each_with_index do |v, idx|
               placeholders.store("${#{key}[#{idx}]}", v)
               placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
-            }
+            end
+          elsif value.kind_of?(Hash) # record, etc
+            value.each do |k, v|
+              placeholders.store("${#{k}}", v) # foo
+              placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
+            end
           else # string, interger, float, and others?
             placeholders.store("${#{key}}", value)
           end
@@ -194,22 +211,27 @@ def prepare_placeholders(time, record, opts)
         @placeholders = placeholders
       end
 
-      def expand(str, force_stringify=false)
+      # Expand string with placeholders
+      #
+      # @param [String] str
+      # @param [Boolean] force_stringify the value must be string, used for hash key
+      def expand(str, force_stringify = false)
         if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
-            log_unknown_placeholder($1)
+            log_if_unknown_placeholder($1)
             return @placeholders[single_placeholder_matched[1]]
           end
         end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log_unknown_placeholder($1)
+          log_if_unknown_placeholder($1)
           @placeholders[$1]
         }
       end
 
       private
-      def log_unknown_placeholder(placeholder)
+
+      def log_if_unknown_placeholder(placeholder)
         unless @placeholders.include?(placeholder)
           log.warn "unknown placeholder `#{placeholder}` found"
         end
@@ -217,41 +239,96 @@ def log_unknown_placeholder(placeholder)
     end
 
     class RubyPlaceholderExpander
-      attr_reader :placeholders, :log
+      attr_reader :log
 
       def initialize(params)
         @log = params[:log]
         @auto_typecast = params[:auto_typecast]
+        @cleanroom_expander = CleanroomExpander.new
       end
 
-      # Get placeholders as a struct
-      #
-      # @param [Time]   time        the time
-      # @param [Hash]   record      the record
-      # @param [Hash]   opts        others
-      def prepare_placeholders(time, record, opts)
-        struct = UndefOpenStruct.new(record)
-        struct.time = Time.at(time)
-        opts.each {|key, value| struct.__send__("#{key}=", value) }
-        @placeholders = struct
+      def time_value(time)
+        Time.at(time)
       end
 
-      def expand(_str_for_eval_, force_stringify=false)
-        if @auto_typecast and !force_stringify
-          _single_placeholder_matched_ = _str_for_eval_.match(/\A\${([^}]+)}\z/)
-          if _single_placeholder_matched_
-            return eval _single_placeholder_matched_[1], @placeholders.instance_eval { binding }
+      # Preprocess record map to convert into ruby string expansion
+      #
+      # @param [Hash|String|Array] value record map config
+      # @param [Boolean] force_stringify the value must be string, used for hash key
+      def preprocess_map(value, force_stringify = false)
+        new_value = nil
+        if value.is_a?(String)
+          if @auto_typecast and !force_stringify
+            if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
+              new_value = single_placeholder_matched[1]
+            end
+          end
+          unless new_value
+            new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
+          end
+        elsif value.is_a?(Hash)
+          new_value = {}
+          value.each_pair do |k, v|
+            new_value[preprocess_map(k, true)] = preprocess_map(v)
           end
+        elsif value.is_a?(Array)
+          new_value = []
+          value.each_with_index do |v, i|
+            new_value[i] = preprocess_map(v)
+          end
+        else
+          new_value = value
         end
-        _interpolated_for_eval_ = _str_for_eval_.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-        eval "\"#{_interpolated_for_eval_}\"", @placeholders.instance_eval { binding }
+        new_value
+      end
+
+      def prepare_placeholders(placeholder_values)
+        @tag = placeholder_values['tag']
+        @time = placeholder_values['time']
+        @record = placeholder_values['record']
+        @tag_parts = placeholder_values['tag_parts']
+        @tag_prefix = placeholder_values['tag_prefix']
+        @tag_suffix = placeholder_values['tag_suffix']
+        @hostname = placeholder_values['hostname']
+      end
+
+      # Expand string with placeholders
+      #
+      # @param [String] str
+      def expand(str, force_stringify = false)
+        @cleanroom_expander.expand(
+          str,
+          @tag,
+          @time,
+          @record,
+          @tag_parts,
+          @tag_prefix,
+          @tag_suffix,
+          @hostname,
+        )
       rescue => e
-        log.warn "failed to expand `#{_str_for_eval_}`", :error_class => e.class, :error => e.message
+        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
         log.warn_backtrace
         nil
       end
 
-      class UndefOpenStruct < OpenStruct
+      class CleanroomExpander
+        def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
+          tags = tag_parts # for old version compatibility
+          @record = record # for old version compatibility
+          instance_eval(__str_to_eval__)
+        end
+
+        # for old version compatibility
+        def method_missing(name)
+          key = name.to_s
+          if @record.has_key?(key)
+            @record[key]
+          else
+            raise NameError, "undefined local variable or method `#{key}'"
+          end
+        end
+
         (Object.instance_methods).each do |m|
           undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
         end
diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
index f1af131f0c..957d2dc645 100644
--- a/test/plugin/test_filter_record_transformer.rb
+++ b/test/plugin/test_filter_record_transformer.rb
@@ -340,6 +340,7 @@ def emit(config, msgs = [''])
             multiple    ${source}${source}
             with_prefix prefix-${source}
             with_suffix ${source}-suffix
+            with_quote  source[""]
           </record>
         ]
         msgs = [
@@ -353,23 +354,28 @@ def emit(config, msgs = [''])
           { :single      => "string",
             :multiple    => "stringstring",
             :with_prefix => "prefix-string",
-            :with_suffix => "string-suffix" },
+            :with_suffix => "string-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => 123.to_s,
             :multiple    => "#{123.to_s}#{123.to_s}",
             :with_prefix => "prefix-#{123.to_s}",
-            :with_suffix => "#{123.to_s}-suffix" },
+            :with_suffix => "#{123.to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => [1, 2].to_s,
             :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
             :with_prefix => "prefix-#{[1, 2].to_s}",
-            :with_suffix => "#{[1, 2].to_s}-suffix" },
+            :with_suffix => "#{[1, 2].to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => {a:1, b:2}.to_s,
             :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
             :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
-            :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
+            :with_suffix => "#{{a:1, b:2}.to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => nil.to_s,
             :multiple    => "#{nil.to_s}#{nil.to_s}",
             :with_prefix => "prefix-#{nil.to_s}",
-            :with_suffix => "#{nil.to_s}-suffix" },
+            :with_suffix => "#{nil.to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
         ]
         actual_results = []
         es = emit(config, msgs)
@@ -379,6 +385,7 @@ def emit(config, msgs = [''])
             :multiple    => r["multiple"],
             :with_prefix => r["with_prefix"],
             :with_suffix => r["with_suffix"],
+            :with_quote  => r["with_quote"],
           }
         end
         assert_equal(expected_results, actual_results)
@@ -436,6 +443,27 @@ def emit(config, msgs = [''])
         end
         assert_equal(expected_results, actual_results)
       end
+
+      test %Q[record["key"] with enable_ruby #{enable_ruby}] do
+        config = %[
+          enable_ruby #{enable_ruby}
+          auto_typecast yes
+          <record>
+            _timestamp ${record["@timestamp"]}
+            _foo_bar   ${record["foo.bar"]}
+          </record>
+        ]
+        d = create_driver(config)
+        record = {
+          "foo.bar"    => "foo.bar",
+          "@timestamp" => 10,
+        }
+        es = d.run { d.emit(record, @time) }.filtered
+        es.each do |t, r|
+          assert { r['_timestamp'] == record['@timestamp'] }
+          assert { r['_foo_bar'] == record['foo.bar'] }
+        end
+      end
     end
 
     test 'unknown placeholder (enable_ruby no)' do
@@ -458,7 +486,7 @@ def emit(config, msgs = [''])
         </record>
       ]
       es = emit(config) { |d|
-        mock(d.instance.log).warn("failed to expand `${unknown['bar']}`", anything)
+        mock(d.instance.log).warn("failed to expand `%Q[\#{unknown['bar']}]`", anything)
       }
       es.each do |t, r|
         assert_nil(r['message'])
@@ -495,4 +523,36 @@ def emit(config, msgs = [''])
       end
     end
   end
+
+  test "compatibility test (enable_ruby yes)" do
+    config = %[
+      enable_ruby yes
+      auto_typecast yes
+      <record>
+        _message   prefix-${message}-suffix
+        _time      ${Time.at(time)}
+        _number    ${number == '-' ? 0 : number}
+        _match     ${/0x[0-9a-f]+/.match(hex)[0]}
+        _timestamp ${__send__("@timestamp")}
+        _foo_bar   ${__send__('foo.bar')}
+      </record>
+    ]
+    d = create_driver(config)
+    record = {
+      "number"     => "-",
+      "hex"        => "0x10",
+      "foo.bar"    => "foo.bar",
+      "@timestamp" => 10,
+      "message"    => "10",
+    }
+    es = d.run { d.emit(record, @time) }.filtered
+    es.each do |t, r|
+      assert { r['_message'] == "prefix-#{record['message']}-suffix" }
+      assert { r['_time'] == Time.at(@time) }
+      assert { r['_number'] == 0 }
+      assert { r['_match'] == record['hex'] }
+      assert { r['_timestamp'] == record['@timestamp'] }
+      assert { r['_foo_bar'] == record['foo.bar'] }
+    end
+  end
 end

From 8a246d31287ef2ec3a3751df94c51bf7dc4a3dbd Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 28 Dec 2015 07:43:59 +0900
Subject: [PATCH 053/193] Merge pull request #757 from riywo/patch-1

Add on_exit_process(i) hook point
---
 lib/fluent/process.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index 517a5e4b8f..6130e472a4 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -289,6 +289,9 @@ module DetachProcessImpl
     def on_detach_process(i)
     end
 
+    def on_exit_process(i)
+    end
+
     private
 
     def detach_process_impl(num, &block)
@@ -326,6 +329,7 @@ def detach_process_impl(num, &block)
           #forward_thread.join  # TODO this thread won't stop because parent doesn't close pipe
           fin.wait
 
+          on_exit_process(i)
           exit! 0
         ensure
           $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid

From 7f6b47a89a7426e9f5625bb0674645319f485b05 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Tue, 29 Dec 2015 16:18:38 +0900
Subject: [PATCH 054/193] Merge pull request #761 from okkez/suppress-warning

Use `Timeout.timeout` instead of `Object#timeout`
---
 test/test_buffer.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_buffer.rb b/test/test_buffer.rb
index 1350b273f8..dcb9436a79 100644
--- a/test/test_buffer.rb
+++ b/test/test_buffer.rb
@@ -574,7 +574,7 @@ def test_emit_with_block
 
         begin
           # with block, emit events to full queue causes sleep loop
-          timeout(1) {
+          Timeout.timeout(1) {
             assert db.emit('key', data, chain)
           }
           flunk("timeout must happen")

From d063ac8ef7d74272bbc383fb6a41b29a010d7c36 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Wed, 30 Dec 2015 03:09:40 +0900
Subject: [PATCH 055/193] Merge pull request #759 from
 okkez/add-descriptions-output

Add descriptions to BufferedOutput
---
 lib/fluent/output.rb | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index e6ad6180e8..ad79889971 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -179,14 +179,22 @@ def initialize
       @emit_count = 0
     end
 
+    desc 'The buffer type (memory, file)'
     config_param :buffer_type, :string, :default => 'memory'
+    desc 'The interval between data flushes.'
     config_param :flush_interval, :time, :default => 60
     config_param :try_flush_interval, :float, :default => 1
+    desc 'If true, the value of `retry_value` is ignored and there is no limit'
     config_param :disable_retry_limit, :bool, :default => false
+    desc 'The limit on the number of retries before buffered data is discarded'
     config_param :retry_limit, :integer, :default => 17
+    desc 'The initial intervals between write retries.'
     config_param :retry_wait, :time, :default => 1.0
+    desc 'The maximum intervals between write retries.'
     config_param :max_retry_wait, :time, :default => nil
+    desc 'The number of threads to flush the buffer.'
     config_param :num_threads, :integer, :default => 1
+    desc 'The interval between data flushes for queued chunk.'
     config_param :queued_chunk_flush_interval, :time, :default => 1
 
     def configure(conf)

From b835c07f80079f8a60e4a3af98c60341c42da157 Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Wed, 30 Dec 2015 03:10:08 +0900
Subject: [PATCH 056/193] Merge pull request #760 from
 okkez/add-descriptions-for-time-sliced-output

Add descriptions for TimeSlicedOutput
---
 lib/fluent/output.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index ad79889971..1a9fe5c24b 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -482,8 +482,11 @@ def initialize
       #@ignore_old = false   # TODO
     end
 
+    desc 'The time format used as part of the file name.'
     config_param :time_slice_format, :string, :default => '%Y%m%d'
+    desc 'The amount of time Fluentd will wait for old logs to arrive.'
     config_param :time_slice_wait, :time, :default => 10*60
+    desc 'Parse the time value in the specified timezone'
     config_param :timezone, :string, :default => nil
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit

From 2e945607bc0aff1638dbf2d4e0e90066e9a114b9 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 7 Jan 2016 05:06:36 +0900
Subject: [PATCH 057/193] Merge pull request #765 from
 fluent/fix-regression-of-unused-parameter-warning

Fix the regression of unused parameter warning inside sub section
---
 lib/fluent/config/element.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/fluent/config/element.rb b/lib/fluent/config/element.rb
index f4a9ffc058..ba1f124621 100644
--- a/lib/fluent/config/element.rb
+++ b/lib/fluent/config/element.rb
@@ -75,11 +75,13 @@ def each_element(*names, &block)
       end
 
       def has_key?(key)
+        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in chgeck_not_fetched.
         @unused.delete(key)
         super
       end
 
       def [](key)
+        @unused_in = false # ditto
         @unused.delete(key)
         super
       end

From 03865d4b0d3809d09f129572393c50eef178b234 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 7 Jan 2016 05:06:50 +0900
Subject: [PATCH 058/193] Merge pull request #763 from
 cosmo0920/travis-ruby-2.3.0

Try to use Ruby 2.3.0 in Travis
---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index cf8a942d3c..0a74ee88b7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ rvm:
   - 2.0.0
   - 2.1
   - 2.2.3
+  - 2.3.0
   - ruby-head
   - rbx-2
 
@@ -30,3 +31,4 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx-2
+    - rvm: 2.3.0

From c3b7d866ffa1c13e571a9bedb268fa905aaf72b7 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 7 Jan 2016 05:07:25 +0900
Subject: [PATCH 059/193] Merge pull request #762 from
 fluent/input_output_test_driver_num_waits

Enable to control wait period of d.run in input and output test driver
---
 lib/fluent/test/input_test.rb  | 4 ++--
 lib/fluent/test/output_test.rb | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/fluent/test/input_test.rb b/lib/fluent/test/input_test.rb
index 062c4a46b0..341211cd5f 100644
--- a/lib/fluent/test/input_test.rb
+++ b/lib/fluent/test/input_test.rb
@@ -102,7 +102,7 @@ def run_should_stop?
         false
       end
 
-      def run(&block)
+      def run(num_waits = 10, &block)
         m = method(:emit_stream)
         Engine.define_singleton_method(:emit_stream) {|tag,es|
           m.call(tag, es)
@@ -110,7 +110,7 @@ def run(&block)
         instance.router.define_singleton_method(:emit_stream) {|tag,es|
           m.call(tag, es)
         }
-        super {
+        super(num_waits) {
           block.call if block
 
           if @expected_emits_length || @expects || @run_post_conditions
diff --git a/lib/fluent/test/output_test.rb b/lib/fluent/test/output_test.rb
index bca6c3e911..83d749e94a 100644
--- a/lib/fluent/test/output_test.rb
+++ b/lib/fluent/test/output_test.rb
@@ -69,9 +69,9 @@ def expect_format(str)
         (@expected_buffer ||= '') << str
       end
 
-      def run(&block)
+      def run(num_waits = 10, &block)
         result = nil
-        super {
+        super(num_waits) {
           es = ArrayEventStream.new(@entries)
           buffer = @instance.format_stream(@tag, es)
 

From 3611106efb9261fdf740bc1b1d85e8d2db70b91e Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 18 Jan 2016 06:18:08 +0900
Subject: [PATCH 060/193] Merge pull request #778 from fluent/fix-773

Don't use BigDecimal in JSON parsing result. fix #773
---
 lib/fluent/parser.rb | 1 +
 test/test_parser.rb  | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 7f6b4e7375..bd6674b8d6 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -250,6 +250,7 @@ def configure(conf)
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
+          Oj.default_options = {:bigdecimal_load => :float}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError
diff --git a/test/test_parser.rb b/test/test_parser.rb
index ba96e319f9..56e247a599 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -386,6 +386,14 @@ def test_parse(data)
       }
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_large_float(data)
+      @parser.configure('json_parser' => data)
+      @parser.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
+        assert_equal(Float, record['num'].class)
+      }
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_parse_without_time(data)
       time_at_start = Time.now.to_i

From 2346830ef662c37cf99638a82b3a53161e0c51f8 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 18 Jan 2016 06:44:21 +0900
Subject: [PATCH 061/193] Merge pull request #775 from
 kikumoto/multiline_periodic_flush

Periodic flush on in_tail plugin for multiline format
---
 lib/fluent/plugin/in_tail.rb | 37 +++++++++++++++++++--
 test/plugin/test_in_tail.rb  | 62 ++++++++++++++++++++++++++++++++----
 2 files changed, 90 insertions(+), 9 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 6c78c6c0e5..c25239810d 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -40,6 +40,8 @@ def initialize
     config_param :refresh_interval, :time, :default => 60
     desc 'The number of reading lines at each IO.'
     config_param :read_lines_limit, :integer, :default => 1000
+    desc 'The interval of flushing the buffer for multiline format'
+    config_param :multiline_flush_interval, :time, :default => nil
 
     attr_reader :paths
 
@@ -147,7 +149,8 @@ def refresh_watchers
     end
 
     def setup_watcher(path, pe)
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), &method(:receive_lines))
+      line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
       tw.attach(@loop)
       tw
     end
@@ -277,6 +280,7 @@ def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
       es = MultiEventStream.new
       if @parser.has_firstline?
+        tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
         lines.each { |line|
           if @parser.firstline?(line)
             if lb
@@ -308,7 +312,7 @@ def parse_multilines(lines, tail_watcher)
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
@@ -323,10 +327,12 @@ def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, upd
         @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
         @io_handler = nil
         @log = log
+
+        @line_buffer_timer_flusher = line_buffer_timer_flusher
       end
 
       attr_reader :path
-      attr_accessor :line_buffer
+      attr_accessor :line_buffer, :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
 
       def tag
@@ -358,6 +364,7 @@ def close(close_io = true)
 
       def on_notify
         @rotate_handler.on_notify if @rotate_handler
+        @line_buffer_timer_flusher.on_notify(self) if @line_buffer_timer_flusher
         return unless @io_handler
         @io_handler.on_notify
       end
@@ -595,6 +602,30 @@ def on_notify
           @log.error_backtrace
         end
       end
+
+
+      class LineBufferTimerFlusher
+        def initialize(log, flush_interval, &flush_method)
+          @log = log
+          @flush_interval = flush_interval
+          @flush_method = flush_method
+          @start = nil
+        end
+
+        def on_notify(tw)
+          if @start && @flush_interval
+            if Time.now - @start >= @flush_interval
+              @flush_method.call(tw)
+              tw.line_buffer = nil
+              @start = nil
+            end
+          end
+        end
+
+        def reset_timer
+          @start = Time.now
+        end
+      end
     end
 
 
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 233b0fbcd5..caac6a0922 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -12,6 +12,11 @@ def setup
     FileUtils.mkdir_p(TMP_DIR)
   end
 
+  def teardown
+    super
+    Fluent::Engine.stop
+  end
+
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = %[
@@ -288,16 +293,61 @@ def test_multiline
         f.puts "s test8"
       }
       sleep 1
+
+      emits = d.emits
+      assert(emits.length == 3)
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+      assert_equal({"message1" => "test5"}, emits[1][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+
+      sleep 3
+      emits = d.emits
+      assert(emits.length == 3)
     end
 
     emits = d.emits
-    assert(emits.length > 0)
-    assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
-    assert_equal({"message1" => "test5"}, emits[1][2])
-    assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+    assert(emits.length == 4)
     assert_equal({"message1" => "test8"}, emits[3][2])
   end
 
+  def test_multiline_with_flush_interval
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      multiline_flush_interval 2s
+    ]
+
+    assert_equal 2, d.instance.multiline_flush_interval
+
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+
+      emits = d.emits
+      assert(emits.length == 3)
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+      assert_equal({"message1" => "test5"}, emits[1][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+
+      sleep 3
+      emits = d.emits
+      assert(emits.length == 4)
+      assert_equal({"message1" => "test8"}, emits[3][2])
+    end
+  end
+
   def test_multiline_with_multiple_formats
     File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
 
@@ -434,7 +484,7 @@ def test_refresh_watchers
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -450,7 +500,7 @@ def test_refresh_watchers
       end
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times

From 413738b4b5c0fa2f58c8e6cb885f1c3eb8769514 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 22 Jan 2016 06:47:59 +0900
Subject: [PATCH 062/193] Merge pull request #781 from ysksn/add_some_objects

Add some objects cannot be parsed
---
 test/test_parser.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_parser.rb b/test/test_parser.rb
index 56e247a599..7307636cad 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -85,7 +85,7 @@ def test_parse_with_strptime
     def test_parse_with_invalid_argument
       parser = TextParser::TimeParser.new(nil)
 
-      [[], {}, nil, true, 10000].each { |v|
+      [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
         assert_raise Fluent::ParserError do
           parser.parse(v)
         end

From b4bc7090a2524502ea130ab3be4258684ac6383f Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sat, 23 Jan 2016 11:03:51 +0900
Subject: [PATCH 063/193] Merge pull request #776 from
 fluent/add-skip-invalid-event

Add skip_invalid_event parameter to in_forward. fixes #774
---
 lib/fluent/plugin/in_forward.rb | 48 +++++++++++++++++++------
 test/plugin/test_in_forward.rb  | 64 +++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+), 10 deletions(-)

diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index 48581f5029..ce91d0fe02 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -38,6 +38,8 @@ def initialize
     config_param :chunk_size_warn_limit, :size, :default => nil
     desc 'Received chunk is dropped if it is larger than this value.'
     config_param :chunk_size_limit, :size, :default => nil
+    desc 'Skip an event if incoming event is invalid.'
+    config_param :skip_invalid_event, :bool, :default => false
 
     def configure(conf)
       super
@@ -136,7 +138,7 @@ def on_message(msg, chunk_size, source)
         return
       end
 
-      tag = msg[0].to_s
+      tag = msg[0]
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
@@ -149,27 +151,37 @@ def on_message(msg, chunk_size, source)
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries)
+        es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
         router.emit_stream(tag, es)
         option = msg[2]
 
       elsif entries.class == Array
         # Forward
-        es = MultiEventStream.new
-        entries.each {|e|
-          record = e[1]
-          next if record.nil?
-          time = e[0].to_i
-          time = (now ||= Engine.now) if time == 0
-          es.add(time, record)
-        }
+        es = if @skip_invalid_event
+               check_and_skip_invalid_event(tag, entries, source)
+             else
+               es = MultiEventStream.new
+               entries.each { |e|
+                 record = e[1]
+                 next if record.nil?
+                 time = e[0]
+                 time = (now ||= Engine.now) if time.to_i == 0
+                 es.add(time, record)
+               }
+               es
+             end
         router.emit_stream(tag, es)
         option = msg[2]
 
       else
         # Message
+        time = msg[1]
         record = msg[2]
+        if @skip_invalid_event && invalid_event?(tag, time, record)
+          log.warn "got invalid event and drop it:", source: source, tag: tag, time: time, record: record
+          return msg[3] # retry never succeeded so return ack and drop incoming event.
+        end
         return if record.nil?
-        time = msg[1]
         time = Engine.now if time == 0
         router.emit(tag, time, record)
         option = msg[3]
@@ -179,6 +191,22 @@ def on_message(msg, chunk_size, source)
       option
     end
 
+    def invalid_event?(tag, time, record)
+      !(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
+    end
+
+    def check_and_skip_invalid_event(tag, es, source)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        if invalid_event?(tag, time, record)
+          log.warn "skip invalid event:", source: source, tag: tag, time: time, record: record
+          next
+        end
+        new_es.add(time, record)
+      }
+      new_es
+    end
+
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
diff --git a/test/plugin/test_in_forward.rb b/test/plugin/test_in_forward.rb
index a012ba7214..c4285b6856 100644
--- a/test/plugin/test_in_forward.rb
+++ b/test/plugin/test_in_forward.rb
@@ -64,6 +64,28 @@ def test_message
     end
   end
 
+  def test_message_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map { |tag, time, record| [tag, time, record] }
+      # These entries are skipped
+      entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
+
+      entries.each { |tag, time, record|
+        # Without ack, logs are sometimes not saved to logs during test.
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s, true
+      }
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
+  end
+
   def test_forward
     d = create_driver
 
@@ -81,6 +103,25 @@ def test_forward
     end
   end
 
+  def test_forward_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag1", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map { |tag, time, record| [time, record] }
+      # These entries are skipped
+      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+  end
+
   def test_packed_forward
     d = create_driver
 
@@ -98,6 +139,29 @@ def test_packed_forward
     end
   end
 
+  def test_packed_forward_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag1", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map { |tag ,time, record| [time, record] }
+      # These entries are skipped
+      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+      packed_entries = ''
+      entries.each { |time, record|
+        Fluent::Engine.msgpack_factory.packer(packed_entries).write([time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", packed_entries]).to_s
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+  end
+
   def test_message_json
     d = create_driver
 

From d20398cf219f81d5d285b3f6081f1834e8937c27 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 26 Jan 2016 00:04:33 +0900
Subject: [PATCH 064/193] v0.12 doesn't have EventTime

---
 test/plugin/test_in_forward.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/plugin/test_in_forward.rb b/test/plugin/test_in_forward.rb
index c4285b6856..b103d8d3e7 100644
--- a/test/plugin/test_in_forward.rb
+++ b/test/plugin/test_in_forward.rb
@@ -67,7 +67,7 @@ def test_message
   def test_message_with_skip_invalid_event
     d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
     d.expect_emit "tag1", time, {"a" => 1}
     d.expect_emit "tag2", time, {"a" => 2}
@@ -106,7 +106,7 @@ def test_forward
   def test_forward_with_skip_invalid_event
     d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
     d.expect_emit "tag1", time, {"a" => 1}
     d.expect_emit "tag1", time, {"a" => 2}
@@ -142,7 +142,7 @@ def test_packed_forward
   def test_packed_forward_with_skip_invalid_event
     d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
     d.expect_emit "tag1", time, {"a" => 1}
     d.expect_emit "tag1", time, {"a" => 2}

From e89da82dd3744aada7a0268d06b583c40c9299c4 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 26 Jan 2016 07:08:41 +0900
Subject: [PATCH 065/193] v0.12.20

---
 ChangeLog             | 23 +++++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d773eb25ef..d6a373baca 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,28 @@
 # v0.12
 
+## Release 0.12.20 - 2015/01/26
+
+### New features / Enhancement
+
+* in_forward: Add skip_invalid_event paramter to check and skip invalid event
+  https://github.com/fluent/fluentd/pull/776
+* in_tail: Add multiline_flush_interval parameter for periodic flush with multiline format
+  https://github.com/fluent/fluentd/pull/775
+* filter_record_transformer: Improve ruby placeholder performance and adding 'record["key"]' syntax
+  https://github.com/fluent/fluentd/pull/766
+* Add on_exit_process hook point to DetachProcessMixin
+  https://github.com/fluent/fluentd/pull/757
+* Add descriptions to BufferedOutput and TimeSlicedOutput
+  https://github.com/fluent/fluentd/pull/759
+  https://github.com/fluent/fluentd/pull/760
+
+### Bug fixes
+
+* parser: Don't use BigDecimal in JSON parsing result with Oj
+  https://github.com/fluent/fluentd/pull/778
+* config: Fix the regression of unused parameter warning inside sub section
+  https://github.com/fluent/fluentd/pull/765
+
 ## Release 0.12.19 - 2015/12/21
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 3ad7d0359a..0b0a8cf2d2 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.19'
+  VERSION = '0.12.20'
 
 end

From 0da3442d7e66654ae005f8c6c09b684d8715115d Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 4 Feb 2016 21:52:05 +0900
Subject: [PATCH 066/193] Merge pull request #791 from
 fluent/use-error-stream-in-tsoutput-emit

Use error stream in TimeSlicedOutput#emit
---
 lib/fluent/output.rb    | 19 ++++++++++++-------
 lib/fluent/test/base.rb |  1 +
 test/test_output.rb     | 30 +++++++++++++++++++++++++++---
 3 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index 1a9fe5c24b..0d9b3c70fa 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -552,14 +552,19 @@ def emit(tag, es, chain)
       @emit_count += 1
       formatted_data = {}
       es.each {|time,record|
-        tc = time / @time_slice_cache_interval
-        if @before_tc == tc
-          key = @before_key
-        else
-          @before_tc = tc
-          key = @time_slicer.call(time)
-          @before_key = key
+        begin
+          tc = time / @time_slice_cache_interval
+          if @before_tc == tc
+            key = @before_key
+          else
+            @before_tc = tc
+            key = @time_slicer.call(time)
+            @before_key = key
+          end
+        rescue => e
+          @router.emit_error_event(tag, Engine.now, {'time' => time, 'record' => record}, e)
         end
+
         formatted_data[key] ||= ''
         formatted_data[key] << format(tag, time, record)
       }
diff --git a/lib/fluent/test/base.rb b/lib/fluent/test/base.rb
index 43be9002a5..8d74878739 100644
--- a/lib/fluent/test/base.rb
+++ b/lib/fluent/test/base.rb
@@ -47,6 +47,7 @@ def initialize(klass, &block)
         end
         @instance.router = Engine.root_agent.event_router
         @instance.log = TestLogger.new
+        Engine.root_agent.instance_variable_set(:@log, @instance.log)
 
         @config = Config.new
       end
diff --git a/test/test_output.rb b/test/test_output.rb
index 351e6ed284..774fd34a20 100644
--- a/test/test_output.rb
+++ b/test/test_output.rb
@@ -215,13 +215,13 @@ def setup
 
     TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/time_sliced_output")
 
-    CONFIG = %[]
+    CONFIG = %[buffer_path #{TMP_DIR}/foo]
 
     def create_driver(conf=CONFIG)
       Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::TimeSlicedOutput).configure(conf, true)
     end
 
-    sub_test_case "test_force_flush" do
+    sub_test_case "force_flush test" do
       setup do
         time = Time.parse("2011-01-02 13:14:15 UTC")
         Timecop.freeze(time)
@@ -235,7 +235,6 @@ def create_driver(conf=CONFIG)
       test "force_flush immediately flushes" do
         d = create_driver(CONFIG + %[
           time_format %Y%m%d%H%M%S
-          buffer_path #{TMP_DIR}/foo
         ])
         d.instance.start
         # buffer should be popped (flushed) immediately
@@ -246,5 +245,30 @@ def create_driver(conf=CONFIG)
         10.times { sleep 0.05 }
       end
     end
+
+    sub_test_case "test emit" do
+      setup do
+        @time = Time.parse("2011-01-02 13:14:15 UTC")
+        Timecop.freeze(@time)
+      end
+
+      teardown do
+        Timecop.return
+      end
+
+      test "emit with valid event" do
+        d = create_driver
+        d.instance.start
+        d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
+        assert_equal 0, d.instance.log.logs.size
+      end
+
+      test "emit with invalid event" do
+        d = create_driver
+        d.instance.start
+        d.instance.emit('test', OneEventStream.new('string', 10), NullOutputChain.instance)
+        assert_equal 1, d.instance.log.logs.count { |line| line =~ /dump an error event/ }
+      end
+    end
   end
 end

From aee8086e9fcd3b45fa11b83e866fd758cb79bffb Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 9 Feb 2016 19:40:09 +0900
Subject: [PATCH 067/193] Merge pull request #797 from
 fluent/skip-broken-event-in-tsoutput

Skip broken event which is routed to error stream
---
 lib/fluent/output.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index 0d9b3c70fa..2b8523b283 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -563,6 +563,7 @@ def emit(tag, es, chain)
           end
         rescue => e
           @router.emit_error_event(tag, Engine.now, {'time' => time, 'record' => record}, e)
+          next
         end
 
         formatted_data[key] ||= ''

From d1ba7d868d18b00f9947f17a316f137aead04456 Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Tue, 9 Feb 2016 16:48:55 +0900
Subject: [PATCH 068/193] Merge pull request #794 from
 fluent/separate-system-config

Separate system config object from supervisor
---
 lib/fluent/configurable.rb        |  3 +-
 lib/fluent/engine.rb              | 14 +++--
 lib/fluent/root_agent.rb          |  7 ++-
 lib/fluent/supervisor.rb          | 18 ++----
 lib/fluent/system_config.rb       | 92 +++++++++++++++++++++++++++++++
 lib/fluent/test/base.rb           |  2 +-
 test/config/test_system_config.rb | 10 ++--
 test/test_root_agent.rb           |  7 ++-
 8 files changed, 121 insertions(+), 32 deletions(-)
 create mode 100644 lib/fluent/system_config.rb

diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
index a17008ed2a..262e062690 100644
--- a/lib/fluent/configurable.rb
+++ b/lib/fluent/configurable.rb
@@ -19,6 +19,7 @@ module Fluent
   require 'fluent/config/section'
   require 'fluent/config/error'
   require 'fluent/registry'
+  require 'fluent/plugin'
 
   module Configurable
     def self.included(mod)
@@ -51,7 +52,7 @@ def configure(conf)
       conf.corresponding_proxies << proxy
 
       # In the nested section, can't get plugin class through proxies so get plugin class here
-      plugin_class = Plugin.lookup_name_from_class(proxy.name.to_s)
+      plugin_class = Fluent::Plugin.lookup_name_from_class(proxy.name.to_s)
       root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
       @config_root_section = root
 
diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index 48acb7c4df..3f0d7a27cf 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -18,6 +18,7 @@ module Fluent
   require 'fluent/event_router'
   require 'fluent/root_agent'
   require 'fluent/time'
+  require 'fluent/system_config'
 
   class EngineClass
     class DummyMessagePackFactory
@@ -51,8 +52,11 @@ def initialize
     attr_reader :root_agent
     attr_reader :matches, :sources
     attr_reader :msgpack_factory
+    attr_reader :system_config
+
+    def init(system_config)
+      @system_config = system_config
 
-    def init(opts = {})
       BasicSocket.do_not_reverse_lookup = true
       Plugin.load_plugins
       if defined?(Encoding)
@@ -60,11 +64,11 @@ def init(opts = {})
         Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
       end
 
-      suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
-      @suppress_config_dump = opts[:suppress_config_dump] if opts[:suppress_config_dump]
-      @without_source = opts[:without_source] if opts[:without_source]
+      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
+      @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
+      @without_source = system_config.without_source unless system_config.without_source.nil?
 
-      @root_agent = RootAgent.new(opts)
+      @root_agent = RootAgent.new(@system_config)
 
       self
     end
diff --git a/lib/fluent/root_agent.rb b/lib/fluent/root_agent.rb
index a0851c78b7..2445ec2ea7 100644
--- a/lib/fluent/root_agent.rb
+++ b/lib/fluent/root_agent.rb
@@ -19,6 +19,7 @@ module Fluent
 
   require 'fluent/agent'
   require 'fluent/label'
+  require 'fluent/system_config'
 
   #
   # Fluentd forms a tree structure to manage plugins:
@@ -43,7 +44,7 @@ module Fluent
   class RootAgent < Agent
     ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
 
-    def initialize(opts = {})
+    def initialize(system_config = SystemConfig.new)
       super
 
       @labels = {}
@@ -52,8 +53,8 @@ def initialize(opts = {})
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
 
-      suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
-      @without_source = opts[:without_source] if opts[:without_source]
+      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
+      @without_source = system_config.without_source unless system_config.without_source.nil?
     end
 
     attr_reader :inputs
diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index 10299ba3ea..17f4130b52 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -15,6 +15,7 @@
 #
 
 require 'fluent/load'
+require 'fluent/system_config'
 require 'etc'
 
 module Fluent
@@ -128,7 +129,8 @@ def start
       @log.init
       show_plugin_config if @show_plugin_config
       read_config
-      apply_system_config
+      @system_config = SystemConfig.create(@conf) # @conf is set in read_config
+      @system_config.apply(self)
 
       dry_run if @dry_run
       start_daemonize if @daemonize
@@ -500,17 +502,6 @@ def apply(supervisor)
       end
     end
 
-    # TODO: this method should be moved to SystemConfig class method
-    def apply_system_config
-      systems = @conf.elements.select { |e|
-        e.name == 'system'
-      }
-      return if systems.empty?
-      raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
-
-      SystemConfig.new(systems.first).apply(self)
-    end
-
     def run_configure
       Fluent::Engine.run_configure(@conf)
     end
@@ -533,8 +524,7 @@ def change_privilege
     end
 
     def init_engine
-      init_opts = {:suppress_interval => @suppress_interval, :suppress_config_dump => @suppress_config_dump, :without_source => @without_source}
-      Fluent::Engine.init(init_opts)
+      Fluent::Engine.init(@system_config)
 
       @libs.each {|lib|
         require lib
diff --git a/lib/fluent/system_config.rb b/lib/fluent/system_config.rb
new file mode 100644
index 0000000000..4452e9dd86
--- /dev/null
+++ b/lib/fluent/system_config.rb
@@ -0,0 +1,92 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  require 'fluent/configurable'
+
+  module SystemConfigMixin
+    def system_config
+      @_system_config || Fluent::Engine.system_config
+    end
+
+    def system_config_override(opts={})
+      unless @_system_config
+        @_system_config = Fluent::Engine.system_config.dup
+      end
+      opts.each_pair do |key, value|
+        @_system_config.send(:"#{key.to_s}=", value)
+      end
+    end
+  end
+
+  class SystemConfig
+    include Configurable
+
+    config_param :log_level, default: nil do |level|
+      Log.str_to_level(level)
+    end
+    config_param :suppress_repeated_stacktrace, :bool, default: nil
+    config_param :emit_error_log_interval, :time, default: nil
+    config_param :suppress_config_dump, :bool, default: nil
+    config_param :without_source, :bool, default: nil
+    config_param :rpc_endpoint, :string, default: nil
+    config_param :enable_get_dump, :bool, default: nil
+    config_param :process_name, default: nil
+
+    def self.create(conf)
+      systems = conf.elements.select { |e|
+        e.name == 'system'
+      }
+      return SystemConfig.new if systems.empty?
+      raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
+
+      SystemConfig.new(systems.first)
+    end
+
+    def initialize(conf={})
+      super()
+      configure(conf)
+    end
+
+    def dup
+      s = SystemConfig.new
+      s.log_level = @log_level
+      s.suppress_repeated_stacktrace = @suppress_repeated_stacktrace
+      s.emit_error_log_interval = @emit_error_log_interval
+      s.suppress_config_dump = @suppress_config_dump
+      s.without_source = @without_source
+      s.rpc_endpoint = @rpc_endpoint
+      s.enable_get_dump = @enable_get_dump
+      s.process_name = @process_name
+
+      s
+    end
+
+    def apply(supervisor)
+      system = self
+      supervisor.instance_eval {
+        @log.level = @log_level = system.log_level unless system.log_level.nil?
+        @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
+        @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
+        @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
+        @without_source = system.without_source unless system.without_source.nil?
+        @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
+        @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
+        @process_name = system.process_name unless system.process_name.nil?
+      }
+    end
+  end
+end
diff --git a/lib/fluent/test/base.rb b/lib/fluent/test/base.rb
index 8d74878739..8c40ac77ce 100644
--- a/lib/fluent/test/base.rb
+++ b/lib/fluent/test/base.rb
@@ -18,7 +18,7 @@ module Fluent
   module Test
     def self.setup
       Fluent.__send__(:remove_const, :Engine)
-      engine = Fluent.const_set(:Engine, EngineClass.new).init
+      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
 
       engine.define_singleton_method(:now=) {|n|
         @now = n.to_i
diff --git a/test/config/test_system_config.rb b/test/config/test_system_config.rb
index 0775415d67..e4cfb13f01 100644
--- a/test/config/test_system_config.rb
+++ b/test/config/test_system_config.rb
@@ -2,7 +2,7 @@
 require 'fluent/configurable'
 require 'fluent/config/element'
 require 'fluent/config/section'
-require 'fluent/supervisor'
+require 'fluent/system_config'
 
 module Fluent::Config
   class FakeLoggerInitializer
@@ -36,7 +36,7 @@ def parse_text(text)
         </system>
       EOS
       s = FakeSupervisor.new
-      sc = Fluent::Supervisor::SystemConfig.new(conf)
+      sc = Fluent::SystemConfig.new(conf)
       sc.apply(s)
       assert_nil(sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
@@ -63,7 +63,7 @@ def parse_text(text)
           </system>
         EOS
         s = FakeSupervisor.new
-        sc = Fluent::Supervisor::SystemConfig.new(conf)
+        sc = Fluent::SystemConfig.new(conf)
         sc.apply(s)
         assert_not_nil(sc.instance_variable_get("@#{k}"))
         key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
@@ -74,7 +74,7 @@ def parse_text(text)
     {'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
       test "should not affect settable parameters with unknown #{k} parameter" do
         s = FakeSupervisor.new
-        sc = Fluent::Supervisor::SystemConfig.new({k => v})
+        sc = Fluent::SystemConfig.new({k => v})
         sc.apply(s)
         assert_nil(s.instance_variable_get(:@log_level))
         assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
@@ -91,7 +91,7 @@ def parse_text(text)
         </system>
       EOS
       s = FakeSupervisor.new
-      sc = Fluent::Supervisor::SystemConfig.new(conf)
+      sc = Fluent::SystemConfig.new(conf)
       sc.apply(s)
       assert_equal(Fluent::Log::LEVEL_WARN, s.instance_variable_get("@log").level)
     end
diff --git a/test/test_root_agent.rb b/test/test_root_agent.rb
index 0b7cc784ed..50ff51b55e 100644
--- a/test/test_root_agent.rb
+++ b/test/test_root_agent.rb
@@ -1,4 +1,5 @@
 require 'fluent/event_router'
+require 'fluent/system_config'
 require_relative 'test_plugin_classes'
 
 class RootAgentTest < ::Test::Unit::TestCase
@@ -12,12 +13,12 @@ def test_initialize
   end
 
   data(
-    'suppress interval' => [{:suppress_interval => 30}, {:@suppress_emit_error_log_interval => 30}],
-    'without source' => [{:without_source => true}, {:@without_source => true}]
+    'suppress interval' => [{'emit_error_log_interval' => 30}, {:@suppress_emit_error_log_interval => 30}],
+    'without source' => [{'without_source' => true}, {:@without_source => true}]
     )
   def test_initialize_with_opt(data)
     opt, expected = data
-    ra = RootAgent.new(opt)
+    ra = RootAgent.new(SystemConfig.new(opt))
     expected.each { |k, v|
       assert_equal v, ra.instance_variable_get(k)
     }

From eb1478d5dead8cc036a2cf3970b7c3b138ce6e3f Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Tue, 9 Feb 2016 18:28:18 +0900
Subject: [PATCH 069/193] Merge pull request #795 from fluent/hash-syntax

enforce new hash syntax by `rubocop --only HashSyntax --auto-correct`
---
 Rakefile                                      |   4 +-
 lib/fluent/agent.rb                           |  12 +-
 lib/fluent/buffer.rb                          |   6 +-
 lib/fluent/engine.rb                          |   4 +-
 lib/fluent/formatter.rb                       |  46 ++++----
 lib/fluent/log.rb                             |   2 +-
 lib/fluent/output.rb                          |  45 ++++----
 lib/fluent/parser.rb                          |  46 ++++----
 lib/fluent/plugin/buf_file.rb                 |   4 +-
 lib/fluent/plugin/buf_memory.rb               |   2 +-
 lib/fluent/plugin/filter_grep.rb              |   6 +-
 .../plugin/filter_record_transformer.rb       |  22 ++--
 lib/fluent/plugin/filter_stdout.rb            |   2 +-
 lib/fluent/plugin/in_debug_agent.rb           |  10 +-
 lib/fluent/plugin/in_dummy.rb                 |   6 +-
 lib/fluent/plugin/in_exec.rb                  |  18 +--
 lib/fluent/plugin/in_forward.rb               |  22 ++--
 lib/fluent/plugin/in_gc_stat.rb               |   4 +-
 lib/fluent/plugin/in_http.rb                  |  26 ++---
 lib/fluent/plugin/in_monitor_agent.rb         |  20 ++--
 lib/fluent/plugin/in_object_space.rb          |   8 +-
 lib/fluent/plugin/in_stream.rb                |  12 +-
 lib/fluent/plugin/in_syslog.rb                |  22 ++--
 lib/fluent/plugin/in_tail.rb                  |  18 +--
 lib/fluent/plugin/in_tcp.rb                   |   2 +-
 lib/fluent/plugin/in_udp.rb                   |   2 +-
 lib/fluent/plugin/out_copy.rb                 |   2 +-
 lib/fluent/plugin/out_exec.rb                 |  10 +-
 lib/fluent/plugin/out_exec_filter.rb          |  50 ++++----
 lib/fluent/plugin/out_file.rb                 |  10 +-
 lib/fluent/plugin/out_forward.rb              |  42 +++----
 lib/fluent/plugin/out_stdout.rb               |   2 +-
 lib/fluent/plugin/out_stream.rb               |   4 +-
 lib/fluent/plugin/socket_util.rb              |  16 +--
 lib/fluent/process.rb                         |  16 +--
 lib/fluent/root_agent.rb                      |  10 +-
 lib/fluent/rpc.rb                             |  10 +-
 lib/fluent/supervisor.rb                      |  58 +++++-----
 test/config/test_configurable.rb              |  22 ++--
 test/config/test_configure_proxy.rb           |   2 +-
 test/plugin/test_filter_record_transformer.rb | 108 +++++++++---------
 test/test_config.rb                           |   4 +-
 42 files changed, 368 insertions(+), 369 deletions(-)

diff --git a/Rakefile b/Rakefile
index 9c968f3a6c..8b0184e462 100755
--- a/Rakefile
+++ b/Rakefile
@@ -5,7 +5,7 @@ require 'fileutils'
 require 'rake/testtask'
 require 'rake/clean'
 
-task :test => [:base_test]
+task test: [:base_test]
 
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
@@ -30,4 +30,4 @@ task :coverage do |t|
   Rake::Task["test"].invoke
 end
 
-task :default => [:test, :build]
+task default: [:test, :build]
diff --git a/lib/fluent/agent.rb b/lib/fluent/agent.rb
index 5080f2377c..1c766fe197 100644
--- a/lib/fluent/agent.rb
+++ b/lib/fluent/agent.rb
@@ -81,7 +81,7 @@ def shutdown
             log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
             f.shutdown
           rescue => e
-            log.warn "unexpected error while shutting down filter plugins", :plugin => f.class, :plugin_id => f.plugin_id, :error_class => e.class, :error => e
+            log.warn "unexpected error while shutting down filter plugins", plugin: f.class, plugin_id: f.plugin_id, error_class: e.class, error: e
             log.warn_backtrace
           end
         end
@@ -95,7 +95,7 @@ def shutdown
             log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
             o.shutdown
           rescue => e
-            log.warn "unexpected error while shutting down output plugins", :plugin => o.class, :plugin_id => o.plugin_id, :error_class => e.class, :error => e
+            log.warn "unexpected error while shutting down output plugins", plugin: o.class, plugin_id: o.plugin_id, error_class: e.class, error: e
             log.warn_backtrace
           end
         end
@@ -115,7 +115,7 @@ def flush_recursive(array)
             flush_recursive(o.outputs)
           end
         rescue => e
-          log.debug "error while force flushing", :error_class => e.class, :error => e
+          log.debug "error while force flushing", error_class: e.class, error: e
           log.debug_backtrace
         end
       }
@@ -163,16 +163,16 @@ def emit(tag, es, chain)
         c = (@count += 1)
         if c < 512
           if Math.log(c) / Math.log(2) % 1.0 == 0
-            @log.warn "no patterns matched", :tag => tag
+            @log.warn "no patterns matched", tag: tag
             return
           end
         else
           if c % 512 == 0
-            @log.warn "no patterns matched", :tag => tag
+            @log.warn "no patterns matched", tag: tag
             return
           end
         end
-        @log.on_trace { @log.trace "no patterns matched", :tag => tag }
+        @log.on_trace { @log.trace "no patterns matched", tag: tag }
       end
 
       def start
diff --git a/lib/fluent/buffer.rb b/lib/fluent/buffer.rb
index 97f10d2ad0..2187d835f1 100644
--- a/lib/fluent/buffer.rb
+++ b/lib/fluent/buffer.rb
@@ -140,11 +140,11 @@ def enable_parallel(b=true)
     # Local file based plugins which should provide more reliability and efficiency
     # should override buffer_chunk_limit with a larger size.
     desc 'The size of each buffer chunk.'
-    config_param :buffer_chunk_limit, :size, :default => 8*1024*1024
+    config_param :buffer_chunk_limit, :size, default: 8*1024*1024
     desc 'The length limit of the chunk queue.'
-    config_param :buffer_queue_limit, :integer, :default => 256
+    config_param :buffer_queue_limit, :integer, default: 256
     desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
-    config_param :buffer_queue_full_action, :enum, :list => [:exception, :block], :default => :exception
+    config_param :buffer_queue_full_action, :enum, list: [:exception, :block], default: :exception
 
     alias chunk_limit buffer_chunk_limit
     alias chunk_limit= buffer_chunk_limit=
diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index 48acb7c4df..4bb8927a61 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -164,7 +164,7 @@ def log_event_loop
           begin
             @event_router.emit(tag, time, record)
           rescue => e
-            $log.error "failed to emit fluentd's log event", :tag => tag, :event => record, :error_class => e.class, :error => e
+            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error_class: e.class, error: e
           end
         }
       end
@@ -193,7 +193,7 @@ def run
         end
 
       rescue => e
-        $log.error "unexpected error", :error_class=>e.class, :error=>e
+        $log.error "unexpected error", error_class: e.class, error: e
         $log.error_backtrace
       ensure
         $log.info "shutting down fluentd"
diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index 5b14892f3a..dbc4c86b4b 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -33,13 +33,13 @@ module TextFormatter
     module HandleTagAndTimeMixin
       def self.included(klass)
         klass.instance_eval {
-          config_param :include_time_key, :bool, :default => false
-          config_param :time_key, :string, :default => 'time'
-          config_param :time_format, :string, :default => nil
-          config_param :include_tag_key, :bool, :default => false
-          config_param :tag_key, :string, :default => 'tag'
-          config_param :localtime, :bool, :default => true
-          config_param :timezone, :string, :default => nil
+          config_param :include_time_key, :bool, default: false
+          config_param :time_key, :string, default: 'time'
+          config_param :time_format, :string, default: nil
+          config_param :include_tag_key, :bool, default: false
+          config_param :tag_key, :string, default: 'tag'
+          config_param :localtime, :bool, default: true
+          config_param :timezone, :string, default: nil
         }
       end
 
@@ -65,9 +65,9 @@ def filter_record(tag, time, record)
     class OutFileFormatter < Formatter
       include HandleTagAndTimeMixin
 
-      config_param :output_time, :bool, :default => true
-      config_param :output_tag, :bool, :default => true
-      config_param :delimiter, :default => "\t" do |val|
+      config_param :output_time, :bool, default: true
+      config_param :output_tag, :bool, default: true
+      config_param :delimiter, default: "\t" do |val|
         case val
         when /SPACE/i then ' '
         when /COMMA/i then ','
@@ -85,7 +85,7 @@ def format(tag, time, record)
     end
 
     class StdoutFormatter < Formatter
-      config_param :output_type, :string, :default => 'json'
+      config_param :output_type, :string, default: 'json'
 
       def configure(conf)
         super
@@ -103,7 +103,7 @@ def format(tag, time, record)
     module StructuredFormatMixin
       def self.included(klass)
         klass.instance_eval {
-          config_param :time_as_epoch, :bool, :default => false
+          config_param :time_as_epoch, :bool, default: false
         }
       end
 
@@ -131,7 +131,7 @@ class JSONFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
-      config_param :json_parser, :string, :default => 'oj'
+      config_param :json_parser, :string, default: 'oj'
 
       def configure(conf)
         super
@@ -139,7 +139,7 @@ def configure(conf)
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {:mode => :compat}
+          Oj.default_options = {mode: :compat}
           @dump_proc = Oj.method(:dump)
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
@@ -172,8 +172,8 @@ def format_record(record)
     class LabeledTSVFormatter < Formatter
       include HandleTagAndTimeMixin
 
-      config_param :delimiter, :string, :default => "\t"
-      config_param :label_delimiter, :string, :default =>  ":"
+      config_param :delimiter, :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
 
       def format(tag, time, record)
         filter_record(tag, time, record)
@@ -189,11 +189,11 @@ def format(tag, time, record)
     class CsvFormatter < Formatter
       include HandleTagAndTimeMixin
 
-      config_param :delimiter, :default => ',' do |val|
+      config_param :delimiter, default: ',' do |val|
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
-      config_param :force_quotes, :bool, :default => true
-      config_param :fields, :default => [] do |val|
+      config_param :force_quotes, :bool, default: true
+      config_param :fields, default: [] do |val|
         val.split(',').map do |f|
           f.strip!
           f.size > 0 ? f : nil
@@ -211,14 +211,14 @@ def format(tag, time, record)
             memo << record[key]
             memo
         end
-        CSV.generate_line(row, :col_sep => @delimiter,
-                          :force_quotes => @force_quotes)
+        CSV.generate_line(row, col_sep: @delimiter,
+                          force_quotes: @force_quotes)
       end
     end
 
     class SingleValueFormatter < Formatter
-      config_param :message_key, :string, :default => 'message'
-      config_param :add_newline, :bool, :default => true
+      config_param :message_key, :string, default: 'message'
+      config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup
diff --git a/lib/fluent/log.rb b/lib/fluent/log.rb
index f14f27c32f..978da489d1 100644
--- a/lib/fluent/log.rb
+++ b/lib/fluent/log.rb
@@ -344,7 +344,7 @@ module PluginLoggerMixin
     def self.included(klass)
       klass.instance_eval {
         desc 'Allows the user to set different levels of logging for each plugin.'
-        config_param :log_level, :string, :default => nil, :alias => :@log_level
+        config_param :log_level, :string, default: nil, alias: :@log_level
       }
     end
 
diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index 2b8523b283..ebeea00c34 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -88,7 +88,7 @@ def shutdown
 
     def secondary_init(primary)
       if primary.class != self.class
-        $log.warn "type of secondary output should be same as primary output", :primary=>primary.class.to_s, :secondary=>self.class.to_s
+        $log.warn "type of secondary output should be same as primary output", primary: primary.class.to_s, secondary: self.class.to_s
       end
     end
   end
@@ -152,7 +152,7 @@ def run
         @mutex.unlock
       end
     rescue
-      $log.error "error on output thread", :error=>$!.to_s
+      $log.error "error on output thread", error: $!.to_s
       $log.error_backtrace
       raise
     ensure
@@ -180,22 +180,22 @@ def initialize
     end
 
     desc 'The buffer type (memory, file)'
-    config_param :buffer_type, :string, :default => 'memory'
+    config_param :buffer_type, :string, default: 'memory'
     desc 'The interval between data flushes.'
-    config_param :flush_interval, :time, :default => 60
-    config_param :try_flush_interval, :float, :default => 1
+    config_param :flush_interval, :time, default: 60
+    config_param :try_flush_interval, :float, default: 1
     desc 'If true, the value of `retry_value` is ignored and there is no limit'
-    config_param :disable_retry_limit, :bool, :default => false
+    config_param :disable_retry_limit, :bool, default: false
     desc 'The limit on the number of retries before buffered data is discarded'
-    config_param :retry_limit, :integer, :default => 17
+    config_param :retry_limit, :integer, default: 17
     desc 'The initial intervals between write retries.'
-    config_param :retry_wait, :time, :default => 1.0
+    config_param :retry_wait, :time, default: 1.0
     desc 'The maximum intervals between write retries.'
-    config_param :max_retry_wait, :time, :default => nil
+    config_param :max_retry_wait, :time, default: nil
     desc 'The number of threads to flush the buffer.'
-    config_param :num_threads, :integer, :default => 1
+    config_param :num_threads, :integer, default: 1
     desc 'The interval between data flushes for queued chunk.'
-    config_param :queued_chunk_flush_interval, :time, :default => 1
+    config_param :queued_chunk_flush_interval, :time, default: 1
 
     def configure(conf)
       super
@@ -334,7 +334,7 @@ def try_flush
           @num_errors = 0
           # Note: don't notify to other threads to prevent
           #       burst to recovered server
-          $log.warn "retry succeeded.", :plugin_id=>plugin_id
+          $log.warn "retry succeeded.", plugin_id: plugin_id
         end
 
         if has_next
@@ -359,20 +359,20 @@ def try_flush
         end
 
         if @disable_retry_limit || error_count < @retry_limit
-          $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+          $log.warn "temporarily failed to flush the buffer.", next_retry: Time.at(@next_retry_time), error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
           $log.warn_backtrace e.backtrace
 
         elsif @secondary
           if error_count == @retry_limit
-            $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+            $log.warn "failed to flush the buffer.", error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
             $log.warn "retry count exceededs limit. falling back to secondary output."
             $log.warn_backtrace e.backtrace
             retry  # retry immediately
           elsif error_count <= @retry_limit + @secondary_limit
-            $log.warn "failed to flush the buffer, next retry will be with secondary output.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+            $log.warn "failed to flush the buffer, next retry will be with secondary output.", next_retry: Time.at(@next_retry_time), error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
             $log.warn_backtrace e.backtrace
           else
-            $log.warn "failed to flush the buffer.", :error_class=>e.class, :error=>e.to_s, :plugin_id=>plugin_id
+            $log.warn "failed to flush the buffer.", error_class: e.class, error: e.to_s, plugin_id: plugin_id
             $log.warn "secondary retry count exceededs limit."
             $log.warn_backtrace e.backtrace
             write_abort
@@ -380,7 +380,7 @@ def try_flush
           end
 
         else
-          $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+          $log.warn "failed to flush the buffer.", error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
           $log.warn "retry count exceededs limit."
           $log.warn_backtrace e.backtrace
           write_abort
@@ -403,7 +403,7 @@ def before_shutdown
       begin
         @buffer.before_shutdown(self)
       rescue
-        $log.warn "before_shutdown failed", :error=>$!.to_s
+        $log.warn "before_shutdown failed", error: $!.to_s
         $log.warn_backtrace
       end
     end
@@ -425,7 +425,7 @@ def write_abort
       begin
         @buffer.clear!
       rescue
-        $log.error "unexpected error while aborting", :error=>$!.to_s
+        $log.error "unexpected error while aborting", error: $!.to_s
         $log.error_backtrace
       end
     end
@@ -483,11 +483,11 @@ def initialize
     end
 
     desc 'The time format used as part of the file name.'
-    config_param :time_slice_format, :string, :default => '%Y%m%d'
+    config_param :time_slice_format, :string, default: '%Y%m%d'
     desc 'The amount of time Fluentd will wait for old logs to arrive.'
-    config_param :time_slice_wait, :time, :default => 10*60
+    config_param :time_slice_wait, :time, default: 10*60
     desc 'Parse the time value in the specified timezone'
-    config_param :timezone, :string, :default => nil
+    config_param :timezone, :string, default: nil
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit
     config_set_default :flush_interval, nil
@@ -609,4 +609,3 @@ class MultiOutput < Output
     #end
   end
 end
-
diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index bd6674b8d6..ef14b46a09 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -27,7 +27,7 @@ class Parser
     # 'configure()' may raise errors for unexpected configurations
     attr_accessor :estimate_current_event
 
-    config_param :keep_time_key, :bool, :default => false
+    config_param :keep_time_key, :bool, default: false
 
     def initialize
       super
@@ -113,9 +113,9 @@ module TypeConverter
 
       def self.included(klass)
         klass.instance_eval {
-          config_param :types, :string, :default => nil
-          config_param :types_delimiter, :string, :default => ','
-          config_param :types_label_delimiter, :string, :default => ':'
+          config_param :types, :string, default: nil
+          config_param :types_delimiter, :string, default: ','
+          config_param :types_label_delimiter, :string, default: ':'
         }
       end
 
@@ -162,8 +162,8 @@ def parse_types_parameter
     class RegexpParser < Parser
       include TypeConverter
 
-      config_param :time_key, :string, :default => 'time'
-      config_param :time_format, :string, :default => nil
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
 
       def initialize(regexp, conf={})
         super()
@@ -235,9 +235,9 @@ def parse(text)
     end
 
     class JSONParser < Parser
-      config_param :time_key, :string, :default => 'time'
-      config_param :time_format, :string, :default => nil
-      config_param :json_parser, :string, :default => 'oj'
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
+      config_param :json_parser, :string, default: 'oj'
 
       def configure(conf)
         super
@@ -250,7 +250,7 @@ def configure(conf)
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {:bigdecimal_load => :float}
+          Oj.default_options = {bigdecimal_load: :float}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError
@@ -298,17 +298,17 @@ def parse(text)
     class ValuesParser < Parser
       include TypeConverter
 
-      config_param :keys, :default => [] do |val|
+      config_param :keys, default: [] do |val|
         if val.start_with?('[') # This check is enough because keys parameter is simple. No '[' started column name.
           JSON.load(val)
         else
           val.split(",")
         end
       end
-      config_param :time_key, :string, :default => nil
-      config_param :time_format, :string, :default => nil
-      config_param :null_value_pattern, :string, :default => nil
-      config_param :null_empty_string, :bool, :default => false
+      config_param :time_key, :string, default: nil
+      config_param :time_format, :string, default: nil
+      config_param :null_value_pattern, :string, default: nil
+      config_param :null_empty_string, :bool, default: false
 
       def configure(conf)
         super
@@ -377,7 +377,7 @@ def convert_value_to_nil(value)
     end
 
     class TSVParser < ValuesParser
-      config_param :delimiter, :string, :default => "\t"
+      config_param :delimiter, :string, default: "\t"
 
       def configure(conf)
         super
@@ -394,9 +394,9 @@ def parse(text)
     end
 
     class LabeledTSVParser < ValuesParser
-      config_param :delimiter,       :string, :default => "\t"
-      config_param :label_delimiter, :string, :default =>  ":"
-      config_param :time_key, :string, :default =>  "time"
+      config_param :delimiter,       :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
+      config_param :time_key, :string, default: "time"
 
       def configure(conf)
         conf['keys'] = conf['time_key'] || 'time'
@@ -437,7 +437,7 @@ def parse(text)
     end
 
     class NoneParser < Parser
-      config_param :message_key, :string, :default => 'message'
+      config_param :message_key, :string, default: 'message'
 
       def parse(text)
         record = {}
@@ -526,8 +526,8 @@ class SyslogParser < Parser
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
 
-      config_param :time_format, :string, :default => "%b %d %H:%M:%S"
-      config_param :with_priority, :bool, :default => false
+      config_param :time_format, :string, default: "%b %d %H:%M:%S"
+      config_param :with_priority, :bool, default: false
 
       def initialize
         super
@@ -586,7 +586,7 @@ def parse(text)
     end
 
     class MultilineParser < Parser
-      config_param :format_firstline, :string, :default => nil
+      config_param :format_firstline, :string, default: nil
 
       FORMAT_MAX_NUM = 20
 
diff --git a/lib/fluent/plugin/buf_file.rb b/lib/fluent/plugin/buf_file.rb
index 15f161b2f0..c3285da110 100644
--- a/lib/fluent/plugin/buf_file.rb
+++ b/lib/fluent/plugin/buf_file.rb
@@ -85,7 +85,7 @@ def initialize
     desc 'The path where buffer chunks are stored.'
     config_param :buffer_path, :string
     desc 'If true, queued chunks are flushed at shutdown process.'
-    config_param :flush_at_shutdown, :bool, :default => false
+    config_param :flush_at_shutdown, :bool, default: false
 
     # 'symlink_path' is currently only for out_file.
     # That is the reason why this is not config_param, but attr_accessor.
@@ -112,7 +112,7 @@ def configure(conf)
     end
 
     def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), :mode => DEFAULT_DIR_PERMISSION
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), mode: DEFAULT_DIR_PERMISSION
       super
     end
 
diff --git a/lib/fluent/plugin/buf_memory.rb b/lib/fluent/plugin/buf_memory.rb
index 81d69256d8..7ed3a16d35 100644
--- a/lib/fluent/plugin/buf_memory.rb
+++ b/lib/fluent/plugin/buf_memory.rb
@@ -71,7 +71,7 @@ def initialize
     end
 
     desc 'If true, queued chunks are flushed at shutdown process. Otherwise queued chunks are discarded'
-    config_param :flush_at_shutdown, :bool, :default => true
+    config_param :flush_at_shutdown, :bool, default: true
     # Overwrite default BasicBuffer#buffer_queue_limit
     # to limit total memory usage upto 512MB.
     config_set_default :buffer_queue_limit, 64
diff --git a/lib/fluent/plugin/filter_grep.rb b/lib/fluent/plugin/filter_grep.rb
index 704d5d7d6f..95a21bcd3d 100644
--- a/lib/fluent/plugin/filter_grep.rb
+++ b/lib/fluent/plugin/filter_grep.rb
@@ -25,8 +25,8 @@ def initialize
 
     REGEXP_MAX_NUM = 20
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, :default => nil }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
 
     # for test
     attr_reader :regexps
@@ -67,7 +67,7 @@ def filter(tag, time, record)
           result = record
         end
       rescue => e
-        log.warn "failed to grep events", :error_class => e.class, :error => e.message
+        log.warn "failed to grep events", error_class: e.class, error: e.message
         log.warn_backtrace
       end
       result
diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 319917da35..3838c23700 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -26,17 +26,17 @@ def initialize
     end
 
     desc 'A comma-delimited list of keys to delete.'
-    config_param :remove_keys, :string, :default => nil
+    config_param :remove_keys, :string, default: nil
     desc 'A comma-delimited list of keys to keep.'
-    config_param :keep_keys, :string, :default => nil
+    config_param :keep_keys, :string, default: nil
     desc 'Create new Hash to transform incoming data'
-    config_param :renew_record, :bool, :default => false
+    config_param :renew_record, :bool, default: false
     desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
-    config_param :renew_time_key, :string, :default => nil
+    config_param :renew_time_key, :string, default: nil
     desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
-    config_param :enable_ruby, :bool, :default => false
+    config_param :enable_ruby, :bool, default: false
     desc 'Use original value type.'
-    config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
+    config_param :auto_typecast, :bool, default: false # false for lower version compatibility
 
     def configure(conf)
       super
@@ -60,8 +60,8 @@ def configure(conf)
       end
 
       placeholder_expander_params = {
-        :log           => log,
-        :auto_typecast => @auto_typecast,
+        log: log,
+        auto_typecast: @auto_typecast,
       }
       @placeholder_expander =
         if @enable_ruby
@@ -105,7 +105,7 @@ def filter_stream(tag, es)
       end
       new_es
     rescue => e
-      log.warn "failed to reform records", :error_class => e.class, :error => e.message
+      log.warn "failed to reform records", error_class: e.class, error: e.message
       log.warn_backtrace
       log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
@@ -119,7 +119,7 @@ def parse_value(value_str)
         value_str
       end
     rescue => e
-      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", :error_class => e.class, :error => e.message
+      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error_class: e.class, error: e.message
       value_str # emit as string
     end
 
@@ -307,7 +307,7 @@ def expand(str, force_stringify = false)
           @hostname,
         )
       rescue => e
-        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
+        log.warn "failed to expand `#{str}`", error_class: e.class, error: e.message
         log.warn_backtrace
         nil
       end
diff --git a/lib/fluent/plugin/filter_stdout.rb b/lib/fluent/plugin/filter_stdout.rb
index 9e49d3d10c..5ab0290a99 100644
--- a/lib/fluent/plugin/filter_stdout.rb
+++ b/lib/fluent/plugin/filter_stdout.rb
@@ -22,7 +22,7 @@ class StdoutFilter < Filter
     attr_reader :formatter
 
     desc 'The format of the output.'
-    config_param :format, :string, :default => 'stdout'
+    config_param :format, :string, default: 'stdout'
     # config_param :output_type, :string, :default => 'json' (StdoutFormatter defines this)
 
     def configure(conf)
diff --git a/lib/fluent/plugin/in_debug_agent.rb b/lib/fluent/plugin/in_debug_agent.rb
index 17c87c0e56..60a6ffea97 100644
--- a/lib/fluent/plugin/in_debug_agent.rb
+++ b/lib/fluent/plugin/in_debug_agent.rb
@@ -24,11 +24,11 @@ def initialize
       super
     end
 
-    config_param :bind, :string, :default => '0.0.0.0'
-    config_param :port, :integer, :default => 24230
-    config_param :unix_path, :string, :default => nil
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: 24230
+    config_param :unix_path, :string, default: nil
     #config_param :unix_mode  # TODO
-    config_param :object, :string, :default => 'Engine'
+    config_param :object, :string, default: 'Engine'
 
     def configure(conf)
       super
@@ -46,7 +46,7 @@ def start
       else
         uri = "druby://#{@bind}:#{@port}"
       end
-      log.info "listening dRuby", :uri => uri, :object => @object
+      log.info "listening dRuby", uri: uri, object: @object
       obj = eval(@object)
       @server = DRb::DRbServer.new(uri, obj)
     end
diff --git a/lib/fluent/plugin/in_dummy.rb b/lib/fluent/plugin/in_dummy.rb
index 00492ce006..b436eb2210 100644
--- a/lib/fluent/plugin/in_dummy.rb
+++ b/lib/fluent/plugin/in_dummy.rb
@@ -23,11 +23,11 @@ class DummyInput < Input
     desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
     desc "It configures how many events to generate per second."
-    config_param :rate, :integer, :default => 1
+    config_param :rate, :integer, default: 1
     desc "If specified, each generated event has an auto-incremented key field."
-    config_param :auto_increment_key, :string, :default => nil
+    config_param :auto_increment_key, :string, default: nil
     desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
-    config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
+    config_param :dummy, default: [{"message"=>"dummy"}] do |val|
       begin
         parsed = JSON.parse(val)
       rescue JSON::ParserError => e
diff --git a/lib/fluent/plugin/in_exec.rb b/lib/fluent/plugin/in_exec.rb
index bc7bd82178..b5ea3e3e97 100644
--- a/lib/fluent/plugin/in_exec.rb
+++ b/lib/fluent/plugin/in_exec.rb
@@ -27,21 +27,21 @@ def initialize
     desc 'The command (program) to execute.'
     config_param :command, :string
     desc 'The format used to map the program output to the incoming event.(tsv,json,msgpack)'
-    config_param :format, :string, :default => 'tsv'
+    config_param :format, :string, default: 'tsv'
     desc 'Specify the comma-separated keys when using the tsv format.'
-    config_param :keys, :default => [] do |val|
+    config_param :keys, default: [] do |val|
       val.split(',')
     end
     desc 'Tag of the output events.'
-    config_param :tag, :string, :default => nil
+    config_param :tag, :string, default: nil
     desc 'The key to use as the event tag instead of the value in the event record. '
-    config_param :tag_key, :string, :default => nil
+    config_param :tag_key, :string, default: nil
     desc 'The key to use as the event time instead of the value in the event record.'
-    config_param :time_key, :string, :default => nil
+    config_param :time_key, :string, default: nil
     desc 'The format of the event time used for the time_key parameter.'
-    config_param :time_format, :string, :default => nil
+    config_param :time_format, :string, default: nil
     desc 'The interval time between periodic program runs.'
-    config_param :run_interval, :time, :default => nil
+    config_param :run_interval, :time, default: nil
 
     def configure(conf)
       super
@@ -136,7 +136,7 @@ def run_periodic
           Process.waitpid(io.pid)
           sleep @run_interval
         rescue
-          log.error "exec failed to run or shutdown child process", :error => $!.to_s, :error_class => $!.class.to_s
+          log.error "exec failed to run or shutdown child process", error: $!.to_s, error_class: $!.class.to_s
           log.warn_backtrace $!.backtrace
         end
       end
@@ -163,7 +163,7 @@ def on_message(record, parsed_time = nil)
 
       router.emit(tag, time, record)
     rescue => e
-      log.error "exec failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+      log.error "exec failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
     end
   end
 end
diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index ce91d0fe02..c9d6d51a20 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -24,22 +24,22 @@ def initialize
     end
 
     desc 'The port to listen to.'
-    config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
     desc 'The bind address to listen to.'
-    config_param :bind, :string, :default => '0.0.0.0'
-    config_param :backlog, :integer, :default => nil
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :backlog, :integer, default: nil
     # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
     desc 'The timeout time used to set linger option.'
-    config_param :linger_timeout, :integer, :default => 0
+    config_param :linger_timeout, :integer, default: 0
     # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
-    config_param :blocking_timeout, :time, :default => 0.5
+    config_param :blocking_timeout, :time, default: 0.5
 
     desc 'Log warning if received chunk size is larger than this value.'
-    config_param :chunk_size_warn_limit, :size, :default => nil
+    config_param :chunk_size_warn_limit, :size, default: nil
     desc 'Received chunk is dropped if it is larger than this value.'
-    config_param :chunk_size_limit, :size, :default => nil
+    config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
-    config_param :skip_invalid_event, :bool, :default => false
+    config_param :skip_invalid_event, :bool, default: false
 
     def configure(conf)
       super
@@ -97,7 +97,7 @@ def listen
     def run
       @loop.run(@blocking_timeout)
     rescue => e
-      log.error "unexpected error", :error => e, :error_class => e.class
+      log.error "unexpected error", error: e, error_class: e.class
       log.error_backtrace
     end
 
@@ -265,7 +265,7 @@ def on_read_json(data)
         @chunk_counter += data.bytesize
         @y << data
       rescue => e
-        @log.error "forward error", :error => e, :error_class => e.class
+        @log.error "forward error", error: e, error_class: e.class
         @log.error_backtrace
         close
       end
@@ -278,7 +278,7 @@ def on_read_msgpack(data)
           @chunk_counter = 0
         end
       rescue => e
-        @log.error "forward error", :error => e, :error_class => e.class
+        @log.error "forward error", error: e, error_class: e.class
         @log.error_backtrace
         close
       end
diff --git a/lib/fluent/plugin/in_gc_stat.rb b/lib/fluent/plugin/in_gc_stat.rb
index e68297b138..e0dbfdea41 100644
--- a/lib/fluent/plugin/in_gc_stat.rb
+++ b/lib/fluent/plugin/in_gc_stat.rb
@@ -22,7 +22,7 @@ def initialize
       super
     end
 
-    config_param :emit_interval, :time, :default => 60
+    config_param :emit_interval, :time, default: 60
     config_param :tag, :string
 
     class TimerWatcher < Coolio::TimerWatcher
@@ -61,7 +61,7 @@ def shutdown
     def run
       @loop.run
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index a44ce0b3fd..255a5c5fe3 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -31,25 +31,25 @@ def initialize
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
     desc 'The port to listen to.'
-    config_param :port, :integer, :default => 9880
+    config_param :port, :integer, default: 9880
     desc 'The bind address to listen to.'
-    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :bind, :string, default: '0.0.0.0'
     desc 'The size limit of the POSTed element. Default is 32MB.'
-    config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
+    config_param :body_size_limit, :size, default: 32*1024*1024  # TODO default
     desc 'The timeout limit for keeping the connection alive.'
-    config_param :keepalive_timeout, :time, :default => 10   # TODO default
-    config_param :backlog, :integer, :default => nil
+    config_param :keepalive_timeout, :time, default: 10   # TODO default
+    config_param :backlog, :integer, default: nil
     desc 'Add HTTP_ prefix headers to the record.'
-    config_param :add_http_headers, :bool, :default => false
+    config_param :add_http_headers, :bool, default: false
     desc 'Add REMOTE_ADDR header to the record.'
-    config_param :add_remote_addr, :bool, :default => false
+    config_param :add_remote_addr, :bool, default: false
     desc 'The format of the HTTP body.'
-    config_param :format, :string, :default => 'default'
-    config_param :blocking_timeout, :time, :default => 0.5
+    config_param :format, :string, default: 'default'
+    config_param :blocking_timeout, :time, default: 0.5
     desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
-    config_param :cors_allow_origins, :array, :default => nil
+    config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
-    config_param :respond_with_empty_img, :bool, :default => false
+    config_param :respond_with_empty_img, :bool, default: false
 
     def configure(conf)
       super
@@ -121,7 +121,7 @@ def shutdown
     def run
       @loop.run(@blocking_timeout)
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -254,7 +254,7 @@ def on_read(data)
         @idle = 0
         @parser << data
       rescue
-        @log.warn "unexpected error", :error=>$!.to_s
+        @log.warn "unexpected error", error: $!.to_s
         @log.warn_backtrace
         close
       end
diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index 18b3cd4d82..9ec0a5a1d7 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -25,10 +25,10 @@ def initialize
       super
     end
 
-    config_param :bind, :string, :default => '0.0.0.0'
-    config_param :port, :integer, :default => 24220
-    config_param :tag, :string, :default => nil
-    config_param :emit_interval, :time, :default => 60
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: 24220
+    config_param :tag, :string, default: nil
+    config_param :emit_interval, :time, default: 60
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -234,10 +234,10 @@ def on_timer
     def start
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
       @srv = WEBrick::HTTPServer.new({
-          :BindAddress => @bind,
-          :Port => @port,
-          :Logger => WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
-          :AccessLog => [],
+          BindAddress: @bind,
+          Port: @port,
+          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+          AccessLog: [],
         })
       @srv.mount('/api/plugins', LTSVMonitorServlet, self)
       @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
@@ -250,7 +250,7 @@ def start
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
         @loop = Coolio::Loop.new
-        opts = {:with_config => false}
+        opts = {with_config: false}
         timer = TimerWatcher.new(@emit_interval, log) {
           es = MultiEventStream.new
           now = Engine.now
@@ -267,7 +267,7 @@ def start
     def run
       @loop.run
     rescue => e
-      log.error "unexpected error", :error => e.to_s
+      log.error "unexpected error", error: e.to_s
       log.error_backtrace
     end
 
diff --git a/lib/fluent/plugin/in_object_space.rb b/lib/fluent/plugin/in_object_space.rb
index 8f2f3dbdea..d29e90905d 100644
--- a/lib/fluent/plugin/in_object_space.rb
+++ b/lib/fluent/plugin/in_object_space.rb
@@ -22,9 +22,9 @@ def initialize
       super
     end
 
-    config_param :emit_interval, :time, :default => 60
+    config_param :emit_interval, :time, default: 60
     config_param :tag, :string
-    config_param :top, :integer, :default => 15
+    config_param :top, :integer, default: 15
 
     class TimerWatcher < Coolio::TimerWatcher
       def initialize(interval, repeat, log, &callback)
@@ -62,7 +62,7 @@ def shutdown
     def run
       @loop.run
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -110,7 +110,7 @@ def on_timer
 
       router.emit(@tag, now, record)
     rescue => e
-      log.error "object space failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => @tag, :record => Yajl.dump(record)
+      log.error "object space failed to emit", error: e.to_s, error_class: e.class.to_s, tag: @tag, record: Yajl.dump(record)
     end
   end
 end
diff --git a/lib/fluent/plugin/in_stream.rb b/lib/fluent/plugin/in_stream.rb
index a14b76efe1..d2f9c030aa 100644
--- a/lib/fluent/plugin/in_stream.rb
+++ b/lib/fluent/plugin/in_stream.rb
@@ -17,7 +17,7 @@
 module Fluent
   # obsolete
   class StreamInput < Input
-    config_param :blocking_timeout, :time, :default => 0.5
+    config_param :blocking_timeout, :time, default: 0.5
 
     def initialize
       require 'socket'
@@ -45,7 +45,7 @@ def shutdown
     def run
       @loop.run(@blocking_timeout)
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -142,7 +142,7 @@ def on_read(data)
       def on_read_json(data)
         @y << data
       rescue
-        @log.error "unexpected error", :error=>$!.to_s
+        @log.error "unexpected error", error: $!.to_s
         @log.error_backtrace
         close
       end
@@ -150,7 +150,7 @@ def on_read_json(data)
       def on_read_msgpack(data)
         @u.feed_each(data, &@on_message)
       rescue
-        @log.error "unexpected error", :error=>$!.to_s
+        @log.error "unexpected error", error: $!.to_s
         @log.error_backtrace
         close
       end
@@ -165,9 +165,9 @@ class UnixInput < StreamInput
     Plugin.register_input('unix', self)
 
     desc 'The path to your Unix Domain Socket.'
-    config_param :path, :string, :default => DEFAULT_SOCKET_PATH
+    config_param :path, :string, default: DEFAULT_SOCKET_PATH
     desc 'The backlog of Unix Domain Socket.'
-    config_param :backlog, :integer, :default => nil
+    config_param :backlog, :integer, default: nil
 
     def configure(conf)
       super
diff --git a/lib/fluent/plugin/in_syslog.rb b/lib/fluent/plugin/in_syslog.rb
index 0ec6148562..fc7d5e2b9d 100644
--- a/lib/fluent/plugin/in_syslog.rb
+++ b/lib/fluent/plugin/in_syslog.rb
@@ -65,13 +65,13 @@ def initialize
     end
 
     desc 'The port to listen to.'
-    config_param :port, :integer, :default => 5140
+    config_param :port, :integer, default: 5140
     desc 'The bind address to listen to.'
-    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :bind, :string, default: '0.0.0.0'
     desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
-    config_param :protocol_type, :default => :udp do |val|
+    config_param :protocol_type, default: :udp do |val|
       case val.downcase
       when 'tcp'
         :tcp
@@ -82,10 +82,10 @@ def initialize
       end
     end
     desc 'If true, add source host to event record.'
-    config_param :include_source_host, :bool, :default => false
+    config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
-    config_param :source_host_key, :string, :default => 'source_host'.freeze
-    config_param :blocking_timeout, :time, :default => 0.5
+    config_param :source_host_key, :string, default: 'source_host'.freeze
+    config_param :blocking_timeout, :time, default: 0.5
 
     def configure(conf)
       super
@@ -125,7 +125,7 @@ def shutdown
     def run
       @loop.run(@blocking_timeout)
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -150,14 +150,14 @@ def receive_data_parser(data, addr)
         emit(pri, time, record)
       }
     rescue => e
-      log.error data.dump, :error => e.to_s
+      log.error data.dump, error: e.to_s
       log.error_backtrace
     end
 
     def receive_data(data, addr)
       @parser.parse(data) { |time, record|
         unless time && record
-          log.warn "invalid syslog message", :data => data
+          log.warn "invalid syslog message", data: data
           return
         end
 
@@ -166,7 +166,7 @@ def receive_data(data, addr)
         emit(pri, time, record)
       }
     rescue => e
-      log.error data.dump, :error => e.to_s
+      log.error data.dump, error: e.to_s
       log.error_backtrace
     end
 
@@ -192,7 +192,7 @@ def emit(pri, time, record)
 
       router.emit(tag, time, record)
     rescue => e
-      log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+      log.error "syslog failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
     end
   end
 end
diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index c25239810d..fc8c51ac47 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -29,19 +29,19 @@ def initialize
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
-    config_param :exclude_path, :array, :default => []
+    config_param :exclude_path, :array, default: []
     desc 'Specify interval to keep reference to old file when rotate a file.'
-    config_param :rotate_wait, :time, :default => 5
+    config_param :rotate_wait, :time, default: 5
     desc 'Fluentd will record the position it last read into this file.'
-    config_param :pos_file, :string, :default => nil
+    config_param :pos_file, :string, default: nil
     desc 'Start to read the logs from the head of file, not bottom.'
-    config_param :read_from_head, :bool, :default => false
+    config_param :read_from_head, :bool, default: false
     desc 'The interval of refreshing the list of watch file.'
-    config_param :refresh_interval, :time, :default => 60
+    config_param :refresh_interval, :time, default: 60
     desc 'The number of reading lines at each IO.'
-    config_param :read_lines_limit, :integer, :default => 1000
+    config_param :read_lines_limit, :integer, default: 1000
     desc 'The interval of flushing the buffer for multiline format'
-    config_param :multiline_flush_interval, :time, :default => nil
+    config_param :multiline_flush_interval, :time, default: nil
 
     attr_reader :paths
 
@@ -232,7 +232,7 @@ def flush_buffer(tw)
     def run
       @loop.run
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -263,7 +263,7 @@ def convert_line_to_event(line, es)
           end
         }
       rescue => e
-        log.warn line.dump, :error => e.to_s
+        log.warn line.dump, error: e.to_s
         log.debug_backtrace(e.backtrace)
       end
     end
diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb
index 5993d95052..df02c5ecbf 100644
--- a/lib/fluent/plugin/in_tcp.rb
+++ b/lib/fluent/plugin/in_tcp.rb
@@ -22,7 +22,7 @@ class TcpInput < SocketUtil::BaseInput
 
     config_set_default :port, 5170
     desc 'The payload is read up to this character.'
-    config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
     def listen(callback)
       log.info "listening tcp socket on #{@bind}:#{@port}"
diff --git a/lib/fluent/plugin/in_udp.rb b/lib/fluent/plugin/in_udp.rb
index d07ef6cc86..5fe9dd90e8 100644
--- a/lib/fluent/plugin/in_udp.rb
+++ b/lib/fluent/plugin/in_udp.rb
@@ -21,7 +21,7 @@ class UdpInput < SocketUtil::BaseInput
     Plugin.register_input('udp', self)
 
     config_set_default :port, 5160
-    config_param :body_size_limit, :size, :default => 4096
+    config_param :body_size_limit, :size, default: 4096
 
     def listen(callback)
       log.info "listening udp socket on #{@bind}:#{@port}"
diff --git a/lib/fluent/plugin/out_copy.rb b/lib/fluent/plugin/out_copy.rb
index 6a0ec95086..de43630589 100644
--- a/lib/fluent/plugin/out_copy.rb
+++ b/lib/fluent/plugin/out_copy.rb
@@ -19,7 +19,7 @@ class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
 
     desc 'If true, pass different record to each `store` plugin.'
-    config_param :deep_copy, :bool, :default => false
+    config_param :deep_copy, :bool, default: false
 
     def initialize
       super
diff --git a/lib/fluent/plugin/out_exec.rb b/lib/fluent/plugin/out_exec.rb
index cdf4ee496a..2062afda51 100644
--- a/lib/fluent/plugin/out_exec.rb
+++ b/lib/fluent/plugin/out_exec.rb
@@ -29,17 +29,17 @@ def initialize
     desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argumen'
     config_param :command, :string
     desc 'Specify the comma-separated keys when using the tsv format.'
-    config_param :keys, :default => [] do |val|
+    config_param :keys, default: [] do |val|
       val.split(',')
     end
     desc 'The name of the key to use as the event tag. This replaces the value in the event record.'
-    config_param :tag_key, :string, :default => nil
+    config_param :tag_key, :string, default: nil
     desc 'The name of the key to use as the event time. This replaces the the value in the event record.'
-    config_param :time_key, :string, :default => nil
+    config_param :time_key, :string, default: nil
     desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
-    config_param :time_format, :string, :default => nil
+    config_param :time_format, :string, default: nil
     desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :format, :default => :tsv do |val|
+    config_param :format, default: :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f
       f
diff --git a/lib/fluent/plugin/out_exec_filter.rb b/lib/fluent/plugin/out_exec_filter.rb
index 261a7571c1..46021c37bc 100644
--- a/lib/fluent/plugin/out_exec_filter.rb
+++ b/lib/fluent/plugin/out_exec_filter.rb
@@ -28,61 +28,61 @@ def initialize
     desc 'The command (program) to execute.'
     config_param :command, :string
 
-    config_param :remove_prefix, :string, :default => nil
-    config_param :add_prefix, :string, :default => nil
+    config_param :remove_prefix, :string, default: nil
+    config_param :add_prefix, :string, default: nil
 
     desc "The format used to map the incoming event to the program input.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :in_format, :default => :tsv do |val|
+    config_param :in_format, default: :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported in_format '#{val}'" unless f
       f
     end
     desc 'Specify comma-separated values for tsv format.'
-    config_param :in_keys, :default => [] do |val|
+    config_param :in_keys, default: [] do |val|
       val.split(',')
     end
     desc 'The name of the key to use as the event tag.'
-    config_param :in_tag_key, :default => nil
+    config_param :in_tag_key, default: nil
     desc 'The name of the key to use as the event time.'
-    config_param :in_time_key, :default => nil
+    config_param :in_time_key, default: nil
     desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
-    config_param :in_time_format, :default => nil
+    config_param :in_time_format, default: nil
 
     desc "The format used to process the program output.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :out_format, :default => :tsv do |val|
+    config_param :out_format, default: :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported out_format '#{val}'" unless f
       f
     end
     desc 'Specify comma-separated values for tsv format.'
-    config_param :out_keys, :default => [] do |val|  # for tsv format
+    config_param :out_keys, default: [] do |val|  # for tsv format
       val.split(',')
     end
     desc 'The name of the key to use as the event tag.'
-    config_param :out_tag_key, :default => nil
+    config_param :out_tag_key, default: nil
     desc 'The name of the key to use as the event time.'
-    config_param :out_time_key, :default => nil
+    config_param :out_time_key, default: nil
     desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
-    config_param :out_time_format, :default => nil
+    config_param :out_time_format, default: nil
 
-    config_param :tag, :string, :default => nil
+    config_param :tag, :string, default: nil
 
-    config_param :time_key, :string, :default => nil
-    config_param :time_format, :string, :default => nil
+    config_param :time_key, :string, default: nil
+    config_param :time_format, :string, default: nil
 
     desc 'If true, use localtime with in_time_format.'
-    config_param :localtime, :bool, :default => true
+    config_param :localtime, :bool, default: true
     desc 'If true, use timezone with in_time_format.'
-    config_param :timezone, :string, :default => nil
+    config_param :timezone, :string, default: nil
     desc 'The number of spawned process for command.'
-    config_param :num_children, :integer, :default => 1
+    config_param :num_children, :integer, default: 1
 
     desc 'Respawn command when command exit.'
     # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
-    config_param :child_respawn, :string, :default => nil
+    config_param :child_respawn, :string, default: nil
 
     # 0: output logs for all of messages to emit
-    config_param :suppress_error_log_interval, :time, :default => 0
+    config_param :suppress_error_log_interval, :time, default: 0
 
     config_set_default :flush_interval, 1
 
@@ -306,7 +306,7 @@ def write(chunk)
           chunk.write_to(@io)
         rescue Errno::EPIPE => e
           # Broken pipe (child process unexpectedly exited)
-          @log.warn "exec_filter Broken pipe, child process maybe exited.", :command => @command
+          @log.warn "exec_filter Broken pipe, child process maybe exited.", command: @command
           if try_respawn
             retry # retry chunk#write_to with child respawned
           else
@@ -329,19 +329,19 @@ def try_respawn
 
           @respawns -= 1 if @respawns > 0
         end
-        @log.warn "exec_filter child process successfully respawned.", :command => @command, :respawns => @respawns
+        @log.warn "exec_filter child process successfully respawned.", command: @command, respawns: @respawns
         true
       end
 
       def run
         @parser.call(@io)
       rescue
-        @log.error "exec_filter thread unexpectedly failed with an error.", :command=>@command, :error=>$!.to_s
+        @log.error "exec_filter thread unexpectedly failed with an error.", command: @command, error: $!.to_s
         @log.warn_backtrace $!.backtrace
       ensure
         pid, stat = Process.waitpid2(@pid)
         unless @finished
-          @log.error "exec_filter process unexpectedly exited.", :command=>@command, :ecode=>stat.to_i
+          @log.error "exec_filter process unexpectedly exited.", command: @command, ecode: stat.to_i
           unless @respawns == 0
             @log.warn "exec_filter child process will respawn for next input data (respawns #{@respawns})."
           end
@@ -369,7 +369,7 @@ def on_message(record)
       router.emit(tag, time, record)
     rescue
       if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
-        log.error "exec_filter failed to emit", :error=>$!.to_s, :error_class=>$!.class.to_s, :record=>Yajl.dump(record)
+        log.error "exec_filter failed to emit", error: $!.to_s, error_class: $!.class.to_s, record: Yajl.dump(record)
         log.warn_backtrace $!.backtrace
         @next_log_time = Time.now.to_i + @suppress_error_log_interval
       end
diff --git a/lib/fluent/plugin/out_file.rb b/lib/fluent/plugin/out_file.rb
index d5ba98ee14..08e355cd5c 100644
--- a/lib/fluent/plugin/out_file.rb
+++ b/lib/fluent/plugin/out_file.rb
@@ -26,11 +26,11 @@ class FileOutput < TimeSlicedOutput
     desc "The Path of the file."
     config_param :path, :string
     desc "The format of the file content. The default is out_file."
-    config_param :format, :string, :default => 'out_file'
+    config_param :format, :string, default: 'out_file'
     desc "The flushed chunk is appended to existence file or not."
-    config_param :append, :bool, :default => false
+    config_param :append, :bool, default: false
     desc "Compress flushed file."
-    config_param :compress, :default => nil do |val|
+    config_param :compress, default: nil do |val|
       c = SUPPORTED_COMPRESS[val]
       unless c
         raise ConfigError, "Unsupported compression algorithm '#{val}'"
@@ -38,7 +38,7 @@ class FileOutput < TimeSlicedOutput
       c
     end
     desc "Create symlink to temporary buffered file when buffer_type is file."
-    config_param :symlink_path, :string, :default => nil
+    config_param :symlink_path, :string, default: nil
 
     def initialize
       require 'zlib'
@@ -84,7 +84,7 @@ def format(tag, time, record)
 
     def write(chunk)
       path = generate_path(chunk.key)
-      FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
+      FileUtils.mkdir_p File.dirname(path), mode: DEFAULT_DIR_PERMISSION
 
       case @compress
       when nil
diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 9e85b84cce..3063b8caa6 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -37,9 +37,9 @@ def initialize
     end
 
     desc 'The timeout time when sending event logs.'
-    config_param :send_timeout, :time, :default => 60
+    config_param :send_timeout, :time, default: 60
     desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
-    config_param :heartbeat_type, :default => :udp do |val|
+    config_param :heartbeat_type, default: :udp do |val|
       case val.downcase
       when 'tcp'
         :tcp
@@ -52,34 +52,34 @@ def initialize
       end
     end
     desc 'The interval of the heartbeat packer.'
-    config_param :heartbeat_interval, :time, :default => 1
+    config_param :heartbeat_interval, :time, default: 1
     desc 'The wait time before accepting a server fault recovery.'
-    config_param :recover_wait, :time, :default => 10
+    config_param :recover_wait, :time, default: 10
     desc 'The hard timeout used to detect server failure.'
-    config_param :hard_timeout, :time, :default => 60
+    config_param :hard_timeout, :time, default: 60
     desc 'Set TTL to expire DNS cache in seconds.'
-    config_param :expire_dns_cache, :time, :default => nil  # 0 means disable cache
+    config_param :expire_dns_cache, :time, default: nil  # 0 means disable cache
     desc 'The threshold parameter used to detect server faults.'
-    config_param :phi_threshold, :integer, :default => 16
+    config_param :phi_threshold, :integer, default: 16
     desc 'Use the "Phi accrual failure detector" to detect server failure.'
-    config_param :phi_failure_detector, :bool, :default => true
+    config_param :phi_failure_detector, :bool, default: true
 
     # if any options added that requires extended forward api, fix @extend_internal_protocol
 
     desc 'Change the protocol to at-least-once.'
-    config_param :require_ack_response, :bool, :default => false  # require in_forward to respond with ack
+    config_param :require_ack_response, :bool, default: false  # require in_forward to respond with ack
     desc 'This option is used when require_ack_response is true.'
-    config_param :ack_response_timeout, :time, :default => 190  # 0 means do not wait for ack responses
+    config_param :ack_response_timeout, :time, default: 190  # 0 means do not wait for ack responses
     # Linux default tcp_syn_retries is 5 (in many environment)
     # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
     desc 'Enable client-side DNS round robin.'
-    config_param :dns_round_robin, :bool, :default => false # heartbeat_type 'udp' is not available for this
+    config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
     attr_reader :nodes
 
     # backward compatibility
-    config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
-    config_param :host, :string, :default => nil
+    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
+    config_param :host, :string, default: nil
 
     attr_accessor :extend_internal_protocol
 
@@ -138,7 +138,7 @@ def configure(conf)
         else
           @nodes << Node.new(log, node_conf)
         end
-        log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight, :plugin_id=>plugin_id
+        log.info "adding forwarding server '#{name}'", host: host, port: port, weight: weight, plugin_id: plugin_id
       }
 
       if @nodes.empty?
@@ -184,7 +184,7 @@ def shutdown
     def run
       @loop.run if @loop
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -229,13 +229,13 @@ def rebuild_weight_array
           lost_weight += n.weight
         end
       }
-      log.debug "rebuilding weight array", :lost_weight=>lost_weight
+      log.debug "rebuilding weight array", lost_weight: lost_weight
 
       if lost_weight > 0
         standby_nodes.each {|n|
           if n.available?
             regular_nodes << n
-            log.warn "using standby node #{n.host}:#{n.port}", :weight=>n.weight
+            log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight
             lost_weight -= n.weight
             break if lost_weight <= 0
           end
@@ -399,7 +399,7 @@ def on_timer
           end
         rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
           # TODO log
-          log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", :error=>$!.to_s
+          log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", error: $!.to_s
         end
       }
     end
@@ -511,7 +511,7 @@ def tick
         end
 
         if @failure.hard_timeout?(now)
-          @log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :hard_timeout=>true
+          @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, hard_timeout: true
           @available = false
           @resolved_host = nil  # expire cached host
           @failure.clear
@@ -522,7 +522,7 @@ def tick
           phi = @failure.phi(now)
           #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
           if phi > @conf.phi_threshold
-            @log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
+            @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, phi: phi
             @available = false
             @resolved_host = nil  # expire cached host
             @failure.clear
@@ -538,7 +538,7 @@ def heartbeat(detect=true)
         #@log.trace "heartbeat from '#{@name}'", :host=>@host, :port=>@port, :available=>@available, :sample_size=>@failure.sample_size
         if detect && !@available && @failure.sample_size > @conf.recover_sample_size
           @available = true
-          @log.warn "recovered forwarding server '#{@name}'", :host=>@host, :port=>@port
+          @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port
           return true
         else
           return nil
diff --git a/lib/fluent/plugin/out_stdout.rb b/lib/fluent/plugin/out_stdout.rb
index eb289217c2..fbaa700622 100644
--- a/lib/fluent/plugin/out_stdout.rb
+++ b/lib/fluent/plugin/out_stdout.rb
@@ -19,7 +19,7 @@ class StdoutOutput < Output
     Plugin.register_output('stdout', self)
 
     desc 'Output format.(json,hash)'
-    config_param :output_type, :default => 'json'
+    config_param :output_type, default: 'json'
 
     def configure(conf)
       super
diff --git a/lib/fluent/plugin/out_stream.rb b/lib/fluent/plugin/out_stream.rb
index 41db049c6d..2ee222010b 100644
--- a/lib/fluent/plugin/out_stream.rb
+++ b/lib/fluent/plugin/out_stream.rb
@@ -23,7 +23,7 @@ def initialize
       super
     end
 
-    config_param :send_timeout, :time, :default => 60
+    config_param :send_timeout, :time, default: 60
 
     def configure(conf)
       super
@@ -91,7 +91,7 @@ def initialize
       $log.warn "see 'forward' section in http://docs.fluentd.org/ for the high-availability configuration."
     end
 
-    config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
     config_param :host, :string
 
     def configure(conf)
diff --git a/lib/fluent/plugin/socket_util.rb b/lib/fluent/plugin/socket_util.rb
index 0f1e1496a7..7b9829e879 100644
--- a/lib/fluent/plugin/socket_util.rb
+++ b/lib/fluent/plugin/socket_util.rb
@@ -43,7 +43,7 @@ def on_readable
         msg.chomp!
         @callback.call(msg, addr)
       rescue => e
-        @log.error "unexpected error", :error => e, :error_class => e.class
+        @log.error "unexpected error", error: e, error_class: e.class
       end
     end
 
@@ -79,7 +79,7 @@ def on_read(data)
         end
         @buffer.slice!(0, pos) if pos > 0
       rescue => e
-        @log.error "unexpected error", :error => e, :error_class => e.class
+        @log.error "unexpected error", error: e, error_class: e.class
         close
       end
 
@@ -99,12 +99,12 @@ def initialize
       desc 'The format of the payload.'
       config_param :format, :string
       desc 'The port to listen to.'
-      config_param :port, :integer, :default => 5150
+      config_param :port, :integer, default: 5150
       desc 'The bind address to listen to.'
-      config_param :bind, :string, :default => '0.0.0.0'
+      config_param :bind, :string, default: '0.0.0.0'
       desc "The field name of the client's hostname."
-      config_param :source_host_key, :string, :default => nil
-      config_param :blocking_timeout, :time, :default => 0.5
+      config_param :source_host_key, :string, default: nil
+      config_param :blocking_timeout, :time, default: 0.5
 
       def configure(conf)
         super
@@ -130,7 +130,7 @@ def shutdown
       def run
         @loop.run(@blocking_timeout)
       rescue => e
-        log.error "unexpected error", :error => e, :error_class => e.class
+        log.error "unexpected error", error: e, error_class: e.class
         log.error_backtrace
       end
 
@@ -147,7 +147,7 @@ def on_message(msg, addr)
           router.emit(@tag, time, record)
         }
       rescue => e
-        log.error msg.dump, :error => e, :error_class => e.class, :host => addr[3]
+        log.error msg.dump, error: e, error_class: e.class, host: addr[3]
         log.error_backtrace
       end
     end
diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index 6130e472a4..d2f87a0351 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -149,12 +149,12 @@ def output_forward_main(opr, target)
         begin
           target.emit(tag, es, NullOutputChain.instance)
         rescue
-          $log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
+          $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
           $log.warn_backtrace
         end
       }
     rescue
-      $log.error "error on output process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
+      $log.error "error on output process forwarding thread", error: $!.to_s, pid: Process.pid
       $log.error_backtrace
       raise
     end
@@ -165,12 +165,12 @@ def input_forward_main(ipr, pid)
         begin
           Engine.emit_stream(tag, es)
         rescue
-          $log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
+          $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
           $log.warn_backtrace
         end
       }
     rescue
-      $log.error "error on input process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
+      $log.error "error on input process forwarding thread", error: $!.to_s, pid: Process.pid
       $log.error_backtrace
       raise
     end
@@ -264,7 +264,7 @@ def run
           end
         end
       rescue
-        $log.error "error on forwerder thread", :error=>$!.to_s
+        $log.error "error on forwerder thread", error: $!.to_s
         $log.error_backtrace
         raise
       end
@@ -302,7 +302,7 @@ def detach_process_impl(num, &block)
 
         if pid
           # parent process
-          $log.info "detached process", :class=>self.class, :pid=>pid
+          $log.info "detached process", class: self.class, pid: pid
           children << [pid, forward_thread]
           next
         end
@@ -332,7 +332,7 @@ def detach_process_impl(num, &block)
           on_exit_process(i)
           exit! 0
         ensure
-          $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
+          $log.error "unknown error while shutting down this child process", error: $!.to_s, pid: Process.pid
           $log.error_backtrace
         end
 
@@ -353,7 +353,7 @@ def detach_process_impl(num, &block)
               pair[0] = nil
             end
           rescue
-            $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
+            $log.error "unknown error while shutting down remote child process", error: $!.to_s
             $log.error_backtrace
           end
         }
diff --git a/lib/fluent/root_agent.rb b/lib/fluent/root_agent.rb
index a0851c78b7..58462ef478 100644
--- a/lib/fluent/root_agent.rb
+++ b/lib/fluent/root_agent.rb
@@ -121,7 +121,7 @@ def shutdown
             log.info "shutting down input", type: Plugin.lookup_name_from_class(i.class), plugin_id: i.plugin_id
             i.shutdown
           rescue => e
-            log.warn "unexpected error while shutting down input plugin", :plugin => i.class, :plugin_id => i.plugin_id, :error_class => e.class, :error => e
+            log.warn "unexpected error while shutting down input plugin", plugin: i.class, plugin_id: i.plugin_id, error_class: e.class, error: e
             log.warn_backtrace
           end
         end
@@ -168,7 +168,7 @@ def find_label(label_name)
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {:error_class => error.class, :error => error.to_s, :tag => tag, :time => time}
+      error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -180,7 +180,7 @@ def emit_error_event(tag, time, record, error)
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {:error_class => error.class, :error => error.to_s, :tag => tag}
+      error_info = {error_class: error.class, error: error.to_s, tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
@@ -209,14 +209,14 @@ def initialize(root_agent)
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {:error_class => error.class, :error => error.to_s, :tag => tag, :time => time, :record => record}
+        error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 
       def handle_emits_error(tag, es, e)
         now = Engine.now
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed in @ERROR:", :error_class => e.class, :error => e, :tag => tag
+          log.warn "emit transaction failed in @ERROR:", error_class: e.class, error: e, tag: tag
           log.warn_backtrace
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
         end
diff --git a/lib/fluent/rpc.rb b/lib/fluent/rpc.rb
index 752e7f0727..2c7e70a16c 100644
--- a/lib/fluent/rpc.rb
+++ b/lib/fluent/rpc.rb
@@ -26,10 +26,10 @@ def initialize(endpoint, log)
         @log = log
 
         @server = WEBrick::HTTPServer.new(
-          :BindAddress => @bind,
-          :Port => @port,
-          :Logger => WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
-          :AccessLog => [],
+          BindAddress: @bind,
+          Port: @port,
+          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+          AccessLog: [],
         )
       end
 
@@ -43,7 +43,7 @@ def mount_proc(path, &block)
           begin
             code, header, response = block.call(req, res)
           rescue => e
-            @log.warn "failed to handle RPC request", :path => path, :error => e.to_s
+            @log.warn "failed to handle RPC request", path: path, error: e.to_s
             @log.warn_backtrace e.backtrace
 
             code = 500
diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index 10299ba3ea..ce5a364f4a 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -80,20 +80,20 @@ def level=(level)
 
     def self.default_options
       {
-        :config_path => Fluent::DEFAULT_CONFIG_PATH,
-        :plugin_dirs => [Fluent::DEFAULT_PLUGIN_DIR],
-        :log_level => Fluent::Log::LEVEL_INFO,
-        :log_path => nil,
-        :daemonize => nil,
-        :libs => [],
-        :setup_path => nil,
-        :chuser => nil,
-        :chgroup => nil,
-        :suppress_interval => 0,
-        :suppress_repeated_stacktrace => true,
-        :without_source => false,
-        :use_v1_config => true,
-        :supervise => true,
+        config_path: Fluent::DEFAULT_CONFIG_PATH,
+        plugin_dirs: [Fluent::DEFAULT_PLUGIN_DIR],
+        log_level: Fluent::Log::LEVEL_INFO,
+        log_path: nil,
+        daemonize: nil,
+        libs: [],
+        setup_path: nil,
+        chuser: nil,
+        chgroup: nil,
+        suppress_interval: 0,
+        suppress_repeated_stacktrace: true,
+        without_source: false,
+        use_v1_config: true,
+        supervise: true,
       }
     end
 
@@ -118,7 +118,7 @@ def initialize(opt)
       @suppress_config_dump = opt[:suppress_config_dump]
       @without_source = opt[:without_source]
 
-      log_opts = {:suppress_repeated_stacktrace => opt[:suppress_repeated_stacktrace]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace]}
       @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
       @finished = false
       @main_pid = nil
@@ -337,7 +337,7 @@ def supervise(&block)
       @main_pid = nil
       ecode = $?.to_i
 
-      $log.info "process finished", :code=>ecode
+      $log.info "process finished", code: ecode
 
       if !@finished && Time.now - start_time < 1
         $log.warn "process died within 1 second. exit."
@@ -352,20 +352,20 @@ def main_process(&block)
         block.call
 
       rescue Fluent::ConfigError
-        $log.error "config error", :file=>@config_path, :error=>$!.to_s
+        $log.error "config error", file: @config_path, error: $!.to_s
         $log.debug_backtrace
         unless @log.stdout?
           console = Fluent::Log.new(STDOUT, @log_level).enable_debug
-          console.error "config error", :file=>@config_path, :error=>$!.to_s
+          console.error "config error", file: @config_path, error: $!.to_s
           console.debug_backtrace
         end
 
       rescue
-        $log.error "unexpected error", :error=>$!.to_s
+        $log.error "unexpected error", error: $!.to_s
         $log.error_backtrace
         unless @log.stdout?
           console = Fluent::Log.new(STDOUT, @log_level).enable_debug
-          console.error "unexpected error", :error=>$!.to_s
+          console.error "unexpected error", error: $!.to_s
           console.error_backtrace
         end
       end
@@ -454,7 +454,7 @@ def supervisor_get_dump_config_handler
     end
 
     def read_config
-      $log.info "reading config file", :path => @config_path
+      $log.info "reading config file", path: @config_path
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
       @config_data = File.read(@config_path)
@@ -469,16 +469,16 @@ def read_config
     class SystemConfig
       include Configurable
 
-      config_param :log_level, :default => nil do |level|
+      config_param :log_level, default: nil do |level|
         Log.str_to_level(level)
       end
-      config_param :suppress_repeated_stacktrace, :bool, :default => nil
-      config_param :emit_error_log_interval, :time, :default => nil
-      config_param :suppress_config_dump, :bool, :default => nil
-      config_param :without_source, :bool, :default => nil
-      config_param :rpc_endpoint, :string, :default => nil
-      config_param :enable_get_dump, :bool, :default => nil
-      config_param :process_name, :default => nil
+      config_param :suppress_repeated_stacktrace, :bool, default: nil
+      config_param :emit_error_log_interval, :time, default: nil
+      config_param :suppress_config_dump, :bool, default: nil
+      config_param :without_source, :bool, default: nil
+      config_param :rpc_endpoint, :string, default: nil
+      config_param :enable_get_dump, :bool, default: nil
+      config_param :process_name, default: nil
 
       def initialize(conf)
         super()
diff --git a/test/config/test_configurable.rb b/test/config/test_configurable.rb
index 14d509216a..d09c40c5a6 100644
--- a/test/config/test_configurable.rb
+++ b/test/config/test_configurable.rb
@@ -7,14 +7,14 @@ module ConfigurableSpec
   class Base1
     include Fluent::Configurable
 
-    config_param :node, :string, :default => "node"
-    config_param :flag1, :bool, :default => false
-    config_param :flag2, :bool, :default => true
+    config_param :node, :string, default: "node"
+    config_param :flag1, :bool, default: false
+    config_param :flag2, :bool, default: true
 
     config_param :name1, :string
     config_param :name2, :string
-    config_param :name3, :string, :default => "base1"
-    config_param :name4, :string, :default => "base1"
+    config_param :name3, :string, default: "base1"
+    config_param :name4, :string, default: "base1"
 
     config_param :opt1, :enum, list: [:foo, :bar, :baz]
     config_param :opt2, :enum, list: [:foo, :bar, :baz], default: :foo
@@ -29,7 +29,7 @@ class Base2 < Base1
     config_set_default :name4, "base2"
     config_set_default :opt1, :bar
     config_param :name5, :string
-    config_param :name6, :string, :default => "base2"
+    config_param :name6, :string, default: "base2"
     config_param :opt3, :enum, list: [:a, :b]
 
     def get_all
@@ -41,7 +41,7 @@ def get_all
   class Base3 < Base2
     config_set_default :opt3, :a
     config_section :node do
-      config_param :name, :string, :default => "node"
+      config_param :name, :string, default: "node"
       config_param :type, :string
     end
     config_section :branch, required: true, multi: true do
@@ -65,8 +65,8 @@ class Base4 < Base2
     config_set_default :opt3, :a
     config_section :node, param_name: :nodes do
       config_argument :num, :integer
-      config_param :name, :string, :default => "node"
-      config_param :type, :string, :default => "b4"
+      config_param :name, :string, default: "node"
+      config_param :type, :string, default: "b4"
     end
     config_section :description1, required: false, multi: false do
       config_argument :note, :string, default: "desc1"
@@ -151,11 +151,11 @@ class Example5
     include Fluent::Configurable
 
     config_param :normal_param, :string
-    config_param :secret_param, :string, :secret => true
+    config_param :secret_param, :string, secret: true
 
     config_section :section  do
       config_param :normal_param2, :string
-      config_param :secret_param2, :string, :secret => true
+      config_param :secret_param2, :string, secret: true
     end
   end
 end
diff --git a/test/config/test_configure_proxy.rb b/test/config/test_configure_proxy.rb
index b46d85a710..63ee5c7809 100644
--- a/test/config/test_configure_proxy.rb
+++ b/test/config/test_configure_proxy.rb
@@ -127,7 +127,7 @@ class TestConfigureProxy < ::Test::Unit::TestCase
         test 'does not permit description specification twice' do
           @proxy.desc("description1")
           assert_raise(ArgumentError) do
-            @proxy.config_param(:name, :string, :desc => "description2")
+            @proxy.config_param(:name, :string, desc: "description2")
           end
         end
       end
diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
index 957d2dc645..94746196e0 100644
--- a/test/plugin/test_filter_record_transformer.rb
+++ b/test/plugin/test_filter_record_transformer.rb
@@ -351,41 +351,41 @@ def emit(config, msgs = [''])
           { "source" => nil },
         ]
         expected_results = [
-          { :single      => "string",
-            :multiple    => "stringstring",
-            :with_prefix => "prefix-string",
-            :with_suffix => "string-suffix",
-            :with_quote  => %Q{source[""]} },
-          { :single      => 123.to_s,
-            :multiple    => "#{123.to_s}#{123.to_s}",
-            :with_prefix => "prefix-#{123.to_s}",
-            :with_suffix => "#{123.to_s}-suffix",
-            :with_quote  => %Q{source[""]} },
-          { :single      => [1, 2].to_s,
-            :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
-            :with_prefix => "prefix-#{[1, 2].to_s}",
-            :with_suffix => "#{[1, 2].to_s}-suffix",
-            :with_quote  => %Q{source[""]} },
-          { :single      => {a:1, b:2}.to_s,
-            :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
-            :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
-            :with_suffix => "#{{a:1, b:2}.to_s}-suffix",
-            :with_quote  => %Q{source[""]} },
-          { :single      => nil.to_s,
-            :multiple    => "#{nil.to_s}#{nil.to_s}",
-            :with_prefix => "prefix-#{nil.to_s}",
-            :with_suffix => "#{nil.to_s}-suffix",
-            :with_quote  => %Q{source[""]} },
+          { single: "string",
+            multiple: "stringstring",
+            with_prefix: "prefix-string",
+            with_suffix: "string-suffix",
+            with_quote: %Q{source[""]} },
+          { single: 123.to_s,
+            multiple: "#{123.to_s}#{123.to_s}",
+            with_prefix: "prefix-#{123.to_s}",
+            with_suffix: "#{123.to_s}-suffix",
+            with_quote: %Q{source[""]} },
+          { single: [1, 2].to_s,
+            multiple: "#{[1, 2].to_s}#{[1, 2].to_s}",
+            with_prefix: "prefix-#{[1, 2].to_s}",
+            with_suffix: "#{[1, 2].to_s}-suffix",
+            with_quote: %Q{source[""]} },
+          { single: {a:1, b:2}.to_s,
+            multiple: "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
+            with_prefix: "prefix-#{{a:1, b:2}.to_s}",
+            with_suffix: "#{{a:1, b:2}.to_s}-suffix",
+            with_quote: %Q{source[""]} },
+          { single: nil.to_s,
+            multiple: "#{nil.to_s}#{nil.to_s}",
+            with_prefix: "prefix-#{nil.to_s}",
+            with_suffix: "#{nil.to_s}-suffix",
+            with_quote: %Q{source[""]} },
         ]
         actual_results = []
         es = emit(config, msgs)
         es.each_with_index do |(t, r), i|
           actual_results << {
-            :single      => r["single"],
-            :multiple    => r["multiple"],
-            :with_prefix => r["with_prefix"],
-            :with_suffix => r["with_suffix"],
-            :with_quote  => r["with_quote"],
+            single: r["single"],
+            multiple: r["multiple"],
+            with_prefix: r["with_prefix"],
+            with_suffix: r["with_suffix"],
+            with_quote: r["with_quote"],
           }
         end
         assert_equal(expected_results, actual_results)
@@ -410,35 +410,35 @@ def emit(config, msgs = [''])
           { "source" => nil },
         ]
         expected_results = [
-          { :single      => "string",
-            :multiple    => "stringstring",
-            :with_prefix => "prefix-string",
-            :with_suffix => "string-suffix" },
-          { :single      => 123,
-            :multiple    => "#{123.to_s}#{123.to_s}",
-            :with_prefix => "prefix-#{123.to_s}",
-            :with_suffix => "#{123.to_s}-suffix" },
-          { :single      => [1, 2],
-            :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
-            :with_prefix => "prefix-#{[1, 2].to_s}",
-            :with_suffix => "#{[1, 2].to_s}-suffix" },
-          { :single      => {a:1, b:2},
-            :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
-            :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
-            :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
-          { :single      => nil,
-            :multiple    => "#{nil.to_s}#{nil.to_s}",
-            :with_prefix => "prefix-#{nil.to_s}",
-            :with_suffix => "#{nil.to_s}-suffix" },
+          { single: "string",
+            multiple: "stringstring",
+            with_prefix: "prefix-string",
+            with_suffix: "string-suffix" },
+          { single: 123,
+            multiple: "#{123.to_s}#{123.to_s}",
+            with_prefix: "prefix-#{123.to_s}",
+            with_suffix: "#{123.to_s}-suffix" },
+          { single: [1, 2],
+            multiple: "#{[1, 2].to_s}#{[1, 2].to_s}",
+            with_prefix: "prefix-#{[1, 2].to_s}",
+            with_suffix: "#{[1, 2].to_s}-suffix" },
+          { single: {a:1, b:2},
+            multiple: "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
+            with_prefix: "prefix-#{{a:1, b:2}.to_s}",
+            with_suffix: "#{{a:1, b:2}.to_s}-suffix" },
+          { single: nil,
+            multiple: "#{nil.to_s}#{nil.to_s}",
+            with_prefix: "prefix-#{nil.to_s}",
+            with_suffix: "#{nil.to_s}-suffix" },
         ]
         actual_results = []
         es = emit(config, msgs)
         es.each_with_index do |(t, r), i|
           actual_results << {
-            :single      => r["single"],
-            :multiple    => r["multiple"],
-            :with_prefix => r["with_prefix"],
-            :with_suffix => r["with_suffix"],
+            single: r["single"],
+            multiple: r["multiple"],
+            with_prefix: r["with_prefix"],
+            with_suffix: r["with_suffix"],
           }
         end
         assert_equal(expected_results, actual_results)
diff --git a/test/test_config.rb b/test/test_config.rb
index 3b7d495154..e245519203 100644
--- a/test/test_config.rb
+++ b/test/test_config.rb
@@ -159,8 +159,8 @@ def write_config(path, data)
   def test_inline
     prepare_config 
     opts = {
-      :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      config_path: "#{TMP_DIR}/config_test_1.conf",
+      inline_config: "<source>\n  type http\n  port 2222\n </source>"
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)

From 6802dc2a8a282ba2e7c0370ff3dc17302a32be91 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Wed, 10 Feb 2016 00:40:24 +0900
Subject: [PATCH 070/193] Fix to do `require` for required files
 straightforward, per files

* Currently, all required files are loaded by `lib/fluent/load.rb` even for tests
  * This file creates the implicit order of files to be loaded
  * There's no explicit dependency definition, and there are many circular dependencies
  * This situation is totally broken
* Without this change, we can't run tests by `bundle exec rake test TEST=test/file.rb`
  * Running whole tests consumes long time, and it's not acceptable

This change is to make straightforward dependency from all files to others.
We can load any file in fluentd source code, make clean dependencies, and
also can run each files itself to test it.
---
 lib/fluent/agent.rb                           |  9 ++--
 lib/fluent/buffer.rb                          |  6 +++
 lib/fluent/command/bundler_injection.rb       |  3 ++
 lib/fluent/command/cat.rb                     |  2 +-
 lib/fluent/command/debug.rb                   |  5 ++-
 lib/fluent/command/fluentd.rb                 |  3 +-
 lib/fluent/config.rb                          |  9 ++--
 lib/fluent/config/basic_parser.rb             |  7 ++-
 lib/fluent/config/configure_proxy.rb          |  2 +
 lib/fluent/config/dsl.rb                      |  3 ++
 lib/fluent/config/element.rb                  |  5 ++-
 lib/fluent/config/literal_parser.rb           | 13 +++---
 lib/fluent/config/parser.rb                   |  8 ++--
 lib/fluent/config/section.rb                  |  6 +--
 lib/fluent/config/types.rb                    |  7 ++-
 lib/fluent/config/v1_parser.rb                | 14 +++---
 lib/fluent/configurable.rb                    | 12 ++---
 lib/fluent/engine.rb                          | 19 +++++---
 lib/fluent/event.rb                           |  2 +
 lib/fluent/event_router.rb                    |  6 ++-
 lib/fluent/filter.rb                          |  6 +++
 lib/fluent/formatter.rb                       |  6 ++-
 lib/fluent/input.rb                           |  5 +++
 lib/fluent/label.rb                           |  4 +-
 lib/fluent/log.rb                             |  5 ++-
 lib/fluent/mixin.rb                           |  6 ++-
 lib/fluent/output.rb                          | 10 ++++-
 lib/fluent/parser.rb                          | 14 +++++-
 lib/fluent/plugin.rb                          |  6 +++
 lib/fluent/plugin/buf_file.rb                 |  7 +++
 lib/fluent/plugin/buf_memory.rb               |  6 +++
 lib/fluent/plugin/exec_util.rb                |  9 +++-
 lib/fluent/plugin/filter_grep.rb              |  3 ++
 .../plugin/filter_record_transformer.rb       | 12 ++---
 lib/fluent/plugin/filter_stdout.rb            |  3 ++
 lib/fluent/plugin/in_debug_agent.rb           |  2 +
 lib/fluent/plugin/in_dummy.rb                 |  5 +++
 lib/fluent/plugin/in_exec.rb                  |  8 +++-
 lib/fluent/plugin/in_forward.rb               |  7 +++
 lib/fluent/plugin/in_gc_stat.rb               |  4 ++
 lib/fluent/plugin/in_http.rb                  | 11 ++++-
 lib/fluent/plugin/in_monitor_agent.rb         | 17 ++++---
 lib/fluent/plugin/in_object_space.rb          |  5 +++
 lib/fluent/plugin/in_stream.rb                |  9 ++++
 lib/fluent/plugin/in_syslog.rb                |  8 +++-
 lib/fluent/plugin/in_tail.rb                  |  6 +++
 lib/fluent/plugin/in_tcp.rb                   |  2 +
 lib/fluent/plugin/out_copy.rb                 |  4 ++
 lib/fluent/plugin/out_exec.rb                 |  5 ++-
 lib/fluent/plugin/out_exec_filter.rb          |  7 +++
 lib/fluent/plugin/out_file.rb                 |  6 +++
 lib/fluent/plugin/out_forward.rb              | 12 +++--
 lib/fluent/plugin/out_null.rb                 |  2 +
 lib/fluent/plugin/out_relabel.rb              |  2 +
 lib/fluent/plugin/out_roundrobin.rb           |  3 ++
 lib/fluent/plugin/out_stdout.rb               |  2 +
 lib/fluent/plugin/out_stream.rb               | 12 ++---
 lib/fluent/plugin/socket_util.rb              |  6 ++-
 lib/fluent/process.rb                         |  6 +++
 lib/fluent/registry.rb                        |  6 ++-
 lib/fluent/root_agent.rb                      | 13 +++---
 lib/fluent/rpc.rb                             |  4 +-
 lib/fluent/supervisor.rb                      | 44 ++++---------------
 lib/fluent/system_config.rb                   | 12 +++--
 lib/fluent/test.rb                            |  2 +-
 lib/fluent/test/base.rb                       |  4 ++
 lib/fluent/test/filter_test.rb                |  3 ++
 lib/fluent/test/formatter_test.rb             |  5 ++-
 lib/fluent/test/input_test.rb                 |  5 +++
 lib/fluent/test/output_test.rb                |  4 ++
 lib/fluent/test/parser_test.rb                |  5 ++-
 lib/fluent/timezone.rb                        |  2 +
 test/plugin/test_in_forward.rb                |  4 ++
 test/test_event_router.rb                     |  1 +
 test/test_plugin_classes.rb                   |  3 ++
 test/test_process.rb                          |  1 +
 test/test_root_agent.rb                       |  1 +
 77 files changed, 371 insertions(+), 142 deletions(-)

diff --git a/lib/fluent/agent.rb b/lib/fluent/agent.rb
index 1c766fe197..9543b4a277 100644
--- a/lib/fluent/agent.rb
+++ b/lib/fluent/agent.rb
@@ -13,10 +13,13 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
-  require 'fluent/configurable'
-  require 'fluent/engine'
 
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/output'
+
+module Fluent
   #
   # Agent is a resource unit who manages emittable plugins
   #
diff --git a/lib/fluent/buffer.rb b/lib/fluent/buffer.rb
index 2187d835f1..d7327ddcbd 100644
--- a/lib/fluent/buffer.rb
+++ b/lib/fluent/buffer.rb
@@ -13,6 +13,12 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'monitor'
+require 'fileutils'
+
+require 'fluent/configurable'
+
 module Fluent
   class BufferError < StandardError
   end
diff --git a/lib/fluent/command/bundler_injection.rb b/lib/fluent/command/bundler_injection.rb
index 8e244f7ce5..5b90e0684b 100644
--- a/lib/fluent/command/bundler_injection.rb
+++ b/lib/fluent/command/bundler_injection.rb
@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'rbconfig'
+require 'rubygems'
+
 if ENV['BUNDLE_BIN_PATH']
   puts 'error: You seem to use `bundle exec` already.'
   exit 1
diff --git a/lib/fluent/command/cat.rb b/lib/fluent/command/cat.rb
index 8ec8a5e2c1..172a066c86 100644
--- a/lib/fluent/command/cat.rb
+++ b/lib/fluent/command/cat.rb
@@ -66,7 +66,7 @@
   message_key = s
 }
 
-(class<<self;self;end).module_eval do
+(class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
diff --git a/lib/fluent/command/debug.rb b/lib/fluent/command/debug.rb
index 22a21d18a5..1210695337 100644
--- a/lib/fluent/command/debug.rb
+++ b/lib/fluent/command/debug.rb
@@ -34,7 +34,7 @@
   unix = b
 }
 
-(class<<self;self;end).module_eval do
+(class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -60,7 +60,8 @@
   uri = "druby://#{host}:#{port}"
 end
 
-require 'fluent/load'
+require 'fluent/log'
+require 'fluent/engine'
 
 $log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
 Fluent::Engine.init
diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb
index 39fd24bc41..8e4a65c1e5 100644
--- a/lib/fluent/command/fluentd.rb
+++ b/lib/fluent/command/fluentd.rb
@@ -15,6 +15,7 @@
 #
 
 require 'optparse'
+
 require 'fluent/supervisor'
 
 op = OptionParser.new
@@ -119,7 +120,7 @@
   opts[:gem_install_path] = s
 }
 
-(class<<self;self;end).module_eval do
+(class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb
index 5ff253a714..66cc47a69c 100644
--- a/lib/fluent/config.rb
+++ b/lib/fluent/config.rb
@@ -14,10 +14,11 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/error'
-  require 'fluent/config/element'
+require 'fluent/config/error'
+require 'fluent/config/element'
+require 'fluent/configurable'
 
+module Fluent
   module Config
     def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
       if fname =~ /\.rb$/
@@ -39,8 +40,6 @@ def self.new(name = '')
     end
   end
 
-  require 'fluent/configurable'
-
   module PluginId
     def configure(conf)
       @id = conf['@id'] || conf['id']
diff --git a/lib/fluent/config/basic_parser.rb b/lib/fluent/config/basic_parser.rb
index 7d695807fb..2a92f6b9b7 100644
--- a/lib/fluent/config/basic_parser.rb
+++ b/lib/fluent/config/basic_parser.rb
@@ -14,12 +14,11 @@
 #    limitations under the License.
 #
 
+require 'stringio'
+require 'fluent/config/error'
+
 module Fluent
   module Config
-
-    require 'stringio'
-    require 'fluent/config/error'
-
     class BasicParser
       def initialize(strscan)
         @ss = strscan
diff --git a/lib/fluent/config/configure_proxy.rb b/lib/fluent/config/configure_proxy.rb
index 5083da0b4a..af9be8d135 100644
--- a/lib/fluent/config/configure_proxy.rb
+++ b/lib/fluent/config/configure_proxy.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/configurable'
+
 module Fluent
   module Config
     class ConfigureProxy
diff --git a/lib/fluent/config/dsl.rb b/lib/fluent/config/dsl.rb
index 8282007c48..538b13d5d6 100644
--- a/lib/fluent/config/dsl.rb
+++ b/lib/fluent/config/dsl.rb
@@ -14,7 +14,10 @@
 #    limitations under the License.
 #
 
+require 'json'
+
 require 'fluent/config'
+require 'fluent/config/element'
 
 module Fluent
   module Config
diff --git a/lib/fluent/config/element.rb b/lib/fluent/config/element.rb
index ba1f124621..5d47824646 100644
--- a/lib/fluent/config/element.rb
+++ b/lib/fluent/config/element.rb
@@ -14,9 +14,10 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/error'
+require 'fluent/config/error'
+require 'fluent/config/literal_parser'
 
+module Fluent
   module Config
     class Element < Hash
       def initialize(name, arg, attrs, elements, unused = nil)
diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb
index b65479e098..403b7a12ee 100644
--- a/lib/fluent/config/literal_parser.rb
+++ b/lib/fluent/config/literal_parser.rb
@@ -14,13 +14,16 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Config
+require 'stringio'
+
+require 'json'
+require 'yajl'
+require 'irb/ruby-lex'  # RubyLex
 
-    require 'yajl'
-    require 'fluent/config/basic_parser'
-    require 'irb/ruby-lex'  # RubyLex
+require 'fluent/config/basic_parser'
 
+module Fluent
+  module Config
     class LiteralParser < BasicParser
       def self.unescape_char(c)
         case c
diff --git a/lib/fluent/config/parser.rb b/lib/fluent/config/parser.rb
index 6024f5d5b9..98e8cbfbec 100644
--- a/lib/fluent/config/parser.rb
+++ b/lib/fluent/config/parser.rb
@@ -14,11 +14,13 @@
 #    limitations under the License.
 #
 
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+
 module Fluent
   module Config
-    require 'fluent/config/error'
-    require 'fluent/config/element'
-
     class Parser
       def self.parse(io, fname, basepath = Dir.pwd)
         attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
diff --git a/lib/fluent/config/section.rb b/lib/fluent/config/section.rb
index 8549e22fd3..0a0933201c 100644
--- a/lib/fluent/config/section.rb
+++ b/lib/fluent/config/section.rb
@@ -16,10 +16,10 @@
 
 require 'json'
 
-module Fluent
-  require 'fluent/config/error'
-  require 'fluent/config/v1_parser'
+require 'fluent/config/error'
+require 'fluent/config/v1_parser'
 
+module Fluent
   module Config
     class Section < BasicObject
       def self.name
diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
index 5442b66189..13af0b8264 100644
--- a/lib/fluent/config/types.rb
+++ b/lib/fluent/config/types.rb
@@ -14,9 +14,12 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'json'
+require 'json'
+
+require 'fluent/config/error'
+require 'fluent/configurable'
 
+module Fluent
   module Config
     def self.size_value(str)
       case str.to_s
diff --git a/lib/fluent/config/v1_parser.rb b/lib/fluent/config/v1_parser.rb
index 0303e9cd5f..4eceaedc3c 100644
--- a/lib/fluent/config/v1_parser.rb
+++ b/lib/fluent/config/v1_parser.rb
@@ -14,14 +14,16 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Config
+require 'strscan'
+require 'uri'
 
-    require 'strscan'
-    require 'fluent/config/error'
-    require 'fluent/config/literal_parser'
-    require 'fluent/config/element'
+require 'fluent/config/error'
+require 'fluent/config/basic_parser'
+require 'fluent/config/literal_parser'
+require 'fluent/config/element'
 
+module Fluent
+  module Config
     class V1Parser < LiteralParser
       ELEMENT_NAME = /[a-zA-Z0-9_]+/
 
diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
index 262e062690..16bc784d47 100644
--- a/lib/fluent/configurable.rb
+++ b/lib/fluent/configurable.rb
@@ -14,13 +14,13 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/configure_proxy'
-  require 'fluent/config/section'
-  require 'fluent/config/error'
-  require 'fluent/registry'
-  require 'fluent/plugin'
+require 'fluent/config/configure_proxy'
+require 'fluent/config/section'
+require 'fluent/config/error'
+require 'fluent/registry'
+require 'fluent/plugin'
 
+module Fluent
   module Configurable
     def self.included(mod)
       mod.extend(ClassMethods)
diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index 2f942b3ef4..0e98c02494 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -14,12 +14,21 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/event_router'
-  require 'fluent/root_agent'
-  require 'fluent/time'
-  require 'fluent/system_config'
+require 'socket'
+require 'rubygems'
+
+require 'msgpack'
+require 'cool.io'
 
+require 'fluent/config'
+require 'fluent/event'
+require 'fluent/event_router'
+require 'fluent/root_agent'
+require 'fluent/time'
+require 'fluent/system_config'
+require 'fluent/plugin'
+
+module Fluent
   class EngineClass
     class DummyMessagePackFactory
       def packer(*args)
diff --git a/lib/fluent/event.rb b/lib/fluent/event.rb
index f07bfd1a1e..7152b2cd3a 100644
--- a/lib/fluent/event.rb
+++ b/lib/fluent/event.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/engine'
+
 module Fluent
   class EventStream
     include Enumerable
diff --git a/lib/fluent/event_router.rb b/lib/fluent/event_router.rb
index 339e406292..7ff4828606 100644
--- a/lib/fluent/event_router.rb
+++ b/lib/fluent/event_router.rb
@@ -13,9 +13,11 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
-  require 'fluent/match'
 
+require 'fluent/match'
+require 'fluent/event'
+
+module Fluent
   #
   # EventRouter is responsible to route events to a collector.
   #
diff --git a/lib/fluent/filter.rb b/lib/fluent/filter.rb
index b5da9c4982..86d6357f31 100644
--- a/lib/fluent/filter.rb
+++ b/lib/fluent/filter.rb
@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/log'
+
 module Fluent
   class Filter
     include Configurable
diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index dbc4c86b4b..9bbaefd306 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -14,9 +14,11 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/registry'
+require 'fluent/configurable'
+require 'fluent/registry'
+require 'fluent/mixin'
 
+module Fluent
   class Formatter
     include Configurable
 
diff --git a/lib/fluent/input.rb b/lib/fluent/input.rb
index 10a6022bae..7de192019a 100644
--- a/lib/fluent/input.rb
+++ b/lib/fluent/input.rb
@@ -14,6 +14,11 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/log'
+
 module Fluent
   class Input
     include Configurable
diff --git a/lib/fluent/label.rb b/lib/fluent/label.rb
index 2473832b27..d94f102089 100644
--- a/lib/fluent/label.rb
+++ b/lib/fluent/label.rb
@@ -14,9 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/agent'
+require 'fluent/agent'
 
+module Fluent
   class Label < Agent
     def initialize(name, opts = {})
       super(opts)
diff --git a/lib/fluent/log.rb b/lib/fluent/log.rb
index 978da489d1..e0340497c0 100644
--- a/lib/fluent/log.rb
+++ b/lib/fluent/log.rb
@@ -65,6 +65,9 @@ def initialize(out=STDERR, level=LEVEL_TRACE, opts={})
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
 
+      # Fluent::Engine requires Fluent::Log, so we must take that object lazily
+      @engine = Fluent.const_get('Engine')
+
       if opts.has_key?(:suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       end
@@ -285,7 +288,7 @@ def event(level, args)
           record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
         }
         record['message'] = message.dup
-        Engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
+        @engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
       end
 
       return time, message
diff --git a/lib/fluent/mixin.rb b/lib/fluent/mixin.rb
index ff4b1680d1..a2e7ea3a63 100644
--- a/lib/fluent/mixin.rb
+++ b/lib/fluent/mixin.rb
@@ -14,10 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fluent/timezone'
+require 'fluent/time'
+require 'fluent/config/error'
+
 module Fluent
   class TimeFormatter
-    require 'fluent/timezone'
-
     def initialize(format, localtime, timezone = nil)
       @tc1 = 0
       @tc1_str = nil
diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index ebeea00c34..75cd27f9bf 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -14,6 +14,15 @@
 #    limitations under the License.
 #
 
+require 'thread'
+
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/log'
+require 'fluent/plugin'
+require 'fluent/timezone'
+
 module Fluent
   class OutputChain
     def initialize(array, tag, es, chain=NullOutputChain.instance)
@@ -93,7 +102,6 @@ def secondary_init(primary)
     end
   end
 
-
   class OutputThread
     def initialize(output)
       @output = output
diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index ef14b46a09..dd933e358a 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -14,9 +14,19 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/registry'
+require 'time'
+require 'json'
+
+require 'yajl'
 
+require 'fluent/config/error'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/registry'
+require 'fluent/time'
+
+module Fluent
   class ParserError < StandardError
   end
 
diff --git a/lib/fluent/plugin.rb b/lib/fluent/plugin.rb
index da26d4d67c..b4a4fbda41 100644
--- a/lib/fluent/plugin.rb
+++ b/lib/fluent/plugin.rb
@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'rubygems'
+
+require 'fluent/config/error'
+
 module Fluent
   class PluginClass
     # This class is refactored using Fluent::Registry at v0.14
@@ -66,10 +70,12 @@ def new_buffer(type)
     end
 
     def new_parser(type)
+      require 'fluent/parser'
       TextParser.lookup(type)
     end
 
     def new_formatter(type)
+      require 'fluent/formatter'
       TextFormatter.lookup(type)
     end
 
diff --git a/lib/fluent/plugin/buf_file.rb b/lib/fluent/plugin/buf_file.rb
index c3285da110..02c018e976 100644
--- a/lib/fluent/plugin/buf_file.rb
+++ b/lib/fluent/plugin/buf_file.rb
@@ -14,6 +14,13 @@
 #    limitations under the License.
 #
 
+require 'fileutils'
+require 'uri'
+
+require 'fluent/env'
+require 'fluent/plugin'
+require 'fluent/buffer'
+
 module Fluent
   class FileBufferChunk < BufferChunk
     def initialize(key, path, unique_id, mode="a+", symlink_path = nil)
diff --git a/lib/fluent/plugin/buf_memory.rb b/lib/fluent/plugin/buf_memory.rb
index 7ed3a16d35..00b3b6f8a0 100644
--- a/lib/fluent/plugin/buf_memory.rb
+++ b/lib/fluent/plugin/buf_memory.rb
@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'stringio'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/buffer'
+
 module Fluent
   class MemoryBufferChunk < BufferChunk
     def initialize(key, data='')
diff --git a/lib/fluent/plugin/exec_util.rb b/lib/fluent/plugin/exec_util.rb
index 9ed3d9edff..33548dd9ab 100644
--- a/lib/fluent/plugin/exec_util.rb
+++ b/lib/fluent/plugin/exec_util.rb
@@ -14,10 +14,15 @@
 #    limitations under the License.
 #
 
+require 'msgpack'
+require 'yajl'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/parser'
+
 module Fluent
   module ExecUtil
-    require 'fluent/parser'
-
     SUPPORTED_FORMAT = {
       'tsv' => :tsv,
       'json' => :json,
diff --git a/lib/fluent/plugin/filter_grep.rb b/lib/fluent/plugin/filter_grep.rb
index 95a21bcd3d..be8b61a2fa 100644
--- a/lib/fluent/plugin/filter_grep.rb
+++ b/lib/fluent/plugin/filter_grep.rb
@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/filter'
+require 'fluent/config/error'
+
 module Fluent
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 3838c23700..36293d7e7d 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -14,17 +14,19 @@
 #    limitations under the License.
 #
 
+require 'socket'
+require 'json'
 require 'ostruct'
 
+require 'fluent/filter'
+require 'fluent/config/error'
+require 'fluent/event'
+require 'fluent/time'
+
 module Fluent
   class RecordTransformerFilter < Filter
     Fluent::Plugin.register_filter('record_transformer', self)
 
-    def initialize
-      require 'socket'
-      super
-    end
-
     desc 'A comma-delimited list of keys to delete.'
     config_param :remove_keys, :string, default: nil
     desc 'A comma-delimited list of keys to keep.'
diff --git a/lib/fluent/plugin/filter_stdout.rb b/lib/fluent/plugin/filter_stdout.rb
index 5ab0290a99..4fc14d4c43 100644
--- a/lib/fluent/plugin/filter_stdout.rb
+++ b/lib/fluent/plugin/filter_stdout.rb
@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/filter'
+require 'fluent/plugin'
+
 module Fluent
   class StdoutFilter < Filter
     Plugin.register_filter('stdout', self)
diff --git a/lib/fluent/plugin/in_debug_agent.rb b/lib/fluent/plugin/in_debug_agent.rb
index 60a6ffea97..cfb196298b 100644
--- a/lib/fluent/plugin/in_debug_agent.rb
+++ b/lib/fluent/plugin/in_debug_agent.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/input'
+
 module Fluent
   class DebugAgentInput < Input
     Plugin.register_input('debug_agent', self)
diff --git a/lib/fluent/plugin/in_dummy.rb b/lib/fluent/plugin/in_dummy.rb
index b436eb2210..365eb975f4 100644
--- a/lib/fluent/plugin/in_dummy.rb
+++ b/lib/fluent/plugin/in_dummy.rb
@@ -14,6 +14,11 @@
 #    limitations under the License.
 #
 
+require 'json'
+
+require 'fluent/input'
+require 'fluent/config/error'
+
 module Fluent
   class DummyInput < Input
     Fluent::Plugin.register_input('dummy', self)
diff --git a/lib/fluent/plugin/in_exec.rb b/lib/fluent/plugin/in_exec.rb
index b5ea3e3e97..b9274400ca 100644
--- a/lib/fluent/plugin/in_exec.rb
+++ b/lib/fluent/plugin/in_exec.rb
@@ -14,6 +14,13 @@
 #    limitations under the License.
 #
 
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/time'
+require 'fluent/timezone'
+require 'fluent/config/error'
+
 module Fluent
   class ExecInput < Input
     Plugin.register_input('exec', self)
@@ -21,7 +28,6 @@ class ExecInput < Input
     def initialize
       super
       require 'fluent/plugin/exec_util'
-      require 'fluent/timezone'
     end
 
     desc 'The command (program) to execute.'
diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index c9d6d51a20..7a4c2189ea 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -14,6 +14,13 @@
 #    limitations under the License.
 #
 
+require 'fcntl'
+
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+
 module Fluent
   class ForwardInput < Input
     Plugin.register_input('forward', self)
diff --git a/lib/fluent/plugin/in_gc_stat.rb b/lib/fluent/plugin/in_gc_stat.rb
index e0dbfdea41..22690f3e7e 100644
--- a/lib/fluent/plugin/in_gc_stat.rb
+++ b/lib/fluent/plugin/in_gc_stat.rb
@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+
+require 'fluent/input'
+
 module Fluent
   class GCStatInput < Input
     Plugin.register_input('gc_stat', self)
diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index 255a5c5fe3..aee6c3a8fa 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -14,6 +14,16 @@
 #    limitations under the License.
 #
 
+require 'uri'
+require 'socket'
+require 'json'
+
+require 'cool.io'
+
+require 'fluent/input'
+require 'fluent/event'
+require 'fluent/process'
+
 module Fluent
   class HttpInput < Input
     Plugin.register_input('http', self)
@@ -24,7 +34,6 @@ class HttpInput < Input
 
     def initialize
       require 'webrick/httputils'
-      require 'uri'
       super
     end
 
diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index 9ec0a5a1d7..e4389fc4ba 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -14,17 +14,20 @@
 #    limitations under the License.
 #
 
+require 'json'
+require 'webrick'
+require 'cgi'
+
+require 'cool.io'
+
+require 'fluent/input'
+require 'fluent/output'
+require 'fluent/filter'
+
 module Fluent
   class MonitorAgentInput < Input
     Plugin.register_input('monitor_agent', self)
 
-    require 'webrick'
-
-    def initialize
-      require 'cgi'
-      super
-    end
-
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24220
     config_param :tag, :string, default: nil
diff --git a/lib/fluent/plugin/in_object_space.rb b/lib/fluent/plugin/in_object_space.rb
index d29e90905d..e3c28399fb 100644
--- a/lib/fluent/plugin/in_object_space.rb
+++ b/lib/fluent/plugin/in_object_space.rb
@@ -14,6 +14,11 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+
 module Fluent
   class ObjectSpaceInput < Input
     Plugin.register_input('object_space', self)
diff --git a/lib/fluent/plugin/in_stream.rb b/lib/fluent/plugin/in_stream.rb
index d2f9c030aa..3f2d1fcba4 100644
--- a/lib/fluent/plugin/in_stream.rb
+++ b/lib/fluent/plugin/in_stream.rb
@@ -14,6 +14,15 @@
 #    limitations under the License.
 #
 
+require 'fileutils'
+require 'socket'
+
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/event'
+
 module Fluent
   # obsolete
   class StreamInput < Input
diff --git a/lib/fluent/plugin/in_syslog.rb b/lib/fluent/plugin/in_syslog.rb
index fc7d5e2b9d..0f4291fe65 100644
--- a/lib/fluent/plugin/in_syslog.rb
+++ b/lib/fluent/plugin/in_syslog.rb
@@ -14,6 +14,13 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/config/error'
+require 'fluent/parser'
+
 module Fluent
   class SyslogInput < Input
     Plugin.register_input('syslog', self)
@@ -60,7 +67,6 @@ class SyslogInput < Input
 
     def initialize
       super
-      require 'cool.io'
       require 'fluent/plugin/socket_util'
     end
 
diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index fc8c51ac47..b5fb28bf7d 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+
+require 'fluent/input'
+require 'fluent/config/error'
+require 'fluent/event'
+
 module Fluent
   class NewTailInput < Input
     Plugin.register_input('tail', self)
diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb
index df02c5ecbf..cfac648bf9 100644
--- a/lib/fluent/plugin/in_tcp.rb
+++ b/lib/fluent/plugin/in_tcp.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+
 require 'fluent/plugin/socket_util'
 
 module Fluent
diff --git a/lib/fluent/plugin/out_copy.rb b/lib/fluent/plugin/out_copy.rb
index de43630589..c26804b218 100644
--- a/lib/fluent/plugin/out_copy.rb
+++ b/lib/fluent/plugin/out_copy.rb
@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'fluent/output'
+require 'fluent/config/error'
+require 'fluent/event'
+
 module Fluent
   class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
diff --git a/lib/fluent/plugin/out_exec.rb b/lib/fluent/plugin/out_exec.rb
index 2062afda51..653686bf3e 100644
--- a/lib/fluent/plugin/out_exec.rb
+++ b/lib/fluent/plugin/out_exec.rb
@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'tempfile'
+
+require 'fluent/output'
+require 'fluent/config/error'
 require 'fluent/plugin/exec_util'
 
 module Fluent
@@ -22,7 +26,6 @@ class ExecOutput < TimeSlicedOutput
 
     def initialize
       super
-      require 'tempfile'
       @localtime = false
     end
 
diff --git a/lib/fluent/plugin/out_exec_filter.rb b/lib/fluent/plugin/out_exec_filter.rb
index 46021c37bc..4e549a84fe 100644
--- a/lib/fluent/plugin/out_exec_filter.rb
+++ b/lib/fluent/plugin/out_exec_filter.rb
@@ -14,7 +14,14 @@
 #    limitations under the License.
 #
 
+require 'yajl'
+
+require 'fluent/output'
+require 'fluent/env'
+require 'fluent/time'
+require 'fluent/timezone'
 require 'fluent/plugin/exec_util'
+require 'fluent/config/error'
 
 module Fluent
   class ExecFilterOutput < BufferedOutput
diff --git a/lib/fluent/plugin/out_file.rb b/lib/fluent/plugin/out_file.rb
index 08e355cd5c..d2d795acad 100644
--- a/lib/fluent/plugin/out_file.rb
+++ b/lib/fluent/plugin/out_file.rb
@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fileutils'
+require 'zlib'
+
+require 'fluent/output'
+require 'fluent/config/error'
+
 module Fluent
   class FileOutput < TimeSlicedOutput
     Plugin.register_output('file', self)
diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 3063b8caa6..57d747095a 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -14,6 +14,15 @@
 #    limitations under the License.
 #
 
+require 'base64'
+require 'socket'
+require 'fileutils'
+
+require 'cool.io'
+
+require 'fluent/output'
+require 'fluent/config/error'
+
 module Fluent
   class ForwardOutputError < StandardError
   end
@@ -29,9 +38,6 @@ class ForwardOutput < ObjectBufferedOutput
 
     def initialize
       super
-      require "base64"
-      require 'socket'
-      require 'fileutils'
       require 'fluent/plugin/socket_util'
       @nodes = []  #=> [Node]
     end
diff --git a/lib/fluent/plugin/out_null.rb b/lib/fluent/plugin/out_null.rb
index e60ab03af0..032f683a99 100644
--- a/lib/fluent/plugin/out_null.rb
+++ b/lib/fluent/plugin/out_null.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/output'
+
 module Fluent
   class NullOutput < Output
     Plugin.register_output('null', self)
diff --git a/lib/fluent/plugin/out_relabel.rb b/lib/fluent/plugin/out_relabel.rb
index 82196b43f3..9622e66f29 100644
--- a/lib/fluent/plugin/out_relabel.rb
+++ b/lib/fluent/plugin/out_relabel.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/output'
+
 module Fluent
   class RelabelOutput < Output
     Plugin.register_output('relabel', self)
diff --git a/lib/fluent/plugin/out_roundrobin.rb b/lib/fluent/plugin/out_roundrobin.rb
index 34212b4bc9..a17195ea21 100644
--- a/lib/fluent/plugin/out_roundrobin.rb
+++ b/lib/fluent/plugin/out_roundrobin.rb
@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/output'
+require 'fluent/config/error'
+
 module Fluent
   class RoundRobinOutput < MultiOutput
     Plugin.register_output('roundrobin', self)
diff --git a/lib/fluent/plugin/out_stdout.rb b/lib/fluent/plugin/out_stdout.rb
index fbaa700622..47968d3481 100644
--- a/lib/fluent/plugin/out_stdout.rb
+++ b/lib/fluent/plugin/out_stdout.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/output'
+
 module Fluent
   class StdoutOutput < Output
     Plugin.register_output('stdout', self)
diff --git a/lib/fluent/plugin/out_stream.rb b/lib/fluent/plugin/out_stream.rb
index 2ee222010b..d66d2d53d2 100644
--- a/lib/fluent/plugin/out_stream.rb
+++ b/lib/fluent/plugin/out_stream.rb
@@ -14,15 +14,15 @@
 #    limitations under the License.
 #
 
+require 'socket'
+require 'fileutils'
+
+require 'fluent/output'
+require 'fluent/event'
+
 module Fluent
   # obsolete
   class StreamOutput < BufferedOutput
-    def initialize
-      require 'socket'
-      require 'fileutils'
-      super
-    end
-
     config_param :send_timeout, :time, default: 60
 
     def configure(conf)
diff --git a/lib/fluent/plugin/socket_util.rb b/lib/fluent/plugin/socket_util.rb
index 7b9829e879..c5fb7ad98c 100644
--- a/lib/fluent/plugin/socket_util.rb
+++ b/lib/fluent/plugin/socket_util.rb
@@ -14,13 +14,15 @@
 #    limitations under the License.
 #
 
+require 'ipaddr'
+
 require 'cool.io'
 
+require 'fluent/plugin'
+
 module Fluent
   module SocketUtil
     def create_udp_socket(host)
-      require 'ipaddr'
-
       if IPAddr.new(IPSocket.getaddress(host)).ipv4?
         UDPSocket.new
       else
diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index d2f87a0351..87151f8a48 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'thread'
+
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/event'
+
 module Fluent
   class DetachProcessManager
     require 'singleton'
diff --git a/lib/fluent/registry.rb b/lib/fluent/registry.rb
index e5e1c87188..2500db891c 100644
--- a/lib/fluent/registry.rb
+++ b/lib/fluent/registry.rb
@@ -14,9 +14,11 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/error'
+require 'rubygems'
+
+require 'fluent/config/error'
 
+module Fluent
   class Registry
     def initialize(kind, search_prefix)
       @kind = kind
diff --git a/lib/fluent/root_agent.rb b/lib/fluent/root_agent.rb
index 147cea094b..35857652f8 100644
--- a/lib/fluent/root_agent.rb
+++ b/lib/fluent/root_agent.rb
@@ -13,14 +13,17 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
 
-  require 'delegate'
+require 'delegate'
 
-  require 'fluent/agent'
-  require 'fluent/label'
-  require 'fluent/system_config'
+require 'fluent/config/error'
+require 'fluent/agent'
+require 'fluent/engine'
+require 'fluent/label'
+require 'fluent/plugin'
+require 'fluent/system_config'
 
+module Fluent
   #
   # Fluentd forms a tree structure to manage plugins:
   #
diff --git a/lib/fluent/rpc.rb b/lib/fluent/rpc.rb
index 2c7e70a16c..53b899ad81 100644
--- a/lib/fluent/rpc.rb
+++ b/lib/fluent/rpc.rb
@@ -14,10 +14,10 @@
 #    limitations under the License.
 #
 
+require 'webrick'
+
 module Fluent
   module RPC
-    require 'webrick'
-
     class Server
       def initialize(endpoint, log)
         bind, port = endpoint.split(':')
diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index d09883bc23..65712fa669 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -14,10 +14,16 @@
 #    limitations under the License.
 #
 
-require 'fluent/load'
-require 'fluent/system_config'
 require 'etc'
 
+require 'fluent/config'
+require 'fluent/env'
+require 'fluent/engine'
+require 'fluent/log'
+require 'fluent/plugin'
+require 'fluent/rpc'
+require 'fluent/system_config'
+
 module Fluent
   class Supervisor
     def self.get_etc_passwd(user)
@@ -468,40 +474,6 @@ def read_config
       @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
     end
 
-    class SystemConfig
-      include Configurable
-
-      config_param :log_level, default: nil do |level|
-        Log.str_to_level(level)
-      end
-      config_param :suppress_repeated_stacktrace, :bool, default: nil
-      config_param :emit_error_log_interval, :time, default: nil
-      config_param :suppress_config_dump, :bool, default: nil
-      config_param :without_source, :bool, default: nil
-      config_param :rpc_endpoint, :string, default: nil
-      config_param :enable_get_dump, :bool, default: nil
-      config_param :process_name, default: nil
-
-      def initialize(conf)
-        super()
-        configure(conf)
-      end
-
-      def apply(supervisor)
-        system = self
-        supervisor.instance_eval {
-          @log.level = @log_level = system.log_level unless system.log_level.nil?
-          @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
-          @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
-          @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
-          @without_source = system.without_source unless system.without_source.nil?
-          @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
-          @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
-          @process_name = system.process_name unless system.process_name.nil?
-        }
-      end
-    end
-
     def run_configure
       Fluent::Engine.run_configure(@conf)
     end
diff --git a/lib/fluent/system_config.rb b/lib/fluent/system_config.rb
index 4452e9dd86..e7d252c864 100644
--- a/lib/fluent/system_config.rb
+++ b/lib/fluent/system_config.rb
@@ -14,9 +14,10 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/configurable'
+require 'fluent/configurable'
+require 'fluent/config/element'
 
+module Fluent
   module SystemConfigMixin
     def system_config
       @_system_config || Fluent::Engine.system_config
@@ -56,8 +57,13 @@ def self.create(conf)
       SystemConfig.new(systems.first)
     end
 
-    def initialize(conf={})
+    def self.blank_system_config
+      Fluent::Config::Element.new('<SYSTEM>', '', {}, [])
+    end
+
+    def initialize(conf=nil)
       super()
+      conf ||= SystemConfig.blank_system_config
       configure(conf)
     end
 
diff --git a/lib/fluent/test.rb b/lib/fluent/test.rb
index f552f7a20d..d34b4442f7 100644
--- a/lib/fluent/test.rb
+++ b/lib/fluent/test.rb
@@ -15,7 +15,7 @@
 #
 
 require 'test/unit'
-require 'fluent/load'
+require 'fluent/env' # for Fluent.windows?
 require 'fluent/test/base'
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
diff --git a/lib/fluent/test/base.rb b/lib/fluent/test/base.rb
index 8c40ac77ce..713a944ad3 100644
--- a/lib/fluent/test/base.rb
+++ b/lib/fluent/test/base.rb
@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'fluent/engine'
+require 'fluent/system_config'
+require 'fluent/config'
+
 module Fluent
   module Test
     def self.setup
diff --git a/lib/fluent/test/filter_test.rb b/lib/fluent/test/filter_test.rb
index 2a55bee08a..88cca149d6 100644
--- a/lib/fluent/test/filter_test.rb
+++ b/lib/fluent/test/filter_test.rb
@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/test/base'
+require 'fluent/event'
+
 module Fluent
   module Test
     class FilterTestDriver < TestDriver
diff --git a/lib/fluent/test/formatter_test.rb b/lib/fluent/test/formatter_test.rb
index f840cebf52..d7bd2a888c 100644
--- a/lib/fluent/test/formatter_test.rb
+++ b/lib/fluent/test/formatter_test.rb
@@ -14,10 +14,11 @@
 #    limitations under the License.
 #
 
+require 'fluent/formatter'
+require 'fluent/config'
+
 module Fluent
   module Test
-    require 'fluent/config'
-
     class FormatterTestDriver
       def initialize(klass_or_str, proc=nil, &block)
         if klass_or_str.is_a?(Class)
diff --git a/lib/fluent/test/input_test.rb b/lib/fluent/test/input_test.rb
index 341211cd5f..f0c0c90bff 100644
--- a/lib/fluent/test/input_test.rb
+++ b/lib/fluent/test/input_test.rb
@@ -14,6 +14,11 @@
 #    limitations under the License.
 #
 
+require 'fluent/buffer'
+require 'fluent/engine'
+require 'fluent/time'
+require 'fluent/test/base'
+
 module Fluent
   class FileBuffer < BasicBuffer
     def self.clear_buffer_paths
diff --git a/lib/fluent/test/output_test.rb b/lib/fluent/test/output_test.rb
index 83d749e94a..62b971a2f0 100644
--- a/lib/fluent/test/output_test.rb
+++ b/lib/fluent/test/output_test.rb
@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/test/input_test'
+
 module Fluent
   module Test
     class TestOutputChain
diff --git a/lib/fluent/test/parser_test.rb b/lib/fluent/test/parser_test.rb
index 7c6f6ddd6b..79706c192f 100644
--- a/lib/fluent/test/parser_test.rb
+++ b/lib/fluent/test/parser_test.rb
@@ -14,10 +14,11 @@
 #    limitations under the License.
 #
 
+require 'fluent/parser'
+require 'fluent/config'
+
 module Fluent
   module Test
-    require 'fluent/config'
-
     class ParserTestDriver
       def initialize(klass_or_str, format=nil, conf={}, &block)
         if klass_or_str.is_a?(Class)
diff --git a/lib/fluent/timezone.rb b/lib/fluent/timezone.rb
index 4320a728d0..f0383495be 100644
--- a/lib/fluent/timezone.rb
+++ b/lib/fluent/timezone.rb
@@ -16,6 +16,8 @@
 
 require 'tzinfo'
 
+require 'fluent/config/error'
+
 module Fluent
   class Timezone
     # [+-]HH:MM, [+-]HHMM, [+-]HH
diff --git a/test/plugin/test_in_forward.rb b/test/plugin/test_in_forward.rb
index b103d8d3e7..c68a73a374 100644
--- a/test/plugin/test_in_forward.rb
+++ b/test/plugin/test_in_forward.rb
@@ -1,7 +1,11 @@
 require_relative '../helper'
+
 require 'fluent/test'
 require 'base64'
 
+require 'fluent/env'
+require 'fluent/plugin/in_forward'
+
 class ForwardInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
diff --git a/test/test_event_router.rb b/test/test_event_router.rb
index 3e2ff61f96..cc2bd88c45 100644
--- a/test/test_event_router.rb
+++ b/test/test_event_router.rb
@@ -1,3 +1,4 @@
+require_relative 'helper'
 require 'fluent/event_router'
 require_relative 'test_plugin_classes'
 
diff --git a/test/test_plugin_classes.rb b/test/test_plugin_classes.rb
index 77e8c39e30..70cc9416ac 100644
--- a/test/test_plugin_classes.rb
+++ b/test/test_plugin_classes.rb
@@ -1,4 +1,7 @@
 require_relative 'helper'
+require 'fluent/input'
+require 'fluent/output'
+require 'fluent/filter'
 
 module FluentTest
   class FluentTestInput < ::Fluent::Input
diff --git a/test/test_process.rb b/test/test_process.rb
index ae846c3529..7c8c2a072b 100644
--- a/test/test_process.rb
+++ b/test/test_process.rb
@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/event'
+require 'fluent/process'
 require 'stringio'
 require 'msgpack'
 
diff --git a/test/test_root_agent.rb b/test/test_root_agent.rb
index 50ff51b55e..8357a683f7 100644
--- a/test/test_root_agent.rb
+++ b/test/test_root_agent.rb
@@ -1,3 +1,4 @@
+require_relative 'helper'
 require 'fluent/event_router'
 require 'fluent/system_config'
 require_relative 'test_plugin_classes'

From 212f4f8a7737fa64d8eef39990b24c949cf79afd Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 12 Feb 2016 20:31:10 +0900
Subject: [PATCH 071/193] Merge pull request #788 from
 dwoo4dwoo/reduce_tail_opens

in_tail: avoid unnecessary open()s
---
 lib/fluent/plugin/in_tail.rb | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index b5fb28bf7d..8a6fc745ad 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -580,8 +580,7 @@ def initialize(path, log, &on_rotate)
 
         def on_notify
           begin
-            io = File.open(@path)
-            stat = io.stat
+            stat = FileWrapper.stat(@path)
             inode = stat.ino
             fsize = stat.size
           rescue Errno::ENOENT
@@ -593,14 +592,14 @@ def on_notify
           begin
             if @inode != inode || fsize < @fsize
               # rotated or truncated
+              begin
+                io = FileWrapper.open(@path)
+              rescue Errno::ENOENT
+              end
               @on_rotate.call(io)
-              io = nil
             end
-
             @inode = inode
             @fsize = fsize
-          ensure
-            io.close if io
           end
 
         rescue

From 51786e604f126db00cdc8dfb7e5f97e50040da56 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sun, 14 Feb 2016 12:21:08 +0900
Subject: [PATCH 072/193] Merge pull request #802 from
 fluent/fix-in_monitor_agent-debug

Remove unnecessary fields from API response with debug mode. fix #793
---
 lib/fluent/plugin/in_monitor_agent.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index e4389fc4ba..68d10d9be5 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -377,6 +377,9 @@ def plugins_info_all(opts={})
       }
     end
 
+    # TODO: use %i() after drop ruby v1.9.3 support.
+    IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
+
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
       obj = {}
@@ -400,6 +403,7 @@ def get_monitor_info(pe, opts={})
         iv = {}
         pe.instance_eval do
           instance_variables.each {|sym|
+            next if IGNORE_ATTRIBUTES.include?(sym)
             key = sym.to_s[1..-1]  # removes first '@'
             iv[key] = instance_variable_get(sym)
           }

From be46bbff3d2cfebb02f328364fecb896c65e3e29 Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Fri, 19 Feb 2016 11:54:17 +0900
Subject: [PATCH 073/193] Merge pull request #809 from
 okkez/dont-require-rubygems

Don't require rubygems
---
 bin/fluent-cat                          | 1 -
 bin/fluent-debug                        | 1 -
 bin/fluent-gem                          | 1 -
 bin/fluentd                             | 1 -
 lib/fluent/command/bundler_injection.rb | 1 -
 lib/fluent/engine.rb                    | 1 -
 lib/fluent/plugin.rb                    | 2 --
 lib/fluent/registry.rb                  | 2 --
 8 files changed, 10 deletions(-)

diff --git a/bin/fluent-cat b/bin/fluent-cat
index 42e4edb551..16277df0c6 100755
--- a/bin/fluent-cat
+++ b/bin/fluent-cat
@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-require 'rubygems' unless defined?(gem)
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
 require 'fluent/command/cat'
diff --git a/bin/fluent-debug b/bin/fluent-debug
index f4f28cdf85..6f99e88de4 100755
--- a/bin/fluent-debug
+++ b/bin/fluent-debug
@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-require 'rubygems' unless defined?(gem)
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
 require 'fluent/command/debug'
diff --git a/bin/fluent-gem b/bin/fluent-gem
index 6a61974669..124fb2c545 100755
--- a/bin/fluent-gem
+++ b/bin/fluent-gem
@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-require 'rubygems' unless defined?(gem)
 require 'rubygems/gem_runner'
 require 'rubygems/exceptions'
 begin
diff --git a/bin/fluentd b/bin/fluentd
index 6760512df4..9be4ff39aa 100755
--- a/bin/fluentd
+++ b/bin/fluentd
@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-require 'rubygems' unless defined?(gem)
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
 require 'fluent/command/fluentd'
diff --git a/lib/fluent/command/bundler_injection.rb b/lib/fluent/command/bundler_injection.rb
index 5b90e0684b..1037eb9e9b 100644
--- a/lib/fluent/command/bundler_injection.rb
+++ b/lib/fluent/command/bundler_injection.rb
@@ -15,7 +15,6 @@
 #
 
 require 'rbconfig'
-require 'rubygems'
 
 if ENV['BUNDLE_BIN_PATH']
   puts 'error: You seem to use `bundle exec` already.'
diff --git a/lib/fluent/engine.rb b/lib/fluent/engine.rb
index 0e98c02494..2f506e2307 100644
--- a/lib/fluent/engine.rb
+++ b/lib/fluent/engine.rb
@@ -15,7 +15,6 @@
 #
 
 require 'socket'
-require 'rubygems'
 
 require 'msgpack'
 require 'cool.io'
diff --git a/lib/fluent/plugin.rb b/lib/fluent/plugin.rb
index b4a4fbda41..ee7fcb2f7c 100644
--- a/lib/fluent/plugin.rb
+++ b/lib/fluent/plugin.rb
@@ -14,8 +14,6 @@
 #    limitations under the License.
 #
 
-require 'rubygems'
-
 require 'fluent/config/error'
 
 module Fluent
diff --git a/lib/fluent/registry.rb b/lib/fluent/registry.rb
index 2500db891c..2473090705 100644
--- a/lib/fluent/registry.rb
+++ b/lib/fluent/registry.rb
@@ -14,8 +14,6 @@
 #    limitations under the License.
 #
 
-require 'rubygems'
-
 require 'fluent/config/error'
 
 module Fluent

From dd0a2f3871470e40a27e855aa63be259064a4601 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sun, 28 Feb 2016 06:17:45 +0900
Subject: [PATCH 074/193] Merge pull request #811 from
 dwoo4dwoo/disable_watch_timer

Added enable_watch_timer option
---
 lib/fluent/plugin/in_tail.rb | 14 +++++++++-----
 test/plugin/test_in_tail.rb  | 33 +++++++++++++++++++++++++++++++--
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 8a6fc745ad..035899c8ac 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -48,6 +48,8 @@ def initialize
     config_param :read_lines_limit, :integer, default: 1000
     desc 'The interval of flushing the buffer for multiline format'
     config_param :multiline_flush_interval, :time, default: nil
+    desc 'Enable the additional watch timer.'
+    config_param :enable_watch_timer, :bool, default: true
 
     attr_reader :paths
 
@@ -156,7 +158,7 @@ def refresh_watchers
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
       tw.attach(@loop)
       tw
     end
@@ -318,16 +320,18 @@ def parse_multilines(lines, tail_watcher)
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
+        @enable_watch_timer = enable_watch_timer
         @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify))
+        @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify)) if @enable_watch_timer
+
         @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
 
         @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
@@ -350,13 +354,13 @@ def wrap_receive_lines(lines)
       end
 
       def attach(loop)
-        @timer_trigger.attach(loop)
+        @timer_trigger.attach(loop) if @enable_watch_timer
         @stat_trigger.attach(loop)
         on_notify
       end
 
       def detach
-        @timer_trigger.detach if @timer_trigger.attached?
+        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
         @stat_trigger.detach if @stat_trigger.attached?
       end
 
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index caac6a0922..5da4078186 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -30,6 +30,9 @@ def teardown
   CONFIG_READ_FROM_HEAD = %[
     read_from_head true
   ]
+  CONFIG_ENABLE_WATCH_TIMER = %[
+    enable_watch_timer false
+  ]
   SINGLE_LINE_CONFIG = %[
     format /(?<message>.*)/
   ]
@@ -124,6 +127,32 @@ def test_emit_with_read_from_head
     assert_equal({"message" => "test4"}, emits[3][2])
   end
 
+  def test_emit_with_enable_watch_timer
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver(CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG)
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      # according to cool.io's stat_watcher.c, systems without inotify will use
+      # an "automatic" value, typically around 5 seconds
+      sleep 10
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+  end
+
   def test_rotate_file
     emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
     assert_equal(4, emits.length)
@@ -484,7 +513,7 @@ def test_refresh_watchers
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -500,7 +529,7 @@ def test_refresh_watchers
       end
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times

From a1e2005325bcbb24da74abc20af0d1bd1c0fda19 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 1 Mar 2016 22:17:03 +0900
Subject: [PATCH 075/193] Merge pull request #816 from
 fluent/fix_record_transformer_race

Fix record_transformer to be thread-safe
---
 .../plugin/filter_record_transformer.rb       | 62 +++++++++----------
 1 file changed, 30 insertions(+), 32 deletions(-)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 36293d7e7d..69d2baa561 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -126,28 +126,29 @@ def parse_value(value_str)
     end
 
     def reform(record, placeholder_values)
-      @placeholder_expander.prepare_placeholders(placeholder_values)
+      placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
-      new_record.merge!(expand_placeholders(@map))
+      new_record.merge!(expand_placeholders(@map, placeholders))
       @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end
 
-    def expand_placeholders(value)
+    def expand_placeholders(value, placeholders)
       if value.is_a?(String)
-        new_value = @placeholder_expander.expand(value)
+        new_value = @placeholder_expander.expand(value, placeholders)
       elsif value.is_a?(Hash)
         new_value = {}
         value.each_pair do |k, v|
-          new_value[@placeholder_expander.expand(k, true)] = expand_placeholders(v)
+          new_key = @placeholder_expander.expand(k, placeholders, true)
+          new_value[new_key] = expand_placeholders(v, placeholders)
         end
       elsif value.is_a?(Array)
         new_value = []
         value.each_with_index do |v, i|
-          new_value[i] = expand_placeholders(v)
+          new_value[i] = expand_placeholders(v, placeholders)
         end
       else
         new_value = value
@@ -174,6 +175,7 @@ def tag_suffix(tag_parts)
       rev_tag_suffix.reverse!
     end
 
+    # THIS CLASS MUST BE THREAD-SAFE
     class PlaceholderExpander
       attr_reader :placeholders, :log
 
@@ -210,36 +212,37 @@ def prepare_placeholders(placeholder_values)
           end
         end
 
-        @placeholders = placeholders
+        placeholders
       end
 
       # Expand string with placeholders
       #
       # @param [String] str
       # @param [Boolean] force_stringify the value must be string, used for hash key
-      def expand(str, force_stringify = false)
+      def expand(str, placeholders, force_stringify = false)
         if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
-            log_if_unknown_placeholder($1)
-            return @placeholders[single_placeholder_matched[1]]
+            log_if_unknown_placeholder($1, placeholders)
+            return placeholders[single_placeholder_matched[1]]
           end
         end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log_if_unknown_placeholder($1)
-          @placeholders[$1]
+          log_if_unknown_placeholder($1, placeholders)
+          placeholders[$1]
         }
       end
 
       private
 
-      def log_if_unknown_placeholder(placeholder)
-        unless @placeholders.include?(placeholder)
+      def log_if_unknown_placeholder(placeholder, placeholders)
+        unless placeholders.include?(placeholder)
           log.warn "unknown placeholder `#{placeholder}` found"
         end
       end
     end
 
+    # THIS CLASS MUST BE THREAD-SAFE
     class RubyPlaceholderExpander
       attr_reader :log
 
@@ -285,28 +288,22 @@ def preprocess_map(value, force_stringify = false)
       end
 
       def prepare_placeholders(placeholder_values)
-        @tag = placeholder_values['tag']
-        @time = placeholder_values['time']
-        @record = placeholder_values['record']
-        @tag_parts = placeholder_values['tag_parts']
-        @tag_prefix = placeholder_values['tag_prefix']
-        @tag_suffix = placeholder_values['tag_suffix']
-        @hostname = placeholder_values['hostname']
+        placeholder_values
       end
 
       # Expand string with placeholders
       #
       # @param [String] str
-      def expand(str, force_stringify = false)
+      def expand(str, placeholders, force_stringify = false)
         @cleanroom_expander.expand(
           str,
-          @tag,
-          @time,
-          @record,
-          @tag_parts,
-          @tag_prefix,
-          @tag_suffix,
-          @hostname,
+          placeholders['tag'],
+          placeholders['time'],
+          placeholders['record'],
+          placeholders['tag_parts'],
+          placeholders['tag_prefix'],
+          placeholders['tag_suffix'],
+          placeholders['hostname'],
         )
       rescue => e
         log.warn "failed to expand `#{str}`", error_class: e.class, error: e.message
@@ -317,15 +314,16 @@ def expand(str, force_stringify = false)
       class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
           tags = tag_parts # for old version compatibility
-          @record = record # for old version compatibility
+          Thread.current[:record_transformer_record] = record # for old version compatibility
           instance_eval(__str_to_eval__)
         end
 
         # for old version compatibility
         def method_missing(name)
           key = name.to_s
-          if @record.has_key?(key)
-            @record[key]
+          record = Thread.current[:record_transformer_record]
+          if record.has_key?(key)
+            record[key]
           else
             raise NameError, "undefined local variable or method `#{key}'"
           end

From 14716328208a26a75a76926c4db80193f499e5ce Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 2 Mar 2016 10:38:36 +0900
Subject: [PATCH 076/193] Merge pull request #817 from
 fluent/relax_auto_typecast

filter_record_transformer: Relax conditions which auto_typecast is applied for enable_ruby yes
---
 .../plugin/filter_record_transformer.rb       |  7 ++++---
 test/plugin/test_filter_record_transformer.rb | 19 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 69d2baa561..d86664294c 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -264,12 +264,13 @@ def preprocess_map(value, force_stringify = false)
         new_value = nil
         if value.is_a?(String)
           if @auto_typecast and !force_stringify
-            if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
-              new_value = single_placeholder_matched[1]
+            num_placeholders = value.scan('${').size
+            if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
+              new_value = value[2..-2] # ${..} => ..
             end
           end
           unless new_value
-            new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
+            new_value = "%Q[#{value.gsub('${', '#{')}]" # xx${..}xx => %Q[xx#{..}xx]
           end
         elsif value.is_a?(Hash)
           new_value = {}
diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
index 94746196e0..8b94cf7eba 100644
--- a/test/plugin/test_filter_record_transformer.rb
+++ b/test/plugin/test_filter_record_transformer.rb
@@ -508,6 +508,23 @@ def emit(config, msgs = [''])
       end
     end
 
+    test 'auto_typecast placeholder containing {} (enable_ruby yes)' do
+      config = %[
+        tag tag
+        enable_ruby yes
+        auto_typecast yes
+        <record>
+          foo ${record.map{|k,v|v}}
+        </record>
+      ]
+      d = create_driver(config)
+      message = {"@timestamp" => "foo"}
+      es = d.run { d.emit(message, @time) }.filtered
+      es.each do |t, r|
+        assert_equal([message["@timestamp"]], r['foo'])
+      end
+    end
+
     test 'expand fields starting with @ (enable_ruby yes)' do
       config = %[
         enable_ruby yes
@@ -522,7 +539,7 @@ def emit(config, msgs = [''])
         assert_equal(message["@timestamp"], r['foo'])
       end
     end
-  end
+  end # test placeholders
 
   test "compatibility test (enable_ruby yes)" do
     config = %[

From 27cf781b7bfe7aee389a83703d71c4d9686070e9 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 2 Mar 2016 12:49:09 +0900
Subject: [PATCH 077/193] FileWrapper is introduced since v0.14. Don't use in
 v0.12

---
 lib/fluent/plugin/in_tail.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 035899c8ac..db18a28a6d 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -584,7 +584,7 @@ def initialize(path, log, &on_rotate)
 
         def on_notify
           begin
-            stat = FileWrapper.stat(@path)
+            stat = File.stat(@path)
             inode = stat.ino
             fsize = stat.size
           rescue Errno::ENOENT
@@ -597,7 +597,7 @@ def on_notify
             if @inode != inode || fsize < @fsize
               # rotated or truncated
               begin
-                io = FileWrapper.open(@path)
+                io = File.open(@path)
               rescue Errno::ENOENT
               end
               @on_rotate.call(io)

From 41d17240852d0dff8ba811eff1f6c7a501c58af6 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 2 Mar 2016 22:02:38 +0900
Subject: [PATCH 078/193] Merge pull request #810 from
 fluent/fix-in_tail-for-bqle

Fix in_tail input messages loss when receive_lines fail. fix #699
---
 lib/fluent/plugin/in_tail.rb | 52 ++++++++++++++++++++++--------------
 test/plugin/test_in_tail.rb  | 42 +++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 20 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index db18a28a6d..7740c2cb7f 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -244,6 +244,7 @@ def run
       log.error_backtrace
     end
 
+    # @return true if no error or unrecoverable error happens in emit action. false if got BufferQueueLimitError
     def receive_lines(lines, tail_watcher)
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
@@ -254,10 +255,15 @@ def receive_lines(lines, tail_watcher)
               end
         begin
           router.emit_stream(tag, es)
+        rescue BufferQueueLimitError
+          return false
         rescue
-          # ignore errors. Engine shows logs and backtraces.
+          # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
+          return true
         end
       end
+
+      return true
     end
 
     def convert_line_to_event(line, es)
@@ -513,6 +519,7 @@ def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
           @receive_lines = receive_lines
           @buffer = ''.force_encoding('ASCII-8BIT')
           @iobuf = ''.force_encoding('ASCII-8BIT')
+          @lines = []
         end
 
         attr_reader :io
@@ -520,31 +527,36 @@ def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
 
         def on_notify
           begin
-            lines = []
             read_more = false
 
-            begin
-              while true
-                if @buffer.empty?
-                  @io.read_nonblock(2048, @buffer)
-                else
-                  @buffer << @io.read_nonblock(2048, @iobuf)
-                end
-                while line = @buffer.slice!(/.*?\n/m)
-                  lines << line
-                end
-                if lines.size >= @read_lines_limit
-                  # not to use too much memory in case the file is very large
-                  read_more = true
-                  break
+            if @lines.empty?
+              begin
+                while true
+                  if @buffer.empty?
+                    @io.readpartial(2048, @buffer)
+                  else
+                    @buffer << @io.readpartial(2048, @iobuf)
+                  end
+                  while line = @buffer.slice!(/.*?\n/m)
+                    @lines << line
+                  end
+                  if @lines.size >= @read_lines_limit
+                    # not to use too much memory in case the file is very large
+                    read_more = true
+                    break
+                  end
                 end
+              rescue EOFError
               end
-            rescue EOFError
             end
 
-            unless lines.empty?
-              @receive_lines.call(lines)
-              @pe.update_pos(@io.pos - @buffer.bytesize)
+            unless @lines.empty?
+              if @receive_lines.call(@lines)
+                @pe.update_pos(@io.pos - @buffer.bytesize)
+                @lines.clear
+              else
+                read_more = false
+              end
             end
           end while read_more
 
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 5da4078186..18beeff676 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -638,4 +638,46 @@ def test_missing_file
       assert_equal({"message" => "test4"}, emits[1][2])
     end
   end
+
+  sub_test_case 'emit error cases' do
+    def test_emit_error_with_buffer_queue_limit_error
+      emits = execute_test(::Fluent::BufferQueueLimitError, "queue size exceeds limit")
+      assert_equal(10, emits.length)
+      10.times { |i|
+        assert_equal({"message" => "test#{i}"}, emits[i][2])
+      }
+    end
+
+    def test_emit_error_with_non_buffer_queue_limit_error
+      emits = execute_test(StandardError, "non BufferQueueLimitError error")
+      assert_true(emits.size > 0 && emits.size != 10)
+      emits.size.times { |i|
+        assert_equal({"message" => "test#{10 - emits.size + i}"}, emits[i][2])
+      }
+    end
+
+    def execute_test(error_class, error_message)
+      d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
+      # Use define_singleton_method instead of d.emit_stream to capture local variable
+      d.define_singleton_method(:emit_stream) do |tag, es|
+        @test_num_errors ||= 0
+        if @test_num_errors < 5
+          @test_num_errors += 1
+          raise error_class, error_message
+        else
+          @emit_streams << [tag, es.to_a]
+        end
+      end
+
+      d.run do
+        10.times { |i|
+          File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test#{i}" }
+          sleep 0.5
+        }
+        sleep 1
+      end
+
+      d.emits
+    end
+  end
 end

From c90a1b245e758f5862df3a60fd1aaf3d3c5488fa Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Mar 2016 07:42:17 +0900
Subject: [PATCH 079/193] Merge pull request #821 from
 fluent/enum-list-must-be-of-symbols

Enum list must be of symbols
---
 lib/fluent/config/types.rb | 87 +++++++++++++++++---------------------
 test/config/test_types.rb  | 70 ++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 48 deletions(-)

diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
index 13af0b8264..b295ef5816 100644
--- a/lib/fluent/config/types.rb
+++ b/lib/fluent/config/types.rb
@@ -64,54 +64,45 @@ def self.bool_value(str)
         nil
       end
     end
-  end
-
-  Configurable.register_type(:string, Proc.new { |val, opts|
-    val
-  })
-
-  Configurable.register_type(:enum, Proc.new { |val, opts|
-    s = val.to_sym
-    raise "Plugin BUG: config type 'enum' requires :list argument" unless opts[:list].is_a?(Array)
-    unless opts[:list].include?(s)
-      raise ConfigError, "valid options are #{opts[:list].join(',')} but got #{val}"
-    end
-    s
-  })
-
-  Configurable.register_type(:integer, Proc.new { |val, opts|
-    val.to_i
-  })
-
-  Configurable.register_type(:float, Proc.new { |val, opts|
-    val.to_f
-  })
-
-  Configurable.register_type(:size, Proc.new { |val, opts|
-    Config.size_value(val)
-  })
-
-  Configurable.register_type(:bool, Proc.new { |val, opts|
-    Config.bool_value(val)
-  })
 
-  Configurable.register_type(:time, Proc.new { |val, opts|
-    Config.time_value(val)
-  })
-
-  Configurable.register_type(:hash, Proc.new { |val, opts|
-    param = val.is_a?(String) ? JSON.load(val) : val
-    if param.class != Hash
-      raise ConfigError, "hash required but got #{val.inspect}"
-    end
-    param
-  })
+    STRING_TYPE = Proc.new { |val, opts| val }
+    ENUM_TYPE = Proc.new { |val, opts|
+      s = val.to_sym
+      list = opts[:list]
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      unless list.include?(s)
+        raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
+      end
+      s
+    }
+    INTEGER_TYPE = Proc.new { |val, opts| val.to_i }
+    FLOAT_TYPE = Proc.new { |val, opts| val.to_f }
+    SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
+    BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
+    TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+    HASH_TYPE = Proc.new { |val, opts|
+      param = val.is_a?(String) ? JSON.load(val) : val
+      if param.class != Hash
+        raise ConfigError, "hash required but got #{val.inspect}"
+      end
+      param
+    }
+    ARRAY_TYPE = Proc.new { |val, opts|
+      param = val.is_a?(String) ? JSON.load(val) : val
+      if param.class != Array
+        raise ConfigError, "array required but got #{val.inspect}"
+      end
+      param
+    }
+  end
 
-  Configurable.register_type(:array, Proc.new { |val, opts|
-    param = val.is_a?(String) ? JSON.load(val) : val
-    if param.class != Array
-      raise ConfigError, "array required but got #{val.inspect}"
-    end
-    param
-  })
+  Configurable.register_type(:string,  Config::STRING_TYPE)
+  Configurable.register_type(:enum,    Config::ENUM_TYPE)
+  Configurable.register_type(:integer, Config::INTEGER_TYPE)
+  Configurable.register_type(:float,   Config::FLOAT_TYPE)
+  Configurable.register_type(:size,    Config::SIZE_TYPE)
+  Configurable.register_type(:bool,    Config::BOOL_TYPE)
+  Configurable.register_type(:time,    Config::TIME_TYPE)
+  Configurable.register_type(:hash,    Config::HASH_TYPE)
+  Configurable.register_type(:array,   Config::ARRAY_TYPE)
 end
diff --git a/test/config/test_types.rb b/test/config/test_types.rb
index be240dbf52..95c544bc63 100644
--- a/test/config/test_types.rb
+++ b/test/config/test_types.rb
@@ -60,4 +60,74 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_nil Config.bool_value(10)
     end
   end
+
+  sub_test_case 'type converters for config_param definitions' do
+    test 'string' do
+      assert_equal 'test', Config::STRING_TYPE.call('test', {})
+      assert_equal '1', Config::STRING_TYPE.call('1', {})
+      assert_equal '   ', Config::STRING_TYPE.call('   ', {})
+    end
+
+    test 'enum' do
+      assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
+      assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
+      assert_equal :value, Config::ENUM_TYPE.call('value', {list: [:val, :value, :v]})
+      assert_raises(Fluent::ConfigError){ Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) }
+      assert_raises(RuntimeError){ Config::ENUM_TYPE.call('val', {}) }
+      assert_raises(RuntimeError){ Config::ENUM_TYPE.call('val', {list: ["val", "value", "v"]}) }
+    end
+
+    test 'integer' do
+      assert_equal 1, Config::INTEGER_TYPE.call('1', {})
+      assert_equal 1, Config::INTEGER_TYPE.call('1.0', {})
+      assert_equal 1000, Config::INTEGER_TYPE.call('1_000', {})
+      assert_equal 1, Config::INTEGER_TYPE.call('1x', {})
+    end
+
+    test 'float' do
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1', {})
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1.0', {})
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1.00', {})
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1e0', {})
+    end
+
+    test 'size' do
+      assert_equal 1000, Config::SIZE_TYPE.call('1000', {})
+      assert_equal 1024, Config::SIZE_TYPE.call('1k', {})
+      assert_equal 1024*1024, Config::SIZE_TYPE.call('1m', {})
+    end
+
+    test 'bool' do
+      assert_equal true, Config::BOOL_TYPE.call('true', {})
+      assert_equal true, Config::BOOL_TYPE.call('yes', {})
+      assert_equal false, Config::BOOL_TYPE.call('no', {})
+      assert_equal false, Config::BOOL_TYPE.call('false', {})
+
+      assert_equal nil, Config::BOOL_TYPE.call('TRUE', {})
+      assert_equal nil, Config::BOOL_TYPE.call('True', {})
+      assert_equal nil, Config::BOOL_TYPE.call('Yes', {})
+      assert_equal nil, Config::BOOL_TYPE.call('No', {})
+
+      assert_equal true, Config::BOOL_TYPE.call('', {})
+      assert_equal nil, Config::BOOL_TYPE.call('unexpected_string', {})
+    end
+
+    test 'time' do
+      assert_equal 0, Config::TIME_TYPE.call('0', {})
+      assert_equal 1.0, Config::TIME_TYPE.call('1', {})
+      assert_equal 1.01, Config::TIME_TYPE.call('1.01', {})
+      assert_equal 1, Config::TIME_TYPE.call('1s', {})
+      assert_equal 60, Config::TIME_TYPE.call('1m', {})
+      assert_equal 3600, Config::TIME_TYPE.call('1h', {})
+      assert_equal 86400, Config::TIME_TYPE.call('1d', {})
+    end
+
+    test 'hash' do
+      assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {}))
+    end
+
+    test 'array' do
+      assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
+    end
+  end
 end

From a9dfedd72df9e687d127f96f810833a93d511062 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Mar 2016 08:08:02 +0900
Subject: [PATCH 080/193] Merge pull request #822 from
 fluent/fix-set-system-config-in-sighup

Fix the regression of SystemConfig apply
---
 lib/fluent/supervisor.rb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index 65712fa669..961cae84be 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -135,8 +135,7 @@ def start
       @log.init
       show_plugin_config if @show_plugin_config
       read_config
-      @system_config = SystemConfig.create(@conf) # @conf is set in read_config
-      @system_config.apply(self)
+      set_system_config
 
       dry_run if @dry_run
       start_daemonize if @daemonize
@@ -437,7 +436,7 @@ def supervisor_sighup_handler
       # in main thread during trap context
       Thread.new {
         read_config
-        apply_system_config
+        set_system_config
         if pid = @main_pid
           Process.kill(:TERM, pid)
           # don't resuce Erro::ESRSH here (invalid status)
@@ -474,6 +473,11 @@ def read_config
       @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
     end
 
+    def set_system_config
+      @system_config = SystemConfig.create(@conf) # @conf is set in read_config
+      @system_config.apply(self)
+    end
+
     def run_configure
       Fluent::Engine.run_configure(@conf)
     end

From 556cc58fa28c13fc358904285714c1d185bd52f8 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 3 Mar 2016 09:47:20 +0900
Subject: [PATCH 081/193] Add missing require

fluent/output uses fluent/status.
---
 lib/fluent/output.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index 75cd27f9bf..da829c0335 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -21,6 +21,7 @@
 require 'fluent/engine'
 require 'fluent/log'
 require 'fluent/plugin'
+require 'fluent/status'
 require 'fluent/timezone'
 
 module Fluent

From c1e736cffbd7571ce8bff5bbb8e438417f52f93e Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 3 Mar 2016 11:42:34 +0900
Subject: [PATCH 082/193] Add missing require

fluent/version is used in command/fluentd.
---
 lib/fluent/command/fluentd.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb
index 8e4a65c1e5..35fca5f0d8 100644
--- a/lib/fluent/command/fluentd.rb
+++ b/lib/fluent/command/fluentd.rb
@@ -17,6 +17,7 @@
 require 'optparse'
 
 require 'fluent/supervisor'
+require 'fluent/version'
 
 op = OptionParser.new
 op.version = Fluent::VERSION

From 46f825fcb373efc91ac18058b008fb3508088a3f Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 3 Mar 2016 12:14:26 +0900
Subject: [PATCH 083/193] Add missing require

fluent/mixin should include automatically before requiring plugin.
---
 lib/fluent/configurable.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
index 16bc784d47..889d995dc9 100644
--- a/lib/fluent/configurable.rb
+++ b/lib/fluent/configurable.rb
@@ -19,6 +19,7 @@
 require 'fluent/config/error'
 require 'fluent/registry'
 require 'fluent/plugin'
+require 'fluent/mixin'
 
 module Fluent
   module Configurable

From ff0738fd128a34f07ead4de6a493ba3d9bf907ce Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 3 Mar 2016 13:44:39 +0900
Subject: [PATCH 084/193] Add missing require

fluent/filter should require in filter test driver.
---
 lib/fluent/test/filter_test.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/test/filter_test.rb b/lib/fluent/test/filter_test.rb
index 88cca149d6..c5c080d5d7 100644
--- a/lib/fluent/test/filter_test.rb
+++ b/lib/fluent/test/filter_test.rb
@@ -16,6 +16,7 @@
 
 require 'fluent/test/base'
 require 'fluent/event'
+require 'fluent/filter'
 
 module Fluent
   module Test

From b585e9f187be1f2469190873710eb9fbda3effdf Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 4 Mar 2016 11:42:42 +0900
Subject: [PATCH 085/193] Add missing require

This is for input plugin test
---
 lib/fluent/test/input_test.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/test/input_test.rb b/lib/fluent/test/input_test.rb
index f0c0c90bff..feb05651ec 100644
--- a/lib/fluent/test/input_test.rb
+++ b/lib/fluent/test/input_test.rb
@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/input'
 require 'fluent/buffer'
 require 'fluent/engine'
 require 'fluent/time'

From dae72688fb9040f475e5ec17b5c93230afc155ea Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 4 Mar 2016 21:27:30 +0900
Subject: [PATCH 086/193] v0.12.21

---
 ChangeLog             | 23 +++++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d6a373baca..2d4dda137f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,28 @@
 # v0.12
 
+## Release 0.12.21 - 2015/03/03
+
+### New features / Enhancement
+
+* output: Use error stream in TimeSlicedOutput#emit
+  https://github.com/fluent/fluentd/pull/791
+* in_tail: Avoid unnecessary open call
+  https://github.com/fluent/fluentd/pull/788
+* in_tail: Add enable_watch_timer option
+  https://github.com/fluent/fluentd/pull/811
+* filter_record_transformer: Relax conditions which auto_typecast is applied when enable_ruby yes
+  https://github.com/fluent/fluentd/pull/817
+* Backport v0.14 refactoring changes
+
+### Bug fixes
+
+* in_tail: Fix input messages loss when receive_lines fail with BufferQueueLimitError
+  https://github.com/fluent/fluentd/pull/810
+* in_monitor_agent: Remove unnecessary fields from API response with debug mode
+  https://github.com/fluent/fluentd/pull/802
+* filter_record_transformer: Fix record_transformer to be thread-safe
+  https://github.com/fluent/fluentd/pull/816
+
 ## Release 0.12.20 - 2015/01/26
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 0b0a8cf2d2..a7107cfa8c 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.20'
+  VERSION = '0.12.21'
 
 end

From 386ff3eea0e8a619e46825328bcd48a9630d7a32 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 11 Mar 2016 13:40:47 +0900
Subject: [PATCH 087/193] Merge pull request #845 from fluent/fix-missing-fcntl

Require fcntl library for daemonize. fix #841
---
 lib/fluent/supervisor.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index 961cae84be..e344876e7e 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -15,6 +15,7 @@
 #
 
 require 'etc'
+require 'fcntl'
 
 require 'fluent/config'
 require 'fluent/env'

From 282af784d20bdf0502d697d7c4810251cab2ca70 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 11 Mar 2016 14:09:05 +0900
Subject: [PATCH 088/193] v0.12.22

---
 ChangeLog             | 7 +++++++
 lib/fluent/version.rb | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 2d4dda137f..8e82b7b39d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
 # v0.12
 
+## Release 0.12.22 - 2015/03/10
+
+### Bug fixes
+
+* supervisor: Require fcntl library for daemonize
+  https://github.com/fluent/fluentd/pull/845
+
 ## Release 0.12.21 - 2015/03/03
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index a7107cfa8c..6f37a8bf58 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.21'
+  VERSION = '0.12.22'
 
 end

From a227d97f1bbac4df63dd03e4c6ef6b1b5d2e3cb5 Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Mon, 11 Apr 2016 15:50:16 +0900
Subject: [PATCH 089/193] Merge pull request #891 from
 cosmo0920/fix-broken-fluent-debug-command

fluent-debug: Fix broken fluent-debug command
---
 lib/fluent/command/debug.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/command/debug.rb b/lib/fluent/command/debug.rb
index 1210695337..254d6b59f0 100644
--- a/lib/fluent/command/debug.rb
+++ b/lib/fluent/command/debug.rb
@@ -62,9 +62,12 @@
 
 require 'fluent/log'
 require 'fluent/engine'
+require 'fluent/system_config'
+
+include Fluent::SystemConfig::Mixin
 
 $log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
-Fluent::Engine.init
+Fluent::Engine.init(system_config)
 
 DRb::DRbObject.class_eval do
   undef_method :methods

From 099ae2dafa0e35ad55618c98786295a14b251fc5 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 28 Apr 2016 15:08:12 +0900
Subject: [PATCH 090/193] Add test for log related class and module

* `Fluent::Log`
* `Fluent::PluginLogger`
* `Fluent::PluginLoggerMixin`

See #909
---
 test/test_log.rb | 340 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 340 insertions(+)
 create mode 100644 test/test_log.rb

diff --git a/test/test_log.rb b/test/test_log.rb
new file mode 100644
index 0000000000..cd95530415
--- /dev/null
+++ b/test/test_log.rb
@@ -0,0 +1,340 @@
+require_relative 'helper'
+require 'fluent/engine'
+require 'fluent/log'
+
+class LogTest < Test::Unit::TestCase
+  def setup
+    @log_device = Fluent::Test::DummyLogDevice.new
+    @timestamp = Time.parse("2016-04-21 11:58:41 +0900")
+    @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+    stub(Time).now { @timestamp }
+  end
+
+  def teardown
+    Thread.current[:last_repeated_stacktrace] = nil
+  end
+
+  sub_test_case "log level" do
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, 0],
+      debug: [Fluent::Log::LEVEL_DEBUG, 1],
+      info: [Fluent::Log::LEVEL_INFO, 2],
+      warn: [Fluent::Log::LEVEL_WARN, 3],
+      error: [Fluent::Log::LEVEL_ERROR, 4],
+      fatal: [Fluent::Log::LEVEL_FATAL, 5],
+    )
+    def test_output(data)
+      log_level, start = data
+      log = Fluent::Log.new(@log_device, log_level)
+      log.trace "trace log"
+      log.debug "debug log"
+      log.info "info log"
+      log.warn "warn log"
+      log.error "error log"
+      log.fatal "fatal log"
+      expected = [
+        "#{@timestamp_str} [trace]: trace log\n",
+        "#{@timestamp_str} [debug]: debug log\n",
+        "#{@timestamp_str} [info]: info log\n",
+        "#{@timestamp_str} [warn]: warn log\n",
+        "#{@timestamp_str} [error]: error log\n",
+        "#{@timestamp_str} [fatal]: fatal log\n"
+      ][start..-1]
+      assert_equal(expected, log.out.logs)
+    end
+
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, 0],
+      debug: [Fluent::Log::LEVEL_DEBUG, 1],
+      info: [Fluent::Log::LEVEL_INFO, 2],
+      warn: [Fluent::Log::LEVEL_WARN, 3],
+      error: [Fluent::Log::LEVEL_ERROR, 4],
+      fatal: [Fluent::Log::LEVEL_FATAL, 5],
+    )
+    def test_output_with_block(data)
+      log_level, start = data
+      log = Fluent::Log.new(@log_device, log_level)
+      log.trace { "trace log" }
+      log.debug { "debug log" }
+      log.info { "info log" }
+      log.warn { "warn log" }
+      log.error { "error log" }
+      log.fatal { "fatal log" }
+      expected = [
+        "#{@timestamp_str} [trace]: trace log\n",
+        "#{@timestamp_str} [debug]: debug log\n",
+        "#{@timestamp_str} [info]: info log\n",
+        "#{@timestamp_str} [warn]: warn log\n",
+        "#{@timestamp_str} [error]: error log\n",
+        "#{@timestamp_str} [fatal]: fatal log\n"
+      ][start..-1]
+      assert_equal(expected, log.out.logs)
+    end
+
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, { trace: true, debug: true, info: true, warn: true, error: true, fatal: true }],
+      debug: [Fluent::Log::LEVEL_DEBUG, { trace: false, debug: true, info: true, warn: true, error: true, fatal: true }],
+      info: [Fluent::Log::LEVEL_INFO, { trace: false, debug: false, info: true, warn: true, error: true, fatal: true }],
+      warn: [Fluent::Log::LEVEL_WARN, { trace: false, debug: false, info: false, warn: true, error: true, fatal: true }],
+      error: [Fluent::Log::LEVEL_ERROR, { trace: false, debug: false, info: false, warn: false, error: true, fatal: true }],
+      fatal: [Fluent::Log::LEVEL_FATAL, { trace: false, debug: false, info: false, warn: false, error: false, fatal: true }],
+    )
+    def test_execute_block(data)
+      log_level, expected = data
+      log = Fluent::Log.new(@log_device, log_level)
+      block_called = {
+        trace: false,
+        debug: false,
+        info: false,
+        warn: false,
+        error: false,
+        fatal: false,
+      }
+      log.trace { block_called[:trace] = true }
+      log.debug { block_called[:debug] = true }
+      log.info { block_called[:info] = true }
+      log.warn { block_called[:warn] = true }
+      log.error { block_called[:error] = true }
+      log.fatal { block_called[:fatal] = true }
+      assert_equal(expected, block_called)
+    end
+
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, 0],
+      debug: [Fluent::Log::LEVEL_DEBUG, 3],
+      info: [Fluent::Log::LEVEL_INFO, 6],
+      warn: [Fluent::Log::LEVEL_WARN, 9],
+      error: [Fluent::Log::LEVEL_ERROR, 12],
+      fatal: [Fluent::Log::LEVEL_FATAL, 15],
+    )
+    def test_backtrace(data)
+      log_level, start = data
+      backtrace = ["line 1", "line 2", "line 3"]
+      log = Fluent::Log.new(@log_device, log_level)
+      log.trace_backtrace(backtrace)
+      log.debug_backtrace(backtrace)
+      log.info_backtrace(backtrace)
+      log.warn_backtrace(backtrace)
+      log.error_backtrace(backtrace)
+      log.fatal_backtrace(backtrace)
+      expected = [
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [debug]: line 1\n",
+        "  #{@timestamp_str} [debug]: line 2\n",
+        "  #{@timestamp_str} [debug]: line 3\n",
+        "  #{@timestamp_str} [info]: line 1\n",
+        "  #{@timestamp_str} [info]: line 2\n",
+        "  #{@timestamp_str} [info]: line 3\n",
+        "  #{@timestamp_str} [warn]: line 1\n",
+        "  #{@timestamp_str} [warn]: line 2\n",
+        "  #{@timestamp_str} [warn]: line 3\n",
+        "  #{@timestamp_str} [error]: line 1\n",
+        "  #{@timestamp_str} [error]: line 2\n",
+        "  #{@timestamp_str} [error]: line 3\n",
+        "  #{@timestamp_str} [fatal]: line 1\n",
+        "  #{@timestamp_str} [fatal]: line 2\n",
+        "  #{@timestamp_str} [fatal]: line 3\n"
+      ][start..-1]
+      assert_equal(expected, log.out.logs)
+    end
+  end
+
+  sub_test_case "suppress repeated backtrace" do
+    def test_same_log_level
+      backtrace = ["line 1", "line 2", "line 3"]
+      log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE, suppress_repeated_stacktrace: true)
+      log.trace_backtrace(backtrace)
+      log.trace_backtrace(backtrace)
+      log.trace_backtrace(backtrace + ["line 4"])
+      log.trace_backtrace(backtrace)
+      log.trace_backtrace(backtrace)
+      expected = [
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [trace]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [trace]: line 4\n",
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [trace]: suppressed same stacktrace\n",
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+
+    def test_different_log_level
+      backtrace = ["line 1", "line 2", "line 3"]
+      log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE, suppress_repeated_stacktrace: true)
+      log.trace_backtrace(backtrace)
+      log.debug_backtrace(backtrace)
+      log.info_backtrace(backtrace)
+      log.warn_backtrace(backtrace)
+      log.error_backtrace(backtrace)
+      log.fatal_backtrace(backtrace)
+      expected = [
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [debug]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [info]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [warn]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [error]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [fatal]: suppressed same stacktrace\n",
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+  end
+
+  def test_dup
+    log1 = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE)
+    log2 = log1.dup
+    log1.level = Fluent::Log::LEVEL_DEBUG
+    original_tag = log1.tag
+    log1.tag = "changed"
+    assert_equal(Fluent::Log::LEVEL_DEBUG, log1.level)
+    assert_equal(Fluent::Log::LEVEL_TRACE, log2.level)
+    assert_equal("changed", log1.tag)
+    assert_equal(original_tag, log2.tag)
+  end
+
+  def test_disable_events
+    log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE)
+    engine = log.instance_variable_get("@engine")
+    mock(engine).push_log_event(anything, anything, anything).once
+    log.trace "trace log"
+    log.disable_events(Thread.current)
+    log.trace "trace log"
+  end
+end
+
+class PluginLoggerTest < Test::Unit::TestCase
+  def setup
+    @log_device = Fluent::Test::DummyLogDevice.new
+    @timestamp = Time.parse("2016-04-21 11:58:41 +0900")
+    @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+    stub(Time).now { @timestamp }
+    @logger = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE)
+  end
+
+  def teardown
+    Thread.current[:last_repeated_stacktrace] = nil
+  end
+
+  def test_initialize
+    log = Fluent::PluginLogger.new(@logger)
+    logger = log.instance_variable_get("@logger")
+    assert_equal(logger, @logger)
+  end
+
+  def test_level
+    log = Fluent::PluginLogger.new(@logger)
+    assert_equal(log.level, @logger.level)
+    log.level = "fatal"
+    assert_equal(Fluent::Log::LEVEL_FATAL, log.level)
+    assert_equal(Fluent::Log::LEVEL_TRACE, @logger.level)
+  end
+
+  def test_enable_color
+    log = Fluent::PluginLogger.new(@logger)
+    log.enable_color(true)
+    assert_equal(true, log.enable_color?)
+    assert_equal(true, @logger.enable_color?)
+    log.enable_color(false)
+    assert_equal(false, log.enable_color?)
+    assert_equal(false, @logger.enable_color?)
+    log.enable_color
+    assert_equal(true, log.enable_color?)
+    assert_equal(true, @logger.enable_color?)
+  end
+
+  sub_test_case "delegators" do
+    def setup
+      super
+      @log = Fluent::PluginLogger.new(@logger)
+    end
+
+    def test_enable_debug
+      mock(@logger).enable_debug
+      @log.enable_debug
+    end
+
+    def test_enable_event
+      mock(@logger).enable_event
+      @log.enable_event
+    end
+
+    def test_disable_events
+      mock(@logger).disable_events(Thread.current)
+      @log.disable_events(Thread.current)
+    end
+
+    def test_tag
+      assert_equal(@log.tag, @logger.tag)
+      @log.tag = "dummy"
+      assert_equal(@log.tag, @logger.tag)
+    end
+
+    def test_time_format
+      assert_equal(@log.time_format, @logger.time_format)
+      @log.time_format = "time_format"
+      assert_equal(@log.time_format, @logger.time_format)
+    end
+
+    def test_event
+      mock(@logger).event(Fluent::Log::LEVEL_TRACE, { key: "value" })
+      @log.event(Fluent::Log::LEVEL_TRACE, { key: "value" })
+    end
+
+    def test_caller_line
+      mock(@logger).caller_line(Time.now, 1, Fluent::Log::LEVEL_TRACE)
+      @log.caller_line(Time.now, 1, Fluent::Log::LEVEL_TRACE)
+    end
+
+    def test_puts
+      mock(@logger).puts("log")
+      @log.puts("log")
+    end
+
+    def test_write
+      mock(@logger).write("log")
+      @log.write("log")
+    end
+
+    def test_out
+      assert_equal(@log.out, @logger.out)
+      @log.out = Object.new
+      assert_equal(@log.out, @logger.out)
+    end
+  end
+end
+
+class PluginLoggerMixinTest < Test::Unit::TestCase
+  class DummyPlugin < Fluent::Input
+  end
+
+  def create_driver(conf)
+    Fluent::Test::TestDriver.new(DummyPlugin).configure(conf)
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_default_log
+    plugin = DummyPlugin.new
+    log = plugin.log
+    assert_equal($log, log)
+  end
+
+  def test_log_level
+    d = create_driver(%[log_level fatal])
+    log = d.instance.log
+    assert_not_equal($log.level, log.level)
+    assert_equal(Fluent::Log::LEVEL_FATAL, log.level)
+  end
+end

From 25a51a17efac932fa72778728ca891388c09ed19 Mon Sep 17 00:00:00 2001
From: toyama0919 <toyama0919@gmail.com>
Date: Wed, 20 Apr 2016 21:13:45 +0900
Subject: [PATCH 091/193] Fix logger on Fluent::StringUtil to use $log.

---
 lib/fluent/plugin/string_util.rb |  2 +-
 test/plugin/test_string_util.rb  | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 test/plugin/test_string_util.rb

diff --git a/lib/fluent/plugin/string_util.rb b/lib/fluent/plugin/string_util.rb
index 73e419aa50..1a82498007 100644
--- a/lib/fluent/plugin/string_util.rb
+++ b/lib/fluent/plugin/string_util.rb
@@ -21,7 +21,7 @@ def match_regexp(regexp, string)
         return regexp.match(string)
       rescue ArgumentError => e
         raise e unless e.message.index("invalid byte sequence in".freeze).zero?
-        log.info "invalid byte sequence is replaced in `#{string}`"
+        $log.info "invalid byte sequence is replaced in `#{string}`"
         string = string.scrub('?')
         retry
       end
diff --git a/test/plugin/test_string_util.rb b/test/plugin/test_string_util.rb
new file mode 100644
index 0000000000..faead3075d
--- /dev/null
+++ b/test/plugin/test_string_util.rb
@@ -0,0 +1,26 @@
+require_relative '../helper'
+require 'fluent/plugin/string_util'
+
+class StringUtilTest < Test::Unit::TestCase
+  def setup
+    @null_value_pattern = Regexp.new("^(-|null|NULL)$")
+  end
+
+  sub_test_case 'valid string' do
+    test 'null string' do
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "null").to_s, "null"
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "NULL").to_s, "NULL"
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "-").to_s, "-"
+    end
+
+    test 'normal string' do
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "fluentd"), nil
+    end
+  end
+
+  sub_test_case 'invalid string' do
+    test 'normal string' do
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "\xff"), nil
+    end
+  end
+end

From 49bdb09548823ef1a308a6c946d40a98ab546012 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Fri, 22 Apr 2016 13:55:27 +0900
Subject: [PATCH 092/193] fix assigning an object with unconsistent API

---
 test/plugin/test_filter_stdout.rb | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/test/plugin/test_filter_stdout.rb b/test/plugin/test_filter_stdout.rb
index 761d1083a3..b5bf502afe 100644
--- a/test/plugin/test_filter_stdout.rb
+++ b/test/plugin/test_filter_stdout.rb
@@ -1,7 +1,7 @@
 require_relative '../helper'
 require 'fluent/plugin/filter_stdout'
 require 'timecop'
-require 'flexmock'
+require 'flexmock/test_unit'
 
 class StdoutFilterTest < Test::Unit::TestCase
   include Fluent
@@ -13,6 +13,8 @@ def setup
   end
 
   def teardown
+    super # FlexMock::TestCase requires this
+    # http://flexmock.rubyforge.org/FlexMock/TestCase.html
     Timecop.return
   end
 
@@ -101,12 +103,12 @@ def test_format_json
 
   # Capture the log output of the block given
   def capture_log(d, &block)
-    tmp = d.instance.log
-    d.instance.log = StringIO.new
+    tmp = d.instance.log.out
+    d.instance.log.out = StringIO.new
     yield
-    return d.instance.log.string
+    return d.instance.log.out.string
   ensure
-    d.instance.log = tmp
+    d.instance.log.out = tmp
   end
 end
 

From a34f668bdf22fb2639bd9e2207414e1f3b2f2349 Mon Sep 17 00:00:00 2001
From: Ed Robinson <ed@reevoo.com>
Date: Wed, 4 May 2016 11:36:46 +0100
Subject: [PATCH 093/193] Fix string-scrub dependency on ruby > 2

string-scrub 0.1.0 dropped today, and the strict
ruby version constraint stops fluent from being
installed on a modern ruby.
---
 fluentd.gemspec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index ec7eb7a678..5b0de02fe3 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
   gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  gem.add_runtime_dependency("string-scrub", [">= 0.0.3"])
+  gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock", ["~> 1.3.3"])

From dcdbda2b0320526ad17a0601db2ab0ef8a46cb2a Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 4 May 2016 20:02:30 +0900
Subject: [PATCH 094/193] v0.12.23

---
 ChangeLog             | 15 ++++++++++++---
 lib/fluent/version.rb |  2 +-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e82b7b39d..b0abcb897b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,13 +1,22 @@
 # v0.12
 
-## Release 0.12.22 - 2015/03/10
+## Release 0.12.23 - 2016/05/03
+
+### Bug fixes
+
+* Fix string-scrub dependency on ruby > 2
+  https://github.com/fluent/fluentd/pull/933
+* fluent-debug: Fix regression
+  https://github.com/fluent/fluentd/pull/894
+
+## Release 0.12.22 - 2016/03/10
 
 ### Bug fixes
 
 * supervisor: Require fcntl library for daemonize
   https://github.com/fluent/fluentd/pull/845
 
-## Release 0.12.21 - 2015/03/03
+## Release 0.12.21 - 2016/03/03
 
 ### New features / Enhancement
 
@@ -30,7 +39,7 @@
 * filter_record_transformer: Fix record_transformer to be thread-safe
   https://github.com/fluent/fluentd/pull/816
 
-## Release 0.12.20 - 2015/01/26
+## Release 0.12.20 - 2016/01/26
 
 ### New features / Enhancement
 
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 6f37a8bf58..848c03b6b2 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.22'
+  VERSION = '0.12.23'
 
 end

From 2050bd793a28a0b8943fe82533aac6cefa027de6 Mon Sep 17 00:00:00 2001
From: varunkumta <varun.kumta@outlook.com>
Date: Wed, 4 May 2016 16:33:29 -0700
Subject: [PATCH 095/193] Add support for drop_oldest_chunk when buffer queue
 is full

- Drop oldest chunk from queue without writing to output when buffer
queue is full
- Add test
---
 lib/fluent/buffer.rb |  7 +++++--
 test/test_buffer.rb  | 18 ++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/buffer.rb b/lib/fluent/buffer.rb
index d7327ddcbd..ad4c138ddb 100644
--- a/lib/fluent/buffer.rb
+++ b/lib/fluent/buffer.rb
@@ -150,7 +150,7 @@ def enable_parallel(b=true)
     desc 'The length limit of the chunk queue.'
     config_param :buffer_queue_limit, :integer, default: 256
     desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
-    config_param :buffer_queue_full_action, :enum, list: [:exception, :block], default: :exception
+    config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
 
     alias chunk_limit buffer_chunk_limit
     alias chunk_limit= buffer_chunk_limit=
@@ -212,6 +212,9 @@ def emit(key, data, chain)
             $log.debug "buffer queue is full. Wait 1 second to re-emit events"
             sleep 1
             retry
+          when :drop_oldest_chunk
+            $log.debug "buffer queue is full. Dropping oldest chunk"
+            pop(nil)
           end
         end
 
@@ -326,7 +329,7 @@ def pop(out)
       begin
         # #push(key) does not push empty chunks into queue.
         # so this check is nonsense...
-        if !chunk.empty?
+        if !chunk.empty? && !out.nil?
           write_chunk(chunk, out)
         end
 
diff --git a/test/test_buffer.rb b/test/test_buffer.rb
index dcb9436a79..20a6b89668 100644
--- a/test/test_buffer.rb
+++ b/test/test_buffer.rb
@@ -589,6 +589,24 @@ def test_emit_with_block
         assert db.emit('key', data, chain)
       end
 
+      def test_emit_with_drop_oldest_chunk
+        db = dummy_buffer(:drop_oldest_chunk)
+      
+        assert !db.emit('key', data, chain)
+        assert db.emit('key', data, chain)
+        
+        # oldest (and only) chunk in queue is dropped before newer data is enqueued 
+        assert db.emit('key', data, chain)
+        
+        assert db.queue.size == 1
+
+        assert !pop_chunk(db)
+        assert db.queue.size == 0
+
+        # queue is now empty so can emit data again
+        assert db.emit('key', data, chain)
+      end
+
       def pop_chunk(db)
         out = DummyOutput.new
         c1 = DummyChunk.new('k1', 1)

From edc4354d7722fe4b21b52363982fcd1ae84eedf8 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 9 May 2016 10:12:58 +0900
Subject: [PATCH 096/193] Fix string-scrub dependency

`String#scrub` has been added since Ruby 2.1.
We don't need to install string-scrub gem when we use Ruby 2.1 or later.
---
 fluentd.gemspec | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index 5b0de02fe3..1e81fe7825 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -27,7 +27,9 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
   gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
+  unless "".respond_to?(:scrub)
+    gem.add_runtime_dependency("string-scrub", [">= 0.1.1"])
+  end
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock", ["~> 1.3.3"])

From 6e8e0edd8243d8220dec638e869c03bb7759c2f1 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 9 May 2016 10:15:00 +0900
Subject: [PATCH 097/193] Add missing require when we use Ruby 2.0 or 1.9.3

---
 lib/fluent/plugin/string_util.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/fluent/plugin/string_util.rb b/lib/fluent/plugin/string_util.rb
index 73e419aa50..81496d934e 100644
--- a/lib/fluent/plugin/string_util.rb
+++ b/lib/fluent/plugin/string_util.rb
@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require "string-scrub" unless "".respond_to?(:scrub)
+
 module Fluent
   module StringUtil
     def match_regexp(regexp, string)

From 93661a12a896e25b473c12b474bdb02ad6e1ad4f Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Thu, 31 Mar 2016 14:19:30 +0900
Subject: [PATCH 098/193] Add simplified syntax for hash and array  * to
 specify values with very simple comma (and colon for key-value separator)  *
 to re-format values because comma-separated values doesn't have type
 information

It looks to satisfy various use-cases of 3rd party plugins, and this change is just additional
change to allow format which is currently denied as ConfigError.
---
 lib/fluent/config/types.rb | 45 ++++++++++++++++++++++++++++++++++----
 test/config/test_types.rb  | 29 ++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
index b295ef5816..674d7fecc8 100644
--- a/lib/fluent/config/types.rb
+++ b/lib/fluent/config/types.rb
@@ -80,19 +80,56 @@ def self.bool_value(str)
     SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
     BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
     TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+
+    REFORMAT_VALUES = ->(type, value) {
+      if value.nil?
+        value
+      else
+        case type
+        when :string  then value.to_s
+        when :integer then value.to_i
+        when :float   then value.to_f
+        when :size then Config.size_value(value)
+        when :bool then Config.bool_value(value)
+        when :time then Config.time_value(value)
+        end
+      end
+    }
+
     HASH_TYPE = Proc.new { |val, opts|
-      param = val.is_a?(String) ? JSON.load(val) : val
+      param = if val.is_a?(String)
+                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+              else
+                val
+              end
       if param.class != Hash
         raise ConfigError, "hash required but got #{val.inspect}"
       end
-      param
+      if opts.empty?
+        param
+      else
+        newparam = {}
+        param.each_pair do |key, value|
+          new_key = opts[:symbolize_keys] ? key.to_sym : key
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUES.call(opts[:value_type], value) : value
+        end
+        newparam
+      end
     }
     ARRAY_TYPE = Proc.new { |val, opts|
-      param = val.is_a?(String) ? JSON.load(val) : val
+      param = if val.is_a?(String)
+                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+              else
+                val
+              end
       if param.class != Array
         raise ConfigError, "array required but got #{val.inspect}"
       end
-      param
+      if opts[:type]
+        param.map{|v| REFORMAT_VALUES.call(opts[:type], v) }
+      else
+        param
+      end
     }
   end
 
diff --git a/test/config/test_types.rb b/test/config/test_types.rb
index 95c544bc63..b73b3d86db 100644
--- a/test/config/test_types.rb
+++ b/test/config/test_types.rb
@@ -124,10 +124,39 @@ class TestConfigTypes < ::Test::Unit::TestCase
 
     test 'hash' do
       assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {}))
+
+      assert_equal({x: "v", k: 1},   Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true}))
+      assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('x:v,k:1',         {symbolize_keys: true, value_type: :string}))
+      assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true, value_type: :string}))
+      assert_equal({x: 0, k: 1},     Config::HASH_TYPE.call('x:0,k:1',         {symbolize_keys: true, value_type: :integer}))
+
+      assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}))
+      assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('x:1s,y:1m,z:1h',               {value_type: :time}))
     end
 
     test 'array' do
       assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
+      assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))
+
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('["a","b","c"]', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a,b,c', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a, b, c', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a , b , c', {}))
+
+      assert_equal(["a a","b,b"," c "], Config::ARRAY_TYPE.call('["a a","b,b"," c "]', {}))
+
+      assert_equal(["a a","b","c"], Config::ARRAY_TYPE.call('a a,b,c', {}))
+
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('[1,2,1]', {}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {type: :integer}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {type: :integer}))
+
+      array_options = {
+        default: [],
+      }
+      assert_equal(["1","2"], Config::ARRAY_TYPE.call('["1","2"]', array_options))
+      assert_equal(["3"], Config::ARRAY_TYPE.call('["3"]', array_options))
     end
   end
 end

From 87e17d981f450d5cfa8b6235d518b223f6f63dfe Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Thu, 31 Mar 2016 18:22:33 +0900
Subject: [PATCH 099/193] Fix option name from type to value_type (type is
 already used by param's type itself)

---
 lib/fluent/config/types.rb | 6 ++++--
 test/config/test_types.rb  | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
index 674d7fecc8..35907413c1 100644
--- a/lib/fluent/config/types.rb
+++ b/lib/fluent/config/types.rb
@@ -92,6 +92,8 @@ def self.bool_value(str)
         when :size then Config.size_value(value)
         when :bool then Config.bool_value(value)
         when :time then Config.time_value(value)
+        else
+          raise "unknown type in REFORMAT: #{type}"
         end
       end
     }
@@ -125,8 +127,8 @@ def self.bool_value(str)
       if param.class != Array
         raise ConfigError, "array required but got #{val.inspect}"
       end
-      if opts[:type]
-        param.map{|v| REFORMAT_VALUES.call(opts[:type], v) }
+      if opts[:value_type]
+        param.map{|v| REFORMAT_VALUES.call(opts[:value_type], v) }
       else
         param
       end
diff --git a/test/config/test_types.rb b/test/config/test_types.rb
index b73b3d86db..88e2b67890 100644
--- a/test/config/test_types.rb
+++ b/test/config/test_types.rb
@@ -149,8 +149,8 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal(["a a","b","c"], Config::ARRAY_TYPE.call('a a,b,c', {}))
 
       assert_equal([1,2,1], Config::ARRAY_TYPE.call('[1,2,1]', {}))
-      assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {type: :integer}))
-      assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {type: :integer}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {value_type: :integer}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {value_type: :integer}))
 
       array_options = {
         default: [],

From 1a2638af0685c09d79224c662d71c4d53959d5f6 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Thu, 31 Mar 2016 18:26:10 +0900
Subject: [PATCH 100/193] add test cases for LCSV

---
 test/config/test_types.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/config/test_types.rb b/test/config/test_types.rb
index 88e2b67890..ffd96b2bc2 100644
--- a/test/config/test_types.rb
+++ b/test/config/test_types.rb
@@ -125,6 +125,11 @@ class TestConfigTypes < ::Test::Unit::TestCase
     test 'hash' do
       assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {}))
       assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v, k:1', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call(' x:v, k:1 ', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v , k:1 ', {}))
+
+      assert_equal({"x"=>"v:v","k"=>"1"}, Config::HASH_TYPE.call('x:v:v, k:1', {}))
 
       assert_equal({x: "v", k: 1},   Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true}))
       assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('x:v,k:1',         {symbolize_keys: true, value_type: :string}))

From 2188635082fbd7d2003ed310665698887f1ba104 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Mon, 4 Apr 2016 13:08:51 +0900
Subject: [PATCH 101/193] fix misleading name

---
 lib/fluent/config/types.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
index 35907413c1..85499c9910 100644
--- a/lib/fluent/config/types.rb
+++ b/lib/fluent/config/types.rb
@@ -81,7 +81,7 @@ def self.bool_value(str)
     BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
     TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
 
-    REFORMAT_VALUES = ->(type, value) {
+    REFORMAT_VALUE = ->(type, value) {
       if value.nil?
         value
       else
@@ -113,7 +113,7 @@ def self.bool_value(str)
         newparam = {}
         param.each_pair do |key, value|
           new_key = opts[:symbolize_keys] ? key.to_sym : key
-          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUES.call(opts[:value_type], value) : value
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value) : value
         end
         newparam
       end
@@ -128,7 +128,7 @@ def self.bool_value(str)
         raise ConfigError, "array required but got #{val.inspect}"
       end
       if opts[:value_type]
-        param.map{|v| REFORMAT_VALUES.call(opts[:value_type], v) }
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v) }
       else
         param
       end

From 4e0032f0536ff0418e9e6192e3d58f26caba8a07 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Wed, 27 Apr 2016 17:38:13 +0900
Subject: [PATCH 102/193] add error class definition which is used but not
 defined yet

---
 lib/fluent/plugin/out_forward.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 57d747095a..73858b1952 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -30,6 +30,9 @@ class ForwardOutputError < StandardError
   class ForwardOutputResponseError < ForwardOutputError
   end
 
+  class ForwardOutputConnectionClosedError < ForwardOutputError
+  end
+
   class ForwardOutputACKTimeoutError < ForwardOutputResponseError
   end
 

From 1665c549ffc700ae1f55b6cfd5d2a3778aca5ba1 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 5 Apr 2016 21:16:00 +0900
Subject: [PATCH 103/193] Merge pull request #882 from
 HatsuneMiku3939/in-http-add-access-control-allow-origin

Add 'Access-Control-Allow-Origin' to in_http plugin
---
 lib/fluent/plugin/in_http.rb |  1 +
 test/plugin/test_in_http.rb  | 74 ++++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)

diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index aee6c3a8fa..58c804064f 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -362,6 +362,7 @@ def on_message_complete
         code, header, body = *@callback.call(path_info, params)
         body = body.to_s
 
+        header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
         if @keep_alive
           header['Connection'] = 'Keep-Alive'
           send_response(code, header, body)
diff --git a/test/plugin/test_in_http.rb b/test/plugin/test_in_http.rb
index 1861f0cb63..26ad19d4cd 100644
--- a/test/plugin/test_in_http.rb
+++ b/test/plugin/test_in_http.rb
@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_http'
 require 'net/http'
 
 class HttpInputTest < Test::Unit::TestCase
@@ -269,6 +270,79 @@ def test_resonse_with_empty_img
     end
   end
 
+  def test_cors_allowed
+    d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
+    assert_equal ["http://foo.com"], d.instance.cors_allow_origins
+
+    test_in_http_cros_allowed = nil
+    acao = nil
+
+    begin
+      d.run do
+        Net::HTTP.start("127.0.0.1", PORT) do |http|
+          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://foo.com"})
+          res = http.request(req)
+
+          acao = res["Access-Control-Allow-Origin"]
+        end
+      end
+      test_in_http_cros_allowed = true
+    rescue
+      test_in_http_cros_allowed = false
+    end
+
+    assert_equal true, test_in_http_cros_allowed
+    assert_equal "http://foo.com", acao
+  end
+
+  def test_cors_disallowed
+    d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
+    assert_equal ["http://foo.com"], d.instance.cors_allow_origins
+
+    test_in_http_cros_disallowed = nil
+    response_code = nil
+
+    begin
+      d.run do
+        Net::HTTP.start("127.0.0.1", PORT) do |http|
+          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://bar.com"})
+          res = http.request(req)
+
+          response_code = res.code
+        end
+      end
+      test_in_http_cros_disallowed = true
+    rescue
+      test_in_http_cros_disallowed = false
+    end
+
+    assert_equal true, test_in_http_cros_disallowed
+    assert_equal "403", response_code
+  end
+
+  $test_in_http_connection_object_ids = []
+  $test_in_http_content_types = []
+  $test_in_http_content_types_flag = false
+  module ContentTypeHook
+    def on_headers_complete(headers)
+      super
+      if $test_in_http_content_types_flag
+        $test_in_http_content_types << self.content_type
+      end
+    end
+
+    def on_message_begin
+      super
+      if $test_in_http_content_types_flag
+        $test_in_http_connection_object_ids << @io_handler.object_id
+      end
+    end
+  end
+
+  class Fluent::HttpInput::Handler
+    prepend ContentTypeHook
+  end
+
   def test_if_content_type_is_initialized_properly
     # This test is to check if Fluent::HttpInput::Handler's @content_type is initialized properly.
     # Especially when in Keep-Alive and the second request has no 'Content-Type'.

From 1c923cef212920a420b5f359edce449b9d2780c6 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 20 Apr 2016 18:10:40 +0900
Subject: [PATCH 104/193] Merge pull request #889 from
 miyakawataku/encoding-param-for-tail-plugin

Add "encoding" parameter to in_tail plugin
---
 lib/fluent/plugin/in_tail.rb | 10 ++++++
 test/plugin/test_in_tail.rb  | 59 ++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 7740c2cb7f..2809acf552 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -50,6 +50,14 @@ def initialize
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'The encoding of the input.'
+    config_param :encoding, default: nil do |encoding_name|
+      begin
+        Encoding.find(encoding_name)
+      rescue ArgumentError => e
+        raise ConfigError, e.message
+      end
+    end
 
     attr_reader :paths
 
@@ -222,6 +230,7 @@ def close_watcher_after_rotate_wait(tw)
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
+        lb.force_encoding(@encoding) if @encoding
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -269,6 +278,7 @@ def receive_lines(lines, tail_watcher)
     def convert_line_to_event(line, es)
       begin
         line.chomp!  # remove \n
+        line.force_encoding(@encoding) if @encoding
         @parser.parse(line) { |time, record|
           if time && record
             es.add(time, record)
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 18beeff676..9a528aed77 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -51,6 +51,17 @@ def test_configure
     assert_equal 1000, d.instance.read_lines_limit
   end
 
+  def test_configure_encoding
+    # valid encoding
+    d = create_driver(SINGLE_LINE_CONFIG + 'encoding utf-8')
+    assert_equal Encoding::UTF_8, d.instance.encoding
+
+    # invalid encoding
+    assert_raise(Fluent::ConfigError) do
+      create_driver(SINGLE_LINE_CONFIG + 'encoding no-such-encoding')
+    end
+  end
+
   # TODO: Should using more better approach instead of sleep wait
 
   def test_emit
@@ -300,6 +311,27 @@ def test_whitespace
     assert_equal({"message" => "tab	"}, emits[5][2])
   end
 
+  data(
+    'default encoding' => ['', Encoding::ASCII_8BIT],
+    'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
+  def test_encoding(data)
+    encoding_config, encoding = data
+
+    d = create_driver(SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD + encoding_config)
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(encoding, emits[0][2]['message'].encoding)
+  end
+
   # multiline mode test
 
   def test_multiline
@@ -377,6 +409,33 @@ def test_multiline_with_flush_interval
     end
   end
 
+  data(
+    'default encoding' => ['', Encoding::ASCII_8BIT],
+    'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
+  def test_multiline_encoding_of_flushed_record(data)
+    encoding_config, encoding = data
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      multiline_flush_interval 2s
+      read_from_head true
+      #{encoding_config}
+    ]
+
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
+        f.puts "s test"
+      }
+
+      sleep 4
+      emits = d.emits
+      assert_equal(1, emits.length)
+      assert_equal(encoding, emits[0][2]['message1'].encoding)
+    end
+  end
+
   def test_multiline_with_multiple_formats
     File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
 

From 31c7d011eaefd41e85a0493ecba759ce67eeb8ae Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 21 Apr 2016 05:49:43 +0900
Subject: [PATCH 105/193] Merge pull request #908 from yuuki1224/patch-2

Fix typo
---
 lib/fluent/registry.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/fluent/registry.rb b/lib/fluent/registry.rb
index 2473090705..837660a210 100644
--- a/lib/fluent/registry.rb
+++ b/lib/fluent/registry.rb
@@ -40,7 +40,7 @@ def lookup(type)
       if value = @map[type]
         return value
       end
-      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluentd-plugin' to find plugins"  # TODO error class
+      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"  # TODO error class
     end
 
     def search(type)

From 2c461a1f14cc19e6fc1371482ff071c392cf826e Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 16 May 2016 14:06:33 +0900
Subject: [PATCH 106/193] Merge pull request #951 from fluent/tail_path

Add tail_path option to in_tail plugin
---
 lib/fluent/plugin/in_tail.rb |  12 ++--
 test/plugin/test_in_tail.rb  | 119 ++++++++++++++++++++++++++++++++++-
 2 files changed, 126 insertions(+), 5 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 2809acf552..78673f1803 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -58,6 +58,8 @@ def initialize
         raise ConfigError, e.message
       end
     end
+    desc 'Add the log path being tailed to records. Specify the field name to be used.'
+    config_param :path_key, :string, default: nil
 
     attr_reader :paths
 
@@ -238,6 +240,7 @@ def flush_buffer(tw)
                   else
                     @tag
                   end
+            record[@path_key] ||= tw.path unless @path_key.nil?
             router.emit(tag, time, record)
           else
             log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
@@ -275,12 +278,13 @@ def receive_lines(lines, tail_watcher)
       return true
     end
 
-    def convert_line_to_event(line, es)
+    def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
         line.force_encoding(@encoding) if @encoding
         @parser.parse(line) { |time, record|
           if time && record
+            record[@path_key] ||= tail_watcher.path unless @path_key.nil?
             es.add(time, record)
           else
             log.warn "pattern not match: #{line.inspect}"
@@ -295,7 +299,7 @@ def convert_line_to_event(line, es)
     def parse_singleline(lines, tail_watcher)
       es = MultiEventStream.new
       lines.each { |line|
-        convert_line_to_event(line, es)
+        convert_line_to_event(line, es, tail_watcher)
       }
       es
     end
@@ -308,7 +312,7 @@ def parse_multilines(lines, tail_watcher)
         lines.each { |line|
           if @parser.firstline?(line)
             if lb
-              convert_line_to_event(lb, es)
+              convert_line_to_event(lb, es, tail_watcher)
             end
             lb = line
           else
@@ -325,7 +329,7 @@ def parse_multilines(lines, tail_watcher)
           lb << line
           @parser.parse(lb) { |time, record|
             if time && record
-              convert_line_to_event(lb, es)
+              convert_line_to_event(lb, es, tail_watcher)
               lb = ''
             end
           }
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 9a528aed77..5325f29d95 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -1,7 +1,7 @@
 require_relative '../helper'
 require 'fluent/test'
 require 'net/http'
-require 'flexmock'
+require 'flexmock/test_unit'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -739,4 +739,121 @@ def execute_test(error_class, error_message)
       d.emits
     end
   end
+
+  sub_test_case "tail_path" do
+    def test_tail_path_with_singleline
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(%[path_key path] + SINGLE_LINE_CONFIG)
+
+      d.run do
+        sleep 1
+
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      emits.each do |emit|
+        assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
+      end
+    end
+
+    def test_tail_path_with_multiline_with_firstline
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+      d = create_driver %[
+        path_key path
+        format multiline
+        format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+        format_firstline /^[s]/
+      ]
+      d.run do
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert(emits.length == 4)
+      emits.each do |emit|
+        assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
+      end
+    end
+
+    def test_tail_path_with_multiline_without_firstline
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+      d = create_driver %[
+        path_key path
+        format multiline
+        format1 /(?<var1>foo \\d)\\n/
+        format2 /(?<var2>bar \\d)\\n/
+        format3 /(?<var3>baz \\d)/
+      ]
+      d.run do
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+          f.puts "foo 1"
+          f.puts "bar 1"
+          f.puts "baz 1"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert(emits.length > 0)
+      emits.each do |emit|
+        assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
+      end
+    end
+
+    def test_tail_path_with_multiline_with_multiple_paths
+      files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
+      files.each { |file| File.open(file, "wb") { |f| } }
+
+      d = create_driver(%[
+        path #{files[0]},#{files[1]}
+        path_key path
+        tag t1
+        format multiline
+        format1 /^[s|f] (?<message>.*)/
+        format_firstline /^[s]/
+      ], false)
+      d.run do
+        files.each do |file|
+          File.open(file, 'ab') { |f|
+            f.puts "f #{file} line should be ignored"
+            f.puts "s test1"
+            f.puts "f test2"
+            f.puts "f test3"
+            f.puts "s test4"
+          }
+        end
+        sleep 1
+      end
+
+      emits = d.emits
+      assert(emits.length == 4)
+      assert_equal("#{TMP_DIR}/tail1.txt", emits[0][2]["path"])
+      assert_equal("#{TMP_DIR}/tail2.txt", emits[1][2]["path"])
+      # "test4" events are here because these events are flushed at shutdown phase
+      assert_equal("#{TMP_DIR}/tail1.txt", emits[2][2]["path"])
+      assert_equal("#{TMP_DIR}/tail2.txt", emits[3][2]["path"])
+    end
+  end
 end

From 688db777271db43d1617503837c31451d6a9274b Mon Sep 17 00:00:00 2001
From: varunkumta <varun.kumta@outlook.com>
Date: Tue, 17 May 2016 14:58:06 -0700
Subject: [PATCH 107/193] Add support for emit_config to in_monitor_agent

Add support for emit_config to in_monitor_agent
---
 lib/fluent/plugin/in_monitor_agent.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index 68d10d9be5..bfd7ea9238 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -32,6 +32,7 @@ class MonitorAgentInput < Input
     config_param :port, :integer, default: 24220
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
+    config_param :emit_config, :bool, default: false
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -253,7 +254,7 @@ def start
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
         @loop = Coolio::Loop.new
-        opts = {with_config: false}
+        opts = {with_config: @emit_config}
         timer = TimerWatcher.new(@emit_interval, log) {
           es = MultiEventStream.new
           now = Engine.now

From d1b590243a5d8cedf676976a9bb6dc7dc7949dfe Mon Sep 17 00:00:00 2001
From: Naotoshi Seo <sonots@gmail.com>
Date: Mon, 16 May 2016 21:18:30 +0900
Subject: [PATCH 108/193] Merge pull request #957 from fluent/fix_test_in_tail

fix unstable test_in_tail
---
 test/plugin/test_in_tail.rb | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 5325f29d95..7688c7f94e 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -849,11 +849,9 @@ def test_tail_path_with_multiline_with_multiple_paths
 
       emits = d.emits
       assert(emits.length == 4)
-      assert_equal("#{TMP_DIR}/tail1.txt", emits[0][2]["path"])
-      assert_equal("#{TMP_DIR}/tail2.txt", emits[1][2]["path"])
+      assert_equal(files, [emits[0][2]["path"], emits[1][2]["path"]].sort)
       # "test4" events are here because these events are flushed at shutdown phase
-      assert_equal("#{TMP_DIR}/tail1.txt", emits[2][2]["path"])
-      assert_equal("#{TMP_DIR}/tail2.txt", emits[3][2]["path"])
+      assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
     end
   end
 end

From 94835ebc352a2f00e96791676387a271201fb54f Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 20 May 2016 06:36:31 +0900
Subject: [PATCH 109/193] travis-ci: Move 1.9.3 test to allow_failures

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 0a74ee88b7..9c79bb8e12 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1
   - 2.2.3
@@ -32,3 +31,4 @@ matrix:
     - rvm: ruby-head
     - rvm: rbx-2
     - rvm: 2.3.0
+    - rvm: 1.9.3

From b7415c2dc64b905bbe7cd976fe56434feacc1e95 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 10 May 2016 17:39:58 +0900
Subject: [PATCH 110/193] Merge pull request #929 from
 llby/support-multiline-string-in-quoted-string

[#772]  Support multiline string in " quoted string
---
 lib/fluent/config/basic_parser.rb   |  5 +++++
 lib/fluent/config/literal_parser.rb |  5 +++++
 test/config/test_config_parser.rb   | 35 +++++++++++++++++++++++++++++
 test/config/test_literal_parser.rb  |  2 ++
 4 files changed, 47 insertions(+)

diff --git a/lib/fluent/config/basic_parser.rb b/lib/fluent/config/basic_parser.rb
index 2a92f6b9b7..01b04e5228 100644
--- a/lib/fluent/config/basic_parser.rb
+++ b/lib/fluent/config/basic_parser.rb
@@ -28,6 +28,7 @@ def initialize(strscan)
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
       ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
+      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/
 
       module ClassMethods
         def symbol(string)
@@ -71,6 +72,10 @@ def prev_match
         @ss[0]
       end
 
+      def check(pattern)
+        @ss.check(pattern)
+      end
+
       def line_end
         skip(LINE_END)
       end
diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb
index 403b7a12ee..67db779461 100644
--- a/lib/fluent/config/literal_parser.rb
+++ b/lib/fluent/config/literal_parser.rb
@@ -81,6 +81,11 @@ def scan_double_quoted_string
         while true
           if skip(/\"/)
             return string.join
+          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+              string << s
+            end
+            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
           elsif s = scan(/\\./)
             string << eval_escape_char(s[1,1])
           elsif skip(/\#\{/)
diff --git a/test/config/test_config_parser.rb b/test/config/test_config_parser.rb
index e1529515c4..7e85aeaee2 100644
--- a/test/config/test_config_parser.rb
+++ b/test/config/test_config_parser.rb
@@ -5,6 +5,7 @@
 require "fluent/config/basic_parser"
 require "fluent/config/literal_parser"
 require "fluent/config/v1_parser"
+require 'fluent/config/parser'
 
 module Fluent::Config
   module V1TestHelper
@@ -115,6 +116,7 @@ def parse_text(text)
         test "requires escaping double quote" do
           assert_text_parsed_as(e('ROOT', '', {"k1" => '"'}), '  k1 "\\""')
           assert_parse_error('  k1 """')
+          assert_parse_error('  k1 ""\'')
         end
 
         test "removes backslash in front of a normal character" do
@@ -124,6 +126,39 @@ def parse_text(text)
         test "accepts escape characters" do
           assert_text_parsed_as(e('ROOT', '', {"k1" => "\n"}), '  k1 "\\n"')
         end
+
+        test "support multiline string" do
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1
+                       line2]
+            }),
+            %[k1      "line1
+                       line2"]
+          )
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1                       line2]
+            }),
+            %[k1      "line1\\
+                       line2"]
+          )
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1
+                       line2
+                       line3]
+            }),
+            %[k1      "line1
+                       line2
+                       line3"]
+          )
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1
+                       line2                       line3]
+            }),
+            %[k1      "line1
+                       line2\\
+                       line3"]
+          )
+        end
       end
 
       sub_test_case 'single quoted string' do
diff --git a/test/config/test_literal_parser.rb b/test/config/test_literal_parser.rb
index 75218b77aa..3e495fe332 100644
--- a/test/config/test_literal_parser.rb
+++ b/test/config/test_literal_parser.rb
@@ -109,6 +109,8 @@ def test_falseX
       test('"\\0"') { assert_parse_error('"\\0"') }  # unknown escaped character
       test('"\\1"') { assert_parse_error('"\\1"') }  # unknown escaped character
       test('"t') { assert_parse_error('"t') }  # non-terminated quoted character
+      test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string
+      test("\"t\\\nt\"") { assert_text_parsed_as("tt", "\"t\\\nt\"" ) } # multiline string
       test('t"') { assert_text_parsed_as('t"', 't"') }
       test('"."') { assert_text_parsed_as('.', '"."') }
       test('"*"') { assert_text_parsed_as('*', '"*"') }

From 5e491f8f9dd93a8ed359f10d612272e7fb835ca2 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 20 May 2016 11:49:23 +0900
Subject: [PATCH 111/193] v0.12.24

---
 ChangeLog             | 24 ++++++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index b0abcb897b..11055eb510 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,29 @@
 # v0.12
 
+## Release 0.12.24 - 2016/05/20
+
+### New features / Enhancement
+
+* buffer: Add drop_oldest_chunk to buffer_queue_full_action parameter
+  https://github.com/fluent/fluentd/pull/934
+* in_tail: Add "encoding" parameter
+  https://github.com/fluent/fluentd/pull/889
+* in_tail: Add "path_key" patameter
+  https://github.com/fluent/fluentd/pull/951
+* in_monitor_agent: Add emit_config parameter to emit plugin configuration together
+  https://github.com/fluent/fluentd/pull/963
+* config: Support multiline string in " quoted string
+  https://github.com/fluent/fluentd/pull/929
+
+### Bug fixes
+
+* in_http: Add 'Access-Control-Allow-Origin' to response header
+  https://github.com/fluent/fluentd/pull/882
+* parser: Fix logger on Fluent::StringUtil to use $log
+  https://github.com/fluent/fluentd/pull/926
+* out_forward: Add missing error class definition
+  https://github.com/fluent/fluentd/pull/953
+
 ## Release 0.12.23 - 2016/05/03
 
 ### Bug fixes
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 848c03b6b2..429287b3fa 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.23'
+  VERSION = '0.12.24'
 
 end

From 9335556ef5268c3ab27299525b633fe4a8857879 Mon Sep 17 00:00:00 2001
From: Chris Broglie <cbroglie@zynga.com>
Date: Mon, 23 May 2016 14:01:27 -0700
Subject: [PATCH 112/193] Make the detach process forward interval configurable

Setting this to a value less than the magic number of 0.2 fixes the
issue in #930
---
 lib/fluent/process.rb | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index 87151f8a48..8cb488e935 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -40,7 +40,7 @@ def initialize
       @parent_uri = DRb.uri
     end
 
-    def fork(delegate_object)
+    def fork(forward_interval, delegate_object)
       ipr, ipw = IO.pipe  # child Engine.emit_stream -> parent Engine.emit_stream
       opr, opw = IO.pipe  # parent target.emit -> child target.emit
 
@@ -49,14 +49,14 @@ def fork(delegate_object)
         # parent process
         ipw.close
         opr.close
-        forward_thread = process_parent(ipr, opw, pid, delegate_object)
+        forward_thread = process_parent(ipr, opw, pid, forward_interval, delegate_object)
         return pid, forward_thread
       end
 
       # child process
       ipr.close
       opw.close
-      forward_thread = process_child(ipw, opr, delegate_object)
+      forward_thread = process_child(ipw, opr, forward_interval, delegate_object)
       return nil, forward_thread
     end
 
@@ -77,14 +77,14 @@ def create_drb_uri
     end
 
     private
-    def process_child(ipw, opr, delegate_object)
+    def process_child(ipw, opr, forward_interval, delegate_object)
       DRb.start_service(create_drb_uri, delegate_object)
       child_uri = DRb.uri
 
       send_header(ipw, child_uri)
 
       # override target.emit_stream to write event stream to the pipe
-      fwd = new_forwarder(ipw, 0.5)  # TODO interval
+      fwd = new_forwarder(ipw, forward_interval)
       Engine.define_singleton_method(:emit_stream) do |tag,es|
         fwd.emit(tag, es)
       end
@@ -115,7 +115,7 @@ def shared_methods
       ]
     end
 
-    def process_parent(ipr, opw, pid, delegate_object)
+    def process_parent(ipr, opw, pid, forward_interval, delegate_object)
       child_uri = read_header(ipr)
 
       # read event stream from the pipe and forward to Engine.emit_stream
@@ -127,7 +127,7 @@ def process_parent(ipr, opw, pid, delegate_object)
 
       # return forwarder for DetachProcessMixin to
       # override target.emit and write event stream to the pipe
-      fwd = new_forwarder(opw, 0.5)  # TODO interval
+      fwd = new_forwarder(opw, forward_interval)
       # note: override emit method on DetachProcessMixin
       forward_thread.define_singleton_method(:forwarder) do
         fwd
@@ -300,11 +300,11 @@ def on_exit_process(i)
 
     private
 
-    def detach_process_impl(num, &block)
+    def detach_process_impl(num, forward_interval, &block)
       children = []
 
       num.times do |i|
-        pid, forward_thread = DetachProcessManager.instance.fork(self)
+        pid, forward_thread = DetachProcessManager.instance.fork(forward_interval, self)
 
         if pid
           # parent process
@@ -438,11 +438,17 @@ def configure(conf)
           @detach_process = false
         end
       end
+
+      if forward_interval = conf['detach_process_forward_interval']
+        @forward_interval = forward_interval.to_f
+      else
+        @forward_interval = 0.5
+      end
     end
 
     def detach_process(&block)
       if @detach_process
-        detach_process_impl(1, &block)
+        detach_process_impl(1, @forward_interval, &block)
       else
         block.call
       end
@@ -480,13 +486,19 @@ def configure(conf)
           @detach_process = false
         end
       end
+
+      if forward_interval = conf['detach_process_forward_interval']
+        @forward_interval = forward_interval.to_f
+      else
+        @forward_interval = 0.5
+      end
     end
 
     private
 
     def detach_multi_process(&block)
       if @detach_process
-        detach_process_impl(@detach_process_num, &block)
+        detach_process_impl(@detach_process_num, @forward_interval, &block)
       else
         block.call
       end

From 91bf6821fc90e339479711e2fd7dfa03a26ccba8 Mon Sep 17 00:00:00 2001
From: sonots <sonots@gmail.com>
Date: Tue, 24 May 2016 22:50:36 +0900
Subject: [PATCH 113/193] Revert: Fix string-scrub dependency #937 (only
 gemspec)

---
 fluentd.gemspec | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index 1e81fe7825..5b0de02fe3 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -27,9 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
   gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  unless "".respond_to?(:scrub)
-    gem.add_runtime_dependency("string-scrub", [">= 0.1.1"])
-  end
+  gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock", ["~> 1.3.3"])

From 6059d56e004348145eb97ae6c6dbe1e6a4522746 Mon Sep 17 00:00:00 2001
From: Chris Broglie <cbroglie@zynga.com>
Date: Tue, 24 May 2016 10:27:12 -0700
Subject: [PATCH 114/193] Use Config.time_value instead of to_f

---
 lib/fluent/process.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/process.rb b/lib/fluent/process.rb
index 8cb488e935..48d08fd92e 100644
--- a/lib/fluent/process.rb
+++ b/lib/fluent/process.rb
@@ -440,7 +440,7 @@ def configure(conf)
       end
 
       if forward_interval = conf['detach_process_forward_interval']
-        @forward_interval = forward_interval.to_f
+        @forward_interval = Config.time_value(forward_interval)
       else
         @forward_interval = 0.5
       end
@@ -488,7 +488,7 @@ def configure(conf)
       end
 
       if forward_interval = conf['detach_process_forward_interval']
-        @forward_interval = forward_interval.to_f
+        @forward_interval = Config.time_value(forward_interval)
       else
         @forward_interval = 0.5
       end

From bd53e876d2a81b6373b4ccfe99d5fe1b679cbb02 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 25 May 2016 09:54:31 +0900
Subject: [PATCH 115/193] v0.12.25

---
 ChangeLog             | 12 ++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 11055eb510..cf92abf023 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,17 @@
 # v0.12
 
+## Release 0.12.25 - 2016/05/25
+
+### New features / Enhancement
+
+* process: Add detach_process_forward_interval parameter to control emit interval
+  https://github.com/fluent/fluentd/pull/982
+
+### Bug fixes
+
+* Fix string-scrub dependency for ruby 2.0 or earlier support
+  https://github.com/fluent/fluentd/pull/986
+
 ## Release 0.12.24 - 2016/05/20
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 429287b3fa..e8120703bb 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.24'
+  VERSION = '0.12.25'
 
 end

From fec6898eeb50ea93fbe9409cd509d2e2f01522a8 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Wed, 27 Apr 2016 17:37:41 +0900
Subject: [PATCH 116/193] fix test drivers to work well for blocks with d.emit,
 and ObjectBufferedOutput

---
 lib/fluent/test/output_test.rb | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/fluent/test/output_test.rb b/lib/fluent/test/output_test.rb
index 62b971a2f0..0c49821547 100644
--- a/lib/fluent/test/output_test.rb
+++ b/lib/fluent/test/output_test.rb
@@ -76,20 +76,24 @@ def expect_format(str)
       def run(num_waits = 10, &block)
         result = nil
         super(num_waits) {
+          block.call if block
+
           es = ArrayEventStream.new(@entries)
           buffer = @instance.format_stream(@tag, es)
 
-          block.call if block
-
           if @expected_buffer
             assert_equal(@expected_buffer, buffer)
           end
 
-          key = ''
-          if @instance.respond_to?(:time_slicer)
+          case
+          when @instance.is_a?(Fluent::ObjectBufferedOutput)
+            key = @tag
+          when @instance.respond_to?(:time_slicer)
             # this block is only for test_out_file
-            time, record = @entries.first
+            time, _record = @entries.first
             key = @instance.time_slicer.call(time)
+          else
+            key = ''
           end
           chunk = @instance.buffer.new_chunk(key)
           chunk << buffer
@@ -128,6 +132,8 @@ def expect_format(str)
       def run(&block)
         result = []
         super {
+          block.call if block
+
           buffer = ''
           @entries.keys.each {|key|
             es = ArrayEventStream.new(@entries[key])
@@ -135,8 +141,6 @@ def run(&block)
             buffer << @instance.format_stream(@tag, es)
           }
 
-          block.call if block
-
           if @expected_buffer
             assert_equal(@expected_buffer, buffer)
           end

From e8200aaf8bdf25da586d21392b62430367cd5aa9 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Wed, 27 Apr 2016 17:38:36 +0900
Subject: [PATCH 117/193] fix test with added (missing) error definition

---
 test/plugin/test_out_forward.rb | 40 ++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb
index f2db81895e..fa440adefb 100644
--- a/test/plugin/test_out_forward.rb
+++ b/test/plugin/test_out_forward.rb
@@ -23,7 +23,7 @@ def setup
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::ForwardOutput) {
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ForwardOutput) {
       attr_reader :responses, :exceptions
 
       def initialize
@@ -231,10 +231,12 @@ def test_require_a_node_not_supporting_responses_to_respond_with_ack
     end
     d.run_timeout = 2
 
-    target_input_driver.run do
-      d.run do
-        records.each do |record|
-          d.emit record, time
+    assert_raise Fluent::ForwardOutputACKTimeoutError do
+      target_input_driver.run do
+        d.run do
+          records.each do |record|
+            d.emit record, time
+          end
         end
       end
     end
@@ -272,10 +274,12 @@ def test_require_a_node_not_supporting_responses_2_to_respond_with_ack
     end
     d.run_timeout = 2
 
-    target_input_driver.run do
-      d.run do
-        records.each do |record|
-          d.emit record, time
+    assert_raise Fluent::ForwardOutputConnectionClosedError do
+      target_input_driver.run do
+        d.run do
+          records.each do |record|
+            d.emit record, time
+          end
         end
       end
     end
@@ -318,7 +322,10 @@ def write(data)
         end
 
         def close
-          @sock.close
+          unless @sock.closed?
+            @sock.close_write
+            @sock.close
+          end
         end
       }
 
@@ -332,6 +339,7 @@ def close
                 handler.on_read(raw_data)
                 # chunk_counter is reset to zero only after all the data have been received and successfully deserialized.
                 break if handler.chunk_counter == 0
+                break if sock.closed?
               end
               if disconnect
                 handler.close
@@ -339,7 +347,10 @@ def close
               end
               sleep  # wait for connection to be closed by client
             ensure
-              sock.close if sock
+              if sock && !sock.closed?
+                sock.close_write
+                sock.close
+              end
             end
           end
         end
@@ -370,10 +381,6 @@ class DummyEngineDriver < Fluent::Test::TestDriver
     def initialize(klass, &block)
       super(klass, &block)
       @engine = DummyEngineClass.new
-      @klass = klass
-      # To avoid accessing Fluent::Engine, set Engine as a plugin's class constant (Fluent::SomePlugin::Engine).
-      # But this makes it impossible to run tests concurrently by threading in a process.
-      @klass.const_set(:Engine, @engine)
     end
 
     def inject_router
@@ -383,9 +390,6 @@ def inject_router
 
     def run(&block)
       super(&block)
-      @klass.class_eval do
-        remove_const(:Engine)
-      end
     end
 
     def emits

From dcfa064b047927291411af2fbe53d3e6d1033c7f Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 30 May 2016 14:53:57 +0900
Subject: [PATCH 118/193] Define `ForwardOutput#format_stream` for test

BufferedOutputTestDriver calls this method with Fluentd v0.12.
---
 test/plugin/test_out_forward.rb | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb
index fa440adefb..e4870cf26b 100644
--- a/test/plugin/test_out_forward.rb
+++ b/test/plugin/test_out_forward.rb
@@ -39,6 +39,13 @@ def send_data(node, tag, chunk)
         @exceptions << e
         raise e
       end
+
+      # Fluentd v0.12 BufferedOutputTestDriver calls this method.
+      # BufferedOutput#format_stream calls format method, but ForwardOutput#format is not defined.
+      # Because ObjectBufferedOutput#emit calls es.to_msgpack_stream directly.
+      def format_stream(tag, es)
+        es.to_msgpack_stream
+      end
     }.configure(conf)
   end
 

From 43c42a9a16c8f521e11931ec63a88a75c78d2848 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 30 May 2016 19:32:07 +0900
Subject: [PATCH 119/193] Fix regression of not require sigdump

---
 lib/fluent/supervisor.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
index e344876e7e..51964a0d01 100644
--- a/lib/fluent/supervisor.rb
+++ b/lib/fluent/supervisor.rb
@@ -17,6 +17,12 @@
 require 'etc'
 require 'fcntl'
 
+begin
+  require 'sigdump/setup'
+rescue Exception
+  # ignore LoadError and others (related with signals): it may raise these errors in Windows
+end
+
 require 'fluent/config'
 require 'fluent/env'
 require 'fluent/engine'

From dc4e6d7dbd28f2c00123a395519114a81f280d31 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 30 May 2016 19:34:24 +0900
Subject: [PATCH 120/193] v0.12.26

---
 ChangeLog             | 6 ++++++
 lib/fluent/version.rb | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index cf92abf023..de9906bf46 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 # v0.12
 
+## Release 0.12.26 - 2016/05/30
+
+### Bug fixes
+
+* Fix regression of not require sigdump
+
 ## Release 0.12.25 - 2016/05/25
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index e8120703bb..d411e81254 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.25'
+  VERSION = '0.12.26'
 
 end

From 35cb7b1891f820623c29a7357d1d35ea30286c42 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 10 Jun 2016 18:36:15 +0900
Subject: [PATCH 121/193] Fix regression of fluent-debug command. fix #1035

---
 lib/fluent/command/debug.rb | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/fluent/command/debug.rb b/lib/fluent/command/debug.rb
index 254d6b59f0..9643f24bb6 100644
--- a/lib/fluent/command/debug.rb
+++ b/lib/fluent/command/debug.rb
@@ -62,12 +62,13 @@
 
 require 'fluent/log'
 require 'fluent/engine'
+require 'fluent/config'
 require 'fluent/system_config'
 
-include Fluent::SystemConfig::Mixin
-
 $log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
-Fluent::Engine.init(system_config)
+conf = Fluent::Config.parse('', 'fluent-debug.conf', '', true)
+system_conf = Fluent::SystemConfig.create(conf)
+Fluent::Engine.init(system_conf)
 
 DRb::DRbObject.class_eval do
   undef_method :methods

From 65ea4351914243d2aba14b519ab19fbef00ec814 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 16 Jun 2016 06:45:01 +0900
Subject: [PATCH 122/193] Merge pull request #1051 from
 HatsuneMiku3939/in_http_x_forwarded-for

add support of X-Forwarded-For header
---
 lib/fluent/plugin/in_http.rb |  2 +
 test/plugin/test_in_http.rb  | 82 +++++++++++++++++++++---------------
 2 files changed, 51 insertions(+), 33 deletions(-)

diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index 58c804064f..2f75174a46 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -300,6 +300,8 @@ def on_headers_complete(headers)
             end
           when /Origin/i
             @origin  = v
+          when /X-Forwarded-For/i
+            @remote_addr = v.split(",").first
           end
         }
         if expect
diff --git a/test/plugin/test_in_http.rb b/test/plugin/test_in_http.rb
index 26ad19d4cd..76e1051ddb 100644
--- a/test/plugin/test_in_http.rb
+++ b/test/plugin/test_in_http.rb
@@ -119,6 +119,39 @@ def test_multi_json_with_add_remote_addr
 
   end
 
+  def test_json_with_add_remote_addr_given_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+    d.expect_emit "tag2", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+
+    d.run do
+      d.expected_emits.each {|tag,_time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
+        assert_equal "200", res.code
+      }
+    end
+
+  end
+
+  def test_multi_json_with_add_remote_addr_given_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>2}
+
+    d.run do
+      res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
+      assert_equal "200", res.code
+    end
+
+  end
+
   def test_multi_json_with_add_http_headers
     d = create_driver(CONFIG + "add_http_headers true")
 
@@ -274,50 +307,33 @@ def test_cors_allowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
-    test_in_http_cros_allowed = nil
-    acao = nil
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    begin
-      d.run do
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
-          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://foo.com"})
-          res = http.request(req)
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>1}
 
-          acao = res["Access-Control-Allow-Origin"]
-        end
-      end
-      test_in_http_cros_allowed = true
-    rescue
-      test_in_http_cros_allowed = false
+    d.run do
+      d.expected_emits.each {|tag,_time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"Origin"=>"http://foo.com"})
+        assert_equal "200", res.code
+        assert_equal "http://foo.com", res["Access-Control-Allow-Origin"]
+      }
     end
-
-    assert_equal true, test_in_http_cros_allowed
-    assert_equal "http://foo.com", acao
   end
 
   def test_cors_disallowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
-    test_in_http_cros_disallowed = nil
-    response_code = nil
-
-    begin
-      d.run do
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
-          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://bar.com"})
-          res = http.request(req)
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-          response_code = res.code
-        end
-      end
-      test_in_http_cros_disallowed = true
-    rescue
-      test_in_http_cros_disallowed = false
+    d.expected_emits_length = 0
+    d.run do
+      res = post("/tag1", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
+      assert_equal "403", res.code
+      res = post("/tag2", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
+      assert_equal "403", res.code
     end
-
-    assert_equal true, test_in_http_cros_disallowed
-    assert_equal "403", response_code
   end
 
   $test_in_http_connection_object_ids = []

From 3e5e4e82efff3979af2b03ffea14e59a4af1edab Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Thu, 16 Jun 2016 13:50:41 +0900
Subject: [PATCH 123/193] Merge pull request #950 from
 llby/add-include-support-for-ruby-dsl

[#360] Add include support for Ruby DSL
---
 lib/fluent/config/dsl.rb       |  19 +++-
 lib/fluent/config/v1_parser.rb |  12 ++-
 test/config/test_dsl.rb        | 182 ++++++++++++++++++++++++++++++++-
 3 files changed, 205 insertions(+), 8 deletions(-)

diff --git a/lib/fluent/config/dsl.rb b/lib/fluent/config/dsl.rb
index 538b13d5d6..c0771c9a02 100644
--- a/lib/fluent/config/dsl.rb
+++ b/lib/fluent/config/dsl.rb
@@ -35,14 +35,19 @@ def self.parse(source, source_path="config.rb")
       end
 
       class Proxy
-        def initialize(name, arg)
+        def initialize(name, arg, include_basepath = Dir.pwd)
           @element = Element.new(name, arg, self)
+          @include_basepath = include_basepath
         end
 
         def element
           @element
         end
 
+        def include_basepath
+          @include_basepath
+        end
+
         def eval(source, source_path)
           @element.instance_eval(source, source_path)
           self
@@ -96,6 +101,18 @@ def method_missing(name, *args, &block)
           self
         end
 
+        def include(*args)
+          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
+          if args.first =~ /\.rb$/
+            path = File.expand_path(args.first)
+            data = File.read(path)
+            self.instance_eval(data, path)
+          else
+            ss = StringScanner.new('')
+            Config::V1Parser.new(ss, @proxy.include_basepath, '', nil).eval_include(@attrs, @elements, args.first)
+          end
+        end
+
         def source(&block)
           @proxy.add_element('source', nil, block)
         end
diff --git a/lib/fluent/config/v1_parser.rb b/lib/fluent/config/v1_parser.rb
index 4eceaedc3c..68502e6779 100644
--- a/lib/fluent/config/v1_parser.rb
+++ b/lib/fluent/config/v1_parser.rb
@@ -148,7 +148,9 @@ def parse_include(attrs, elems)
 
       def eval_include(attrs, elems, uri)
         u = URI.parse(uri)
-        if u.scheme == 'file' || u.path == uri  # file path
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
+          # When the Windows absolute path then u.scheme.length == 1
+          # e.g. C:
           path = u.path
           if path[0] != ?/
             pattern = File.expand_path("#{@include_basepath}/#{path}")
@@ -156,10 +158,10 @@ def eval_include(attrs, elems, uri)
             pattern = path
           end
 
-          Dir.glob(pattern).sort.each { |path|
-            basepath = File.dirname(path)
-            fname = File.basename(path)
-            data = File.read(path)
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            data = File.read(entry)
             data.force_encoding('UTF-8')
             ss = StringScanner.new(data)
             V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
diff --git a/test/config/test_dsl.rb b/test/config/test_dsl.rb
index f81b97756d..ffb23931c0 100644
--- a/test/config/test_dsl.rb
+++ b/test/config/test_dsl.rb
@@ -2,6 +2,66 @@
 require 'fluent/config/element'
 require "fluent/config/dsl"
 
+TMP_DIR = File.dirname(__FILE__) + "/tmp/config_dsl#{ENV['TEST_ENV_NUMBER']}"
+def write_config(path, data)
+  FileUtils.mkdir_p(File.dirname(path))
+  File.open(path, "w") {|f| f.write data }
+end
+
+def prepare_config1
+  write_config "#{TMP_DIR}/config_test_1.conf", %[
+  k1 root_config
+  include dir/config_test_2.conf  #
+  @include #{TMP_DIR}/config_test_4.conf
+  include file://#{TMP_DIR}/config_test_5.conf
+  @include config.d/*.conf
+]
+  write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+  k2 relative_path_include
+  @include ../config_test_3.conf
+]
+  write_config "#{TMP_DIR}/config_test_3.conf", %[
+  k3 relative_include_in_included_file
+]
+  write_config "#{TMP_DIR}/config_test_4.conf", %[
+  k4 absolute_path_include
+]
+  write_config "#{TMP_DIR}/config_test_5.conf", %[
+  k5 uri_include
+]
+  write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+  k6 wildcard_include_1
+  <elem1 name>
+    include normal_parameter
+  </elem1>
+]
+  write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+  k7 wildcard_include_2
+]
+  write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+  <elem2 name>
+    @include ../dir/config_test_9.conf
+  </elem2>
+]
+  write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+  k9 embeded
+  <elem3 name>
+    nested nested_value
+    include hoge
+  </elem3>
+]
+  write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+  k8 wildcard_include_3
+  <elem4 name>
+    include normal_parameter
+  </elem4>
+]
+end
+
+def prepare_config2
+  write_config "#{TMP_DIR}/config_test_1.rb", DSL_CONFIG_EXAMPLE
+end
+
 DSL_CONFIG_EXAMPLE = %q[
 worker {
   hostname = "myhostname"
@@ -57,6 +117,14 @@
 }
 ]
 
+DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF = %q[
+include "#{TMP_DIR}/config_test_1.conf"
+]
+
+DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB = %q[
+include "#{TMP_DIR}/config_test_1.rb"
+]
+
 DSL_CONFIG_RETURNS_NON_ELEMENT = %q[
 worker {
 }
@@ -74,6 +142,9 @@
 DSL_CONFIG_WRONG_SYNTAX3 = %q[
 match('aa','bb')
 ]
+DSL_CONFIG_WRONG_SYNTAX4 = %q[
+include
+]
 
 module Fluent::Config
   class TestDSLParser < ::Test::Unit::TestCase
@@ -158,6 +229,112 @@ def setup
       end
     end
 
+    sub_test_case 'with include conf' do
+      def setup
+        prepare_config1
+        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF, 'dsl_config_for_include.conf')
+      end
+      test 'include config' do
+        assert_equal('root_config', @root['k1'])
+        assert_equal('relative_path_include', @root['k2'])
+        assert_equal('relative_include_in_included_file', @root['k3'])
+        assert_equal('absolute_path_include', @root['k4'])
+        assert_equal('uri_include', @root['k5'])
+        assert_equal('wildcard_include_1', @root['k6'])
+        assert_equal('wildcard_include_2', @root['k7'])
+        assert_equal('wildcard_include_3', @root['k8'])
+        assert_equal([
+            'k1',
+            'k2',
+            'k3',
+            'k4',
+            'k5',
+            'k8', # Because of the file name this comes first.
+            'k6',
+            'k7',
+          ], @root.keys)
+
+        elem1 = @root.elements.find { |e| e.name == 'elem1' }
+        assert(elem1)
+        assert_equal('name', elem1.arg)
+        assert_equal('normal_parameter', elem1['include'])
+
+        elem2 = @root.elements.find { |e| e.name == 'elem2' }
+        assert(elem2)
+        assert_equal('name', elem2.arg)
+        assert_equal('embeded', elem2['k9'])
+        assert_not_include(elem2, 'include')
+
+        elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+        assert(elem3)
+        assert_equal('nested_value', elem3['nested'])
+        assert_equal('hoge', elem3['include'])
+      end
+
+      # TODO: Add uri based include spec
+    end
+
+    sub_test_case 'with include rb' do
+      def setup
+        prepare_config2
+        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB, 'dsl_config_for_include.rb')
+      end
+      sub_test_case '.parse' do
+        test 'makes root element' do
+          assert_equal('ROOT', @root.name)
+          assert_predicate(@root.arg, :empty?)
+          assert_equal(0, @root.keys.size)
+        end
+
+        test 'makes worker element for worker tag' do
+          assert_equal(1, @root.elements.size)
+
+          worker = @root.elements.first
+          assert_equal('worker', worker.name)
+          assert_predicate(worker.arg, :empty?)
+          assert_equal(0, worker.keys.size)
+          assert_equal(10, worker.elements.size)
+        end
+
+        test 'makes subsections for blocks, with variable substitution' do
+          ele4 = @root.elements.first.elements[4]
+
+          assert_equal('source', ele4.name)
+          assert_predicate(ele4.arg, :empty?)
+          assert_equal(2, ele4.keys.size)
+          assert_equal('tail', ele4['type'])
+          assert_equal("/var/log/httpd/access.part4.log", ele4['path'])
+        end
+
+        test 'makes user-defined sections with blocks' do
+          filter0 = @root.elements.first.elements[4].elements.first
+
+          assert_equal('filter', filter0.name)
+          assert_equal('bar.**', filter0.arg)
+          assert_equal('hoge', filter0['type'])
+          assert_equal('moge', filter0['val1'])
+          assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2'])
+          assert_equal('10', filter0['val3'])
+          assert_equal('hoge', filter0['id'])
+
+          assert_equal(2, filter0.elements.size)
+          assert_equal('subsection', filter0.elements[0].name)
+          assert_equal('bar', filter0.elements[0]['foo'])
+          assert_equal('subsection', filter0.elements[1].name)
+          assert_equal('baz', filter0.elements[1]['foo'])
+        end
+
+        test 'makes values with user-assigned variable substitutions' do
+          match0 = @root.elements.first.elements[4].elements.last
+
+          assert_equal('match', match0.name)
+          assert_equal('{foo,bar}.**', match0.arg)
+          assert_equal('file', match0['type'])
+          assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path'])
+        end
+      end
+    end
+
     sub_test_case 'with configuration that returns non element on top' do
       sub_test_case '.parse' do
         test 'does not crash' do
@@ -170,8 +347,9 @@ def setup
       sub_test_case '.parse' do
         test 'raises ArgumentError correctly' do
           assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX1, 'dsl_config_wrong_syntax1') }
-          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax1') }
-          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax1') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax2') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax3') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX4, 'dsl_config_wrong_syntax4') }
         end
       end
     end

From 8db2a39698c435734b8e9e528724f09aa8eda0e3 Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Tue, 26 Apr 2016 14:06:08 +0900
Subject: [PATCH 124/193] fix test driver for TimeSlicedOutput * to call
 Buffer#push to call chunk#enqueued! correctly * to call Chunk#purge not to
 reuse chunks on buffer incorrectly after driver#run

---
 lib/fluent/test/output_test.rb | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/lib/fluent/test/output_test.rb b/lib/fluent/test/output_test.rb
index 0c49821547..a3d246ae44 100644
--- a/lib/fluent/test/output_test.rb
+++ b/lib/fluent/test/output_test.rb
@@ -145,17 +145,19 @@ def run(&block)
             assert_equal(@expected_buffer, buffer)
           end
 
-          chunks = @instance.instance_eval {
-            @buffer.instance_eval {
-              chunks = []
-              @map.keys.each {|key|
-                chunks.push(@map.delete(key))
-              }
-              chunks
-            }
-          }
+          chunks = []
+          @instance.instance_eval do
+            @buffer.instance_eval{ @map.keys }.each do |key|
+              @buffer.push(key)
+              chunks << @buffer.instance_eval{ @queue.pop }
+            end
+          end
           chunks.each { |chunk|
-            result.push(@instance.write(chunk))
+            begin
+              result.push(@instance.write(chunk))
+            ensure
+              chunk.purge
+            end
           }
         }
         result

From 75dfa1e41792275cabe1ea20e30796337154a92b Mon Sep 17 00:00:00 2001
From: TAGOMORI Satoshi <tagomoris@gmail.com>
Date: Tue, 26 Apr 2016 14:07:41 +0900
Subject: [PATCH 125/193] fix to use TimeSlicedOutputTestDriver in tests of
 out_file

---
 test/plugin/test_out_file.rb | 113 +++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 50 deletions(-)

diff --git a/test/plugin/test_out_file.rb b/test/plugin/test_out_file.rb
index 4d10107845..206e3e99b5 100644
--- a/test/plugin/test_out_file.rb
+++ b/test/plugin/test_out_file.rb
@@ -20,7 +20,7 @@ def setup
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
   end
 
   def with_timezone(timezone = 'UTC', &block)
@@ -143,11 +143,11 @@ def test_write
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    expect_path = "#{TMP_DIR}/out_file_test.20110102_0.log.gz"
-    assert_equal expect_path, path
+    paths = d.run
+    expect_paths = ["#{TMP_DIR}/out_file_test.20110102_0.log.gz"]
+    assert_equal expect_paths, paths
 
-    check_gzipped_result(path, %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
+    check_gzipped_result(paths[0], %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
   end
 
   def test_write_with_format_json
@@ -158,8 +158,8 @@ def test_write_with_format_json
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time})}\n])
+    paths = d.run
+    check_gzipped_result(paths[0], %[#{Yajl.dump({"a" => 1, 'time' => time})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time})}\n])
   end
 
   def test_write_with_format_ltsv
@@ -170,8 +170,8 @@ def test_write_with_format_ltsv
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z\n] + %[a:2\ttime:2011-01-02T13:14:15Z\n])
+    paths = d.run
+    check_gzipped_result(paths[0], %[a:1\ttime:2011-01-02T13:14:15Z\n] + %[a:2\ttime:2011-01-02T13:14:15Z\n])
   end
 
   def test_write_with_format_single_value
@@ -182,53 +182,66 @@ def test_write_with_format_single_value
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    check_gzipped_result(path, %[1\n] + %[2\n])
+    paths = d.run
+    check_gzipped_result(paths[0], %[1\n] + %[2\n])
   end
 
   def test_write_path_increment
-    d = create_driver
-
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
 
+    write_once = ->(){
+      d = create_driver
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+      d.run
+    }
+
+    assert !File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
+
     # FileOutput#write returns path
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102_0.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102_1.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102_2.log.gz", path
-    check_gzipped_result(path, formatted_lines)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102_0.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    assert_equal 1, Dir.glob("#{TMP_DIR}/out_file_test.*").size
+
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102_1.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    assert_equal 2, Dir.glob("#{TMP_DIR}/out_file_test.*").size
+
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102_2.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
   end
 
   def test_write_with_append
-    d = create_driver %[
-      path #{TMP_DIR}/out_file_test
-      compress gz
-      utc
-      append true
-    ]
-
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
 
+    write_once = ->(){
+      d = create_driver %[
+        path #{TMP_DIR}/out_file_test
+        compress gz
+        utc
+        append true
+      ]
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+      d.run
+    }
+
     # FileOutput#write returns path
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 2)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 3)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines * 2)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines * 3)
   end
 
   def test_write_with_symlink
@@ -270,8 +283,8 @@ def test_write_with_symlink
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
       # FileOutput#write returns path
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.log", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13_0.log"], paths
     end
 
     test 'normal with append' do
@@ -283,8 +296,8 @@ def test_write_with_symlink
       ])
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.log", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13.log"], paths
     end
 
     test '*' do
@@ -295,8 +308,8 @@ def test_write_with_symlink
       ])
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt"], paths
     end
 
     test '* with append' do
@@ -308,8 +321,8 @@ def test_write_with_symlink
       ])
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.txt", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13.txt"], paths
     end
   end
 end

From 78201512d3507b43dc1bceafd2ee2e362c489c65 Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Thu, 28 Jul 2016 12:58:50 +0900
Subject: [PATCH 126/193] Merge pull request #1127 from
 fluent/configurable-body-size-limit-for-syslog-input

Add option to receive longer udp syslog messages
---
 lib/fluent/plugin/in_syslog.rb |  3 ++-
 test/plugin/test_in_syslog.rb  | 36 +++++++++++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/lib/fluent/plugin/in_syslog.rb b/lib/fluent/plugin/in_syslog.rb
index 0f4291fe65..11a9326ff4 100644
--- a/lib/fluent/plugin/in_syslog.rb
+++ b/lib/fluent/plugin/in_syslog.rb
@@ -92,6 +92,7 @@ def initialize
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
     config_param :blocking_timeout, :time, default: 0.5
+    config_param :message_length_limit, :size, default: 2048
 
     def configure(conf)
       super
@@ -183,7 +184,7 @@ def listen(callback)
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, 2048, callback)
+        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback)
       else
         # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
         Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback)
diff --git a/test/plugin/test_in_syslog.rb b/test/plugin/test_in_syslog.rb
index 3e1a20e106..1cad0fc1d4 100755
--- a/test/plugin/test_in_syslog.rb
+++ b/test/plugin/test_in_syslog.rb
@@ -79,6 +79,24 @@ def test_msg_size
     compare_test_result(d.emits, tests)
   end
 
+  def test_msg_size_udp_for_large_msg
+    d = create_driver(CONFIG + %[
+      message_length_limit 5k
+    ])
+    tests = create_test_case(true)
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
   def test_msg_size_with_tcp
     d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
     tests = create_test_case
@@ -149,12 +167,20 @@ def test_msg_size_with_include_source_host
     compare_test_result(d.emits, tests, host)
   end
 
-  def create_test_case
+  def create_test_case(large_message = false)
     # actual syslog message has "\n"
-    [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
-    ]
+    if large_message
+      [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 4096 + "\n", 'expected' => 'x' * 4096},
+      ]
+    else
+      [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+      ]
+    end
   end
 
   def compare_test_result(emits, tests, host = nil)

From 17fa0d66cf68c2f507b55b4a40c66211eb39037d Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Fri, 29 Jul 2016 12:20:58 +0900
Subject: [PATCH 127/193] Add json 2.0 or later error message to regexp

---
 test/plugin/test_in_dummy.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/plugin/test_in_dummy.rb b/test/plugin/test_in_dummy.rb
index 828a5a5601..d2a7998ec5 100644
--- a/test/plugin/test_in_dummy.rb
+++ b/test/plugin/test_in_dummy.rb
@@ -42,6 +42,10 @@ def create_driver(conf)
       assert_equal 10, d.instance.rate
     end
 
+    def json_error_messages_regexp
+      /JSON::ParserError|got incomplete JSON|0th element of dummy, foo, is not a hash/
+    end
+
     test 'dummy' do
       # hash is okay
       d = create_driver(config + %[dummy {"foo":"bar"}])
@@ -51,7 +55,7 @@ def create_driver(conf)
       d = create_driver(config + %[dummy [{"foo":"bar"}]])
       assert_equal [{"foo"=>"bar"}], d.instance.dummy
 
-      assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
+      assert_raise_message(json_error_messages_regexp) do
         create_driver(config + %[dummy "foo"])
       end
 

From f1d12c29b25784bea4356d36c71bd1d7ae24fdb8 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 29 Jul 2016 21:03:48 +0900
Subject: [PATCH 128/193] Fix wrong message for unsupported parameter. fix
 #1039

---
 lib/fluent/config/section.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/fluent/config/section.rb b/lib/fluent/config/section.rb
index 0a0933201c..618988fef3 100644
--- a/lib/fluent/config/section.rb
+++ b/lib/fluent/config/section.rb
@@ -151,6 +151,10 @@ def self.check_unused_section(proxy, conf, plugin_class)
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          # In v0.12, buffer and output parameters are defined in same place.
+          # It causes same configuration is used in different buffer / output
+          # and buffer plugin's sections are always empty. It should be skipped.
+          next if proxy.sections.empty?
 
           unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
             parent_name = if conf.arg.empty?

From 1be5b7b63b9260bd3a3bd2ff351d8aee4822a995 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 9 Jun 2016 17:52:28 +0900
Subject: [PATCH 129/193] Merge pull request #1043 from
 cosmo0920/use-at-type-in-example-conf

example: Use `@type` in example confs
---
 example/filter_stdout.conf | 10 +++++-----
 example/in_forward.conf    |  4 ++--
 example/in_http.conf       |  4 ++--
 example/in_syslog.conf     |  4 ++--
 example/in_tail.conf       |  4 ++--
 example/in_tcp.conf        |  4 ++--
 example/in_udp.conf        |  4 ++--
 example/out_copy.conf      |  8 ++++----
 example/out_file.conf      |  4 ++--
 example/out_forward.conf   |  4 ++--
 example/v0_12_filter.conf  | 16 ++++++++--------
 11 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/example/filter_stdout.conf b/example/filter_stdout.conf
index e25bdc3c68..5754fea023 100644
--- a/example/filter_stdout.conf
+++ b/example/filter_stdout.conf
@@ -1,22 +1,22 @@
 <source>
-  type dummy
+  @type dummy
   tag dummy
 </source>
 
 <filter **>
-  type stdout
+  @type stdout
 </filter>
 
 <filter **>
-  type stdout
+  @type stdout
   output_type hash
 </filter>
 
 <filter **>
-  type stdout
+  @type stdout
   format ltsv
 </filter>
 
 <match **>
-  type null
+  @type null
 </match>
diff --git a/example/in_forward.conf b/example/in_forward.conf
index 97a99257d2..91a6d80185 100644
--- a/example/in_forward.conf
+++ b/example/in_forward.conf
@@ -1,7 +1,7 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>
diff --git a/example/in_http.conf b/example/in_http.conf
index e2402dcf70..fd93ee33f8 100644
--- a/example/in_http.conf
+++ b/example/in_http.conf
@@ -1,5 +1,5 @@
 <source>
-  type http
+  @type http
   bind 0.0.0.0
   port 9880
   body_size_limit 32MB
@@ -10,5 +10,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>
diff --git a/example/in_syslog.conf b/example/in_syslog.conf
index 73c5ca6775..9bff2289e9 100644
--- a/example/in_syslog.conf
+++ b/example/in_syslog.conf
@@ -1,5 +1,5 @@
 <source>
-  type syslog
+  @type syslog
   bind 0.0.0.0
   port 5140
   tag test
@@ -11,5 +11,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>
diff --git a/example/in_tail.conf b/example/in_tail.conf
index b3bc22f092..31e4d6ee61 100644
--- a/example/in_tail.conf
+++ b/example/in_tail.conf
@@ -1,5 +1,5 @@
 <source>
-  type tail
+  @type tail
   format none
   path /var/log/fluentd_test.log
   pos_file /var/log/fluentd_test.pos
@@ -10,5 +10,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>
diff --git a/example/in_tcp.conf b/example/in_tcp.conf
index df2663e993..3b40bfa01b 100644
--- a/example/in_tcp.conf
+++ b/example/in_tcp.conf
@@ -1,5 +1,5 @@
 <source>
-  type tcp
+  @type tcp
   format none
   bind 0.0.0.0
   port 5170
@@ -9,5 +9,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>
diff --git a/example/in_udp.conf b/example/in_udp.conf
index c7c8dd85b4..5de0b8dfaf 100644
--- a/example/in_udp.conf
+++ b/example/in_udp.conf
@@ -1,5 +1,5 @@
 <source>
-  type udp
+  @type udp
   format none
   bind 0.0.0.0
   port 5160
@@ -9,5 +9,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>
diff --git a/example/out_copy.conf b/example/out_copy.conf
index f07d815df5..73b6725665 100644
--- a/example/out_copy.conf
+++ b/example/out_copy.conf
@@ -1,15 +1,15 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test>
-  type copy
+  @type copy
   deep_copy false
   <store>
-    type stdout
+    @type stdout
   </store>
   <store>
-    type file
+    @type file
     path /var/log/fluentd/out_file_test
     format json
     buffer_type memory
diff --git a/example/out_file.conf b/example/out_file.conf
index 0f44774973..3e731cf7a8 100644
--- a/example/out_file.conf
+++ b/example/out_file.conf
@@ -1,9 +1,9 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test>
-  type file
+  @type file
   path /var/log/fluentd/out_file_test
   format json
   buffer_type memory
diff --git a/example/out_forward.conf b/example/out_forward.conf
index 79adc67b14..a116dc1e3c 100644
--- a/example/out_forward.conf
+++ b/example/out_forward.conf
@@ -1,10 +1,10 @@
 <source>
-  type dummy
+  @type dummy
   tag test
 </source>
 
 <match test>
-  type forward
+  @type forward
 
   <server>
     # first server
diff --git a/example/v0_12_filter.conf b/example/v0_12_filter.conf
index d74b5831cb..1b8c87047e 100644
--- a/example/v0_12_filter.conf
+++ b/example/v0_12_filter.conf
@@ -9,7 +9,7 @@
 # $ echo '{"message":"hello world"}' | fluent-cat foo
 
 <source>
-  type forward
+  @type forward
   port 24224
 </source>
 
@@ -21,13 +21,13 @@
 # - {"messag22":"keep this please"} is filtered out.
 
 <filter foo>
-  type grep
+  @type grep
   regexp1  message keep this
 </filter>
 
 # Matches the events that was kept by the above filter
 <match foo>
-  type stdout
+  @type stdout
 </match>
 
 # For all events with the tag "bar", add the machine's hostname with
@@ -35,7 +35,7 @@
 # at 123.4.2.4:24224.
 
 <filter bar>
-  type record_transformer
+  @type record_transformer
   <record>
     hostname ${hostname}
   </record>
@@ -44,7 +44,7 @@
 # By the time it is getting matched here, the event has
 # the "hostname" field.
 <match bar>
-  type forward
+  @type forward
   <server>
     host 123.4.2.4
     port 24225
@@ -61,12 +61,12 @@
 # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
 
 <filter foo.bar>
-  type grep
+  @type grep
   exclude1 hello .
 </filter>
 
 <filter foo.bar>
-  type record_transformer
+  @type record_transformer
   enable_ruby true
   <record>
     name ${name.downcase}
@@ -74,5 +74,5 @@
 </filter>
 
 <match foo.bar>
-  type stdout
+  @type stdout
 </match>

From 1f7cdffdb7bcae2560e536ec9cd796eff5777fc2 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 3 Aug 2016 17:20:04 +0900
Subject: [PATCH 130/193] v0.12.27

---
 ChangeLog             | 24 ++++++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index de9906bf46..d498042fb1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,29 @@
 # v0.12
 
+## Release 0.12.27 - 2016/08/03
+
+### New features / Enhancement
+
+* config: Add simplified syntax for hash and array
+  https://github.com/fluent/fluentd/pull/939
+* config: Add include support for Ruby DSL
+  https://github.com/fluent/fluentd/pull/950
+* in_http: Add support of X-Forwarded-For header
+  https://github.com/fluent/fluentd/pull/1051
+* in_syslog: Add message_length_limit option to receive longer udp syslog messages
+  https://github.com/fluent/fluentd/pull/1128
+
+### Bug fixes
+
+* test: Fix test for TimeSlicedOutput
+  https://github.com/fluent/fluentd/pull/941
+* example: Use `@type` in example confs
+  https://github.com/fluent/fluentd/pull/1043
+* Fix regression of fluent-debug command
+  https://github.com/fluent/fluentd/pull/1046
+* config: Fix wrong message for unsupported parameter
+  https://github.com/fluent/fluentd/pull/1132
+
 ## Release 0.12.26 - 2016/05/30
 
 ### Bug fixes
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index d411e81254..ac75f0b9a0 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.26'
+  VERSION = '0.12.27'
 
 end

From b1e5a3ab499ec06dde56cf4be20761c6130abbf3 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 8 Aug 2016 09:57:06 +0900
Subject: [PATCH 131/193] Merge pull request #1147 from edevil/use_string_oj

---
 lib/fluent/parser.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index dd933e358a..7608f2ef3f 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -260,7 +260,7 @@ def configure(conf)
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {bigdecimal_load: :float}
+          Oj.default_options = {bigdecimal_load: :float, mode: :strict}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError

From 1371c1189fb32e413ca37d8d8c2f3336a912e393 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 10 Aug 2016 19:34:06 +0900
Subject: [PATCH 132/193] Merge pull request #807 from
 fluent/add-source-host-key-to-forward

---
 lib/fluent/plugin/in_forward.rb | 46 ++++++++++++++++++--------
 test/plugin/test_in_forward.rb  | 58 ++++++++++++++++++++++++++++++---
 2 files changed, 85 insertions(+), 19 deletions(-)

diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index 7a4c2189ea..f303757548 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -47,6 +47,8 @@ def initialize
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
 
     def configure(conf)
       super
@@ -133,7 +135,7 @@ def run
     #   3: object record
     #   4: object option (optional)
     # }
-    def on_message(msg, chunk_size, source)
+    def on_message(msg, chunk_size, peeraddr)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -141,7 +143,7 @@ def on_message(msg, chunk_size, source)
 
       # TODO: raise an exception if broken chunk is generated by recoverable situation
       unless msg.is_a?(Array)
-        log.warn "incoming chunk is broken:", source: source, msg: msg
+        log.warn "incoming chunk is broken:", source: source_message(peeraddr), msg: msg
         return
       end
 
@@ -149,23 +151,24 @@ def on_message(msg, chunk_size, source)
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source, limit: @chunk_size_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source_message(peeraddr), limit: @chunk_size_limit, size: chunk_size
         return
       elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source, limit: @chunk_size_warn_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
       end
 
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries)
-        es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
+        es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
+        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)
         option = msg[2]
 
       elsif entries.class == Array
         # Forward
         es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, source)
+               check_and_skip_invalid_event(tag, entries, peeraddr)
              else
                es = MultiEventStream.new
                entries.each { |e|
@@ -177,6 +180,7 @@ def on_message(msg, chunk_size, source)
                }
                es
              end
+        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -185,11 +189,12 @@ def on_message(msg, chunk_size, source)
         time = msg[1]
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
-          log.warn "got invalid event and drop it:", source: source, tag: tag, time: time, record: record
+          log.warn "got invalid event and drop it:", source: source_message(peeraddr), tag: tag, time: time, record: record
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
         time = Engine.now if time == 0
+        record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -202,11 +207,11 @@ def invalid_event?(tag, time, record)
       !(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, source)
+    def check_and_skip_invalid_event(tag, es, peeraddr)
       new_es = MultiEventStream.new
       es.each { |time, record|
         if invalid_event?(tag, time, record)
-          log.warn "skip invalid event:", source: source, tag: tag, time: time, record: record
+          log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
           next
         end
         new_es.add(time, record)
@@ -214,16 +219,29 @@ def check_and_skip_invalid_event(tag, es, source)
       new_es
     end
 
+    def add_source_host(es, host)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        record[@source_hostname_key] = host
+        new_es.add(time, record)
+      }
+      new_es
+    end
+
+    def source_message(peeraddr)
+      _, port, host, addr = peeraddr
+      "host: #{host}, addr: #{addr}, port: #{port}"
+    end
+
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
       def initialize(io, linger_timeout, log, on_message)
         super(io)
 
+        @peeraddr = nil
         if io.is_a?(TCPSocket) # for unix domain socket support in the future
-          proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
-          @source = "host: #{host}, addr: #{addr}, port: #{port}"
-
+          @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
@@ -252,7 +270,7 @@ def on_read(data)
           @serializer = :to_json.to_proc
           @y = Yajl::Parser.new
           @y.on_parse_complete = lambda { |obj|
-            option = @on_message.call(obj, @chunk_counter, @source)
+            option = @on_message.call(obj, @chunk_counter, @peeraddr)
             respond option if option
             @chunk_counter = 0
           }
@@ -280,7 +298,7 @@ def on_read_json(data)
       def on_read_msgpack(data)
         @chunk_counter += data.bytesize
         @u.feed_each(data) do |obj|
-          option = @on_message.call(obj, @chunk_counter, @source)
+          option = @on_message.call(obj, @chunk_counter, @peeraddr)
           respond option if option
           @chunk_counter = 0
         end
diff --git a/test/plugin/test_in_forward.rb b/test/plugin/test_in_forward.rb
index c68a73a374..6d793f72d9 100644
--- a/test/plugin/test_in_forward.rb
+++ b/test/plugin/test_in_forward.rb
@@ -17,6 +17,7 @@ def setup
     port #{PORT}
     bind 127.0.0.1
   ]
+  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
 
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
@@ -197,7 +198,7 @@ def test_send_large_chunk_warning
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -226,7 +227,7 @@ def test_send_large_chunk_only_warning
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -253,7 +254,7 @@ def test_send_large_chunk_limit
     # d.run => send_data
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -275,7 +276,7 @@ def test_send_broken_chunk(data)
 
     # d.run => send_data
     d.run do
-      d.instance.send(:on_message, data, 1000000000, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+      d.instance.send(:on_message, data, 1000000000, PEERADDR)
     end
 
     # check emitted data
@@ -391,7 +392,6 @@ def test_respond_to_message_json_requiring_ack
 
     assert_equal events, d.emits
     assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
-
   end
 
   def test_not_respond_to_message_not_requiring_ack
@@ -498,5 +498,53 @@ def send_data(data, try_to_receive_response=false, response_timeout=1)
     @responses << res if try_to_receive_response
   end
 
+  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
+  test 'message protocol with source_hostname_key' do
+    execute_test { |events|
+      events.each { |tag, time, record|
+        send_data [tag, time, record].to_msgpack
+      }
+    }
+  end
+
+  test 'forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = []
+      events.each {|tag,time,record|
+        entries << [time, record]
+      }
+      send_data ['tag1', entries].to_msgpack
+    }
+  end
+
+  test 'packed forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = ''
+      events.each { |tag, time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    }
+  end
+
+  def execute_test(&block)
+    d = create_driver(CONFIG + 'source_hostname_key source')
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      block.call(events)
+    end
+
+    d.emits.each { |tag, _time, record|
+      assert_true record.has_key?('source')
+    }
+  end
+
   # TODO heartbeat
 end

From 1f261c7ce0821432767d7246a5111fbea3f29480 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 15 Aug 2016 15:29:16 +0900
Subject: [PATCH 133/193] v0.12.28

---
 ChangeLog             | 12 ++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d498042fb1..91feb3c10e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,17 @@
 # v0.12
 
+## Release 0.12.28 - 2016/08/14
+
+### New features / Enhancement
+
+* in_forward: Add source_hostname_key parameter
+  https://github.com/fluent/fluentd/pull/807
+
+### Bug fixes
+
+* parser: Ensure strict mode is used for Oj parser
+  https://github.com/fluent/fluentd/pull/1147
+
 ## Release 0.12.27 - 2016/08/03
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index ac75f0b9a0..744c2963a7 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.27'
+  VERSION = '0.12.28'
 
 end

From c804a5bc5708c00914d91dcabcd336e7b551d9c5 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 15 Aug 2016 23:11:44 +0900
Subject: [PATCH 134/193] Use link to refer v0.10 changelog

---
 ChangeLog | 709 +-----------------------------------------------------
 1 file changed, 1 insertion(+), 708 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 91feb3c10e..3f21084304 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -474,711 +474,4 @@ Here is the announcement for above new features: http://www.fluentd.org/blog/flu
 * out_file: Don't create world writable directory in daemon mode
   https://github.com/fluent/fluentd/pull/483
 
-# v0.10
-
-Release 0.10.57 - 2014/11/13
-
-* buffer: Make `total_queued_chunk_size` thread-safe to avoid race condition with `in_monitor_agent`
-   https://github.com/fluent/fluentd/pull/469
-* Add `--no-supervisor` option to disable the supervisor for supervision tools like runit
-   https://github.com/fluent/fluentd/pull/449
-
-0.10.57 was a bug fix release.
-
-Release 0.10.56 - 2014/10/31
-
-* config: Add Fluent::Config::Section#object_id to avoid NoMethodError by object dump tools
-  https://github.com/fluent/fluentd/pull/460
-* mixin: Fix SetTagKeyMixin and SetTimeKeyMixin which ignores config_set_default since 0.10.53
-  https://github.com/fluent/fluentd/pull/462
-* input/output: Support 'timezone' configuration parameter (e.g. "+09:00" for JST)
-  https://github.com/fluent/fluentd/pull/430
-* log: Use plugin_id instead of object_id in log message for consistency with in_monitor_agent
-  https://github.com/fluent/fluentd/pull/459
-
-Release 0.10.55 - 2014/10/16
-
-* config: Change v1's non-quoted string literal behaviour to v0's behaviour
-  https://github.com/fluent/fluentd/pull/436
-* config: Relax '@' handling in the plugin configuration for backward compatibility
-  https://github.com/fluent/fluentd/pull/455
-* config: Add --use-v0-config for keeping old configuration
-  https://github.com/fluent/fluentd/pull/434
-* config: Add single quoted literal in v1 configuration
-  https://github.com/fluent/fluentd/pull/437
-* config: Improve error message of Array / Hash parse error
-  https://github.com/fluent/fluentd/pull/457
-* input: Reduce shutdown time in some network related plugins when use Cool.io 1.2
-  https://github.com/fluent/fluentd/pull/440
-* parser: Use ParserError instead of general exception classes
-  https://github.com/fluent/fluentd/pull/453
-* Remove unused zfile buffer plugin
-  https://github.com/fluent/fluentd/pull/445
-
-Release 0.10.54 - skipped
-
-Release 0.10.53 - 2014/08/21
-
-* in_tail: Fix forget to detach Closer timer object
-* in_debug_agent: Fix config_param type of unix_path
-* in_syslog: Add include_source_host and source_host_Key options
-* config: Dump V1 escaped parameter correctly at start phase
-* config: Fix bug for config_param of bool value without defaults
-* config: Add 'alias' option to config_section and config_param
-* buffer: Add 'disable_retry_limit' option
-* engine: Reduce memory usage when retry_limit is large
-* parser: Add apache_error format
-* supervisor: Dump Fluentd version at dry run mode
-* test: InputTestDriver wait 'run' until input plugin emits all test data
-
-Release 0.10.52 - 2014/07/15
-
-* in_tail: Fix typo of warning argument. lb to line
-* config: Fix 'uninitialized constant Fluent::Config::V1Parser::URI' error
-* config: Reject '@' prefix of parameter name in V1 configuration. @ is reserved prefix.
-* config; Accept '#' comment in array and hash types
-* parser: Add time_format option to SyslogParser
-* parser: SyslogParser now makes ':' optional after pid
-* process: Fix 'mutex can't lock' issue of DetachMultiProcessMixin on Ruby 2.x
-* Add --without-source option to ignore <source> sections
-
-Release 0.10.51 - 2014/07/04
-
-* in_tail: Fix rotation handling when new file not found
-* in_syslog: Change debug log level to warn at pattern mismatch
-* formatter: Use localtime by default
-* parser: Add SyslogParser
-* parser: Add "estimate_current_event" attribute to TextParsers not to set current time into parse result
-* config: Add <system> section to set core parameters via configuration file
-* test: Support v1 configuration in TestDriver
-* Add udp and tcp input plugins
-
-Release 0.10.50 - 2014/06/17
-
-* in_tail: Fix race condition at shutdown
-* in_tail: read_from_head now works without pos_file
-* in_tail: Fix multiline format parsing without format_firstline
-* formatter: Add add_newline option to SingleValueFormatter
-* parser: Add block based API to support multi-events parsing
-* config: Fix encoding mismatch when @include used in v1
-* engine: Serialize shutdown order. Now shutdown input plugins first
-* Update rspec to 3.x
-
-Release 0.10.49 - 2014/06/05
-
-* in_http: Add format option to support various input data format
-* in_http: Accept json / msgpack array in default format
-* in_tail: Print warning message when file not exist with 'read_from_head true'
-* out_file: Add append option to disable path increment
-* out_file: Add format option to support various output data format
-* config: Fix broken 'include' processing in V1 configuration. Now add @include
-* fluentd-debug: Fix undefined method 'usage' error when invalid option passed
-* supervisor: Fix incorrect --group option handling
-* Add TextFormatter module for output plugins
-
-Release 0.10.48 - 2014/05/18
-
-* config: Add inspect method to Section for dumping the status
-* test: Fix the regression which forget to initialize $log in fluent/test.rb
-* Engine: Remove undocumented legacy plugin load mechanizm
-
-Release 0.10.47 - 2014/05/15
-
-* in_tail: Fix typo in flush_buffer
-* in_forward: Add chunk_size_warn_limit and chunk_size_limit for large chunks
-* in_monitor_agent: Add /api/config APIs to expose Fluentd configurations
-* supervisor: Add Supervisor.default_options to return default Fluentd options
-* test: Add DummyLogDevise and improve log handling in tests
-* config: Allow empty string value like `tag_mapped` in v1 configuration
-
-Release 0.10.46 - 2014/04/21
-
-* in_tail: Fix typo in rescue for dumping backtrace
-* in_forward: Add blocking_timeout option to avoid Cool.io loop stuck in shutdown
-* out_forward: Add phi_failure_detector option to disable "The Phi accrual failure detector"
-* buf_memory: Add flush_at_shutdown option to discard buffered event at shutdown
-* parser: Use Engine.now when time_key is missing in ValuesParser
-* parser: TextParser loads fluent/plugin/parser_xxx.rb automatically when 'xxx' format not found
-* Fix match cache expiration
-* Add new v1 configuration format and --use-v1-config option to enable v1 configuration format
-* Add config_section to Configurable to support nested parameters in plugins
-* Relax http_parser.rb gem version to support 0.6 series
-
-Release 0.10.45 - 2014/03/28
-
-* in_tail: Merge in_tail_ex and in_tail_multiline features
-* in_forward: Add linger_timeout option
-* out_exec: Support json and msgpack format
-* Engine: Fix signal handling when receives SIGTERM before loop starts
-* Use correct out_forward option in fluent.conf with --setup option
-* Ignore sigdump exception on signal unsupported environment
-
-Release 0.10.44 - 2014/02/26
-
-* in_forward: Print remote address and port in trace log
-* Remove several methods from fluentd-debug to call remote object method directly
-* Add --gemfile option to manage Fluentd and plugins using Gemfile and Bundler
-* Add <include path /> directive to include the configuration file under the elements
-* Add 'ruby' keyword to call Kernel method in configuration DSL
-* Relax Cool.io gem version to allow 1.2.x
-
-Release 0.10.43 - 2014/02/03
-
-* Input/Output: Add 'log_level' option to plugins
-* parser/in_tail: Fix RegexpParser forget to call configure method
-* parser/in_tail: Improve type conversion performance
-* in_http: Add add_http_headers option to add HTTP headers to record
-* Buffer: Improve submit_buffer throughput using round-robin writer selection
-* add sigdump gem to dump useful information via SIGCONT
-
-Release 0.10.42 - 2013/12/25
-
-* parser/in_tail: Add 'types' and related parameters to convert record value type
-* in_syslog: Add 'protocol_type' option for TCP support.
-* Buffer: Extract 'storable?' method for hook judge mechanizm by 3rd-party buffer plugins.
-* Output: Add 'try_flush_interval' parameter for high frequency flush.
-  It is undocumented because almost users never need this feature.
-* Change configuration unused checking to prevent memory leak when a plugin accesses configuratino keys.
-
-Release 0.10.41 - 2013/12/05
-
-* BufferedOutput: Add 'max_retry_wait' to limit retry interval
-* in_exec/object_space/syslog: Improve emit error message
-* Log: Add --suppress-repeated-stacktrace option to prevent log size explosion
-
-Release 0.10.40 - 2013/11/08
-
-* Dump gem version of fluent-plugin-xxx and fluent-mixin-xxx
-* in_http: Doesn't set REMOTE_ADDR when IO#getpeername failed
-* in_tail/parser: Fix RegexpParser which ignores time_format parameter
-* in_exec: Catch error and dumps error message in run_periodic
-* in_forward: Show error class in the error message
-* in_exec/out_exec_filter: Catch more exceptions, not only Errno::ESRCH, at shutdown
-* Log: Catch exception for log event handling at shutdown
-
-Release 0.10.39 - 2013/09/18
-
-* out_file: Improve symlink handling with buf_file
-* out_copy: Add deep_copy parameter for actual record copy
-* in_tail/parser: Add none format for supporting non-parse tailing like fluent-agent-lite
-* in_tail/parser: Improve performance using parsed time caching
-* Engine: Fix signal related exception in trap context at Ruby 2.0
-* buffer: Fix race condition when remove flushed chunk
-* add --suppress-config-dump option to disable dump configuration at start
-
-Release 0.10.38 - 2013/08/29
-
-* Fix require related problem in running test
-
-Release 0.10.37 - 2013/08/29
-
-* in_exec: Add format option to support tsv, json and msgpack. Default is tsv
-* out_stdout: Add output_type to print non-jsonable object
-* Engine: Skip nil record in emit
-* Check buffere_path duplication across buffer configurations.
-* Add experimental DSL to configuration format
-
-Release 0.10.36 - 2013/07/31
-
-* Log: add class name on each error logs
-* out_foward: change log level of node detachment/recovering/standby-failover from info to warn
-* out_file: add symlink_path parameter to point to latest written file
-* in_forward, in_http, in_unix: add backlog parameter to change resource backlog size
-* add --dry-run option to check fluentd setup without actual run
-* Fluentd now assumes configuration file written in UTF-8
-* update cool.io version to 1.1.1
-
-Release 0.10.35 - 2013/06/12
-
-* Log: emits fluentd's log via background thread
-* Log: use #inspect for non to_msgpack-able object in log event processing
-* FailureDetector/out_forward: fixed wrong phi calculation
-* in_stream: fixed unix domain connection leak when parsing json or msgpack failed
-* buf_file/out_file: sets 0644 permission to each created file for preventing
-  File.umask(0) effect at daemonize mode
-
-Release 0.10.34 - 2013/06/03
-
-* fluentd: supports --emit-error-log-interval to suppress log messages with the
-  same message for a certain seconds
-* BufferedOutput: fixed warning messages so that errors with the same meaning
-  use the same message
-* Log: ignores ThreadError which could happen in signal handler
-* Engine: prevents signal handlers from calling stop() method of plugins twice
-* MessagePackEventStream/in_forward: doesn't cache MessagePack::Unpacker (if
-  it uses msgpack-0.5.5 which doesn't cause performance impact) to not cause
-  problems with out_copy
-* TextParser/in_tail,in_syslog: 'json' format type removes time key from records
-* TextParser/in_tail,in_syslog: raises exception if time_format is set while
-  time_key is not set
-* in_tail: fixed minor bugs
-
-
-Release 0.10.33 - 2013/03/28
-
-* added in_monitor_agent
-* switched from jeweler to bundler
-* updated dependency for msgpack from '~> 0.4.4' to '~> 0.4.4 OR ~> 0.5.4'
-* added PluginId module so that in_monitor_agent can search plugins by id
-* BufferedOutput: added queued_chunk_flush_interval option to insert delay
-time before flushing queued chunks
-* BufferedOutput: randomize retry interval to prevent multiple servers from
-bursting network traffic at the same time
-* Configurable: sets @config instance variable
-
-
-Release 0.10.32 - 2013/03/04
-
-* in_tail: fixed the nil error problem which occurs with pos_file option
-* TextParser/in_tail: supports 'ltsv' (Labeled Tab-Separated Values) format: http://ltsv.org
-* #105: TextParser/in_tail: fixed 'nginx' format
-* #102: show log level in log messages
-* Gemfile: uses https source of rubygems instead of http
-
-
-Release 0.10.31 - 2013/01/31
-
-* out_forward: supports TCP heartbeat
-* in_tail reads rotated files from its head rather than its end when pos_file option is set
-* out_/in_ forward set O_NONBLOCK flag to the UDP heartbeat socket
-* Improved match cache expiration algorithm
-
-
-Release 0.10.30 - 2012/12/06
-
-* Shows configuration file at start time
-* Changed ruby version dependency from ~> 1.9.2 to => 1.9.2 so that it works
-  with ruby-2.0.
-* TimeSlicedOutput: overwrites default value of buffer_chunk_limit to 256m.
-  fluent-plugin-s3 will be affected.
-* TextParser/in_tail: supports 'nginx' preset formats
-* in/out_forward and in_syslog: support IPv6
-* out_forward: fixed a out of bounds bug
-
-
-Release 0.10.29 - 2012/11/26
-
-* buf_file: added flush_at_shutdown option to force flush before shuttingdown
-* TextParser shows message in warn level if text doesn't match the pattern
-
-
-Release 0.10.28 - 2012/10/30
-
-* in_http: on_request method can get headers as HTTP_* key and remote address
-  as REMOTE_ADDR key of the params argument
-
-
-Release 0.10.27 - 2012/10/11
-
-* added BufferChunk#unique_id which is an unique identifier of a buffered chunk
-* BufferedOutput: show 'temporarily failed' message
-
-
-Release 0.10.26 - 2012/09/26
-
-* added in_debug_agent plugin and fluent-debug command which enable you to
-  gather internal information of running Fluentd process using interactive shell
-* supervisor: --user commandline argument also inherits secondary groups (id -G)
-* out_forward: supports 'expire_dns_cache' option to set DNS expire time or disable
-  caching. Default behavior is 'never expire' which is same as former versions
-* out_forward: assumes successful data transfer via TCP as a successful
-  heartbeat.
-* out_forward: lazier failure detection: default phi_threshold 8 -> 10
-* in_forward: uses other nodes aggressively if it couldn't send data to the first
-  candidate to not stall during the wait time before detecting failure
-* in_forward: fixed shutdown procedure
-* in_tail: seeks to the saved position if detected log rotation actually
-  didn't rotate the file (meaning inode doesn't change). this change enables
-  cronolog to work with Fluentd.
-* in_tail: fixed a problem that it couldn't recover from uncaught exceptions
-  caused by output plugins
-* fluentd command supports '-q' option to decrease log level
-
-
-Release 0.10.25 - 2012/07/23
-
-* shutdown won't block
-* out_exec_filter supports 'child_respawn' option
-* in_tail doesn't remove whitespaces
-* out_roundrobin supports 'weight' option
-
-
-Release 0.10.24 - 2012/06/28
-
-* TextParser/in_tail: supports 'tsv' and 'csv' preset formats
-* supervisor set FD_CLOEXEC to @wait_daemonize_pipe_w in case the child
-process forked during run_configure
-
-
-Release 0.10.23 - 2012/06/25
-
-* out_exec_filter: show more error messages when child process unexpectedly exited
-* in_tail: Added 'apache2' format template which converts fields into integers or nil
-* TextParser creates a parser instance for each output for proper configuration
-* BufferedOutput: reset @flush_now flag
-* changed default buffer limit parameters:
-    Fluentd core:
-      buffer_chunk_limit 256MB -> 8MB
-      buffer_queue_limit 128 -> 256
-    buf_memory:
-      buffer_chunk_limit 32MB -> 8MB
-      buffer_queue_limit 32 -> 64
-    out_file:
-      buffer_chunk_limit -> 256MB
-
-
-Release 0.10.22 - 2012/05/02
-
-* Fixed in_tail and TextParser not to raise "time_format is required" error
-* out_exec_filter: fixed JSONFormatter to add \n at the end of records
-* out_exec_filter: separated {tag_key,time_key,time_format} parameters to
-  {in_,out_}{tag_key,time_key,time_format parameters while keeping
-  compatibility
-
-
-Release 0.10.21 - 2012/05/01
-
-* in_tail and TextParser support 'format json'
-* in_tail and TextParser support 'time_key' and 'time_format' parameters for 'format json'
-* in_tail: improved the file reader not to consume too much memory if the file is huge
-* out_stdout: uses logger instead of STDOUT to show records even if Fluentd is daemonized
-* Fixed the type of 'retry_wait' parameter of buffered output plugins from 'float' to 'time'
-* Fixed owner of log files for log rotation to work properly with --user and --group options
-
-
-Release 0.10.20 - 2012/04/27
-
-* Fixed Config#to_s
-* out_exec_filter: supports 'out_format json' option
-* out_exec_filter: supports 'in_format json' option
-
-
-Release 0.10.19 - 2012/04/17
-
-* Fixed in_syslog
-* in_tail: fixed "invalid byte sequence in US-ASCII" error
-* in_unix is not obsoleted
-
-
-Release 0.10.18 - 2012/04/13
-
-* Use Yajl instead of standard json library to avoid encoding problem
-* buf_memory: call force_encoding('ASCII-8BIT') to avoid encoding problem
-
-
-Release 0.10.17 - 2012/04/02
-
-* in_tail: don't ignore last line without LF [#44]
-* Added HandleTagNameMixin
-* Removed automake/conf style build support
-    Maintainig automake/conf style files are complex, but no user. We've decided
-    not to have those files anymore. Otherwise it's still used at rpm/deb packaging
-    scripts.
-
-
-Release 0.10.16 - 2012/03/26
-
-* Added in_gc plugin
-* Added in_object_space plugin
-* Changed dependency yajl-ruby from '~> 1.0.0' to '~> 1.0'
-
-
-Release 0.10.15 - 2012/03/09
-
-* Added experimental in_status plugin
-* out_forward: normalize the phi value of the failure detector into 1sec
-* out_forward: improved memory efficiency
-* out_forward: use weighted moving average for the phi accrual failure detector
-
-
-Release 0.10.14 - 2012/03/05
-
-* Show warnings if in_tcp or out_unix is used
-* BasicBuffer shows warnings when data.bytesize > @buffer_chunk_limit
-* out_forward: randomize roundrobin queue correctly
-* Added development dependencies to the gemspec
-
-
-Release 0.10.13 - 2012/02/21
-
-* Rewrote in_tail
-* Fixed SIGUSR1 handler to force flush logs
-
-
-Release 0.10.12 - 2012/02/13
-
-* Engine shows warnings when emitted record doesn't match any outputs
-* in_tail is rewritten to follow symbolic links correctly
-* out_forward uses independent default value as 'hard_timeout' parameter
-
-
-Release 0.10.11 - 2012/02/10
-
-* out_forward supports 'standby' parameter
-* out_forward handles 'hard_timeout' correctly
-
-
-Release 0.10.10 - 2012/02/09
-
-* in_forward and out_forward don't raise exceptions in callback handlers of
-  cool.io not to stop Loop#run
-* TimeSlicedOutput ignores time_slice_wait if flush_interval is specified
-* in_tail follows symbolic links correctly
-* in_http supports "Content-Type: application/json"
-* TestDriver#run sleeps 0.5 seconds for the out_exec_filter plugin
-
-
-Release 0.10.9 - 2012/01/19
-
-* Fixed TimeSlicedOutputTestDriver
-* Updated cool.io 1.0.0 -> 1.1.0
-* TextParser: fixed regexp of syslog parser to work with rsyslog on CentOS
-* out_exec_filter: improve performance by using buffering
-* out_exec_filter: added num_children parameter
-* out_exec_filter: added remove_prefix/add_prefix parameters
-* out_tail: show warning if pos_file parameter is not set
-* out_copy: fixed problems when event stream is not repeatable
-
-
-Release 0.10.8 - 2011/12/03
-
-* Added Supervisor: restart process on SIGHUP or unexpected end of process
-* Added -i commandline option which allows inline config
-* Added TimeSlicedOutputTestDriver
-* BufferedOutput outputs 'retry succeeded' message
-* Use Gem::Specification instead of Gem.searcher which is obsoleted
-* Added BasicBuffer#chunk_limit -> buffer_chunk_limit alias for backward
-  compatibility
-* buf_file: fixed to work with keys which contains '/'
-
-
-Release 0.10.7 - 2011/11/16
-
-* Supports multi-threaded on buffered output plugins ('num_threads')
-* Supports multi-process on input plugins ('detach_process')
-* Added ObjectBufferedOutput
-* Ensure to call 'shutdown' method of started plugins
-* in_tail supports 'pos_file' option that stores read position to a file
-
-
-Release 0.10.6 - 2011/11/11
-
-* Fixed --group NAME cmdline argument
-* Run configure(conf) after changing user and group
-* Fixed SIGHUP handling
-* BasicBuffer plugin uses Configurable module
-* buf_file uses microseconds + 12 bits random number for unique file name
-
-
-Release 0.10.5 - 2011/11/08
-
-* Added out_forward, in_forward
-* Added out_exec, in_exec
-* Added out_exec_filter
-
-
-Release 0.10.4 - 2011/11/07
-
-* TestDriver uses klass.dup.module_eval instead of inheriting class to
-  override methods
-* fixed encoding problem of String#[] in FileBufferChunk#<<(data)
-
-
-Release 0.10.3 - 2011/10/21
-
-  * Supports 'include' in config file
-    * Supports "http://" schema
-	* Supports wildcards
-
-
-Release 0.10.2 - 2011/10/18
-
-  * Fixed EventStream#to_msgpack_stream (thanks CkReal)
-  * Added gemspec.required_ruby_version = '~> 1.9.2' (thanks sakuro)
-
-
-Release 0.10.1 - 2011/10/16
-
-  * SetTimeKeyMixin accepts include_time_key parameter
-  * SetTagKeyMixin accepts include_tag_key parameter
-  * Fixed Makefile.am
-  * Fixed MemoryBufferChunk#msgpack_each
-
-
-Release 0.10.0 - 2011/10/16
-
-  * Removed Event class
-    * def format(tag, event) -> format(time, tag, record)
-	* def emit(tag, event) -> emit(tag, time, record)
-  * Added plugin APIs
-    * Added Configurable
-    * Added TimeFormatter
-    * Added SetTimeKeyMixin
-    * Added SetTagKeyMixin
-    * Added InputTestDriver
-  * Added Gemfile, gemspec and .rvmrc files
-
-
-Release 0.9.20 - 2011/10/07
-
-  * Config#has_key? sets used flag not to show warning
-  * in_tcp supports JSON
-  * Depends on Yajl
-  * Fixed fluentd -r option
-  * Fixed autogen.sh to create plugins directory
-
-
-Release 0.9.19 - 2011/10/07
-
-  * Secondary outputs work only when error count exceededs retry_limit
-  * Secondary outputs retry upto secondary_limit
-  * Added Output#secondary_init
-  * Added unit test driver for output plugins
-  * Added Fluent::Test.test? method
-  * Added comparision methods (<=>, ==, eql? and hash) to Event class
-  * Added test cases for out_copy, out_roundrobin and out_file
-
-
-Release 0.9.18 - 2011/10/04
-
-  * Performance improvements
-    * in_tcp uses lazy deserialization
-	* Engine caches results of Match#match
-	* out_file caches results of strftime
-  * Fixed TextParser.register_template
-
-
-Release 0.9.17 - 2011/10/03
-
-  * TimeSlicedOutput caches results of strftime() for performance
-  * Fixed TextParser.register_template (thanks eiichiroi)
-  * Fixed fluent-cat --port argument to work
-
-
-Release 0.9.16 - 2011/10/01
-
-  * Fixed it can't use relative path for --plugin command line argument
-  * Fixed out_copy to work with non-repeatable event streams
-
-
-Release 0.9.15 - 2011/09/21
-
-  * Set default encoding to ASCII-8BIT
-  * Fixed thread issues in BasicBuffer
-  * Performance improvements
-
-
-Release 0.9.14 - 2011/09/21
-
-  * Fixed to write unexpected error to log files
-  * Plugin loader uses GemSpec if it is available to load newer plugin
-  * Added Config.bool_value method for plugins
-
-
-Release 0.9.13 - 2011/09/21
-
-  * Updated out_file plugin
-    * Uses localtime by default
-    * Uses \t to separate time, tag and JSON.
-    * Uses ISO-8601 for the time format
-
-
-Release 0.9.12 - 2011/09/20
-
-  * Updated out_file plugin
-    * Supports log compression
-	* Slices log files every day by default
-
-
-Release 0.9.11 - 2011/09/19
-
-  * Added out_null plugin
-  * Added out_roundrobin plugin
-  * in_http supports keep-alive
-  * in_tcp and in_unix detect network failure
-
-
-Release 0.9.10 - 2011/09/07
-
-  * Periodic timer enqueues buffer chunks only when queue is empty
-
-
-Release 0.9.9 - 2011/09/06
-
-  * in_tail strips \n at the end of log line
-  * Uses newer one if multiple plugins are found
-
-
-Release 0.9.8 - 2011/08/16
-
-  * in_http plugin supports 'Expect: 100-continue' header
-  * Added --user and --group command line arguments to change privilege
-  * Changed default retry_limit parameter from 8 to 17
-  * Changed default buffer_chunk_limit parameter from 1m to 16m
-  * Changed default buffer_queue_limit parameter from 100 to 64
-
-
-Release 0.9.7 - 2011/08/06
-
-  * Load plugins named fluent_plugin.rb installed by RubyGems
-  * Load plugins located on fluent/plugin/{buf,in,out}_{type} installed by RubyGems
-  * Removed out_tdlog plugin to fluent-plugin-td gem
-
-
-Release 0.9.6 - 2011/08/06
-
-  * Force flush buffered events on SIGUSR1
-  * out_tdlog uses event tag for the database and table name
-  * Fluent can log Fluent logs
-  * Colorize Fluent logs on console
-  * Changed default unix domain socket path to /var/run/fluent/fluent.sock
-  * Makefile.am and configure.in aware fakeroot
-
-
-Release 0.9.5 - 2011/07/23
-
-  * Added <secondary> tag that fallbacks to secondary output when it can't
-    write to primary output.
-  * <match> pattern supports **; matches any fragments recursively.
-  * <match> pattern supports {x,y,...}; matches any of the inner patterns.
-
-
-Release 0.9.4 - 2011/07/22
-
-  * Switched EventMachine to Cool.io
-  * in_tail continues to follow file if it is deleted on Linux
-  * Added fluentd --setup option that installs sample configuration file
-  * Added out_tdlog plugin
-
-
-Release 0.9.3 - 2011/07/05
-
-  * Disable EventMachine.epoll because it doesn't work correctly on
-    Ruby 1.9 with threads
-  * in_tcp and in_unix uses EventMachine instead of threads
-  * Set BasicSocket.do_not_reverse_lookup = true
-
-
-Release 0.9.2 - 2011/07/03
-
-  * Added TimeSlicedOutput that splits buffer chunk exactly based on the time
-  * Changed structure of buffers from queue to map + queue
-  * Show waring if configuration parameters are not used
-  * Added out_time_file plugin
-
-
-Release 0.9.1 - 2011/06/26
-
-  * Added in_tail plugin
-  * Added in_syslog plugin
-  * Added out_stdout plugin
-  * Added out_tcp plugin
-  * Added out_unix plugin
-  * Improved performance of in_http plugin by replacing WEBrick + thread
-     with EventMachine
-
-
-Release 0.9.0 - 2011/06/20
-
-  * First release
-
+See https://github.com/fluent/fluentd/blob/v0.10/ChangeLog for v0.10 changelog

From 3e7dc6c260a6625a025557bbe4fb831594cb7908 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 5 Sep 2016 16:13:21 +0900
Subject: [PATCH 135/193] Port parser filter to v0.12

---
 lib/fluent/plugin/filter_parser.rb | 111 +++++
 test/plugin/test_filter_parser.rb  | 719 +++++++++++++++++++++++++++++
 2 files changed, 830 insertions(+)
 create mode 100644 lib/fluent/plugin/filter_parser.rb
 create mode 100644 test/plugin/test_filter_parser.rb

diff --git a/lib/fluent/plugin/filter_parser.rb b/lib/fluent/plugin/filter_parser.rb
new file mode 100644
index 0000000000..0353c6072f
--- /dev/null
+++ b/lib/fluent/plugin/filter_parser.rb
@@ -0,0 +1,111 @@
+require 'fluent/parser'
+
+class Fluent::ParserFilter < Fluent::Filter
+  Fluent::Plugin.register_filter('parser', self)
+
+  config_param :key_name, :string
+  config_param :reserve_data, :bool, default: false
+  config_param :inject_key_prefix, :string, default: nil
+  config_param :replace_invalid_sequence, :bool, default: false
+  config_param :hash_value_field, :string, default: nil
+  config_param :suppress_parse_error_log, :bool, default: false
+  config_param :time_parse, :bool, default: true
+  config_param :ignore_key_not_exist, :bool, default: false
+
+  attr_reader :parser
+
+  def initialize
+    super
+    require 'time'
+  end
+
+  def configure(conf)
+    super
+
+    @parser = Fluent::TextParser.new
+    #@parser.estimate_current_event = false
+    @parser.configure(conf)
+    if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
+      # disable parse time
+      @parser.parser.time_key = nil
+    end
+
+    self
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    es.each do |time,record|
+      raw_value = record[@key_name]
+      if raw_value.nil?
+        log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+        new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
+        next
+      end
+      begin
+        @parser.parse(raw_value) do |t,values|
+          if values
+            t ||= time
+            r = handle_parsed(tag, record, t, values)
+            new_es.add(t, r)
+          else
+            log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+            if @reserve_data
+              t = time
+              r = handle_parsed(tag, record, time, {})
+              new_es.add(t, r)
+            end
+          end
+        end
+      rescue Fluent::TextParser::ParserError => e
+        log.warn e.message unless @suppress_parse_error_log
+      rescue ArgumentError => e
+        if @replace_invalid_sequence
+          unless e.message.index("invalid byte sequence in") == 0
+            raise
+          end
+          replaced_string = replace_invalid_byte(raw_value)
+          @parser.parse(replaced_string) do |t,values|
+            if values
+              t ||= time
+              r = handle_parsed(tag, record, t, values)
+              new_es.add(t, r)
+            else
+              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              if @reserve_data
+                t = time
+                r = handle_parsed(tag, record, time, {})
+                new_es.add(t, r)
+              end
+            end
+          end
+        else
+          raise
+        end
+      rescue => e
+        log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+      end
+    end
+    new_es
+  end
+
+  private
+
+  def handle_parsed(tag, record, t, values)
+    if values && @inject_key_prefix
+      values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+    end
+    r = @hash_value_field ? {@hash_value_field => values} : values
+    if @reserve_data
+      r = r ? record.merge(r) : record
+    end
+    r
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
+end
diff --git a/test/plugin/test_filter_parser.rb b/test/plugin/test_filter_parser.rb
new file mode 100644
index 0000000000..64b413b000
--- /dev/null
+++ b/test/plugin/test_filter_parser.rb
@@ -0,0 +1,719 @@
+require_relative '../helper'
+require 'timecop'
+require 'fluent/plugin/filter_parser'
+
+class ParserFilterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @time = Time.parse("2012-01-02 13:14:15")
+    Timecop.freeze(@time)
+  end
+
+  teardown do
+    Timecop.return
+  end
+
+  CONFIG = %[
+    key_name      message
+    format        /^(?<x>.)(?<y>.) (?<time>.+)$/
+    time_format   %Y%m%d%H%M%S
+    reserve_data  true
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::ParserFilter, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        format unknown_format_that_will_never_be_implemented
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format json
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format ltsv
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+    d = create_driver %[
+        format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+        key_name message
+        suppress_parse_error_log true
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+        key_name message
+        suppress_parse_error_log false
+      ]
+    }
+    d = create_driver %[
+      key_name foo
+      format /(?<x>.)/
+    ]
+    assert_equal false, d.instance.reserve_data
+  end
+
+  # CONFIG = %[
+  #   remove_prefix test
+  #   add_prefix    parsed
+  #   key_name      message
+  #   format        /^(?<x>.)(?<y>.) (?<time>.+)$/
+  #   time_format   %Y%m%d%H%M%S
+  #   reserve_data  true
+  # ]
+  def test_filter
+    d1 = create_driver(CONFIG, 'test.no.change')
+    time = @time.to_i
+    d1.run do
+      d1.filter({'message' => '12 20120402182059'}, time)
+      d1.filter({'message' => '34 20120402182100'}, time)
+      d1.filter({'message' => '56 20120402182100'}, time)
+      d1.filter({'message' => '78 20120402182101'}, time)
+      d1.filter({'message' => '90 20120402182100'}, time)
+    end
+    filtered = d1.filtered_as_array
+    assert_equal 5, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal Time.parse("2012-04-02 18:20:59").to_i, first[1]
+    assert_equal '1', first[2]['x']
+    assert_equal '2', first[2]['y']
+    assert_equal '12 20120402182059', first[2]['message']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal Time.parse("2012-04-02 18:21:00").to_i, second[1]
+    assert_equal '3', second[2]['x']
+    assert_equal '4', second[2]['y']
+
+    third = filtered[2]
+    assert_equal 'test.no.change', third[0]
+    assert_equal Time.parse("2012-04-02 18:21:00").to_i, third[1]
+    assert_equal '5', third[2]['x']
+    assert_equal '6', third[2]['y']
+
+    fourth = filtered[3]
+    assert_equal 'test.no.change', fourth[0]
+    assert_equal Time.parse("2012-04-02 18:21:01").to_i, fourth[1]
+    assert_equal '7', fourth[2]['x']
+    assert_equal '8', fourth[2]['y']
+
+    fifth = filtered[4]
+    assert_equal 'test.no.change', fifth[0]
+    assert_equal Time.parse("2012-04-02 18:21:00").to_i, fifth[1]
+    assert_equal '9', fifth[2]['x']
+    assert_equal '0', fifth[2]['y']
+
+    d2 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>.)(?<y>.) (?<t>.+)$/
+    ], 'test.no.change')
+    time = @time.to_i
+    d2.run do
+      d2.filter({'data' => '12 20120402182059'}, time)
+      d2.filter({'data' => '34 20120402182100'}, time)
+    end
+    filtered = d2.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal '1', first[2]['x']
+    assert_equal '2', first[2]['y']
+    assert_equal '20120402182059', first[2]['t']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_nil second[2]['data']
+    assert_equal '3', second[2]['x']
+    assert_equal '4', second[2]['y']
+    assert_equal '20120402182100', second[2]['t']
+
+    d3 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/
+    ], 'test.no.change')
+    time = @time.to_i
+    d3.run do
+      d3.filter({'data' => '12 20120402182059'}, time)
+      d3.filter({'data' => '34 20120402182100'}, time)
+      d3.filter({'data' => 'xy 20120402182101'}, time)
+    end
+    filtered = d3.filtered_as_array
+    assert_equal 2, filtered.length
+
+    d3x = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
+      reserve_data  yes
+    ], 'test.no.change')
+    time = @time.to_i
+    d3x.run do
+      d3x.filter({'data' => '12 20120402182059'}, time)
+      d3x.filter({'data' => '34 20120402182100'}, time)
+      d3x.filter({'data' => 'xy 20120402182101'}, time)
+    end
+    filtered = d3x.filtered_as_array
+    assert_equal 3, filtered.length
+
+    d4 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        json
+    ], 'test.no.change')
+    time = @time.to_i
+    d4.run do
+      d4.filter({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d4.filter({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d4.filtered_as_array
+    assert_equal 1, filtered.length
+
+    d4x = create_driver(%[
+      tag parsed
+      key_name      data
+      format        json
+      reserve_data  yes
+    ], 'test.no.change')
+    time = @time.to_i
+    d4x.run do
+      d4x.filter({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d4x.filter({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d4x.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_equal 'foobar', second[2]['data']
+    assert_equal 'x', second[2]['xxx']
+    assert_equal 'y', second[2]['yyy']
+  end
+
+  CONFIG_LTSV =  %[
+    format ltsv
+    key_name data
+  ]
+  def test_filter_ltsv
+    d = create_driver(CONFIG_LTSV, 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d.filter({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_nil first[2]['data']
+    assert_equal 'first', second[2]['xxx']
+    assert_equal 'second2', second[2]['yyy']
+
+    d = create_driver(CONFIG_LTSV + %[
+      reserve_data yes
+    ], 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d.filter({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal "xxx:first\tyyy:second", first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_equal "xxx:first\tyyy:second", first[2]['data']
+    assert_equal 'first', second[2]['xxx']
+    assert_equal 'second2', second[2]['yyy']
+
+    # convert types
+    d = create_driver(CONFIG_LTSV + %[
+      types i:integer,s:string,f:float,b:bool
+    ], 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter({'data' => "i:1\ts:2\tf:3\tb:true\tx:123"}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal 1, first[2]['i']
+    assert_equal '2', first[2]['s']
+    assert_equal 3.0, first[2]['f']
+    assert_equal true, first[2]['b']
+    assert_equal '123', first[2]['x']
+  end
+
+  CONFIG_TSV =  %[
+    format tsv
+    key_name data
+    keys key1,key2,key3
+  ]
+  def test_filter_tsv
+    d = create_driver(CONFIG_TSV, 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter({'data' => "value1\tvalue2\tvalueThree", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'value1', first[2]['key1']
+    assert_equal 'value2', first[2]['key2']
+    assert_equal 'valueThree', first[2]['key3']
+  end
+
+  CONFIG_CSV =  %[
+    format csv
+    key_name data
+    keys key1,key2,key3
+  ]
+  def test_filter_csv
+    d = create_driver(CONFIG_CSV, 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter({'data' => 'value1,"value2","value""ThreeYes!"', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'value1', first[2]['key1']
+    assert_equal 'value2', first[2]['key2']
+    assert_equal 'value"ThreeYes!', first[2]['key3']
+  end
+
+  CONFIG_HASH_VALUE_FIELD = %[
+    format       json
+    key_name     data
+    hash_value_field parsed
+  ]
+  CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[
+    format       json
+    key_name     data
+    reserve_data yes
+    hash_value_field parsed
+  ]
+  CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[
+    format       json
+    key_name     data
+    hash_value_field parsed
+    inject_key_prefix data.
+  ]
+  def test_filter_inject_hash_value_field
+    original = {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD, 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter(original, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 1, record.keys.size
+    assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA, 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter(original, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 4, record.keys.size
+    assert_equal original['data'], record['data']
+    assert_equal original['xxx'], record['xxx']
+    assert_equal original['yyy'], record['yyy']
+    assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX, 'test.no.change')
+    time = @time.to_i
+    d.run do
+      d.filter(original, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 1, record.keys.size
+    assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
+  end
+
+  CONFIG_DONT_PARSE_TIME = %[
+    key_name data
+    format json
+    time_parse no
+  ]
+  def test_time_should_be_reserved
+    t = Time.now.to_i
+    d = create_driver(CONFIG_DONT_PARSE_TIME, 'test.no.change')
+
+    d.run do
+      d.filter({'data' => '{"time":1383190430, "f1":"v1"}'}, t)
+      d.filter({'data' => '{"time":"1383190430", "f1":"v1"}'}, t)
+      d.filter({'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'}, t)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 3, filtered.length
+
+    assert_equal 'test.no.change', filtered[0][0]
+    assert_equal 'v1', filtered[0][2]['f1']
+    assert_equal 1383190430, filtered[0][2]['time']
+    assert_equal t, filtered[0][1]
+
+    assert_equal 'test.no.change', filtered[1][0]
+    assert_equal 'v1', filtered[1][2]['f1']
+    assert_equal "1383190430", filtered[1][2]['time']
+    assert_equal t, filtered[1][1]
+
+    assert_equal 'test.no.change', filtered[2][0]
+    assert_equal 'v1', filtered[2][2]['f1']
+    assert_equal '2013-10-31 12:34:03 +0900', filtered[2][2]['time']
+    assert_equal t, filtered[2][1]
+  end
+
+  CONFIG_INVALID_TIME_VALUE = %[
+    remove_prefix test
+    key_name data
+    format json
+  ] # 'time' is implicit @time_key
+  def test_filter_invalid_time_data
+    # should not raise errors
+    t = Time.now.to_i
+    d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.no.change')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'data' => '{"time":[], "f1":"v1"}'}, t)
+        d.filter({'data' => '{"time":"thisisnottime", "f1":"v1"}'}, t)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    assert_equal 'test.no.change', filtered[0][0]
+    assert_equal 0, filtered[0][1]
+    assert_equal 'v1', filtered[0][2]['f1']
+    assert_equal 0, filtered[0][2]['time'].to_i
+  end
+
+  # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+
+  CONFIG_NOT_REPLACE = %[
+    remove_prefix test
+    key_name      data
+    format        /^(?<message>.*)$/
+  ]
+  CONFIG_INVALID_BYTE = CONFIG_NOT_REPLACE + %[
+    replace_invalid_sequence true
+  ]
+  def test_filter_invalid_byte
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+
+    d = create_driver(CONFIG_NOT_REPLACE, 'test.no.change')
+    assert_raise(ArgumentError) {
+      d.run do
+        d.filter({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+
+    d = create_driver(CONFIG_INVALID_BYTE, 'test.in')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_nil filtered[0][2]['data']
+    assert_equal '?'.force_encoding('UTF-8'), filtered[0][2]['message']
+
+    d = create_driver(CONFIG_INVALID_BYTE + %[
+      reserve_data yes
+    ], 'test.no.change')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_equal invalid_utf8, filtered[0][2]['data']
+    assert_equal '?'.force_encoding('UTF-8'), filtered[0][2]['message']
+
+    invalid_ascii = "\xff".force_encoding('US-ASCII')
+    d = create_driver(CONFIG_INVALID_BYTE, 'test.no.change')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'data' => invalid_ascii}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_nil filtered[0][2]['data']
+    assert_equal '?'.force_encoding('US-ASCII'), filtered[0][2]['message']
+  end
+
+  CONFIG_NOT_IGNORE = %[
+    remove_prefix    test
+    key_name         data
+    format           json
+    hash_value_field parsed
+  ]
+  CONFIG_IGNORE = CONFIG_NOT_IGNORE + %[
+    ignore_key_not_exist true
+  ]
+  CONFIG_PASS_SAME_RECORD = CONFIG_IGNORE + %[
+    reserve_data true
+  ]
+  def test_filter_key_not_exist
+    d = create_driver(CONFIG_NOT_IGNORE, 'test.no.ignore')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    assert_match /data does not exist/, d.instance.log.out.logs.first
+
+    d = create_driver(CONFIG_IGNORE, 'test.ignore')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    assert_not_match /data does not exist/, d.instance.log.out.logs.first
+
+    d = create_driver(CONFIG_PASS_SAME_RECORD, 'test.pass_same_record')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_nil filtered[0][2]['data']
+    assert_equal 'bar', filtered[0][2]['foo']
+  end
+
+  # suppress_parse_error_log test
+  CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+    suppress_parse_error_log false
+  ]
+  CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+    suppress_parse_error_log true
+  ]
+  CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+  ]
+
+  INVALID_MESSAGE = 'foo bar'
+  VALID_MESSAGE   = 'col1=foo col2=bar'
+
+  # if call warn() raise exception
+  class DummyLoggerWarnedException < StandardError; end
+  class DummyLogger
+    def reset
+    end
+    def warn(message)
+      raise DummyLoggerWarnedException
+    end
+  end
+
+  def swap_logger(instance)
+    raise "use with block" unless block_given?
+    dummy = DummyLogger.new
+    saved_logger = instance.log
+    instance.log = dummy
+    restore = lambda{ instance.log = saved_logger }
+
+    yield
+
+    restore.call
+  end
+
+  def test_parser_error_warning
+    d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.no.change')
+    swap_logger(d.instance) do
+      assert_raise(DummyLoggerWarnedException) {
+        d.run do
+          d.filter({'data' => '{"time":[], "f1":"v1"}'}, Time.now.to_i)
+        end
+      }
+    end
+  end
+
+  class DefaultSuppressParseErrorLogTest < self
+    def setup
+      # default(disabled) 'suppress_parse_error_log' is not specify
+      @d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.no.change')
+    end
+
+    def test_raise_exception
+      swap_logger(@d.instance) do
+        assert_raise(DummyLoggerWarnedException) {
+          @d.run do
+            @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          end
+        }
+      end
+    end
+
+    def test_nothing_raised
+      swap_logger(@d.instance) do
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => VALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+      end
+    end
+  end
+
+  class DisabledSuppressParseErrorLogTest < self
+    def setup
+      # disabled 'suppress_parse_error_log'
+      @d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.no.change')
+    end
+
+    def test_raise_exception
+      swap_logger(@d.instance) do
+        assert_raise(DummyLoggerWarnedException) {
+          @d.run do
+            @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          end
+        }
+      end
+    end
+
+    def test_nothing_raised
+      swap_logger(@d.instance) do
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => VALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+      end
+    end
+  end
+
+  class EnabledSuppressParseErrorLogTest < self
+    def setup
+      # enabled 'suppress_parse_error_log'
+      @d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.no.change')
+    end
+
+    def test_nothing_raised
+      swap_logger(@d.instance) do
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          @d.filter({'message' => VALID_MESSAGE},   Time.now.to_i)
+        end
+      }
+      end
+    end
+  end
+end

From baea2c1444529b69b0b316c988b6ecbd23ae2265 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 5 Sep 2016 17:53:57 +0900
Subject: [PATCH 136/193] Move require to top and fix comment out

---
 lib/fluent/plugin/filter_parser.rb | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/lib/fluent/plugin/filter_parser.rb b/lib/fluent/plugin/filter_parser.rb
index 0353c6072f..3d57ae3f0d 100644
--- a/lib/fluent/plugin/filter_parser.rb
+++ b/lib/fluent/plugin/filter_parser.rb
@@ -1,3 +1,4 @@
+require 'time'
 require 'fluent/parser'
 
 class Fluent::ParserFilter < Fluent::Filter
@@ -14,16 +15,11 @@ class Fluent::ParserFilter < Fluent::Filter
 
   attr_reader :parser
 
-  def initialize
-    super
-    require 'time'
-  end
-
   def configure(conf)
     super
 
     @parser = Fluent::TextParser.new
-    #@parser.estimate_current_event = false
+    @parser.estimate_current_event = false
     @parser.configure(conf)
     if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
       # disable parse time

From 5f0c0b97e2176f9abbf894c41d64b894855a763d Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Mon, 29 Aug 2016 20:03:48 +0900
Subject: [PATCH 137/193] Merge pull request #1186 from
 fluent/deprecated-parameter

config_param options to specify deprecated/obsoleted parameters
---
 lib/fluent/config/configure_proxy.rb |  27 ++-
 lib/fluent/config/error.rb           |   3 +
 lib/fluent/config/section.rb         |  15 ++
 lib/fluent/plugin/out_forward.rb     |   5 +-
 test/config/test_configurable.rb     | 271 +++++++++++++++++++++++++++
 test/config/test_types.rb            |  10 +-
 6 files changed, 322 insertions(+), 9 deletions(-)

diff --git a/lib/fluent/config/configure_proxy.rb b/lib/fluent/config/configure_proxy.rb
index af9be8d135..b47b16e5af 100644
--- a/lib/fluent/config/configure_proxy.rb
+++ b/lib/fluent/config/configure_proxy.rb
@@ -129,6 +129,12 @@ def merge_for_finalized(other)
         merged
       end
 
+      def option_value_type!(name, opts, key, klass)
+        if opts.has_key?(key) && !opts[key].is_a?(klass)
+          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+        end
+      end
+
       def parameter_configuration(name, *args, &block)
         name = name.to_sym
 
@@ -145,7 +151,7 @@ def parameter_configuration(name, *args, &block)
 
         type = opts[:type]
         if block && type
-          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
+          raise ArgumentError, "#{name}: both of block and type cannot be specified"
         end
 
         begin
@@ -153,8 +159,19 @@ def parameter_configuration(name, *args, &block)
           block ||= Configurable.lookup_type(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
+          raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
+        end
+
+        option_value_type!(name, opts, :desc, String)
+        option_value_type!(name, opts, :alias, Symbol)
+        option_value_type!(name, opts, :deprecated, String)
+        option_value_type!(name, opts, :obsoleted, String)
+        if type == :enum
+          if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
+            raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
+          end
         end
+        option_value_type!(name, opts, :value_type, Symbol) # hash, array
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])
@@ -164,6 +181,10 @@ def parameter_configuration(name, *args, &block)
           config_set_desc(name, opts[:desc])
         end
 
+        if opts[:deprecated] && opts[:obsoleted]
+          raise ArgumentError, "#{name}: both of deprecated and obsoleted cannot be specified at once"
+        end
+
         [name, block, opts]
       end
 
@@ -218,7 +239,7 @@ def desc(description)
 
       def config_section(name, *args, &block)
         unless block_given?
-          raise ArgumentError, "#{self.name}: config_section requires block parameter"
+          raise ArgumentError, "#{name}: config_section requires block parameter"
         end
         name = name.to_sym
 
diff --git a/lib/fluent/config/error.rb b/lib/fluent/config/error.rb
index 1b7809630e..30374611ae 100644
--- a/lib/fluent/config/error.rb
+++ b/lib/fluent/config/error.rb
@@ -20,4 +20,7 @@ class ConfigError < StandardError
 
   class ConfigParseError < ConfigError
   end
+
+  class ObsoletedParameterError < ConfigError
+  end
 end
diff --git a/lib/fluent/config/section.rb b/lib/fluent/config/section.rb
index 618988fef3..b9856343e8 100644
--- a/lib/fluent/config/section.rb
+++ b/lib/fluent/config/section.rb
@@ -100,6 +100,7 @@ def self.generate(proxy, conf, logger, plugin_class, stack = [])
             logger.error "config error in:\n#{conf}"
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
+          # argument should NOT be deprecated... (argument always has a value: '')
         end
 
         proxy.params.each_pair do |name, defval|
@@ -112,6 +113,20 @@ def self.generate(proxy, conf, logger, plugin_class, stack = [])
                     conf[opts[:alias].to_s]
                   end
             section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
           end
           unless section_params.has_key?(varname)
             logger.error "config error in:\n#{conf}"
diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 73858b1952..6e6322f2a8 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -86,9 +86,8 @@ def initialize
 
     attr_reader :nodes
 
-    # backward compatibility
-    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
-    config_param :host, :string, default: nil
+    config_param :port, :integer, default: LISTEN_PORT, deprecated: "User <server> host xxx </server> instead."
+    config_param :host, :string, default: nil, deprecated: "Use <server> port xxx </server> instead."
 
     attr_accessor :extend_internal_protocol
 
diff --git a/test/config/test_configurable.rb b/test/config/test_configurable.rb
index d09c40c5a6..ccf34cdb3d 100644
--- a/test/config/test_configurable.rb
+++ b/test/config/test_configurable.rb
@@ -158,6 +158,195 @@ class Example5
       config_param :secret_param2, :string, secret: true
     end
   end
+
+  class Example6
+    include Fluent::Configurable
+    config_param :obj1, :hash, default: {}
+    config_param :obj2, :array, default: []
+  end
+
+  module Overwrite
+    class Base
+      include Fluent::Configurable
+
+      config_param :name, :string, alias: :fullname
+      config_param :bool, :bool, alias: :flag
+      config_section :detail, required: false, multi: false, alias: "information" do
+        config_param :address, :string, default: "x"
+      end
+    end
+
+    class Required < Base
+      config_section :detail, required: true do
+        config_param :address, :string, default: "x"
+      end
+    end
+
+    class Multi < Base
+      config_section :detail, multi: true do
+        config_param :address, :string, default: "x"
+      end
+    end
+
+    class Alias < Base
+      config_section :detail, alias: "information2" do
+        config_param :address, :string, default: "x"
+      end
+    end
+
+    class DefaultOptions < Base
+      config_section :detail do
+        config_param :address, :string, default: "x"
+      end
+    end
+
+    class DetailAddressDefault < Base
+      config_section :detail do
+        config_param :address, :string, default: "y"
+      end
+    end
+
+    class AddParam < Base
+      config_section :detail do
+        config_param :phone_no, :string
+      end
+    end
+
+    class AddParamOverwriteAddress < Base
+      config_section :detail do
+        config_param :address, :string, default: "y"
+        config_param :phone_no, :string
+      end
+    end
+  end
+
+  module Final
+    # Show what is allowed in finalized sections
+    # InheritsFinalized < Finalized < Base
+    class Base
+      include Fluent::Configurable
+      config_section :appendix, multi: false, final: false do
+        config_param :code, :string
+        config_param :name, :string
+        config_param :address, :string, default: ""
+      end
+    end
+
+    class Finalized < Base
+      # to non-finalized section
+      # subclass can change type (code)
+      #          add default value (name)
+      #          change default value (address)
+      #          add field (age)
+      config_section :appendix, final: true do
+        config_param :code, :integer
+        config_set_default :name, "y"
+        config_set_default :address, "-"
+        config_param :age, :integer, default: 10
+      end
+    end
+
+    class InheritsFinalized < Finalized
+      # to finalized section
+      # subclass can add default value (code)
+      #              change default value (age)
+      #              add field (phone_no)
+      config_section :appendix do
+        config_set_default :code, 2
+        config_set_default :age, 0
+        config_param :phone_no, :string
+      end
+    end
+
+    # Show what is allowed/prohibited for finalized sections
+    class FinalizedBase
+      include Fluent::Configurable
+      config_section :appendix, param_name: :apd, init: false, required: true, multi: false, alias: "options", final: true do
+        config_param :name, :string
+      end
+    end
+
+    class FinalizedBase2
+      include Fluent::Configurable
+      config_section :appendix, param_name: :apd, init: false, required: false, multi: false, alias: "options", final: true do
+        config_param :name, :string
+      end
+    end
+
+    # subclass can change init with adding default values
+    class OverwriteInit < FinalizedBase2
+      config_section :appendix, init: true do
+        config_set_default :name, "moris"
+        config_param :code, :integer, default: 0
+      end
+    end
+
+    # subclass cannot change type (name)
+    class Subclass < FinalizedBase
+      config_section :appendix do
+        config_param :name, :integer
+      end
+    end
+
+    # subclass cannot change param_name
+    class OverwriteParamName < FinalizedBase
+      config_section :appendix, param_name: :adx do
+      end
+    end
+
+    # subclass cannot change final (section)
+    class OverwriteFinal < FinalizedBase
+      config_section :appendix, final: false do
+        config_param :name, :integer
+      end
+    end
+
+    # subclass cannot change required
+    class OverwriteRequired < FinalizedBase
+      config_section :appendix, required: false do
+      end
+    end
+
+    # subclass cannot change multi
+    class OverwriteMulti < FinalizedBase
+      config_section :appendix, multi: true do
+      end
+    end
+
+    # subclass cannot change alias
+    class OverwriteAlias < FinalizedBase
+      config_section :appendix, alias: "options2" do
+      end
+    end
+  end
+
+  module OverwriteDefaults
+    class Owner
+      include Fluent::Configurable
+      config_set_default :key1, "V1"
+      config_section :buffer do
+        config_set_default :size_of_something, 1024
+      end
+    end
+
+    class SubOwner < Owner
+      config_section :buffer do
+        config_set_default :size_of_something, 2048
+      end
+    end
+
+    class FlatChild
+      include Fluent::Configurable
+      attr_accessor :owner
+      config_param :key1, :string, default: "v1"
+    end
+  end
+  class UnRecommended
+    include Fluent::Configurable
+    attr_accessor :log
+    config_param :key1, :string, default: 'deprecated', deprecated: "key1 will be removed."
+    config_param :key2, :string, default: 'obsoleted', obsoleted: "key2 has been removed."
+  end
 end
 
 module Fluent::Config
@@ -740,5 +929,87 @@ def assert_secret_param(key, value)
         end
       end
     end
+    sub_test_case 'non-required options for config_param' do
+      test 'desc must be a string if specified' do
+        assert_raise ArgumentError.new("key: desc must be a String, but Symbol") do
+          class InvalidDescClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', desc: :invalid_description
+          end
+        end
+      end
+      test 'alias must be a symbol if specified' do
+        assert_raise ArgumentError.new("key: alias must be a Symbol, but String") do
+          class InvalidAliasClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', alias: 'yay'
+          end
+        end
+      end
+      test 'deprecated must be a string if specified' do
+        assert_raise ArgumentError.new("key: deprecated must be a String, but TrueClass") do
+          class InvalidDeprecatedClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', deprecated: true
+          end
+        end
+      end
+      test 'obsoleted must be a string if specified' do
+        assert_raise ArgumentError.new("key: obsoleted must be a String, but TrueClass") do
+          class InvalidObsoletedClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', obsoleted: true
+          end
+        end
+      end
+      test 'value_type for hash must be a symbol' do
+        assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do
+          class InvalidValueTypeOfHashClass
+            include Fluent::Configurable
+            config_param :key, :hash, value_type: 'yay'
+          end
+        end
+      end
+      test 'value_type for array must be a symbol' do
+        assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do
+          class InvalidValueTypeOfArrayClass
+            include Fluent::Configurable
+            config_param :key, :array, value_type: 'yay'
+          end
+        end
+      end
+    end
+    sub_test_case 'enum parameters' do
+      test 'list must be specified as an array of symbols'
+    end
+    sub_test_case 'deprecated/obsoleted parameters' do
+      test 'both cannot be specified at once' do
+        assert_raise ArgumentError.new("param1: both of deprecated and obsoleted cannot be specified at once") do
+          class Buggy1
+            include Fluent::Configurable
+            config_param :param1, :string, default: '', deprecated: 'yay', obsoleted: 'foo!'
+          end
+        end
+      end
+
+      test 'warned if deprecated parameter is configured' do
+        obj = ConfigurableSpec::UnRecommended.new
+        obj.log = Fluent::Test::TestLogger.new
+        obj.configure(Fluent::Config::Element.new('ROOT', '', {'key1' => 'yay'}, []))
+
+        assert_equal 'yay', obj.key1
+        first_log = obj.log.logs.first
+        assert{ first_log && first_log.include?("[warn]") && first_log.include?("'key1' parameter is deprecated: key1 will be removed.") }
+      end
+
+      test 'error raised if obsoleted parameter is configured' do
+        obj = ConfigurableSpec::UnRecommended.new
+        obj.log = Fluent::Test::TestLogger.new
+
+        assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do
+          obj.configure(Fluent::Config::Element.new('ROOT', '', {'key2' => 'yay'}, []))
+        end
+      end
+    end
   end
 end
diff --git a/test/config/test_types.rb b/test/config/test_types.rb
index ffd96b2bc2..6d826f4eec 100644
--- a/test/config/test_types.rb
+++ b/test/config/test_types.rb
@@ -72,9 +72,9 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
       assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
       assert_equal :value, Config::ENUM_TYPE.call('value', {list: [:val, :value, :v]})
-      assert_raises(Fluent::ConfigError){ Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) }
-      assert_raises(RuntimeError){ Config::ENUM_TYPE.call('val', {}) }
-      assert_raises(RuntimeError){ Config::ENUM_TYPE.call('val', {list: ["val", "value", "v"]}) }
+      assert_raises(Fluent::ConfigError.new("valid options are val,value,v but got x")){ Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) }
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")){ Config::ENUM_TYPE.call('val', {}) }
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")){ Config::ENUM_TYPE.call('val', {list: ["val", "value", "v"]}) }
     end
 
     test 'integer' do
@@ -138,6 +138,8 @@ class TestConfigTypes < ::Test::Unit::TestCase
 
       assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}))
       assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('x:1s,y:1m,z:1h',               {value_type: :time}))
+
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
     end
 
     test 'array' do
@@ -162,6 +164,8 @@ class TestConfigTypes < ::Test::Unit::TestCase
       }
       assert_equal(["1","2"], Config::ARRAY_TYPE.call('["1","2"]', array_options))
       assert_equal(["3"], Config::ARRAY_TYPE.call('["3"]', array_options))
+
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::ARRAY_TYPE.call("1,2", {value_type: :foo}) }
     end
   end
 end

From c7198308d320e24ce12a9a945b55da1803953321 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 5 Sep 2016 20:21:25 +0900
Subject: [PATCH 138/193] Fix constant name and removing useless log line

---
 lib/fluent/plugin/out_forward.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 6e6322f2a8..a19733b11e 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -86,7 +86,7 @@ def initialize
 
     attr_reader :nodes
 
-    config_param :port, :integer, default: LISTEN_PORT, deprecated: "User <server> host xxx </server> instead."
+    config_param :port, :integer, default: DEFAULT_LISTEN_PORT, deprecated: "User <server> host xxx </server> instead."
     config_param :host, :string, default: nil, deprecated: "Use <server> port xxx </server> instead."
 
     attr_accessor :extend_internal_protocol
@@ -96,7 +96,6 @@ def configure(conf)
 
       # backward compatibility
       if host = conf['host']
-        log.warn "'host' option in forward output is obsoleted. Use '<server> host xxx </server>' instead."
         port = conf['port']
         port = port ? port.to_i : DEFAULT_LISTEN_PORT
         e = conf.add_element('server')

From de89b154bc190f5b3a14f6548b87642e2c8b2071 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 5 Sep 2016 20:24:09 +0900
Subject: [PATCH 139/193] Remove unused test classes

---
 test/config/test_configurable.rb | 182 -------------------------------
 1 file changed, 182 deletions(-)

diff --git a/test/config/test_configurable.rb b/test/config/test_configurable.rb
index ccf34cdb3d..c2f1d3bbfd 100644
--- a/test/config/test_configurable.rb
+++ b/test/config/test_configurable.rb
@@ -159,188 +159,6 @@ class Example5
     end
   end
 
-  class Example6
-    include Fluent::Configurable
-    config_param :obj1, :hash, default: {}
-    config_param :obj2, :array, default: []
-  end
-
-  module Overwrite
-    class Base
-      include Fluent::Configurable
-
-      config_param :name, :string, alias: :fullname
-      config_param :bool, :bool, alias: :flag
-      config_section :detail, required: false, multi: false, alias: "information" do
-        config_param :address, :string, default: "x"
-      end
-    end
-
-    class Required < Base
-      config_section :detail, required: true do
-        config_param :address, :string, default: "x"
-      end
-    end
-
-    class Multi < Base
-      config_section :detail, multi: true do
-        config_param :address, :string, default: "x"
-      end
-    end
-
-    class Alias < Base
-      config_section :detail, alias: "information2" do
-        config_param :address, :string, default: "x"
-      end
-    end
-
-    class DefaultOptions < Base
-      config_section :detail do
-        config_param :address, :string, default: "x"
-      end
-    end
-
-    class DetailAddressDefault < Base
-      config_section :detail do
-        config_param :address, :string, default: "y"
-      end
-    end
-
-    class AddParam < Base
-      config_section :detail do
-        config_param :phone_no, :string
-      end
-    end
-
-    class AddParamOverwriteAddress < Base
-      config_section :detail do
-        config_param :address, :string, default: "y"
-        config_param :phone_no, :string
-      end
-    end
-  end
-
-  module Final
-    # Show what is allowed in finalized sections
-    # InheritsFinalized < Finalized < Base
-    class Base
-      include Fluent::Configurable
-      config_section :appendix, multi: false, final: false do
-        config_param :code, :string
-        config_param :name, :string
-        config_param :address, :string, default: ""
-      end
-    end
-
-    class Finalized < Base
-      # to non-finalized section
-      # subclass can change type (code)
-      #          add default value (name)
-      #          change default value (address)
-      #          add field (age)
-      config_section :appendix, final: true do
-        config_param :code, :integer
-        config_set_default :name, "y"
-        config_set_default :address, "-"
-        config_param :age, :integer, default: 10
-      end
-    end
-
-    class InheritsFinalized < Finalized
-      # to finalized section
-      # subclass can add default value (code)
-      #              change default value (age)
-      #              add field (phone_no)
-      config_section :appendix do
-        config_set_default :code, 2
-        config_set_default :age, 0
-        config_param :phone_no, :string
-      end
-    end
-
-    # Show what is allowed/prohibited for finalized sections
-    class FinalizedBase
-      include Fluent::Configurable
-      config_section :appendix, param_name: :apd, init: false, required: true, multi: false, alias: "options", final: true do
-        config_param :name, :string
-      end
-    end
-
-    class FinalizedBase2
-      include Fluent::Configurable
-      config_section :appendix, param_name: :apd, init: false, required: false, multi: false, alias: "options", final: true do
-        config_param :name, :string
-      end
-    end
-
-    # subclass can change init with adding default values
-    class OverwriteInit < FinalizedBase2
-      config_section :appendix, init: true do
-        config_set_default :name, "moris"
-        config_param :code, :integer, default: 0
-      end
-    end
-
-    # subclass cannot change type (name)
-    class Subclass < FinalizedBase
-      config_section :appendix do
-        config_param :name, :integer
-      end
-    end
-
-    # subclass cannot change param_name
-    class OverwriteParamName < FinalizedBase
-      config_section :appendix, param_name: :adx do
-      end
-    end
-
-    # subclass cannot change final (section)
-    class OverwriteFinal < FinalizedBase
-      config_section :appendix, final: false do
-        config_param :name, :integer
-      end
-    end
-
-    # subclass cannot change required
-    class OverwriteRequired < FinalizedBase
-      config_section :appendix, required: false do
-      end
-    end
-
-    # subclass cannot change multi
-    class OverwriteMulti < FinalizedBase
-      config_section :appendix, multi: true do
-      end
-    end
-
-    # subclass cannot change alias
-    class OverwriteAlias < FinalizedBase
-      config_section :appendix, alias: "options2" do
-      end
-    end
-  end
-
-  module OverwriteDefaults
-    class Owner
-      include Fluent::Configurable
-      config_set_default :key1, "V1"
-      config_section :buffer do
-        config_set_default :size_of_something, 1024
-      end
-    end
-
-    class SubOwner < Owner
-      config_section :buffer do
-        config_set_default :size_of_something, 2048
-      end
-    end
-
-    class FlatChild
-      include Fluent::Configurable
-      attr_accessor :owner
-      config_param :key1, :string, default: "v1"
-    end
-  end
   class UnRecommended
     include Fluent::Configurable
     attr_accessor :log

From 1fba5bc73613164a3d43892e6f7f9b721d91440b Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 22 Aug 2016 08:56:55 +0900
Subject: [PATCH 140/193] Merge pull request #1176 from
 sonots/prevent_overwriting_reserved_placeholders

Fix to prevent overwriting reserved placeholder keys such as tag
---
 .../plugin/filter_record_transformer.rb       |  4 +++-
 test/plugin/test_filter_record_transformer.rb | 22 +++++++++++++++----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index d86664294c..9396e38ad9 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -204,7 +204,9 @@ def prepare_placeholders(placeholder_values)
             end
           elsif value.kind_of?(Hash) # record, etc
             value.each do |k, v|
-              placeholders.store("${#{k}}", v) # foo
+              unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
+                placeholders.store("${#{k}}", v) # foo
+              end
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
           else # string, interger, float, and others?
diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
index 8b94cf7eba..1a4a66659d 100644
--- a/test/plugin/test_filter_record_transformer.rb
+++ b/test/plugin/test_filter_record_transformer.rb
@@ -172,10 +172,7 @@ def emit(config, msgs = [''])
       yield d if block_given?
       d.run {
         msgs.each do |msg|
-          record = {
-            'eventType0' => 'bar',
-            'message'    => msg,
-          }
+          record = {'eventType0' => 'bar', 'message' => msg}
           record = record.merge(msg) if msg.is_a?(Hash)
           d.emit(record, @time)
         end
@@ -269,6 +266,23 @@ def emit(config, msgs = [''])
         end
       end
 
+      test "Prevent overwriting reserved keys such as tag with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            new_tag ${tag}
+            new_record_tag ${record["tag"]}
+          </record>
+        ]
+        records = [{'tag' => 'tag', 'time' => 'time'}]
+        filtered = filter(config, records)
+        filtered.each_with_index do |(_t, r), i|
+          assert_not_equal('tag', r['new_tag'])
+          assert_equal(@tag, r['new_tag'])
+          assert_equal('tag', r['new_record_tag'])
+        end
+      end
+
       test "hash values with placeholders with enable_ruby #{enable_ruby}" do
         config = %[
           enable_ruby #{enable_ruby}

From 7ba559b28586e00f4a1861b556d79ad3bf53e16e Mon Sep 17 00:00:00 2001
From: panage <age.of.pan+github@gmail.com>
Date: Thu, 18 Aug 2016 18:50:50 +0900
Subject: [PATCH 141/193] Add from_encoding parameter to in_tail plugin (#1067)

* Add test code to check from_encoding param

Add configuration check new param "from_encoding",
and test convert from input source is "Hello world" in japanese Hiragana
that is encoded in cp932 to UTF-8.

* Add from_encoding param to in_tail plugin

Add new param "from_encoding".
If "encoding" param is only specified, process is same way as ever
to keep backword compatibility.
If two params, "encoding" and "from_encoding" are specified, process uses
```String.encode!(to, from)```.

* Specify type of encoding and from_encoding

* Fix configuration encoding and from_encoding

* Fix test code

Add test pattern from_encoding is only specified.

* Fix configure_encoding

Use log instead of $log.
Change the log messages so that the users are easy to understand.
Return as soon as possible, if encoding parameters are not specified.

* Use ConfigError instead of warnning-log

I deleted 2nd log message.
Changed to raise error if 'encoding' and 'from_encoding' paramters are bad cofiguration.

* Fix test code

Corresponding to ConfigError

* Change implementation of encode statement

For almost users, default setting should be faster than
the setting is specified encoding.
---
 lib/fluent/plugin/in_tail.rb | 46 ++++++++++++++++++-----
 test/plugin/test_in_tail.rb  | 73 ++++++++++++++++++++++++++++++++++++
 2 files changed, 110 insertions(+), 9 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 78673f1803..b5c647074d 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -50,14 +50,10 @@ def initialize
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'The encoding after conversion of the input.'
+    config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
-    config_param :encoding, default: nil do |encoding_name|
-      begin
-        Encoding.find(encoding_name)
-      rescue ArgumentError => e
-        raise ConfigError, e.message
-      end
-    end
+    config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
 
@@ -78,6 +74,7 @@ def configure(conf)
 
       configure_parser(conf)
       configure_tag
+      configure_encoding
 
       @multiline_mode = conf['format'] =~ /multiline/
       @receive_handler = if @multiline_mode
@@ -102,6 +99,25 @@ def configure_tag
       end
     end
 
+    def configure_encoding
+      unless @encoding
+        if @from_encoding
+          raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
+        end
+      end
+
+      @encoding = parse_encoding_param(@encoding) if @encoding
+      @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
+    end
+
+    def parse_encoding_param(encoding_name)
+      begin
+        Encoding.find(encoding_name) if encoding_name
+      rescue ArgumentError => e
+        raise ConfigError, e.message
+      end
+    end
+
     def start
       if @pos_file
         @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
@@ -232,7 +248,13 @@ def close_watcher_after_rotate_wait(tw)
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        lb.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            lb.encode!(@encoding, @from_encoding)
+          else
+            lb.force_encoding(@encoding)
+          end
+        end
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -281,7 +303,13 @@ def receive_lines(lines, tail_watcher)
     def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
-        line.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            line.encode!(@encoding, @from_encoding)
+          else
+            line.force_encoding(@encoding)
+          end
+        end
         @parser.parse(line) { |time, record|
           if time && record
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 7688c7f94e..fcb1ad92fc 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -62,6 +62,32 @@ def test_configure_encoding
     end
   end
 
+  def test_configure_from_encoding
+    # If only specified from_encoding raise ConfigError
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(SINGLE_LINE_CONFIG + 'from_encoding utf-8')
+    end
+
+    # valid setting
+    d = create_driver %[
+      format /(?<message>.*)/
+      read_from_head true
+      from_encoding utf-8
+      encoding utf-8
+    ]
+    assert_equal Encoding::UTF_8, d.instance.from_encoding
+
+    # invalid from_encoding
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver %[
+        format /(?<message>.*)/
+        read_from_head true
+        from_encoding no-such-encoding
+        encoding utf-8
+      ]
+    end
+  end
+
   # TODO: Should using more better approach instead of sleep wait
 
   def test_emit
@@ -332,6 +358,28 @@ def test_encoding(data)
     assert_equal(encoding, emits[0][2]['message'].encoding)
   end
 
+  def test_from_encoding
+    d = create_driver %[
+      format /(?<message>.*)/
+      read_from_head true
+      from_encoding cp932
+      encoding utf-8
+    ]
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
+        f.puts "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message'])
+    assert_equal(Encoding::UTF_8, emits[0][2]['message'].encoding)
+  end
+
   # multiline mode test
 
   def test_multiline
@@ -436,6 +484,31 @@ def test_multiline_encoding_of_flushed_record(data)
     end
   end
 
+  def test_multiline_from_encoding_of_flushed_record
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      multiline_flush_interval 2s
+      read_from_head true
+      from_encoding cp932
+      encoding utf-8
+    ]
+
+    d.run do
+      sleep 1
+      File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
+        f.puts "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
+      }
+
+      sleep 4
+      emits = d.emits
+      assert_equal(1, emits.length)
+      assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message1'])
+      assert_equal(Encoding::UTF_8, emits[0][2]['message1'].encoding)
+    end
+  end
+
   def test_multiline_with_multiple_formats
     File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
 

From 1e49ea6bde7f6b0bf86ab233cb30ca9e2e4a2b9c Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Fri, 2 Sep 2016 13:46:37 +0900
Subject: [PATCH 142/193] Merge pull request #1202 from
 fluent/raise-configuration-errors-for-missing-type

fix to raise ConfigError for type missing, and its tests
---
 lib/fluent/agent.rb      |  1 +
 lib/fluent/root_agent.rb |  2 +-
 test/test_root_agent.rb  | 42 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/agent.rb b/lib/fluent/agent.rb
index 9543b4a277..7bef4d97c3 100644
--- a/lib/fluent/agent.rb
+++ b/lib/fluent/agent.rb
@@ -57,6 +57,7 @@ def configure(conf)
       conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
         pattern = e.arg.empty? ? '**' : e.arg
         type = e['@type'] || e['type']
+        raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
         if e.name == 'filter'
           add_filter(type, pattern, e)
         else
diff --git a/lib/fluent/root_agent.rb b/lib/fluent/root_agent.rb
index 35857652f8..c505f57986 100644
--- a/lib/fluent/root_agent.rb
+++ b/lib/fluent/root_agent.rb
@@ -91,7 +91,7 @@ def configure(conf)
       else
         conf.elements.select { |e| e.name == 'source' }.each { |e|
           type = e['@type'] || e['type']
-          raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
+          raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
         }
       end
diff --git a/test/test_root_agent.rb b/test/test_root_agent.rb
index 8357a683f7..5978a5c612 100644
--- a/test/test_root_agent.rb
+++ b/test/test_root_agent.rb
@@ -50,6 +50,48 @@ def configure_ra(conf_str)
       assert_nil ra.error_collector
     end
 
+    test 'raises configuration error for missing type of source' do
+      conf = <<-EOC
+<source>
+</source>
+EOC
+      errmsg = "Missing '@type' parameter on <source> directive"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for missing type of match' do
+      conf = <<-EOC
+<source>
+  @type test_in
+</source>
+<match *.**>
+</match>
+EOC
+      errmsg = "Missing '@type' parameter on <match> directive"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for missing type of filter' do
+      conf = <<-EOC
+<source>
+  @type test_in
+</source>
+<filter *.**>
+</filter>
+<match *.**>
+  @type test_out
+</match>
+EOC
+      errmsg = "Missing '@type' parameter on <filter> directive"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
     test 'with plugins' do
       # check @type and type in one configuration
       conf = <<-EOC

From eb6bb0a57d08b1423c3acee3724d275a1694b848 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 6 Sep 2016 05:52:09 +0900
Subject: [PATCH 143/193] Fix test to use v0.12 test helper

---
 test/plugin/test_filter_record_transformer.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
index 1a4a66659d..cfc47fd913 100644
--- a/test/plugin/test_filter_record_transformer.rb
+++ b/test/plugin/test_filter_record_transformer.rb
@@ -275,7 +275,7 @@ def emit(config, msgs = [''])
           </record>
         ]
         records = [{'tag' => 'tag', 'time' => 'time'}]
-        filtered = filter(config, records)
+        filtered = emit(config, records)
         filtered.each_with_index do |(_t, r), i|
           assert_not_equal('tag', r['new_tag'])
           assert_equal(@tag, r['new_tag'])

From cd458194c949851a2b76c06b994330e964b55cc0 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 7 Sep 2016 02:42:07 +0900
Subject: [PATCH 144/193] v0.12.29

---
 ChangeLog             | 18 ++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 3f21084304..fe7ed43262 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,23 @@
 # v0.12
 
+## Release 0.12.29 - 2016/09/05
+
+### New features / Enhancement
+
+* in_tail: Add from_encoding parameter
+  https://github.com/fluent/fluentd/pull/1067
+* filter: Port parser filter to v0.12
+  https://github.com/fluent/fluentd/pull/1203
+* config: New config_param options to specify deprecated/obsoleted parameters
+  https://github.com/fluent/fluentd/pull/1186
+
+### Bug fixes
+
+* config: fix to raise ConfigError for type missing, and its tests
+  https://github.com/fluent/fluentd/pull/1202
+* filter_record_transformer: Fix to prevent overwriting reserved placeholder keys such as tag
+  https://github.com/fluent/fluentd/pull/1176
+
 ## Release 0.12.28 - 2016/08/14
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 744c2963a7..298a68829b 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.28'
+  VERSION = '0.12.29'
 
 end

From 703ee38b15c2df3193a09517f906ee97d7c6afbf Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 3 Nov 2016 05:20:57 +0900
Subject: [PATCH 145/193] Port 1239 oj mode changes to v0.12

---
 lib/fluent/env.rb       |  1 +
 lib/fluent/formatter.rb |  3 ++-
 lib/fluent/parser.rb    |  3 ++-
 test/test_formatter.rb  | 12 ++++++++++++
 test/test_parser.rb     |  8 ++++++++
 5 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/env.rb b/lib/fluent/env.rb
index 2916ff669d..b535f17ec5 100644
--- a/lib/fluent/env.rb
+++ b/lib/fluent/env.rb
@@ -18,6 +18,7 @@ module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat}
   DEFAULT_LISTEN_PORT = 24224
   DEFAULT_FILE_PERMISSION = 0644
   DEFAULT_DIR_PERMISSION = 0755
diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index 9bbaefd306..0278f8dcf7 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -15,6 +15,7 @@
 #
 
 require 'fluent/configurable'
+require 'fluent/env'
 require 'fluent/registry'
 require 'fluent/mixin'
 
@@ -141,7 +142,7 @@ def configure(conf)
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {mode: :compat}
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
           @dump_proc = Oj.method(:dump)
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 7608f2ef3f..a2d01ceb34 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -22,6 +22,7 @@
 require 'fluent/config/error'
 require 'fluent/config/element'
 require 'fluent/configurable'
+require 'fluent/env'
 require 'fluent/engine'
 require 'fluent/registry'
 require 'fluent/time'
@@ -260,7 +261,7 @@ def configure(conf)
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {bigdecimal_load: :float, mode: :strict}
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError
diff --git a/test/test_formatter.rb b/test/test_formatter.rb
index 416b92917b..6eec9f9d01 100644
--- a/test/test_formatter.rb
+++ b/test/test_formatter.rb
@@ -29,6 +29,10 @@ def record
     {'message' => 'awesome'}
   end
 
+  def symbolic_record
+    {:message => :awesome}
+  end
+
   def with_timezone(tz)
     oldtz, ENV['TZ'] = ENV['TZ'], tz
     yield
@@ -138,6 +142,14 @@ def test_format(data)
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_symbolic_record(data)
+      @formatter.configure('json_parser' => data)
+      formatted = @formatter.format(tag, @time, symbolic_record)
+
+      assert_equal("#{JSON.generate(record)}\n", formatted)
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_format_with_include_tag(data)
       @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo', 'json_parser' => data)
diff --git a/test/test_parser.rb b/test/test_parser.rb
index 7307636cad..54da34a180 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -421,6 +421,14 @@ def test_parse_without_time(data)
       }
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_colon_string(data)
+      @parser.configure('json_parser' => data)
+      @parser.parse('{"time":1362020400,"log":":message"}') { |time, record|
+        assert_equal(record['log'], ':message')
+      }
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_parse_with_invalid_time(data)
       @parser.configure('json_parser' => data)

From 04311ac8748f168acded6c956355998a2c3a9ff8 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 24 Nov 2016 10:19:59 +0900
Subject: [PATCH 146/193] Port #1317 into v0.12

---
 lib/fluent/plugin/in_monitor_agent.rb | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index bfd7ea9238..dd05608c98 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -33,6 +33,7 @@ class MonitorAgentInput < Input
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
     config_param :emit_config, :bool, default: false
+    config_param :include_config, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -76,12 +77,16 @@ def build_object(req, res)
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {}
+        opts = {with_config: @agent.include_config}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
         end
 
+        if with_config = get_search_parameter(qs, 'with_config'.freeze)
+          opts[:with_config] = Fluent::Config.bool_value(with_config)
+        end
+
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -389,7 +394,7 @@ def get_monitor_info(pe, opts={})
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
       obj['type'] = pe.config['@type'] || pe.config['type']
-      obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
+      obj['config'] = pe.config if opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|

From fb368a1ed66f44f96fa425216a77e3a1fb4429ff Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 24 Nov 2016 11:21:54 +0900
Subject: [PATCH 147/193] Add Fluent::Test::Helpers module for easy to write
 tests

---
 lib/fluent/test/helpers.rb       | 86 ++++++++++++++++++++++++++++++++
 test/config/test_configurable.rb | 12 ++---
 test/helper.rb                   |  3 ++
 test/test_parser.rb              | 68 +++++++++++--------------
 4 files changed, 125 insertions(+), 44 deletions(-)
 create mode 100644 lib/fluent/test/helpers.rb

diff --git a/lib/fluent/test/helpers.rb b/lib/fluent/test/helpers.rb
new file mode 100644
index 0000000000..dc52e906dd
--- /dev/null
+++ b/lib/fluent/test/helpers.rb
@@ -0,0 +1,86 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/element'
+require 'time'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Helpers
+      # See "Example Custom Assertion: http://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
+      def assert_equal_event_time(expected, actual, message = nil)
+        expected_s = "#{expected}"
+        actual_s   = "#{actual}"
+        message = build_message(message, <<EOT, expected_s, actual_s)
+<?> time expected but was
+<?>.
+EOT
+        assert_block(message) do
+          expected.is_a?(Integer) && actual.is_a?(Integer) && expected == actual
+        end
+      end
+
+      def config_element(name = 'test', argument = '', params = {}, elements = [])
+        Fluent::Config::Element.new(name, argument, params, elements)
+      end
+
+      def event_time(str = nil, format: nil)
+        if str
+          if format
+            Time.strptime(str, format).to_i
+          else
+            Time.parse(str).to_i
+          end
+        else
+          Time.now.to_i
+        end
+      end
+
+      def with_timezone(tz)
+        oldtz, ENV['TZ'] = ENV['TZ'], tz
+        yield
+      ensure
+        ENV['TZ'] = oldtz
+      end
+
+      def time2str(time, localtime: false, format: nil)
+        if format
+          if localtime
+            Time.at(time).strftime(format)
+          else
+            Time.at(time).utc.strftime(format)
+          end
+        else
+          if localtime
+            Time.at(time).iso8601
+          else
+            Time.at(time).utc.iso8601
+          end
+        end
+      end
+
+      def capture_log(driver)
+        tmp = driver.instance.log.out
+        driver.instance.log.out = StringIO.new
+        yield
+        return driver.instance.log.out.string
+      ensure
+        driver.instance.log.out = tmp
+      end
+    end
+  end
+end
diff --git a/test/config/test_configurable.rb b/test/config/test_configurable.rb
index c2f1d3bbfd..1018a7ed7a 100644
--- a/test/config/test_configurable.rb
+++ b/test/config/test_configurable.rb
@@ -296,7 +296,7 @@ def e(attrs)
         def e(name, arg = '', attrs = {}, elements = [])
           attrs_str_keys = {}
           attrs.each{|key, value| attrs_str_keys[key.to_s] = value }
-          Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+          config_element(name, arg, attrs_str_keys, elements)
         end
 
         BASE_ATTRS = {
@@ -577,7 +577,7 @@ def e(name, arg = '', attrs = {}, elements = [])
         def e(name, arg = '', attrs = {}, elements = [])
           attrs_str_keys = {}
           attrs.each{|key, value| attrs_str_keys[key.to_s] = value }
-          Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+          config_element(name, arg, attrs_str_keys, elements)
         end
 
         test 'subclass configuration spec can overwrite superclass specs' do
@@ -682,7 +682,7 @@ def e(name, arg = '', attrs = {}, elements = [])
         def e(name, arg = '', attrs = {}, elements = [])
           attrs_str_keys = {}
           attrs.each{|key, value| attrs_str_keys[key.to_s] = value }
-          Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+          config_element(name, arg, attrs_str_keys, elements)
         end
 
         test 'provides accessible data for alias attribute keys' do
@@ -701,7 +701,7 @@ def e(name, arg = '', attrs = {}, elements = [])
       def e(name, arg = '', attrs = {}, elements = [])
         attrs_str_keys = {}
         attrs.each { |key, value| attrs_str_keys[key.to_s] = value }
-        Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+        config_element(name, arg, attrs_str_keys, elements)
       end
 
       setup do
@@ -813,7 +813,7 @@ class Buggy1
       test 'warned if deprecated parameter is configured' do
         obj = ConfigurableSpec::UnRecommended.new
         obj.log = Fluent::Test::TestLogger.new
-        obj.configure(Fluent::Config::Element.new('ROOT', '', {'key1' => 'yay'}, []))
+        obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, []))
 
         assert_equal 'yay', obj.key1
         first_log = obj.log.logs.first
@@ -825,7 +825,7 @@ class Buggy1
         obj.log = Fluent::Test::TestLogger.new
 
         assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do
-          obj.configure(Fluent::Config::Element.new('ROOT', '', {'key2' => 'yay'}, []))
+          obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, []))
         end
       end
     end
diff --git a/test/helper.rb b/test/helper.rb
index 86acd6953b..b09d0ebf51 100644
--- a/test/helper.rb
+++ b/test/helper.rb
@@ -41,12 +41,15 @@ def to_masked_element
 require 'fileutils'
 require 'fluent/log'
 require 'fluent/test'
+require 'fluent/test/helpers'
 
 unless defined?(Test::Unit::AssertionFailedError)
   class Test::Unit::AssertionFailedError < StandardError
   end
 end
 
+include Fluent::Test::Helpers
+
 def unused_port
   s = TCPServer.open(0)
   port = s.addr[1]
diff --git a/test/test_parser.rb b/test/test_parser.rb
index 54da34a180..29e8ac7c10 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -6,14 +6,6 @@
 module ParserTest
   include Fluent
 
-  def str2time(str_time, format = nil)
-    if format
-      Time.strptime(str_time, format).to_i
-    else
-      Time.parse(str_time).to_i
-    end
-  end
-
   class BaseParserTest < ::Test::Unit::TestCase
     include ParserTest
 
@@ -71,15 +63,15 @@ class TimeParserTest < ::Test::Unit::TestCase
     def test_call_with_parse
       parser = TextParser::TimeParser.new(nil)
 
-      time = str2time('2013-09-18 12:00:00 +0900')
-      assert_equal(time, parser.parse('2013-09-18 12:00:00 +0900'))
+      time = event_time('2013-09-18 12:00:00 +0900')
+      assert_equal_event_time(time, parser.parse('2013-09-18 12:00:00 +0900'))
     end
 
     def test_parse_with_strptime
       parser = TextParser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')
 
-      time = str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z')
-      assert_equal(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
+      time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
+      assert_equal_event_time(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
     end
 
     def test_parse_with_invalid_argument
@@ -99,13 +91,13 @@ class RegexpParserTest < ::Test::Unit::TestCase
     def internal_test_case(parser)
       text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
       [parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal({
           'user' => '-',
           'flag' => true,
           'code' => 200.0,
           'size' => 777,
-          'date' => str2time('14/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
+          'date' => event_time('14/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'),
           'host' => '192.168.0.1',
           'path' => ['/', '/user']
         }, record)
@@ -139,12 +131,12 @@ def test_parse_with_time_key
       )
       text = '2013-02-28 12:00:00 +0900'
       parser.parse(text) do |time, record|
-        assert_equal Time.parse(text).to_i, time
+        assert_equal event_time(text), time
       end
     end
 
     def test_parse_without_time
-      time_at_start = Time.now.to_i
+      time_at_start = event_time()
       text = "tagomori_satoshi tagomoris 34\n"
 
       parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
@@ -206,7 +198,7 @@ def setup
     def test_call(method_name)
       m = @parser.method(method_name)
       m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal({
           'user'    => '-',
           'method'  => 'GET',
@@ -245,21 +237,21 @@ def setup
 
     def test_parse
       @parser.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
-        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
         assert_equal(@expected, record)
       }
     end
 
     def test_parse_with_pid
       @parser.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
-        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
         assert_equal(@expected.merge('pid' => '1000'), record)
       }
     end
 
     def test_parse_without_client
       @parser.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
-        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
         assert_equal({
           'level' => 'notice',
           'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
@@ -287,7 +279,7 @@ def setup
 
     def test_parse
       @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
       assert_equal(TextParser::ApacheParser::REGEXP, @parser.patterns['format'])
@@ -296,7 +288,7 @@ def test_parse
 
     def test_parse_without_http_version
       @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
     end
@@ -318,7 +310,7 @@ def setup
     def test_parse
       @parser.configure({})
       @parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
-        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
       assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
@@ -328,7 +320,7 @@ def test_parse
     def test_parse_with_time_format
       @parser.configure('time_format' => '%b %d %M:%S:%H')
       @parser.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
-        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
       assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
@@ -337,7 +329,7 @@ def test_parse_with_time_format
     def test_parse_with_priority
       @parser.configure('with_priority' => true)
       @parser.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
-        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       }
       assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
@@ -347,7 +339,7 @@ def test_parse_with_priority
     def test_parse_without_colon
       @parser.configure({})
       @parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
-        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
       assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
@@ -377,7 +369,7 @@ def setup
     def test_parse(data)
       @parser.configure('json_parser' => data)
       @parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
-        assert_equal(str2time('2013-02-28 12:00:00 +0900').to_i, time)
+        assert_equal_event_time(event_time('2013-02-28 12:00:00 +0900'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'size'   => 777,
@@ -472,21 +464,21 @@ def setup
 
     def test_parse
       @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
     end
 
     def test_parse_with_empty_included_path
       @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected.merge('path' => '/a[ ]b'), record)
       }
     end
 
     def test_parse_without_http_version
       @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
     end
@@ -515,7 +507,7 @@ def test_parse(param)
       parser = TextParser::TSVParser.new
       parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
-        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'a' => '192.168.0.1',
           'b' => '111',
@@ -620,7 +612,7 @@ def test_parse(param)
       parser = TextParser::CSVParser.new
       parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
-        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'c' => '192.168.0.1',
           'd' => '111',
@@ -723,7 +715,7 @@ def test_parse
       parser = TextParser::LabeledTSVParser.new
       parser.configure({})
       parser.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
-        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'req_id' => '111',
@@ -738,7 +730,7 @@ def test_parse_with_customized_delimiter
         'label_delimiter' => '=',
       )
       parser.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
-        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'req_id' => '111',
@@ -753,7 +745,7 @@ def test_parse_with_customized_time_format
         'time_format' => '%d/%b/%Y:%H:%M:%S %z',
       )
       parser.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
-        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'req_id' => '111',
@@ -898,7 +890,7 @@ def test_parse
 \tat Main.main(Main.java:16) ~[bin/:na]
 EOS
 
-        assert_equal(str2time('2013-3-03 14:27:33').to_i, time)
+        assert_equal_event_time(event_time('2013-3-03 14:27:33'), time)
         assert_equal({
           "thread"  => "main",
           "level"   => "ERROR",
@@ -916,7 +908,7 @@ def test_parse_with_firstline
 EOS
 
         assert(parser.firstline?('----'))
-        assert_equal(str2time('2013-3-03 14:27:33').to_i, time)
+        assert_equal_event_time(event_time('2013-3-03 14:27:33'), time)
         assert_equal({"message" => "test1"}, record)
       }
     end
@@ -938,7 +930,7 @@ def test_parse_with_multiple_formats
 EOS
 
         assert(parser.firstline?('Started GET "/users/123/" for 127.0.0.1...'))
-        assert_equal(str2time('2013-06-14 12:00:11 +0900').to_i, time)
+        assert_equal_event_time(event_time('2013-06-14 12:00:11 +0900'), time)
         assert_equal({
           "method" => "GET",
           "path" => "/users/123/",

From dbc5392f15f13377de1e7bbc28662472da0de6a2 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 29 Nov 2016 18:23:18 +0900
Subject: [PATCH 148/193] Port #1331 into v0.12

---
 lib/fluent/env.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/fluent/env.rb b/lib/fluent/env.rb
index b535f17ec5..4c1b22d654 100644
--- a/lib/fluent/env.rb
+++ b/lib/fluent/env.rb
@@ -18,7 +18,7 @@ module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
-  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat}
+  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
   DEFAULT_LISTEN_PORT = 24224
   DEFAULT_FILE_PERMISSION = 0644
   DEFAULT_DIR_PERMISSION = 0755

From a702b489c9b4c110e54ca7b1fe8f8866063f8718 Mon Sep 17 00:00:00 2001
From: "Satoshi \"Moris\" Tagomori" <tagomoris@gmail.com>
Date: Mon, 28 Nov 2016 11:02:56 +0900
Subject: [PATCH 149/193] Merge pull request #1325 from
 fluent/optimize-split-lines-in_tail

optimize to split lines
---
 lib/fluent/plugin/in_tail.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index b5c647074d..f5292524fd 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -579,8 +579,8 @@ def on_notify
                   else
                     @buffer << @io.readpartial(2048, @iobuf)
                   end
-                  while line = @buffer.slice!(/.*?\n/m)
-                    @lines << line
+                  while idx = @buffer.index("\n".freeze)
+                    @lines << @buffer.slice!(0, idx + 1)
                   end
                   if @lines.size >= @read_lines_limit
                     # not to use too much memory in case the file is very large

From 5a1a245da0e04dfbafcda0595db348a673231318 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 1 Dec 2016 03:53:42 +0900
Subject: [PATCH 150/193] filter_parser: Add emit_invalid_record_to_error
 parameter

v0.14's parser filter routes invalid record to @ERROR label.
This parameter enables same behaviour in v0.12.
Include refactoring.
---
 lib/fluent/plugin/filter_parser.rb | 52 +++++++++++++++---------------
 test/plugin/test_filter_parser.rb  | 33 ++++++++++++++++---
 2 files changed, 55 insertions(+), 30 deletions(-)

diff --git a/lib/fluent/plugin/filter_parser.rb b/lib/fluent/plugin/filter_parser.rb
index 3d57ae3f0d..7e3a812409 100644
--- a/lib/fluent/plugin/filter_parser.rb
+++ b/lib/fluent/plugin/filter_parser.rb
@@ -12,6 +12,7 @@ class Fluent::ParserFilter < Fluent::Filter
   config_param :suppress_parse_error_log, :bool, default: false
   config_param :time_parse, :bool, default: true
   config_param :ignore_key_not_exist, :bool, default: false
+  config_param :emit_invalid_record_to_error, :bool, default: false
 
   attr_reader :parser
 
@@ -34,7 +35,11 @@ def filter_stream(tag, es)
     es.each do |time,record|
       raw_value = record[@key_name]
       if raw_value.nil?
-        log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+        if @emit_invalid_record_to_error
+          router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        else
+          log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+        end
         new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
         next
       end
@@ -45,7 +50,11 @@ def filter_stream(tag, es)
             r = handle_parsed(tag, record, t, values)
             new_es.add(t, r)
           else
-            log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+            if @emit_invalid_record_to_error
+              router.emit_error_event(tag, time, record, ::Fluent::ParserError.new("pattern not match with data '#{raw_value}'"))
+            else
+              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+            end
             if @reserve_data
               t = time
               r = handle_parsed(tag, record, time, {})
@@ -53,33 +62,24 @@ def filter_stream(tag, es)
             end
           end
         end
-      rescue Fluent::TextParser::ParserError => e
-        log.warn e.message unless @suppress_parse_error_log
-      rescue ArgumentError => e
-        if @replace_invalid_sequence
-          unless e.message.index("invalid byte sequence in") == 0
-            raise
-          end
-          replaced_string = replace_invalid_byte(raw_value)
-          @parser.parse(replaced_string) do |t,values|
-            if values
-              t ||= time
-              r = handle_parsed(tag, record, t, values)
-              new_es.add(t, r)
-            else
-              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
-              if @reserve_data
-                t = time
-                r = handle_parsed(tag, record, time, {})
-                new_es.add(t, r)
-              end
-            end
-          end
+      rescue Fluent::ParserError => e
+        if @emit_invalid_record_to_error
+          router.emit_error_event(tag, time, record, e)
         else
-          raise
+          log.warn e.message unless @suppress_parse_error_log
         end
+      rescue ArgumentError => e
+        raise unless @replace_invalid_sequence
+        raise unless e.message.index("invalid byte sequence in") == 0
+
+        raw_value = replace_invalid_byte(raw_value)
+        retry
       rescue => e
-        log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+        if @emit_invalid_record_to_error
+          router.emit_error_event(tag, time, record, Fluent::ParserError.new("parse failed #{e.message}"))
+        else
+          log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+        end
       end
     end
     new_es
diff --git a/test/plugin/test_filter_parser.rb b/test/plugin/test_filter_parser.rb
index 64b413b000..e469da6037 100644
--- a/test/plugin/test_filter_parser.rb
+++ b/test/plugin/test_filter_parser.rb
@@ -1,8 +1,11 @@
 require_relative '../helper'
 require 'timecop'
 require 'fluent/plugin/filter_parser'
+require 'flexmock/test_unit'
 
 class ParserFilterTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   setup do
     Fluent::Test.setup
     @time = Time.parse("2012-01-02 13:14:15")
@@ -588,6 +591,15 @@ def test_filter_key_not_exist
     assert_equal 1, filtered.length
     assert_nil filtered[0][2]['data']
     assert_equal 'bar', filtered[0][2]['foo']
+
+    d = create_driver(CONFIG_NOT_IGNORE + "emit_invalid_record_to_error true", 'test.no.ignore')
+    assert_nothing_raised {
+      flexmock(d.instance.router).should_receive(:emit_error_event).
+        with(String, Integer, Hash, ArgumentError).once
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
   end
 
   # suppress_parse_error_log test
@@ -689,13 +701,11 @@ def test_raise_exception
     end
 
     def test_nothing_raised
-      swap_logger(@d.instance) do
       assert_nothing_raised {
         @d.run do
           @d.filter({'message' => VALID_MESSAGE}, Time.now.to_i)
         end
       }
-      end
     end
   end
 
@@ -706,14 +716,29 @@ def setup
     end
 
     def test_nothing_raised
-      swap_logger(@d.instance) do
       assert_nothing_raised {
         @d.run do
           @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
           @d.filter({'message' => VALID_MESSAGE},   Time.now.to_i)
         end
       }
-      end
+    end
+  end
+
+  class EmitInvalidRecordErrorTest < self
+    def setup
+      # enabled 'suppress_parse_error_log'
+      @d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG + "emit_invalid_record_to_error true", 'test.no.change')
+    end
+
+    def test_nothing_raised
+      flexmock(@d.instance.router).should_receive(:emit_error_event).
+        with(String, Integer, Hash, Fluent::ParserError).once
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+        end
+      }
     end
   end
 end

From 0c44166f8d1c355c6095de411006ab963c8b6dc4 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 1 Dec 2016 13:54:45 +0900
Subject: [PATCH 151/193] v0.12.30

---
 ChangeLog             | 20 ++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index fe7ed43262..9eb11e7c9f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,25 @@
 # v0.12
 
+## Release 0.12.30 - 2016/12/01
+
+### New features / Enhancement
+
+* in_tail: Optimize to split lines
+  https://github.com/fluent/fluentd/pull/1325
+* in_monitor_agent: Port #1317 changes
+  https://github.com/fluent/fluentd/pull/1317
+* filter_parser: Add emit_invalid_record_to_error parameter
+  https://github.com/fluent/fluentd/pull/1339
+* test: Add v0.14 compatible test helpers
+  https://github.com/fluent/fluentd/pull/1322
+
+### Bug fixes
+
+* Use compat mode instead of stric mode for oj
+  https://github.com/fluent/fluentd/pull/1304
+* Port #1331 changes for oj
+  https://github.com/fluent/fluentd/pull/1331
+
 ## Release 0.12.29 - 2016/09/05
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 298a68829b..ea64886bf7 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.29'
+  VERSION = '0.12.30'
 
 end

From 0d5b43452a3d4dc19bb78aaa7dae50a4bba6585c Mon Sep 17 00:00:00 2001
From: autopp <autopp@gmail.com>
Date: Fri, 2 Dec 2016 19:42:52 +0900
Subject: [PATCH 152/193] ensure to sleep even if command or parsing fails

---
 lib/fluent/plugin/in_exec.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_exec.rb b/lib/fluent/plugin/in_exec.rb
index b9274400ca..d9806cd602 100644
--- a/lib/fluent/plugin/in_exec.rb
+++ b/lib/fluent/plugin/in_exec.rb
@@ -140,10 +140,11 @@ def run_periodic
           io = IO.popen(@command, "r")
           @parser.call(io)
           Process.waitpid(io.pid)
-          sleep @run_interval
         rescue
           log.error "exec failed to run or shutdown child process", error: $!.to_s, error_class: $!.class.to_s
           log.warn_backtrace $!.backtrace
+        ensure
+          sleep @run_interval
         end
       end
     end

From 8ec6af0def08d8f4ec895688d9f75c52f1f38271 Mon Sep 17 00:00:00 2001
From: autopp <autopp@gmail.com>
Date: Fri, 2 Dec 2016 19:45:49 +0900
Subject: [PATCH 153/193] add tests for the case of invalid command given

For testing, add new mode to exec_script.rb.
In this mode, exec_script.rb output a charactor to stderr and then
output a invalid json string to stdout.

The test case checks that the total length of command's stderr (this
means the count of command execution) is appropriate.
---
 test/plugin/test_in_exec.rb | 27 +++++++++++++++++++++++++++
 test/scripts/exec_script.rb |  7 +++++++
 2 files changed, 34 insertions(+)

diff --git a/test/plugin/test_in_exec.rb b/test/plugin/test_in_exec.rb
index 83c47166bb..70c975e7e1 100644
--- a/test/plugin/test_in_exec.rb
+++ b/test/plugin/test_in_exec.rb
@@ -7,6 +7,11 @@ def setup
     Fluent::Test.setup
     @test_time = Time.parse("2011-01-02 13:14:15").to_i
     @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
+    @count_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'count.txt'))
+  end
+
+  def teardown
+    FileUtils.rm_f(@count_file)
   end
 
   def create_driver(conf = tsv_config)
@@ -53,6 +58,17 @@ def regexp_config
     ]
   end
 
+  def invalid_json_config
+    # For counting command execution, redirect stderr to file
+    %[
+      command ruby #{@script} #{@test_time} 4 2>>#{@count_file}
+      format json
+      tag_key tag
+      time_key time
+      run_interval 0.5
+    ]
+  end
+
   def test_configure
     d = create_driver
     assert_equal 'tsv', d.instance.format
@@ -130,4 +146,15 @@ def test_emit_regexp
     assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
     assert_equal @test_time, emits[0][1]
   end
+
+  def test_emit_with_invalid_script
+    d = create_driver invalid_json_config
+
+    d.run do
+      sleep 2
+    end
+
+    assert_equal true, d.emits.empty?
+    assert_equal true, File.read(@count_file).length.between?(1, 4)
+  end
 end
diff --git a/test/scripts/exec_script.rb b/test/scripts/exec_script.rb
index 0e41ab0fe3..569bd39aee 100644
--- a/test/scripts/exec_script.rb
+++ b/test/scripts/exec_script.rb
@@ -17,6 +17,11 @@ def gen_raw_string(time)
   "#{time} hello"
 end
 
+def gen_invalid_json(time)
+  $stderr.print '*' # For counting command exections
+  %({"tag": "tag1", "time": #{time}, "k1": "ok", })
+end
+
 time = ARGV.first
 time = Integer(time) rescue time
 
@@ -29,4 +34,6 @@ def gen_raw_string(time)
   print gen_msgpack(time)
 when 3
   print gen_raw_string(time)
+when 4
+  print gen_invalid_json(time)
 end

From b68a05dbfb72e4918e769b5e4f6dca8be8b64495 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 9 Dec 2016 20:50:30 +0900
Subject: [PATCH 154/193] formatter_csv: Change fields parameter to required

---
 lib/fluent/formatter.rb | 12 ++++++------
 test/test_formatter.rb  |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index 0278f8dcf7..cb05db38a8 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -196,18 +196,18 @@ class CsvFormatter < Formatter
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
       config_param :force_quotes, :bool, default: true
-      config_param :fields, default: [] do |val|
-        val.split(',').map do |f|
-          f.strip!
-          f.size > 0 ? f : nil
-        end.compact
-      end
+      config_param :fields, :array, value_type: :string
 
       def initialize
         super
         require 'csv'
       end
 
+      def configure(conf)
+        super
+        @fields = fields.select { |f| !f.empty? }
+      end
+
       def format(tag, time, record)
         filter_record(tag, time, record)
         row = @fields.inject([]) do |memo, key|
diff --git a/test/test_formatter.rb b/test/test_formatter.rb
index 6eec9f9d01..a2744d0793 100644
--- a/test/test_formatter.rb
+++ b/test/test_formatter.rb
@@ -289,7 +289,7 @@ def setup
     def test_config_params
       assert_equal ',', @formatter.delimiter
       assert_equal true, @formatter.force_quotes
-      assert_equal [], @formatter.fields
+      assert_nil @formatter.fields
     end
 
     data(
@@ -298,7 +298,7 @@ def test_config_params
       'pipe' => ['|', '|'])
     def test_config_params_with_customized_delimiters(data)
       expected, target = data
-      @formatter.configure('delimiter' => target)
+      @formatter.configure('fields' => 'f1', 'delimiter' => target)
       assert_equal expected, @formatter.delimiter
     end
 

From 8376ea4f098d894df1a1ccc47dd8c7fb971981b4 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 13 Dec 2016 15:25:52 +0900
Subject: [PATCH 155/193] Add slow_flush_threshold parameter

---
 lib/fluent/output.rb | 10 +++++++
 test/test_output.rb  | 62 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index da829c0335..466e00d95c 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -203,6 +203,8 @@ def initialize
     config_param :max_retry_wait, :time, default: nil
     desc 'The number of threads to flush the buffer.'
     config_param :num_threads, :integer, default: 1
+    desc 'The threshold to show slow flush logs'
+    config_param :slow_flush_threshold, :float, default: 5.0
     desc 'The interval between data flushes for queued chunk.'
     config_param :queued_chunk_flush_interval, :time, default: 1
 
@@ -332,12 +334,20 @@ def try_flush
           end
         end
 
+        chunk_write_start = Time.now
+
         if @secondary && !@disable_retry_limit && @num_errors > @retry_limit
           has_next = flush_secondary(@secondary)
         else
           has_next = @buffer.pop(self)
         end
 
+        elapsed_time = Time.now - chunk_write_start
+        if elapsed_time > @slow_flush_threshold
+          $log.warn "buffer flush took longer time than slow_flush_threshold",
+                    plugin_id: plugin_id, elapsed_time: elapsed_time, slow_flush_threshold: @slow_flush_threshold
+        end
+
         # success
         if retrying
           @num_errors = 0
diff --git a/test/test_output.rb b/test/test_output.rb
index 774fd34a20..bc081ed1d5 100644
--- a/test/test_output.rb
+++ b/test/test_output.rb
@@ -201,6 +201,68 @@ def test_secondary
         10.times { sleep 0.05 }
       end
     end
+
+    sub_test_case "test slow_flush_threshold" do
+      def create_slow_driver(conf, sleep_time)
+        d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+          attr_accessor :sleep_time
+
+          def initialize
+            super
+            @_written_check_mutex = Mutex.new
+            @_written_check = false
+          end
+
+          def written_check
+            @_written_check_mutex.synchronize { @_written_check }
+          end
+
+          def write(chunk)
+            sleep @sleep_time
+          end
+
+          def try_flush
+            super
+            @_written_check_mutex.synchronize { @_written_check = true }
+          end
+        end.configure(conf)
+        d.instance.sleep_time = sleep_time
+        d
+      end
+
+      def setup
+        # output#try_flush uses $log, so it should be replaced
+        @orig_log = $log
+        $log = Fluent::Test::TestLogger.new
+      end
+
+      def teardown
+        $log = @orig_log
+      end
+
+      test "flush took longer time than slow_flush_threshold" do
+        d = create_slow_driver(%[
+          slow_flush_threshold 0.5
+        ], 1.5)
+        run_driver(d)
+        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_threshold/ }
+      end
+
+      test "flush took lower time than slow_flush_threshold. No logs" do
+        d = create_slow_driver(%[
+          slow_flush_threshold 5.0
+        ], 0)
+        run_driver(d)
+        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_threshold/ }
+      end
+
+      def run_driver(d)
+        d.instance.start
+        d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
+        d.instance.force_flush
+        until d.instance.written_check; end
+      end
+    end
   end
 
   class TimeSlicedOutputTest < ::Test::Unit::TestCase

From 856708bdba62e982e95f13d09753e28476da239a Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 13 Dec 2016 16:23:43 +0900
Subject: [PATCH 156/193] Rename to slow_flush_log_threshold

---
 lib/fluent/output.rb |  8 ++++----
 test/test_output.rb  | 14 +++++++-------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index 466e00d95c..e833c56f6f 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -204,7 +204,7 @@ def initialize
     desc 'The number of threads to flush the buffer.'
     config_param :num_threads, :integer, default: 1
     desc 'The threshold to show slow flush logs'
-    config_param :slow_flush_threshold, :float, default: 5.0
+    config_param :slow_flush_log_threshold, :float, default: 5.0
     desc 'The interval between data flushes for queued chunk.'
     config_param :queued_chunk_flush_interval, :time, default: 1
 
@@ -343,9 +343,9 @@ def try_flush
         end
 
         elapsed_time = Time.now - chunk_write_start
-        if elapsed_time > @slow_flush_threshold
-          $log.warn "buffer flush took longer time than slow_flush_threshold",
-                    plugin_id: plugin_id, elapsed_time: elapsed_time, slow_flush_threshold: @slow_flush_threshold
+        if elapsed_time > @slow_flush_log_threshold
+          $log.warn "buffer flush took longer time than slow_flush_log_threshold",
+                    plugin_id: plugin_id, elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold
         end
 
         # success
diff --git a/test/test_output.rb b/test/test_output.rb
index bc081ed1d5..c4fb6effc5 100644
--- a/test/test_output.rb
+++ b/test/test_output.rb
@@ -202,7 +202,7 @@ def test_secondary
       end
     end
 
-    sub_test_case "test slow_flush_threshold" do
+    sub_test_case "test slow_flush_log_threshold" do
       def create_slow_driver(conf, sleep_time)
         d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
           attr_accessor :sleep_time
@@ -240,20 +240,20 @@ def teardown
         $log = @orig_log
       end
 
-      test "flush took longer time than slow_flush_threshold" do
+      test "flush took longer time than slow_flush_log_threshold" do
         d = create_slow_driver(%[
-          slow_flush_threshold 0.5
+          slow_flush_log_threshold 0.5
         ], 1.5)
         run_driver(d)
-        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_threshold/ }
+        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold/ }
       end
 
-      test "flush took lower time than slow_flush_threshold. No logs" do
+      test "flush took lower time than slow_flush_log_threshold. No logs" do
         d = create_slow_driver(%[
-          slow_flush_threshold 5.0
+          slow_flush_log_threshold 5.0
         ], 0)
         run_driver(d)
-        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_threshold/ }
+        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold/ }
       end
 
       def run_driver(d)

From 6e31e61fcf7bdc4fcae69da4cd075f9306e3522e Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 13 Dec 2016 16:28:09 +0900
Subject: [PATCH 157/193] Change to larger value to suppress warning for
 external services

---
 lib/fluent/output.rb | 3 ++-
 test/test_output.rb  | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index e833c56f6f..4ffdbe0a63 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -204,7 +204,7 @@ def initialize
     desc 'The number of threads to flush the buffer.'
     config_param :num_threads, :integer, default: 1
     desc 'The threshold to show slow flush logs'
-    config_param :slow_flush_log_threshold, :float, default: 5.0
+    config_param :slow_flush_log_threshold, :float, default: 20.0
     desc 'The interval between data flushes for queued chunk.'
     config_param :queued_chunk_flush_interval, :time, default: 1
 
@@ -510,6 +510,7 @@ def initialize
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit
     config_set_default :flush_interval, nil
+    config_set_default :slow_flush_log_threshold, 40.0
 
     attr_accessor :localtime
     attr_reader :time_slicer # for test
diff --git a/test/test_output.rb b/test/test_output.rb
index c4fb6effc5..c6495a3c67 100644
--- a/test/test_output.rb
+++ b/test/test_output.rb
@@ -47,6 +47,7 @@ def test_configure
       assert_equal nil, d.instance.max_retry_wait
       assert_equal 1.0, d.instance.retry_wait
       assert_equal 1, d.instance.num_threads
+      assert_equal 20.0, d.instance.slow_flush_log_threshold
       assert_equal 1, d.instance.queued_chunk_flush_interval
 
       # max_retry_wait

From 06707c9bd85fe0021f69ce72bcc1f3e737cccc18 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 13 Dec 2016 17:00:49 +0900
Subject: [PATCH 158/193] raise ConfigError when fields are empty

---
 lib/fluent/formatter.rb | 1 +
 test/test_formatter.rb  | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index cb05db38a8..f9b52b036a 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -206,6 +206,7 @@ def initialize
       def configure(conf)
         super
         @fields = fields.select { |f| !f.empty? }
+        raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
       end
 
       def format(tag, time, record)
diff --git a/test/test_formatter.rb b/test/test_formatter.rb
index a2744d0793..8d50b34f60 100644
--- a/test/test_formatter.rb
+++ b/test/test_formatter.rb
@@ -292,6 +292,14 @@ def test_config_params
       assert_nil @formatter.fields
     end
 
+    data('empty array' => [],
+         'array including empty string' => ['', ''])
+    def test_empty_fields(param)
+      assert_raise ConfigError do
+        @formatter.configure('fields' => param)
+      end
+    end
+
     data(
       'tab_char' => ["\t", '\t'],
       'tab_string' => ["\t", 'TAB'],

From c78bc7f85b1aad59b15857ff94d7a5df86e2f5cc Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 13 Dec 2016 17:01:25 +0900
Subject: [PATCH 159/193] Reduce memory allocation in each format call

---
 lib/fluent/formatter.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index f9b52b036a..2fb8b70694 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -207,6 +207,8 @@ def configure(conf)
         super
         @fields = fields.select { |f| !f.empty? }
         raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
+
+        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
       end
 
       def format(tag, time, record)
@@ -215,8 +217,7 @@ def format(tag, time, record)
             memo << record[key]
             memo
         end
-        CSV.generate_line(row, col_sep: @delimiter,
-                          force_quotes: @force_quotes)
+        CSV.generate_line(row, @generate_opts)
       end
     end
 

From c26a308b76a09f8253fb5d6cc320af212410caf0 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 14 Dec 2016 12:12:17 +0900
Subject: [PATCH 160/193] Add : to log

---
 lib/fluent/output.rb | 2 +-
 test/test_output.rb  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/fluent/output.rb b/lib/fluent/output.rb
index 4ffdbe0a63..5d3eb13317 100644
--- a/lib/fluent/output.rb
+++ b/lib/fluent/output.rb
@@ -344,7 +344,7 @@ def try_flush
 
         elapsed_time = Time.now - chunk_write_start
         if elapsed_time > @slow_flush_log_threshold
-          $log.warn "buffer flush took longer time than slow_flush_log_threshold",
+          $log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                     plugin_id: plugin_id, elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold
         end
 
diff --git a/test/test_output.rb b/test/test_output.rb
index c6495a3c67..249fa08275 100644
--- a/test/test_output.rb
+++ b/test/test_output.rb
@@ -246,7 +246,7 @@ def teardown
           slow_flush_log_threshold 0.5
         ], 1.5)
         run_driver(d)
-        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold/ }
+        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold:/ }
       end
 
       test "flush took lower time than slow_flush_log_threshold. No logs" do
@@ -254,7 +254,7 @@ def teardown
           slow_flush_log_threshold 5.0
         ], 0)
         run_driver(d)
-        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold/ }
+        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold:/ }
       end
 
       def run_driver(d)

From 1863653ed47c925780aa54f667ed5fc21d8618d2 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 8 Dec 2016 11:13:18 +0900
Subject: [PATCH 161/193] Merge pull request #1351 from knqyf263/add_options

---
 lib/fluent/plugin/in_syslog.rb | 31 ++++++++++++------
 test/plugin/test_in_syslog.rb  | 57 ++++++++++++++++++++++++++++++----
 2 files changed, 73 insertions(+), 15 deletions(-)

diff --git a/lib/fluent/plugin/in_syslog.rb b/lib/fluent/plugin/in_syslog.rb
index 11a9326ff4..f982cbd0b7 100644
--- a/lib/fluent/plugin/in_syslog.rb
+++ b/lib/fluent/plugin/in_syslog.rb
@@ -91,6 +91,10 @@ def initialize
     config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'The field name of the priority.'
+    config_param :priority_key, :string, default: nil
+    desc 'The field name of the facility.'
+    config_param :facility_key, :string, default: nil
     config_param :blocking_timeout, :time, default: 0.5
     config_param :message_length_limit, :size, default: 2048
 
@@ -153,8 +157,15 @@ def receive_data_parser(data, addr)
           return
         end
 
+        facility = FACILITY_MAP[pri >> 3]
+        priority = PRIORITY_MAP[pri & 0b111]
+
+        record[@priority_key] = priority if @priority_key
+        record[@facility_key] = facility if @facility_key
         record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
+
+        tag = "#{@tag}.#{facility}.#{priority}"
+        emit(tag, time, record)
       }
     rescue => e
       log.error data.dump, error: e.to_s
@@ -168,9 +179,16 @@ def receive_data(data, addr)
           return
         end
 
-        pri = record.delete('pri')
+        pri = record.delete('pri'.freeze)
+        facility = FACILITY_MAP[pri >> 3]
+        priority = PRIORITY_MAP[pri & 0b111]
+
+        record[@priority_key] = priority if @priority_key
+        record[@facility_key] = facility if @facility_key
         record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
+
+        tag = "#{@tag}.#{facility}.#{priority}"
+        emit(tag, time, record)
       }
     rescue => e
       log.error data.dump, error: e.to_s
@@ -191,12 +209,7 @@ def listen(callback)
       end
     end
 
-    def emit(pri, time, record)
-      facility = FACILITY_MAP[pri >> 3]
-      priority = PRIORITY_MAP[pri & 0b111]
-
-      tag = "#{@tag}.#{facility}.#{priority}"
-
+    def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
       log.error "syslog failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
diff --git a/test/plugin/test_in_syslog.rb b/test/plugin/test_in_syslog.rb
index 1cad0fc1d4..d1be8a68db 100755
--- a/test/plugin/test_in_syslog.rb
+++ b/test/plugin/test_in_syslog.rb
@@ -57,6 +57,7 @@ def test_time_format
       end
 
       emits = d.emits
+      assert_equal 2, emits.size
       emits.each_index {|i|
         assert_equal(tests[i]['expected'], emits[i][1])
       }
@@ -76,6 +77,7 @@ def test_msg_size
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -94,6 +96,7 @@ def test_msg_size_udp_for_large_msg
       sleep 1
     end
 
+    assert_equal 3, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -110,6 +113,7 @@ def test_msg_size_with_tcp
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -126,6 +130,7 @@ def test_msg_size_with_same_tcp_connection
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -146,6 +151,7 @@ def test_msg_size_with_json_format
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -164,7 +170,44 @@ def test_msg_size_with_include_source_host
       sleep 1
     end
 
-    compare_test_result(d.emits, tests, host)
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests, {host: host})
+  end
+
+  def test_msg_size_with_include_priority
+    d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
+    tests = create_test_case
+
+    priority = 'info'
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests, {priority: priority})
+  end
+
+  def test_msg_size_with_include_facility
+    d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
+    tests = create_test_case
+
+    facility = 'kern'
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests, {facility: facility})
   end
 
   def create_test_case(large_message = false)
@@ -183,11 +226,13 @@ def create_test_case(large_message = false)
     end
   end
 
-  def compare_test_result(emits, tests, host = nil)
-    emits.each_index { |i|
-      assert_equal('syslog.kern.info', emits[0][0]) # <6> means kern.info
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
-      assert_equal(host, emits[i][2]['source_host']) if host
+  def compare_test_result(events, tests, options = {})
+    events.each_index { |i|
+      assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
+      assert_equal(tests[i]['expected'], events[i][2]['message'])
+      assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
+      assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
+      assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
 end

From 419fd9ea7cb491a12ad44c344265f3927f66c2de Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 14 Dec 2016 21:09:29 +0900
Subject: [PATCH 162/193] Update development gems to follow v0.14

---
 fluentd.gemspec | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index 5b0de02fe3..469f245cf0 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -30,12 +30,12 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock", ["~> 1.3.3"])
-  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
-  gem.add_development_dependency("simplecov", ["~> 0.6.4"])
-  gem.add_development_dependency("rr", [">= 1.0.0"])
-  gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
-  gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
+  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.7"])
+  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("timecop", ["~> 0.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", ["~> 2.14"])
 end

From 55f0d0f785abbf7b09ca2fe4a47df6dfeaed3bb8 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 15 Dec 2016 08:25:47 +0900
Subject: [PATCH 163/193] v0.12.31

---
 ChangeLog             | 11 +++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 9eb11e7c9f..09b65839e2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,16 @@
 # v0.12
 
+## Release 0.12.31 - 2016/12/14
+
+### New features / Enhancement
+
+* output: Add slow_flush_log_threshold parameter
+  https://github.com/fluent/fluentd/pull/1366
+* formatter_csv: Change fields parameter to required. Now accepts both a,b and ["a", "b"]
+  https://github.com/fluent/fluentd/pull/1361
+* in_syslog: Add "priority_key" and "facility_key" parameters
+  https://github.com/fluent/fluentd/pull/1351
+
 ## Release 0.12.30 - 2016/12/01
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index ea64886bf7..1a497debd7 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.30'
+  VERSION = '0.12.31'
 
 end

From 649f7baebb4bd022ef27302b9509300b3f7cb8fc Mon Sep 17 00:00:00 2001
From: Akira Tanimura <atanimur@yahoo-corp.jp>
Date: Thu, 15 Dec 2016 18:30:12 +0900
Subject: [PATCH 164/193] check log lines instead of stderr for counting
 command exectuion

---
 test/plugin/test_in_exec.rb | 10 +++-------
 test/scripts/exec_script.rb |  1 -
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/test/plugin/test_in_exec.rb b/test/plugin/test_in_exec.rb
index 70c975e7e1..8b9667fc2b 100644
--- a/test/plugin/test_in_exec.rb
+++ b/test/plugin/test_in_exec.rb
@@ -7,11 +7,6 @@ def setup
     Fluent::Test.setup
     @test_time = Time.parse("2011-01-02 13:14:15").to_i
     @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
-    @count_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'count.txt'))
-  end
-
-  def teardown
-    FileUtils.rm_f(@count_file)
   end
 
   def create_driver(conf = tsv_config)
@@ -61,7 +56,7 @@ def regexp_config
   def invalid_json_config
     # For counting command execution, redirect stderr to file
     %[
-      command ruby #{@script} #{@test_time} 4 2>>#{@count_file}
+      command ruby #{@script} #{@test_time} 4
       format json
       tag_key tag
       time_key time
@@ -155,6 +150,7 @@ def test_emit_with_invalid_script
     end
 
     assert_equal true, d.emits.empty?
-    assert_equal true, File.read(@count_file).length.between?(1, 4)
+    logs = d.instance.log.logs
+    assert_equal true, logs.count { |line| line =~ /exec failed to run or shutdown child process/ }.between?(1, 4)
   end
 end
diff --git a/test/scripts/exec_script.rb b/test/scripts/exec_script.rb
index 569bd39aee..256ffbd620 100644
--- a/test/scripts/exec_script.rb
+++ b/test/scripts/exec_script.rb
@@ -18,7 +18,6 @@ def gen_raw_string(time)
 end
 
 def gen_invalid_json(time)
-  $stderr.print '*' # For counting command exections
   %({"tag": "tag1", "time": #{time}, "k1": "ok", })
 end
 

From cbf8fc287262dcf36fc4b914db0aaf533449f8b0 Mon Sep 17 00:00:00 2001
From: autopp <autopp.inc@gmail.com>
Date: Thu, 5 Jan 2017 18:29:10 +0900
Subject: [PATCH 165/193] port query parameter with_ivars to v0.12

---
 lib/fluent/plugin/in_monitor_agent.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index dd05608c98..7a06657fa4 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -83,6 +83,10 @@ def build_object(req, res)
           opts[:pretty_json] = true
         end
 
+        if ivars = (qs['with_ivars'] || []).first
+          opts[:ivars] = ivars.split(',')
+        end
+
         if with_config = get_search_parameter(qs, 'with_config'.freeze)
           opts[:with_config] = Fluent::Config.bool_value(with_config)
         end
@@ -415,6 +419,13 @@ def get_monitor_info(pe, opts={})
           }
         end
         obj['instance_variables'] = iv
+      elsif ivars = opts[:ivars]
+        iv = {}
+        ivars.each {|name|
+          iname = "@#{name}"
+          iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
+        }
+        obj['instance_variables'] = iv
       end
 
       obj

From f57a6aa1431e38a5d3ee5582a7e536f0a64a9bef Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 26 Jan 2017 07:16:03 +0900
Subject: [PATCH 166/193] Merge pull request #1421 from
 jitran/emit-unmatched-lines

Capture unmatched lines
---
 lib/fluent/plugin/in_tail.rb | 10 ++++++
 test/plugin/test_in_tail.rb  | 67 +++++++++++++++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index f5292524fd..9e901c5f31 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -48,6 +48,8 @@ def initialize
     config_param :read_lines_limit, :integer, default: 1000
     desc 'The interval of flushing the buffer for multiline format'
     config_param :multiline_flush_interval, :time, default: nil
+    desc 'Enable the option to emit unmatched lines.'
+    config_param :emit_unmatched_lines, :bool, default: false
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
     desc 'The encoding after conversion of the input.'
@@ -315,6 +317,11 @@ def convert_line_to_event(line, es, tail_watcher)
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?
             es.add(time, record)
           else
+            if @emit_unmatched_lines
+              record = {'unmatched_line' => line}
+              record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+              es.add(::Fluent::Engine.now, record)
+            end
             log.warn "pattern not match: #{line.inspect}"
           end
         }
@@ -345,6 +352,9 @@ def parse_multilines(lines, tail_watcher)
             lb = line
           else
             if lb.nil?
+              if @emit_unmatched_lines
+                convert_line_to_event(line, es, tail_watcher)
+              end
               log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
             else
               lb << line
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index fcb1ad92fc..3999c02738 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -97,7 +97,6 @@ def test_emit
     }
 
     d = create_driver
-
     d.run do
       sleep 1
 
@@ -115,6 +114,34 @@ def test_emit
     assert_equal(1, d.emit_streams.size)
   end
 
+  def test_emit_with_emit_unmatched_lines_true
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+    d = create_driver(%[
+      format /^(?<message>test.*)/
+      emit_unmatched_lines true
+    ])
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+        f.puts "test line 1"
+        f.puts "test line 2"
+        f.puts "bad line 1"
+        f.puts "test line 3"
+      }
+
+      sleep 1
+    end
+
+    events = d.emits
+    assert_equal(4, events.length)
+    assert_equal({"message" => "test line 1"}, events[0][2])
+    assert_equal({"message" => "test line 2"}, events[1][2])
+    assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
+    assert_equal({"message" => "test line 3"}, events[3][2])
+  end
+
   data('1' => [1, 2], '10' => [10, 1])
   def test_emit_with_read_lines_limit(data)
     limit, num_emits = data
@@ -419,6 +446,44 @@ def test_multiline
     assert_equal({"message1" => "test8"}, emits[3][2])
   end
 
+  def test_multiline_with_emit_unmatched_lines_true
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      emit_unmatched_lines true
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+
+      events = d.emits
+      assert_equal(4, events.length)
+      assert_equal({"unmatched_line" => "f test1"}, events[0][2])
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
+      assert_equal({"message1" => "test5"}, events[2][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
+
+      sleep 3
+      assert_equal(4, d.emits.length)
+    end
+
+    emits = d.emits
+    assert_equal(5, emits.length)
+    assert_equal({"message1" => "test8"}, emits[4][2])
+  end
+
   def test_multiline_with_flush_interval
     File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 

From c66ef656ceb20055a6f15b449a351d2b96acfa1f Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 26 Jan 2017 12:05:43 +0900
Subject: [PATCH 167/193] Merge pull request #1433 from
 fluent/move-remove-keys-to-last

record_transformer: remove_keys processing should be last
---
 .../plugin/filter_record_transformer.rb       |  8 +++-----
 test/plugin/test_filter_record_transformer.rb | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
index 9396e38ad9..1320d70fa6 100644
--- a/lib/fluent/plugin/filter_record_transformer.rb
+++ b/lib/fluent/plugin/filter_record_transformer.rb
@@ -95,14 +95,13 @@ def filter_stream(tag, es)
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        placeholder_values.merge!({
-          'time'     => @placeholder_expander.time_value(time),
-          'record'   => record,
-        })
+        placeholder_values['time'] = @placeholder_expander.time_value(time)
+        placeholder_values['record'] = record
         new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = new_record[@renew_time_key].to_i
         end
+        @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
         new_es.add(time, new_record)
       end
       new_es
@@ -131,7 +130,6 @@ def reform(record, placeholder_values)
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
-      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end
diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
index cfc47fd913..a8408d1227 100644
--- a/test/plugin/test_filter_record_transformer.rb
+++ b/test/plugin/test_filter_record_transformer.rb
@@ -99,6 +99,25 @@ def emit(config, msgs = [''])
       es = emit(config, msgs)
       es.each_with_index do |(time, record), i|
         assert_equal(times[i].to_i, time)
+        assert_true(record.has_key?('message'))
+      end
+    end
+
+    test 'renew_time_key and remove_keys' do
+      config = %[
+                 renew_time_key event_time_key
+                 remove_keys event_time_key
+                 auto_typecast true
+                 <record>
+                   event_time_key ${record["message"]}
+                 </record>
+               ]
+      times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]
+      msgs = times.map { |t| t.to_f.to_s }
+      filtered = emit(config, msgs)
+      filtered.each_with_index do |(time, _record), i|
+        assert_equal(times[i].to_i, time)
+        assert_false(_record.has_key?('event_time_key'))
       end
     end
 

From 13f731375235579443b036e3a6bba742691b1526 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 2 Feb 2017 12:11:03 +0900
Subject: [PATCH 168/193] Port #1420 to v0.12

---
 lib/fluent/formatter.rb | 26 +++++++++++++++++++++++---
 test/test_formatter.rb  | 24 ++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/lib/fluent/formatter.rb b/lib/fluent/formatter.rb
index 2fb8b70694..1fa6e5fc4f 100644
--- a/lib/fluent/formatter.rb
+++ b/lib/fluent/formatter.rb
@@ -135,6 +135,7 @@ class JSONFormatter < Formatter
       include StructuredFormatMixin
 
       config_param :json_parser, :string, default: 'oj'
+      config_param :add_newline, :bool, default: true
 
       def configure(conf)
         super
@@ -147,19 +148,34 @@ def configure(conf)
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
         end
+
+        # format json is used on various highload environment, so re-define method to skip if check
+        unless @add_newline
+          define_singleton_method(:format, method(:format_without_nl))
+        end
       end
 
       def format_record(record)
         "#{@dump_proc.call(record)}\n"
       end
+
+      def format_without_nl(tag, time, record)
+        @dump_proc.call(record)
+      end
     end
 
     class HashFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
+      config_param :add_newline, :bool, default: true
+
       def format_record(record)
-        "#{record.to_s}\n"
+        if @add_newline
+          "#{record.to_s}\n"
+        else
+          record.to_s
+        end
       end
     end
 
@@ -177,6 +193,7 @@ class LabeledTSVFormatter < Formatter
 
       config_param :delimiter, :string, default: "\t"
       config_param :label_delimiter, :string, default: ":"
+      config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         filter_record(tag, time, record)
@@ -184,7 +201,7 @@ def format(tag, time, record)
           result << @delimiter if result.length.nonzero?
           result << "#{pair.first}#{@label_delimiter}#{pair.last}"
         }
-        formatted << "\n"
+        formatted << "\n".freeze if @add_newline
         formatted
       end
     end
@@ -197,6 +214,7 @@ class CsvFormatter < Formatter
       end
       config_param :force_quotes, :bool, default: true
       config_param :fields, :array, value_type: :string
+      config_param :add_newline, :bool, default: true
 
       def initialize
         super
@@ -217,7 +235,9 @@ def format(tag, time, record)
             memo << record[key]
             memo
         end
-        CSV.generate_line(row, @generate_opts)
+        line = CSV.generate_line(row, @generate_opts)
+        line.chomp! unless @add_newline
+        line
       end
     end
 
diff --git a/test/test_formatter.rb b/test/test_formatter.rb
index 8d50b34f60..a92d463f17 100644
--- a/test/test_formatter.rb
+++ b/test/test_formatter.rb
@@ -142,6 +142,14 @@ def test_format(data)
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_without_nl(data)
+      @formatter.configure('json_parser' => data, 'add_newline' => false)
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal(Yajl.dump(record), formatted)
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_format_with_symbolic_record(data)
       @formatter.configure('json_parser' => data)
@@ -252,6 +260,13 @@ def test_format
       assert_equal("message:awesome\n", formatted)
     end
 
+    def test_format_without_nl
+      @formatter.configure('add_newline' => false)
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message:awesome", formatted)
+    end
+
     def test_format_with_tag
       @formatter.configure('include_tag_key' => 'true')
       formatted = @formatter.format(tag, @time, record)
@@ -319,6 +334,15 @@ def test_format
       assert_equal("\"awesome\",\"awesome2\"\n", formatted)
     end
 
+    def test_format_without_nl
+      @formatter.configure('fields' => 'message,message2', 'add_newline' => false)
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("\"awesome\",\"awesome2\"", formatted)
+    end
+
     def test_format_with_tag
       @formatter.configure(
         'fields' => 'tag,message,message2',

From e50a98ce518cadb3a37f76cde67ba8c6e076f638 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 2 Feb 2017 14:33:59 +0900
Subject: [PATCH 169/193] Allow msgpack v1

---
 fluentd.gemspec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fluentd.gemspec b/fluentd.gemspec
index 469f245cf0..697af2779d 100644
--- a/fluentd.gemspec
+++ b/fluentd.gemspec
@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.5.11", "< 0.6.0"])
+  gem.add_runtime_dependency("msgpack", [">= 0.5.11", "< 2"])
   gem.add_runtime_dependency("json", [">= 1.4.3"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.2.2", "< 2.0.0"])

From be13675ee9a8eafd62764e0828473c5426e34e26 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 2 Feb 2017 14:34:34 +0900
Subject: [PATCH 170/193] travis-ci: Update testing ruby versions

---
 .travis.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9c79bb8e12..9a5ec3a25b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,10 +3,10 @@ language: ruby
 rvm:
   - 2.0.0
   - 2.1
-  - 2.2.3
-  - 2.3.0
+  - 2.2.6
+  - 2.3.3
+  - 2.4.0
   - ruby-head
-  - rbx-2
 
 os:
   - linux
@@ -15,7 +15,6 @@ os:
 branches:
   only:
     - master
-    - v0.10
     - v0.12
     - v0.14
 
@@ -29,6 +28,3 @@ sudo: false
 matrix:
   allow_failures:
     - rvm: ruby-head
-    - rvm: rbx-2
-    - rvm: 2.3.0
-    - rvm: 1.9.3

From 956714cec8d8891034409e363b0019325353d677 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 2 Feb 2017 19:57:47 +0900
Subject: [PATCH 171/193] travis: Remove ruby 2.2 test from osx

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 9a5ec3a25b..3e056734ef 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,3 +28,6 @@ sudo: false
 matrix:
   allow_failures:
     - rvm: ruby-head
+  exclude:
+    - rvm: 2.2.6
+      os: osx

From 2ad2dca824ad8659e23c94c201c19a77b6c7f5b4 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Mon, 26 Dec 2016 01:32:16 +0900
Subject: [PATCH 172/193] Merge pull request #1387 from
 CSharpRU/get-real-retry-info-from-buffered-output

Ability to get real retry state of buffered output
---
 lib/fluent/plugin/in_monitor_agent.rb | 29 +++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index 7a06657fa4..93d2ce9d7b 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -34,6 +34,7 @@ class MonitorAgentInput < Input
     config_param :emit_interval, :time, default: 60
     config_param :emit_config, :bool, default: false
     config_param :include_config, :bool, default: true
+    config_param :include_retry, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -77,7 +78,7 @@ def build_object(req, res)
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {with_config: @agent.include_config}
+        opts = {with_config: @agent.include_config, with_retry: @agent.include_retry}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
@@ -91,6 +92,10 @@ def build_object(req, res)
           opts[:with_config] = Fluent::Config.bool_value(with_config)
         end
 
+        if with_retry = get_search_parameter(qs, 'with_retry'.freeze)
+          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
+        end
+
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -263,7 +268,7 @@ def start
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
         @loop = Coolio::Loop.new
-        opts = {with_config: @emit_config}
+        opts = {with_config: @emit_config, with_retry: false}
         timer = TimerWatcher.new(@emit_interval, log) {
           es = MultiEventStream.new
           now = Engine.now
@@ -408,6 +413,11 @@ def get_monitor_info(pe, opts={})
         end
       }
 
+      if opts[:with_retry] && pe.instance_variable_defined?(:@num_errors) &&
+         (pe.instance_variable_get(:@num_errors) > 0)
+        obj['retry'] = get_retry_info(pe)
+      end
+
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]
         iv = {}
@@ -431,6 +441,21 @@ def get_monitor_info(pe, opts={})
       obj
     end
 
+    RETRY_INFO = {
+      'steps' => '@num_errors',
+      'next_time' => '@next_retry_time',
+    }.freeze
+
+    def get_retry_info(pe)
+      retry_variables = {}
+
+      RETRY_INFO.each_pair { |key, param|
+        retry_variables[key] = pe.instance_variable_get(param)
+      }
+
+      retry_variables
+    end
+
     def plugin_category(pe)
       case pe
       when Fluent::Input

From cd164b481b1a3943352e87a7e443a29d38d560d8 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 3 Feb 2017 05:55:30 +0900
Subject: [PATCH 173/193] in_monitor_agent: optimize with_retry code

---
 lib/fluent/plugin/in_monitor_agent.rb | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
index 93d2ce9d7b..e7f573e5c1 100644
--- a/lib/fluent/plugin/in_monitor_agent.rb
+++ b/lib/fluent/plugin/in_monitor_agent.rb
@@ -394,6 +394,7 @@ def plugins_info_all(opts={})
 
     # TODO: use %i() after drop ruby v1.9.3 support.
     IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
+    EMPTY_RESULT = {}
 
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
@@ -413,9 +414,11 @@ def get_monitor_info(pe, opts={})
         end
       }
 
-      if opts[:with_retry] && pe.instance_variable_defined?(:@num_errors) &&
-         (pe.instance_variable_get(:@num_errors) > 0)
-        obj['retry'] = get_retry_info(pe)
+      if opts[:with_retry]
+        num_errors = pe.instance_variable_get(:@num_errors)
+        if num_errors
+          obj['retry'] = num_errors.zero? ? EMPTY_RESULT : get_retry_info(pe, num_errors)
+        end
       end
 
       # include all instance variables if :with_debug_info is set
@@ -441,18 +444,10 @@ def get_monitor_info(pe, opts={})
       obj
     end
 
-    RETRY_INFO = {
-      'steps' => '@num_errors',
-      'next_time' => '@next_retry_time',
-    }.freeze
-
-    def get_retry_info(pe)
+    def get_retry_info(pe, num_errors)
       retry_variables = {}
-
-      RETRY_INFO.each_pair { |key, param|
-        retry_variables[key] = pe.instance_variable_get(param)
-      }
-
+      retry_variables['steps'] = num_errors
+      retry_variables['next_time'] = Time.at(pe.instance_variable_get('@next_retry_time'.freeze))
       retry_variables
     end
 

From 79a2cbe26815aa8964cec25f0b2da62071da2c43 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 3 Feb 2017 11:20:59 +0900
Subject: [PATCH 174/193] v0.12.32

---
 ChangeLog             | 21 +++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 09b65839e2..99684bde19 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,26 @@
 # v0.12
 
+## Release 0.12.32 - 2017/02/02
+
+### New features / Enhancement
+
+* formatter: Port add_newline parameter to v012
+  https://github.com/fluent/fluentd/pull/1447
+* record_transformer: remove_keys processing should be last
+  https://github.com/fluent/fluentd/pull/1433
+* in_tail: Capture unmatched lines
+  https://github.com/fluent/fluentd/pull/1421
+* in_monitor_agent: Port #1393 to v0.12 for with_ivars query paretemer
+  https://github.com/fluent/fluentd/pull/1402
+* in_monitor_agent: Port #1387 to v0.12 for retry field
+  https://github.com/fluent/fluentd/pull/1387
+* Allow msgpack-ruby v1
+
+### Bug fixes
+
+* in_exec: Ensure to sleep for prevending fork bomb
+  https://github.com/fluent/fluentd/pull/1348
+
 ## Release 0.12.31 - 2016/12/14
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index 1a497debd7..fb5f64b94d 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.31'
+  VERSION = '0.12.32'
 
 end

From 2249683f6196f7d8ea7e4583aeea9089716cdaca Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 3 Mar 2017 15:22:25 +0900
Subject: [PATCH 175/193] in_forward: backport source / address key features

- Fix source_hostname_key feature to get hostname, not address.
- Add source_address_key feature
---
 lib/fluent/plugin/in_forward.rb |  57 ++++++++++++---
 test/plugin/test_in_forward.rb  | 126 ++++++++++++++++++++++----------
 2 files changed, 136 insertions(+), 47 deletions(-)

diff --git a/lib/fluent/plugin/in_forward.rb b/lib/fluent/plugin/in_forward.rb
index f303757548..8945af0b4d 100644
--- a/lib/fluent/plugin/in_forward.rb
+++ b/lib/fluent/plugin/in_forward.rb
@@ -47,11 +47,24 @@ def initialize
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
+    desc 'Try to resolve hostname from IP addresses or not.'
+    config_param :resolve_hostname, :bool, default: nil
+    desc "The field name of the client's source address."
+    config_param :source_address_key, :string, default: nil
     desc "The field name of the client's hostname."
     config_param :source_hostname_key, :string, default: nil
 
     def configure(conf)
       super
+
+      if @source_hostname_key
+        if @resolve_hostname.nil?
+          @resolve_hostname = true
+        elsif !@resolve_hostname # user specifies "false" in configure
+          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
+        end
+      end
+      @enable_field_injection = @source_address_key || @source_hostname_key
     end
 
     def start
@@ -88,7 +101,7 @@ def shutdown
 
     def listen
       log.info "listening fluent socket on #{@bind}:#{@port}"
-      s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, method(:on_message))
+      s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, @resolve_hostname, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s
     end
@@ -161,7 +174,7 @@ def on_message(msg, chunk_size, peeraddr)
         # PackedForward
         es = MessagePackEventStream.new(entries)
         es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
-        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
+        es = add_source_host(es, peeraddr) if @enable_field_injection
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -180,7 +193,7 @@ def on_message(msg, chunk_size, peeraddr)
                }
                es
              end
-        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
+        es = add_source_host(es, peeraddr) if @enable_field_injection
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -194,7 +207,10 @@ def on_message(msg, chunk_size, peeraddr)
         end
         return if record.nil?
         time = Engine.now if time == 0
-        record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
+        if @enable_field_injection
+          record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
+          record[@source_address_key] = peeraddr[3] if @source_address_key
+        end
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -219,12 +235,31 @@ def check_and_skip_invalid_event(tag, es, peeraddr)
       new_es
     end
 
-    def add_source_host(es, host)
+    def add_source_host(es, peeraddr)
       new_es = MultiEventStream.new
-      es.each { |time, record|
-        record[@source_hostname_key] = host
-        new_es.add(time, record)
-      }
+      if @source_address_key && @source_hostname_key
+        address = peeraddr[3]
+        hostname = peeraddr[2]
+        es.each { |time, record|
+          record[@source_address_key] = address
+          record[@source_hostname_key] = hostname
+          new_es.add(time, record)
+        }
+      elsif @source_address_key
+        address = peeraddr[3]
+        es.each { |time, record|
+          record[@source_address_key] = address
+          new_es.add(time, record)
+        }
+      elsif @source_hostname_key
+        hostname = peeraddr[2]
+        es.each { |time, record|
+          record[@source_hostname_key] = hostname
+          new_es.add(time, record)
+        }
+      else
+        raise "BUG: don't call this method in this case"
+      end
       new_es
     end
 
@@ -236,11 +271,13 @@ def source_message(peeraddr)
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
-      def initialize(io, linger_timeout, log, on_message)
+      def initialize(io, linger_timeout, log, resolve_hostname, on_message)
         super(io)
 
         @peeraddr = nil
         if io.is_a?(TCPSocket) # for unix domain socket support in the future
+          io.do_not_reverse_lookup = !resolve_hostname unless resolve_hostname.nil?
+
           @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
diff --git a/test/plugin/test_in_forward.rb b/test/plugin/test_in_forward.rb
index 6d793f72d9..d043b5f772 100644
--- a/test/plugin/test_in_forward.rb
+++ b/test/plugin/test_in_forward.rb
@@ -498,52 +498,104 @@ def send_data(data, try_to_receive_response=false, response_timeout=1)
     @responses << res if try_to_receive_response
   end
 
-  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
-  test 'message protocol with source_hostname_key' do
-    execute_test { |events|
-      events.each { |tag, time, record|
-        send_data [tag, time, record].to_msgpack
+  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
+  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
+  DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
+
+  sub_test_case 'source_hostname_key and source_address_key features' do
+    test 'resolve_hostname must be true with source_hostname_key' do
+      assert_raise(Fluent::ConfigError) {
+        create_driver(CONFIG + <<EOS)
+resolve_hostname false
+source_hostname_key hostname
+EOS
       }
-    }
-  end
+    end
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'message protocol' do |keys|
+      execute_test(*keys) { |events|
+        events.each { |tag, time, record|
+          send_data [tag, time, record].to_msgpack
+        }
+      }
+    end
 
-  test 'forward protocol with source_hostname_key' do
-    execute_test { |events|
-      entries = []
-      events.each {|tag,time,record|
-        entries << [time, record]
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'forward protocol' do |keys|
+      execute_test(*keys) { |events|
+        entries = []
+        events.each {|tag,time,record|
+          entries << [time, record]
+        }
+        send_data ['tag1', entries].to_msgpack
       }
-      send_data ['tag1', entries].to_msgpack
-    }
-  end
+    end
 
-  test 'packed forward protocol with source_hostname_key' do
-    execute_test { |events|
-      entries = ''
-      events.each { |tag, time, record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'packed forward protocol' do |keys|
+      execute_test(*keys) { |events|
+        entries = ''
+        events.each { |tag, time, record|
+          Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+        }
+        send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
       }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
-    }
-  end
+    end
 
-  def execute_test(&block)
-    d = create_driver(CONFIG + 'source_hostname_key source')
+    def execute_test(*keys, &block)
+      conf = CONFIG.dup
+      if keys.include?(:hostname)
+        conf << <<EOL
+source_hostname_key source_hostname
+EOL
+      end
+      if keys.include?(:address)
+        conf << <<EOL
+source_address_key source_address
+EOL
+      end
+      d = create_driver(conf)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
 
-    d.run do
-      block.call(events)
-    end
+      d.run do
+        block.call(events)
+      end
 
-    d.emits.each { |tag, _time, record|
-      assert_true record.has_key?('source')
-    }
+      d.emits.each { |tag, _time, record|
+        if keys.include?(:hostname)
+          assert_true record.has_key?('source_hostname')
+          assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
+          unless keys.include?(:address)
+            assert_false record.has_key?('source_address')
+          end
+        end
+        if keys.include?(:address)
+          assert_true record.has_key?('source_address')
+          assert_equal DUMMY_SOCK.remote_addr, record['source_address']
+          unless keys.include?(:hostname)
+            assert_false record.has_key?('source_hostname')
+          end
+        end
+      }
+    end
   end
 
   # TODO heartbeat

From c279a3ed920f27194021987289a60d6892482f76 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 2 Mar 2017 10:08:33 +0900
Subject: [PATCH 176/193] Merge pull request #1477 from
 okkez/fix-undefined-conversion-error

Use `String#force_encoding` instead of `String#encode`
---
 lib/fluent/config/types.rb |  4 ++--
 test/config/test_types.rb  | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
index 85499c9910..067a988428 100644
--- a/lib/fluent/config/types.rb
+++ b/lib/fluent/config/types.rb
@@ -65,7 +65,7 @@ def self.bool_value(str)
       end
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val }
+    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -86,7 +86,7 @@ def self.bool_value(str)
         value
       else
         case type
-        when :string  then value.to_s
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
         when :integer then value.to_i
         when :float   then value.to_f
         when :size then Config.size_value(value)
diff --git a/test/config/test_types.rb b/test/config/test_types.rb
index 6d826f4eec..0895f07a28 100644
--- a/test/config/test_types.rb
+++ b/test/config/test_types.rb
@@ -68,6 +68,17 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal '   ', Config::STRING_TYPE.call('   ', {})
     end
 
+    data('latin' => 'Märch',
+         'ascii' => 'ascii',
+         'space' => '     ',
+         'number' => '1',
+         'Hiragana' => 'あいうえお')
+    test 'string w/ binary' do |str|
+      actual = Config::STRING_TYPE.call(str.b, {})
+      assert_equal str, actual
+      assert_equal Encoding::UTF_8, actual.encoding
+    end
+
     test 'enum' do
       assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
       assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
@@ -142,6 +153,14 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
     end
 
+    data('latin' => ['3:Märch', {"3"=>"Märch"}],
+         'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
+         'number' => ['number:1', {"number"=>"1"}],
+         'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
+    test 'hash w/ binary' do |(target, expected)|
+      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
+    end
+
     test 'array' do
       assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
       assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))

From 0fe9916e1c0b510140c1972fb6eac52456227091 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 2 Mar 2017 09:59:02 +0900
Subject: [PATCH 177/193] Merge pull request #1479 from
 sergeyfedotov/allow_escape_sequence

Allow escape sequence in Apache access log
---
 lib/fluent/parser.rb | 2 +-
 test/test_parser.rb  | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index a2d01ceb34..738020e1fa 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -463,7 +463,7 @@ def parse(text)
     end
 
     class ApacheParser < Parser
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
diff --git a/test/test_parser.rb b/test/test_parser.rb
index 29e8ac7c10..03d906bf9f 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -292,6 +292,14 @@ def test_parse_without_http_version
         assert_equal(@expected, record)
       }
     end
+
+    def test_parse_with_escape_sequence
+      @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
+        assert_equal('/\"', record['path'])
+        assert_equal('referer \\\ \"', record['referer'])
+        assert_equal('user agent \\\ \"', record['agent'])
+      }
+    end
   end
 
   class SyslogParserTest < ::Test::Unit::TestCase

From d4f92a5ef5081f815133eaf5984a59609b97b918 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 3 Mar 2017 17:38:02 +0900
Subject: [PATCH 178/193] Fix tests of TSVParser

---
 test/test_parser.rb | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/test_parser.rb b/test/test_parser.rb
index 03d906bf9f..99cf8e30be 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -580,11 +580,10 @@ def test_parse_with_keep_time_key
     end
 
     data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-    def test_parse_with_null_value_pattern
+    def test_parse_with_null_value_pattern(param)
       parser = TextParser::TSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_value_pattern'=>'^(-|null|NULL)$'
       )
       parser.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
@@ -598,11 +597,10 @@ def test_parse_with_null_value_pattern
     end
 
     data('array param' => '["a","b"]', 'string param' => 'a,b')
-    def test_parse_with_null_empty_string
+    def test_parse_with_null_empty_string(param)
       parser = TextParser::TSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_empty_string'=>true
       )
       parser.parse("\t ") do |time, record|

From 6fc58ddb2a075f069c81b9f44a04386a0117c6cb Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Wed, 22 Feb 2017 02:11:21 +0900
Subject: [PATCH 179/193] Merge pull request #1478 from
 okkez/add-log-concat-method

Add `Fluent::Log#<<`
---
 lib/fluent/log.rb | 3 +++
 test/test_log.rb  | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/lib/fluent/log.rb b/lib/fluent/log.rb
index e0340497c0..5166c682d4 100644
--- a/lib/fluent/log.rb
+++ b/lib/fluent/log.rb
@@ -240,6 +240,9 @@ def puts(msg)
     def write(data)
       @out.write(data)
     end
+    # We need `#<<` method to use this logger class with other
+    # libraries such as aws-sdk
+    alias << write
 
     def flush
       @out.flush
diff --git a/test/test_log.rb b/test/test_log.rb
index cd95530415..8dd28b9b4e 100644
--- a/test/test_log.rb
+++ b/test/test_log.rb
@@ -305,6 +305,10 @@ def test_write
       @log.write("log")
     end
 
+    def test_write_alias
+      assert(@log.respond_to?(:<<))
+    end
+
     def test_out
       assert_equal(@log.out, @logger.out)
       @log.out = Object.new

From ba84f03746e6dc770dd4d073dd4b9057ce6bb322 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 14 Feb 2017 09:28:38 +0900
Subject: [PATCH 180/193] Merge pull request #1464 from angry-cellophane/master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fix issue #1451. The in_tail plugin tries to tail directories when th…
---
 lib/fluent/plugin/in_tail.rb  |  2 +-
 test/plugin/data/log/foo/bar2 |  0
 test/plugin/test_in_tail.rb   | 19 +++++++++++++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 test/plugin/data/log/foo/bar2

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 9e901c5f31..93003056c9 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -153,7 +153,7 @@ def expand_paths
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            if File.readable?(p)
+            if File.readable?(p) && !File.directory?(p)
               true
             else
               log.warn "#{p} unreadable. It is excluded and would be examined next time."
diff --git a/test/plugin/data/log/foo/bar2 b/test/plugin/data/log/foo/bar2
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 3999c02738..e613b4f2ab 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -697,6 +697,25 @@ def test_expand_paths
     assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
   end
 
+  def test_log_file_without_extension
+    expected_files = [
+      'test/plugin/data/log/bar',
+      'test/plugin/data/log/foo/bar.log',
+      'test/plugin/data/log/foo/bar2',
+      'test/plugin/data/log/test.log'
+    ]
+
+    config = config_element("", "", {
+      "tag" => "tail",
+      "path" => "test/plugin/data/log/**/*",
+      "format" => "none",
+      "pos_file" => "#{TMP_DIR}/tail.pos"
+    })
+
+    plugin = create_driver(config, false).instance
+    assert_equal expected_files, plugin.expand_paths.sort
+  end
+
   def test_refresh_watchers
     plugin = create_driver(EX_CONFIG, false).instance
     sio = StringIO.new

From 8ff1b8d55268aa3cc6f78a8c29e7f6fd1b1100c7 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Fri, 17 Feb 2017 20:58:01 +0900
Subject: [PATCH 181/193] Merge pull request #1474 from
 t-osakada/watch_only_recently_modified_files

Watch only recently modified files
---
 lib/fluent/plugin/in_tail.rb |  8 +++++++-
 test/plugin/test_in_tail.rb  | 25 +++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index 93003056c9..bca84e8e48 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -58,6 +58,8 @@ def initialize
     config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
+    desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
+    config_param :limit_recently_modified, :time, default: nil
 
     attr_reader :paths
 
@@ -154,7 +156,11 @@ def expand_paths
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
             if File.readable?(p) && !File.directory?(p)
-              true
+              if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
+                false
+              else
+                true
+              end
             else
               log.warn "#{p} unreadable. It is excluded and would be examined next time."
               false
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index e613b4f2ab..07d9b656c0 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -2,6 +2,7 @@
 require 'fluent/test'
 require 'net/http'
 require 'flexmock/test_unit'
+require 'timecop'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -1011,4 +1012,28 @@ def test_tail_path_with_multiline_with_multiple_paths
       assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
     end
   end
+
+  def test_limit_recently_modified
+    now = Time.new(2010, 1, 2, 3, 4, 5)
+    FileUtils.touch("#{TMP_DIR}/tail_unwatch.txt", mtime: (now - 3601))
+    FileUtils.touch("#{TMP_DIR}/tail_watch1.txt", mtime: (now - 3600))
+    FileUtils.touch("#{TMP_DIR}/tail_watch2.txt", mtime: now)
+
+    config = config_element('', '', {
+      'tag' => 'tail',
+      'path' => "#{TMP_DIR}/*.txt",
+      'format' => 'none',
+      'limit_recently_modified' => '3600s'
+    })
+
+    expected_files = [
+      "#{TMP_DIR}/tail_watch1.txt",
+      "#{TMP_DIR}/tail_watch2.txt"
+    ]
+
+    Timecop.freeze(now) do
+      plugin = create_driver(config, false).instance
+      assert_equal expected_files, plugin.expand_paths.sort
+    end
+  end
 end

From 2c83da30eb32af0255bd8506d29014a1480f86bf Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sun, 5 Mar 2017 00:52:08 +0900
Subject: [PATCH 182/193] Fix tests of CSVParser

---
 test/test_parser.rb | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/test/test_parser.rb b/test/test_parser.rb
index 99cf8e30be..ef15c3bb8c 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -667,29 +667,27 @@ def test_parse_with_keep_time_key
     end
 
     data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-    def test_parse_with_null_value_pattern
+    def test_parse_with_null_value_pattern(param)
       parser = TextParser::CSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_value_pattern'=>'^(-|null|NULL)$'
       )
       parser.parse("-,null,NULL,,--,nuLL") do |time, record|
         assert_nil record['a']
         assert_nil record['b']
         assert_nil record['c']
-        assert_equal record['d'], ''
+        assert_nil record['d']
         assert_equal record['e'], '--'
         assert_equal record['f'], 'nuLL'
       end
     end
 
     data('array param' => '["a","b"]', 'string param' => 'a,b')
-    def test_parse_with_null_empty_string
+    def test_parse_with_null_empty_string(param)
       parser = TextParser::CSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_empty_string'=>true
       )
       parser.parse(", ") do |time, record|

From db71a89e1f55f97cd3859651d98a9dbbe87ffaea Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <cosmo0920.oucc@gmail.com>
Date: Thu, 9 Mar 2017 13:27:18 +0900
Subject: [PATCH 183/193] Backport #1492 to v0.12

---
 lib/fluent/parser.rb |  41 +++++++++++
 test/test_parser.rb  | 162 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 203 insertions(+)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 738020e1fa..083e78bf52 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -536,9 +536,13 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_param :time_format, :string, default: "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize
         super
@@ -549,6 +553,26 @@ def configure(conf)
         super
 
         @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
+        @regexp = case @message_format
+                  when :rfc3164
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                  when :rfc5424
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    REGEXP_RFC5424
+                  when :auto
+                    class << self
+                      alias_method :parse, :parse_auto
+                    end
+                    @time_parser_rfc3164 = TextParser::TimeParser.new(@time_format)
+                    @time_parser_rfc5424 = TextParser::TimeParser.new(@rfc5424_time_format)
+                    nil
+                  end
         @time_parser = TextParser::TimeParser.new(@time_format)
       end
 
@@ -557,6 +581,23 @@ def patterns
       end
 
       def parse(text)
+        # This is overwritten in configure
+      end
+
+      def parse_auto(text, &block)
+        if @message_format == :auto
+          if REGEXP_DETECT_RFC5424.match(text)
+            @regexp = REGEXP_RFC5424
+            @time_parser = @time_parser_rfc5424
+          else
+            @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+            @time_parser = @time_parser_rfc3164
+          end
+        end
+        parse_plain(text, &block)
+      end
+
+      def parse_plain(text, &block)
         m = @regexp.match(text)
         unless m
           if block_given?
diff --git a/test/test_parser.rb b/test/test_parser.rb
index ef15c3bb8c..1396043a7d 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -364,6 +364,168 @@ def test_parse_with_keep_time_key
         assert_equal "Feb 28 00:00:12", record['time']
       end
     end
+
+    class TestRFC5424Regexp < self
+      def test_parse_with_rfc5424_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'rfc5424',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_rfc5424_structured_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'rfc5424',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+    end
+
+    class TestAutoRegexp < self
+      def test_auto_with_legacy_syslog_message
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'mseeage_format' => 'auto',
+        )
+        text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected, record)
+        end
+      end
+
+      def test_auto_with_legacy_syslog_priority_message
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'with_priority' => true,
+          'mseeage_format' => 'auto',
+        )
+        text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected.merge('pri' => 6), record)
+        end
+      end
+
+      def test_parse_with_rfc5424_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'auto',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_rfc5424_structured_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'auto',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_both_message_type
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'auto',
+        )
+        text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected, record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+        text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected, record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_both_message_type_and_priority
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'with_priority' => true,
+          'message_format' => 'auto',
+        )
+        text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected.merge('pri' => 6), record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+        text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected.merge('pri' => 16), record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+    end
   end
 
   class JsonParserTest < ::Test::Unit::TestCase

From eaf8b99e955f1c2865f1a8e78f0949646fa10a2f Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 9 Mar 2017 14:16:26 +0900
Subject: [PATCH 184/193] Remove needless assignment

---
 lib/fluent/parser.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 083e78bf52..37227e4638 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -552,7 +552,6 @@ def initialize
       def configure(conf)
         super
 
-        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
         @time_parser_rfc3164 = @time_parser_rfc5424 = nil
         @regexp = case @message_format
                   when :rfc3164

From 3c807653184d4f823659e88270876181baf84912 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Tue, 7 Mar 2017 00:28:53 +0900
Subject: [PATCH 185/193] Merge pull request #1487 from
 t-osakada/skip_refresh_on_startup

Skip the refresh of watching list on startup.
---
 lib/fluent/plugin/in_tail.rb |  4 +++-
 test/plugin/test_in_tail.rb  | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb
index bca84e8e48..fb0f2e317f 100644
--- a/lib/fluent/plugin/in_tail.rb
+++ b/lib/fluent/plugin/in_tail.rb
@@ -60,6 +60,8 @@ def initialize
     config_param :path_key, :string, default: nil
     desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
     config_param :limit_recently_modified, :time, default: nil
+    desc 'Enable the option to skip the refresh of watching list on startup.'
+    config_param :skip_refresh_on_startup, :bool, default: false
 
     attr_reader :paths
 
@@ -130,7 +132,7 @@ def start
       end
 
       @loop = Coolio::Loop.new
-      refresh_watchers
+      refresh_watchers unless @skip_refresh_on_startup
 
       @refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
       @refresh_trigger.attach(@loop)
diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb
index 07d9b656c0..3ff841acee 100644
--- a/test/plugin/test_in_tail.rb
+++ b/test/plugin/test_in_tail.rb
@@ -1036,4 +1036,19 @@ def test_limit_recently_modified
       assert_equal expected_files, plugin.expand_paths.sort
     end
   end
+
+  def test_skip_refresh_on_startup
+    FileUtils.touch("#{TMP_DIR}/tail.txt")
+    config = config_element('', '', {
+                              'tag' => 'tail',
+                              'path' => "#{TMP_DIR}/*.txt",
+                              'format' => 'none',
+                              'refresh_interval' => 1,
+                              'skip_refresh_on_startup' => true
+                            })
+    d = create_driver(config, false)
+    d.run {
+      sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1
+    }
+  end
 end

From 99110c73232a5d3002ec94103f89c62fc6b3d1e6 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 9 Mar 2017 15:37:58 +0900
Subject: [PATCH 186/193] Remove needless if condition

`@message_format` is always :auto in #parse_auto.
---
 lib/fluent/parser.rb | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 37227e4638..9aed1894c5 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -584,14 +584,12 @@ def parse(text)
       end
 
       def parse_auto(text, &block)
-        if @message_format == :auto
-          if REGEXP_DETECT_RFC5424.match(text)
-            @regexp = REGEXP_RFC5424
-            @time_parser = @time_parser_rfc5424
-          else
-            @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
-            @time_parser = @time_parser_rfc3164
-          end
+        if REGEXP_DETECT_RFC5424.match(text)
+          @regexp = REGEXP_RFC5424
+          @time_parser = @time_parser_rfc5424
+        else
+          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          @time_parser = @time_parser_rfc3164
         end
         parse_plain(text, &block)
       end

From 58a44e8c24bd46e9c44f995b05ddf1b54521eaa4 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 9 Mar 2017 16:35:29 +0900
Subject: [PATCH 187/193] Set default time_format value for rfc5424 when users
 do not set time_format key

---
 lib/fluent/parser.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fluent/parser.rb b/lib/fluent/parser.rb
index 9aed1894c5..ed53a3c544 100644
--- a/lib/fluent/parser.rb
+++ b/lib/fluent/parser.rb
@@ -563,6 +563,7 @@ class << self
                     class << self
                       alias_method :parse, :parse_plain
                     end
+                    @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
                     REGEXP_RFC5424
                   when :auto
                     class << self

From 223d249d212563809b5752a90c2c10b55b992075 Mon Sep 17 00:00:00 2001
From: Hiroshi Hatake <hatake@clear-code.com>
Date: Thu, 9 Mar 2017 16:42:17 +0900
Subject: [PATCH 188/193] Add a rfc5424 test case without time_format conf key

---
 test/test_parser.rb | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/test/test_parser.rb b/test/test_parser.rb
index 1396043a7d..f90af77cf8 100644
--- a/test/test_parser.rb
+++ b/test/test_parser.rb
@@ -381,6 +381,20 @@ def test_parse_with_rfc5424_message
         end
       end
 
+      def test_parse_with_rfc5424_message_without_time_format
+        @parser.configure(
+          'message_format' => 'rfc5424',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.instance.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
       def test_parse_with_rfc5424_structured_message
         @parser.configure(
           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',

From 67910e5caae54a17ca123a427a225906b67bf851 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 9 Mar 2017 14:30:26 +0900
Subject: [PATCH 189/193] Add Development section for developers

---
 README.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/README.md b/README.md
index a853d77a1a..cdedce16ca 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,31 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
     $ fluentd -c conf/fluent.conf &
     $ echo '{"json":"message"}' | fluent-cat debug.test
 
+## Development
+
+### Prerequisites
+
+- Ruby 2.1 or later
+- git
+
+`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
+
+### Install dependent gems
+
+Use bundler:
+
+    $ gem install bundler
+    $ bundle install --path vendor/bundle
+
+### Run test
+
+    $ bundle exec rake test
+
+You can run specified test via `TEST` environment variable:
+
+    $ bundle exec rake test TEST=test/test_specified_path.rb
+    $ bundle exec rake test TEST=test/test_*.rb
+
 ## Fluentd UI: Admin GUI
 
 [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.

From fb743e3f851b56fd76c3804738ed64d647e387ef Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Thu, 9 Mar 2017 18:23:41 +0900
Subject: [PATCH 190/193] v0.12.33

---
 ChangeLog             | 24 ++++++++++++++++++++++++
 lib/fluent/version.rb |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 99684bde19..fde39681bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,29 @@
 # v0.12
 
+## Release 0.12.33 - 2017/03/09
+
+### New features / Enhancement
+
+* in_syslog: Backport 'Support rfc5424 syslog format'
+  https://github.com/fluent/fluentd/pull/1495
+* in_forward: Add source_address_key and fix source_hostname_key parameters
+  https://github.com/fluent/fluentd/pull/1490
+* in_tail: Skip directories when use **/* in path
+  https://github.com/fluent/fluentd/pull/1464
+* in_tail: Add limit_recently_modified parameter to limit watching files
+  https://github.com/fluent/fluentd/pull/1474
+* in_tail: Skip the refresh of watching list on startup
+  https://github.com/fluent/fluentd/pull/1487
+* parser: Allow escape sequence in Apache access log
+  https://github.com/fluent/fluentd/pull/1479
+* log: Add Fluent::Log#<< to support some SDKs
+  https://github.com/fluent/fluentd/pull/1478
+
+### Bug fixes
+
+* config: Set encoding forcefully to avoid UndefinedConversionError
+  https://github.com/fluent/fluentd/pull/1477
+
 ## Release 0.12.32 - 2017/02/02
 
 ### New features / Enhancement
diff --git a/lib/fluent/version.rb b/lib/fluent/version.rb
index fb5f64b94d..2502ebc00f 100644
--- a/lib/fluent/version.rb
+++ b/lib/fluent/version.rb
@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.32'
+  VERSION = '0.12.33'
 
 end

From d0a7b2f991324c0467dc307d8eba03e6dae40eb9 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sat, 11 Mar 2017 01:16:18 +0900
Subject: [PATCH 191/193] Fix typo of v0.12.33 ChangeLog

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index fde39681bb..8b8531e16e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,7 +4,7 @@
 
 ### New features / Enhancement
 
-* in_syslog: Backport 'Support rfc5424 syslog format'
+* parser_syslog: Backport 'Support rfc5424 syslog format'
   https://github.com/fluent/fluentd/pull/1495
 * in_forward: Add source_address_key and fix source_hostname_key parameters
   https://github.com/fluent/fluentd/pull/1490

From 6c4b90f111f62ba2cfdad3e57da8df0dab44d6d5 Mon Sep 17 00:00:00 2001
From: Masahiro Nakagawa <repeatedly@gmail.com>
Date: Sat, 11 Mar 2017 01:52:15 +0900
Subject: [PATCH 192/193] Add syslog parser message_format tests to in_syslog

---
 test/plugin/test_in_syslog.rb | 45 +++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/test/plugin/test_in_syslog.rb b/test/plugin/test_in_syslog.rb
index d1be8a68db..3de45b629d 100755
--- a/test/plugin/test_in_syslog.rb
+++ b/test/plugin/test_in_syslog.rb
@@ -235,4 +235,49 @@ def compare_test_result(events, tests, options = {})
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
+
+  class SyslogMessageFormatTest < self
+    def test_syslog_rfc5424_format
+      d = create_driver(CONFIG + 'message_format rfc5424')
+      tests = [
+        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
+        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      ]
+
+      run_tests(d, tests)
+      compare_test_result(d)
+    end
+
+    def test_syslog_auto_format
+      d = create_driver(CONFIG + 'message_format auto')
+      tests = [
+        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
+        '<6>Sep 11 00:00:00 localhost fluentd: Hi, from Fluentd!'
+      ]
+
+      run_tests(d, tests)
+      compare_test_result(d)
+    end
+
+    def run_tests(d, tests)
+      d.run do
+        u = Fluent::SocketUtil.create_udp_socket('127.0.0.1')
+        u.connect('127.0.0.1', PORT)
+        tests.each {|test|
+          u.send(test, 0)
+        }
+        sleep 1
+      end
+    end
+
+    def compare_test_result(d)
+      emits = d.emits
+      assert_equal 2, emits.size
+      emits.each_index {|i|
+        record =  emits[i][2]
+        assert_equal('fluentd', record['ident'])
+        assert_equal('Hi, from Fluentd!', record['message'])
+      }
+    end
+  end
 end

From a36090d3c9d6185bf9e10c5bb71186548a627a31 Mon Sep 17 00:00:00 2001
From: JD Conley <jd.conley@gmail.com>
Date: Thu, 16 Mar 2017 16:49:46 -0700
Subject: [PATCH 193/193] msgpack format chosen with content-type header

---
 lib/fluent/plugin/in_http.rb |  2 ++
 test/plugin/test_in_http.rb  | 16 ++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/lib/fluent/plugin/in_http.rb b/lib/fluent/plugin/in_http.rb
index 2f75174a46..6cb250f51e 100644
--- a/lib/fluent/plugin/in_http.rb
+++ b/lib/fluent/plugin/in_http.rb
@@ -355,6 +355,8 @@ def on_message_complete
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif @content_type =~ /^application\/json/
           params['json'] = @body
+        elsif @content_type =~ /^application\/msgpack/
+          params['msgpack'] = @body
         end
         path_info = uri.path
 
diff --git a/test/plugin/test_in_http.rb b/test/plugin/test_in_http.rb
index 76e1051ddb..8ea174691a 100644
--- a/test/plugin/test_in_http.rb
+++ b/test/plugin/test_in_http.rb
@@ -205,6 +205,22 @@ def test_application_json
     end
   end
 
+  def test_application_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}?time=#{time.to_s}", record.to_msgpack, {"content-type"=>"application/msgpack; charset=utf-8"})
+        assert_equal "200", res.code
+      }
+    end
+  end
+  
   def test_msgpack
     d = create_driver