Introduce v14 test drivers

lib/fluent/compat/output.rb

@@ -21,6 +21,8 @@
 require 'fluent/compat/output_chain'
 require 'fluent/timezone'
 
+require 'fluent/plugin_helper/compat_parameters'
+
 require 'time'
 
 module Fluent
@@ -201,21 +203,7 @@ def support_in_v12_style?(feature)
 
       config_param :flush_at_shutdown, :bool, default: true
 
-      PARAMS_MAP = {
-        "buffer_type" => "@type",
-        "buffer_path" => "path",
-        "num_threads"                 => "flush_threads",
-        "flush_interval"              => "flush_interval",
-        "try_flush_interval"          => "flush_thread_interval",
-        "queued_chunk_flush_interval" => "flush_burst_interval",
-        "disable_retry_limit" => "retry_forever",
-        "retry_limit"         => "retry_max_times",
-        "max_retry_wait"      => "retry_max_interval",
-        "buffer_chunk_limit"  => "chunk_bytes_limit",
-        "buffer_queue_limit"  => "queue_length_limit",
-        "buffer_queue_full_action" => "overflow_action",
-        "flush_at_shutdown" => "flush_at_shutdown",
-      }
+      PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP
 
       def configure(conf)
         bufconf = CompatOutputUtils.buffer_section(conf)
@@ -226,6 +214,7 @@ def configure(conf)
             "retry_type" => "exponential_backoff",
           }
           PARAMS_MAP.each do |older, newer|
+            next unless newer
             buf_params[newer] = conf[older] if conf.has_key?(older)
           end
 
@@ -330,21 +319,7 @@ def support_in_v12_style?(feature)
 
       config_set_default :time_as_integer, true
 
-      PARAMS_MAP = {
-        "buffer_type" => "@type",
-        "buffer_path" => "path",
-        "num_threads"                 => "flush_threads",
-        "flush_interval"              => "flush_interval",
-        "try_flush_interval"          => "flush_thread_interval",
-        "queued_chunk_flush_interval" => "flush_burst_interval",
-        "disable_retry_limit" => "retry_forever",
-        "retry_limit"         => "retry_max_times",
-        "max_retry_wait"      => "retry_max_interval",
-        "buffer_chunk_limit"  => "chunk_bytes_limit",
-        "buffer_queue_limit"  => "queue_length_limit",
-        "buffer_queue_full_action" => "overflow_action",
-        "flush_at_shutdown" => "flush_at_shutdown",
-      }
+      PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP
 
       def configure(conf)
         bufconf = CompatOutputUtils.buffer_section(conf)
@@ -355,6 +330,7 @@ def configure(conf)
             "retry_type" => "exponential_backoff",
           }
           PARAMS_MAP.each do |older, newer|
+            next unless newer
             buf_params[newer] = conf[older] if conf.has_key?(older)
           end
 
@@ -450,22 +426,7 @@ def support_in_v12_style?(feature)
         config_set_default :@type, 'file'
       end
 
-      PARAMS_MAP = {
-        "buffer_type" => "@type",
-        "buffer_path" => "path",
-        "num_threads"                 => "flush_threads",
-        "flush_interval"              => "flush_interval",
-        "try_flush_interval"          => "flush_thread_interval",
-        "queued_chunk_flush_interval" => "flush_burst_interval",
-        "disable_retry_limit" => "retry_forever",
-        "retry_limit"         => "retry_max_times",
-        "max_retry_wait"      => "retry_max_interval",
-        "buffer_chunk_limit"  => "chunk_bytes_limit",
-        "buffer_queue_limit"  => "queue_length_limit",
-        "buffer_queue_full_action" => "overflow_action",
-        "flush_at_shutdown" => "flush_at_shutdown",
-        "time_slice_wait" => "timekey_wait",
-      }
+      PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP.merge(Fluent::PluginHelper::CompatParameters::TIME_SLICED_PARAMS)
 
       def initialize
         super
@@ -488,6 +449,7 @@ def configure(conf)
             "retry_type" => "exponential_backoff",
           }
           PARAMS_MAP.each do |older, newer|
+            next unless newer
             buf_params[newer] = conf[older] if conf.has_key?(older)
           end
           unless buf_params.has_key?("@type")

lib/fluent/config.rb

@@ -20,18 +20,32 @@
 
 module Fluent
   module Config
-    def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
-      if fname =~ /\.rb$/
+    def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
+      parser = if fname =~ /\.rb$/ || syntax == :ruby
+                 :ruby
+               elsif v1_config.nil?
+                 case syntax
+                 when :v1 then :v1
+                 when :v0 then :v0
+                 else
+                   raise ArgumentError, "Unknown Fluentd configuration syntax: '#{syntax}'"
+                 end
+               elsif v1_config then :v1
+               else :v0
+               end
+      case parser
+      when :v1
+        require 'fluent/config/v1_parser'
+        V1Parser.parse(str, fname, basepath, Kernel.binding)
+      when :v0
+        # TODO: show deprecated message in v1
+        require 'fluent/config/parser'
+        Parser.parse(str, fname, basepath)
+      when :ruby
         require 'fluent/config/dsl'
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
-        if v1_config
-          require 'fluent/config/v1_parser'
-          V1Parser.parse(str, fname, basepath, Kernel.binding)
-        else
-          require 'fluent/config/parser'
-          Parser.parse(str, fname, basepath)
-        end
+        raise "[BUG] unknown configuration parser specification:'#{parser}'"
       end
     end
 

lib/fluent/config/configure_proxy.rb

@@ -41,7 +41,7 @@ def initialize(name, param_name: nil, final: nil, init: nil, required: nil, mult
         @name = name.to_sym
         @final = final
 
-        @param_name = (param_name || @name).to_sym
+        @param_name = param_name && param_name.to_sym
         @init = init
         @required = required
         @multi = multi
@@ -63,6 +63,10 @@ def initialize(name, param_name: nil, final: nil, init: nil, required: nil, mult
         @current_description = nil
       end
 
+      def variable_name
+        @param_name || @name
+      end
+
       def init?
         @init.nil? ? false : @init
       end
@@ -82,26 +86,17 @@ def final?
       def merge(other) # self is base class, other is subclass
         return merge_for_finalized(other) if self.final?
 
-        if overwrite?(other, :init)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: init"
-        end
-        if overwrite?(other, :required)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: required"
-        end
-        if overwrite?(other, :multi)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: multi"
-        end
-        if overwrite?(other, :alias)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: alias"
-        end
-        if overwrite?(other, :configured_in_section)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: configured_in"
+        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|
+          if overwrite?(other, prohibited_name)
+            raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}"
+          end
         end
 
         options = {}
-        # param_name is used not to ovewrite plugin's instance
-        # varible, so this should be able to be overwritten
-        options[:param_name] = other.param_name
+        # param_name affects instance variable name, which is just "internal" of each plugins.
+        # so it must not be changed. base class's name (or param_name) is always used.
+        options[:param_name] = @param_name
+
         # subclass cannot overwrite base class's definition
         options[:init] = @init.nil? ? other.init : self.init
         options[:required] = @required.nil? ? other.required : self.required
@@ -110,7 +105,7 @@ def merge(other) # self is base class, other is subclass
         options[:final] = @final || other.final
         options[:type_lookup] = @type_lookup
 
-        merged = self.class.new(other.name, options)
+        merged = self.class.new(@name, options)
 
         # configured_in MUST be kept
         merged.configured_in_section = self.configured_in_section
@@ -137,45 +132,36 @@ def merge(other) # self is base class, other is subclass
 
       def merge_for_finalized(other)
         # list what subclass can do for finalized section
-        #  * overwrite param_name to escape duplicated name of instance variable
         #  * append params/defaults/sections which are missing in superclass
+        #  * change default values of superclass
+        #  * overwrite init to make it enable to instantiate section objects with added default values
 
         if other.final == false && overwrite?(other, :final)
           raise ConfigError, "BUG: subclass cannot overwrite finalized base class's config_section"
         end
 
-        if overwrite?(other, :init)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: init"
-        end
-        if overwrite?(other, :required)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: required"
-        end
-        if overwrite?(other, :multi)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: multi"
-        end
-        if overwrite?(other, :alias)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: alias"
-        end
-        if overwrite?(other, :configured_in_section)
-          raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: configured_in"
+        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|
+          if overwrite?(other, prohibited_name)
+            raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}"
+          end
         end
 
         options = {}
-        options[:param_name] = other.param_name
-        options[:init] = @init.nil? ? other.init : self.init
+        options[:param_name] = @param_name
+        options[:init] = @init || other.init
         options[:required] = @required.nil? ? other.required : self.required
         options[:multi] = @multi.nil? ? other.multi : self.multi
         options[:alias] = @alias.nil? ? other.alias : self.alias
         options[:final]  = true
         options[:type_lookup] = @type_lookup
 
-        merged = self.class.new(other.name, options)
+        merged = self.class.new(@name, options)
 
         merged.configured_in_section = self.configured_in_section
 
         merged.argument = self.argument || other.argument
         merged.params = other.params.merge(self.params)
-        merged.defaults = other.defaults.merge(self.defaults)
+        merged.defaults = self.defaults.merge(other.defaults)
         merged.sections = {}
         (self.sections.keys + other.sections.keys).uniq.each do |section_key|
           self_section = self.sections[section_key]

lib/fluent/config/section.rb

@@ -144,7 +144,7 @@ def self.generate(proxy, conf, logger, plugin_class, stack = [])
         check_unused_section(proxy, conf, plugin_class)
 
         proxy.sections.each do |name, subproxy|
-          varname = subproxy.param_name.to_sym
+          varname = subproxy.variable_name
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
           if elements.empty? && subproxy.init? && !subproxy.multi?
             elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])

lib/fluent/configurable.rb

@@ -41,9 +41,9 @@ def initialize
         next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?
-          instance_variable_set("@#{subproxy.param_name}".to_sym, [])
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
         else
-          instance_variable_set("@#{subproxy.param_name}".to_sym, nil)
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
         end
       end
     end
@@ -145,7 +145,7 @@ def config_set_desc(name, desc)
 
       def config_section(name, **kwargs, &block)
         configure_proxy(self.name).config_section(name, **kwargs, &block)
-        attr_accessor configure_proxy(self.name).sections[name].param_name
+        attr_accessor configure_proxy(self.name).sections[name].variable_name
       end
 
       def desc(description)