From 3754e61d85cc9b11ab7ac80d3318aea806d553e0 Mon Sep 17 00:00:00 2001 From: Yuta Iwama Date: Wed, 5 Feb 2020 18:38:36 +0900 Subject: [PATCH 01/15] yaml config file Signed-off-by: Yuta Iwama --- lib/fluent/command/fluentd.rb | 8 ++ lib/fluent/config.rb | 8 +- lib/fluent/config/yaml_parser.rb | 44 ++++++ lib/fluent/config/yaml_parser/fluent_value.rb | 47 +++++++ lib/fluent/config/yaml_parser/loader.rb | 91 +++++++++++++ lib/fluent/config/yaml_parser/parser.rb | 126 ++++++++++++++++++ .../config/yaml_parser/section_builder.rb | 107 +++++++++++++++ lib/fluent/supervisor.rb | 9 +- 8 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 lib/fluent/config/yaml_parser.rb create mode 100644 lib/fluent/config/yaml_parser/fluent_value.rb create mode 100644 lib/fluent/config/yaml_parser/loader.rb create mode 100644 lib/fluent/config/yaml_parser/parser.rb create mode 100644 lib/fluent/config/yaml_parser/section_builder.rb diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb index 2b2d229a01..7880b86eb6 100644 --- a/lib/fluent/command/fluentd.rb +++ b/lib/fluent/command/fluentd.rb @@ -126,6 +126,14 @@ opts[:without_source] = b } +op.on('--config-file-type VALU', 'file type of fluentd. yaml or yml') { |s| + if (s == 'yaml') || (s == 'yml') + opts[:config_file_type] = s.to_sym + else + usage '--config-file-type accepts yaml or yml' + end +} + op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b| opts[:use_v1_config] = b } diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index 8ca5cc6618..59ca90f3c8 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -17,6 +17,7 @@ require 'fluent/config/error' require 'fluent/config/element' require 'fluent/configurable' +require 'fluent/config/yaml_parser' module Fluent module Config @@ -25,7 +26,11 @@ module Config # @param additional_config [String] config which is added to last of config body # @param use_v1_config [Bool] config is formatted with v1 or not # @return [Fluent::Config] - def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true) + def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil) + if type == :yaml + return Fluent::Config::YamlParser.parse(config_path) + end + config_fname = File.basename(config_path) config_basedir = File.dirname(config_path) config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f| @@ -36,6 +41,7 @@ def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_c end s end + Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config) end diff --git a/lib/fluent/config/yaml_parser.rb b/lib/fluent/config/yaml_parser.rb new file mode 100644 index 0000000000..07ef8a414c --- /dev/null +++ b/lib/fluent/config/yaml_parser.rb @@ -0,0 +1,44 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/config/yaml_parser/loader' +require 'fluent/config/yaml_parser/parser' +require 'pathname' + +module Fluent + module Config + module YamlParser + def self.parse(path) + context = Kernel.binding + + unless context.respond_to?(:use_nil) + context.define_singleton_method(:use_nil) do + raise Fluent::SetNil + end + end + + unless context.respond_to?(:use_default) + context.define_singleton_method(:use_default) do + raise Fluent::SetDefault + end + end + + s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path)) + Fluent::Config::YamlParser::Parser.new(s).build.to_element + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/fluent_value.rb b/lib/fluent/config/yaml_parser/fluent_value.rb new file mode 100644 index 0000000000..69108d8bd1 --- /dev/null +++ b/lib/fluent/config/yaml_parser/fluent_value.rb @@ -0,0 +1,47 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module Config + module YamlParser + module FluentValue + JsonValue = Struct.new(:val) do + def to_s + val.to_json + end + + def to_element + to_s + end + end + + StringValue = Struct.new(:val, :context) do + def to_s + context.instance_eval("\"#{val}\"") + rescue Fluent::SetNil => _ + '' + rescue Fluent::SetDefault => _ + ':default' + end + + def to_element + to_s + end + end + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/loader.rb b/lib/fluent/config/yaml_parser/loader.rb new file mode 100644 index 0000000000..22f32885e4 --- /dev/null +++ b/lib/fluent/config/yaml_parser/loader.rb @@ -0,0 +1,91 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'psych' +require 'json' +require 'fluent/config/error' +require 'fluent/config/yaml_parser/fluent_value' + +# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb + +module Fluent + module Config + module YamlParser + class Loader + INCLUDE_TAG = 'tag:include'.freeze + FLUENT_JSON_TAG = 'tag:fluent/json'.freeze + FLUENT_STR_TAG = 'tag:fluent/s'.freeze + SHOVEL = '<<'.freeze + + def initialize(context = Kernel.binding) + @context = context + @current_path = nil + end + + # @param [String] path + # @return [Hash] + def load(path) + class_loader = Psych::ClassLoader.new + scanner = Psych::ScalarScanner.new(class_loader) + + visitor = Visitor.new(scanner, class_loader) + + visitor._register_domain(INCLUDE_TAG) do |_, val| + load(path.parent.join(val)) + end + + visitor._register_domain(FLUENT_JSON_TAG) do |_, val| + Fluent::Config::YamlParser::FluentValue::JsonValue.new(val) + end + + visitor._register_domain(FLUENT_STR_TAG) do |_, val| + Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context) + end + + path.open do |f| + visitor.accept(Psych.parse(f)) + end + end + + class Visitor < Psych::Visitors::ToRuby + def initialize(scanner, class_loader) + super(scanner, class_loader) + end + + def _register_domain(name, &block) + @domain_types.merge!({ name => [name, block] }) + end + + def revive_hash(hash, o) + super(hash, o).tap do |r| + if r[SHOVEL].is_a?(Hash) + h2 = {} + r.each do |k, v| + if k == SHOVEL + h2.merge!(v) + else + h2[k] = v + end + end + r.replace(h2) + end + end + end + end + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/parser.rb b/lib/fluent/config/yaml_parser/parser.rb new file mode 100644 index 0000000000..939798690a --- /dev/null +++ b/lib/fluent/config/yaml_parser/parser.rb @@ -0,0 +1,126 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/config/yaml_parser/section_builder' + +module Fluent + module Config + module YamlParser + class Parser + def initialize(config, indent: 2) + @base_indent = indent + @config = config + end + + def build + s = @config['system'] && system_config_build(@config['system']) + c = @config['config'] && config_build(@config['config'], root: true) + RootBuilder.new(s, c) + end + + private + + def system_config_build(config) + section_build('system', config) + end + + def config_build(config, indent: 0, root: false) + sb = SectionBodyBuilder.new(indent, root: root) + config.each do |c| + if (lc = c.delete('label')) + sb.add_section(label_build(lc, indent: indent)) + end + + if (sc = c.delete('source')) + sb.add_section(source_build(sc, indent: indent)) + end + + if (fc = c.delete('filter')) + sb.add_section(filter_build(fc, indent: indent)) + end + + if (mc = c.delete('match')) + sb.add_section(match_build(mc, indent: indent)) + end + + if (wc = c.delete('worker')) + sb.add_section(worker_build(wc, indent: indent)) + end + end + + sb + end + + def label_build(config, indent: 0) + config = config.dup + name = config.delete('$name') + c = config.delete('config') + SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name) + end + + def worker_build(config, indent: 0) + config = config.dup + num = config.delete('$arg') + c = config.delete('config') + SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num) + end + + def source_build(config, indent: 0) + section_build('source', config, indent: indent) + end + + def filter_build(config, indent: 0) + config = config.dup + tag = config.delete('$tag') + section_build('filter', config, indent: indent, arg: tag) + end + + def match_build(config, indent: 0) + config = config.dup + tag = config.delete('$tag') + section_build('match', config, indent: indent, arg: tag) + end + + def section_build(name, config, indent: 0, arg: nil) + sb = SectionBodyBuilder.new(indent + @base_indent) + + if (v = config.delete('$type')) + sb.add_line('@type', v) + end + + if (v = config.delete('$label')) + sb.add_line('@label', v) + end + + config.each do |key, val| + if val.is_a?(Array) + val.each do |v| + sb.add_section(section_build(key, v, indent: indent + @base_indent)) + end + elsif val.is_a?(Hash) + harg = val.delete('$arg') + sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg)) + else + sb.add_line(key, val) + end + end + + SectionBuilder.new(name, sb, indent, arg) + end + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/section_builder.rb b/lib/fluent/config/yaml_parser/section_builder.rb new file mode 100644 index 0000000000..58a89e6cd9 --- /dev/null +++ b/lib/fluent/config/yaml_parser/section_builder.rb @@ -0,0 +1,107 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module Config + module YamlParser + SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do + def to_s + indent = ' ' * indent_size + + if arg && !arg.to_s.empty? + "#{indent}<#{name} #{arg}>\n#{body}\n#{indent}" + else + "#{indent}<#{name}>\n#{body}\n#{indent}" + end + end + + def to_element + elem = body.to_element + elem.name = name + elem.arg = arg.to_s if arg + elem.v1_config = true + elem + end + end + + class RootBuilder + def initialize(system, conf) + @system = system + @conf = conf + end + + attr_reader :system, :conf + + def to_element + Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten) + end + + def to_s + s = StringIO.new(+'') + s.puts(@system.to_s) if @system + s.puts(@conf.to_s) if @conf + + s.string + end + end + + class SectionBodyBuilder + Row = Struct.new(:key, :value, :indent) do + def to_s + "#{indent}#{key} #{value}" + end + end + + def initialize(indent, root: false) + @indent = ' ' * indent + @bodies = [] + @root = root + end + + def add_line(k, v) + @bodies << Row.new(k, v, @indent) + end + + def add_section(section) + @bodies << section + end + + def to_element + if @root + return @bodies.map(&:to_element) + end + + not_section, section = @bodies.partition { |e| e.is_a?(Row) } + r = {} + not_section.each do |e| + v = e.value + r[e.key] = v.respond_to?(:to_element) ? v.to_element : v + end + + if @root + section.map(&:to_element) + else + Fluent::Config::Element.new('', '', r, section.map(&:to_element)) + end + end + + def to_s + @bodies.map(&:to_s).join("\n") + end + end + end + end +end diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 4c57048809..5bc533da1c 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -593,6 +593,7 @@ def self.cleanup_resources end def initialize(opt) + @config_file_type = opt[:config_file_type] @daemonize = opt[:daemonize] @standalone_worker= opt[:standalone_worker] @config_path = opt[:config_path] @@ -744,7 +745,13 @@ def configure(supervisor: false) $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711') @inline_config = STDIN.read end - @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config) + @conf = Fluent::Config.build( + config_path: @config_path, + encoding: @conf_encoding, + additional_config: @inline_config, + use_v1_config: @use_v1_config, + type: @config_file_type, + ) @system_config = build_system_config(@conf) @log.level = @system_config.log_level From 38e4ef08ecba8c532b22d9131975190194cf2e76 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Mon, 14 Feb 2022 17:17:42 +0900 Subject: [PATCH 02/15] yaml config: Handle Array object as concatenated string Signed-off-by: Hiroshi Hatake --- lib/fluent/config/yaml_parser/parser.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/fluent/config/yaml_parser/parser.rb b/lib/fluent/config/yaml_parser/parser.rb index 939798690a..62ecdc211a 100644 --- a/lib/fluent/config/yaml_parser/parser.rb +++ b/lib/fluent/config/yaml_parser/parser.rb @@ -112,7 +112,14 @@ def section_build(name, config, indent: 0, arg: nil) end elsif val.is_a?(Hash) harg = val.delete('$arg') - sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg)) + if harg.is_a?(Array) + # To prevent to generate invalid configuration for arg. + # "arg" should be String object and concatenated by "," + # when two or more objects are specified there. + sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg&.join(','))) + else + sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg)) + end else sb.add_line(key, val) end From 963cac40d7c1ca1ae3e6ffe6ffbef6617c1b350d Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Mon, 14 Feb 2022 17:31:18 +0900 Subject: [PATCH 03/15] Add mechanism for guessing config type Signed-off-by: Hiroshi Hatake --- lib/fluent/command/fluentd.rb | 4 ++++ lib/fluent/config.rb | 11 +++++++++-- lib/fluent/supervisor.rb | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb index 7880b86eb6..cb7c737ac2 100644 --- a/lib/fluent/command/fluentd.rb +++ b/lib/fluent/command/fluentd.rb @@ -134,6 +134,10 @@ end } +op.on('--guess-config-file-type', "Guess config file type that is yaml or not (default)", TrueClass) {|b| + opts[:guess_config_file_type] = b +} + op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b| opts[:use_v1_config] = b } diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index 59ca90f3c8..cc53d95c63 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -26,8 +26,15 @@ module Config # @param additional_config [String] config which is added to last of config body # @param use_v1_config [Bool] config is formatted with v1 or not # @return [Fluent::Config] - def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil) - if type == :yaml + def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil, guess_type: nil) + if guess_type + config_fext = File.extname(config_path.dup) + if config_fext == '.yaml' || config_fext == '.yml' + type = :yaml + end + end + + if type == :yaml || type == :yml return Fluent::Config::YamlParser.parse(config_path) end diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 5bc533da1c..91b408c16b 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -580,7 +580,8 @@ def self.default_options standalone_worker: false, signame: nil, conf_encoding: 'utf-8', - disable_shared_socket: nil + disable_shared_socket: nil, + guess_config_file_type: true, } end @@ -594,6 +595,7 @@ def self.cleanup_resources def initialize(opt) @config_file_type = opt[:config_file_type] + @guess_config_file_type = opt[:guess_config_file_type] @daemonize = opt[:daemonize] @standalone_worker= opt[:standalone_worker] @config_path = opt[:config_path] @@ -751,6 +753,7 @@ def configure(supervisor: false) additional_config: @inline_config, use_v1_config: @use_v1_config, type: @config_file_type, + guess_type: @guess_config_file_type, ) @system_config = build_system_config(@conf) @@ -936,6 +939,8 @@ def reload_config encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config, + type: @config_file_type, + guess_type: @guess_config_file_type, ) Fluent::VariableStore.try_to_reset do From 5edf1eb9b4000c101c90ade1d3e281d1319f9a19 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 15 Feb 2022 15:58:19 +0900 Subject: [PATCH 04/15] test: supervisor: Add testcases for Yaml configuration format Signed-off-by: Hiroshi Hatake --- lib/fluent/supervisor.rb | 5 ++- test/test_supervisor.rb | 90 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 91b408c16b..64a6461227 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -621,7 +621,10 @@ def initialize(opt) @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding ? @conf_encoding : 'utf-8', additional_config: @inline_config ? @inline_config : nil, - use_v1_config: !!@use_v1_config) + use_v1_config: !!@use_v1_config, + type: @config_file_type, + guess_type: @guess_config_file_type, + ) @system_config = build_system_config(@conf) if @system_config.log @log_rotate_age ||= @system_config.log.rotate_age diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 9b6ca82195..62f379d45b 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -90,6 +90,71 @@ def test_system_config assert_equal 2, counter_client.timeout end + sub_test_case "yaml config" do + def parse_yaml(yaml) + context = Kernel.binding + + config = nil + Tempfile.open do |file| + file.puts(yaml) + file.flush + s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(file)) + config = Fluent::Config::YamlParser::Parser.new(s).build.to_element + end + config + end + + def test_system_config + opts = Fluent::Supervisor.default_options + sv = Fluent::Supervisor.new(opts) + conf_data = <<-EOC + system: + rpc_endpoint: 127.0.0.1:24445 + suppress_repeated_stacktrace: true + suppress_config_dump: true + without_source: true + enable_get_dump: true + process_name: "process_name" + log_level: info + root_dir: !fluent/s "#{TMP_ROOT_DIR}" + log: + format: json + time_format: "%Y" + counter_server: + bind: 127.0.0.1 + port: 24321 + scope: server1 + backup_path: /tmp/backup + counter_client: + host: 127.0.0.1 + port: 24321 + timeout: 2 + EOC + conf = parse_yaml(conf_data) + sys_conf = sv.__send__(:build_system_config, conf) + + assert_equal '127.0.0.1:24445', sys_conf.rpc_endpoint + assert_equal true, sys_conf.suppress_repeated_stacktrace + assert_equal true, sys_conf.suppress_config_dump + assert_equal true, sys_conf.without_source + assert_equal true, sys_conf.enable_get_dump + assert_equal "process_name", sys_conf.process_name + assert_equal 2, sys_conf.log_level + assert_equal TMP_ROOT_DIR, sys_conf.root_dir + assert_equal :json, sys_conf.log.format + assert_equal '%Y', sys_conf.log.time_format + counter_server = sys_conf.counter_server + assert_equal '127.0.0.1', counter_server.bind + assert_equal 24321, counter_server.port + assert_equal 'server1', counter_server.scope + assert_equal '/tmp/backup', counter_server.backup_path + counter_client = sys_conf.counter_client + assert_equal '127.0.0.1', counter_client.host + assert_equal 24321, counter_client.port + assert_equal 2, counter_client.timeout + end + end + def test_main_process_signal_handlers omit "Windows cannot handle signals" if Fluent.windows? @@ -557,6 +622,31 @@ def test_override_default_log_rotate logger.instance_variable_get(:@rotate_size)]) end end + + def test_override_default_log_rotate_with_yaml_config + Tempfile.open do |file| + config = <<-EOS + system: + log: + rotate_age: 3 + rotate_size: 300 + EOS + file.puts(config) + file.flush + opts = Fluent::Supervisor.default_options.merge( + log_path: "#{TMP_DIR}/test.log", config_path: file.path, config_file_type: :yaml, + ) + sv = Fluent::Supervisor.new(opts) + + log = sv.instance_variable_get(:@log) + log.init(:standalone, 0) + logger = $log.instance_variable_get(:@logger) + + assert_equal([3, 300], + [logger.instance_variable_get(:@rotate_age), + logger.instance_variable_get(:@rotate_size)]) + end + end end def test_inline_config From 701267efb9f2794c35e11d16efecf61d451790d6 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 8 Apr 2022 15:42:35 +0900 Subject: [PATCH 05/15] test: config: Add yaml config format testcase Signed-off-by: Hiroshi Hatake --- test/test_config.rb | 61 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/test/test_config.rb b/test/test_config.rb index e9d1005647..b19e7e816f 100644 --- a/test/test_config.rb +++ b/test/test_config.rb @@ -10,11 +10,18 @@ class ConfigTest < Test::Unit::TestCase TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}" - def read_config(path) + def read_config(path, use_yaml: false) path = File.expand_path(path) - File.open(path) { |io| - Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path)) - } + if use_yaml + context = Kernel.binding + + s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path)) + Fluent::Config::YamlParser::Parser.new(s).build.to_element + else + File.open(path) { |io| + Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path)) + } + end end def prepare_config @@ -151,6 +158,52 @@ def test_check_not_fetchd assert_equal before_size, match_conf.unused.size end + sub_test_case "yaml config" do + def test_check_not_fetchd + write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS + config: + - match: + $arg: dummy + $type: rewrite + add_prefix: filtered + rule: + key: path + pattern: "^[A-Z]+" + replace: true + EOS + root_conf = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true) + match_conf = root_conf.elements.first + rule_conf = match_conf.elements.first + + not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key } + assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched + + not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key } + assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched + + not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key } + assert_equal %w[key pattern replace], not_fetched + + # accessing should delete + match_conf['type'] + rule_conf['key'] + + not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key } + assert_equal %w[@type $arg add_prefix pattern replace], not_fetched + + not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key } + assert_equal %w[@type $arg add_prefix pattern replace], not_fetched + + not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key } + assert_equal %w[pattern replace], not_fetched + + # repeatedly accessing should not grow memory usage + before_size = match_conf.unused.size + 10.times { match_conf['type'] } + assert_equal before_size, match_conf.unused.size + end + end + def write_config(path, data, encoding: 'utf-8') FileUtils.mkdir_p(File.dirname(path)) File.open(path, "w:#{encoding}:utf-8") {|f| f.write data } From cdde6608902e6237d873eeed0fdb333df37ed172 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 16 Feb 2022 10:55:41 +0900 Subject: [PATCH 06/15] yaml config: Handle pipeline-level !include yaml tag Signed-off-by: Hiroshi Hatake --- lib/fluent/config/yaml_parser/parser.rb | 21 ++++++++ test/test_config.rb | 65 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/lib/fluent/config/yaml_parser/parser.rb b/lib/fluent/config/yaml_parser/parser.rb index 62ecdc211a..ff7518053c 100644 --- a/lib/fluent/config/yaml_parser/parser.rb +++ b/lib/fluent/config/yaml_parser/parser.rb @@ -59,6 +59,8 @@ def config_build(config, indent: 0, root: false) if (wc = c.delete('worker')) sb.add_section(worker_build(wc, indent: indent)) end + + included_sections_build(c, sb, indent: indent) end sb @@ -94,6 +96,25 @@ def match_build(config, indent: 0) section_build('match', config, indent: indent, arg: tag) end + def included_sections_build(config, section_builder, indent: 0) + config.each_entry do |e| + k = e.keys.first + cc = e.delete(k) + case k + when 'label' + section_builder.add_section(label_build(cc, indent: indent)) + when 'worker' + section_builder.add_section(worker_build(cc, indent: indent)) + when 'source' + section_builder.add_section(source_build(cc, indent: indent)) + when 'filter' + section_builder.add_section(filter_build(cc, indent: indent)) + when 'match' + section_builder.add_section(match_build(cc, indent: indent)) + end + end + end + def section_build(name, config, indent: 0, arg: nil) sb = SectionBodyBuilder.new(indent + @base_indent) diff --git a/test/test_config.rb b/test/test_config.rb index b19e7e816f..3616fc00b9 100644 --- a/test/test_config.rb +++ b/test/test_config.rb @@ -159,6 +159,71 @@ def test_check_not_fetchd end sub_test_case "yaml config" do + def test_included + write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS + config: + - source: + $type: dummy + tag: tag.dummy + - source: + $type: tcp + tag: tag.tcp + parse: + $arg: + - why.parse.section.doesnot.have.arg + - huh + $type: none + - match: + $tag: tag.* + $type: stdout + buffer: + $type: memory + flush_interval: 1s + - !include fluent-included.yaml + EOS + write_config "#{TMP_DIR}/fluent-included.yaml", <<-EOS + - label: + $name: '@FLUENT_LOG' + config: + - match: + $type: "null" + $tag: "**" + buffer: + $type: memory + flush_mode: interval + flush_interval: 1s + EOS + root_conf = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true) + dummy_source_conf = root_conf.elements.first + tcp_source_conf = root_conf.elements[1] + parse_tcp_conf = tcp_source_conf.elements.first + match_conf = root_conf.elements[2] + label_conf = root_conf.elements[3] + fluent_log_conf = label_conf.elements.first + fluent_log_buffer_conf = fluent_log_conf.elements.first + + assert_equal 'dummy', dummy_source_conf['@type'] + assert_equal 'tag.dummy', dummy_source_conf['tag'] + + assert_equal 'tcp', tcp_source_conf['@type'] + assert_equal 'tag.tcp', tcp_source_conf['tag'] + + assert_equal 'none', parse_tcp_conf['@type'] + assert_equal 'why.parse.section.doesnot.have.arg,huh', parse_tcp_conf.arg + + assert_equal 'stdout', match_conf['@type'] + assert_equal 'tag.*', match_conf.arg + + assert_equal 'null', fluent_log_conf['@type'] + assert_equal '**', fluent_log_conf.arg + + assert_equal '@FLUENT_LOG', label_conf.arg + + assert_equal 'memory', fluent_log_buffer_conf['@type'] + assert_equal 'interval', fluent_log_buffer_conf['flush_mode'] + assert_equal '1s', fluent_log_buffer_conf['flush_interval'] + end + def test_check_not_fetchd write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS config: From 39b10cfb6126bacd1efd225c2892550938bd51a6 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 12 Apr 2022 10:35:19 +0900 Subject: [PATCH 07/15] Don't use abbreviation for file_ext Signed-off-by: Hiroshi Hatake --- lib/fluent/config.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index cc53d95c63..be1485f493 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -28,8 +28,8 @@ module Config # @return [Fluent::Config] def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil, guess_type: nil) if guess_type - config_fext = File.extname(config_path.dup) - if config_fext == '.yaml' || config_fext == '.yml' + config_file_ext = File.extname(config_path.dup) + if config_file_ext == '.yaml' || config_file_ext == '.yml' type = :yaml end end From ebe9e8c42aba46dc15658c5c746fda19b3ab39a9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 12 Apr 2022 10:36:12 +0900 Subject: [PATCH 08/15] Remove needless #dup Signed-off-by: Hiroshi Hatake --- lib/fluent/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index be1485f493..246703ea60 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -28,7 +28,7 @@ module Config # @return [Fluent::Config] def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil, guess_type: nil) if guess_type - config_file_ext = File.extname(config_path.dup) + config_file_ext = File.extname(config_path) if config_file_ext == '.yaml' || config_file_ext == '.yml' type = :yaml end From e3a2be8ddfd6a8a696de531db9a8e4ddd992a1e9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 13 Apr 2022 19:05:43 +0900 Subject: [PATCH 09/15] config: yaml: Flatten array object for $tag Signed-off-by: Hiroshi Hatake --- lib/fluent/config/yaml_parser/parser.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/fluent/config/yaml_parser/parser.rb b/lib/fluent/config/yaml_parser/parser.rb index ff7518053c..2fc44bb6d5 100644 --- a/lib/fluent/config/yaml_parser/parser.rb +++ b/lib/fluent/config/yaml_parser/parser.rb @@ -87,13 +87,21 @@ def source_build(config, indent: 0) def filter_build(config, indent: 0) config = config.dup tag = config.delete('$tag') - section_build('filter', config, indent: indent, arg: tag) + if tag.is_a?(Array) + section_build('filter', config, indent: indent, arg: tag&.join(',')) + else + section_build('filter', config, indent: indent, arg: tag) + end end def match_build(config, indent: 0) config = config.dup tag = config.delete('$tag') - section_build('match', config, indent: indent, arg: tag) + if tag.is_a?(Array) + section_build('match', config, indent: indent, arg: tag&.join(',')) + else + section_build('match', config, indent: indent, arg: tag) + end end def included_sections_build(config, section_builder, indent: 0) From dc597c03ee0f13269082b4828db17268e38bc3d0 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 13 Apr 2022 22:42:20 +0900 Subject: [PATCH 10/15] config: yaml: Support short-hand `hostname` and `worker_id` methods Signed-off-by: Hiroshi Hatake --- lib/fluent/config/yaml_parser.rb | 12 ++++++++++++ test/command/test_fluentd.rb | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/lib/fluent/config/yaml_parser.rb b/lib/fluent/config/yaml_parser.rb index 07ef8a414c..e000bfa446 100644 --- a/lib/fluent/config/yaml_parser.rb +++ b/lib/fluent/config/yaml_parser.rb @@ -36,6 +36,18 @@ def self.parse(path) end end + unless context.respond_to?(:hostname) + context.define_singleton_method(:hostname) do + Socket.gethostname + end + end + + unless context.respond_to?(:worker_id) + context.define_singleton_method(:worker_id) do + ENV['SERVERENGINE_WORKER_ID'] || '' + end + end + s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path)) Fluent::Config::YamlParser::Parser.new(s).build.to_element end diff --git a/test/command/test_fluentd.rb b/test/command/test_fluentd.rb index 643588ffdb..2ad023bf90 100644 --- a/test/command/test_fluentd.rb +++ b/test/command/test_fluentd.rb @@ -589,6 +589,39 @@ def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10) ) end + sub_test_case "YAML config format" do + test 'success to start the number of workers specified in configuration' do + conf = <<'CONF' + system: + workers: 2 + root_dir: "#{@root_path}" + config: + - source: + $type: dummy + $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command + $label: '@dummydata' + tag: dummy + dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"} + + - label: + $name: '@dummydata' + config: + - match: + $tag: dummy + $type: "null" + $id: blackhole +CONF + conf_path = create_conf_file('workers1.yaml', conf) + assert Dir.exist?(@root_path) + + assert_log_matches( + create_cmdline(conf_path), + "#0 fluentd worker is now running worker=0", + "#1 fluentd worker is now running worker=1" + ) + end + end + test 'success to start the number of workers specified by command line option' do conf = < From 79c88a35079cdb11207d618ff3650e38a9830336 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 13 Apr 2022 23:00:44 +0900 Subject: [PATCH 11/15] config: yaml: Handle $id elements for interpreting as `@id` Signed-off-by: Hiroshi Hatake --- lib/fluent/config/yaml_parser/parser.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/fluent/config/yaml_parser/parser.rb b/lib/fluent/config/yaml_parser/parser.rb index 2fc44bb6d5..dc1caaf46d 100644 --- a/lib/fluent/config/yaml_parser/parser.rb +++ b/lib/fluent/config/yaml_parser/parser.rb @@ -134,6 +134,10 @@ def section_build(name, config, indent: 0, arg: nil) sb.add_line('@label', v) end + if (v = config.delete('$id')) + sb.add_line('@id', v) + end + config.each do |key, val| if val.is_a?(Array) val.each do |v| From 49ec02315311c94ba3550c0b752ad24deaaaee0e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 17 May 2022 19:31:02 +0900 Subject: [PATCH 12/15] config: test: Guess configuration file type by default Signed-off-by: Hiroshi Hatake --- lib/fluent/command/fluentd.rb | 10 +++------- lib/fluent/config.rb | 10 ++++------ lib/fluent/supervisor.rb | 12 ++++-------- test/test_supervisor.rb | 2 +- 4 files changed, 12 insertions(+), 22 deletions(-) diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb index cb7c737ac2..6abf30a076 100644 --- a/lib/fluent/command/fluentd.rb +++ b/lib/fluent/command/fluentd.rb @@ -126,18 +126,14 @@ opts[:without_source] = b } -op.on('--config-file-type VALU', 'file type of fluentd. yaml or yml') { |s| +op.on('--guess-config-file-type VALU', 'guessing file type of fluentd configuration. yaml or yml') { |s| if (s == 'yaml') || (s == 'yml') - opts[:config_file_type] = s.to_sym + opts[:guess_config_file_type] = s.to_sym else - usage '--config-file-type accepts yaml or yml' + usage '--guess-config-file-type accepts yaml or yml' end } -op.on('--guess-config-file-type', "Guess config file type that is yaml or not (default)", TrueClass) {|b| - opts[:guess_config_file_type] = b -} - op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b| opts[:use_v1_config] = b } diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index 246703ea60..acbef5ab17 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -26,12 +26,10 @@ module Config # @param additional_config [String] config which is added to last of config body # @param use_v1_config [Bool] config is formatted with v1 or not # @return [Fluent::Config] - def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil, guess_type: nil) - if guess_type - config_file_ext = File.extname(config_path) - if config_file_ext == '.yaml' || config_file_ext == '.yml' - type = :yaml - end + def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil) + config_file_ext = File.extname(config_path) + if config_file_ext == '.yaml' || config_file_ext == '.yml' + type = :yaml end if type == :yaml || type == :yml diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 64a6461227..035193f664 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -581,7 +581,7 @@ def self.default_options signame: nil, conf_encoding: 'utf-8', disable_shared_socket: nil, - guess_config_file_type: true, + guess_config_file_type: nil, } end @@ -594,7 +594,6 @@ def self.cleanup_resources end def initialize(opt) - @config_file_type = opt[:config_file_type] @guess_config_file_type = opt[:guess_config_file_type] @daemonize = opt[:daemonize] @standalone_worker= opt[:standalone_worker] @@ -622,8 +621,7 @@ def initialize(opt) encoding: @conf_encoding ? @conf_encoding : 'utf-8', additional_config: @inline_config ? @inline_config : nil, use_v1_config: !!@use_v1_config, - type: @config_file_type, - guess_type: @guess_config_file_type, + type: @guess_config_file_type, ) @system_config = build_system_config(@conf) if @system_config.log @@ -755,8 +753,7 @@ def configure(supervisor: false) encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config, - type: @config_file_type, - guess_type: @guess_config_file_type, + type: @guess_config_file_type, ) @system_config = build_system_config(@conf) @@ -942,8 +939,7 @@ def reload_config encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config, - type: @config_file_type, - guess_type: @guess_config_file_type, + type: @guess_config_file_type, ) Fluent::VariableStore.try_to_reset do diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 62f379d45b..22794b44ae 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -634,7 +634,7 @@ def test_override_default_log_rotate_with_yaml_config file.puts(config) file.flush opts = Fluent::Supervisor.default_options.merge( - log_path: "#{TMP_DIR}/test.log", config_path: file.path, config_file_type: :yaml, + log_path: "#{TMP_DIR}/test.log", config_path: file.path, guess_config_file_type: :yaml, ) sv = Fluent::Supervisor.new(opts) From 6574387ef2c78363dc23640df14d9886d5606bba Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 18 May 2022 10:42:20 +0900 Subject: [PATCH 13/15] config: test: Specify guess type as default value of config-file-type Signed-off-by: Hiroshi Hatake --- lib/fluent/command/fluentd.rb | 8 +++++--- lib/fluent/config.rb | 8 +++++--- lib/fluent/supervisor.rb | 10 +++++----- test/test_supervisor.rb | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb index 6abf30a076..2150ece521 100644 --- a/lib/fluent/command/fluentd.rb +++ b/lib/fluent/command/fluentd.rb @@ -126,11 +126,13 @@ opts[:without_source] = b } -op.on('--guess-config-file-type VALU', 'guessing file type of fluentd configuration. yaml or yml') { |s| +op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s| if (s == 'yaml') || (s == 'yml') - opts[:guess_config_file_type] = s.to_sym + opts[:config_file_type] = s.to_sym + elsif (s == 'guess') + opts[:config_file_type] = s.to_sym else - usage '--guess-config-file-type accepts yaml or yml' + usage '--config-file-type accepts yaml/yml or guess' end } diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index acbef5ab17..d471114830 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -27,9 +27,11 @@ module Config # @param use_v1_config [Bool] config is formatted with v1 or not # @return [Fluent::Config] def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil) - config_file_ext = File.extname(config_path) - if config_file_ext == '.yaml' || config_file_ext == '.yml' - type = :yaml + if type == :guess + config_file_ext = File.extname(config_path) + if config_file_ext == '.yaml' || config_file_ext == '.yml' + type = :yaml + end end if type == :yaml || type == :yml diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 035193f664..1df23053d6 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -581,7 +581,7 @@ def self.default_options signame: nil, conf_encoding: 'utf-8', disable_shared_socket: nil, - guess_config_file_type: nil, + config_file_type: :guess, } end @@ -594,7 +594,7 @@ def self.cleanup_resources end def initialize(opt) - @guess_config_file_type = opt[:guess_config_file_type] + @config_file_type = opt[:config_file_type] @daemonize = opt[:daemonize] @standalone_worker= opt[:standalone_worker] @config_path = opt[:config_path] @@ -621,7 +621,7 @@ def initialize(opt) encoding: @conf_encoding ? @conf_encoding : 'utf-8', additional_config: @inline_config ? @inline_config : nil, use_v1_config: !!@use_v1_config, - type: @guess_config_file_type, + type: @config_file_type, ) @system_config = build_system_config(@conf) if @system_config.log @@ -753,7 +753,7 @@ def configure(supervisor: false) encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config, - type: @guess_config_file_type, + type: @config_file_type, ) @system_config = build_system_config(@conf) @@ -939,7 +939,7 @@ def reload_config encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config, - type: @guess_config_file_type, + type: @config_file_type, ) Fluent::VariableStore.try_to_reset do diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 22794b44ae..62f379d45b 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -634,7 +634,7 @@ def test_override_default_log_rotate_with_yaml_config file.puts(config) file.flush opts = Fluent::Supervisor.default_options.merge( - log_path: "#{TMP_DIR}/test.log", config_path: file.path, guess_config_file_type: :yaml, + log_path: "#{TMP_DIR}/test.log", config_path: file.path, config_file_type: :yaml, ) sv = Fluent::Supervisor.new(opts) From 7c7d92fd19334b271890ea96249afb7bb6214ca9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 18 May 2022 10:45:52 +0900 Subject: [PATCH 14/15] config: test: Assert parameters at once Co-authored-by: Takuro Ashie Signed-off-by: Hiroshi Hatake --- test/test_config.rb | 53 ++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/test/test_config.rb b/test/test_config.rb index 3616fc00b9..74b4dc5d7f 100644 --- a/test/test_config.rb +++ b/test/test_config.rb @@ -202,26 +202,39 @@ def test_included fluent_log_conf = label_conf.elements.first fluent_log_buffer_conf = fluent_log_conf.elements.first - assert_equal 'dummy', dummy_source_conf['@type'] - assert_equal 'tag.dummy', dummy_source_conf['tag'] - - assert_equal 'tcp', tcp_source_conf['@type'] - assert_equal 'tag.tcp', tcp_source_conf['tag'] - - assert_equal 'none', parse_tcp_conf['@type'] - assert_equal 'why.parse.section.doesnot.have.arg,huh', parse_tcp_conf.arg - - assert_equal 'stdout', match_conf['@type'] - assert_equal 'tag.*', match_conf.arg - - assert_equal 'null', fluent_log_conf['@type'] - assert_equal '**', fluent_log_conf.arg - - assert_equal '@FLUENT_LOG', label_conf.arg - - assert_equal 'memory', fluent_log_buffer_conf['@type'] - assert_equal 'interval', fluent_log_buffer_conf['flush_mode'] - assert_equal '1s', fluent_log_buffer_conf['flush_interval'] + assert_equal( + [ + 'dummy', + 'tag.dummy', + 'tcp', + 'tag.tcp', + 'none', + 'why.parse.section.doesnot.have.arg,huh', + 'stdout', + 'tag.*', + 'null', + '**', + '@FLUENT_LOG', + 'memory', + 'interval', + '1s', + ], + [ + dummy_source_conf['@type'], + dummy_source_conf['tag'], + tcp_source_conf['@type'], + tcp_source_conf['tag'], + parse_tcp_conf['@type'], + parse_tcp_conf.arg, + match_conf['@type'], + match_conf.arg, + fluent_log_conf['@type'], + fluent_log_conf.arg, + label_conf.arg, + fluent_log_buffer_conf['@type'], + fluent_log_buffer_conf['flush_mode'], + fluent_log_buffer_conf['flush_interval'], + ]) end def test_check_not_fetchd From db8b245a1fe22bf4063510aea6e740f5273bd625 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 18 May 2022 10:47:58 +0900 Subject: [PATCH 15/15] supervisor: test: Assert parameters at once Signed-off-by: Hiroshi Hatake Co-authored-by: Takuro Ashie --- test/test_supervisor.rb | 60 ++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 62f379d45b..86159c2cb7 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -133,25 +133,47 @@ def test_system_config conf = parse_yaml(conf_data) sys_conf = sv.__send__(:build_system_config, conf) - assert_equal '127.0.0.1:24445', sys_conf.rpc_endpoint - assert_equal true, sys_conf.suppress_repeated_stacktrace - assert_equal true, sys_conf.suppress_config_dump - assert_equal true, sys_conf.without_source - assert_equal true, sys_conf.enable_get_dump - assert_equal "process_name", sys_conf.process_name - assert_equal 2, sys_conf.log_level - assert_equal TMP_ROOT_DIR, sys_conf.root_dir - assert_equal :json, sys_conf.log.format - assert_equal '%Y', sys_conf.log.time_format - counter_server = sys_conf.counter_server - assert_equal '127.0.0.1', counter_server.bind - assert_equal 24321, counter_server.port - assert_equal 'server1', counter_server.scope - assert_equal '/tmp/backup', counter_server.backup_path - counter_client = sys_conf.counter_client - assert_equal '127.0.0.1', counter_client.host - assert_equal 24321, counter_client.port - assert_equal 2, counter_client.timeout + counter_client = sys_conf.counter_client + counter_server = sys_conf.counter_server + assert_equal( + [ + '127.0.0.1:24445', + true, + true, + true, + true, + "process_name", + 2, + TMP_ROOT_DIR, + :json, + '%Y', + '127.0.0.1', + 24321, + 'server1', + '/tmp/backup', + '127.0.0.1', + 24321, + 2, + ], + [ + sys_conf.rpc_endpoint, + sys_conf.suppress_repeated_stacktrace, + sys_conf.suppress_config_dump, + sys_conf.without_source, + sys_conf.enable_get_dump, + sys_conf.process_name, + sys_conf.log_level, + sys_conf.root_dir, + sys_conf.log.format, + sys_conf.log.time_format, + counter_server.bind, + counter_server.port, + counter_server.scope, + counter_server.backup_path, + counter_client.host, + counter_client.port, + counter_client.timeout, + ]) end end