diff --git a/bin/fluent-plugin-config-format b/bin/fluent-plugin-config-format new file mode 100755 index 0000000000..3aa955cc5d --- /dev/null +++ b/bin/fluent-plugin-config-format @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +$LOAD_PATH.unshift(File.join(__dir__, 'lib')) +require 'fluent/command/plugin_config_formatter' + +FluentPluginConfigFormatter.new.call diff --git a/bin/fluent-plugin-generate b/bin/fluent-plugin-generate new file mode 100755 index 0000000000..3ee0e7adfd --- /dev/null +++ b/bin/fluent-plugin-generate @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +$LOAD_PATH << File.expand_path(File.join(__dir__, '..', 'lib')) +require 'fluent/command/plugin_generator' + +FluentPluginGenerator.new.call diff --git a/lib/fluent/command/plugin_config_formatter.rb b/lib/fluent/command/plugin_config_formatter.rb new file mode 100644 index 0000000000..c1b66210aa --- /dev/null +++ b/lib/fluent/command/plugin_config_formatter.rb @@ -0,0 +1,258 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "erb" +require "optparse" +require "pathname" +require "fluent/plugin" +require "fluent/env" +require "fluent/engine" +require "fluent/system_config" +require "fluent/config/element" + +class FluentPluginConfigFormatter + + AVAILABLE_FORMATS = [:markdown, :txt, :json] + SUPPORTED_TYPES = [ + "input", "output", "filter", + "buffer", "parser", "formatter", "storage" + ] + + def initialize(argv = ARGV) + @argv = argv + + @compact = false + @format = :markdown + @verbose = false + @libs = [] + @plugin_dirs = [] + @options = {} + + prepare_option_parser + end + + def call + parse_options! + init_libraries + @plugin = Fluent::Plugin.__send__("new_#{@plugin_type}", @plugin_name) + dumped_config = {} + if @plugin.class.respond_to?(:plugin_helpers) + dumped_config[:plugin_helpers] = @plugin.class.plugin_helpers + end + @plugin.class.ancestors.reverse_each do |plugin_class| + next unless plugin_class.respond_to?(:dump_config_definition) + unless @verbose + next if plugin_class.name =~ /::PluginHelper::/ + end + dumped_config_definition = plugin_class.dump_config_definition + dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty? + end + case @format + when :txt + puts dump_txt(dumped_config) + when :markdown + puts dump_markdown(dumped_config) + when :json + puts dump_json(dumped_config) + end + end + + private + + def dump_txt(dumped_config) + dumped = "" + plugin_helpers = dumped_config.delete(:plugin_helpers) + if plugin_helpers && !plugin_helpers.empty? + dumped << "helpers: #{plugin_helpers.join(',')}\n" + end + if @verbose + dumped_config.each do |name, config| + dumped << "#{name}\n" + dumped << dump_section_txt(config) + end + else + configs = dumped_config.values + root_section = configs.shift + configs.each do |config| + root_section.update(config) + end + dumped << dump_section_txt(root_section) + end + dumped + end + + def dump_section_txt(base_section, level = 0) + dumped = "" + indent = " " * level + if base_section[:section] + sections = [] + params = base_section + else + sections, params = base_section.partition {|_name, value| value[:section] } + end + params.each do |name, config| + next if name == :section + dumped << "#{indent}#{name}: #{config[:type]}: (#{config[:default].inspect})" + dumped << " # #{config[:description]}" if config.key?(:description) + dumped << "\n" + end + sections.each do |section_name, sub_section| + required = sub_section.delete(:required) + multi = sub_section.delete(:multi) + alias_name = sub_section.delete(:alias) + required_label = required ? "required" : "optional" + multi_label = multi ? "multiple" : "single" + alias_label = "alias: #{alias_name}" + dumped << "#{indent}<#{section_name}>: #{required_label}, #{multi_label}" + if alias_name + dumped << ", #{alias_label}\n" + else + dumped << "\n" + end + sub_section.delete(:section) + dumped << dump_section_txt(sub_section, level + 1) + end + dumped + end + + def dump_markdown(dumped_config) + dumped = "" + plugin_helpers = dumped_config.delete(:plugin_helpers) + if plugin_helpers && !plugin_helpers.empty? + dumped = "## Plugin helpers\n\n" + plugin_helpers.each do |plugin_helper| + dumped << "* #{plugin_helper}\n" + end + dumped << "\n" + end + dumped_config.each do |name, config| + if name == @plugin.class.name + dumped << "## #{name}\n\n" + dumped << dump_section_markdown(config) + else + dumped << "* See also: #{name}\n\n" + end + end + dumped + end + + def dump_section_markdown(base_section, level = 0) + dumped = "" + if base_section[:section] + sections = [] + params = base_section + else + sections, params = base_section.partition {|_name, value| value[:section] } + end + params.each do |name, config| + next if name == :section + template_name = @compact ? "param.md-compact.erb" : "param.md.erb" + dumped << ERB.new(template_path(template_name).read, nil, "-").result(binding) + end + dumped << "\n" + sections.each do |section_name, sub_section| + required = sub_section.delete(:required) + multi = sub_section.delete(:multi) + alias_name = sub_section.delete(:alias) + $log.trace(name: section_name, required: required, multi: multi, alias_name: alias_name) + sub_section.delete(:section) + dumped << ERB.new(template_path("section.md.erb").read, nil, "-").result(binding) + end + dumped + end + + def dump_json(dumped_config) + if @compact + JSON.generate(dumped_config) + else + JSON.pretty_generate(dumped_config) + end + end + + def usage(message = nil) + puts @parser.to_s + puts + puts "Error: #{message}" if message + exit(false) + end + + def prepare_option_parser + @parser = OptionParser.new + @parser.banner = < + +Output plugin config definitions + +Arguments: +\ttype: #{SUPPORTED_TYPES.join(",")} +\tname: registered plugin name + +Options: +BANNER + @parser.on("--verbose", "Be verbose") do + @verbose = true + end + @parser.on("-c", "--compact", "Compact output") do + @compact = true + end + @parser.on("-f", "--format=FORMAT", "Specify format. (#{AVAILABLE_FORMATS.join(',')})") do |s| + format = s.to_sym + usage("Unsupported format: #{s}") unless AVAILABLE_FORMATS.include?(format) + @format = format + end + @parser.on("-I PATH", "Add PATH to $LOAD_PATH") do |s| + $LOAD_PATH.unshift(s) + end + @parser.on("-r NAME", "Load library") do |s| + @libs << s + end + @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s| + @plugin_dirs << s + end + end + + def parse_options! + @parser.parse!(@argv) + + raise "Must specify plugin type and name" unless @argv.size == 2 + + @plugin_type, @plugin_name = @argv + @options = { + compact: @compact, + format: @format, + verbose: @verbose, + } + rescue => e + usage(e) + end + + def init_libraries + @libs.each do |lib| + require lib + end + + @plugin_dirs.each do |dir| + if Dir.exist?(dir) + dir = File.expand_path(dir) + Fluent::Plugin.add_plugin_dir(dir) + end + end + end + + def template_path(name) + (Pathname(__dir__) + "../../../templates/plugin_config_formatter/#{name}").realpath + end +end diff --git a/lib/fluent/command/plugin_generator.rb b/lib/fluent/command/plugin_generator.rb new file mode 100644 index 0000000000..dda98fe0f8 --- /dev/null +++ b/lib/fluent/command/plugin_generator.rb @@ -0,0 +1,301 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "optparse" +require "pathname" +require "fileutils" +require "erb" +require "open-uri" + +require "fluent/registry" + +class FluentPluginGenerator + attr_reader :type, :name + attr_reader :license_name + + SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"] + + def initialize(argv = ARGV) + @argv = argv + @parser = prepare_parser + + @license_name = "Apache-2.0" + @overwrite_all = false + end + + def call + parse_options! + FileUtils.mkdir_p(gem_name) + Dir.chdir(gem_name) do + copy_license + template_directory.find do |path| + next if path.directory? + dest_dir = path.dirname.sub(/\A#{Regexp.quote(template_directory.to_s)}\/?/, "") + dest_file = dest_filename(path) + if path.extname == ".erb" + if path.fnmatch?("*/plugin/*") + next unless path.basename.fnmatch?("*#{type}*") + end + template(path, dest_dir + dest_file) + else + file(path, dest_dir + dest_file) + end + end + pid = spawn("git", "init", ".") + Process.wait(pid) + end + end + + private + + def template_directory + (Pathname(__dir__) + "../../../templates/new_gem").realpath + end + + def template_file(filename) + template_directory + filename + end + + def template(source, dest) + dest.dirname.mkpath + contents = ERB.new(source.read, nil, "-").result(binding) + label = create_label(dest, contents) + puts "\t#{label} #{dest}" + if label == "conflict" + return unless overwrite?(dest) + end + File.write(dest, contents) + end + + def file(source, dest) + label = create_label(dest, source.read) + puts "\t#{label} #{dest}" + if label == "conflict" + return unless overwrite?(dest) + end + FileUtils.cp(source, dest) + end + + def prepare_parser + @parser = OptionParser.new + @parser.banner = < + +Generate a project skeleton for creating a Fluentd plugin + +Arguments: +\ttype: #{SUPPORTED_TYPES.join(",")} +\tname: Your plugin name + +Options: +BANNER + + @parser.on("--[no-]license=NAME", "Specify license name (default: Apache-2.0)") do |v| + @license_name = v || "no-license" + end + @parser + end + + def parse_options! + @parser.parse!(@argv) + unless @argv.size == 2 + raise ArgumentError, "Missing arguments" + end + @type, @name = @argv + rescue => e + usage("#{e.class}:#{e.message}") + end + + def usage(message = "") + puts message + puts + puts @parser.help + exit(false) + end + + def user_name + v = `git config --get user.name`.chomp + v.empty? ? "TODO: Write your name" : v + end + + def user_email + v = `git config --get user.email`.chomp + v.empty? ? "TODO: Write your email" : v + end + + def gem_name + "fluent-plugin-#{name}" + end + + def class_name + "#{name.capitalize}#{type.capitalize}" + end + + def plugin_filename + case type + when "input" + "in_#{name}.rb" + when "output" + "out_#{name}.rb" + else + "#{type}_#{name}.rb" + end + end + + def test_filename + case type + when "input" + "test_in_#{name}.rb" + when "output" + "test_out_#{name}.rb" + else + "test_#{type}_#{name}.rb" + end + end + + def dest_filename(path) + case path.to_s + when %r!\.gemspec! + "#{gem_name}.gemspec" + when %r!lib/fluent/plugin! + plugin_filename + when %r!test/plugin! + test_filename + else + path.basename.sub_ext("") + end + end + + def preamble + @license.preamble(user_name) + end + + def copy_license + # in gem_name directory + return unless license_name + puts "License: #{license_name}" + license_class = self.class.lookup_license(license_name) + @license = license_class.new + Pathname("LICENSE").write(@license.text) unless @license.text.empty? + rescue Fluent::ConfigError + usage("Unknown license: #{license_name}") + rescue => ex + usage("#{ex.class}: #{ex.message}") + end + + def create_label(dest, contents) + if dest.exist? + if dest.read == contents + "identical" + else + "conflict" + end + else + "create" + end + end + + def overwrite?(dest) + return true if @overwrite_all + loop do + print "Overwrite #{dest}? (enter \"h\" for help) [Ynaqh]" + answer = $stdin.gets.chomp + return true if /\Ay\z/i =~ answer || answer.empty? + case answer + when "n" + return false + when "a" + @overwrite_all = true + return true + when "q" + exit + when "h" + puts < NoLicense, + "Apache-2.0" => ApacheLicense + }.each do |license, klass| + register_license(license, klass) + end +end diff --git a/lib/fluent/compat/filter.rb b/lib/fluent/compat/filter.rb index ab542607e6..617e800633 100644 --- a/lib/fluent/compat/filter.rb +++ b/lib/fluent/compat/filter.rb @@ -25,7 +25,7 @@ module Compat class Filter < Fluent::Plugin::Filter # TODO: warn when deprecated - helpers :inject + helpers_internal :inject def initialize super diff --git a/lib/fluent/compat/output.rb b/lib/fluent/compat/output.rb index 87be76af18..39613e79a4 100644 --- a/lib/fluent/compat/output.rb +++ b/lib/fluent/compat/output.rb @@ -149,7 +149,7 @@ def write(chunk) class Output < Fluent::Plugin::Output # TODO: warn when deprecated - helpers :event_emitter, :inject + helpers_internal :event_emitter, :inject def support_in_v12_style?(feature) case feature @@ -205,7 +205,7 @@ def detach_multi_process(&block) class MultiOutput < Fluent::Plugin::BareOutput # TODO: warn when deprecated - helpers :event_emitter + helpers_internal :event_emitter def process(tag, es) emit(tag, es, NULL_OUTPUT_CHAIN) @@ -215,7 +215,7 @@ def process(tag, es) class BufferedOutput < Fluent::Plugin::Output # TODO: warn when deprecated - helpers :event_emitter, :inject + helpers_internal :event_emitter, :inject def support_in_v12_style?(feature) case feature @@ -425,7 +425,7 @@ def detach_multi_process(&block) class ObjectBufferedOutput < Fluent::Plugin::Output # TODO: warn when deprecated - helpers :event_emitter, :inject + helpers_internal :event_emitter, :inject # This plugin cannot inherit BufferedOutput because #configure sets chunk_key 'tag' # to flush chunks per tags, but BufferedOutput#configure doesn't allow setting chunk_key @@ -562,7 +562,7 @@ def detach_multi_process(&block) class TimeSlicedOutput < Fluent::Plugin::Output # TODO: warn when deprecated - helpers :event_emitter, :inject + helpers_internal :event_emitter, :inject def support_in_v12_style?(feature) case feature diff --git a/lib/fluent/config/configure_proxy.rb b/lib/fluent/config/configure_proxy.rb index 2522d88c15..24acb50c5b 100644 --- a/lib/fluent/config/configure_proxy.rb +++ b/lib/fluent/config/configure_proxy.rb @@ -368,16 +368,40 @@ def config_section(name, **kwargs, &block) name end - def dump(level = 0) - dumped_config = "" - indent = " " * level + def dump_config_definition + dumped_config = {} @params.each do |name, config| - dumped_config << "#{indent}#{name}: #{config[1][:type]}: <#{@defaults[name].inspect}>" - dumped_config << " # #{@descriptions[name]}" if @descriptions[name] - dumped_config << "\n" + dumped_config[name] = config[1] + dumped_config[name][:required] = !@defaults.key?(name) + dumped_config[name][:default] = @defaults[name] if @defaults.key?(name) + dumped_config[name][:desc] = @descriptions[name] if @descriptions.key?(name) + end + # Overwrite by config_set_default + @defaults.each do |name, value| + if @params.key?(name) + dumped_config[name][:default] = value + else + dumped_config[name] = { default: value } + end + end + # Overwrite by config_set_desc + @descriptions.each do |name, value| + if @params.key?(name) + dumped_config[name][:desc] = value + else + dumped_config[name] = { desc: value } + end end @sections.each do |section_name, sub_proxy| - dumped_config << "#{indent}#{section_name}\n#{sub_proxy.dump(level + 1)}" + if dumped_config.key?(section_name) + dumped_config[section_name].update(sub_proxy.dump_config_definition) + else + dumped_config[section_name] = sub_proxy.dump_config_definition + dumped_config[section_name][:required] = sub_proxy.required? + dumped_config[section_name][:multi] = sub_proxy.multi? + dumped_config[section_name][:alias] = sub_proxy.alias + dumped_config[section_name][:section] = true + end end dumped_config end @@ -392,4 +416,3 @@ def overwrite?(other, attribute_name) end end end - diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb index 7d5d4cf60b..0812f9a698 100644 --- a/lib/fluent/configurable.rb +++ b/lib/fluent/configurable.rb @@ -192,8 +192,8 @@ def merged_configure_proxy configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge) end - def dump(level = 0) - configure_proxy_map[self.to_s].dump(level) + def dump_config_definition + configure_proxy_map[self.to_s].dump_config_definition end end end diff --git a/lib/fluent/plugin/filter.rb b/lib/fluent/plugin/filter.rb index 1a509e2b62..6d496c9267 100644 --- a/lib/fluent/plugin/filter.rb +++ b/lib/fluent/plugin/filter.rb @@ -28,7 +28,7 @@ class Filter < Base include PluginLoggerMixin include PluginHelper::Mixin - helpers :event_emitter + helpers_internal :event_emitter attr_reader :has_filter_with_time diff --git a/lib/fluent/plugin/input.rb b/lib/fluent/plugin/input.rb index f9911298e6..5a2f7c1b7b 100644 --- a/lib/fluent/plugin/input.rb +++ b/lib/fluent/plugin/input.rb @@ -27,7 +27,7 @@ class Input < Base include PluginLoggerMixin include PluginHelper::Mixin - helpers :event_emitter + helpers_internal :event_emitter def multi_workers_ready? false diff --git a/lib/fluent/plugin/output.rb b/lib/fluent/plugin/output.rb index 02bf0e442a..7ed30db946 100644 --- a/lib/fluent/plugin/output.rb +++ b/lib/fluent/plugin/output.rb @@ -33,7 +33,7 @@ class Output < Base include PluginHelper::Mixin include UniqueId::Mixin - helpers :thread, :retry_state + helpers_internal :thread, :retry_state CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/ CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/ diff --git a/lib/fluent/plugin_helper.rb b/lib/fluent/plugin_helper.rb index 94870e0058..0428e3ef15 100644 --- a/lib/fluent/plugin_helper.rb +++ b/lib/fluent/plugin_helper.rb @@ -37,7 +37,15 @@ def self.included(mod) end end - def helpers(*snake_case_symbols) + def self.extended(mod) + def mod.inherited(subclass) + subclass.module_eval do + @_plugin_helpers_list = [] + end + end + end + + def helpers_internal(*snake_case_symbols) helper_modules = [] snake_case_symbols.each do |name| begin @@ -48,5 +56,15 @@ def helpers(*snake_case_symbols) end include(*helper_modules) end + + def helpers(*snake_case_symbols) + @_plugin_helpers_list ||= [] + @_plugin_helpers_list.concat(snake_case_symbols) + helpers_internal(*snake_case_symbols) + end + + def plugin_helpers + @_plugin_helpers_list || [] + end end end diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index c75d5952a7..cb40e98889 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -517,28 +517,9 @@ def dry_run end def show_plugin_config - $log.info "Show config for #{@show_plugin_config}" - @system_config = SystemConfig.new - init_engine - name, type = @show_plugin_config.split(":") - plugin = Plugin.__send__("new_#{name}", type) - dumped_config = "\n" - level = 0 - plugin.class.ancestors.reverse_each do |plugin_class| - if plugin_class.respond_to?(:dump) - $log.on_debug do - dumped_config << plugin_class.name - dumped_config << "\n" - level = 1 - end - dumped_config << plugin_class.dump(level) - end - end - $log.info dumped_config + name, type = @show_plugin_config.split(":") # input:tail + $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}" exit 0 - rescue => e - $log.error "show config failed: #{e}" - exit 1 end def supervise diff --git a/lib/fluent/test/helpers.rb b/lib/fluent/test/helpers.rb index 0d57294c30..8626d9a2ea 100644 --- a/lib/fluent/test/helpers.rb +++ b/lib/fluent/test/helpers.rb @@ -120,6 +120,15 @@ def capture_log(driver) ensure driver.instance.log.out = tmp end + + def capture_stdout + out = StringIO.new + $stdout = out + yield + out.string.force_encoding('utf-8') + ensure + $stdout = STDOUT + end end end end diff --git a/templates/new_gem/Gemfile b/templates/new_gem/Gemfile new file mode 100644 index 0000000000..b4e2a20bb6 --- /dev/null +++ b/templates/new_gem/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gemspec diff --git a/templates/new_gem/README.md.erb b/templates/new_gem/README.md.erb new file mode 100644 index 0000000000..75755df868 --- /dev/null +++ b/templates/new_gem/README.md.erb @@ -0,0 +1,43 @@ +# <%= gem_name %> + +[Fluentd](http://fluentd.org/) <%= type %> plugin to do something. + +TODO: write description for you plugin. + +## Installation + +### RubyGems + +``` +$ gem install <%= gem_name %> +``` + +### Bundler + +Add following line to your Gemfile: + +```ruby +gem "<%= gem_name %>" +``` + +And then execute: + +``` +$ bundle +``` + +## Configuration + +You can generate configuration template: + +``` +$ fluent-plugin-format-config <%= type %> <%= name %> +``` + +You can copy and paste generated documents here. + +## Copyright + +* Copyright(c) <%= Date.today.year %>- <%= user_name %> +* License + * <%= @license.full_name %> diff --git a/templates/new_gem/Rakefile b/templates/new_gem/Rakefile new file mode 100644 index 0000000000..9756c6eeb6 --- /dev/null +++ b/templates/new_gem/Rakefile @@ -0,0 +1,13 @@ +require "bundler" +Bundler::GemHelper.install_tasks + +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs.push("lib", "test") + t.test_files = FileList["test/**/test_*.rb"] + t.verbose = true + t.warning = true +end + +task default: [:test] diff --git a/templates/new_gem/fluent-plugin.gemspec.erb b/templates/new_gem/fluent-plugin.gemspec.erb new file mode 100644 index 0000000000..c2cb616b42 --- /dev/null +++ b/templates/new_gem/fluent-plugin.gemspec.erb @@ -0,0 +1,27 @@ +lib = File.expand_path("../lib", __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) + +Gem::Specification.new do |spec| + spec.name = "<%= gem_name %>" + spec.version = "0.1.0" + spec.authors = ["<%= user_name %>"] + spec.email = ["<%= user_email %>"] + + spec.summary = %q{TODO: Write a short summary, because Rubygems requires one.} + spec.description = %q{TODO: Write a longer description or delete this line.} + spec.homepage = "TODO: Put your gem's website or public repo URL here." + spec.license = "<%= @license.name %>" + + test_files, files = `git ls-files -z`.split("\x0").partition do |f| + f.match(%r{^(test|spec|features)/}) + end + spec.files = files + spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) } + spec.test_files = test_files + spec.require_paths = ["lib"] + + spec.add_development_dependency "bundler", "~> 1.14" + spec.add_development_dependency "rake", "~> 12.0" + spec.add_development_dependency "test-unit", "~> 3.0" + spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"] +end diff --git a/templates/new_gem/lib/fluent/plugin/filter.rb.erb b/templates/new_gem/lib/fluent/plugin/filter.rb.erb new file mode 100644 index 0000000000..5c560183a3 --- /dev/null +++ b/templates/new_gem/lib/fluent/plugin/filter.rb.erb @@ -0,0 +1,14 @@ +<%= preamble %> + +require "fluent/plugin/filter" + +module Fluent + module Plugin + class <%= class_name %> < Fluent::Plugin::Filter + Fluent::Plugin.register_filter("<%= name %>", self) + + def filter(tag, time, record) + end + end + end +end diff --git a/templates/new_gem/lib/fluent/plugin/formatter.rb.erb b/templates/new_gem/lib/fluent/plugin/formatter.rb.erb new file mode 100644 index 0000000000..37b759d30a --- /dev/null +++ b/templates/new_gem/lib/fluent/plugin/formatter.rb.erb @@ -0,0 +1,14 @@ +<%= preamble %> + +require "fluent/plugin/formatter" + +module Fluent + module Plugin + class <%= class_name %> < Fluent::Plugin::Formatter + Fluent::Plugin.register_formatter("<%= name %>", self) + + def format(tag, time, record) + end + end + end +end diff --git a/templates/new_gem/lib/fluent/plugin/input.rb.erb b/templates/new_gem/lib/fluent/plugin/input.rb.erb new file mode 100644 index 0000000000..495055c1a4 --- /dev/null +++ b/templates/new_gem/lib/fluent/plugin/input.rb.erb @@ -0,0 +1,11 @@ +<%= preamble %> + +require "fluent/plugin/input" + +module Fluent + module Plugin + class <%= class_name %> < Fluent::Plugin::Input + Fluent::Plugin.register_input("<%= name %>", self) + end + end +end diff --git a/templates/new_gem/lib/fluent/plugin/output.rb.erb b/templates/new_gem/lib/fluent/plugin/output.rb.erb new file mode 100644 index 0000000000..2d5373ed0d --- /dev/null +++ b/templates/new_gem/lib/fluent/plugin/output.rb.erb @@ -0,0 +1,11 @@ +<%= preamble %> + +require "fluent/plugin/output" + +module Fluent + module Plugin + class <%= class_name %> < Fluent::Plugin::Output + Fluent::Plugin.register_output("<%= name %>", self) + end + end +end diff --git a/templates/new_gem/lib/fluent/plugin/parser.rb.erb b/templates/new_gem/lib/fluent/plugin/parser.rb.erb new file mode 100644 index 0000000000..74b6e749d8 --- /dev/null +++ b/templates/new_gem/lib/fluent/plugin/parser.rb.erb @@ -0,0 +1,15 @@ +<%= preamble %> + +require "fluent/plugin/parser" + +module Fluent + module Plugin + class <%= class_name %> < Fluent::Plugin::Parser + Fluent::Plugin.register_parser("<%= name %>", self) + end + + def parse(text) + yield time, record + end + end +end diff --git a/templates/new_gem/test/helper.rb.erb b/templates/new_gem/test/helper.rb.erb new file mode 100644 index 0000000000..c4c6da6928 --- /dev/null +++ b/templates/new_gem/test/helper.rb.erb @@ -0,0 +1,8 @@ +$LOAD_PATH.unshift(File.expand_path("../../", __FILE__)) +require "test-unit" +require "fluent/test" +require "fluent/test/driver/<%= type %>" +require "fluent/test/helpers" + +Test::Unit::TestCase.include(Fluent::Test::Helpers) +Test::Unit::TestCase.extend(Fluent::Test::Helpers) diff --git a/templates/new_gem/test/plugin/test_filter.rb.erb b/templates/new_gem/test/plugin/test_filter.rb.erb new file mode 100644 index 0000000000..49728f2257 --- /dev/null +++ b/templates/new_gem/test/plugin/test_filter.rb.erb @@ -0,0 +1,18 @@ +require "helper" +require "fluent/plugin/<%= plugin_filename %>" + +class <%= class_name %>Test < Test::Unit::TestCase + setup do + Fluent::Test.setup + end + + test "failure" do + flunk + end + + private + + def create_driver(conf) + Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf) + end +end diff --git a/templates/new_gem/test/plugin/test_formatter.rb.erb b/templates/new_gem/test/plugin/test_formatter.rb.erb new file mode 100644 index 0000000000..a3f1fa38a4 --- /dev/null +++ b/templates/new_gem/test/plugin/test_formatter.rb.erb @@ -0,0 +1,18 @@ +require "helper" +require "fluent/plugin/<%= plugin_filename %>" + +class <%= class_name %>Test < Test::Unit::TestCase + setup do + Fluent::Test.setup + end + + test "failure" do + flunk + end + + private + + def create_driver(conf) + Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf) + end +end diff --git a/templates/new_gem/test/plugin/test_input.rb.erb b/templates/new_gem/test/plugin/test_input.rb.erb new file mode 100644 index 0000000000..fd0042997c --- /dev/null +++ b/templates/new_gem/test/plugin/test_input.rb.erb @@ -0,0 +1,18 @@ +require "helper" +require "fluent/plugin/<%= plugin_filename %>" + +class <%= class_name %>Test < Test::Unit::TestCase + setup do + Fluent::Test.setup + end + + test "failure" do + flunk + end + + private + + def create_driver(conf) + Fluent::Test::Driver::Input.new(Fluent::Plugin::<%= class_name %>).configure(conf) + end +end diff --git a/templates/new_gem/test/plugin/test_output.rb.erb b/templates/new_gem/test/plugin/test_output.rb.erb new file mode 100644 index 0000000000..234b4ce236 --- /dev/null +++ b/templates/new_gem/test/plugin/test_output.rb.erb @@ -0,0 +1,18 @@ +require "helper" +require "fluent/plugin/<%= plugin_filename %>" + +class <%= class_name %>Test < Test::Unit::TestCase + setup do + Fluent::Test.setup + end + + test "failure" do + flunk + end + + private + + def create_driver(conf) + Fluent::Test::Driver::Output.new(Fluent::Plugin::<%= class_name %>).configure(conf) + end +end diff --git a/templates/new_gem/test/plugin/test_parser.rb.erb b/templates/new_gem/test/plugin/test_parser.rb.erb new file mode 100644 index 0000000000..cb03c051ba --- /dev/null +++ b/templates/new_gem/test/plugin/test_parser.rb.erb @@ -0,0 +1,18 @@ +require "helper" +require "fluent/plugin/<%= plugin_filename %>" + +class <%= class_name %>Test < Test::Unit::TestCase + setup do + Fluent::Test.setup + end + + test "failure" do + flunk + end + + private + + def create_driver(conf) + Fluent::Test::Driver::Parser.new(Fluent::Plugin::<%= class_name %>).configure(conf) + end +end diff --git a/templates/plugin_config_formatter/param.md-compact.erb b/templates/plugin_config_formatter/param.md-compact.erb new file mode 100644 index 0000000000..3b52a6dc56 --- /dev/null +++ b/templates/plugin_config_formatter/param.md-compact.erb @@ -0,0 +1,25 @@ +<%- +type = config[:type] +required_label = config[:required] ? "required" : "optional" +default = config[:default] +alias_name = config[:alias] +deprecated = config[:deprecated] +obsoleted = config[:obsoleted] +description = config[:desc] +-%> +* **<%= name %>** (<%= type %>) (<%= required_label %>): <%= description %> +<%- if type == :enum -%> + * Available values: <%= config[:list].join(", ") %> +<%- end -%> +<%- if default -%> + * Default value: `<%= default %>`. +<%- end -%> +<%- if alias_name -%> + * Alias: <%= alias_name %> +<%- end -%> +<%- if deprecated -%> + * Deprecated: <%= deprecated %> +<%- end -%> +<%- if obsoleted -%> + * Obsoleted: <%= :obsoleted %> +<%- end -%> diff --git a/templates/plugin_config_formatter/param.md.erb b/templates/plugin_config_formatter/param.md.erb new file mode 100644 index 0000000000..70d9e6eebb --- /dev/null +++ b/templates/plugin_config_formatter/param.md.erb @@ -0,0 +1,34 @@ +<%- +type = config[:type] +required_label = config[:required] ? "required" : "optional" +default = config[:default] +alias_name = config[:alias] +deprecated = config[:deprecated] +obsoleted = config[:obsoleted] +description = config[:desc] +param_header = "#" * (3 + level) +-%> +<%= param_header %> <%= name %> (<%= type %>) (<%= required_label %>) + +<%= description %> +<%- if type == :enum -%> + +Available values: <%= config[:list].join(", ") %> +<%- end -%> +<%- if default -%> + +Default value: `<%= default %>`. +<%- end -%> +<%- if alias_name -%> + +Alias: <%= alias_name %> +<%- end -%> +<%- if deprecated -%> + +Deprecated: <%= deprecated %> +<%- end -%> +<%- if obsoleted -%> + +Obsoleted: <%= :obsoleted %> +<%- end -%> + diff --git a/templates/plugin_config_formatter/section.md.erb b/templates/plugin_config_formatter/section.md.erb new file mode 100644 index 0000000000..09f832ef91 --- /dev/null +++ b/templates/plugin_config_formatter/section.md.erb @@ -0,0 +1,12 @@ +<%- +section_header = "#" * (3 + level) +required_label = required ? "required" : "optional" +multiple_label = multi ? "multiple" : "single" +-%> +<%= section_header %> \<<%= section_name %>\> section (<%= required_label %>) (<%= multiple_label %>) +<%- if alias_name -%> + +Alias: <%= alias_name %> +<%- end -%> + +<%= dump_section_markdown(sub_section, level + 1) %> diff --git a/test/command/test_binlog_reader.rb b/test/command/test_binlog_reader.rb index 51ff0c7d28..3457f8df02 100644 --- a/test/command/test_binlog_reader.rb +++ b/test/command/test_binlog_reader.rb @@ -79,15 +79,6 @@ def timezone(timezone = 'UTC') ensure ENV['TZ'] = old end - - def capture_stdout - out = StringIO.new - $stdout = out - yield - out.string.force_encoding('utf-8') - ensure - $stdout = STDOUT - end end class TestHead < TestBaseCommand diff --git a/test/command/test_plugin_config_formatter.rb b/test/command/test_plugin_config_formatter.rb new file mode 100644 index 0000000000..eae3739be4 --- /dev/null +++ b/test/command/test_plugin_config_formatter.rb @@ -0,0 +1,275 @@ +require_relative '../helper' + +require 'pathname' +require 'fluent/command/plugin_config_formatter' +require 'fluent/plugin/input' +require 'fluent/plugin/output' +require 'fluent/plugin/filter' +require 'fluent/plugin/parser' +require 'fluent/plugin/formatter' + +class TestFluentPluginConfigFormatter < Test::Unit::TestCase + class FakeInput < ::Fluent::Plugin::Input + ::Fluent::Plugin.register_input("fake", self) + + desc "path to something" + config_param :path, :string + end + + class FakeOutput < ::Fluent::Plugin::Output + ::Fluent::Plugin.register_output("fake", self) + + desc "path to something" + config_param :path, :string + + def process(tag, es) + end + end + + class FakeFilter < ::Fluent::Plugin::Filter + ::Fluent::Plugin.register_filter("fake", self) + + desc "path to something" + config_param :path, :string + + def filter(tag, time, record) + end + end + + class FakeParser < ::Fluent::Plugin::Parser + ::Fluent::Plugin.register_parser("fake", self) + + desc "path to something" + config_param :path, :string + + def parse(text) + end + end + + class FakeFormatter < ::Fluent::Plugin::Formatter + ::Fluent::Plugin.register_formatter("fake", self) + + desc "path to something" + config_param :path, :string + + def format(tag, time, record) + end + end + + class SimpleInput < ::Fluent::Plugin::Input + ::Fluent::Plugin.register_input("simple", self) + helpers :inject, :compat_parameters + + desc "path to something" + config_param :path, :string + end + + class ComplexOutput < ::Fluent::Plugin::Output + ::Fluent::Plugin.register_output("complex", self) + helpers :inject, :compat_parameters + + config_section :authentication, required: true, multi: false do + desc "username" + config_param :username, :string + desc "password" + config_param :passowrd, :string, secret: true + end + + config_section :parent do + config_section :child do + desc "names" + config_param :names, :array + desc "difficulty" + config_param :difficulty, :enum, list: [:easy, :normal, :hard], default: :normal + end + end + + def process(tag, es) + end + end + + sub_test_case "json" do + data(input: [FakeInput, "input"], + output: [FakeOutput, "output"], + filter: [FakeFilter, "filter"], + parser: [FakeParser, "parser"], + formatter: [FakeFormatter, "formatter"]) + test "dumped config should be valid JSON" do |(klass, type)| + dumped_config = capture_stdout do + FluentPluginConfigFormatter.new(["--format=json", type, "fake"]).call + end + expected = { + path: { + desc: "path to something", + type: "string", + required: true + } + } + assert_equal(expected, JSON.parse(dumped_config, symbolize_names: true)[klass.name.to_sym]) + end + end + + sub_test_case "text" do + test "input simple" do + dumped_config = capture_stdout do + FluentPluginConfigFormatter.new(["--format=txt", "input", "simple"]).call + end + expected = <: optional, single + @type: string: ("memory") + timekey: time: (nil) + timekey_wait: time: (600) + timekey_use_utc: bool: (false) + timekey_zone: string: ("#{Time.now.strftime('%z')}") + flush_at_shutdown: bool: (nil) + flush_mode: enum: (:default) + flush_interval: time: (60) + flush_thread_count: integer: (1) + flush_thread_interval: float: (1.0) + flush_thread_burst_interval: float: (1.0) + delayed_commit_timeout: time: (60) + overflow_action: enum: (:throw_exception) + retry_forever: bool: (false) + retry_timeout: time: (259200) + retry_max_times: integer: (nil) + retry_secondary_threshold: float: (0.8) + retry_type: enum: (:exponential_backoff) + retry_wait: time: (1) + retry_exponential_backoff_base: float: (2) + retry_max_interval: time: (nil) + retry_randomize: bool: (true) + chunk_keys: : ([]) +: optional, single + @type: string: (nil) + : optional, single + : optional, single +: required, single + username: string: (nil) + passowrd: string: (nil) +: optional, multiple + : optional, multiple + names: array: (nil) + difficulty: enum: (:normal) +TEXT + assert_equal(expected, dumped_config) + end + end + + sub_test_case "markdown" do + test "input simple" do + dumped_config = capture_stdout do + FluentPluginConfigFormatter.new(["--format=markdown", "input", "simple"]).call + end + expected = < section (required) (single) + +#### username (string) (required) + +username + +#### passowrd (string) (required) + +password + + + +### \\ section (optional) (multiple) + + +#### \\ section (optional) (multiple) + +##### names (array) (required) + +names + +##### difficulty (enum) (optional) + +difficulty + +Available values: easy, normal, hard + +Default value: `normal`. + + + + +TEXT + assert_equal(expected, dumped_config) + end + end + + sub_test_case "arguments" do + data do + hash = {} + ["input", "output", "filter", "parser", "formatter"].each do |type| + ["txt", "json", "markdown"].each do |format| + argv = ["--format=#{format}"] + [ + ["--verbose", "--compact"], + ["--verbose"], + ["--compact"] + ].each do |options| + hash[(argv + options).join(" ")] = argv + options + [type, "fake"] + end + end + end + hash + end + test "dump txt" do |argv| + capture_stdout do + assert_nothing_raised do + FluentPluginConfigFormatter.new(argv).call + end + end + end + end +end diff --git a/test/command/test_plugin_generator.rb b/test/command/test_plugin_generator.rb new file mode 100644 index 0000000000..ccc10c0643 --- /dev/null +++ b/test/command/test_plugin_generator.rb @@ -0,0 +1,66 @@ +require_relative '../helper' + +require 'pathname' +require 'fluent/command/plugin_generator' + +class TestFluentPluginGenerator < Test::Unit::TestCase + TEMP_DIR = "tmp/plugin_generator" + setup do + FileUtils.mkdir_p(TEMP_DIR) + @pwd = Dir.pwd + Dir.chdir(TEMP_DIR) + end + + teardown do + Dir.chdir(@pwd) + FileUtils.rm_rf(TEMP_DIR) + end + + data(input: ["input", "in"], + output: ["output", "out"], + filter: ["filter", "filter"], + parser: ["parser", "parser"], + formatter: ["formatter", "formatter"]) + test "generate plugin" do |(type, part)| + capture_stdout do + FluentPluginGenerator.new([type, "fake"]).call + end + plugin_base_dir = Pathname("fluent-plugin-fake") + assert { plugin_base_dir.directory? } + expected = [ + "fluent-plugin-fake", + "fluent-plugin-fake/Gemfile", + "fluent-plugin-fake/LICENSE", + "fluent-plugin-fake/README.md", + "fluent-plugin-fake/Rakefile", + "fluent-plugin-fake/fluent-plugin-fake.gemspec", + "fluent-plugin-fake/lib", + "fluent-plugin-fake/lib/fluent", + "fluent-plugin-fake/lib/fluent/plugin", + "fluent-plugin-fake/lib/fluent/plugin/#{part}_fake.rb", + "fluent-plugin-fake/test", + "fluent-plugin-fake/test/helper.rb", + "fluent-plugin-fake/test/plugin", + "fluent-plugin-fake/test/plugin/test_#{part}_fake.rb", + ] + actual = plugin_base_dir.find.reject {|f| f.fnmatch("*/.git*") }.map(&:to_s).sort + assert_equal(expected, actual) + end + + test "no license" do + capture_stdout do + FluentPluginGenerator.new(["--no-license", "filter", "fake"]).call + end + assert { !Pathname("fluent-plugin-fake/LICENSE").exist? } + assert { Pathname("fluent-plugin-fake/Gemfile").exist? } + end + + test "unknown license" do + out = capture_stdout do + assert_raise(SystemExit) do + FluentPluginGenerator.new(["--license=unknown", "filter", "fake"]).call + end + end + assert { out.lines.include?("License: unknown\n") } + end +end diff --git a/test/config/test_configure_proxy.rb b/test/config/test_configure_proxy.rb index 24e865164c..0881b43763 100644 --- a/test/config/test_configure_proxy.rb +++ b/test/config/test_configure_proxy.rb @@ -400,45 +400,54 @@ class TestConfigureProxy < ::Test::Unit::TestCase end end - sub_test_case '#dump' do + sub_test_case '#dump_config_definition' do setup do @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) end test 'empty proxy' do - assert_equal("", @proxy.dump) + assert_equal({}, @proxy.dump_config_definition) end test 'plain proxy w/o default value' do @proxy.config_param(:name, :string) - assert_equal(< -CONFIG + expected = { + name: { type: :string, required: true } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ default value' do @proxy.config_param(:name, :string, default: "name1") - assert_equal(< -CONFIG + expected = { + name: { type: :string, default: "name1", required: false } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ default value using config_set_default' do @proxy.config_param(:name, :string) @proxy.config_set_default(:name, "name1") - assert_equal(< -CONFIG + expected = { + name: { type: :string, default: "name1", required: false } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'single sub proxy' do @proxy.config_section(:sub) do config_param(:name, :string, default: "name1") end - assert_equal(< -CONFIG + expected = { + sub: { + alias: nil, + multi: true, + required: false, + section: true, + name: { type: :string, default: "name1", required: false } + } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'nested sub proxy' do @@ -450,30 +459,43 @@ class TestConfigureProxy < ::Test::Unit::TestCase config_param(:name4, :string, default: "name4") end end - assert_equal(< - name2: string: <"name2"> - sub2 - name3: string: <"name3"> - name4: string: <"name4"> -CONFIG + expected = { + sub: { + alias: nil, + multi: true, + required: false, + section: true, + name1: { type: :string, default: "name1", required: false }, + name2: { type: :string, default: "name2", required: false }, + sub2: { + alias: nil, + multi: true, + required: false, + section: true, + name3: { type: :string, default: "name3", required: false }, + name4: { type: :string, default: "name4", required: false }, + } + } + } + assert_equal(expected, @proxy.dump_config_definition) end sub_test_case 'w/ description' do test 'single proxy' do @proxy.config_param(:name, :string, desc: "description for name") - assert_equal(< # description for name -CONFIG + expected = { + name: { type: :string, desc: "description for name", required: true } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'single proxy using config_set_desc' do @proxy.config_param(:name, :string) @proxy.config_set_desc(:name, "description for name") - assert_equal(< # description for name -CONFIG + expected = { + name: { type: :string, desc: "description for name", required: true } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'sub proxy' do @@ -485,14 +507,25 @@ class TestConfigureProxy < ::Test::Unit::TestCase config_param(:name4, :string, default: "name4", desc: "desc4") end end - assert_equal(< # desc1 - name2: string: <"name2"> # desc2 - sub2 - name3: string: <"name3"> - name4: string: <"name4"> # desc4 -CONFIG + expected = { + sub: { + alias: nil, + multi: true, + required: false, + section: true, + name1: { type: :string, default: "name1", desc: "desc1", required: false }, + name2: { type: :string, default: "name2", desc: "desc2", required: false }, + sub2: { + alias: nil, + multi: true, + required: false, + section: true, + name3: { type: :string, default: "name3", required: false }, + name4: { type: :string, default: "name4", desc: "desc4", required: false }, + } + } + } + assert_equal(expected, @proxy.dump_config_definition) end test 'sub proxy w/ desc method' do @@ -506,14 +539,25 @@ class TestConfigureProxy < ::Test::Unit::TestCase config_param(:name4, :string, default: "name4") end end - assert_equal(< # desc1 - name2: string: <"name2"> # desc2 - sub2 - name3: string: <"name3"> - name4: string: <"name4"> # desc4 -CONFIG + expected = { + sub: { + alias: nil, + multi: true, + required: false, + section: true, + name1: { type: :string, default: "name1", desc: "desc1", required: false }, + name2: { type: :string, default: "name2", desc: "desc2", required: false }, + sub2: { + alias: nil, + multi: true, + required: false, + section: true, + name3: { type: :string, default: "name3", required: false }, + name4: { type: :string, default: "name4", desc: "desc4", required: false }, + } + } + } + assert_equal(expected, @proxy.dump_config_definition) end end end