From 93661a12a896e25b473c12b474bdb02ad6e1ad4f Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 31 Mar 2016 14:19:30 +0900 Subject: [PATCH 1/4] Add simplified syntax for hash and array * to specify values with very simple comma (and colon for key-value separator) * to re-format values because comma-separated values doesn't have type information It looks to satisfy various use-cases of 3rd party plugins, and this change is just additional change to allow format which is currently denied as ConfigError. --- lib/fluent/config/types.rb | 45 ++++++++++++++++++++++++++++++++++---- test/config/test_types.rb | 29 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb index b295ef5816..674d7fecc8 100644 --- a/lib/fluent/config/types.rb +++ b/lib/fluent/config/types.rb @@ -80,19 +80,56 @@ def self.bool_value(str) SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) } BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) } TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) } + + REFORMAT_VALUES = ->(type, value) { + if value.nil? + value + else + case type + when :string then value.to_s + when :integer then value.to_i + when :float then value.to_f + when :size then Config.size_value(value) + when :bool then Config.bool_value(value) + when :time then Config.time_value(value) + end + end + } + HASH_TYPE = Proc.new { |val, opts| - param = val.is_a?(String) ? JSON.load(val) : val + param = if val.is_a?(String) + val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}] + else + val + end if param.class != Hash raise ConfigError, "hash required but got #{val.inspect}" end - param + if opts.empty? + param + else + newparam = {} + param.each_pair do |key, value| + new_key = opts[:symbolize_keys] ? key.to_sym : key + newparam[new_key] = opts[:value_type] ? REFORMAT_VALUES.call(opts[:value_type], value) : value + end + newparam + end } ARRAY_TYPE = Proc.new { |val, opts| - param = val.is_a?(String) ? JSON.load(val) : val + param = if val.is_a?(String) + val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/) + else + val + end if param.class != Array raise ConfigError, "array required but got #{val.inspect}" end - param + if opts[:type] + param.map{|v| REFORMAT_VALUES.call(opts[:type], v) } + else + param + end } end diff --git a/test/config/test_types.rb b/test/config/test_types.rb index 95c544bc63..b73b3d86db 100644 --- a/test/config/test_types.rb +++ b/test/config/test_types.rb @@ -124,10 +124,39 @@ class TestConfigTypes < ::Test::Unit::TestCase test 'hash' do assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {})) + assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {})) + + assert_equal({x: "v", k: 1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true})) + assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('x:v,k:1', {symbolize_keys: true, value_type: :string})) + assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true, value_type: :string})) + assert_equal({x: 0, k: 1}, Config::HASH_TYPE.call('x:0,k:1', {symbolize_keys: true, value_type: :integer})) + + assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('{"x":"1s","y":"1m","z":"1h"}', {value_type: :time})) + assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('x:1s,y:1m,z:1h', {value_type: :time})) end test 'array' do assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {})) + assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {})) + + assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('["a","b","c"]', {})) + assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a,b,c', {})) + assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a, b, c', {})) + assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a , b , c', {})) + + assert_equal(["a a","b,b"," c "], Config::ARRAY_TYPE.call('["a a","b,b"," c "]', {})) + + assert_equal(["a a","b","c"], Config::ARRAY_TYPE.call('a a,b,c', {})) + + assert_equal([1,2,1], Config::ARRAY_TYPE.call('[1,2,1]', {})) + assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {type: :integer})) + assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {type: :integer})) + + array_options = { + default: [], + } + assert_equal(["1","2"], Config::ARRAY_TYPE.call('["1","2"]', array_options)) + assert_equal(["3"], Config::ARRAY_TYPE.call('["3"]', array_options)) end end end From 87e17d981f450d5cfa8b6235d518b223f6f63dfe Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 31 Mar 2016 18:22:33 +0900 Subject: [PATCH 2/4] Fix option name from type to value_type (type is already used by param's type itself) --- lib/fluent/config/types.rb | 6 ++++-- test/config/test_types.rb | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb index 674d7fecc8..35907413c1 100644 --- a/lib/fluent/config/types.rb +++ b/lib/fluent/config/types.rb @@ -92,6 +92,8 @@ def self.bool_value(str) when :size then Config.size_value(value) when :bool then Config.bool_value(value) when :time then Config.time_value(value) + else + raise "unknown type in REFORMAT: #{type}" end end } @@ -125,8 +127,8 @@ def self.bool_value(str) if param.class != Array raise ConfigError, "array required but got #{val.inspect}" end - if opts[:type] - param.map{|v| REFORMAT_VALUES.call(opts[:type], v) } + if opts[:value_type] + param.map{|v| REFORMAT_VALUES.call(opts[:value_type], v) } else param end diff --git a/test/config/test_types.rb b/test/config/test_types.rb index b73b3d86db..88e2b67890 100644 --- a/test/config/test_types.rb +++ b/test/config/test_types.rb @@ -149,8 +149,8 @@ class TestConfigTypes < ::Test::Unit::TestCase assert_equal(["a a","b","c"], Config::ARRAY_TYPE.call('a a,b,c', {})) assert_equal([1,2,1], Config::ARRAY_TYPE.call('[1,2,1]', {})) - assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {type: :integer})) - assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {type: :integer})) + assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {value_type: :integer})) + assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {value_type: :integer})) array_options = { default: [], From 1a2638af0685c09d79224c662d71c4d53959d5f6 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 31 Mar 2016 18:26:10 +0900 Subject: [PATCH 3/4] add test cases for LCSV --- test/config/test_types.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/config/test_types.rb b/test/config/test_types.rb index 88e2b67890..ffd96b2bc2 100644 --- a/test/config/test_types.rb +++ b/test/config/test_types.rb @@ -125,6 +125,11 @@ class TestConfigTypes < ::Test::Unit::TestCase test 'hash' do assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {})) assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {})) + assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v, k:1', {})) + assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call(' x:v, k:1 ', {})) + assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v , k:1 ', {})) + + assert_equal({"x"=>"v:v","k"=>"1"}, Config::HASH_TYPE.call('x:v:v, k:1', {})) assert_equal({x: "v", k: 1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true})) assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('x:v,k:1', {symbolize_keys: true, value_type: :string})) From 2188635082fbd7d2003ed310665698887f1ba104 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 4 Apr 2016 13:08:51 +0900 Subject: [PATCH 4/4] fix misleading name --- lib/fluent/config/types.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb index 35907413c1..85499c9910 100644 --- a/lib/fluent/config/types.rb +++ b/lib/fluent/config/types.rb @@ -81,7 +81,7 @@ def self.bool_value(str) BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) } TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) } - REFORMAT_VALUES = ->(type, value) { + REFORMAT_VALUE = ->(type, value) { if value.nil? value else @@ -113,7 +113,7 @@ def self.bool_value(str) newparam = {} param.each_pair do |key, value| new_key = opts[:symbolize_keys] ? key.to_sym : key - newparam[new_key] = opts[:value_type] ? REFORMAT_VALUES.call(opts[:value_type], value) : value + newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value) : value end newparam end @@ -128,7 +128,7 @@ def self.bool_value(str) raise ConfigError, "array required but got #{val.inspect}" end if opts[:value_type] - param.map{|v| REFORMAT_VALUES.call(opts[:value_type], v) } + param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v) } else param end