diff --git a/lib/orocos/configurations.rb b/lib/orocos/configurations.rb index ce8ed7d9..e71a9e1f 100644 --- a/lib/orocos/configurations.rb +++ b/lib/orocos/configurations.rb @@ -1,6 +1,7 @@ require 'stringio' require 'yaml' require 'utilrb/hash/map_key' +require 'digest' module Orocos # Class handling multiple possible configuration for a single task @@ -18,7 +19,7 @@ module Orocos # # The following options are possible: # - # name:: + # name:: # it is optional for the first section and mandatory for further # sections. It gives a name to the section, that can then be used # to refer to the configuration information in TaskConfigurations#apply @@ -34,7 +35,7 @@ module Orocos # always be merged with the ones listed. The name of the current # configuration section can be listed, in which case it will be merged in # the specified order. Otherwise, it is added at the end. - # + # class TaskConfigurations # Exception raised when the user asks for a non-existent configuration class SectionNotFound < ArgumentError @@ -61,7 +62,7 @@ def initialize(section_name) # The toplevel value (i.e. the value of e.g. sections['default']) is # always a hash whose keys are the task's property names. # - # @return [{String=>{String=>Object}}] + # @return [{String=>{String=>Object}}] attr_reader :sections # @return [OroGen::Spec::TaskContext] the task context model for which self holds @@ -88,7 +89,7 @@ def initialize_copy(source) @context = Array.new end - # Retrieves the configuration for the given section name + # Retrieves the configuration for the given section name # # @return [Object] see the description of {#sections} for the description # of formatting @@ -97,7 +98,7 @@ def [](section_name) end # @api private - # + # # Evaluate ruby content that has been embedded into the configuration file # inbetween <%= ... %> def evaluate_dynamic_content(filename, value) @@ -129,7 +130,7 @@ def evaluate_dynamic_content(filename, value) def self.load_raw_sections_from_file(file) document_lines = File.readlines(file) - headers = document_lines.enum_for(:each_with_index). + headers = document_lines.each_with_index. find_all { |line, _| line =~ /^---/ } if headers.empty? || headers.first[1] != 0 headers.unshift ["--- name:default", -1] @@ -153,7 +154,7 @@ def self.load_raw_sections_from_file(file) if !line_options.empty? ConfigurationManager.warn "unrecognized options #{line_options.keys.sort.join(", ")} in #{file}" end - + [section_options, line_number] end options[0][0][:name] ||= 'default' @@ -173,6 +174,31 @@ def self.load_raw_sections_from_file(file) options.map(&:first).zip(sections) end + # @api private + # + # Read the YAML from the cache directory, if available + def read_yaml_from_cache(cache_dir, doc) + cache_id = Digest::SHA256.hexdigest(doc) + path = File.join(cache_dir, cache_id) + return cache_id unless File.exist?(path) + + begin + [cache_id, Marshal.load(File.read(path))] + rescue Exception + cache_id + end + end + + # @api private + # + # Write the YAML to the cache directory, if available + def save_yaml_to_cache(cache_dir, cache_id, contents) + path = File.join(cache_dir, cache_id) + File.open(path, 'w') do |io| + io.write Marshal.dump(contents) + end + end + # Loads the configurations from a YAML file # # Multiple configurations can be saved in the file, in which case each @@ -184,7 +210,7 @@ def self.load_raw_sections_from_file(file) # also be provided if needed. # # @return [Array] the names of the sections that have been modified - def load_from_yaml(file) + def load_from_yaml(file, cache_dir: nil) sections = self.class.load_raw_sections_from_file(file) changed_sections = [] @@ -192,11 +218,23 @@ def load_from_yaml(file) doc = doc.join("") doc = evaluate_dynamic_content(file, doc) + if cache_dir + cache_id, cached_yaml = read_yaml_from_cache(cache_dir, doc) + end + unless cached_yaml + loaded_yaml = YAML.load(StringIO.new(doc)) || Hash.new + end + begin - result = normalize_conf(YAML.load(StringIO.new(doc)) || Hash.new) + result = normalize_conf(cached_yaml || loaded_yaml || Hash.new) rescue ConversionFailed => e raise e, "while loading section #{conf_options[:name] || 'default'} #{e.message}", e.backtrace end + + if cache_id && !cached_yaml + save_yaml_to_cache(cache_dir, cache_id, loaded_yaml) + end + name = conf_options.delete(:name) chain = conf(conf_options.delete(:chain), true) result = Orocos::TaskConfigurations.merge_conf(result, chain, true) @@ -328,7 +366,7 @@ def add(name, conf, normalize: true, merge: true) end # Remove a configuration section - # + # # @param [String] name the section name # @return [Boolean] true if such as section existed, and false otherwise def remove(name) @@ -479,7 +517,7 @@ def normalize_conf_terminal_value(value, value_t) # @api private # - # Helper for {.normalize_conf_value}. See it for details + # Helper for {.}. See it for details def normalize_conf_array(array, value_t) if value_t.respond_to?(:length) && value_t.length < array.size raise ConversionFailed.new, "array too big (got #{array.size} for a maximum of #{value_t.length}" @@ -492,7 +530,13 @@ def normalize_conf_array(array, value_t) begin packed_array = array.pack("#{element_t.pack_code}*") if value_t.respond_to?(:length) - return value_t.from_buffer(packed_array) + if value_t.length == array.size + return value_t.from_buffer(packed_array) + else + return array.map do |v| + normalize_conf_terminal_value(v, element_t) + end + end else return value_t.from_buffer([array.size].pack("Q") + packed_array) end @@ -635,16 +679,16 @@ def each_resolved_conf # speed: 1 # # Then - # + # # configuration(['default', 'fast']) # # returns { 'threshold' => 20, 'speed' => 10 } regardless of the value # of the override parameter, while - # + # # configuration(['default', 'fast', 'slow']) # - # will raise ArgumentError and - # + # will raise ArgumentError and + # # configuration(['default', 'fast', 'slow'], true) # # returns { 'threshold' => 20, 'speed' => 1 } @@ -736,7 +780,7 @@ def apply(task, config, override = false) raise ArgumentError, "no configuration #{names.join(", ")} for #{task.model.name}" end end - + timestamp = Time.now config.each do |prop_name, conf| p = task.property(prop_name) @@ -782,7 +826,8 @@ def self.apply_conf_array_on_typelib_value(value, conf) def self.apply_conf_on_typelib_value(value, conf) if conf.kind_of?(Hash) conf.each do |conf_key, conf_value| - value.raw_set(conf_key, apply_conf_on_typelib_value(value.raw_get(conf_key), conf_value)) + value.raw_set(conf_key, + apply_conf_on_typelib_value(value.raw_get(conf_key), conf_value)) end value elsif conf.respond_to?(:to_ary) @@ -914,7 +959,7 @@ def save(*args, task_model: self.model, replace: false) # directory, the generated file will be named based on the task's # model name # @param [String,nil] name the name of the new section. If nil is given, - # defaults to task.name + # defaults to task.name # @return [Hash] the task configuration in YAML representation, as # returned by {.config_as_hash} def self.save(config, file, name, task_model: nil, replace: false) @@ -1160,7 +1205,7 @@ def resolve_requested_configuration_names(model_name, task_conf, names) end end - # If no names are given try to figure them out + # If no names are given try to figure them out if !names || names.empty? if(task_conf.sections.size == 1) [task_conf.sections.keys.first] @@ -1230,4 +1275,3 @@ def resolve(task_model_name, conf_names = Array.new, override = false) end end end - diff --git a/test/test_configurations.rb b/test/test_configurations.rb index 7a81d290..3205710e 100644 --- a/test/test_configurations.rb +++ b/test/test_configurations.rb @@ -59,10 +59,11 @@ def verify_applied_conf(task, *base_path) def get_conf_value(*path) path.inject(@conf_context) do |result, field| - if !result[field] - raise ArgumentError, "no #{field} in #{result.inspect}" + if result.respond_to?(:raw_get) + result.raw_get(field) + else + result.fetch(field) end - result[field] end end @@ -148,6 +149,67 @@ def assert_conf_value(*path) end end + describe "the loaded yaml cache" do + before do + @root_dir = make_tmpdir + @cache_dir = FileUtils.mkdir File.join(@root_dir, 'cache') + @conf_file = File.join(@root_dir, "conf.yml") + write_fixture_conf <<~CONF + --- name:default + intg: 20 + CONF + end + def write_fixture_conf(content) + File.open(@conf_file, 'w') { |io| io.write(content) } + end + it "auto-saves a marshalled version in the provided cache directory" do + @conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + flexmock(YAML).should_receive(:load).never + conf = Orocos::TaskConfigurations.new(model) + conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + default = conf.conf('default') + assert_equal 20, Typelib.to_ruby(default['intg']) + end + it "ignores the cache if the document changed" do + # Generate the cache + @conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + write_fixture_conf <<~CONF + --- name:default + intg: 30 + CONF + + flexmock(YAML).should_receive(:load).at_least.once.pass_thru + conf = Orocos::TaskConfigurations.new(model) + conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + default = conf.conf('default') + assert_equal 30, Typelib.to_ruby(default['intg']) + end + it "does not use the cache if the dynamic content is different" do + write_fixture_conf <<~CONF + --- name:default + intg: <%= Time.now.tv_usec %> + CONF + @conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + flexmock(YAML).should_receive(:load).at_least.once.pass_thru + conf = Orocos::TaskConfigurations.new(model) + conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + end + it "properly deals with an invalid cache" do + write_fixture_conf <<~CONF + --- name:default + intg: 20 + CONF + @conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + Dir.glob(File.join(@cache_dir, "*")) do |file| + File.truncate(file, 0) if File.file?(file) + end + conf = Orocos::TaskConfigurations.new(model) + conf.load_from_yaml(@conf_file, cache_dir: @cache_dir) + default = conf.conf('default') + assert_equal 20, Typelib.to_ruby(default['intg']) + end + end + it "should be able to load complex structures" do conf.load_from_yaml(File.join(data_dir, 'configurations', 'complex_config.yml')) @@ -341,7 +403,7 @@ def assert_conf_value(*path) end conf.apply(task, ['default', 'compound', 'simple_container']) - simple_container[0, 3] = [10, 20, 30] + simple_container = [10, 20, 30] verify_applied_conf task do assert_conf_value 'simple_container', '/std/vector', Typelib::ContainerType, simple_container do |v| v.to_a @@ -687,13 +749,20 @@ def assert_conf_value(*path) @registry = Typelib::CXXRegistry.new end - it "maps arrays passing on the deference'd type" do + it "maps the elements of a ruby array if the sizes do not match" do type = registry.build('/int[5]') - result = conf.normalize_conf_value([1, 2, 3, 4, 5], type) - result.each do |v| - assert_kind_of type.deference, v + result = conf.normalize_conf_value([1, 2, 3, 4], type) + assert_kind_of Array, result + result.each do |e| + assert_kind_of type.deference, e end - assert_equal [1, 2, 3, 4, 5], Typelib.to_ruby(result) + assert_equal [1, 2, 3, 4], result.map { |e| Typelib.to_ruby(e) } + end + it "maps arrays to a typelib array if the sizes match" do + type = registry.build('/int[5]') + result = conf.normalize_conf_value([1, 2, 3, 4, 5], type) + assert_kind_of type, result + assert_equal [1, 2, 3, 4, 5], result.to_a end it "maps hashes passing on the field types" do type = registry.create_compound '/Test' do |c| @@ -708,7 +777,8 @@ def assert_conf_value(*path) end it "converts numerical values using evaluate_numeric_field" do type = registry.get '/int' - flexmock(conf).should_receive(:evaluate_numeric_field).with('42', type).and_return(42).once + flexmock(conf).should_receive(:evaluate_numeric_field). + with('42', type).and_return(42).once result = conf.normalize_conf_value('42', type) assert_kind_of type, result assert_equal 42, Typelib.to_ruby(result) @@ -719,38 +789,42 @@ def assert_conf_value(*path) assert_kind_of string_t, normalized assert_equal "bla", Typelib.to_ruby(normalized) end - it "converts typelib compound values to hashes" do + it "keeps typelib compound values that match the target value" do compound_t = registry.create_compound('/S') { |c| c.add 'a', '/int' } compound = compound_t.new(a: 10) normalized = conf.normalize_conf_value(compound, compound_t) - assert_kind_of Hash, normalized - assert_kind_of compound_t['a'], normalized['a'] - assert_equal 10, normalized['a'] + assert_equal compound, normalized + end + it "normalizes a compound's field" do + compound_t = registry.create_compound('/S') { |c| c.add 'a', '/int' } + compound = compound_t.new(a: 10) + normalized = conf.normalize_conf_value(Hash['a' => compound.a], compound_t) + assert_equal compound.a, normalized['a'] end - it "converts typelib container values to arrays" do + it "keeps typelib container values" do container_t = registry.create_container('/std/vector', '/int') container = container_t.new container << 0 normalized = conf.normalize_conf_value(container, container_t) - assert_kind_of Array, normalized - normalized.each { |v| assert_kind_of(container_t.deference, v) } - normalized.each_with_index { |v, i| assert_equal(container[i], v) } + assert_kind_of container_t, normalized + normalized.raw_each { |v| assert_kind_of(container_t.deference, v) } + normalized.raw_each.each_with_index { |v, i| assert_equal(container[i], v) } end - it "converts typelib array values to arrays" do + it "keeps typelib array values" do array_t = registry.create_array('/int', 3) array = array_t.new normalized = conf.normalize_conf_value(array, array_t) - assert_kind_of Array, normalized - normalized.each { |v| assert_kind_of(array_t.deference, v) } - normalized.each_with_index { |v, i| assert_equal(array[i], v) } + assert_kind_of array_t, normalized + normalized.raw_each { |v| assert_kind_of(array_t.deference, v) } + normalized.raw_each.each_with_index { |v, i| assert_equal(array[i], v) } end it "properly handles Ruby objects that are converted from a complex Typelib type" do klass = Class.new compound_t = registry.create_compound('/S') { |c| c.add 'a', '/int' } compound_t.convert_from_ruby(klass) { |v| compound_t.new(a: 10) } normalized = conf.normalize_conf_value(klass.new, compound_t) - assert_kind_of Hash, normalized - assert_kind_of compound_t['a'], normalized['a'] + assert_kind_of compound_t, normalized + assert_kind_of compound_t['a'], normalized.raw_get('a') assert_equal 10, normalized['a'] end @@ -843,7 +917,7 @@ def assert_conf_value(*path) assert_equal expected_conf, conf.conf('sec') end end - + describe "#save(name, file)" do attr_reader :section before do @@ -1042,4 +1116,3 @@ def test_override_arrays assert_equal({ 'gyrorrw' => default_conf['gyrorrw'], 'gyrorw' => xsens_conf['gyrorw'] }, result) end end -