From 2f1f8955d53984e3d9718b0ec6e253f5695915bc Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Sat, 23 Mar 2024 22:14:54 +0100 Subject: [PATCH 1/7] Optional configuration file updater --- nano/lib/tomlconfig.cpp | 35 +++++++++++++++++++++++++++++++++++ nano/lib/tomlconfig.hpp | 1 + nano/node/cli.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/nano/lib/tomlconfig.cpp b/nano/lib/tomlconfig.cpp index 3cace0d655..d0e880fd26 100644 --- a/nano/lib/tomlconfig.cpp +++ b/nano/lib/tomlconfig.cpp @@ -186,6 +186,41 @@ void nano::tomlconfig::erase_default_values (tomlconfig & defaults_a) erase_defaults (defaults_l.get_tree (), self.get_tree (), get_tree ()); } +// Merges two TOML configurations and commenting values that are identical +std::string nano::tomlconfig::merge_defaults (nano::tomlconfig & current_config, nano::tomlconfig & default_config) +{ + auto default_tree = default_config.get_tree (); + auto current_tree = current_config.get_tree (); + + debug_assert (default_tree != nullptr && current_tree != nullptr); + + // Serialize both configs to commented strings + std::string defaults_str = default_config.to_string (true); + std::string current_str = current_config.to_string (true); + + // Read both configs line by line + std::istringstream stream_defaults (defaults_str); + std::istringstream stream_current (current_str); + std::string line_defaults, line_current, result; + + while (std::getline (stream_defaults, line_defaults) && std::getline (stream_current, line_current)) + { + if (line_defaults == line_current) + { + // Use default value + result += line_defaults + "\n"; + } + else + { + // Non default value. Removing the # to uncomment + size_t pos = line_current.find ('#'); + result += line_current.substr (0, pos) + line_current.substr (pos + 1) + "\n"; + } + } + + return result; +} + std::string nano::tomlconfig::to_string (bool comment_values) { std::stringstream ss, ss_processed; diff --git a/nano/lib/tomlconfig.hpp b/nano/lib/tomlconfig.hpp index 41ac0647b1..3670c7dc11 100644 --- a/nano/lib/tomlconfig.hpp +++ b/nano/lib/tomlconfig.hpp @@ -48,6 +48,7 @@ class tomlconfig : public nano::configbase std::shared_ptr create_array (std::string const & key, boost::optional documentation_a); void erase_default_values (tomlconfig & defaults_a); std::string to_string (bool comment_values); + std::string merge_defaults (nano::tomlconfig & current_config, nano::tomlconfig & default_config); /** Set value for the given key. Any existing value will be overwritten. */ template diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index f58722cfc2..fbf124c49a 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -62,6 +62,7 @@ void nano::add_node_options (boost::program_options::options_description & descr ("migrate_database_lmdb_to_rocksdb", "Migrates LMDB database to RocksDB") ("diagnostics", "Run internal diagnostics") ("generate_config", boost::program_options::value (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node, rpc or log. See also use_defaults.") + ("update_config", boost::program_options::value (), "Reads the current node configuration and updates it with missing keys and values and delete keys that are no longer used. Updated configuration is written to stdout.") ("key_create", "Generates a adhoc random keypair and prints it to stdout") ("key_expand", "Derive public key and account number from ") ("wallet_add_adhoc", "Insert in to ") @@ -711,6 +712,31 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map } } } + else if (vm.count ("update_config")) + { + nano::network_params network_params{ nano::network_constants::active_network }; + nano::tomlconfig default_toml; + nano::tomlconfig current_toml; + nano::daemon_config default_config{ data_path, network_params }; + nano::daemon_config current_config{ data_path, network_params }; + + std::vector config_overrides; + auto error = nano::read_node_config_toml (data_path, current_config, config_overrides); + if (error) + { + std::cerr << "Could not read existing config file\n"; + ec = nano::error_cli::invalid_arguments; + } + else + { + current_config.serialize_toml (current_toml); + default_config.serialize_toml (default_toml); + + auto output = current_toml.merge_defaults (current_toml, default_toml); + + std::cout << output; + } + } else if (vm.count ("diagnostics")) { auto inactive_node = nano::default_inactive_node (data_path, vm); From 39f45284d6f7a0ad4f2af485dac6ab1c233388bd Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:13:32 +0200 Subject: [PATCH 2/7] Removed unused parameters. Changed return code --- nano/lib/tomlconfig.cpp | 5 +---- nano/node/cli.cpp | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nano/lib/tomlconfig.cpp b/nano/lib/tomlconfig.cpp index d0e880fd26..dd2dfebdfd 100644 --- a/nano/lib/tomlconfig.cpp +++ b/nano/lib/tomlconfig.cpp @@ -189,10 +189,7 @@ void nano::tomlconfig::erase_default_values (tomlconfig & defaults_a) // Merges two TOML configurations and commenting values that are identical std::string nano::tomlconfig::merge_defaults (nano::tomlconfig & current_config, nano::tomlconfig & default_config) { - auto default_tree = default_config.get_tree (); - auto current_tree = current_config.get_tree (); - - debug_assert (default_tree != nullptr && current_tree != nullptr); + debug_assert (current_config != nullptr && default_config != nullptr); // Serialize both configs to commented strings std::string defaults_str = default_config.to_string (true); diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index fbf124c49a..41281a9552 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -62,7 +62,7 @@ void nano::add_node_options (boost::program_options::options_description & descr ("migrate_database_lmdb_to_rocksdb", "Migrates LMDB database to RocksDB") ("diagnostics", "Run internal diagnostics") ("generate_config", boost::program_options::value (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node, rpc or log. See also use_defaults.") - ("update_config", boost::program_options::value (), "Reads the current node configuration and updates it with missing keys and values and delete keys that are no longer used. Updated configuration is written to stdout.") + ("update_config", "Reads the current node configuration and updates it with missing keys and values and delete keys that are no longer used. Updated configuration is written to stdout.") ("key_create", "Generates a adhoc random keypair and prints it to stdout") ("key_expand", "Derive public key and account number from ") ("wallet_add_adhoc", "Insert in to ") @@ -725,7 +725,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map if (error) { std::cerr << "Could not read existing config file\n"; - ec = nano::error_cli::invalid_arguments; + ec = nano::error_cli::reading_config; } else { From ccefffc114c2928e4c1ab8a057e68bee3c6b0b28 Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Wed, 24 Apr 2024 20:20:21 +0200 Subject: [PATCH 3/7] Added unit test --- nano/core_test/toml.cpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 7de673fbd8..1c7648dbc2 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -1038,3 +1038,39 @@ TEST (toml, log_config_no_required) ASSERT_FALSE (toml.get_error ()) << toml.get_error ().get_message (); } + +TEST (toml, merge_config_files) +{ + nano::network_params network_params{ nano::network_constants::active_network }; + nano::tomlconfig default_toml; + nano::tomlconfig current_toml; + nano::tomlconfig merged_toml; + nano::daemon_config default_config{ ".", network_params }; + nano::daemon_config current_config{ ".", network_params }; + + std::stringstream ss; + + ss << R"toml( + [node] + active_elections_size = 999 + # backlog_scan_batch_size = 7777 + [node.bootstrap_ascending] + block_wait_count = 33333 + old_entry = 34 + )toml"; + + current_toml.read (ss); + current_config.deserialize_toml (current_toml); + + current_config.serialize_toml (current_toml); + default_config.serialize_toml (default_toml); + + auto merged_config = current_toml.merge_defaults (current_toml, default_toml); + + ASSERT_TRUE (merged_config.find ("active_elections_size = 999") != std::string::npos); + ASSERT_FALSE (merged_config.find ("active_elections_size = 5000") != std::string::npos); + ASSERT_TRUE (merged_config.find ("# backlog_scan_batch_size = 10000") != std::string::npos); + ASSERT_FALSE (merged_config.find ("backlog_scan_batch_size = 7777") != std::string::npos); + ASSERT_TRUE (merged_config.find ("block_wait_count = 33333") != std::string::npos); + ASSERT_FALSE (merged_config.find ("old_entry") != std::string::npos); +} \ No newline at end of file From 41edc272b7f1994daffbd7f2d7b484e769fab65f Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Wed, 24 Apr 2024 20:36:49 +0200 Subject: [PATCH 4/7] Removed faulty assert --- nano/lib/tomlconfig.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/nano/lib/tomlconfig.cpp b/nano/lib/tomlconfig.cpp index dd2dfebdfd..462d2368dd 100644 --- a/nano/lib/tomlconfig.cpp +++ b/nano/lib/tomlconfig.cpp @@ -189,8 +189,6 @@ void nano::tomlconfig::erase_default_values (tomlconfig & defaults_a) // Merges two TOML configurations and commenting values that are identical std::string nano::tomlconfig::merge_defaults (nano::tomlconfig & current_config, nano::tomlconfig & default_config) { - debug_assert (current_config != nullptr && default_config != nullptr); - // Serialize both configs to commented strings std::string defaults_str = default_config.to_string (true); std::string current_str = current_config.to_string (true); From 48ebf8de619b8949d53cbe59e6e658ba25ec1cb5 Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Wed, 24 Apr 2024 21:27:53 +0200 Subject: [PATCH 5/7] Improved unit test --- nano/core_test/toml.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 1c7648dbc2..2e041eb692 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -1047,6 +1047,7 @@ TEST (toml, merge_config_files) nano::tomlconfig merged_toml; nano::daemon_config default_config{ ".", network_params }; nano::daemon_config current_config{ ".", network_params }; + nano::daemon_config merged_config{ ".", network_params }; std::stringstream ss; @@ -1065,12 +1066,19 @@ TEST (toml, merge_config_files) current_config.serialize_toml (current_toml); default_config.serialize_toml (default_toml); - auto merged_config = current_toml.merge_defaults (current_toml, default_toml); + auto merged_config_string = current_toml.merge_defaults (current_toml, default_toml); - ASSERT_TRUE (merged_config.find ("active_elections_size = 999") != std::string::npos); - ASSERT_FALSE (merged_config.find ("active_elections_size = 5000") != std::string::npos); - ASSERT_TRUE (merged_config.find ("# backlog_scan_batch_size = 10000") != std::string::npos); - ASSERT_FALSE (merged_config.find ("backlog_scan_batch_size = 7777") != std::string::npos); - ASSERT_TRUE (merged_config.find ("block_wait_count = 33333") != std::string::npos); - ASSERT_FALSE (merged_config.find ("old_entry") != std::string::npos); + // Configs have been merged. Let's read and parse the new config file and verify the values + + std::stringstream ss2; + ss2 << merged_config_string; + + merged_toml.read (ss2); + merged_config.deserialize_toml (merged_toml); + + ASSERT_NE (merged_config.node.active_elections_size, default_config.node.active_elections_size); + ASSERT_EQ (merged_config.node.active_elections_size, 999); + ASSERT_NE (merged_config.node.backlog_scan_batch_size, 7777); + ASSERT_EQ (merged_config.node.active_elections_size, 999); + ASSERT_TRUE (merged_config_string.find ("old_entry") == std::string::npos); } \ No newline at end of file From aa9782921969abd15fbe18ae64cdeb2de0c78d70 Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Wed, 24 Apr 2024 21:33:40 +0200 Subject: [PATCH 6/7] Fixed testing of block_wait_count --- nano/core_test/toml.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 2e041eb692..5a371cfcd9 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -1079,6 +1079,6 @@ TEST (toml, merge_config_files) ASSERT_NE (merged_config.node.active_elections_size, default_config.node.active_elections_size); ASSERT_EQ (merged_config.node.active_elections_size, 999); ASSERT_NE (merged_config.node.backlog_scan_batch_size, 7777); - ASSERT_EQ (merged_config.node.active_elections_size, 999); + ASSERT_EQ (merged_config.node.bootstrap_ascending.block_wait_count, 33333); ASSERT_TRUE (merged_config_string.find ("old_entry") == std::string::npos); } \ No newline at end of file From 0b79fdf41908d2886b0eb553d43f2917b451fc3f Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Fri, 26 Apr 2024 20:29:19 +0200 Subject: [PATCH 7/7] Added asserts --- nano/lib/tomlconfig.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nano/lib/tomlconfig.cpp b/nano/lib/tomlconfig.cpp index 462d2368dd..ff998dbab9 100644 --- a/nano/lib/tomlconfig.cpp +++ b/nano/lib/tomlconfig.cpp @@ -209,6 +209,8 @@ std::string nano::tomlconfig::merge_defaults (nano::tomlconfig & current_config, { // Non default value. Removing the # to uncomment size_t pos = line_current.find ('#'); + debug_assert (pos != std::string::npos); + debug_assert (pos < line_current.length ()); result += line_current.substr (0, pos) + line_current.substr (pos + 1) + "\n"; } }