From 93b359e3af899e1c311aa280ecbdbd29fc1ba54d Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 11 Dec 2024 11:43:46 +0100 Subject: [PATCH] datastore: kvdb schema and DHII entities --- cmd/capi/execute.cpp | 4 +- cmd/dev/check_changes.cpp | 2 +- cmd/dev/db_toolbox.cpp | 4 +- cmd/dev/staged_pipeline.cpp | 52 +++---- silkworm/capi/fork_validator.cpp | 10 +- silkworm/capi/instance.hpp | 5 + silkworm/capi/rpcdaemon.cpp | 5 +- silkworm/capi/silkworm.cpp | 12 +- silkworm/db/data_store.cpp | 29 ++++ silkworm/db/data_store.hpp | 48 ++++-- silkworm/db/datastore/data_store.hpp | 18 +-- silkworm/db/datastore/kvdb/database.cpp | 70 +++++++++ silkworm/db/datastore/kvdb/database.hpp | 104 +++++++++++++ silkworm/db/datastore/{ => kvdb}/domain.hpp | 12 +- silkworm/db/datastore/{ => kvdb}/history.hpp | 10 +- .../datastore/{ => kvdb}/inverted_index.hpp | 9 +- silkworm/db/datastore/kvdb/schema.cpp | 75 ++++++++++ silkworm/db/datastore/kvdb/schema.hpp | 141 ++++++++++++++++++ silkworm/db/datastore/schema.hpp | 2 + .../datastore/snapshots/snapshot_bundle.cpp | 10 +- .../snapshots/snapshot_repository.cpp | 6 - .../snapshots/snapshot_repository.hpp | 1 - silkworm/db/kv/api/local_transaction.hpp | 2 +- silkworm/db/snapshot_sync.cpp | 4 +- silkworm/db/state/schema_config.cpp | 19 +++ silkworm/db/state/schema_config.hpp | 42 ++++-- silkworm/db/test_util/temp_chain_data.hpp | 9 +- .../db/test_util/test_database_context.hpp | 9 +- silkworm/execution/local_state.hpp | 2 +- silkworm/node/node.cpp | 6 +- silkworm/node/stagedsync/stages_test.cpp | 2 +- silkworm/rpc/daemon.cpp | 4 +- .../sync/internals/header_retrieval_test.cpp | 2 +- .../messages/inbound_get_block_bodies.cpp | 2 +- .../messages/inbound_get_block_headers.cpp | 2 +- 35 files changed, 602 insertions(+), 132 deletions(-) create mode 100644 silkworm/db/datastore/kvdb/database.cpp create mode 100644 silkworm/db/datastore/kvdb/database.hpp rename silkworm/db/datastore/{ => kvdb}/domain.hpp (74%) rename silkworm/db/datastore/{ => kvdb}/history.hpp (76%) rename silkworm/db/datastore/{ => kvdb}/inverted_index.hpp (79%) create mode 100644 silkworm/db/datastore/kvdb/schema.cpp create mode 100644 silkworm/db/datastore/kvdb/schema.hpp diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp index 4de9fea9e3..ca6b83e4e4 100644 --- a/cmd/capi/execute.cpp +++ b/cmd/capi/execute.cpp @@ -315,9 +315,9 @@ int execute_blocks(SilkwormHandle handle, ExecuteBlocksSettings settings, const // Execute blocks if (settings.use_internal_txn) { - return execute_with_internal_txn(handle, settings, data_store.chaindata_rw()); + return execute_with_internal_txn(handle, settings, data_store.chaindata().access_rw()); } - return execute_with_external_txn(handle, settings, data_store.chaindata_rw().start_rw_tx()); + return execute_with_external_txn(handle, settings, data_store.chaindata().access_rw().start_rw_tx()); } int build_indexes(SilkwormHandle handle, const BuildIndexesSettings& settings, const DataDirectory& data_dir) { diff --git a/cmd/dev/check_changes.cpp b/cmd/dev/check_changes.cpp index 2330d64b83..7243ba489b 100644 --- a/cmd/dev/check_changes.cpp +++ b/cmd/dev/check_changes.cpp @@ -109,7 +109,7 @@ int main(int argc, char* argv[]) { data_dir.snapshots().path(), }; - auto txn = data_store.chaindata_rw().start_rw_tx(); + auto txn = data_store.chaindata().access_rw().start_rw_tx(); auto chain_config{db::read_chain_config(txn)}; if (!chain_config) { throw std::runtime_error("Unable to retrieve chain config"); diff --git a/cmd/dev/db_toolbox.cpp b/cmd/dev/db_toolbox.cpp index e7d9786854..0cca88635d 100644 --- a/cmd/dev/db_toolbox.cpp +++ b/cmd/dev/db_toolbox.cpp @@ -1460,10 +1460,10 @@ void do_freeze(EnvConfig& config, const DataDirectory& data_dir, bool keep_block config, data_dir.snapshots().path(), }; - StageSchedulerAdapter stage_scheduler{data_store.chaindata_rw()}; + StageSchedulerAdapter stage_scheduler{data_store.chaindata().access_rw()}; Freezer freezer{ - data_store.chaindata(), + data_store.chaindata().access_ro(), data_store.ref().blocks_repository, stage_scheduler, data_dir.temp().path(), diff --git a/cmd/dev/staged_pipeline.cpp b/cmd/dev/staged_pipeline.cpp index b38eb90028..a47944fcfb 100644 --- a/cmd/dev/staged_pipeline.cpp +++ b/cmd/dev/staged_pipeline.cpp @@ -281,18 +281,15 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t Environment::set_start_at_stage(start_at_stage); Environment::set_stop_before_stage(stop_before_stage); - auto env = silkworm::datastore::kvdb::open_env(config); + auto data_directory = std::make_unique(datadir_path); + db::DataStore data_store{config, data_directory->snapshots().path()}; - datastore::kvdb::ROTxnManaged ro_txn{env}; + datastore::kvdb::ROTxnManaged ro_txn = data_store.chaindata().access_ro().start_ro_tx(); const auto chain_config = db::read_chain_config(ro_txn); ensure(chain_config.has_value(), "Uninitialized Silkworm db or unknown/custom chain"); ro_txn.abort(); - auto data_directory = std::make_unique(datadir_path); - auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/false); - auto state_repository = db::state::make_state_repository(data_directory->snapshots().path(), /*open=*/false); - db::DataStoreRef data_store{datastore::kvdb::RWAccess{env}, blocks_repository, state_repository}; - db::DataModelFactory data_model_factory{data_store}; + db::DataModelFactory data_model_factory{data_store.ref()}; // We need full snapshot sync to take place to have database tables properly updated snapshots::SnapshotSettings snapshot_settings{ @@ -307,7 +304,7 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t db::SnapshotSync snapshot_sync{ std::move(snapshot_settings), chain_config->chain_id, - data_store, + data_store.ref(), std::filesystem::path{}, empty_scheduler}; @@ -324,7 +321,7 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t snap_sync_future.get(); // Commit is enabled by default in RWTxn(Managed), so we need to check here - RWTxnManaged txn{env}; + RWTxnManaged txn = data_store.chaindata().access_rw().start_rw_tx(); if (dry) { txn.disable_commit(); } else { @@ -373,6 +370,7 @@ void debug_unwind(datastore::kvdb::EnvConfig& config, BlockNum height, uint32_t // Unwind has just set progress for pre-Execution stages back to unwind_point even if it is within the snapshots // We need to reset progress for such stages to the max block in snapshots to avoid database update on next start + auto& blocks_repository = data_store.blocks_repository(); db::stages::write_stage_progress(txn, db::stages::kHeadersKey, blocks_repository.max_block_available()); db::stages::write_stage_progress(txn, db::stages::kBlockBodiesKey, blocks_repository.max_block_available()); db::stages::write_stage_progress(txn, db::stages::kBlockHashesKey, blocks_repository.max_block_available()); @@ -386,8 +384,10 @@ void unwind(datastore::kvdb::EnvConfig& config, BlockNum unwind_point, const boo config.readonly = false; - auto env{silkworm::datastore::kvdb::open_env(config)}; - RWTxnManaged txn{env}; + auto data_directory = std::make_unique(); + db::DataStore data_store{config, data_directory->snapshots().path()}; + + RWTxnManaged txn = data_store.chaindata().access_rw().start_rw_tx(); // Commit is enabled by default in RWTxn(Managed), so we need to check here if (dry) { @@ -402,11 +402,7 @@ void unwind(datastore::kvdb::EnvConfig& config, BlockNum unwind_point, const boo const auto chain_config = db::read_chain_config(txn); ensure(chain_config.has_value(), "Not an initialized Silkworm db or unknown/custom chain"); - auto data_directory = std::make_unique(); - auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/true); - auto state_repository = db::state::make_state_repository(data_directory->path(), /*open=*/true); - db::DataStoreRef data_store{datastore::kvdb::RWAccess{env}, blocks_repository, state_repository}; - db::DataModelFactory data_model_factory{data_store}; + db::DataModelFactory data_model_factory{data_store.ref()}; boost::asio::io_context io_context; @@ -491,8 +487,10 @@ void forward(datastore::kvdb::EnvConfig& config, BlockNum forward_point, const b Environment::set_start_at_stage(start_at_stage); Environment::set_stop_before_stage(stop_before_stage); - auto env = silkworm::datastore::kvdb::open_env(config); - RWTxnManaged txn{env}; + auto data_directory = std::make_unique(); + db::DataStore data_store{config, data_directory->snapshots().path()}; + + RWTxnManaged txn = data_store.chaindata().access_rw().start_rw_tx(); // Commit is enabled by default in RWTxn(Managed), so we need to check here if (dry) { @@ -510,11 +508,7 @@ void forward(datastore::kvdb::EnvConfig& config, BlockNum forward_point, const b const auto datadir_path = std::filesystem::path{config.path}.parent_path(); SILK_INFO << "Forward: datadir=" << datadir_path.string(); - auto data_directory = std::make_unique(); - auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/true); - auto state_repository = db::state::make_state_repository(data_directory->path(), /*open=*/true); - db::DataStoreRef data_store{datastore::kvdb::RWAccess{env}, blocks_repository, state_repository}; - db::DataModelFactory data_model_factory{data_store}; + db::DataModelFactory data_model_factory{data_store.ref()}; boost::asio::io_context io_context; @@ -561,8 +555,10 @@ void bisect_pipeline(datastore::kvdb::EnvConfig& config, BlockNum start, BlockNu Environment::set_start_at_stage(start_at_stage); Environment::set_stop_before_stage(stop_before_stage); - auto env = silkworm::datastore::kvdb::open_env(config); - RWTxnManaged txn{env}; + auto data_directory = std::make_unique(); + db::DataStore data_store{config, data_directory->snapshots().path()}; + + RWTxnManaged txn = data_store.chaindata().access_rw().start_rw_tx(); // Commit is enabled by default in RWTxn(Managed), so we need to check here if (dry) { @@ -580,11 +576,7 @@ void bisect_pipeline(datastore::kvdb::EnvConfig& config, BlockNum start, BlockNu const auto datadir_path = std::filesystem::path{config.path}.parent_path(); SILK_INFO << "Bisect: datadir=" << datadir_path.string(); - auto data_directory = std::make_unique(); - auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/true); - auto state_repository = db::state::make_state_repository(data_directory->path(), /*open=*/true); - db::DataStoreRef data_store{datastore::kvdb::RWAccess{env}, blocks_repository, state_repository}; - db::DataModelFactory data_model_factory{data_store}; + db::DataModelFactory data_model_factory{data_store.ref()}; boost::asio::io_context io_context; diff --git a/silkworm/capi/fork_validator.cpp b/silkworm/capi/fork_validator.cpp index bfed95e6ec..9c3201c04a 100644 --- a/silkworm/capi/fork_validator.cpp +++ b/silkworm/capi/fork_validator.cpp @@ -124,10 +124,12 @@ SILKWORM_EXPORT int silkworm_start_fork_validator(SilkwormHandle handle, MDBX_en SILK_INFO << "Starting fork validator"; set_node_settings(handle, *settings, mdbx_env); - silkworm::datastore::kvdb::EnvUnmanaged unmanaged_env{mdbx_env}; - silkworm::datastore::kvdb::RWAccess rw_access{unmanaged_env}; + handle->chaindata = std::make_unique( + silkworm::db::DataStore::make_chaindata_database(silkworm::datastore::kvdb::EnvUnmanaged{mdbx_env})); + auto& chaindata = *handle->chaindata; + silkworm::db::DataStoreRef data_store{ - rw_access, + chaindata.ref(), *handle->blocks_repository, *handle->state_repository, }; @@ -139,7 +141,7 @@ SILKWORM_EXPORT int silkworm_start_fork_validator(SilkwormHandle handle, MDBX_en data_model_factory, /* log_timer_factory = */ std::nullopt, make_stages_factory(handle->node_settings, data_model_factory), - rw_access); + chaindata.access_rw()); SILK_DEBUG << "Execution engine created"; diff --git a/silkworm/capi/instance.hpp b/silkworm/capi/instance.hpp index 1469731615..d693438fd5 100644 --- a/silkworm/capi/instance.hpp +++ b/silkworm/capi/instance.hpp @@ -28,11 +28,16 @@ #include #include +namespace silkworm::datastore::kvdb { +class DatabaseUnmanaged; +} // namespace silkworm::datastore::kvdb + struct SilkwormInstance { silkworm::log::Settings log_settings; silkworm::concurrency::ContextPoolSettings context_pool_settings; std::filesystem::path data_dir_path; silkworm::NodeSettings node_settings; + std::unique_ptr chaindata; std::unique_ptr blocks_repository; std::unique_ptr state_repository; std::unique_ptr rpcdaemon; diff --git a/silkworm/capi/rpcdaemon.cpp b/silkworm/capi/rpcdaemon.cpp index 0d43c6222f..5946d817e7 100644 --- a/silkworm/capi/rpcdaemon.cpp +++ b/silkworm/capi/rpcdaemon.cpp @@ -101,8 +101,11 @@ SILKWORM_EXPORT int silkworm_start_rpcdaemon(SilkwormHandle handle, MDBX_env* en } auto daemon_settings = make_daemon_settings(handle, *settings); + handle->chaindata = std::make_unique( + db::DataStore::make_chaindata_database(datastore::kvdb::EnvUnmanaged{env})); + db::DataStoreRef data_store{ - datastore::kvdb::RWAccess{datastore::kvdb::EnvUnmanaged{env}}, + handle->chaindata->ref(), *handle->blocks_repository, *handle->state_repository, }; diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 1cffbf57c7..1892169063 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -616,9 +616,13 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, try { // Wrap MDBX env into an internal *unmanaged* env, i.e. MDBX env is only used but its lifecycle is untouched datastore::kvdb::EnvUnmanaged unmanaged_env{mdbx_env}; - datastore::kvdb::RWAccess rw_access{unmanaged_env}; - auto txn = rw_access.start_rw_tx(); const auto env_path = unmanaged_env.get_path(); + handle->chaindata = std::make_unique( + db::DataStore::make_chaindata_database(std::move(unmanaged_env))); + auto& chaindata = *handle->chaindata; + + datastore::kvdb::RWAccess rw_access = chaindata.access_rw(); + auto txn = rw_access.start_rw_tx(); db::Buffer state_buffer{txn, std::make_unique(db::DataModel{txn, *handle->blocks_repository})}; state_buffer.set_memory_limit(batch_size); @@ -627,7 +631,7 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, [[maybe_unused]] auto _ = gsl::finally([&block_buffer] { block_buffer.terminate_and_release_all(); }); db::DataStoreRef data_store{ - rw_access, + chaindata.ref(), *handle->blocks_repository, *handle->state_repository, }; @@ -635,7 +639,7 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, BlockProvider block_provider{ &block_buffer, - datastore::kvdb::ROAccess{unmanaged_env}, + chaindata.access_ro(), std::move(data_model_factory), start_block, max_block, diff --git a/silkworm/db/data_store.cpp b/silkworm/db/data_store.cpp index e3d8180177..a6c106740b 100644 --- a/silkworm/db/data_store.cpp +++ b/silkworm/db/data_store.cpp @@ -19,15 +19,44 @@ namespace silkworm::db { datastore::Schema DataStore::make_schema() { + datastore::kvdb::Schema kvdb; + kvdb.default_database() = make_chaindata_database_schema(); + snapshots::Schema snapshots; snapshots.repository(blocks::kBlocksRepositoryName) = blocks::make_blocks_repository_schema(); snapshots.repository(state::kStateRepositoryName) = state::make_state_repository_schema(); return { + std::move(kvdb), std::move(snapshots), }; } +datastore::kvdb::Schema::DatabaseDef DataStore::make_chaindata_database_schema() { + return state::make_state_database_schema(); +} + +datastore::kvdb::Database DataStore::make_chaindata_database(mdbx::env_managed chaindata_env) { + return { + std::move(chaindata_env), + make_chaindata_database_schema(), + }; +} + +datastore::kvdb::DatabaseUnmanaged DataStore::make_chaindata_database(datastore::kvdb::EnvUnmanaged chaindata_env) { + return { + std::move(chaindata_env), + make_chaindata_database_schema(), + }; +} + +std::map> DataStore::make_databases_map( + datastore::kvdb::Database chaindata_database) { + std::map> databases; + databases.emplace(datastore::kvdb::Schema::kDefaultEntityName, std::make_unique(std::move(chaindata_database))); + return databases; +} + std::map> DataStore::make_repositories_map( snapshots::SnapshotRepository blocks_repository, snapshots::SnapshotRepository state_repository) { diff --git a/silkworm/db/data_store.hpp b/silkworm/db/data_store.hpp index eddffdd3e9..442649dde2 100644 --- a/silkworm/db/data_store.hpp +++ b/silkworm/db/data_store.hpp @@ -25,51 +25,69 @@ namespace silkworm::db { struct DataStoreRef { - datastore::kvdb::RWAccess chaindata; + datastore::kvdb::DatabaseRef chaindata; + state::StateDatabaseRef state_db() const { return {chaindata}; } snapshots::SnapshotRepository& blocks_repository; snapshots::SnapshotRepository& state_repository; }; class DataStore { - public: - explicit DataStore(datastore::DataStore store) : store_{std::move(store)} {} DataStore( - mdbx::env_managed chaindata_env, + datastore::kvdb::Database chaindata_database, snapshots::SnapshotRepository blocks_repository, snapshots::SnapshotRepository state_repository) : store_{ make_schema(), - std::move(chaindata_env), + make_databases_map(std::move(chaindata_database)), make_repositories_map(std::move(blocks_repository), std::move(state_repository)), } {} + public: + explicit DataStore(datastore::DataStore store) : store_{std::move(store)} {} + DataStore( - const datastore::kvdb::EnvConfig& chaindata_env_config, + mdbx::env_managed chaindata_env, const std::filesystem::path& repository_path) : DataStore{ - datastore::kvdb::open_env(chaindata_env_config), + make_chaindata_database(std::move(chaindata_env)), blocks::make_blocks_repository(repository_path), state::make_state_repository(repository_path), } {} - void close() { - store_.close(); - } + DataStore( + const datastore::kvdb::EnvConfig& chaindata_env_config, + const std::filesystem::path& repository_path) + : DataStore{ + datastore::kvdb::open_env(chaindata_env_config), + repository_path, + } {} DataStoreRef ref() const { return { - store_.chaindata_rw(), - store_.repository(blocks::kBlocksRepositoryName), - store_.repository(state::kStateRepositoryName), + chaindata().ref(), + blocks_repository(), + state_repository(), }; } - datastore::kvdb::ROAccess chaindata() const { return store_.chaindata(); } - datastore::kvdb::RWAccess chaindata_rw() const { return store_.chaindata_rw(); } + datastore::kvdb::Database& chaindata() const { return store_.default_database(); } + + snapshots::SnapshotRepository& blocks_repository() const { + return store_.repository(blocks::kBlocksRepositoryName); + } + snapshots::SnapshotRepository& state_repository() const { + return store_.repository(state::kStateRepositoryName); + } + + static datastore::kvdb::Schema::DatabaseDef make_chaindata_database_schema(); + static datastore::kvdb::Database make_chaindata_database(mdbx::env_managed chaindata_env); + static datastore::kvdb::DatabaseUnmanaged make_chaindata_database(datastore::kvdb::EnvUnmanaged chaindata_env); private: static datastore::Schema make_schema(); + static std::map> make_databases_map( + datastore::kvdb::Database chaindata_database); static std::map> make_repositories_map( snapshots::SnapshotRepository blocks_repository, snapshots::SnapshotRepository state_repository); diff --git a/silkworm/db/datastore/data_store.hpp b/silkworm/db/datastore/data_store.hpp index 74b56d3c61..2592e17b2d 100644 --- a/silkworm/db/datastore/data_store.hpp +++ b/silkworm/db/datastore/data_store.hpp @@ -20,7 +20,7 @@ #include #include "common/entity_name.hpp" -#include "kvdb/mdbx.hpp" +#include "kvdb/database.hpp" #include "schema.hpp" #include "snapshots/snapshot_repository.hpp" @@ -30,27 +30,21 @@ class DataStore { public: DataStore( Schema schema, - mdbx::env_managed chaindata_env, + std::map> databases, std::map> repositories) : schema_{std::move(schema)}, - chaindata_env_{std::move(chaindata_env)}, + databases_{std::move(databases)}, repositories_{std::move(repositories)} {} - void close() { - chaindata_env_.close(); - for (auto& entry : repositories_) - entry.second->close(); - } - const Schema& schema() const { return schema_; } - datastore::kvdb::ROAccess chaindata() const { return datastore::kvdb::ROAccess{chaindata_env_}; } - datastore::kvdb::RWAccess chaindata_rw() const { return datastore::kvdb::RWAccess{chaindata_env_}; } + kvdb::Database& default_database() const { return database(kvdb::Schema::kDefaultEntityName); } + kvdb::Database& database(const EntityName& name) const { return *databases_.at(name); } snapshots::SnapshotRepository& repository(const EntityName& name) const { return *repositories_.at(name); } private: Schema schema_; - mdbx::env_managed chaindata_env_; + std::map> databases_; std::map> repositories_; }; diff --git a/silkworm/db/datastore/kvdb/database.cpp b/silkworm/db/datastore/kvdb/database.cpp new file mode 100644 index 0000000000..19f1cd551b --- /dev/null +++ b/silkworm/db/datastore/kvdb/database.cpp @@ -0,0 +1,70 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "database.hpp" + +namespace silkworm::datastore::kvdb { + +static MapConfig make_table_config(const Schema::TableDef& table) { + return MapConfig{ + table.name().c_str(), + mdbx::key_mode::usual, + table.is_multi_value() ? mdbx::value_mode::multi : mdbx::value_mode::single, + }; +} + +static std::map make_table_configs( + const Schema::EntityDef& entity) { + std::map results; + for (auto& [name, def] : entity.tables()) { + results.emplace(name, make_table_config(def)); + } + return results; +} + +DatabaseRef::EntitiesMap make_entities( + const Schema::DatabaseDef& schema) { + DatabaseRef::EntitiesMap results; + for (auto& [name, def] : schema.entities()) { + results.emplace(name, make_table_configs(*def)); + } + return results; +} + +Domain DatabaseRef::domain(datastore::EntityName name) const { + auto& entity = entities_.at(name); + Domain domain{ + entity.at(Schema::kDomainValuesName), + std::nullopt, + }; + if (entity.contains(Schema::kHistoryValuesName)) { + domain.history.emplace(History{ + entity.at(Schema::kHistoryValuesName), + inverted_index(name), + }); + } + return domain; +} + +InvertedIndex DatabaseRef::inverted_index(datastore::EntityName name) const { + auto& entity = entities_.at(name); + return InvertedIndex{ + entity.at(Schema::kInvIdxKeysName), + entity.at(Schema::kInvIdxIndexName), + }; +} + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/kvdb/database.hpp b/silkworm/db/datastore/kvdb/database.hpp new file mode 100644 index 0000000000..dc67719e3f --- /dev/null +++ b/silkworm/db/datastore/kvdb/database.hpp @@ -0,0 +1,104 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +#include "domain.hpp" +#include "inverted_index.hpp" +#include "mdbx.hpp" +#include "schema.hpp" + +namespace silkworm::datastore::kvdb { + +class Database; +class DatabaseUnmanaged; + +class DatabaseRef { + public: + using EntitiesMap = std::map>; + + ROAccess access_ro() const { return ROAccess{env_}; } + RWAccess access_rw() const { return RWAccess{env_}; } + + Domain domain(datastore::EntityName name) const; + InvertedIndex inverted_index(datastore::EntityName name) const; + + private: + // this is private, use Database.ref() or DatabaseUnmanaged.ref() to create + DatabaseRef( + mdbx::env env, + const EntitiesMap& entities) + : env_{env}, + entities_{entities} {} + + friend class Database; + friend class DatabaseUnmanaged; + + mdbx::env env_; + const EntitiesMap& entities_; +}; + +DatabaseRef::EntitiesMap make_entities(const Schema::DatabaseDef& schema); + +class Database { + public: + Database( + mdbx::env_managed env, + Schema::DatabaseDef schema) + : env_{std::move(env)}, + schema_{std::move(schema)}, + entities_{make_entities(schema_)} {} + + ROAccess access_ro() const { return ref().access_ro(); } + RWAccess access_rw() const { return ref().access_rw(); } + + Domain domain(datastore::EntityName name) const { return ref().domain(name); } + InvertedIndex inverted_index(datastore::EntityName name) { return ref().inverted_index(name); } + + DatabaseRef ref() const { return {env_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + + private: + mdbx::env_managed env_; + Schema::DatabaseDef schema_; + std::map> entities_; +}; + +class DatabaseUnmanaged { + public: + DatabaseUnmanaged( + EnvUnmanaged env, + Schema::DatabaseDef schema) + : env_{std::move(env)}, + schema_{std::move(schema)}, + entities_{make_entities(schema_)} {} + + ROAccess access_ro() const { return ref().access_ro(); } + RWAccess access_rw() const { return ref().access_rw(); } + + Domain domain(datastore::EntityName name) const { return ref().domain(name); } + InvertedIndex inverted_index(datastore::EntityName name) { return ref().inverted_index(name); } + + DatabaseRef ref() const { return {env_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + + private: + EnvUnmanaged env_; + Schema::DatabaseDef schema_; + std::map> entities_; +}; + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/domain.hpp b/silkworm/db/datastore/kvdb/domain.hpp similarity index 74% rename from silkworm/db/datastore/domain.hpp rename to silkworm/db/datastore/kvdb/domain.hpp index b8b6216cbe..18ff378fbf 100644 --- a/silkworm/db/datastore/domain.hpp +++ b/silkworm/db/datastore/kvdb/domain.hpp @@ -16,12 +16,16 @@ #pragma once -#include "snapshots/domain.hpp" +#include -namespace silkworm::datastore { +#include "history.hpp" +#include "mdbx.hpp" + +namespace silkworm::datastore::kvdb { struct Domain { - snapshots::Domain snapshots; + const MapConfig& values_table; + std::optional history; }; -} // namespace silkworm::datastore +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/history.hpp b/silkworm/db/datastore/kvdb/history.hpp similarity index 76% rename from silkworm/db/datastore/history.hpp rename to silkworm/db/datastore/kvdb/history.hpp index a4e36fd4ec..7f684545e2 100644 --- a/silkworm/db/datastore/history.hpp +++ b/silkworm/db/datastore/kvdb/history.hpp @@ -16,12 +16,14 @@ #pragma once -#include "snapshots/history.hpp" +#include "inverted_index.hpp" +#include "mdbx.hpp" -namespace silkworm::datastore { +namespace silkworm::datastore::kvdb { struct History { - snapshots::History snapshots; + const MapConfig& values_table; + InvertedIndex inverted_index; }; -} // namespace silkworm::datastore +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/inverted_index.hpp b/silkworm/db/datastore/kvdb/inverted_index.hpp similarity index 79% rename from silkworm/db/datastore/inverted_index.hpp rename to silkworm/db/datastore/kvdb/inverted_index.hpp index 6a0adfa5f8..363c794f0e 100644 --- a/silkworm/db/datastore/inverted_index.hpp +++ b/silkworm/db/datastore/kvdb/inverted_index.hpp @@ -16,12 +16,13 @@ #pragma once -#include "snapshots/inverted_index.hpp" +#include "mdbx.hpp" -namespace silkworm::datastore { +namespace silkworm::datastore::kvdb { struct InvertedIndex { - snapshots::InvertedIndex snapshots; + const MapConfig& keys_table; + const MapConfig& index_table; }; -} // namespace silkworm::datastore +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/kvdb/schema.cpp b/silkworm/db/datastore/kvdb/schema.cpp new file mode 100644 index 0000000000..c30d841db3 --- /dev/null +++ b/silkworm/db/datastore/kvdb/schema.cpp @@ -0,0 +1,75 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "schema.hpp" + +namespace silkworm::datastore::kvdb { + +static std::string make_table_name(datastore::EntityName base_name, std::string_view suffix) { + return base_name.to_string() + std::string{suffix}; +} + +Schema::DomainDef Schema::DatabaseDef::make_domain_schema(datastore::EntityName name) { + Schema::DomainDef schema; + schema.table(kDomainValuesName) + .name(make_table_name(name, "Vals")) + .enable_multi_value(); + define_history_schema(name, schema); + return schema; +} + +Schema::EntityDef Schema::DatabaseDef::make_history_schema(datastore::EntityName name) { + Schema::EntityDef schema; + define_history_schema(name, schema); + return schema; +} + +void Schema::DatabaseDef::define_history_schema(datastore::EntityName name, EntityDef& schema) { + schema.table(kHistoryValuesName) + .name(make_table_name(name, "HistoryVals")) + .enable_multi_value(); + define_inverted_index_schema(name, schema); + // update the inverted index table name to have a "HistoryKeys" suffix + schema.table(kInvIdxKeysName) + .name(make_table_name(name, "HistoryKeys")); +} + +void Schema::DatabaseDef::undefine_history_schema(EntityDef& schema) { + schema.undefine(kHistoryValuesName); + undefine_inverted_index_schema(schema); +} + +Schema::EntityDef Schema::DatabaseDef::make_inverted_index_schema(datastore::EntityName name) { + Schema::EntityDef schema; + define_inverted_index_schema(name, schema); + return schema; +} + +void Schema::DatabaseDef::define_inverted_index_schema(datastore::EntityName name, EntityDef& schema) { + schema.table(kInvIdxKeysName) + .name(make_table_name(name, "Keys")) + .enable_multi_value(); + schema.table(kInvIdxIndexName) + .name(make_table_name(name, "Idx")) + .enable_multi_value(); +} + +void Schema::DatabaseDef::undefine_inverted_index_schema(EntityDef& schema) { + schema.undefine(kInvIdxKeysName); + schema.undefine(kInvIdxIndexName); +} + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/kvdb/schema.hpp b/silkworm/db/datastore/kvdb/schema.hpp new file mode 100644 index 0000000000..d1ef39e58b --- /dev/null +++ b/silkworm/db/datastore/kvdb/schema.hpp @@ -0,0 +1,141 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include + +#include "../common/entity_name.hpp" + +namespace silkworm::datastore::kvdb { + +class Schema { + public: + class TableDef { + public: + TableDef& name(std::string_view name) { + name_ = name; + return *this; + } + + TableDef& enable_multi_value() { + is_multi_value_ = true; + return *this; + } + + TableDef& disable_multi_value() { + is_multi_value_ = false; + return *this; + } + + const std::string& name() const { return name_.value(); } + bool is_multi_value() const { return is_multi_value_; } + + private: + std::optional name_; + bool is_multi_value_{false}; + }; + + class EntityDef { + public: + virtual ~EntityDef() = default; + + TableDef& table(datastore::EntityName name) { + return table_defs_[name]; + } + + EntityDef& undefine(datastore::EntityName name) { + table_defs_.erase(name); + return *this; + } + + const std::map& tables() const { return table_defs_; } + + private: + std::map table_defs_; + }; + + class DomainDef : public EntityDef { + public: + ~DomainDef() override = default; + + DomainDef& values_disable_multi_value() { + table(kDomainValuesName).disable_multi_value(); + table(kHistoryValuesName).disable_multi_value(); + return *this; + } + + DomainDef& without_history() { + DatabaseDef::undefine_history_schema(*this); + return *this; + } + }; + + class DatabaseDef { + public: + EntityDef& default_entity() { + entity_defs_.try_emplace(kDefaultEntityName, std::make_shared()); + return *entity_defs_.at(kDefaultEntityName); + } + + DomainDef& domain(datastore::EntityName name) { + entity_defs_.try_emplace(name, std::make_shared(make_domain_schema(name))); + return dynamic_cast(*entity_defs_.at(name)); + } + + EntityDef& inverted_index(datastore::EntityName name) { + entity_defs_.try_emplace(name, std::make_shared(make_inverted_index_schema(name))); + return *entity_defs_.at(name); + } + + const std::map>& entities() const { return entity_defs_; } + + private: + friend DomainDef; + static DomainDef make_domain_schema(datastore::EntityName name); + static EntityDef make_history_schema(datastore::EntityName name); + static void define_history_schema(datastore::EntityName name, EntityDef& schema); + static void undefine_history_schema(EntityDef& schema); + static EntityDef make_inverted_index_schema(datastore::EntityName name); + static void define_inverted_index_schema(datastore::EntityName name, EntityDef& schema); + static void undefine_inverted_index_schema(EntityDef& schema); + + std::map> entity_defs_; + }; + + DatabaseDef& database(datastore::EntityName name) { + return database_defs_[name]; + } + + DatabaseDef& default_database() { + return database(kDefaultEntityName); + } + + static constexpr datastore::EntityName kDefaultEntityName{"_"}; + + static constexpr datastore::EntityName kDomainValuesName{"DomainValues"}; + static constexpr datastore::EntityName kHistoryValuesName{"HistoryValues"}; + static constexpr datastore::EntityName kInvIdxKeysName{"InvIdxKeys"}; + static constexpr datastore::EntityName kInvIdxIndexName{"InvIdxIndex"}; + + private: + std::map database_defs_; +}; + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/schema.hpp b/silkworm/db/datastore/schema.hpp index 5cf37cf8bb..3e4d35e115 100644 --- a/silkworm/db/datastore/schema.hpp +++ b/silkworm/db/datastore/schema.hpp @@ -16,11 +16,13 @@ #pragma once +#include "kvdb/schema.hpp" #include "snapshots/schema.hpp" namespace silkworm::datastore { struct Schema { + kvdb::Schema kvdb; snapshots::Schema snapshots; }; diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index a3ba6b93ca..01521d00fd 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -117,7 +117,7 @@ SnapshotBundleData open_bundle_data( open_existence_indexes(entity_schema, dir_path, step_range), open_btree_indexes(entity_schema, dir_path, step_range), }); - }; + } return data; } @@ -153,15 +153,13 @@ Domain SnapshotBundle::domain(datastore::EntityName name) const { data.accessor_indexes.at(Schema::kDomainAccessorIndexName), data.existence_indexes.at(Schema::kDomainExistenceIndexName), data.btree_indexes.at(Schema::kDomainBTreeIndexName), + std::nullopt, }; if (data.segments.contains(Schema::kHistorySegmentName)) { domain.history.emplace(History{ data.segments.at(Schema::kHistorySegmentName), data.accessor_indexes.at(Schema::kHistoryAccessorIndexName), - InvertedIndex{ - data.kv_segments.at(Schema::kInvIdxKVSegmentName), - data.accessor_indexes.at(Schema::kInvIdxAccessorIndexName), - }, + inverted_index(name), }); } return domain; @@ -169,7 +167,7 @@ Domain SnapshotBundle::domain(datastore::EntityName name) const { InvertedIndex SnapshotBundle::inverted_index(datastore::EntityName name) const { auto& data = data_.entities.at(name); - return { + return InvertedIndex{ data.kv_segments.at(Schema::kInvIdxKVSegmentName), data.accessor_indexes.at(Schema::kInvIdxAccessorIndexName), }; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 0743c084c0..00db030b9f 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -67,12 +67,6 @@ size_t SnapshotRepository::bundles_count() const { return bundles_->size(); } -void SnapshotRepository::close() { - SILK_TRACE << "Close snapshot repository folder: " << dir_path_.string(); - std::scoped_lock lock(*bundles_mutex_); - bundles_ = std::make_shared(); -} - BlockNum SnapshotRepository::max_block_available() const { Step end_step = max_end_step(); if (end_step.value == 0) return 0; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index f56794b705..3ade1d7815 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -60,7 +60,6 @@ class SnapshotRepository { const Schema::RepositoryDef& schema() const { return schema_; }; void reopen_folder(); - void close(); void add_snapshot_bundle(SnapshotBundle bundle); diff --git a/silkworm/db/kv/api/local_transaction.hpp b/silkworm/db/kv/api/local_transaction.hpp index 0de9c8bb70..8b518a4184 100644 --- a/silkworm/db/kv/api/local_transaction.hpp +++ b/silkworm/db/kv/api/local_transaction.hpp @@ -40,7 +40,7 @@ class LocalTransaction : public BaseTransaction { StateCache* state_cache) : BaseTransaction(state_cache), data_store_{std::move(data_store)}, - txn_{data_store_.chaindata.start_ro_tx()} {} + txn_{data_store_.chaindata.access_ro().start_ro_tx()} {} ~LocalTransaction() override = default; diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp index 1ee392eed7..3943b54b63 100644 --- a/silkworm/db/snapshot_sync.cpp +++ b/silkworm/db/snapshot_sync.cpp @@ -71,7 +71,7 @@ SnapshotSync::SnapshotSync( data_store_{std::move(data_store)}, client_{settings_.bittorrent_settings}, snapshot_freezer_{ - data_store_.chaindata, + data_store_.chaindata.access_ro(), data_store_.blocks_repository, stage_scheduler, tmp_dir_path, @@ -138,7 +138,7 @@ Task SnapshotSync::setup() { blocks_repository().reopen_folder(); // Update chain and stage progresses in database according to available snapshots - datastore::kvdb::RWTxnManaged rw_txn = data_store_.chaindata.start_rw_tx(); + datastore::kvdb::RWTxnManaged rw_txn = data_store_.chaindata.access_rw().start_rw_tx(); update_database(rw_txn, blocks_repository().max_block_available(), [this] { return is_stopping_latch_.try_wait(); }); rw_txn.commit_and_stop(); diff --git a/silkworm/db/state/schema_config.cpp b/silkworm/db/state/schema_config.cpp index bf15944ad2..867405f4f2 100644 --- a/silkworm/db/state/schema_config.cpp +++ b/silkworm/db/state/schema_config.cpp @@ -44,6 +44,25 @@ snapshots::Schema::RepositoryDef make_state_repository_schema() { return schema; } +datastore::kvdb::Schema::DatabaseDef make_state_database_schema() { + datastore::kvdb::Schema::DatabaseDef schema; + + schema.domain(kDomainNameAccounts); + schema.domain(kDomainNameStorage); + schema.domain(kDomainNameCode) + .values_disable_multi_value(); + schema.domain(kDomainNameCommitment) + .without_history(); + schema.domain(kDomainNameReceipts); + + schema.inverted_index(kInvIdxNameLogAddress); + schema.inverted_index(kInvIdxNameLogTopics); + schema.inverted_index(kInvIdxNameTracesFrom); + schema.inverted_index(kInvIdxNameTracesTo); + + return schema; +} + std::unique_ptr make_state_index_builders_factory() { return std::make_unique(make_state_repository_schema()); } diff --git a/silkworm/db/state/schema_config.hpp b/silkworm/db/state/schema_config.hpp index ebde80f523..44aceea977 100644 --- a/silkworm/db/state/schema_config.hpp +++ b/silkworm/db/state/schema_config.hpp @@ -17,9 +17,13 @@ #pragma once #include "../datastore/common/entity_name.hpp" -#include "../datastore/domain.hpp" -#include "../datastore/inverted_index.hpp" +#include "../datastore/kvdb/database.hpp" +#include "../datastore/kvdb/domain.hpp" +#include "../datastore/kvdb/inverted_index.hpp" +#include "../datastore/kvdb/schema.hpp" +#include "../datastore/snapshots/domain.hpp" #include "../datastore/snapshots/index_builders_factory.hpp" +#include "../datastore/snapshots/inverted_index.hpp" #include "../datastore/snapshots/schema.hpp" #include "../datastore/snapshots/snapshot_repository.hpp" @@ -28,6 +32,7 @@ namespace silkworm::db::state { inline constexpr datastore::EntityName kStateRepositoryName{"State"}; snapshots::Schema::RepositoryDef make_state_repository_schema(); +datastore::kvdb::Schema::DatabaseDef make_state_database_schema(); std::unique_ptr make_state_index_builders_factory(); @@ -52,16 +57,31 @@ inline constexpr std::string_view kInvIdxLogAddressTag{"logaddrs"}; struct BundleDataRef { const snapshots::SnapshotBundle& bundle; - datastore::Domain accounts_domain() const { return {bundle.domain(kDomainNameAccounts)}; } - datastore::Domain storage_domain() const { return {bundle.domain(kDomainNameStorage)}; } - datastore::Domain code_domain() const { return {bundle.domain(kDomainNameCode)}; } - datastore::Domain commitment_domain() const { return {bundle.domain(kDomainNameCommitment)}; } - datastore::Domain receipts_domain() const { return {bundle.domain(kDomainNameReceipts)}; } + snapshots::Domain accounts_domain() const { return {bundle.domain(kDomainNameAccounts)}; } + snapshots::Domain storage_domain() const { return {bundle.domain(kDomainNameStorage)}; } + snapshots::Domain code_domain() const { return {bundle.domain(kDomainNameCode)}; } + snapshots::Domain commitment_domain() const { return {bundle.domain(kDomainNameCommitment)}; } + snapshots::Domain receipts_domain() const { return {bundle.domain(kDomainNameReceipts)}; } - datastore::InvertedIndex log_address_inverted_index() const { return {bundle.inverted_index(kInvIdxNameLogAddress)}; } - datastore::InvertedIndex log_topics_inverted_index() const { return {bundle.inverted_index(kInvIdxNameLogTopics)}; } - datastore::InvertedIndex traces_from_inverted_index() const { return {bundle.inverted_index(kInvIdxNameTracesFrom)}; } - datastore::InvertedIndex traces_to_inverted_index() const { return {bundle.inverted_index(kInvIdxNameTracesTo)}; } + snapshots::InvertedIndex log_address_inverted_index() const { return {bundle.inverted_index(kInvIdxNameLogAddress)}; } + snapshots::InvertedIndex log_topics_inverted_index() const { return {bundle.inverted_index(kInvIdxNameLogTopics)}; } + snapshots::InvertedIndex traces_from_inverted_index() const { return {bundle.inverted_index(kInvIdxNameTracesFrom)}; } + snapshots::InvertedIndex traces_to_inverted_index() const { return {bundle.inverted_index(kInvIdxNameTracesTo)}; } +}; + +struct StateDatabaseRef { + const datastore::kvdb::DatabaseRef& database; + + datastore::kvdb::Domain accounts_domain() const { return {database.domain(kDomainNameAccounts)}; } + datastore::kvdb::Domain storage_domain() const { return {database.domain(kDomainNameStorage)}; } + datastore::kvdb::Domain code_domain() const { return {database.domain(kDomainNameCode)}; } + datastore::kvdb::Domain commitment_domain() const { return {database.domain(kDomainNameCommitment)}; } + datastore::kvdb::Domain receipts_domain() const { return {database.domain(kDomainNameReceipts)}; } + + datastore::kvdb::InvertedIndex log_address_inverted_index() const { return {database.inverted_index(kInvIdxNameLogAddress)}; } + datastore::kvdb::InvertedIndex log_topics_inverted_index() const { return {database.inverted_index(kInvIdxNameLogTopics)}; } + datastore::kvdb::InvertedIndex traces_from_inverted_index() const { return {database.inverted_index(kInvIdxNameTracesFrom)}; } + datastore::kvdb::InvertedIndex traces_to_inverted_index() const { return {database.inverted_index(kInvIdxNameTracesTo)}; } }; } // namespace silkworm::db::state diff --git a/silkworm/db/test_util/temp_chain_data.hpp b/silkworm/db/test_util/temp_chain_data.hpp index 2923e8bc42..ac1dcf465a 100644 --- a/silkworm/db/test_util/temp_chain_data.hpp +++ b/silkworm/db/test_util/temp_chain_data.hpp @@ -88,10 +88,7 @@ class TempChainDataStore : public TempChainData { TempChainDataStore() : data_store_{ move_env(), - blocks::make_blocks_repository( - data_dir_.snapshots().path()), - state::make_state_repository( - data_dir_.snapshots().path()), + data_dir_.snapshots().path(), } {} ~TempChainDataStore() override { // need to destroy a started RWTxn in the base class before destroying env_managed inside the data_store_ @@ -102,10 +99,10 @@ class TempChainDataStore : public TempChainData { db::DataStore* operator->() { return &data_store_; } datastore::kvdb::ROAccess chaindata() const override { - return data_store_.chaindata(); + return data_store_.chaindata().access_ro(); } datastore::kvdb::RWAccess chaindata_rw() const override { - return data_store_.chaindata_rw(); + return data_store_.chaindata().access_rw(); } db::DataModelFactory data_model_factory() { diff --git a/silkworm/db/test_util/test_database_context.hpp b/silkworm/db/test_util/test_database_context.hpp index 8ae9257527..7ff763280a 100644 --- a/silkworm/db/test_util/test_database_context.hpp +++ b/silkworm/db/test_util/test_database_context.hpp @@ -66,10 +66,7 @@ class TestDataStore : public TestDatabaseContext { : TestDatabaseContext{tmp_dir}, data_store_{ move_env(), - blocks::make_blocks_repository( - DataDirectory{tmp_dir.path(), true}.snapshots().path()), - state::make_state_repository( - DataDirectory{tmp_dir.path(), false}.snapshots().path()), + DataDirectory{tmp_dir.path(), true}.snapshots().path(), } {} ~TestDataStore() override = default; @@ -77,10 +74,10 @@ class TestDataStore : public TestDatabaseContext { db::DataStore* operator->() { return &data_store_; } datastore::kvdb::ROAccess chaindata() const override { - return data_store_.chaindata(); + return data_store_.chaindata().access_ro(); } datastore::kvdb::RWAccess chaindata_rw() const override { - return data_store_.chaindata_rw(); + return data_store_.chaindata().access_rw(); } db::DataModelFactory data_model_factory() { diff --git a/silkworm/execution/local_state.hpp b/silkworm/execution/local_state.hpp index a17126ee02..2d84f6806f 100644 --- a/silkworm/execution/local_state.hpp +++ b/silkworm/execution/local_state.hpp @@ -36,7 +36,7 @@ class LocalState : public State { explicit LocalState(std::optional block_num, std::optional txn_id, db::DataStoreRef data_store) : block_num_{block_num}, txnid_{txn_id}, - txn_{data_store.chaindata.start_ro_tx()}, + txn_{data_store.chaindata.access_ro().start_ro_tx()}, data_model_{txn_, data_store.blocks_repository} {} std::optional read_account(const evmc::address& address) const noexcept override; diff --git a/silkworm/node/node.cpp b/silkworm/node/node.cpp index 54509b65a9..73ddcbd196 100644 --- a/silkworm/node/node.cpp +++ b/silkworm/node/node.cpp @@ -197,7 +197,7 @@ NodeImpl::NodeImpl( data_model_factory(), make_log_timer_factory(context_pool.any_executor(), settings_.node_settings.sync_loop_log_interval_seconds), make_stages_factory(settings_.node_settings, data_model_factory(), *this), - data_store_.chaindata_rw(), + data_store_.chaindata().access_rw(), }, execution_service_{std::make_shared(execution_engine_, execution_ioc_)}, execution_server_{make_execution_server_settings(settings_.node_settings.exec_api_address), execution_service_}, @@ -215,7 +215,7 @@ NodeImpl::NodeImpl( settings.node_settings.remote_sentry_addresses, context_pool.as_executor_pool(), context_pool, - make_sentry_eth_status_data_provider(data_store_.chaindata(), chain_config()))}, + make_sentry_eth_status_data_provider(data_store_.chaindata().access_ro(), chain_config()))}, chain_sync_{ context_pool.any_executor(), data_store(), @@ -226,7 +226,7 @@ NodeImpl::NodeImpl( make_sync_engine_rpc_settings(settings.rpcdaemon_settings, settings.log_settings.log_verbosity), }, resource_usage_log_{*settings_.node_settings.data_directory} { - backend_ = std::make_unique(settings_.node_settings, data_store_.chaindata(), std::get<0>(sentry_)); + backend_ = std::make_unique(settings_.node_settings, data_store_.chaindata().access_ro(), std::get<0>(sentry_)); backend_->set_node_name(settings_.node_settings.build_info.node_name); backend_kv_rpc_server_ = std::make_unique(settings_.server_settings, *backend_); bittorrent_client_ = std::make_unique(settings_.snapshot_settings.bittorrent_settings); diff --git a/silkworm/node/stagedsync/stages_test.cpp b/silkworm/node/stagedsync/stages_test.cpp index 1b58190b17..91a8d4fc2b 100644 --- a/silkworm/node/stagedsync/stages_test.cpp +++ b/silkworm/node/stagedsync/stages_test.cpp @@ -91,7 +91,7 @@ TEST_CASE("Sync Stages") { node_settings.data_directory->snapshots().path(), }; - auto txn = data_store.chaindata_rw().start_rw_tx(); + auto txn = data_store.chaindata().access_rw().start_rw_tx(); table::check_or_create_chaindata_tables(txn); txn.commit_and_renew(); const auto initial_tx_sequence{read_map_sequence(txn, table::kBlockTransactions.name)}; diff --git a/silkworm/rpc/daemon.cpp b/silkworm/rpc/daemon.cpp index 11af1d059f..ae70d0d106 100644 --- a/silkworm/rpc/daemon.cpp +++ b/silkworm/rpc/daemon.cpp @@ -138,7 +138,7 @@ int Daemon::run(const DaemonSettings& settings) { }); // At startup check that chain configuration is valid - datastore::kvdb::ROTxnManaged ro_txn = data_store->chaindata().start_ro_tx(); + datastore::kvdb::ROTxnManaged ro_txn = data_store->chaindata().access_ro().start_ro_tx(); db::DataModel data_access = db::DataModelFactory{data_store->ref()}(ro_txn); if (const auto chain_config{data_access.read_chain_config()}; !chain_config) { throw std::runtime_error{"invalid chain configuration"}; @@ -353,7 +353,7 @@ void Daemon::start() { // Put the interface logs into the data folder std::filesystem::path data_folder{}; if (data_store_) { - datastore::kvdb::RWAccess& chaindata = data_store_->chaindata; + datastore::kvdb::RWAccess chaindata = data_store_->chaindata.access_rw(); mdbx::env& chaindata_env = *chaindata; auto chaindata_path = chaindata_env.get_path(); // Trick to remove any empty filename because MDBX chaindata path ends with '/' diff --git a/silkworm/sync/internals/header_retrieval_test.cpp b/silkworm/sync/internals/header_retrieval_test.cpp index b3657228f1..e23a9fa166 100644 --- a/silkworm/sync/internals/header_retrieval_test.cpp +++ b/silkworm/sync/internals/header_retrieval_test.cpp @@ -28,7 +28,7 @@ TEST_CASE("HeaderRetrieval") { context.add_genesis_data(); context.commit_txn(); - datastore::kvdb::ROTxnManaged tx = context->chaindata().start_ro_tx(); + datastore::kvdb::ROTxnManaged tx = context->chaindata().access_ro().start_ro_tx(); db::DataModel data_model = context.data_model_factory()(tx); HeaderRetrieval header_retrieval{data_model}; diff --git a/silkworm/sync/messages/inbound_get_block_bodies.cpp b/silkworm/sync/messages/inbound_get_block_bodies.cpp index 5a3235c2c1..233b5a51ac 100644 --- a/silkworm/sync/messages/inbound_get_block_bodies.cpp +++ b/silkworm/sync/messages/inbound_get_block_bodies.cpp @@ -40,7 +40,7 @@ void InboundGetBlockBodies::execute(db::DataStoreRef db, HeaderChain&, BodySeque if (bs.max_block_in_output() == 0) return; - datastore::kvdb::ROTxnManaged tx = db.chaindata.start_ro_tx(); + datastore::kvdb::ROTxnManaged tx = db.chaindata.access_ro().start_ro_tx(); BodyRetrieval body_retrieval{tx}; BlockBodiesPacket66 reply; diff --git a/silkworm/sync/messages/inbound_get_block_headers.cpp b/silkworm/sync/messages/inbound_get_block_headers.cpp index eaedbe277e..b6b951d6d9 100644 --- a/silkworm/sync/messages/inbound_get_block_headers.cpp +++ b/silkworm/sync/messages/inbound_get_block_headers.cpp @@ -40,7 +40,7 @@ void InboundGetBlockHeaders::execute(db::DataStoreRef db, HeaderChain&, BodySequ if (bs.max_block_in_output() == 0) // skip requests in the first sync even if we already saved some headers return; - datastore::kvdb::ROTxnManaged tx = db.chaindata.start_ro_tx(); + datastore::kvdb::ROTxnManaged tx = db.chaindata.access_ro().start_ro_tx(); db::DataModel data_model{tx, db.blocks_repository}; HeaderRetrieval header_retrieval(data_model);