From 82d480c3e9919b78617ce0fe57e9f5c2add9359f Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Fri, 29 Mar 2024 08:50:27 +0000 Subject: [PATCH] Add ledger account iterator for the unconfirmed ledger set. --- nano/core_test/system.cpp | 6 +-- nano/nano_node/entry.cpp | 6 +-- nano/node/bootstrap/bootstrap_frontier.cpp | 4 +- nano/node/bootstrap/bootstrap_server.cpp | 2 +- nano/node/bootstrap_ascending/iterators.cpp | 5 +- nano/node/epoch_upgrader.cpp | 2 +- nano/node/json_handler.cpp | 10 ++-- nano/qt/qt.cpp | 2 +- nano/rpc_test/rpc.cpp | 12 ++--- nano/secure/CMakeLists.txt | 3 ++ nano/secure/account_iterator.cpp | 4 ++ nano/secure/account_iterator.hpp | 43 +++++++++++++++++ nano/secure/account_iterator_impl.hpp | 51 +++++++++++++++++++++ nano/secure/ledger_set_any.cpp | 25 ++++++++++ nano/secure/ledger_set_any.hpp | 12 +++++ nano/slow_test/node.cpp | 8 ++-- nano/test_common/system.cpp | 8 ++-- 17 files changed, 170 insertions(+), 33 deletions(-) create mode 100644 nano/secure/account_iterator.cpp create mode 100644 nano/secure/account_iterator.hpp create mode 100644 nano/secure/account_iterator_impl.hpp diff --git a/nano/core_test/system.cpp b/nano/core_test/system.cpp index 9dcf72cf41..8dc0264394 100644 --- a/nano/core_test/system.cpp +++ b/nano/core_test/system.cpp @@ -94,10 +94,10 @@ TEST (system, DISABLED_generate_send_new) system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); { auto transaction (node1.store.tx_begin_read ()); - auto iterator1 (node1.store.account.begin (transaction)); - ASSERT_NE (node1.store.account.end (), iterator1); + auto iterator1 (node1.ledger.any.account_begin (transaction)); + ASSERT_NE (node1.ledger.any.account_end (), iterator1); ++iterator1; - ASSERT_EQ (node1.store.account.end (), iterator1); + ASSERT_EQ (node1.ledger.any.account_end (), iterator1); } nano::keypair stake_preserver; auto send_block (system.wallet (0)->send_action (nano::dev::genesis_key.pub, stake_preserver.pub, nano::dev::constants.genesis_amount / 3 * 2, true)); diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 4faa3c9113..564e4686e5 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -430,7 +430,7 @@ int main (int argc, char * const * argv) // Cache the account heads to make searching quicker against unchecked keys. auto transaction (node->store.tx_begin_read ()); std::unordered_set frontier_hashes; - for (auto i (node->store.account.begin (transaction)), n (node->store.account.end ()); i != n; ++i) + for (auto i (node->ledger.any.account_begin (transaction)), n (node->ledger.any.account_end ()); i != n; ++i) { frontier_hashes.insert (i->second.head); } @@ -1629,7 +1629,7 @@ int main (int argc, char * const * argv) } size_t const accounts_deque_overflow (32 * 1024); auto transaction (node->store.tx_begin_read ()); - for (auto i (node->store.account.begin (transaction)), n (node->store.account.end ()); i != n; ++i) + for (auto i (node->ledger.any.account_begin (transaction)), n (node->ledger.any.account_end ()); i != n; ++i) { { nano::unique_lock lock{ mutex }; @@ -1797,7 +1797,7 @@ int main (int argc, char * const * argv) auto transaction (source_node->store.tx_begin_read ()); block_count = source_node->ledger.block_count (); std::cout << boost::str (boost::format ("Performing bootstrap emulation, %1% blocks in ledger...") % block_count) << std::endl; - for (auto i (source_node->store.account.begin (transaction)), n (source_node->store.account.end ()); i != n; ++i) + for (auto i (source_node->ledger.any.account_begin (transaction)), n (source_node->ledger.any.account_end ()); i != n; ++i) { nano::account const & account (i->first); nano::account_info const & info (i->second); diff --git a/nano/node/bootstrap/bootstrap_frontier.cpp b/nano/node/bootstrap/bootstrap_frontier.cpp index e4181761ff..f37d7b90ce 100644 --- a/nano/node/bootstrap/bootstrap_frontier.cpp +++ b/nano/node/bootstrap/bootstrap_frontier.cpp @@ -238,7 +238,7 @@ void nano::frontier_req_client::next () { std::size_t max_size (128); auto transaction (node->store.tx_begin_read ()); - for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end ()); i != n && accounts.size () != max_size; ++i) + for (auto i (node->ledger.any.account_upper_bound (transaction, current.number ())), n (node->ledger.any.account_end ()); i != n && accounts.size () != max_size; ++i) { nano::account_info const & info (i->second); nano::account const & account (i->first); @@ -380,7 +380,7 @@ void nano::frontier_req_server::next () auto transaction (node->store.tx_begin_read ()); if (!send_confirmed ()) { - for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end ()); i != n && accounts.size () != max_size; ++i) + for (auto i (node->ledger.any.account_upper_bound (transaction, current.number ())), n (node->ledger.any.account_end ()); i != n && accounts.size () != max_size; ++i) { nano::account_info const & info (i->second); if (disable_age_filter || (now - info.modified) <= request->age) diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index 93c99fa3b0..249203f6d2 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -331,7 +331,7 @@ nano::asc_pull_ack nano::bootstrap_server::process (const store::transaction & t nano::asc_pull_ack::frontiers_payload response_payload{}; - for (auto it = store.account.begin (transaction, request.start), end = store.account.end (); it != end && response_payload.frontiers.size () < request.count; ++it) + for (auto it = ledger.any.account_lower_bound (transaction, request.start), end = ledger.any.account_end (); it != end && response_payload.frontiers.size () < request.count; ++it) { response_payload.frontiers.emplace_back (it->first, it->second.head); } diff --git a/nano/node/bootstrap_ascending/iterators.cpp b/nano/node/bootstrap_ascending/iterators.cpp index 29dbeb5aa7..b8f4065d9a 100644 --- a/nano/node/bootstrap_ascending/iterators.cpp +++ b/nano/node/bootstrap_ascending/iterators.cpp @@ -28,9 +28,8 @@ void nano::bootstrap_ascending::database_iterator::next (store::transaction & tx { case table_type::account: { - auto i = current.number () + 1; - auto item = ledger.store.account.begin (tx, i); - if (item != ledger.store.account.end ()) + auto item = ledger.any.account_upper_bound (tx, current.number ()); + if (item != ledger.any.account_end ()) { current = item->first; } diff --git a/nano/node/epoch_upgrader.cpp b/nano/node/epoch_upgrader.cpp index a33d0e9abc..45fba60649 100644 --- a/nano/node/epoch_upgrader.cpp +++ b/nano/node/epoch_upgrader.cpp @@ -116,7 +116,7 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc { auto transaction (store.tx_begin_read ()); // Collect accounts to upgrade - for (auto i (store.account.begin (transaction)), n (store.account.end ()); i != n && accounts_list.size () < count_limit; ++i) + for (auto i (ledger.any.account_begin (transaction)), n (ledger.any.account_end ()); i != n && accounts_list.size () < count_limit; ++i) { nano::account const & account (i->first); nano::account_info const & info (i->second); diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 833f6ead02..357f425ecf 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -2256,7 +2256,7 @@ void nano::json_handler::delegators () { auto transaction (node.store.tx_begin_read ()); boost::property_tree::ptree delegators; - for (auto i (node.store.account.begin (transaction, start_account.number () + 1)), n (node.store.account.end ()); i != n && delegators.size () < count; ++i) + for (auto i (node.ledger.any.account_upper_bound (transaction, start_account)), n (node.ledger.any.account_end ()); i != n && delegators.size () < count; ++i) { nano::account_info const & info (i->second); if (info.representative == representative) @@ -2282,7 +2282,7 @@ void nano::json_handler::delegators_count () { uint64_t count (0); auto transaction (node.store.tx_begin_read ()); - for (auto i (node.store.account.begin (transaction)), n (node.store.account.end ()); i != n; ++i) + for (auto i (node.ledger.any.account_begin (transaction)), n (node.ledger.any.account_end ()); i != n; ++i) { nano::account_info const & info (i->second); if (info.representative == account) @@ -2393,7 +2393,7 @@ void nano::json_handler::frontiers () { boost::property_tree::ptree frontiers; auto transaction (node.store.tx_begin_read ()); - for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end ()); i != n && frontiers.size () < count; ++i) + for (auto i (node.ledger.any.account_lower_bound (transaction, start)), n (node.ledger.any.account_end ()); i != n && frontiers.size () < count; ++i) { frontiers.put (i->first.to_account (), i->second.head.to_string ()); } @@ -2794,7 +2794,7 @@ void nano::json_handler::ledger () auto transaction (node.store.tx_begin_read ()); if (!ec && !sorting) // Simple { - for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end ()); i != n && accounts.size () < count; ++i) + for (auto i (node.ledger.any.account_lower_bound (transaction, start)), n (node.ledger.any.account_end ()); i != n && accounts.size () < count; ++i) { nano::account_info const & info (i->second); if (info.modified >= modified_since && (receivable || info.balance.number () >= threshold.number ())) @@ -2835,7 +2835,7 @@ void nano::json_handler::ledger () else if (!ec) // Sorting { std::vector> ledger_l; - for (auto i (node.store.account.begin (transaction, start)), n (node.store.account.end ()); i != n; ++i) + for (auto i (node.ledger.any.account_lower_bound (transaction, start)), n (node.ledger.any.account_end ()); i != n; ++i) { nano::account_info const & info (i->second); nano::uint128_union balance (info.balance); diff --git a/nano/qt/qt.cpp b/nano/qt/qt.cpp index e2c860f80f..8ba0d156e8 100644 --- a/nano/qt/qt.cpp +++ b/nano/qt/qt.cpp @@ -1972,7 +1972,7 @@ void nano_qt::advanced_actions::refresh_ledger () { ledger_model->removeRows (0, ledger_model->rowCount ()); auto transaction (wallet.node.store.tx_begin_read ()); - for (auto i (wallet.node.ledger.store.account.begin (transaction)), j (wallet.node.ledger.store.account.end ()); i != j; ++i) + for (auto i (wallet.node.ledger.any.account_begin (transaction)), j (wallet.node.ledger.any.account_end ()); i != j; ++i) { QList items; items.push_back (new QStandardItem (QString (i->first.to_account ().c_str ()))); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 425dbc2529..c85869324a 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -6123,7 +6123,7 @@ TEST (rpc, epoch_upgrade) { auto transaction (node->store.tx_begin_read ()); ASSERT_EQ (2, node->store.account.count (transaction)); - for (auto i (node->store.account.begin (transaction)); i != node->store.account.end (); ++i) + for (auto i (node->ledger.any.account_begin (transaction)); i != node->ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_0); @@ -6141,7 +6141,7 @@ TEST (rpc, epoch_upgrade) { auto transaction (node->store.tx_begin_read ()); ASSERT_EQ (4, node->store.account.count (transaction)); - for (auto i (node->store.account.begin (transaction)); i != node->store.account.end (); ++i) + for (auto i (node->ledger.any.account_begin (transaction)); i != node->ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); @@ -6204,7 +6204,7 @@ TEST (rpc, epoch_upgrade) { auto transaction (node->store.tx_begin_read ()); ASSERT_EQ (5, node->store.account.count (transaction)); - for (auto i (node->store.account.begin (transaction)); i != node->store.account.end (); ++i) + for (auto i (node->ledger.any.account_begin (transaction)); i != node->ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_2); @@ -6286,7 +6286,7 @@ TEST (rpc, epoch_upgrade_multithreaded) { auto transaction (node->store.tx_begin_read ()); ASSERT_EQ (2, node->store.account.count (transaction)); - for (auto i (node->store.account.begin (transaction)); i != node->store.account.end (); ++i) + for (auto i (node->ledger.any.account_begin (transaction)); i != node->ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_0); @@ -6305,7 +6305,7 @@ TEST (rpc, epoch_upgrade_multithreaded) { auto transaction (node->store.tx_begin_read ()); ASSERT_EQ (4, node->store.account.count (transaction)); - for (auto i (node->store.account.begin (transaction)); i != node->store.account.end (); ++i) + for (auto i (node->ledger.any.account_begin (transaction)); i != node->ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); @@ -6368,7 +6368,7 @@ TEST (rpc, epoch_upgrade_multithreaded) { auto transaction (node->store.tx_begin_read ()); ASSERT_EQ (5, node->store.account.count (transaction)); - for (auto i (node->store.account.begin (transaction)); i != node->store.account.end (); ++i) + for (auto i (node->ledger.any.account_begin (transaction)); i != node->ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_2); diff --git a/nano/secure/CMakeLists.txt b/nano/secure/CMakeLists.txt index d6bdfd29d9..10dc452e23 100644 --- a/nano/secure/CMakeLists.txt +++ b/nano/secure/CMakeLists.txt @@ -41,6 +41,9 @@ add_library( ${CMAKE_BINARY_DIR}/bootstrap_weights_beta.cpp account_info.hpp account_info.cpp + account_iterator.cpp + account_iterator.hpp + account_iterator_impl.hpp common.hpp common.cpp generate_cache_flags.hpp diff --git a/nano/secure/account_iterator.cpp b/nano/secure/account_iterator.cpp new file mode 100644 index 0000000000..e61ad5783d --- /dev/null +++ b/nano/secure/account_iterator.cpp @@ -0,0 +1,4 @@ +#include +#include + +template class nano::account_iterator; diff --git a/nano/secure/account_iterator.hpp b/nano/secure/account_iterator.hpp new file mode 100644 index 0000000000..dc0d6bade7 --- /dev/null +++ b/nano/secure/account_iterator.hpp @@ -0,0 +1,43 @@ +#pragma once +#include +#include + +#include +#include + +namespace nano::store +{ +class transaction; +} + +namespace nano +{ +// This class iterates account entries +template +class account_iterator +{ +public: + // Creates an end () iterator + // 'transaction' and 'set' are nullptr so all end () iterators compare equal + account_iterator (); + account_iterator (store::transaction const & transaction, Set const & set, std::optional> const & item); + + // Compares if these iterators hold the same 'item'. + // Undefined behavior if this and other don't hold the same 'set' and 'transaction' + bool operator== (account_iterator const & other) const; + +public: // Dereferencing, undefined behavior when called on an end () iterator + // Advances the iterator to the next greater nano::account + // If there are no more accounts, convert this to an end () iterator + account_iterator & operator++ (); + std::pair const & operator* () const; + std::pair const * operator->() const; + +private: + store::transaction const * transaction; + Set const * set{ nullptr }; + // Current item at the position of the iterator + // std::nullopt if an end () iterator + std::optional> item; +}; +} diff --git a/nano/secure/account_iterator_impl.hpp b/nano/secure/account_iterator_impl.hpp new file mode 100644 index 0000000000..6645d51e0f --- /dev/null +++ b/nano/secure/account_iterator_impl.hpp @@ -0,0 +1,51 @@ +#include +#include + +template +nano::account_iterator::account_iterator () +{ +} + +template +nano::account_iterator::account_iterator (store::transaction const & transaction, Set const & set, std::optional> const & item) : + transaction{ &transaction }, + set{ &set }, + item{ item } +{ +} + +template +bool nano::account_iterator::operator== (account_iterator const & other) const +{ + debug_assert (set == nullptr || other.set == nullptr || set == other.set); + return item == other.item; +} + +// Iteration is performed by calling set->account_lower_bound (tx, next) where next is one higher than the current iterator +template +auto nano::account_iterator::operator++ () -> account_iterator & +{ + auto next = item.value ().first.number () + 1; + if (next != 0) + { + *this = set->account_lower_bound (*transaction, next); + } + else + { + // Convert to and end iterator if there are no more items + *this = account_iterator{}; + } + return *this; +} + +template +std::pair const & nano::account_iterator::operator* () const +{ + return item.value (); +} + +template +std::pair const * nano::account_iterator::operator->() const +{ + return &item.value (); +} diff --git a/nano/secure/ledger_set_any.cpp b/nano/secure/ledger_set_any.cpp index 782ca23dd5..2257d14ac9 100644 --- a/nano/secure/ledger_set_any.cpp +++ b/nano/secure/ledger_set_any.cpp @@ -20,6 +20,16 @@ std::optional nano::ledger_set_any::account_balance (store::tra return block->balance ().number (); } +auto nano::ledger_set_any::account_begin (store::transaction const & transaction) const -> account_iterator +{ + return account_lower_bound (transaction, 0); +} + +auto nano::ledger_set_any::account_end () const -> account_iterator +{ + return account_iterator{}; +} + std::optional nano::ledger_set_any::account_get (store::transaction const & transaction, nano::account const & account) const { return ledger.store.account.get (transaction, account); @@ -47,6 +57,21 @@ uint64_t nano::ledger_set_any::account_height (store::transaction const & transa return block->sideband ().height; } +auto nano::ledger_set_any::account_lower_bound (store::transaction const & transaction, nano::account const & account) const -> account_iterator +{ + auto disk = ledger.store.account.begin (transaction, account); + if (disk == ledger.store.account.end ()) + { + return account_iterator{}; + } + return account_iterator{ transaction, *this, *disk }; +} + +auto nano::ledger_set_any::account_upper_bound (store::transaction const & transaction, nano::account const & account) const -> account_iterator +{ + return account_lower_bound (transaction, account.number () + 1); +} + std::optional nano::ledger_set_any::block_account (store::transaction const & transaction, nano::block_hash const & hash) const { auto block_l = block_get (transaction, hash); diff --git a/nano/secure/ledger_set_any.hpp b/nano/secure/ledger_set_any.hpp index 1b4eecad4e..366c575111 100644 --- a/nano/secure/ledger_set_any.hpp +++ b/nano/secure/ledger_set_any.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -24,15 +25,25 @@ namespace nano class ledger_set_any { public: + using account_iterator = nano::account_iterator; using receivable_iterator = nano::receivable_iterator; ledger_set_any (nano::ledger const & ledger); public: // Operations on accounts std::optional account_balance (store::transaction const & transaction, nano::account const & account) const; + account_iterator account_begin (store::transaction const & transaction) const; + account_iterator account_end () const; std::optional account_get (store::transaction const & transaction, nano::account const & account) const; nano::block_hash account_head (store::transaction const & transaction, nano::account const & account) const; uint64_t account_height (store::transaction const & transaction, nano::account const & account) const; + // Returns the next account entry equal or greater than 'account' + // Mirrors std::map::lower_bound + account_iterator account_lower_bound (store::transaction const & transaction, nano::account const & account) const; + // Returns the next account entry greater than 'account' + // Returns account_lower_bound (transaction, account + 1) + // Mirrors std::map::upper_bound + account_iterator account_upper_bound (store::transaction const & transaction, nano::account const & account) const; public: // Operations on blocks std::optional block_account (store::transaction const & transaction, nano::block_hash const & hash) const; @@ -47,6 +58,7 @@ class ledger_set_any public: // Operations on pending entries std::optional pending_get (store::transaction const & transaction, nano::pending_key const & key) const; + bool receivable_any (store::transaction const & transaction, nano::account const & account) const; receivable_iterator receivable_end () const; bool receivable_exists (store::transaction const & transaction, nano::account const & account) const; // Returns the next receivable entry equal or greater than 'key' diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 98c3dbe8e6..3b15874b6d 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -53,7 +53,7 @@ TEST (system, generate_mass_activity) uint32_t count (20); system.generate_mass_activity (count, *system.nodes[0]); auto transaction (system.nodes[0]->store.tx_begin_read ()); - for (auto i (system.nodes[0]->store.account.begin (transaction)), n (system.nodes[0]->store.account.end ()); i != n; ++i) + for (auto i (system.nodes[0]->ledger.any.account_begin (transaction)), n (system.nodes[0]->ledger.any.account_end ()); i != n; ++i) { } } @@ -75,7 +75,7 @@ TEST (system, generate_mass_activity_long) } system.generate_mass_activity (count, *system.nodes[0]); auto transaction (system.nodes[0]->store.tx_begin_read ()); - for (auto i (system.nodes[0]->store.account.begin (transaction)), n (system.nodes[0]->store.account.end ()); i != n; ++i) + for (auto i (system.nodes[0]->ledger.any.account_begin (transaction)), n (system.nodes[0]->ledger.any.account_end ()); i != n; ++i) { } system.stop (); @@ -693,7 +693,7 @@ TEST (confirmation_height, many_accounts_single_confirmation) // All frontiers (except last) should have 2 blocks and both should be confirmed auto transaction = node->store.tx_begin_read (); - for (auto i (node->store.account.begin (transaction)), n (node->store.account.end ()); i != n; ++i) + for (auto i (node->ledger.any.account_begin (transaction)), n (node->ledger.any.account_end ()); i != n; ++i) { auto & account = i->first; auto & account_info = i->second; @@ -1754,7 +1754,7 @@ TEST (node, mass_epoch_upgrader) { auto transaction (node.store.tx_begin_read ()); size_t block_count_sum = 0; - for (auto i (node.store.account.begin (transaction)); i != node.store.account.end (); ++i) + for (auto i (node.ledger.any.account_begin (transaction)); i != node.ledger.any.account_end (); ++i) { nano::account_info info (i->second); ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); diff --git a/nano/test_common/system.cpp b/nano/test_common/system.cpp index 66fbea6a26..a347f7a498 100644 --- a/nano/test_common/system.cpp +++ b/nano/test_common/system.cpp @@ -538,12 +538,12 @@ void nano::test::system::generate_send_existing (nano::node & node_a, std::vecto nano::account account; random_pool::generate_block (account.bytes.data (), sizeof (account.bytes)); auto transaction (node_a.store.tx_begin_read ()); - store::iterator entry (node_a.store.account.begin (transaction, account)); - if (entry == node_a.store.account.end ()) + auto entry = node_a.ledger.any.account_lower_bound (transaction, account); + if (entry == node_a.ledger.any.account_end ()) { - entry = node_a.store.account.begin (transaction); + entry = node_a.ledger.any.account_lower_bound (transaction, 0); } - debug_assert (entry != node_a.store.account.end ()); + debug_assert (entry != node_a.ledger.any.account_end ()); destination = nano::account (entry->first); source = get_random_account (accounts_a); amount = get_random_amount (transaction, node_a, source);