From b7888ac2652ed719b75839f605e542cf493d6950 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 16:08:09 +0000 Subject: [PATCH 01/15] Add iterators for receivable entries to nano::ledger. --- nano/core_test/ledger.cpp | 95 ++++++++++++++++++++++++++++++++++++ nano/secure/ledger.cpp | 30 ++++++++++++ nano/secure/ledger.hpp | 10 ++++ nano/secure/pending_info.cpp | 45 +++++++++++++++++ nano/secure/pending_info.hpp | 29 +++++++++++ 5 files changed, 209 insertions(+) diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 4c021abdd1..e8ef9b67a3 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -5559,3 +5559,98 @@ TEST (ledger, head_block) auto tx = store.tx_begin_read (); ASSERT_EQ (*nano::dev::genesis, *ledger.head_block (tx, nano::dev::genesis_key.pub)); } + +// Test that nullopt can be returned when there are no receivable entries +TEST (ledger_receivable, upper_bound_account_none) +{ + auto ctx = nano::test::context::ledger_empty (); + ASSERT_EQ (ctx.ledger ().receivable_end (), ctx.ledger ().receivable_upper_bound (ctx.store ().tx_begin_read (), 0)); +} + +// Test behavior of ledger::receivable_upper_bound when there are receivable entries for multiple accounts +TEST (ledger_receivable, upper_bound_account_key) +{ + auto ctx = nano::test::context::ledger_empty (); + nano::block_builder builder; + nano::keypair key; + auto send1 = builder + .state () + .account (nano::dev::genesis_key.pub) + .previous (nano::dev::genesis->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .link (key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*ctx.pool ().generate (nano::dev::genesis->hash ())) + .build (); + ASSERT_EQ (nano::block_status::progress, ctx.ledger ().process (ctx.store ().tx_begin_write (), send1)); + auto send2 = builder + .state () + .account (nano::dev::genesis_key.pub) + .previous (send1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio) + .link (nano::dev::genesis_key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*ctx.pool ().generate (send1->hash ())) + .build (); + ASSERT_EQ (nano::block_status::progress, ctx.ledger ().process (ctx.store ().tx_begin_write (), send2)); + auto tx = ctx.store ().tx_begin_read (); + auto & ledger = ctx.ledger (); + auto next1 = ledger.receivable_upper_bound (tx, nano::dev::genesis_key.pub); + auto next2 = ledger.receivable_upper_bound (tx, key.pub); + // Depending on which is greater but only one should have a value + ASSERT_TRUE (next1 == ledger.receivable_end () xor next2 == ledger.receivable_end ()); + // The account returned should be after the one we searched for + ASSERT_TRUE (next1 == ledger.receivable_end () || next1->first.account == key.pub); + ASSERT_TRUE (next2 == ledger.receivable_end () || next2->first.account == nano::dev::genesis_key.pub); + auto next3 = ledger.receivable_upper_bound (tx, nano::dev::genesis_key.pub, 0); + auto next4 = ledger.receivable_upper_bound (tx, key.pub, 0); + // Neither account has more than one receivable + ASSERT_TRUE (next3 != ledger.receivable_end () && next4 != ledger.receivable_end ()); + auto next5 = ledger.receivable_upper_bound (tx, next3->first.account, next3->first.hash); + auto next6 = ledger.receivable_upper_bound (tx, next4->first.account, next4->first.hash); + ASSERT_TRUE (next5 == ledger.receivable_end () && next6 == ledger.receivable_end ()); + ASSERT_EQ (ledger.receivable_end (), ++next3); + ASSERT_EQ (ledger.receivable_end (), ++next4); +} + +// Test that multiple receivable entries for the same account +TEST (ledger_receivable, key_two) +{ + auto ctx = nano::test::context::ledger_empty (); + nano::block_builder builder; + nano::keypair key; + auto send1 = builder + .state () + .account (nano::dev::genesis_key.pub) + .previous (nano::dev::genesis->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .link (key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*ctx.pool ().generate (nano::dev::genesis->hash ())) + .build (); + ASSERT_EQ (nano::block_status::progress, ctx.ledger ().process (ctx.store ().tx_begin_write (), send1)); + auto send2 = builder + .state () + .account (nano::dev::genesis_key.pub) + .previous (send1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio) + .link (key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*ctx.pool ().generate (send1->hash ())) + .build (); + ASSERT_EQ (nano::block_status::progress, ctx.ledger ().process (ctx.store ().tx_begin_write (), send2)); + auto tx = ctx.store ().tx_begin_read (); + auto & ledger = ctx.ledger (); + auto next1 = ledger.receivable_upper_bound (tx, key.pub, 0); + ASSERT_TRUE (next1 != ledger.receivable_end () && next1->first.account == key.pub); + auto next2 = ledger.receivable_upper_bound (tx, key.pub, next1->first.hash); + ASSERT_TRUE (next2 != ledger.receivable_end () && next2->first.account == key.pub); + ASSERT_NE (next1->first.hash, next2->first.hash); + ASSERT_EQ (next2, ++next1); + ASSERT_EQ (ledger.receivable_end (), ++next1); + ASSERT_EQ (ledger.receivable_end (), ++next2); +} diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 8b412717b8..cc256bdf9e 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -1545,6 +1545,36 @@ uint64_t nano::ledger::height (store::transaction const & transaction, nano::blo return block_l->sideband ().height; } +std::optional> nano::ledger::receivable_lower_bound (store::transaction const & tx, nano::account const & account, nano::block_hash const & hash) const +{ + auto result = store.pending.begin (tx, { account, hash }); + if (result == store.pending.end ()) + { + return std::nullopt; + } + return *result; +} + +nano::receivable_iterator nano::ledger::receivable_end () const +{ + return nano::receivable_iterator{}; +} + +nano::receivable_iterator nano::ledger::receivable_upper_bound (store::transaction const & tx, nano::account const & account) const +{ + return receivable_iterator{ *this, tx, receivable_lower_bound (tx, account.number () + 1, 0) }; +} + +nano::receivable_iterator nano::ledger::receivable_upper_bound (store::transaction const & tx, nano::account const & account, nano::block_hash const & hash) const +{ + auto result = receivable_lower_bound (tx, account, hash.number () + 1); + if (!result || result.value ().first.account != account) + { + return nano::receivable_iterator{ *this, tx, std::nullopt }; + } + return nano::receivable_iterator{ *this, tx, result }; +} + nano::uncemented_info::uncemented_info (nano::block_hash const & cemented_frontier, nano::block_hash const & frontier, nano::account const & account) : cemented_frontier (cemented_frontier), frontier (frontier), account (account) { diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index f5009b6648..8ceaedf364 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -36,6 +37,8 @@ class uncemented_info class ledger final { + friend class receivable_iterator; + public: ledger (nano::store::component &, nano::stats &, nano::ledger_constants & constants, nano::generate_cache_flags const & = nano::generate_cache_flags{}); /** @@ -84,6 +87,11 @@ class ledger final static nano::epoch version (nano::block const & block); nano::epoch version (store::transaction const & transaction, nano::block_hash const & hash) const; uint64_t height (store::transaction const & transaction, nano::block_hash const & hash) const; + nano::receivable_iterator receivable_end () const; + // Returns the next receivable entry for an account greater than 'account' + nano::receivable_iterator receivable_upper_bound (store::transaction const & tx, nano::account const & account) const; + // Returns the next receivable entry for the account 'account' with hash greater than 'hash' + nano::receivable_iterator receivable_upper_bound (store::transaction const & tx, nano::account const & account, nano::block_hash const & hash) const; static nano::uint128_t const unit; nano::ledger_constants & constants; nano::store::component & store; @@ -95,6 +103,8 @@ class ledger final bool pruning{ false }; private: + // Returns the next receivable entry equal or greater than 'key' + std::optional> receivable_lower_bound (store::transaction const & tx, nano::account const & account, nano::block_hash const & hash) const; void initialize (nano::generate_cache_flags const &); }; diff --git a/nano/secure/pending_info.cpp b/nano/secure/pending_info.cpp index 637f4adf74..61241f2162 100644 --- a/nano/secure/pending_info.cpp +++ b/nano/secure/pending_info.cpp @@ -1,3 +1,4 @@ +#include #include nano::pending_info::pending_info (nano::account const & source_a, nano::amount const & amount_a, nano::epoch epoch_a) : @@ -70,3 +71,47 @@ bool nano::pending_key::operator< (nano::pending_key const & other_a) const { return account == other_a.account ? hash < other_a.hash : account < other_a.account; } + +nano::receivable_iterator::receivable_iterator (nano::ledger const & ledger, nano::store::transaction const & tx, std::optional> item) : + ledger{ &ledger }, + tx{ &tx }, + item{ item } +{ + if (item.has_value ()) + { + account = item.value ().first.account; + } +} + +bool nano::receivable_iterator::operator== (receivable_iterator const & other) const +{ + debug_assert (ledger == nullptr || other.ledger == nullptr || ledger == other.ledger); + debug_assert (tx == nullptr || other.tx == nullptr || tx == other.tx); + debug_assert (account.is_zero () || other.account.is_zero () || account == other.account); + return item == other.item; +} + +bool nano::receivable_iterator::operator!= (receivable_iterator const & other) const +{ + return !(*this == other); +} + +nano::receivable_iterator & nano::receivable_iterator::operator++ () +{ + item = ledger->receivable_lower_bound (*tx, item.value ().first.account, item.value ().first.hash.number () + 1); + if (item && item.value ().first.account != account) + { + item = std::nullopt; + } + return *this; +} + +std::pair const & nano::receivable_iterator::operator* () const +{ + return item.value (); +} + +std::pair const * nano::receivable_iterator::operator->() const +{ + return &item.value (); +} diff --git a/nano/secure/pending_info.hpp b/nano/secure/pending_info.hpp index 584337e95e..0136da020d 100644 --- a/nano/secure/pending_info.hpp +++ b/nano/secure/pending_info.hpp @@ -4,6 +4,16 @@ #include #include +namespace nano +{ +class ledger; +} + +namespace nano::store +{ +class transaction; +} + namespace nano { /** @@ -33,6 +43,25 @@ class pending_key final nano::account account{}; nano::block_hash hash{ 0 }; }; +// This class iterates receivable enttries for an account +class receivable_iterator +{ +public: + receivable_iterator () = default; + receivable_iterator (nano::ledger const & ledger, nano::store::transaction const & tx, std::optional> item); + bool operator== (receivable_iterator const & other) const; + bool operator!= (receivable_iterator const & other) const; + // Advances to the next receivable entry for the same account + receivable_iterator & operator++ (); + std::pair const & operator* () const; + std::pair const * operator->() const; + +private: + nano::ledger const * ledger{ nullptr }; + nano::store::transaction const * tx{ nullptr }; + nano::account account{ 0 }; + std::optional> item; +}; } // namespace nano namespace std From 994bef05d4796c530653ed7b3330b53e44c11b5c Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 16:24:58 +0000 Subject: [PATCH 02/15] Add receivable_any function to nano::ledger which returns whether there is any receivable entry for an account. --- nano/core_test/ledger.cpp | 26 ++++++++++++++++++++++++++ nano/secure/ledger.cpp | 8 +++++++- nano/secure/ledger.hpp | 2 ++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index e8ef9b67a3..9557954f61 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -5654,3 +5654,29 @@ TEST (ledger_receivable, key_two) ASSERT_EQ (ledger.receivable_end (), ++next1); ASSERT_EQ (ledger.receivable_end (), ++next2); } + +TEST (ledger_receivable, any_none) +{ + auto ctx = nano::test::context::ledger_empty (); + ASSERT_FALSE (ctx.ledger ().receivable_any (ctx.store ().tx_begin_read (), nano::dev::genesis_key.pub)); +} + +TEST (ledger_receivable, any_one) +{ + auto ctx = nano::test::context::ledger_empty (); + nano::block_builder builder; + nano::keypair key; + auto send1 = builder + .state () + .account (nano::dev::genesis_key.pub) + .previous (nano::dev::genesis->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .link (nano::dev::genesis_key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*ctx.pool ().generate (nano::dev::genesis->hash ())) + .build (); + ASSERT_EQ (nano::block_status::progress, ctx.ledger ().process (ctx.store ().tx_begin_write (), send1)); + ASSERT_TRUE (ctx.ledger ().receivable_any (ctx.store ().tx_begin_read (), nano::dev::genesis_key.pub)); + ASSERT_FALSE (ctx.ledger ().receivable_any (ctx.store ().tx_begin_read (), key.pub)); +} diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index cc256bdf9e..e539b4500f 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -413,7 +413,7 @@ void ledger_processor::epoch_block_impl (nano::state_block & block_a) // Non-exisitng account should have pending entries if (result == nano::block_status::progress) { - bool pending_exists = ledger.store.pending.any (transaction, block_a.hashables.account); + bool pending_exists = ledger.receivable_any (transaction, block_a.hashables.account); result = pending_exists ? nano::block_status::progress : nano::block_status::gap_epoch_open_pending; } } @@ -1545,6 +1545,12 @@ uint64_t nano::ledger::height (store::transaction const & transaction, nano::blo return block_l->sideband ().height; } +bool nano::ledger::receivable_any (store::transaction const & tx, nano::account const & account) const +{ + auto next = receivable_upper_bound (tx, account, 0); + return next != receivable_end (); +} + std::optional> nano::ledger::receivable_lower_bound (store::transaction const & tx, nano::account const & account, nano::block_hash const & hash) const { auto result = store.pending.begin (tx, { account, hash }); diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index 8ceaedf364..6c325a0296 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -87,6 +87,8 @@ class ledger final static nano::epoch version (nano::block const & block); nano::epoch version (store::transaction const & transaction, nano::block_hash const & hash) const; uint64_t height (store::transaction const & transaction, nano::block_hash const & hash) const; + // Returns whether there are any receivable entries for 'account' + bool receivable_any (store::transaction const & tx, nano::account const & account) const; nano::receivable_iterator receivable_end () const; // Returns the next receivable entry for an account greater than 'account' nano::receivable_iterator receivable_upper_bound (store::transaction const & tx, nano::account const & account) const; From a5e150068088e3a36bd3128542cdf71518d2119d Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Mon, 18 Mar 2024 12:48:20 +0000 Subject: [PATCH 03/15] Implement ledger::account_receivable in terms of receivable iterators --- nano/secure/ledger.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index e539b4500f..3df9fd9cf2 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -855,19 +855,11 @@ nano::uint128_t nano::ledger::account_balance (store::transaction const & transa nano::uint128_t nano::ledger::account_receivable (store::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a) { - nano::uint128_t result (0); - nano::account end (account_a.number () + 1); - for (auto i (store.pending.begin (transaction_a, nano::pending_key (account_a, 0))), n (store.pending.begin (transaction_a, nano::pending_key (end, 0))); i != n; ++i) + nano::uint128_t result{ 0 }; + for (auto i = receivable_upper_bound (transaction_a, account_a, 0), n = receivable_end (); i != n; ++i) { - nano::pending_info const & info (i->second); - if (only_confirmed_a) - { - if (block_confirmed (transaction_a, i->first.hash)) - { - result += info.amount.number (); - } - } - else + auto const & [key, info] = *i; + if (!only_confirmed_a || block_confirmed (transaction_a, key.hash)) { result += info.amount.number (); } From b08d6ff5228f71abed7785fdab187a6e8a5525cc Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Fri, 15 Mar 2024 23:00:15 +0000 Subject: [PATCH 04/15] Reimplement json_handler::accounts_receivable in terms of receivable iterators. --- nano/node/json_handler.cpp | 55 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 78eb9502f9..d69e57e58f 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1041,42 +1041,41 @@ void nano::json_handler::accounts_receivable () bool const sorting = request.get ("sorting", false); auto simple (threshold.is_zero () && !source && !sorting); // if simple, response is a list of hashes for each account boost::property_tree::ptree pending; - auto transaction (node.store.tx_begin_read ()); + auto transaction = node.store.tx_begin_read (); for (auto & accounts : request.get_child ("accounts")) { auto account (account_impl (accounts.second.data ())); if (!ec) { boost::property_tree::ptree peers_l; - for (auto i (node.store.pending.begin (transaction, nano::pending_key (account, 0))), n (node.store.pending.end ()); i != n && nano::pending_key (i->first).account == account && peers_l.size () < count; ++i) + for (auto current = node.ledger.receivable_upper_bound (transaction, account, 0), end = node.ledger.receivable_end (); current != end; ++current) { - nano::pending_key const & key (i->first); - if (block_confirmed (node, transaction, key.hash, include_active, include_only_confirmed)) + auto const & [key, info] = *current; + if (include_only_confirmed && !node.ledger.block_confirmed (transaction, key.hash)) { - if (simple) - { - boost::property_tree::ptree entry; - entry.put ("", key.hash.to_string ()); - peers_l.push_back (std::make_pair ("", entry)); - } - else - { - nano::pending_info const & info (i->second); - if (info.amount.number () >= threshold.number ()) - { - if (source) - { - boost::property_tree::ptree pending_tree; - pending_tree.put ("amount", info.amount.number ().convert_to ()); - pending_tree.put ("source", info.source.to_account ()); - peers_l.add_child (key.hash.to_string (), pending_tree); - } - else - { - peers_l.put (key.hash.to_string (), info.amount.number ().convert_to ()); - } - } - } + continue; + } + if (simple) + { + boost::property_tree::ptree entry; + entry.put ("", key.hash.to_string ()); + peers_l.push_back (std::make_pair ("", entry)); + continue; + } + if (info.amount.number () < threshold.number ()) + { + continue; + } + if (source) + { + boost::property_tree::ptree pending_tree; + pending_tree.put ("amount", info.amount.number ().template convert_to ()); + pending_tree.put ("source", info.source.to_account ()); + peers_l.add_child (key.hash.to_string (), pending_tree); + } + else + { + peers_l.put (key.hash.to_string (), info.amount.number ().template convert_to ()); } } if (sorting && !simple) From 9d962e6e462c3ec1a2d26b9720b9f06d718d39a4 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Fri, 15 Mar 2024 23:10:11 +0000 Subject: [PATCH 05/15] Rewrite json_handler::receivable in terms of receivable iterators --- nano/node/json_handler.cpp | 95 +++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index d69e57e58f..d956ea9cb0 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -3051,66 +3051,65 @@ void nano::json_handler::receivable () { auto offset_counter = offset; boost::property_tree::ptree peers_l; - auto transaction (node.store.tx_begin_read ()); + auto transaction = node.store.tx_begin_read (); // The ptree container is used if there are any children nodes (e.g source/min_version) otherwise the amount container is used. std::vector> hash_ptree_pairs; std::vector> hash_amount_pairs; - for (auto i (node.store.pending.begin (transaction, nano::pending_key (account, 0))), n (node.store.pending.end ()); i != n && nano::pending_key (i->first).account == account && (should_sort || peers_l.size () < count); ++i) + for (auto current = node.ledger.receivable_upper_bound (transaction, account, 0), end = node.ledger.receivable_end (); current != end && (should_sort || peers_l.size () < count); ++current) { - nano::pending_key const & key (i->first); - if (block_confirmed (node, transaction, key.hash, include_active, include_only_confirmed)) + auto const & [key, info] = *current; + if (include_only_confirmed && !node.ledger.block_confirmed (transaction, key.hash)) + { + continue; + } + if (!should_sort && offset_counter > 0) + { + --offset_counter; + continue; + } + + if (simple) + { + boost::property_tree::ptree entry; + entry.put ("", key.hash.to_string ()); + peers_l.push_back (std::make_pair ("", entry)); + continue; + } + if (info.amount.number () < threshold.number ()) + { + continue; + } + if (source || min_version) { - if (!should_sort && offset_counter > 0) + boost::property_tree::ptree pending_tree; + pending_tree.put ("amount", info.amount.number ().template convert_to ()); + if (source) { - --offset_counter; - continue; + pending_tree.put ("source", info.source.to_account ()); + } + if (min_version) + { + pending_tree.put ("min_version", epoch_as_string (info.epoch)); } - if (simple) + if (should_sort) { - boost::property_tree::ptree entry; - entry.put ("", key.hash.to_string ()); - peers_l.push_back (std::make_pair ("", entry)); + hash_ptree_pairs.emplace_back (key.hash.to_string (), pending_tree); } else { - nano::pending_info const & info (i->second); - if (info.amount.number () >= threshold.number ()) - { - if (source || min_version) - { - boost::property_tree::ptree pending_tree; - pending_tree.put ("amount", info.amount.number ().convert_to ()); - if (source) - { - pending_tree.put ("source", info.source.to_account ()); - } - if (min_version) - { - pending_tree.put ("min_version", epoch_as_string (info.epoch)); - } - - if (should_sort) - { - hash_ptree_pairs.emplace_back (key.hash.to_string (), pending_tree); - } - else - { - peers_l.add_child (key.hash.to_string (), pending_tree); - } - } - else - { - if (should_sort) - { - hash_amount_pairs.emplace_back (key.hash.to_string (), info.amount.number ()); - } - else - { - peers_l.put (key.hash.to_string (), info.amount.number ().convert_to ()); - } - } - } + peers_l.add_child (key.hash.to_string (), pending_tree); + } + } + else + { + if (should_sort) + { + hash_amount_pairs.emplace_back (key.hash.to_string (), info.amount.number ()); + } + else + { + peers_l.put (key.hash.to_string (), info.amount.number ().template convert_to ()); } } } From 873760f345bfb15b4b991fc1de4bd604ed0c5ee4 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Fri, 15 Mar 2024 23:18:03 +0000 Subject: [PATCH 06/15] Reimplement json_handler::wallet_receivable in terms of receivable iterators --- nano/node/json_handler.cpp | 61 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index d956ea9cb0..c5b6d5056a 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -4788,47 +4788,46 @@ void nano::json_handler::wallet_receivable () { boost::property_tree::ptree pending; auto transaction (node.wallets.tx_begin_read ()); - auto block_transaction (node.store.tx_begin_read ()); + auto block_transaction = node.store.tx_begin_read (); for (auto i (wallet->store.begin (transaction)), n (wallet->store.end ()); i != n; ++i) { nano::account const & account (i->first); boost::property_tree::ptree peers_l; - for (auto ii (node.store.pending.begin (block_transaction, nano::pending_key (account, 0))), nn (node.store.pending.end ()); ii != nn && nano::pending_key (ii->first).account == account && peers_l.size () < count; ++ii) + for (auto current = node.ledger.receivable_upper_bound (block_transaction, account, 0), end = node.ledger.receivable_end (); current != end && (peers_l.size () < count); ++current) { - nano::pending_key key (ii->first); - if (block_confirmed (node, block_transaction, key.hash, include_active, include_only_confirmed)) + auto const & [key, info] = *current; + if (include_only_confirmed && !node.ledger.block_confirmed (block_transaction, key.hash)) + { + continue; + } + if (threshold.is_zero () && !source) + { + boost::property_tree::ptree entry; + entry.put ("", key.hash.to_string ()); + peers_l.push_back (std::make_pair ("", entry)); + continue; + } + if (info.amount.number () < threshold.number ()) { - if (threshold.is_zero () && !source) + continue; + } + if (source || min_version) + { + boost::property_tree::ptree pending_tree; + pending_tree.put ("amount", info.amount.number ().template convert_to ()); + if (source) { - boost::property_tree::ptree entry; - entry.put ("", key.hash.to_string ()); - peers_l.push_back (std::make_pair ("", entry)); + pending_tree.put ("source", info.source.to_account ()); } - else + if (min_version) { - nano::pending_info info (ii->second); - if (info.amount.number () >= threshold.number ()) - { - if (source || min_version) - { - boost::property_tree::ptree pending_tree; - pending_tree.put ("amount", info.amount.number ().convert_to ()); - if (source) - { - pending_tree.put ("source", info.source.to_account ()); - } - if (min_version) - { - pending_tree.put ("min_version", epoch_as_string (info.epoch)); - } - peers_l.add_child (key.hash.to_string (), pending_tree); - } - else - { - peers_l.put (key.hash.to_string (), info.amount.number ().convert_to ()); - } - } + pending_tree.put ("min_version", epoch_as_string (info.epoch)); } + peers_l.add_child (key.hash.to_string (), pending_tree); + } + else + { + peers_l.put (key.hash.to_string (), info.amount.number ().template convert_to ()); } } if (!peers_l.empty ()) From 30a3cc2462a7083c710f7314dfdaed5c3915312d Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Fri, 15 Mar 2024 23:31:57 +0000 Subject: [PATCH 07/15] Rewrite wallet::search_receivable in terms of receivable iterators --- nano/node/wallet.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 6d7b6cd7a6..927d22f52e 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -1182,15 +1182,14 @@ bool nano::wallet::search_receivable (store::transaction const & wallet_transact // Don't search pending for watch-only accounts if (!nano::wallet_value (i->second).key.is_zero ()) { - for (auto j (wallets.node.store.pending.begin (block_transaction, nano::pending_key (account, 0))), k (wallets.node.store.pending.end ()); j != k && nano::pending_key (j->first).account == account; ++j) + for (auto i = wallets.node.ledger.receivable_upper_bound (block_transaction, account, 0), n = wallets.node.ledger.receivable_end (); i != n; ++i) { - nano::pending_key key (j->first); - auto hash (key.hash); - nano::pending_info pending (j->second); - auto amount (pending.amount.number ()); + auto const & [key, info] = *i; + auto hash = key.hash; + auto amount = info.amount.number (); if (wallets.node.config.receive_minimum.number () <= amount) { - wallets.node.logger.info (nano::log::type::wallet, "Found a receivable block {} for account {}", hash.to_string (), pending.source.to_account ()); + wallets.node.logger.info (nano::log::type::wallet, "Found a receivable block {} for account {}", hash.to_string (), info.source.to_account ()); if (wallets.node.ledger.block_confirmed (block_transaction, hash)) { From 8087454387d510ee112d0e13e3367c6ba91295b1 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Fri, 15 Mar 2024 23:41:21 +0000 Subject: [PATCH 08/15] Rewrite wallet::deterministic_check in terms of receivable iterators --- nano/node/wallet.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 927d22f52e..864ca4f016 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -1248,11 +1248,11 @@ uint32_t nano::wallet::deterministic_check (store::transaction const & transacti else { // Check if there are pending blocks for account - for (auto ii (wallets.node.store.pending.begin (block_transaction, nano::pending_key (pair.pub, 0))), nn (wallets.node.store.pending.end ()); ii != nn && nano::pending_key (ii->first).account == pair.pub; ++ii) + auto current = wallets.node.ledger.receivable_upper_bound (block_transaction, pair.pub, 0); + if (current != wallets.node.ledger.receivable_end ()) { index = i; n = i + 64 + (i / 64); - break; } } } From b34317004258d30fe38bdc64325a6ae20c8790c2 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 18:07:30 +0000 Subject: [PATCH 09/15] Rewrite system::generate_receive in terms of receivable iterators --- nano/test_common/system.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nano/test_common/system.cpp b/nano/test_common/system.cpp index 19a57c812c..f4522c62b3 100644 --- a/nano/test_common/system.cpp +++ b/nano/test_common/system.cpp @@ -416,11 +416,10 @@ void nano::test::system::generate_receive (nano::node & node_a) auto transaction (node_a.store.tx_begin_read ()); nano::account random_account; random_pool::generate_block (random_account.bytes.data (), sizeof (random_account.bytes)); - auto i (node_a.store.pending.begin (transaction, nano::pending_key (random_account, 0))); - if (i != node_a.store.pending.end ()) + auto item = node_a.ledger.receivable_upper_bound (transaction, random_account); + if (item != node_a.ledger.receivable_end ()) { - nano::pending_key const & send_hash (i->first); - send_block = node_a.ledger.block (transaction, send_hash.hash); + send_block = node_a.ledger.block (transaction, item->first.hash); } } if (send_block != nullptr) From 4439f46d0c2f932e06f4c2ad984914c31ba348e6 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 13:18:14 +0000 Subject: [PATCH 10/15] Rewrite bulk_pull_account_server in terms of receivable iterators --- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 26 +++++---------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index c5ca04b28b..65f392b580 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -753,30 +753,16 @@ std::pair, std::unique_ptrstore.tx_begin_read ()); - auto stream (node->store.pending.begin (stream_transaction, current_key)); - - if (stream == store::iterator (nullptr)) - { - break; - } - - nano::pending_key key (stream->first); - nano::pending_info info (stream->second); + auto tx = node->store.tx_begin_read (); + auto & ledger = node->ledger; + auto stream = ledger.receivable_upper_bound (tx, current_key.account, current_key.hash); - /* - * Get the key for the next value, to use in the next call or iteration - */ - current_key.account = key.account; - current_key.hash = key.hash.number () + 1; - - /* - * Finish up if the response is for a different account - */ - if (key.account != request->account) + if (stream == ledger.receivable_end ()) { break; } + auto const & [key, info] = *stream; + current_key = key; /* * Skip entries where the amount is less than the requested From 35ab9d408f0ba881c55c6766d1fb230f04b36090 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 19:07:32 +0000 Subject: [PATCH 11/15] Rewrite bootstrap database_iterator in terms of receivable iterators --- nano/node/bootstrap_ascending/iterators.cpp | 24 ++++++++++----------- nano/node/bootstrap_ascending/iterators.hpp | 13 +++++++---- nano/node/bootstrap_ascending/service.cpp | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/nano/node/bootstrap_ascending/iterators.cpp b/nano/node/bootstrap_ascending/iterators.cpp index d35c682567..c940218a36 100644 --- a/nano/node/bootstrap_ascending/iterators.cpp +++ b/nano/node/bootstrap_ascending/iterators.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -9,8 +10,8 @@ * database_iterator */ -nano::bootstrap_ascending::database_iterator::database_iterator (nano::store::component & store_a, table_type table_a) : - store{ store_a }, +nano::bootstrap_ascending::database_iterator::database_iterator (nano::ledger & ledger, table_type table_a) : + ledger{ ledger }, table{ table_a } { } @@ -27,8 +28,8 @@ void nano::bootstrap_ascending::database_iterator::next (store::transaction & tx case table_type::account: { auto i = current.number () + 1; - auto item = store.account.begin (tx, i); - if (item != store.account.end ()) + auto item = ledger.store.account.begin (tx, i); + if (item != ledger.store.account.end ()) { current = item->first; } @@ -40,9 +41,8 @@ void nano::bootstrap_ascending::database_iterator::next (store::transaction & tx } case table_type::pending: { - auto i = current.number () + 1; - auto item = store.pending.begin (tx, nano::pending_key{ i, 0 }); - if (item != store.pending.end ()) + auto item = ledger.receivable_upper_bound (tx, current); + if (item != ledger.receivable_end ()) { current = item->first.account; } @@ -59,10 +59,10 @@ void nano::bootstrap_ascending::database_iterator::next (store::transaction & tx * buffered_iterator */ -nano::bootstrap_ascending::buffered_iterator::buffered_iterator (nano::store::component & store_a) : - store{ store_a }, - accounts_iterator{ store, database_iterator::table_type::account }, - pending_iterator{ store, database_iterator::table_type::pending } +nano::bootstrap_ascending::buffered_iterator::buffered_iterator (nano::ledger & ledger) : + ledger{ ledger }, + accounts_iterator{ ledger, database_iterator::table_type::account }, + pending_iterator{ ledger, database_iterator::table_type::pending } { } @@ -95,7 +95,7 @@ void nano::bootstrap_ascending::buffered_iterator::fill () debug_assert (buffer.empty ()); // Fill half from accounts table and half from pending table - auto transaction = store.tx_begin_read (); + auto transaction = ledger.store.tx_begin_read (); for (int n = 0; n < size / 2; ++n) { diff --git a/nano/node/bootstrap_ascending/iterators.hpp b/nano/node/bootstrap_ascending/iterators.hpp index 58d7c6c690..e963164de5 100644 --- a/nano/node/bootstrap_ascending/iterators.hpp +++ b/nano/node/bootstrap_ascending/iterators.hpp @@ -4,6 +4,11 @@ #include +namespace nano +{ +class ledger; +} + namespace nano::store { class component; @@ -21,12 +26,12 @@ class database_iterator pending }; - explicit database_iterator (nano::store::component & store, table_type); + explicit database_iterator (nano::ledger & ledger, table_type); nano::account operator* () const; void next (nano::store::transaction & tx); private: - nano::store::component & store; + nano::ledger & ledger; nano::account current{ 0 }; const table_type table; }; @@ -34,7 +39,7 @@ class database_iterator class buffered_iterator { public: - explicit buffered_iterator (nano::store::component & store); + explicit buffered_iterator (nano::ledger & ledger); nano::account operator* () const; nano::account next (); // Indicates if a full ledger iteration has taken place e.g. warmed up @@ -44,7 +49,7 @@ class buffered_iterator void fill (); private: - nano::store::component & store; + nano::ledger & ledger; std::deque buffer; bool warmup_m{ true }; diff --git a/nano/node/bootstrap_ascending/service.cpp b/nano/node/bootstrap_ascending/service.cpp index e75a26c247..c3e9bd2d15 100644 --- a/nano/node/bootstrap_ascending/service.cpp +++ b/nano/node/bootstrap_ascending/service.cpp @@ -24,7 +24,7 @@ nano::bootstrap_ascending::service::service (nano::node_config & config_a, nano: network{ network_a }, stats{ stat_a }, accounts{ stats }, - iterator{ ledger.store }, + iterator{ ledger }, throttle{ compute_throttle_size () }, scoring{ config.bootstrap_ascending, config.network_params.network }, database_limiter{ config.bootstrap_ascending.database_requests_limit, 1.0 } From 773cd91f1bff8da6f097203c38881924d29d7000 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 19:13:51 +0000 Subject: [PATCH 12/15] Rewrite epoch_upgrader in terms of receivable iterators. --- nano/node/epoch_upgrader.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nano/node/epoch_upgrader.cpp b/nano/node/epoch_upgrader.cpp index 58db6213d0..495d09dc54 100644 --- a/nano/node/epoch_upgrader.cpp +++ b/nano/node/epoch_upgrader.cpp @@ -210,14 +210,13 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc std::atomic upgraded_pending (0); uint64_t workers (0); uint64_t attempts (0); - auto transaction (store.tx_begin_read ()); - for (auto i (store.pending.begin (transaction, nano::pending_key (1, 0))), n (store.pending.end ()); i != n && attempts < upgrade_batch_size && attempts < count_limit && !stopped;) + auto transaction = store.tx_begin_read (); + for (auto current = ledger.receivable_upper_bound (transaction, 0), end = ledger.receivable_end (); current != end && attempts < upgrade_batch_size && attempts < count_limit && !stopped;) { bool to_next_account (false); - nano::pending_key const & key (i->first); + auto const & [key, info] = *current; if (!store.account.exists (transaction, key.account)) { - nano::pending_info const & info (i->second); if (info.epoch < epoch_a) { ++attempts; @@ -272,13 +271,13 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc } else { - i = store.pending.begin (transaction, nano::pending_key (key.account.number () + 1, 0)); + current = ledger.receivable_upper_bound (transaction, key.account); } } else { // Move to next pending item - ++i; + current = ledger.receivable_upper_bound (transaction, key.account, key.hash); } } { From cc9006563031f830aa1c3eb8bebf5db29eb1f022 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 19:16:48 +0000 Subject: [PATCH 13/15] Fuse branches in epoch_upgrader::upgrade_impl --- nano/node/epoch_upgrader.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/nano/node/epoch_upgrader.cpp b/nano/node/epoch_upgrader.cpp index 495d09dc54..c4f683d498 100644 --- a/nano/node/epoch_upgrader.cpp +++ b/nano/node/epoch_upgrader.cpp @@ -213,7 +213,6 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc auto transaction = store.tx_begin_read (); for (auto current = ledger.receivable_upper_bound (transaction, 0), end = ledger.receivable_end (); current != end && attempts < upgrade_batch_size && attempts < count_limit && !stopped;) { - bool to_next_account (false); auto const & [key, info] = *current; if (!store.account.exists (transaction, key.account)) { @@ -257,12 +256,10 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc upgrader_process (upgraded_pending, epoch, difficulty, signer, root, account); } } + // Move to next pending item + current = ledger.receivable_upper_bound (transaction, key.account, key.hash); } else - { - to_next_account = true; - } - if (to_next_account) { // Move to next account if pending account exists or was upgraded if (key.account.number () == std::numeric_limits::max ()) @@ -274,11 +271,6 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc current = ledger.receivable_upper_bound (transaction, key.account); } } - else - { - // Move to next pending item - current = ledger.receivable_upper_bound (transaction, key.account, key.hash); - } } { nano::unique_lock lock{ upgrader_mutex }; From d58316451abd507a5a328ffc024651fa8a129ba2 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 19:24:17 +0000 Subject: [PATCH 14/15] Rewrite json_handler::unopened in terms of receivable iterators The algorithm had a non-trivial change to implementation. The intent is to sum the amount of balance receivable for given set of accounts. The region selected starts at "account" and ends when "count" is reached. --- nano/node/json_handler.cpp | 50 ++++++++++++-------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index c5b6d5056a..230638b71c 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -4238,7 +4238,7 @@ void nano::json_handler::unopened () { auto count (count_optional_impl ()); auto threshold (threshold_optional_impl ()); - nano::account start (1); // exclude burn account by default + nano::account start{ 1 }; // exclude burn account by default boost::optional account_text (request.get_optional ("account")); if (account_text.is_initialized ()) { @@ -4246,48 +4246,28 @@ void nano::json_handler::unopened () } if (!ec) { - auto transaction (node.store.tx_begin_read ()); - auto iterator (node.store.pending.begin (transaction, nano::pending_key (start, 0))); - auto end (node.store.pending.end ()); - nano::account current_account (start); - nano::uint128_t current_account_sum{ 0 }; + auto transaction = node.store.tx_begin_read (); + auto & ledger = node.ledger; boost::property_tree::ptree accounts; - while (iterator != end && accounts.size () < count) + for (auto iterator = ledger.receivable_upper_bound (transaction, start, 0), end = ledger.receivable_end (); iterator != end && accounts.size () < count;) { - nano::pending_key key (iterator->first); - nano::account account (key.account); - nano::pending_info info (iterator->second); - if (node.store.account.exists (transaction, account)) + auto const & [key, info] = *iterator; + nano::account account = key.account; + if (!node.store.account.exists (transaction, account)) { - if (account.number () == std::numeric_limits::max ()) + nano::uint128_t current_account_sum{ 0 }; + while (iterator != end) { - break; + auto const & [key, info] = *iterator; + current_account_sum += info.amount.number (); + ++iterator; } - // Skip existing accounts - iterator = node.store.pending.begin (transaction, nano::pending_key (account.number () + 1, 0)); - } - else - { - if (account != current_account) + if (current_account_sum >= threshold.number ()) { - if (current_account_sum > 0) - { - if (current_account_sum >= threshold.number ()) - { - accounts.put (current_account.to_account (), current_account_sum.convert_to ()); - } - current_account_sum = 0; - } - current_account = account; + accounts.put (account.to_account (), current_account_sum.convert_to ()); } - current_account_sum += info.amount.number (); - ++iterator; } - } - // last one after iterator reaches end - if (accounts.size () < count && current_account_sum > 0 && current_account_sum >= threshold.number ()) - { - accounts.put (current_account.to_account (), current_account_sum.convert_to ()); + iterator = ledger.receivable_upper_bound (transaction, account); } response_l.add_child ("accounts", accounts); } From 65913505856bc3c227a242840ba29f2da90b452f Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 16 Mar 2024 14:09:18 +0000 Subject: [PATCH 15/15] Rewrite bootstrap.trace_base in terms of receivable iterators --- nano/core_test/bootstrap_ascending.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nano/core_test/bootstrap_ascending.cpp b/nano/core_test/bootstrap_ascending.cpp index 257b35c60f..59c0eb381d 100644 --- a/nano/core_test/bootstrap_ascending.cpp +++ b/nano/core_test/bootstrap_ascending.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -251,7 +252,7 @@ TEST (bootstrap_ascending, trace_base) // std::cerr << "--------------- Start ---------------\n"; ASSERT_EQ (nano::block_status::progress, node0.process (send1)); ASSERT_EQ (nano::block_status::progress, node0.process (receive1)); - ASSERT_EQ (node1.store.pending.begin (node1.store.tx_begin_read (), nano::pending_key{ key.pub, 0 }), node1.store.pending.end ()); + ASSERT_EQ (node1.ledger.receivable_end (), node1.ledger.receivable_upper_bound (node1.store.tx_begin_read (), key.pub, 0)); // std::cerr << "node0: " << node0.network.endpoint () << std::endl; // std::cerr << "node1: " << node1.network.endpoint () << std::endl; ASSERT_TIMELY (10s, node1.block (receive1->hash ()) != nullptr);