From c6060d830249d1127da870eb814759403c33cf36 Mon Sep 17 00:00:00 2001 From: Brad Chase Date: Thu, 7 Dec 2017 16:16:05 -0500 Subject: [PATCH] [FOLD] Add LedgerTrie and tests --- Builds/VisualStudio2015/RippleD.vcxproj | 6 + .../VisualStudio2015/RippleD.vcxproj.filters | 6 + src/ripple/consensus/LedgerTrie.h | 699 ++++++++++++++++++ src/test/consensus/Consensus_test.cpp | 6 +- src/test/consensus/LedgerTrie_test.cpp | 537 ++++++++++++++ src/test/csf/Peer.h | 4 +- src/test/csf/impl/ledgers.cpp | 65 +- src/test/csf/ledgers.h | 84 ++- src/test/unity/consensus_test_unity.cpp | 1 + 9 files changed, 1385 insertions(+), 23 deletions(-) create mode 100644 src/ripple/consensus/LedgerTrie.h create mode 100644 src/test/consensus/LedgerTrie_test.cpp diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj index 9f3e25ebe8c..611b3ea7ff9 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj +++ b/Builds/VisualStudio2015/RippleD.vcxproj @@ -1873,6 +1873,8 @@ + + @@ -4501,6 +4503,10 @@ True True + + True + True + True True diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters index 1e641ad67e5..80a5bd98817 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2015/RippleD.vcxproj.filters @@ -2526,6 +2526,9 @@ ripple\consensus + + ripple\consensus + ripple\consensus @@ -5262,6 +5265,9 @@ test\consensus + + test\consensus + test\consensus diff --git a/src/ripple/consensus/LedgerTrie.h b/src/ripple/consensus/LedgerTrie.h new file mode 100644 index 00000000000..31a68ff08aa --- /dev/null +++ b/src/ripple/consensus/LedgerTrie.h @@ -0,0 +1,699 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2017 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_CONSENSUS_LEDGERS_TRIE_H_INCLUDED +#define RIPPLE_APP_CONSENSUS_LEDGERS_TRIE_H_INCLUDED + +#include +#include +#include +#include + +namespace ripple { + +/** Ancestry trie of ledgers + + Combination of a compressed trie and merkle-ish tree that maintains + validation support of recent ledgers based on their ancestry. + + The compressed trie structure comes from recognizing that ledger history + can be viewed as a string over the alphabet of ledger ids. That is, + a given ledger with sequence number `seq` defines a length `seq` string, + with i-th entry equal to the id of the ancestor ledger with sequence + number i. "Sequence" strings with a common prefix share those ancestor + ledgers in common. Tracking this ancestry information and relations across + all validated ledgers is done conveniently in a compressed trie. A node in + the trie is an ancestor of all its children. If a parent node has sequence + number `seq`, each child node has a different ledger starting at `seq+1`. + The compression comes from the invariant that any non-root node with 0 tip + support has either no children or multiple children. In other words, a + non-root 0-tip-support node can be combined with its single child. + + The merkle-ish property is based on the branch support calculation. Each + node has a tipSupport, which is the number of current validations for that + particular ledger. The branch support is the sum of the tip support and + the branch support of that node's children: + + @code + node->branchSupport = node->tipSupport + + sum_(child : node->children) child->branchSupport + @endcode + + This is analagous to the merkle tree property in which a node's hash is + the hash of the concatenation of its child node hashes. + + The templated Ledger type represents a ledger which has a unique history. + It should be lightweight and cheap to copy. + + @code + // Identifier types that should be equality-comparable and copyable + struct ID; + struct Seq; + + struct Ledger + { + // The default ledger represents a ledger that prefixes all other + // ledgers, e.g. the genesis ledger + Ledger(); + + Ledger(Ledger &); + Ledger& operator=(Ledger ); + + // Return the sequence number of this ledger + Seq seq() const; + + // Return the ID of this ledger's ancestor with given sequence number + // or ID{0} if unknown + ID + operator[](Seq s); + + }; + + // Return the sequence number of the first possible mismatching ancestor + // between two ledgers + Seq + mismatch(ledgerA, ledgerB); + @endcode + + The unique history invariant of ledgers requires any ledgers that agree + on the id of a given sequence number agree on ALL ancestors before that + ledger: + + @code + Ledger a,b; + // For all Seq s: + if(a[s] == b[s]); + for(Seq p = 0; p < s; ++p) + assert(a[p] == b[p]); + @endcode + + @tparam Ledger A type representing a ledger and its history +*/ +template +class LedgerTrie +{ + using Seq = typename Ledger::Seq; + using ID = typename Ledger::ID; + + /// Represents a span of ancestry of a ledger + class Span + { + // The span is the half-open interval [start,end) of ledger_ + Seq start_{0}; + Seq end_{1}; + Ledger ledger_; + + public: + Span() + { + // Require default ledger to be genesis seq + assert(ledger_.seq() == start_); + } + + Span(Ledger ledger) + : start_{0}, end_{ledger.seq() + Seq{1}}, ledger_{std::move(ledger)} + { + } + + Span(Span const& s) = default; + Span(Span&& s) = default; + Span& + operator=(Span const&) = default; + Span& + operator=(Span&&) = default; + + Seq + end() const + { + return end_; + } + + // Return the Span from (spot,end_] + Span + from(Seq spot) + { + return sub(spot, end_); + } + + // Return the Span from (start_,spot] + Span + before(Seq spot) + { + return sub(start_, spot); + } + + bool + empty() const + { + return start_ == end_; + } + + //Return the ID of the ledger that starts this span + ID + startID() const + { + return ledger_[start_]; + } + + // Return the ledger sequence number of the first possible difference + // between this span and a given ledger. + Seq + diff(Ledger const& o) const + { + return clamp(mismatch(ledger_, o)); + } + + // The Seq and ID of the end of the span + std::pair + tip() const + { + Seq tipSeq{end_ -Seq{1}}; + return {tipSeq, ledger_[tipSeq]}; + } + + private: + Span(Seq start, Seq end, Ledger const& l) + : start_{start}, end_{end}, ledger_{l} + { + assert(start <= end); + } + + Seq + clamp(Seq val) const + { + return std::min(std::max(start_, val), end_); + }; + + // Return a span of this over the half-open interval [from,to) + Span + sub(Seq from, Seq to) + { + return Span(clamp(from), clamp(to), ledger_); + } + + friend std::ostream& + operator<<(std::ostream& o, Span const& s) + { + return o << s.tip().second << "(" << s.start_ << "," << s.end_ + << "]"; + } + + friend Span + merge(Span const& a, Span const& b) + { + // Return combined span, using ledger_ from higher sequence span + if (a.end_ < b.end_) + return Span(std::min(a.start_, b.start_), b.end_, b.ledger_); + + return Span(std::min(a.start_, b.start_), a.end_, a.ledger_); + } + }; + + // A node in the trie + struct Node + { + Node() : span{}, tipSupport{0}, branchSupport{0} + { + } + + Node(Ledger const& l) : span{l}, tipSupport{1}, branchSupport{1} + { + } + + Node(Span s) : span{std::move(s)} + { + } + + Span span; + std::uint32_t tipSupport = 0; + std::uint32_t branchSupport = 0; + + std::vector> children; + Node* parent = nullptr; + + /** Remove the given node from this Node's children + + @param child The address of the child node to remove + @note The child must be a member of the vector. The passed pointer + will be dangling as a result of this call + */ + void + erase(Node const* child) + { + auto it = std::remove_if( + children.begin(), + children.end(), + [child](std::unique_ptr const& curr) { + return curr.get() == child; + }); + assert(it != children.end()); + children.erase(it, children.end()); + } + + friend std::ostream& + operator<<(std::ostream& o, Node const& s) + { + return o << s.span << "(T:" << s.tipSupport + << ",B:" << s.branchSupport << ")"; + } + + Json::Value + getJson() const + { + Json::Value res; + res["id"] = to_string(span.tip().second); + res["seq"] = static_cast(span.tip().first); + res["tipSupport"] = tipSupport; + res["branchSupport"] = branchSupport; + if(!children.empty()) + { + Json::Value &cs = (res["children"] = Json::arrayValue); + for(auto const & child : children) + { + cs.append(child->getJson()); + } + } + return res; + } + }; + + // The root of the trie. The root is allowed to break the no-single child + // invariant. + std::unique_ptr root; + + /** Find the node in the trie that represents the longest common ancestry + with the given ledger. + + @return Pair of the found node and the sequence number of the first + ledger difference. + */ + std::pair + find(Ledger const& ledger) const + { + Node* curr = root.get(); + + // Root is always defined and is in common with all ledgers + assert(curr); + Seq pos = curr->span.diff(ledger); + + bool done = false; + + // Continue searching for a better span as long as the current position + // matches the entire span + while (!done && pos == curr->span.end()) + { + done = true; + // Find the child with the longest ancestry match + for (std::unique_ptr const& child : curr->children) + { + auto childPos = child->span.diff(ledger); + if (childPos > pos) + { + done = false; + pos = childPos; + curr = child.get(); + break; + } + } + } + return std::make_pair(curr, pos); + } + + void + dumpImpl(std::ostream& o, std::unique_ptr const& curr, int offset) + const + { + if (curr) + { + if (offset > 0) + o << std::setw(offset) << "|-"; + + std::stringstream ss; + ss << *curr; + o << ss.str() << std::endl; + for (std::unique_ptr const& child : curr->children) + dumpImpl(o, child, offset + 1 + ss.str().size() + 2); + } + } + +public: + LedgerTrie() : root{std::make_unique()} + { + } + + /** Insert and/or increment the support for the given ledger. + + @param ledger A ledger and its ancestry + @param count The count of support for this ledger + */ + void + insert(Ledger const& ledger, std::uint32_t count = 1) + { + Node* loc; + Seq diffSeq; + std::tie(loc, diffSeq) = find(ledger); + + // There is always a place to insert + assert(loc); + + Span lTmp{ledger}; + Span prefix = lTmp.before(diffSeq); + Span oldSuffix = loc->span.from(diffSeq); + Span newSuffix = lTmp.from(diffSeq); + Node* incNode = loc; + + if (!oldSuffix.empty()) + { + // new is a prefix of current + // e.g. abcdef->..., adding abcd + // becomes abcd->ef->... + + // Create oldSuffix node that takes over loc + std::unique_ptr newNode{std::make_unique(oldSuffix)}; + newNode->tipSupport = loc->tipSupport; + newNode->branchSupport = loc->branchSupport; + using std::swap; + swap(newNode->children, loc->children); + for(std::unique_ptr & child : newNode->children) + child->parent = newNode.get(); + + // Loc truncates to prefix and newNode is its child + loc->span = prefix; + newNode->parent = loc; + loc->children.emplace_back(std::move(newNode)); + loc->tipSupport = 0; + } + if (!newSuffix.empty()) + { + // current is a substring of new + // e.g. abc->... adding abcde + // -> abc-> ... + // -> de + + std::unique_ptr newNode{std::make_unique(newSuffix)}; + newNode->parent = loc; + // increment support starting from the new node + incNode = newNode.get(); + loc->children.push_back(std::move(newNode)); + } + + incNode->tipSupport += count; + while (incNode) + { + incNode->branchSupport += count; + incNode = incNode->parent; + } + } + + /** Decrease support for a ledger, removing and compressing if possible. + + @param ledger The ledger history to remove + @param count The amount of tip support to remove + + @return Whether a matching node was decremented and possibly removed. + */ + bool + remove(Ledger const& ledger, std::uint32_t count = 1) + { + Node* loc; + Seq diffSeq; + std::tie(loc, diffSeq) = find(ledger); + + // Cannot erase root + if (loc && loc != root.get()) + { + // Must be exact match with tip support + if (diffSeq == loc->span.end() && diffSeq > ledger.seq() && + loc->tipSupport > 0) + { + count = std::min(count, loc->tipSupport); + loc->tipSupport -= count; + + Node* decNode = loc; + while (decNode) + { + decNode->branchSupport -= count; + decNode = decNode->parent; + } + + while (loc->tipSupport == 0 && loc != root.get()) + { + Node* parent = loc->parent; + if (loc->children.empty()) + { + // this node can be erased + parent->erase(loc); + } + else if (loc->children.size() == 1) + { + // This node can be combined with its child + std::unique_ptr child = + std::move(loc->children.front()); + child->span = merge(loc->span, child->span); + child->parent = parent; + parent->children.emplace_back(std::move(child)); + parent->erase(loc); + } + else + break; + loc = parent; + } + return true; + } + } + return false; + } + + /** Return count of tip support for the specific ledger. + + @param ledger The ledger to lookup + @return The number of entries in the trie for this *exact* ledger + */ + std::uint32_t + tipSupport(Ledger const& ledger) const + { + Node const* loc; + Seq diffSeq; + std::tie(loc, diffSeq) = find(ledger); + + // Exact match + if (loc && diffSeq == loc->span.end() && diffSeq > ledger.seq()) + return loc->tipSupport; + return 0; + } + + /** Return the count of branch support for the specific ledger + + @param ledger The ledger to lookup + @return The number of entries in the trie for this ledger or a descendent + */ + std::uint32_t + branchSupport(Ledger const& ledger) const + { + Node const* loc; + Seq diffSeq; + std::tie(loc, diffSeq) = find(ledger); + + // Check that ledger is is an exact match or proper + // prefix of loc + if (loc && diffSeq > ledger.seq() && + ledger.seq() < loc->span.end()) + { + return loc->branchSupport; + } + return 0; + } + + /** Return the preferred ledger ID + + The preferred ledger is used to determine the working ledger + for consensus amongst competing alternatives. + + Recall that each validator is normally validating a chain of ledgers, + e.g. A->B->C->D. However, if due to network connectivity or other issues, + validators generate different chains + + @code + /->C + A->B + \->D->E + @endcode + + we need a way for validators to converge on the chain with the most + support. We call this the preferred ledger. Intuitively, the idea is to + be conservative and only switch to a different branch when you see enough + peer validations to *know* another branch won't have preferred support. + This ensures the preferred branch has monotonically increasing support. + + The preferred ledger is found by walking this tree of validated ledgers + starting from the common ancestor ledger. + + At each sequence number, we have + + - The prior sequence preferred ledger (B). + - The support of ledgers that have been explicitly validated by a + validator (C,D), or are an ancestor of that validators current + validated ledger (E). + - The number of validators that have yet to validate a ledger + with this sequence number (prefixSupport). + + The preferred ledger for this sequence number is then the ledger + with relative majority of support, where prefixSupport can be given to + ANY ledger at that sequence number (including one not yet known). If no + such preferred ledger exists, than prior sequence preferred ledger is + the overall preferred ledger. If one does exist, then we continue + with the next sequence but increase prefixSupport with the non + preferred ones this round, e.g. if C were preferred over D, then + prefixSupport would incerase by the support of D and E. + + */ + std::pair + getPreferred() + { + Node* curr = root.get(); + + bool done = false; + std::uint32_t prefixSupport = curr->tipSupport; + while (curr && !done) + { + Node* best = nullptr; + std::uint32_t margin = 0; + + if (curr->children.size() == 1) + { + best = curr->children[0].get(); + margin = best->branchSupport; + } + else if (!curr->children.empty()) + { + // Sort placing children with largest branch support in the + // front, breaking ties with the span's starting ID + std::partial_sort( + curr->children.begin(), + curr->children.begin() + 2, + curr->children.end(), + [](std::unique_ptr const& a, + std::unique_ptr const& b) { + return std::make_tuple(a->branchSupport, a->span.startID()) > + std::make_tuple(b->branchSupport, b->span.startID()); + }); + + best = curr->children[0].get(); + margin = curr->children[0]->branchSupport - + curr->children[1]->branchSupport; + + // If best holds the tie-breaker, gets one larger margin + // since the second best needs additional branchSupport + // to overcome the tie + if (best->span.startID() > curr->children[1]->span.startID()) + margin++; + } + + // If the best child has margin exceeding the prefix support, + // continue from that child, otherwise we are done + if (best && ((margin > prefixSupport) || (prefixSupport == 0))) + { + // Prefix support is all the support not on the branch we + // are moving to + // curr + // _/ | \_ + // A B best + // At curr, the prefix support already includes the tip support + // of curr and its ancestors, along with the branch support of + // any of its siblings that are inconsistent. + // + // The additional prefix suppport that is carried to best is + // A->branchSupport + B->branchSupport + best->tipSupport + // This is the amount of support that has not yet voted + // on a descendent of best, or has voted on a conflicting + // descendent and will switch to best in the future. This means + // that they may support an arbitrary descendent of best. + // + // The calculation is simplified using + // A->branchSupport+B->branchSupport + // = curr->branchSupport - best->branchSupport + // - curr->tipSupport + // + // This will not overflow by definition of the above quantities + prefixSupport += (curr->branchSupport - best->branchSupport + - curr->tipSupport) + best->tipSupport; + + curr = best; + } + else // current is the best + done = true; + } + return curr->span.tip(); + } + + /** Dump an ascii representation of the trie to the stream + */ + void + dump(std::ostream& o) const + { + dumpImpl(o, root, 0); + } + + /** Dump JSON representation of trie state + */ + Json::Value + getJson() const + { + return root->getJson(); + } + + /** Check the compressed trie and support invariants. + */ + bool + checkInvariants() const + { + std::stack nodes; + nodes.push(root.get()); + while (!nodes.empty()) + { + Node const* curr = nodes.top(); + nodes.pop(); + if (!curr) + continue; + + // Node with 0 tip support must have multiple children + // unless it is the root node + if (curr != root.get() && curr->tipSupport == 0 && + curr->children.size() < 2) + return false; + + // branchSupport = tipSupport + sum(child->branchSupport) + std::size_t support = curr->tipSupport; + for (auto const& child : curr->children) + { + if(child->parent != curr) + return false; + + support += child->branchSupport; + nodes.push(child.get()); + } + if (support != curr->branchSupport) + return false; + } + return true; + } +}; + +} // namespace ripple +#endif diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp index 788d2b46bab..5df162aa8b6 100644 --- a/src/test/consensus/Consensus_test.cpp +++ b/src/test/consensus/Consensus_test.cpp @@ -512,8 +512,7 @@ class Consensus_test : public beast::unit_test::suite peerJumps.closeJumps.front(); // Jump is to a different chain BEAST_EXPECT(jump.from.seq() <= jump.to.seq()); - BEAST_EXPECT( - !sim.oracle.isAncestor(jump.from, jump.to)); + BEAST_EXPECT(!jump.to.isAncestor(jump.from)); } } // fully validated jump forward in same chain @@ -525,8 +524,7 @@ class Consensus_test : public beast::unit_test::suite peerJumps.fullyValidatedJumps.front(); // Jump is to a different chain with same seq BEAST_EXPECT(jump.from.seq() < jump.to.seq()); - BEAST_EXPECT( - sim.oracle.isAncestor(jump.from, jump.to)); + BEAST_EXPECT(jump.to.isAncestor(jump.from)); } } } diff --git a/src/test/consensus/LedgerTrie_test.cpp b/src/test/consensus/LedgerTrie_test.cpp new file mode 100644 index 00000000000..f149bb45b4d --- /dev/null +++ b/src/test/consensus/LedgerTrie_test.cpp @@ -0,0 +1,537 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2017 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +class LedgerTrie_test : public beast::unit_test::suite +{ + beast::Journal j; + + + void + testInsert() + { + using namespace csf; + // Single entry by itself + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); + + t.insert(h["abc"]); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + } + // Suffix of existing (extending tree) + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + BEAST_EXPECT(t.checkInvariants()); + // extend with no siblings + t.insert(h["abcd"]); + BEAST_EXPECT(t.checkInvariants()); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); + + // extend with existing sibling + t.insert(h["abce"]); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abce"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abce"]) == 1); + } + // Prefix of existing node + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + BEAST_EXPECT(t.checkInvariants()); + // prefix with no siblings + t.insert(h["abcdf"]); + BEAST_EXPECT(t.checkInvariants()); + + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abcdf"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcdf"]) == 1); + + // prefix with existing child + t.insert(h["abc"]); + BEAST_EXPECT(t.checkInvariants()); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abcdf"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcdf"]) == 1); + } + // Suffix + prefix of existing node + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + BEAST_EXPECT(t.checkInvariants()); + t.insert(h["abce"]); + BEAST_EXPECT(t.checkInvariants()); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abce"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abce"]) == 1); + } + // Suffix + prefix with existing child + { + // abcd : abcde, abcf + + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + BEAST_EXPECT(t.checkInvariants()); + t.insert(h["abcde"]); + BEAST_EXPECT(t.checkInvariants()); + t.insert(h["abcf"]); + BEAST_EXPECT(t.checkInvariants()); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abcf"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcf"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abcde"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcde"]) == 1); + } + + // Multiple counts + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"],4); + BEAST_EXPECT(t.tipSupport(h["ab"]) == 4); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 4); + BEAST_EXPECT(t.tipSupport(h["a"]) == 0); + BEAST_EXPECT(t.branchSupport(h["a"]) == 4); + + t.insert(h["abc"],2); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + BEAST_EXPECT(t.tipSupport(h["ab"]) == 4); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 6); + BEAST_EXPECT(t.tipSupport(h["a"]) == 0); + BEAST_EXPECT(t.branchSupport(h["a"]) == 6); + + } + } + + void + testRemove() + { + using namespace csf; + // Not in trie + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + + BEAST_EXPECT(!t.remove(h["ab"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(!t.remove(h["a"])); + BEAST_EXPECT(t.checkInvariants()); + } + // In trie but with 0 tip support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + t.insert(h["abce"]); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + BEAST_EXPECT(!t.remove(h["abc"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + } + // In trie with > 1 tip support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"],2); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); + BEAST_EXPECT(t.remove(h["abc"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + + t.insert(h["abc"], 1); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); + BEAST_EXPECT(t.remove(h["abc"], 2)); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + + t.insert(h["abc"], 3); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 3); + BEAST_EXPECT(t.remove(h["abc"], 300)); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + + } + // In trie with = 1 tip support, no children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + + BEAST_EXPECT(t.tipSupport(h["ab"]) == 1); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); + + BEAST_EXPECT(t.remove(h["abc"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["ab"]) == 1); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 0); + } + // In trie with = 1 tip support, 1 child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abcd"]); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); + + BEAST_EXPECT(t.remove(h["abc"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); + } + // In trie with = 1 tip support, > 1 children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"]); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); + + BEAST_EXPECT(t.remove(h["abc"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); + } + + // In trie with = 1 tip support, parent compaction + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abd"]); + BEAST_EXPECT(t.checkInvariants()); + t.remove(h["ab"]); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abd"]) == 1); + BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); + + t.remove(h["abd"]); + BEAST_EXPECT(t.checkInvariants()); + + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); + + } + } + + void + testTipAndBranchSupport() + { + using namespace csf; + LedgerTrie t; + LedgerHistoryHelper h; + BEAST_EXPECT(t.tipSupport(h["a"]) == 0); + BEAST_EXPECT(t.tipSupport(h["axy"]) == 0); + BEAST_EXPECT(t.branchSupport(h["a"]) == 0); + BEAST_EXPECT(t.branchSupport(h["axy"]) == 0); + + t.insert(h["abc"]); + BEAST_EXPECT(t.tipSupport(h["a"]) == 0); + BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abcd"]) == 0); + BEAST_EXPECT(t.branchSupport(h["a"]) == 1); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abcd"]) == 0); + + t.insert(h["abe"]); + BEAST_EXPECT(t.tipSupport(h["a"]) == 0); + BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); + BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); + BEAST_EXPECT(t.tipSupport(h["abe"]) == 1); + + BEAST_EXPECT(t.branchSupport(h["a"]) == 2); + BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); + BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); + BEAST_EXPECT(t.branchSupport(h["abe"]) == 1); + } + + void + testGetPreferred() + { + using namespace csf; + // Empty + { + LedgerTrie t; + LedgerHistoryHelper h; + BEAST_EXPECT(t.getPreferred().second == Ledger::ID{0}); + } + // Single node no children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + } + // Single node smaller child support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + + t.insert(h["abc"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + } + // Single node larger child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"],2); + BEAST_EXPECT(t.getPreferred().second == h["abcd"].id()); + } + // Single node smaller children support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + + t.insert(h["abc"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + } + // Single node larger children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"],2); + t.insert(h["abce"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + t.insert(h["abcd"]); + BEAST_EXPECT(t.getPreferred().second == h["abcd"].id()); + } + // Tie-breaker by id + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"],2); + t.insert(h["abce"],2); + + BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); + BEAST_EXPECT(t.getPreferred().second == h["abce"].id()); + + t.insert(h["abcd"]); + BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); + BEAST_EXPECT(t.getPreferred().second == h["abcd"].id()); + } + + // Tie-breaker not needed + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"],2); + // abce only has a margin of 1, but it owns the tie-breaker + BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); + BEAST_EXPECT(t.getPreferred().second == h["abce"].id()); + + // Switch support from abce to abcd, tie-breaker now needed + t.remove(h["abce"]); + t.insert(h["abcd"]); + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + } + + // Single node larger grand child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"],2); + t.insert(h["abcde"],4); + BEAST_EXPECT(t.getPreferred().second == h["abcde"].id()); + } + + // Too much prefix support from competing branches + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcde"],2); + t.insert(h["abcfg"],2); + // 'de' and 'fg' are tied without 'abc' vote + BEAST_EXPECT(t.getPreferred().second == h["abc"].id()); + t.remove(h["abc"]); + t.insert(h["abcd"]); + // 'de' branch has 3 votes to 2, but not enough suport for 'e' + // since the node on 'd' and the 2 on 'fg' could go in a + // different direction + BEAST_EXPECT(t.getPreferred().second == h["abcd"].id()); + } + } + + void + testRootRelated() + { + using namespace csf; + // Since the root is a special node that breaks the no-single child + // invariant, do some tests that exercise it. + + LedgerTrie t; + LedgerHistoryHelper h; + BEAST_EXPECT(!t.remove(h[""])); + BEAST_EXPECT(t.branchSupport(h[""]) == 0); + BEAST_EXPECT(t.tipSupport(h[""]) == 0); + + t.insert(h["a"]); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.branchSupport(h[""]) == 1); + BEAST_EXPECT(t.tipSupport(h[""]) == 0); + + t.insert(h["e"]); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.branchSupport(h[""]) == 2); + BEAST_EXPECT(t.tipSupport(h[""]) == 0); + + BEAST_EXPECT(t.remove(h["e"])); + BEAST_EXPECT(t.checkInvariants()); + BEAST_EXPECT(t.branchSupport(h[""]) == 1); + BEAST_EXPECT(t.tipSupport(h[""]) == 0); + } + + void + testStress() + { + using namespace csf; + LedgerTrie t; + LedgerHistoryHelper h; + + // Test quasi-randomly add/remove supporting for different ledgers + // from a branching history. + + // Ledgers have sequence 1,2,3,4 + std::uint32_t const depth = 4; + // Each ledger has 4 possible children + std::uint32_t const width = 4; + + std::uint32_t const iterations = 10000; + + // Use explicit seed to have same results for CI + std::mt19937 gen{ 42 }; + std::uniform_int_distribution<> depthDist(0, depth-1); + std::uniform_int_distribution<> widthDist(0, width-1); + std::uniform_int_distribution<> flip(0, 1); + for(std::uint32_t i = 0; i < iterations; ++i) + { + // pick a random ledger history + std::string curr = ""; + char depth = depthDist(gen); + char offset = 0; + for(char d = 0; d < depth; ++d) + { + char a = offset + widthDist(gen); + curr += a; + offset = (a + 1) * width; + } + + // 50-50 to add remove + if(flip(gen) == 0) + t.insert(h[curr]); + else + t.remove(h[curr]); + if(!BEAST_EXPECT(t.checkInvariants())) + return; + } + } + + void + run() + { + testInsert(); + testRemove(); + testTipAndBranchSupport(); + testGetPreferred(); + testRootRelated(); + testStress(); + + } +}; + +BEAST_DEFINE_TESTSUITE(LedgerTrie, consensus, ripple); +} // namespace test +} // namespace ripple diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 6813201a247..1216c5607d9 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -555,7 +555,7 @@ struct Peer // Only send validation if the new ledger is compatible with our // fully validated ledger bool const isCompatible = - oracle.isAncestor(fullyValidatedLedger, newLedger); + newLedger.isAncestor(fullyValidatedLedger); if (runAsValidator && isCompatible) { @@ -691,7 +691,7 @@ struct Peer std::size_t const count = validations.numTrustedForLedger(ledger.id()); std::size_t const numTrustedPeers = trustGraph.graph().outDegree(this); quorum = static_cast(std::ceil(numTrustedPeers * 0.8)); - if (count >= quorum && oracle.isAncestor(fullyValidatedLedger, ledger)) + if (count >= quorum && ledger.isAncestor(fullyValidatedLedger)) { issue(FullyValidateLedger{ledger, fullyValidatedLedger}); fullyValidatedLedger = ledger; diff --git a/src/test/csf/impl/ledgers.cpp b/src/test/csf/impl/ledgers.cpp index eb8e9a7adab..1fffb902574 100644 --- a/src/test/csf/impl/ledgers.cpp +++ b/src/test/csf/impl/ledgers.cpp @@ -18,6 +18,7 @@ //============================================================================== #include #include +#include #include @@ -36,6 +37,53 @@ Ledger::getJson() const return res; } +bool +Ledger::isAncestor(Ledger const& ancestor) const +{ + if (ancestor.seq() < seq()) + return operator[](ancestor.seq()) == ancestor.id(); + return false; +} + +Ledger::ID +Ledger::operator[](Seq s) const +{ + if(s > seq()) + return {}; + if(s== seq()) + return id(); + return instance_->ancestors[static_cast(s)]; + +} + +Ledger::Seq +mismatch(Ledger const& a, Ledger const& b) +{ + using Seq = Ledger::Seq; + + // end is 1 past end of range + Seq start{0}; + Seq end = std::min(a.seq() + Seq{1}, b.seq() + Seq{1}); + + // Find mismatch in [start,end) + // Binary search + Seq count = end - start; + while(count > Seq{0}) + { + Seq step = count/Seq{2}; + Seq curr = start + step; + if(a[curr] == b[curr]) + { + // go to second half + start = ++curr; + count -= step + Seq{1}; + } + else + count = step; + } + return start; +} + LedgerOracle::LedgerOracle() { instances_.insert(InstanceEntry{Ledger::genesis, nextID()}); @@ -67,6 +115,8 @@ LedgerOracle::accept( next.parentCloseTime = parent.closeTime(); next.parentID = parent.id(); + next.ancestors.push_back(parent.id()); + auto it = instances_.left.find(next); if (it == instances_.left.end()) { @@ -88,19 +138,6 @@ LedgerOracle::lookup(Ledger::ID const & id) const } -bool -LedgerOracle::isAncestor(Ledger const & ancestor, Ledger const& descendant) const -{ - // The ancestor must have an earlier sequence number than the descendent - if(ancestor.seq() >= descendant.seq()) - return false; - - boost::optional current{descendant}; - while(current && current->seq() > ancestor.seq()) - current = lookup(current->parentID()); - return current && (current->id() == ancestor.id()); -} - std::size_t LedgerOracle::branches(std::set const & ledgers) const { @@ -121,7 +158,7 @@ LedgerOracle::branches(std::set const & ledgers) const bool const idxEarlier = tips[idx].seq() < ledger.seq(); Ledger const & earlier = idxEarlier ? tips[idx] : ledger; Ledger const & later = idxEarlier ? ledger : tips[idx] ; - if (isAncestor(earlier, later)) + if (later.isAncestor(earlier)) { tips[idx] = later; found = true; diff --git a/src/test/csf/ledgers.h b/src/test/csf/ledgers.h index 9e48d933144..6d706f1a769 100644 --- a/src/test/csf/ledgers.h +++ b/src/test/csf/ledgers.h @@ -94,6 +94,11 @@ class Ledger //! Parent ledger close time NetClock::time_point parentCloseTime; + //! IDs of this ledgers ancestors. Since each ledger already has unique + //! ancestors based on the parentID, this member is not needed foor any + //! of the operators below. + std::vector ancestors; + auto asTie() const { @@ -189,6 +194,21 @@ class Ledger return instance_->txs; } + /** Determine whether ancestor is really an ancestor of this ledger */ + bool + isAncestor(Ledger const& ancestor) const; + + /** Return the id of the ancestor with the given seq (if exists/known) + */ + ID + operator[](Seq seq) const; + + /** Return the sequence number of the first mismatching ancestor + */ + friend + Ledger::Seq + mismatch(Ledger const & a, Ledger const & o); + Json::Value getJson() const; friend bool @@ -238,9 +258,16 @@ class LedgerOracle NetClock::duration closeTimeResolution, NetClock::time_point const& consensusCloseTime); - /** Determine whether ancestor is really an ancestor of descendent */ - bool - isAncestor(Ledger const & ancestor, Ledger const& descendant) const; + Ledger + accept(Ledger const& curr, Tx tx) + { + using namespace std::chrono_literals; + return accept( + curr, + TxSetType{tx}, + curr.closeTimeResolution(), + curr.closeTime() + 1s); + } /** Determine the number of distinct branches for the set of ledgers. @@ -256,6 +283,57 @@ class LedgerOracle }; +/** Helper for writing unit tests with controlled ledger histories. + + This class allows clients to refer to distinct ledgers as strings, where + each character in the string indicates a unique ledger. It enforces the + uniqueness at runtime, but this simplifies creation of alternate ledger + histories, e.g. + + HistoryHelper hh; + hh["a"] + hh["ab"] + hh["ac"] + hh["abd"] + + Creates a history like + b - d + / + a - c + +*/ +struct LedgerHistoryHelper +{ + csf::LedgerOracle oracle; + csf::Tx::ID nextTx{0}; + std::unordered_map ledgers; + std::set seen; + + LedgerHistoryHelper() + { + ledgers[""] = csf::Ledger{}; + } + + /** Get or create the ledger with the given string history. + + Creates an necessary intermediate ledgers, but asserts if + a letter is re-used (e.g. "abc" then "adc" would assert) + */ + csf::Ledger const& operator[](std::string const& s) + { + auto it = ledgers.find(s); + if (it != ledgers.end()) + return it->second; + + // enforce that the new suffix has never been seen + assert(seen.emplace(s.back()).second); + + csf::Ledger const& parent = (*this)[s.substr(0, s.size() - 1)]; + return ledgers.emplace(s, oracle.accept(parent, ++nextTx)) + .first->second; + } +}; + } // csf } // test } // ripple diff --git a/src/test/unity/consensus_test_unity.cpp b/src/test/unity/consensus_test_unity.cpp index 1a9a347a336..f73a01d919e 100644 --- a/src/test/unity/consensus_test_unity.cpp +++ b/src/test/unity/consensus_test_unity.cpp @@ -21,5 +21,6 @@ #include #include #include +#include #include #include