diff --git a/API-CHANGELOG.md b/API-CHANGELOG.md index 968a03c0784..e5d30b3a3d6 100644 --- a/API-CHANGELOG.md +++ b/API-CHANGELOG.md @@ -134,6 +134,38 @@ Currently (prior to the release of 2.0), it is available as a "beta" version, me Since `api_version` 2 is in "beta", breaking changes to V2 can currently be made at any time. +#### Removed methods + +In API version 2, the following methods are no longer available: + +- `tx_history` - Instead, use other methods such as `account_tx` or `ledger` with the `transactions` field set to `true`. +- `ledger_header` - Instead, use the `ledger` method. + +#### Modifications to JSON transaction element in V2 + +In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions: + +- JSON transaction element is named `tx_json` +- Binary transaction element is named `tx_blob` +- JSON transaction metadata element is named `meta` +- Binary transaction metadata element is named `meta_blob` + +Additionally, these elements are now consistently available next to `tx_json` (i.e. sibling elements), where possible: + +- `hash` - Transaction ID. This data was stored inside transaction output in API version 1, but in API version 2 is a sibling element. +- `ledger_index` - Ledger index (only set on validated ledgers) +- `ledger_hash` - Ledger hash (only set on closed or validated ledgers) +- `close_time_iso` - Ledger close time expressed in ISO 8601 time format (only set on validated ledgers) +- `validated` - Bool element set to `true` if the transaction is in a validated ledger, otherwise `false` + +This change affects the following methods: + +- `tx` - Transaction data moved into element `tx_json` (was inline inside `result`) or, if binary output was requested, moved from `tx` to `tx_blob`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements +- `account_tx` - Renamed transaction element from `tx` to `tx_json`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements +- `transaction_entry` - Renamed transaction metadata element from `metadata` to `meta`. Changed location of `hash` and added new elements +- `subscribe` - Renamed transaction element from `transaction` to `tx_json`. Changed location of `hash` and added new elements +- `sign`, `sign_for`, `submit` and `submit_multisigned` - Changed location of `hash` element. + #### Modifications to account_info response in V2 - `signer_lists` is returned in the root of the response. Previously, in API version 1, it was nested under `account_data`. (https://github.com/XRPLF/rippled/pull/3770) diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 4758d0791e8..269c107ca9e 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -1062,6 +1062,7 @@ if (tests) src/test/rpc/KeyGeneration_test.cpp src/test/rpc/LedgerClosed_test.cpp src/test/rpc/LedgerData_test.cpp + src/test/rpc/LedgerHeader_test.cpp src/test/rpc/LedgerRPC_test.cpp src/test/rpc/LedgerRequestRPC_test.cpp src/test/rpc/ManifestRPC_test.cpp @@ -1085,6 +1086,7 @@ if (tests) src/test/rpc/ValidatorInfo_test.cpp src/test/rpc/ValidatorRPC_test.cpp src/test/rpc/Version_test.cpp + src/test/rpc/Handler_test.cpp #[===============================[ test sources: subdir: server diff --git a/docs/Dockerfile b/docs/Dockerfile deleted file mode 100644 index d716ca21315..00000000000 --- a/docs/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt -y update -RUN apt -y upgrade -RUN apt -y install build-essential g++ git libbz2-dev wget python-dev -RUN apt -y install cmake flex bison graphviz graphviz-dev libicu-dev -RUN apt -y install jarwrapper java-common - -RUN cd /tmp -ENV CM_INSTALLER=cmake-3.10.0-rc3-Linux-x86_64.sh -ENV CM_VER_DIR=/opt/local/cmake-3.10.0 -RUN cd /tmp && wget https://cmake.org/files/v3.10/$CM_INSTALLER && chmod a+x $CM_INSTALLER -RUN mkdir -p $CM_VER_DIR -RUN ln -s $CM_VER_DIR /opt/local/cmake -RUN /tmp/$CM_INSTALLER --prefix=$CM_VER_DIR --exclude-subdir -RUN rm -f /tmp/$CM_INSTALLER - -RUN cd /tmp && wget https://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.14.src.tar.gz -RUN cd /tmp && tar xvf doxygen-1.8.14.src.tar.gz -RUN mkdir -p /tmp/doxygen-1.8.14/build -RUN cd /tmp/doxygen-1.8.14/build && /opt/local/cmake/bin/cmake -G "Unix Makefiles" .. -RUN cd /tmp/doxygen-1.8.14/build && make -j2 -RUN cd /tmp/doxygen-1.8.14/build && make install -RUN rm -f /tmp/doxygen-1.8.14.src.tar.gz -RUN rm -rf /tmp/doxygen-1.8.14 - -RUN mkdir -p /opt/plantuml -RUN wget -O /opt/plantuml/plantuml.jar http://sourceforge.net/projects/plantuml/files/plantuml.jar/download -ENV DOXYGEN_PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar - -ENV DOXYGEN_OUTPUT_DIRECTORY=html -CMD cd /opt/rippled && doxygen docs/Doxyfile diff --git a/src/ripple/app/ledger/LedgerMaster.h b/src/ripple/app/ledger/LedgerMaster.h index 26738844536..e2ca3039935 100644 --- a/src/ripple/app/ledger/LedgerMaster.h +++ b/src/ripple/app/ledger/LedgerMaster.h @@ -215,6 +215,8 @@ class LedgerMaster : public AbstractFetchPackContainer void clearLedger(std::uint32_t seq); bool + isValidated(ReadView const& ledger); + bool getValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal); bool getFullValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal); diff --git a/src/ripple/app/ledger/LedgerToJson.h b/src/ripple/app/ledger/LedgerToJson.h index f658583885f..78947ca91d1 100644 --- a/src/ripple/app/ledger/LedgerToJson.h +++ b/src/ripple/app/ledger/LedgerToJson.h @@ -21,8 +21,10 @@ #define RIPPLE_APP_LEDGER_LEDGERTOJSON_H_INCLUDED #include +#include #include #include +#include #include #include #include @@ -41,6 +43,8 @@ struct LedgerFill LedgerEntryType t = ltANY) : ledger(l), options(o), txQueue(std::move(q)), type(t), context(ctx) { + if (context) + closeTime = context->ledgerMaster.getCloseTimeBySeq(ledger.seq()); } enum Options { @@ -58,6 +62,7 @@ struct LedgerFill std::vector txQueue; LedgerEntryType type; RPC::Context* context; + std::optional closeTime; }; /** Given a Ledger and options, fill a Json::Object or Json::Value with a diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp index 050e2f3ef3d..857c0efcc28 100644 --- a/src/ripple/app/ledger/impl/LedgerMaster.cpp +++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp @@ -603,6 +603,54 @@ LedgerMaster::clearLedger(std::uint32_t seq) mCompleteLedgers.erase(seq); } +bool +LedgerMaster::isValidated(ReadView const& ledger) +{ + if (app_.config().reporting()) + return true; // Reporting mode only supports validated ledger + + if (ledger.open()) + return false; + + if (ledger.info().validated) + return true; + + auto const seq = ledger.info().seq; + try + { + // Use the skip list in the last validated ledger to see if ledger + // comes before the last validated ledger (and thus has been + // validated). + auto const hash = walkHashBySeq(seq, InboundLedger::Reason::GENERIC); + + if (!hash || ledger.info().hash != *hash) + { + // This ledger's hash is not the hash of the validated ledger + if (hash) + { + assert(hash->isNonZero()); + uint256 valHash = + app_.getRelationalDatabase().getHashByIndex(seq); + if (valHash == ledger.info().hash) + { + // SQL database doesn't match ledger chain + clearLedger(seq); + } + } + return false; + } + } + catch (SHAMapMissingNode const& mn) + { + JLOG(m_journal.warn()) << "Ledger #" << seq << ": " << mn.what(); + return false; + } + + // Mark ledger as validated to save time if we see it again. + ledger.info().validated = true; + return true; +} + // returns Ledgers we have all the nodes for bool LedgerMaster::getFullValidatedRange( diff --git a/src/ripple/app/ledger/impl/LedgerToJson.cpp b/src/ripple/app/ledger/impl/LedgerToJson.cpp index 55123ba2362..d22cc7cd487 100644 --- a/src/ripple/app/ledger/impl/LedgerToJson.cpp +++ b/src/ripple/app/ledger/impl/LedgerToJson.cpp @@ -17,12 +17,14 @@ */ //============================================================================== +#include #include #include #include #include #include #include +#include #include #include @@ -83,6 +85,7 @@ fillJson(Object& json, bool closed, LedgerInfo const& info, bool bFull) json[jss::close_time_human] = to_string(info.closeTime); if (!getCloseAgree(info)) json[jss::close_time_estimated] = true; + json[jss::close_time_iso] = to_string_iso(info.closeTime); } } @@ -118,8 +121,48 @@ fillJsonTx( if (bBinary) { txJson[jss::tx_blob] = serializeHex(*txn); + if (fill.context->apiVersion > 1) + txJson[jss::hash] = to_string(txn->getTransactionID()); + + auto const json_meta = + (fill.context->apiVersion > 1 ? jss::meta_blob : jss::meta); if (stMeta) - txJson[jss::meta] = serializeHex(*stMeta); + txJson[json_meta] = serializeHex(*stMeta); + } + else if (fill.context->apiVersion > 1) + { + copyFrom( + txJson[jss::tx_json], + txn->getJson(JsonOptions::disable_API_prior_V2, false)); + txJson[jss::hash] = to_string(txn->getTransactionID()); + RPC::insertDeliverMax( + txJson[jss::tx_json], txnType, fill.context->apiVersion); + + if (stMeta) + { + txJson[jss::meta] = stMeta->getJson(JsonOptions::none); + + // If applicable, insert delivered amount + if (txnType == ttPAYMENT || txnType == ttCHECK_CASH) + RPC::insertDeliveredAmount( + txJson[jss::meta], + fill.ledger, + txn, + {txn->getTransactionID(), fill.ledger.seq(), *stMeta}); + } + + if (!fill.ledger.open()) + txJson[jss::ledger_hash] = to_string(fill.ledger.info().hash); + + const bool validated = + fill.context->ledgerMaster.isValidated(fill.ledger); + txJson[jss::validated] = validated; + if (validated) + { + txJson[jss::ledger_index] = to_string(fill.ledger.seq()); + if (fill.closeTime) + txJson[jss::close_time_iso] = to_string_iso(*fill.closeTime); + } } else { @@ -254,7 +297,11 @@ fillJsonQueue(Object& json, LedgerFill const& fill) if (tx.lastResult) txJson["last_result"] = transToken(*tx.lastResult); - txJson[jss::tx] = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr); + auto&& temp = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr); + if (fill.context->apiVersion > 1) + copyFrom(txJson, temp); + else + copyFrom(txJson[jss::tx], temp); } } diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 08ba296b271..8fcbb8a971c 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -1333,9 +1333,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) << "Invalid entry in [" << SECTION_VALIDATOR_LIST_SITES << "]"; return false; } - - // Tell the AmendmentTable who the trusted validators are. - m_amendmentTable->trustChanged(validators_->getQuorumKeys().second); } //---------------------------------------------------------------------- // diff --git a/src/ripple/app/misc/AmendmentTable.h b/src/ripple/app/misc/AmendmentTable.h index d7f7c8b26bb..10396d8591f 100644 --- a/src/ripple/app/misc/AmendmentTable.h +++ b/src/ripple/app/misc/AmendmentTable.h @@ -111,10 +111,6 @@ class AmendmentTable std::set const& enabled, majorityAmendments_t const& majority) = 0; - // Called when the set of trusted validators changes. - virtual void - trustChanged(hash_set const& allTrusted) = 0; - // Called by the consensus code when we need to // inject pseudo-transactions virtual std::map diff --git a/src/ripple/app/misc/FeeEscalation.md b/src/ripple/app/misc/FeeEscalation.md index 30f4dc2784e..b86f8dab945 100644 --- a/src/ripple/app/misc/FeeEscalation.md +++ b/src/ripple/app/misc/FeeEscalation.md @@ -190,11 +190,27 @@ lower) fee to get into the same position as a reference transaction. ### Consensus Health -For consensus to be considered healthy, the consensus process must take -less than 5 seconds. This time limit was chosen based on observed past -behavior of the network. Note that this is not necessarily the time between -ledger closings, as consensus usually starts some amount of time after -a ledger opens. +For consensus to be considered healthy, the peers on the network +should largely remain in sync with one another. It is particularly +important for the validators to remain in sync, because that is required +for participation in consensus. However, the network tolerates some +validators being out of sync. Fundamentally, network health is a +function of validators reaching consensus on sets of recently submitted +transactions. + +Another factor to consider is +the duration of the consensus process itself. This generally takes +under 5 seconds on the main network under low volume. This is based on +historical observations. However factors such as transaction volume +can increase consensus duration. This is because rippled performs +more work as transaction volume increases. Under sufficient load this +tends to increase consensus duration. It's possible that relatively high +consensus duration indicates a problem, but it is not appropriate to +conclude so without investigation. The upper limit for consensus +duration should be roughly 20 seconds. That is far above the normal. +If the network takes this long to close ledgers, then it is almost +certain that there is a problem with the network. This circumstance +often coincides with new ledgers with zero transactions. ### Other Constants diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index a431b5562d3..b20ef5d37f7 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -63,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,7 @@ #include #include +#include #include #include #include @@ -1853,12 +1855,7 @@ NetworkOPsImp::beginConsensus(uint256 const& networkClosed) app_.getHashRouter()); if (!changes.added.empty() || !changes.removed.empty()) - { app_.getValidations().trustChanged(changes.added, changes.removed); - // Update the AmendmentTable so it tracks the current validators. - app_.getAmendmentTable().trustChanged( - app_.validators().getQuorumKeys().second); - } mConsensus.startRound( app_.timeKeeper().closeTime(), @@ -3101,7 +3098,11 @@ NetworkOPsImp::transJson( transResultInfo(result, sToken, sHuman); jvObj[jss::type] = "transaction"; - jvObj[jss::transaction] = transaction->getJson(JsonOptions::none); + // NOTE jvObj which is not a finished object for either API version. After + // it's populated, we need to finish it for a specific API version. This is + // done in a loop, near the end of this function. + jvObj[jss::transaction] = + transaction->getJson(JsonOptions::disable_API_prior_V2, false); if (meta) { @@ -3110,13 +3111,16 @@ NetworkOPsImp::transJson( jvObj[jss::meta], *ledger, transaction, meta->get()); } + if (!ledger->open()) + jvObj[jss::ledger_hash] = to_string(ledger->info().hash); + if (validated) { jvObj[jss::ledger_index] = ledger->info().seq; - jvObj[jss::ledger_hash] = to_string(ledger->info().hash); jvObj[jss::transaction][jss::date] = ledger->info().closeTime.time_since_epoch().count(); jvObj[jss::validated] = true; + jvObj[jss::close_time_iso] = to_string_iso(ledger->info().closeTime); // WRITEME: Put the account next seq here } @@ -3149,6 +3153,7 @@ NetworkOPsImp::transJson( } } + std::string const hash = to_string(transaction->getTransactionID()); MultiApiJson multiObj({jvObj, jvObj}); // Minimum supported API version must match index 0 in MultiApiJson static_assert(apiVersionSelector(RPC::apiMinimumSupportedVersion)() == 0); @@ -3165,11 +3170,21 @@ NetworkOPsImp::transJson( assert(index < MultiApiJson::size); if (index != lastIndex) { - RPC::insertDeliverMax( - multiObj.val[index][jss::transaction], - transaction->getTxnType(), - apiVersion); lastIndex = index; + + Json::Value& jvTx = multiObj.val[index]; + RPC::insertDeliverMax( + jvTx[jss::transaction], transaction->getTxnType(), apiVersion); + + if (apiVersion > 1) + { + jvTx[jss::tx_json] = jvTx.removeMember(jss::transaction); + jvTx[jss::hash] = hash; + } + else + { + jvTx[jss::transaction][jss::hash] = hash; + } } } diff --git a/src/ripple/app/misc/Transaction.h b/src/ripple/app/misc/Transaction.h index 07802becfeb..36815ba0aa0 100644 --- a/src/ripple/app/misc/Transaction.h +++ b/src/ripple/app/misc/Transaction.h @@ -24,10 +24,11 @@ #include #include #include +#include #include #include #include -#include + #include #include @@ -99,13 +100,13 @@ class Transaction : public std::enable_shared_from_this, LedgerIndex getLedger() const { - return mInLedger; + return mLedgerIndex; } bool isValidated() const { - return mInLedger != 0; + return mLedgerIndex != 0; } TransStatus @@ -138,7 +139,7 @@ class Transaction : public std::enable_shared_from_this, void setLedger(LedgerIndex ledger) { - mInLedger = ledger; + mLedgerIndex = ledger; } /** @@ -386,7 +387,7 @@ class Transaction : public std::enable_shared_from_this, uint256 mTransactionID; - LedgerIndex mInLedger = 0; + LedgerIndex mLedgerIndex = 0; TransStatus mStatus = INVALID; TER mResult = temUNCERTAIN; bool mApplying = false; diff --git a/src/ripple/app/misc/impl/AmendmentTable.cpp b/src/ripple/app/misc/impl/AmendmentTable.cpp index 8f4f0321992..6f9ea86fa6c 100644 --- a/src/ripple/app/misc/impl/AmendmentTable.cpp +++ b/src/ripple/app/misc/impl/AmendmentTable.cpp @@ -67,155 +67,6 @@ parseSection(Section const& section) return names; } -/** TrustedVotes records the most recent votes from trusted validators. - We keep a record in an effort to avoid "flapping" while amendment voting - is in process. - - If a trusted validator loses synchronization near a flag ledger their - amendment votes may be lost during that round. If the validator is a - bit flaky, then this can cause an amendment to appear to repeatedly - gain and lose support. - - TrustedVotes addresses the problem by holding on to the last vote seen - from every trusted validator. So if any given validator is off line near - a flag ledger we can assume that they did not change their vote. - - If we haven't seen any STValidations from a validator for several hours we - lose confidence that the validator hasn't changed their position. So - there's a timeout. We remove upVotes if they haven't been updated in - several hours. -*/ -class TrustedVotes -{ -private: - static constexpr NetClock::time_point maxTimeout = - NetClock::time_point::max(); - - // Associates each trusted validator with the last votes we saw from them - // and an expiration for that record. - struct UpvotesAndTimeout - { - std::vector upVotes; - NetClock::time_point timeout = maxTimeout; - }; - hash_map recordedVotes_; - -public: - TrustedVotes() = default; - TrustedVotes(TrustedVotes const& rhs) = delete; - TrustedVotes& - operator=(TrustedVotes const& rhs) = delete; - - // Called when the list of trusted validators changes. - // - // Call with AmendmentTable::mutex_ locked. - void - trustChanged( - hash_set const& allTrusted, - std::lock_guard const& lock) - { - decltype(recordedVotes_) newRecordedVotes; - newRecordedVotes.reserve(allTrusted.size()); - - // Make sure every PublicKey in allTrusted is represented in - // recordedVotes_. Also make sure recordedVotes_ contains - // no additional PublicKeys. - for (auto& trusted : allTrusted) - { - if (recordedVotes_.contains(trusted)) - { - // Preserve this validator's previously saved voting state. - newRecordedVotes.insert(recordedVotes_.extract(trusted)); - } - else - { - // New validators have a starting position of no on everything. - // Add the entry with an empty vector and maxTimeout. - newRecordedVotes[trusted]; - } - } - // The votes of any no-longer-trusted validators will be destroyed - // when changedTrustedVotes goes out of scope. - recordedVotes_.swap(newRecordedVotes); - } - - // Called when we receive the latest votes. - // - // Call with AmendmentTable::mutex_ locked. - void - recordVotes( - Rules const& rules, - std::vector> const& valSet, - NetClock::time_point const closeTime, - std::lock_guard const& lock) - { - // When we get an STValidation we save the upVotes it contains, but - // we also set an expiration for those upVotes. The following constant - // controls the timeout. - // - // There really is no "best" timeout to choose for when we finally - // lose confidence that we know how a validator is voting. But part - // of the point of recording validator votes is to avoid flapping of - // amendment votes. A 24h timeout says that we will change the local - // record of a validator's vote to "no" 24h after the last vote seen - // from that validator. So flapping due to that validator being off - // line will happen less frequently than every 24 hours. - using namespace std::chrono_literals; - static constexpr NetClock::duration expiresAfter = 24h; - - // Walk all validations and replace previous votes from trusted - // validators with these newest votes. - for (auto const& val : valSet) - { - // If this validation comes from one of our trusted validators... - if (auto const iter = recordedVotes_.find(val->getSignerPublic()); - iter != recordedVotes_.end()) - { - iter->second.timeout = closeTime + expiresAfter; - if (val->isFieldPresent(sfAmendments)) - { - auto const& choices = val->getFieldV256(sfAmendments); - iter->second.upVotes.assign(choices.begin(), choices.end()); - } - else - { - // This validator does not upVote any amendments right now. - iter->second.upVotes.clear(); - } - } - } - - // Now remove any expired records from recordedVotes_. - std::for_each( - recordedVotes_.begin(), - recordedVotes_.end(), - [&closeTime](decltype(recordedVotes_)::value_type& votes) { - if (closeTime > votes.second.timeout) - { - votes.second.timeout = maxTimeout; - votes.second.upVotes.clear(); - } - }); - } - - // Return the information needed by AmendmentSet to determine votes. - // - // Call with AmendmentTable::mutex_ locked. - [[nodiscard]] std::pair> - getVotes(Rules const& rules, std::lock_guard const& lock) const - { - hash_map ret; - for (auto& validatorVotes : recordedVotes_) - { - for (uint256 const& amendment : validatorVotes.second.upVotes) - { - ret[amendment] += 1; - } - } - return {recordedVotes_.size(), ret}; - } -}; - /** Current state of an amendment. Tells if a amendment is supported, enabled or vetoed. A vetoed amendment means the node will never announce its support. @@ -253,9 +104,30 @@ class AmendmentSet // number of votes needed int threshold_ = 0; - void - computeThreshold(int trustedValidations, Rules const& rules) +public: + AmendmentSet( + Rules const& rules, + std::vector> const& valSet) + : rules_(rules) { + // process validations for ledger before flag ledger + for (auto const& val : valSet) + { + if (val->isTrusted()) + { + if (val->isFieldPresent(sfAmendments)) + { + auto const choices = val->getFieldV256(sfAmendments); + std::for_each( + choices.begin(), + choices.end(), + [&](auto const& amendment) { ++votes_[amendment]; }); + } + + ++trustedValidations_; + } + } + threshold_ = !rules_.enabled(fixAmendmentMajorityCalc) ? std::max( 1L, @@ -271,22 +143,6 @@ class AmendmentSet postFixAmendmentMajorityCalcThreshold.den)); } -public: - AmendmentSet( - Rules const& rules, - TrustedVotes const& trustedVotes, - std::lock_guard const& lock) - : rules_(rules) - { - // process validations for ledger before flag ledger. - auto [trustedCount, newVotes] = trustedVotes.getVotes(rules, lock); - - trustedValidations_ = trustedCount; - votes_.swap(newVotes); - - computeThreshold(trustedValidations_, rules); - } - bool passes(uint256 const& amendment) const { @@ -347,9 +203,6 @@ class AmendmentTableImpl final : public AmendmentTable hash_map amendmentMap_; std::uint32_t lastUpdateSeq_; - // Record of the last votes seen from trusted validators. - TrustedVotes previousTrustedVotes_; - // Time that an amendment must hold a majority for std::chrono::seconds const majorityTime_; @@ -441,9 +294,6 @@ class AmendmentTableImpl final : public AmendmentTable std::set const& enabled, majorityAmendments_t const& majority) override; - void - trustChanged(hash_set const& allTrusted) override; - std::vector doValidation(std::set const& enabledAmendments) const override; @@ -783,14 +633,8 @@ AmendmentTableImpl::doVoting( << ": " << enabledAmendments.size() << ", " << majorityAmendments.size() << ", " << valSet.size(); - std::lock_guard lock(mutex_); - - // Keep a record of the votes we received. - previousTrustedVotes_.recordVotes(rules, valSet, closeTime, lock); + auto vote = std::make_unique(rules, valSet); - // Tally the most recent votes. - auto vote = - std::make_unique(rules, previousTrustedVotes_, lock); JLOG(j_.debug()) << "Received " << vote->trustedValidations() << " trusted validations, threshold is: " << vote->threshold(); @@ -799,6 +643,8 @@ AmendmentTableImpl::doVoting( // the value of the flags in the pseudo-transaction std::map actions; + std::lock_guard lock(mutex_); + // process all amendments we know of for (auto const& entry : amendmentMap_) { @@ -894,13 +740,6 @@ AmendmentTableImpl::doValidatedLedger( firstUnsupportedExpected_ = *firstUnsupportedExpected_ + majorityTime_; } -void -AmendmentTableImpl::trustChanged(hash_set const& allTrusted) -{ - std::lock_guard lock(mutex_); - previousTrustedVotes_.trustChanged(allTrusted, lock); -} - void AmendmentTableImpl::injectJson( Json::Value& v, diff --git a/src/ripple/app/misc/impl/Transaction.cpp b/src/ripple/app/misc/impl/Transaction.cpp index 9adef982d01..c38a6b7438f 100644 --- a/src/ripple/app/misc/impl/Transaction.cpp +++ b/src/ripple/app/misc/impl/Transaction.cpp @@ -62,7 +62,7 @@ void Transaction::setStatus(TransStatus ts, std::uint32_t lseq) { mStatus = ts; - mInLedger = lseq; + mLedgerIndex = lseq; } TransStatus @@ -167,16 +167,25 @@ Transaction::load( Json::Value Transaction::getJson(JsonOptions options, bool binary) const { - Json::Value ret(mTransaction->getJson(JsonOptions::none, binary)); + // Note, we explicitly suppress `include_date` option here + Json::Value ret( + mTransaction->getJson(options & ~JsonOptions::include_date, binary)); - if (mInLedger) + if (mLedgerIndex) { - ret[jss::inLedger] = mInLedger; // Deprecated. - ret[jss::ledger_index] = mInLedger; + if (!(options & JsonOptions::disable_API_prior_V2)) + { + // Behaviour before API version 2 + ret[jss::inLedger] = mLedgerIndex; + } + + // TODO: disable_API_prior_V3 to disable output of both `date` and + // `ledger_index` elements (taking precedence over include_date) + ret[jss::ledger_index] = mLedgerIndex; - if (options == JsonOptions::include_date) + if (options & JsonOptions::include_date) { - auto ct = mApp.getLedgerMaster().getCloseTimeBySeq(mInLedger); + auto ct = mApp.getLedgerMaster().getCloseTimeBySeq(mLedgerIndex); if (ct) ret[jss::date] = ct->time_since_epoch().count(); } diff --git a/src/ripple/app/paths/Flow.cpp b/src/ripple/app/paths/Flow.cpp index 3d060fdc6bd..83379d34e79 100644 --- a/src/ripple/app/paths/Flow.cpp +++ b/src/ripple/app/paths/Flow.cpp @@ -65,7 +65,7 @@ flow( bool defaultPaths, bool partialPayment, bool ownerPaysTransferFee, - bool offerCrossing, + OfferCrossing offerCrossing, std::optional const& limitQuality, std::optional const& sendMax, beast::Journal j, diff --git a/src/ripple/app/paths/Flow.h b/src/ripple/app/paths/Flow.h index b692c3bdf07..deafd1c7716 100644 --- a/src/ripple/app/paths/Flow.h +++ b/src/ripple/app/paths/Flow.h @@ -44,7 +44,7 @@ struct FlowDebugInfo; @param partialPayment If the payment cannot deliver the entire requested amount, deliver as much as possible, given the constraints @param ownerPaysTransferFee If true then owner, not sender, pays fee - @param offerCrossing If true then flow is executing offer crossing, not + @param offerCrossing If Yes or Sell then flow is executing offer crossing, not payments @param limitQuality Do not use liquidity below this quality threshold @param sendMax Do not spend more than this amount @@ -62,7 +62,7 @@ flow( bool defaultPaths, bool partialPayment, bool ownerPaysTransferFee, - bool offerCrossing, + OfferCrossing offerCrossing, std::optional const& limitQuality, std::optional const& sendMax, beast::Journal j, diff --git a/src/ripple/app/paths/RippleCalc.cpp b/src/ripple/app/paths/RippleCalc.cpp index 6feb276c625..87ef694fa58 100644 --- a/src/ripple/app/paths/RippleCalc.cpp +++ b/src/ripple/app/paths/RippleCalc.cpp @@ -106,7 +106,7 @@ RippleCalc::rippleCalculate( defaultPaths, partialPayment, ownerPaysTransferFee, - /* offerCrossing */ false, + OfferCrossing::no, limitQuality, sendMax, j, diff --git a/src/ripple/app/paths/impl/PaySteps.cpp b/src/ripple/app/paths/impl/PaySteps.cpp index 81c358a2bbc..b96d6ee57b2 100644 --- a/src/ripple/app/paths/impl/PaySteps.cpp +++ b/src/ripple/app/paths/impl/PaySteps.cpp @@ -141,7 +141,7 @@ toStrand( std::optional const& sendMaxIssue, STPath const& path, bool ownerPaysTransferFee, - bool offerCrossing, + OfferCrossing offerCrossing, AMMContext& ammContext, beast::Journal j) { @@ -475,7 +475,7 @@ toStrands( STPathSet const& paths, bool addDefaultPath, bool ownerPaysTransferFee, - bool offerCrossing, + OfferCrossing offerCrossing, AMMContext& ammContext, beast::Journal j) { @@ -588,7 +588,7 @@ StrandContext::StrandContext( std::optional const& limitQuality_, bool isLast_, bool ownerPaysTransferFee_, - bool offerCrossing_, + OfferCrossing offerCrossing_, bool isDefaultPath_, std::array, 2>& seenDirectIssues_, boost::container::flat_set& seenBookOuts_, diff --git a/src/ripple/app/paths/impl/Steps.h b/src/ripple/app/paths/impl/Steps.h index 35c465f18be..1ae2273929d 100644 --- a/src/ripple/app/paths/impl/Steps.h +++ b/src/ripple/app/paths/impl/Steps.h @@ -39,6 +39,7 @@ class AMMContext; enum class DebtDirection { issues, redeems }; enum class QualityDirection { in, out }; enum class StrandDirection { forward, reverse }; +enum OfferCrossing { no = 0, yes = 1, sell = 2 }; inline bool redeems(DebtDirection dir) @@ -398,7 +399,7 @@ toStrand( std::optional const& sendMaxIssue, STPath const& path, bool ownerPaysTransferFee, - bool offerCrossing, + OfferCrossing offerCrossing, AMMContext& ammContext, beast::Journal j); @@ -438,7 +439,7 @@ toStrands( STPathSet const& paths, bool addDefaultPath, bool ownerPaysTransferFee, - bool offerCrossing, + OfferCrossing offerCrossing, AMMContext& ammContext, beast::Journal j); @@ -531,9 +532,10 @@ struct StrandContext bool const isFirst; ///< true if Step is first in Strand bool const isLast = false; ///< true if Step is last in Strand bool const ownerPaysTransferFee; ///< true if owner, not sender, pays fee - bool const offerCrossing; ///< true if offer crossing, not payment - bool const isDefaultPath; ///< true if Strand is default path - size_t const strandSize; ///< Length of Strand + OfferCrossing const + offerCrossing; ///< Yes/Sell if offer crossing, not payment + bool const isDefaultPath; ///< true if Strand is default path + size_t const strandSize; ///< Length of Strand /** The previous step in the strand. Needed to check the no ripple constraint */ @@ -563,7 +565,7 @@ struct StrandContext std::optional const& limitQuality_, bool isLast_, bool ownerPaysTransferFee_, - bool offerCrossing_, + OfferCrossing offerCrossing_, bool isDefaultPath_, std::array, 2>& seenDirectIssues_, ///< For detecting currency loops diff --git a/src/ripple/app/paths/impl/StrandFlow.h b/src/ripple/app/paths/impl/StrandFlow.h index 5e04ef354a0..7817251560f 100644 --- a/src/ripple/app/paths/impl/StrandFlow.h +++ b/src/ripple/app/paths/impl/StrandFlow.h @@ -557,7 +557,7 @@ flow( std::vector const& strands, TOutAmt const& outReq, bool partialPayment, - bool offerCrossing, + OfferCrossing offerCrossing, std::optional const& limitQuality, std::optional const& sendMaxST, beast::Journal j, @@ -817,6 +817,19 @@ flow( JLOG(j.trace()) << "Total flow: in: " << to_string(actualIn) << " out: " << to_string(actualOut); + /* flowCross doesn't handle offer crossing with tfFillOrKill flag correctly. + * 1. If tfFillOrKill is set then the owner must receive the full + * TakerPays. We reverse pays and gets because during crossing + * we are taking, therefore the owner must deliver the full TakerPays and + * the entire TakerGets doesn't have to be spent. + * Pre-fixFillOrKill amendment code fails if the entire TakerGets + * is not spent. fixFillOrKill addresses this issue. + * 2. If tfSell is also set then the owner must spend the entire TakerGets + * even if it means obtaining more than TakerPays. Since the pays and gets + * are reversed, the owner must send the entire TakerGets. + */ + bool const fillOrKillEnabled = baseView.rules().enabled(fixFillOrKill); + if (actualOut != outReq) { if (actualOut > outReq) @@ -827,8 +840,14 @@ flow( if (!partialPayment) { // If we're offerCrossing a !partialPayment, then we're - // handling tfFillOrKill. That case is handled below; not here. - if (!offerCrossing) + // handling tfFillOrKill. + // Pre-fixFillOrKill amendment: + // That case is handled below; not here. + // fixFillOrKill amendment: + // That case is handled here if tfSell is also not set; i.e, + // case 1. + if (!offerCrossing || + (fillOrKillEnabled && offerCrossing != OfferCrossing::sell)) return { tecPATH_PARTIAL, actualIn, @@ -840,11 +859,17 @@ flow( return {tecPATH_DRY, std::move(ofrsToRmOnFail)}; } } - if (offerCrossing && !partialPayment) + if (offerCrossing && + (!partialPayment && + (!fillOrKillEnabled || offerCrossing == OfferCrossing::sell))) { // If we're offer crossing and partialPayment is *not* true, then // we're handling a FillOrKill offer. In this case remainingIn must // be zero (all funds must be consumed) or else we kill the offer. + // Pre-fixFillOrKill amendment: + // Handles both cases 1. and 2. + // fixFillOrKill amendment: + // Handles 2. 1. is handled above and falls through for tfSell. assert(remainingIn); if (remainingIn && *remainingIn != beast::zero) return { diff --git a/src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp b/src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp index 5ee4ce5519d..c57dee30610 100644 --- a/src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp +++ b/src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp @@ -175,8 +175,6 @@ loadLedgerInfos( "total_coins, closing_time, prev_closing_time, close_time_res, " "close_flags, ledger_seq FROM ledgers "; - uint32_t expNumResults = 1; - if (auto ledgerSeq = std::get_if(&whichLedger)) { sql << "WHERE ledger_seq = " + std::to_string(*ledgerSeq); @@ -189,8 +187,6 @@ loadLedgerInfos( auto minAndMax = std::get_if>(&whichLedger)) { - expNumResults = minAndMax->second - minAndMax->first; - sql << ("WHERE ledger_seq >= " + std::to_string(minAndMax->first) + " AND ledger_seq <= " + std::to_string(minAndMax->second)); diff --git a/src/ripple/app/tx/impl/CashCheck.cpp b/src/ripple/app/tx/impl/CashCheck.cpp index b258ae7d9d8..bc3d838540b 100644 --- a/src/ripple/app/tx/impl/CashCheck.cpp +++ b/src/ripple/app/tx/impl/CashCheck.cpp @@ -447,7 +447,7 @@ CashCheck::doApply() true, // default path static_cast(optDeliverMin), // partial payment true, // owner pays transfer fee - false, // offer crossing + OfferCrossing::no, std::nullopt, sleCheck->getFieldAmount(sfSendMax), viewJ); diff --git a/src/ripple/app/tx/impl/CreateOffer.cpp b/src/ripple/app/tx/impl/CreateOffer.cpp index dd01a64b5f2..17f7e2853db 100644 --- a/src/ripple/app/tx/impl/CreateOffer.cpp +++ b/src/ripple/app/tx/impl/CreateOffer.cpp @@ -736,8 +736,10 @@ CreateOffer::flowCross( } // Special handling for the tfSell flag. STAmount deliver = takerAmount.out; + OfferCrossing offerCrossing = OfferCrossing::yes; if (txFlags & tfSell) { + offerCrossing = OfferCrossing::sell; // We are selling, so we will accept *more* than the offer // specified. Since we don't know how much they might offer, // we allow delivery of the largest possible amount. @@ -764,7 +766,7 @@ CreateOffer::flowCross( true, // default path !(txFlags & tfFillOrKill), // partial payment true, // owner pays transfer fee - true, // offer crossing + offerCrossing, threshold, sendMax, j_); diff --git a/src/ripple/app/tx/impl/XChainBridge.cpp b/src/ripple/app/tx/impl/XChainBridge.cpp index 6ef10b0ebfa..59450113d2b 100644 --- a/src/ripple/app/tx/impl/XChainBridge.cpp +++ b/src/ripple/app/tx/impl/XChainBridge.cpp @@ -505,7 +505,7 @@ transferHelper( /*default path*/ true, /*partial payment*/ false, /*owner pays transfer fee*/ true, - /*offer crossing*/ false, + /*offer crossing*/ OfferCrossing::no, /*limit quality*/ std::nullopt, /*sendmax*/ std::nullopt, j); @@ -1211,6 +1211,9 @@ attestationPreflight(PreflightContext const& ctx) if (ctx.tx.getFlags() & tfUniversalMask) return temINVALID_FLAG; + if (!publicKeyType(ctx.tx[sfPublicKey])) + return temMALFORMED; + auto const att = toClaim(ctx.tx); if (!att) return temMALFORMED; diff --git a/src/ripple/basics/chrono.h b/src/ripple/basics/chrono.h index f50d765d58f..ea82f928b7e 100644 --- a/src/ripple/basics/chrono.h +++ b/src/ripple/basics/chrono.h @@ -25,9 +25,12 @@ #include #include #include + #include #include +#include #include +#include namespace ripple { @@ -43,8 +46,19 @@ using weeks = std::chrono:: /** Clock for measuring the network time. The epoch is January 1, 2000 - epoch_offset = days(10957); // 2000-01-01 + + epoch_offset + = date(2000-01-01) - date(1970-0-01) + = days(10957) + = seconds(946684800) */ + +constexpr static std::chrono::seconds epoch_offset = + date::sys_days{date::year{2000} / 1 / 1} - + date::sys_days{date::year{1970} / 1 / 1}; + +static_assert(epoch_offset.count() == 946684800); + class NetClock { public: @@ -71,7 +85,25 @@ to_string(NetClock::time_point tp) // 2000-01-01 00:00:00 UTC is 946684800s from 1970-01-01 00:00:00 UTC using namespace std::chrono; return to_string( - system_clock::time_point{tp.time_since_epoch() + 946684800s}); + system_clock::time_point{tp.time_since_epoch() + epoch_offset}); +} + +template +std::string +to_string_iso(date::sys_time tp) +{ + using namespace std::chrono; + return date::format("%FT%TZ", tp); +} + +inline std::string +to_string_iso(NetClock::time_point tp) +{ + // 2000-01-01 00:00:00 UTC is 946684800s from 1970-01-01 00:00:00 UTC + // Note, NetClock::duration is seconds, as checked by static_assert + static_assert(std::is_same_v>); + return to_string_iso(date::sys_time{ + tp.time_since_epoch() + epoch_offset}); } /** A clock for measuring elapsed time. diff --git a/src/ripple/core/TimeKeeper.h b/src/ripple/core/TimeKeeper.h index 55970ec8227..e239a2f7565 100644 --- a/src/ripple/core/TimeKeeper.h +++ b/src/ripple/core/TimeKeeper.h @@ -22,6 +22,7 @@ #include #include + #include namespace ripple { @@ -37,7 +38,7 @@ class TimeKeeper : public beast::abstract_clock adjust(std::chrono::system_clock::time_point when) { return time_point(std::chrono::duration_cast( - when.time_since_epoch() - days(10957))); + when.time_since_epoch() - epoch_offset)); } public: diff --git a/src/ripple/net/RPCCall.h b/src/ripple/net/RPCCall.h index a97c6177109..600fad28e58 100644 --- a/src/ripple/net/RPCCall.h +++ b/src/ripple/net/RPCCall.h @@ -65,18 +65,22 @@ fromNetwork( std::unordered_map headers = {}); } // namespace RPCCall -/** Given a rippled command line, return the corresponding JSON. - */ Json::Value -cmdLineToJSONRPC(std::vector const& args, beast::Journal j); +rpcCmdToJson( + std::vector const& args, + Json::Value& retParams, + unsigned int apiVersion, + beast::Journal j); /** Internal invocation of RPC client. + * Used by both rippled command line as well as rippled unit tests */ std::pair rpcClient( std::vector const& args, Config const& config, Logs& logs, + unsigned int apiVersion, std::unordered_map const& headers = {}); } // namespace ripple diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index b52545960cc..f4428a3f000 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -1472,10 +1472,11 @@ struct RPCCallImp //------------------------------------------------------------------------------ // Used internally by rpcClient. -static Json::Value -rpcCmdLineToJson( +Json::Value +rpcCmdToJson( std::vector const& args, Json::Value& retParams, + unsigned int apiVersion, beast::Journal j) { Json::Value jvRequest(Json::objectValue); @@ -1493,11 +1494,11 @@ rpcCmdLineToJson( jvRequest = rpParser.parseCommand(args[0], jvRpcParams, true); - auto insert_api_version = [](Json::Value& jr) { + auto insert_api_version = [apiVersion](Json::Value& jr) { if (jr.isObject() && !jr.isMember(jss::error) && !jr.isMember(jss::api_version)) { - jr[jss::api_version] = RPC::apiMaximumSupportedVersion; + jr[jss::api_version] = apiVersion; } }; @@ -1510,35 +1511,6 @@ rpcCmdLineToJson( return jvRequest; } -Json::Value -cmdLineToJSONRPC(std::vector const& args, beast::Journal j) -{ - Json::Value jv = Json::Value(Json::objectValue); - auto const paramsObj = rpcCmdLineToJson(args, jv, j); - - // Re-use jv to return our formatted result. - jv.clear(); - - // Allow parser to rewrite method. - jv[jss::method] = paramsObj.isMember(jss::method) - ? paramsObj[jss::method].asString() - : args[0]; - - // If paramsObj is not empty, put it in a [params] array. - if (paramsObj.begin() != paramsObj.end()) - { - auto& paramsArray = Json::setArray(jv, jss::params); - paramsArray.append(paramsObj); - } - if (paramsObj.isMember(jss::jsonrpc)) - jv[jss::jsonrpc] = paramsObj[jss::jsonrpc]; - if (paramsObj.isMember(jss::ripplerpc)) - jv[jss::ripplerpc] = paramsObj[jss::ripplerpc]; - if (paramsObj.isMember(jss::id)) - jv[jss::id] = paramsObj[jss::id]; - return jv; -} - //------------------------------------------------------------------------------ std::pair @@ -1546,6 +1518,7 @@ rpcClient( std::vector const& args, Config const& config, Logs& logs, + unsigned int apiVersion, std::unordered_map const& headers) { static_assert( @@ -1561,7 +1534,8 @@ rpcClient( try { Json::Value jvRpc = Json::Value(Json::objectValue); - jvRequest = rpcCmdLineToJson(args, jvRpc, logs.journal("RPCParser")); + jvRequest = + rpcCmdToJson(args, jvRpc, apiVersion, logs.journal("RPCParser")); if (jvRequest.isMember(jss::error)) { @@ -1698,7 +1672,8 @@ fromCommandLine( const std::vector& vCmd, Logs& logs) { - auto const result = rpcClient(vCmd, config, logs); + auto const result = + rpcClient(vCmd, config, logs, RPC::apiMaximumSupportedVersion); std::cout << result.second.toStyledString(); diff --git a/src/ripple/perflog/impl/PerfLogImp.cpp b/src/ripple/perflog/impl/PerfLogImp.cpp index db5a188fc3e..3d07d0ed625 100644 --- a/src/ripple/perflog/impl/PerfLogImp.cpp +++ b/src/ripple/perflog/impl/PerfLogImp.cpp @@ -43,7 +43,7 @@ namespace ripple { namespace perf { PerfLogImp::Counters::Counters( - std::vector const& labels, + std::set const& labels, JobTypes const& jobTypes) { { diff --git a/src/ripple/perflog/impl/PerfLogImp.h b/src/ripple/perflog/impl/PerfLogImp.h index 493c1dc1a18..4904126d95f 100644 --- a/src/ripple/perflog/impl/PerfLogImp.h +++ b/src/ripple/perflog/impl/PerfLogImp.h @@ -113,9 +113,7 @@ class PerfLogImp : public PerfLog std::unordered_map methods_; mutable std::mutex methodsMutex_; - Counters( - std::vector const& labels, - JobTypes const& jobTypes); + Counters(std::set const& labels, JobTypes const& jobTypes); Json::Value countersJson() const; Json::Value diff --git a/src/ripple/protocol/Feature.h b/src/ripple/protocol/Feature.h index 6377ce3ac62..3bdfcb15c59 100644 --- a/src/ripple/protocol/Feature.h +++ b/src/ripple/protocol/Feature.h @@ -74,7 +74,7 @@ namespace detail { // Feature.cpp. Because it's only used to reserve storage, and determine how // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than // the actual number of amendments. A LogicError on startup will verify this. -static constexpr std::size_t numFeatures = 64; +static constexpr std::size_t numFeatures = 65; /** Amendments that this server supports and the default voting behavior. Whether they are enabled depends on the Rules defined in the validated @@ -351,6 +351,7 @@ extern uint256 const featureClawback; extern uint256 const featureXChainBridge; extern uint256 const fixDisallowIncomingV1; extern uint256 const featureDID; +extern uint256 const fixFillOrKill; } // namespace ripple diff --git a/src/ripple/protocol/STBase.h b/src/ripple/protocol/STBase.h index 914a3e0f60b..ec8c34a9ddd 100644 --- a/src/ripple/protocol/STBase.h +++ b/src/ripple/protocol/STBase.h @@ -31,7 +31,62 @@ #include namespace ripple { -enum class JsonOptions { none = 0, include_date = 1 }; +/// Note, should be treated as flags that can be | and & +struct JsonOptions +{ + using underlying_t = unsigned int; + underlying_t value; + + enum values : underlying_t { + // clang-format off + none = 0b0000'0000, + include_date = 0b0000'0001, + disable_API_prior_V2 = 0b0000'0010, + + // IMPORTANT `_all` must be union of all of the above; see also operator~ + _all = 0b0000'0011 + // clang-format on + }; + + constexpr JsonOptions(underlying_t v) noexcept : value(v) + { + } + + [[nodiscard]] constexpr explicit operator underlying_t() const noexcept + { + return value; + } + [[nodiscard]] constexpr explicit operator bool() const noexcept + { + return value != 0u; + } + [[nodiscard]] constexpr auto friend + operator==(JsonOptions lh, JsonOptions rh) noexcept -> bool = default; + [[nodiscard]] constexpr auto friend + operator!=(JsonOptions lh, JsonOptions rh) noexcept -> bool = default; + + /// Returns JsonOptions union of lh and rh + [[nodiscard]] constexpr JsonOptions friend + operator|(JsonOptions lh, JsonOptions rh) noexcept + { + return {lh.value | rh.value}; + } + + /// Returns JsonOptions intersection of lh and rh + [[nodiscard]] constexpr JsonOptions friend + operator&(JsonOptions lh, JsonOptions rh) noexcept + { + return {lh.value & rh.value}; + } + + /// Returns JsonOptions binary negation, can be used with & (above) for set + /// difference e.g. `(options & ~JsonOptions::include_date)` + [[nodiscard]] constexpr JsonOptions friend + operator~(JsonOptions v) noexcept + { + return {~v.value & static_cast(_all)}; + } +}; namespace detail { class STVar; diff --git a/src/ripple/protocol/STObject.h b/src/ripple/protocol/STObject.h index 19d4b264734..3e3862bf6c8 100644 --- a/src/ripple/protocol/STObject.h +++ b/src/ripple/protocol/STObject.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include diff --git a/src/ripple/protocol/STTx.h b/src/ripple/protocol/STTx.h index c6a9e053c3d..e166eb20dd4 100644 --- a/src/ripple/protocol/STTx.h +++ b/src/ripple/protocol/STTx.h @@ -29,6 +29,7 @@ #include #include #include + #include namespace ripple { @@ -108,6 +109,7 @@ class STTx final : public STObject, public CountedObject Json::Value getJson(JsonOptions options) const override; + Json::Value getJson(JsonOptions options, bool binary) const; diff --git a/src/ripple/protocol/impl/BuildInfo.cpp b/src/ripple/protocol/impl/BuildInfo.cpp index e6b7b16affa..ec44eddbd42 100644 --- a/src/ripple/protocol/impl/BuildInfo.cpp +++ b/src/ripple/protocol/impl/BuildInfo.cpp @@ -33,7 +33,7 @@ namespace BuildInfo { // and follow the format described at http://semver.org/ //------------------------------------------------------------------------------ // clang-format off -char const* const versionString = "2.0.0-b4" +char const* const versionString = "2.0.0-rc1" // clang-format on #if defined(DEBUG) || defined(SANITIZER) diff --git a/src/ripple/protocol/impl/Feature.cpp b/src/ripple/protocol/impl/Feature.cpp index 77a0a9284ac..25033d4336e 100644 --- a/src/ripple/protocol/impl/Feature.cpp +++ b/src/ripple/protocol/impl/Feature.cpp @@ -458,6 +458,7 @@ REGISTER_FEATURE(AMM, Supported::yes, VoteBehavior::De REGISTER_FEATURE(XChainBridge, Supported::yes, VoteBehavior::DefaultNo); REGISTER_FIX (fixDisallowIncomingV1, Supported::yes, VoteBehavior::DefaultNo); REGISTER_FEATURE(DID, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX(fixFillOrKill, Supported::yes, VoteBehavior::DefaultNo); // The following amendments are obsolete, but must remain supported // because they could potentially get enabled. diff --git a/src/ripple/protocol/impl/STTx.cpp b/src/ripple/protocol/impl/STTx.cpp index 1ce4ddb64b7..8106e997f3a 100644 --- a/src/ripple/protocol/impl/STTx.cpp +++ b/src/ripple/protocol/impl/STTx.cpp @@ -34,6 +34,7 @@ #include #include #include + #include #include #include @@ -236,15 +237,29 @@ Json::Value STTx::getJson(JsonOptions) const Json::Value STTx::getJson(JsonOptions options, bool binary) const { + bool const V1 = !(options & JsonOptions::disable_API_prior_V2); + if (binary) { - Json::Value ret; Serializer s = STObject::getSerializer(); - ret[jss::tx] = strHex(s.peekData()); - ret[jss::hash] = to_string(getTransactionID()); - return ret; + std::string const dataBin = strHex(s.peekData()); + + if (V1) + { + Json::Value ret(Json::objectValue); + ret[jss::tx] = dataBin; + ret[jss::hash] = to_string(getTransactionID()); + return ret; + } + else + return Json::Value{dataBin}; } - return getJson(options); + + Json::Value ret = STObject::getJson(JsonOptions::none); + if (V1) + ret[jss::hash] = to_string(getTransactionID()); + + return ret; } std::string const& diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index d4b213bcb1b..8a701defad8 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -230,6 +230,8 @@ JSS(close); // out: BookChanges JSS(close_flags); // out: LedgerToJson JSS(close_time); // in: Application, out: NetworkOPs, // RCLCxPeerPos, LedgerToJson +JSS(close_time_iso); // out: Tx, NetworkOPs, TransactionEntry + // AccountTx, LedgerToJson JSS(close_time_estimated); // in: Application, out: LedgerToJson JSS(close_time_human); // out: LedgerToJson JSS(close_time_offset); // out: NetworkOPs @@ -460,6 +462,7 @@ JSS(median_fee); // out: TxQ JSS(median_level); // out: TxQ JSS(message); // error. JSS(meta); // out: NetworkOPs, AccountTx*, Tx +JSS(meta_blob); // out: NetworkOPs, AccountTx*, Tx JSS(metaData); JSS(metadata); // out: TransactionEntry JSS(method); // RPC diff --git a/src/ripple/rpc/handlers/AMMInfo.cpp b/src/ripple/rpc/handlers/AMMInfo.cpp index 11e124afb44..a1be636cafd 100644 --- a/src/ripple/rpc/handlers/AMMInfo.cpp +++ b/src/ripple/rpc/handlers/AMMInfo.cpp @@ -16,6 +16,7 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== +#include #include #include #include @@ -66,7 +67,7 @@ to_iso8601(NetClock::time_point tp) return date::format( "%Y-%Om-%dT%H:%M:%OS%z", date::sys_time( - system_clock::time_point{tp.time_since_epoch() + 946684800s})); + system_clock::time_point{tp.time_since_epoch() + epoch_offset})); } Json::Value @@ -244,8 +245,7 @@ doAMMInfo(RPC::JsonContext& context) if (!result.isMember(jss::ledger_index) && !result.isMember(jss::ledger_hash)) result[jss::ledger_current_index] = ledger->info().seq; - result[jss::validated] = - RPC::isValidated(context.ledgerMaster, *ledger, context.app); + result[jss::validated] = context.ledgerMaster.isValidated(*ledger); return result; } diff --git a/src/ripple/rpc/handlers/AccountTx.cpp b/src/ripple/rpc/handlers/AccountTx.cpp index bd939a92f1c..addd9ea0f39 100644 --- a/src/ripple/rpc/handlers/AccountTx.cpp +++ b/src/ripple/rpc/handlers/AccountTx.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include @@ -195,8 +194,8 @@ getLedgerRange( return status; } - bool validated = RPC::isValidated( - context.ledgerMaster, *ledgerView, context.app); + bool validated = + context.ledgerMaster.isValidated(*ledgerView); if (!validated || ledgerView->info().seq > uValidatedMax || ledgerView->info().seq < uValidatedMin) @@ -325,16 +324,39 @@ populateJsonResponse( if (txn) { Json::Value& jvObj = jvTxns.append(Json::objectValue); + jvObj[jss::validated] = true; + + auto const json_tx = + (context.apiVersion > 1 ? jss::tx_json : jss::tx); + if (context.apiVersion > 1) + { + jvObj[json_tx] = txn->getJson( + JsonOptions::include_date | + JsonOptions::disable_API_prior_V2, + false); + jvObj[jss::hash] = to_string(txn->getID()); + jvObj[jss::ledger_index] = txn->getLedger(); + jvObj[jss::ledger_hash] = + to_string(context.ledgerMaster.getHashBySeq( + txn->getLedger())); + + if (auto closeTime = + context.ledgerMaster.getCloseTimeBySeq( + txn->getLedger())) + jvObj[jss::close_time_iso] = + to_string_iso(*closeTime); + } + else + jvObj[json_tx] = + txn->getJson(JsonOptions::include_date); - jvObj[jss::tx] = txn->getJson(JsonOptions::include_date); auto const& sttx = txn->getSTransaction(); RPC::insertDeliverMax( - jvObj[jss::tx], sttx->getTxnType(), context.apiVersion); + jvObj[json_tx], sttx->getTxnType(), context.apiVersion); if (txnMeta) { jvObj[jss::meta] = txnMeta->getJson(JsonOptions::include_date); - jvObj[jss::validated] = true; insertDeliveredAmount( jvObj[jss::meta], context, txn, *txnMeta); insertNFTSyntheticInJson(jvObj, sttx, *txnMeta); @@ -352,7 +374,9 @@ populateJsonResponse( Json::Value& jvObj = jvTxns.append(Json::objectValue); jvObj[jss::tx_blob] = strHex(std::get<0>(binaryData)); - jvObj[jss::meta] = strHex(std::get<1>(binaryData)); + auto const json_meta = + (context.apiVersion > 1 ? jss::meta_blob : jss::meta); + jvObj[json_meta] = strHex(std::get<1>(binaryData)); jvObj[jss::ledger_index] = std::get<2>(binaryData); jvObj[jss::validated] = true; } diff --git a/src/ripple/rpc/handlers/LedgerHandler.cpp b/src/ripple/rpc/handlers/LedgerHandler.cpp index 6b4fc77367b..623cb8d75ac 100644 --- a/src/ripple/rpc/handlers/LedgerHandler.cpp +++ b/src/ripple/rpc/handlers/LedgerHandler.cpp @@ -27,7 +27,6 @@ #include #include #include -#include namespace ripple { namespace RPC { @@ -301,8 +300,7 @@ doLedgerGrpc(RPC::GRPCContext& context) response.set_skiplist_included(true); } - response.set_validated( - RPC::isValidated(context.ledgerMaster, *ledger, context.app)); + response.set_validated(context.ledgerMaster.isValidated(*ledger)); auto end = std::chrono::system_clock::now(); auto duration = diff --git a/src/ripple/rpc/handlers/LedgerHandler.h b/src/ripple/rpc/handlers/LedgerHandler.h index 77b361d3466..b0bca8e6635 100644 --- a/src/ripple/rpc/handlers/LedgerHandler.h +++ b/src/ripple/rpc/handlers/LedgerHandler.h @@ -30,6 +30,7 @@ #include #include #include +#include namespace Json { class Object; @@ -58,23 +59,15 @@ class LedgerHandler void writeResult(Object&); - static char const* - name() - { - return "ledger"; - } + static constexpr char name[] = "ledger"; - static Role - role() - { - return Role::USER; - } + static constexpr unsigned minApiVer = RPC::apiMinimumSupportedVersion; - static Condition - condition() - { - return NO_CONDITION; - } + static constexpr unsigned maxApiVer = RPC::apiMaximumValidVersion; + + static constexpr Role role = Role::USER; + + static constexpr Condition condition = NO_CONDITION; private: JsonContext& context_; diff --git a/src/ripple/rpc/handlers/SignFor.cpp b/src/ripple/rpc/handlers/SignFor.cpp index ac76fa0d8a1..722cf7da157 100644 --- a/src/ripple/rpc/handlers/SignFor.cpp +++ b/src/ripple/rpc/handlers/SignFor.cpp @@ -46,6 +46,7 @@ doSignFor(RPC::JsonContext& context) auto ret = RPC::transactionSignFor( context.params, + context.apiVersion, failType, context.role, context.ledgerMaster.getValidatedLedgerAge(), diff --git a/src/ripple/rpc/handlers/SignHandler.cpp b/src/ripple/rpc/handlers/SignHandler.cpp index 15d433da49c..4d89cdcb2e0 100644 --- a/src/ripple/rpc/handlers/SignHandler.cpp +++ b/src/ripple/rpc/handlers/SignHandler.cpp @@ -45,6 +45,7 @@ doSign(RPC::JsonContext& context) auto ret = RPC::transactionSign( context.params, + context.apiVersion, failType, context.role, context.ledgerMaster.getValidatedLedgerAge(), diff --git a/src/ripple/rpc/handlers/Submit.cpp b/src/ripple/rpc/handlers/Submit.cpp index 8a702c5bd3e..8e998f1ea6c 100644 --- a/src/ripple/rpc/handlers/Submit.cpp +++ b/src/ripple/rpc/handlers/Submit.cpp @@ -63,6 +63,7 @@ doSubmit(RPC::JsonContext& context) auto ret = RPC::transactionSubmit( context.params, + context.apiVersion, failType, context.role, context.ledgerMaster.getValidatedLedgerAge(), diff --git a/src/ripple/rpc/handlers/SubmitMultiSigned.cpp b/src/ripple/rpc/handlers/SubmitMultiSigned.cpp index 9b455a1961f..82fa52a4623 100644 --- a/src/ripple/rpc/handlers/SubmitMultiSigned.cpp +++ b/src/ripple/rpc/handlers/SubmitMultiSigned.cpp @@ -45,6 +45,7 @@ doSubmitMultiSigned(RPC::JsonContext& context) return RPC::transactionSubmitMultiSigned( context.params, + context.apiVersion, failType, context.role, context.ledgerMaster.getValidatedLedgerAge(), diff --git a/src/ripple/rpc/handlers/TransactionEntry.cpp b/src/ripple/rpc/handlers/TransactionEntry.cpp index 677581db6a3..6d157891d1c 100644 --- a/src/ripple/rpc/handlers/TransactionEntry.cpp +++ b/src/ripple/rpc/handlers/TransactionEntry.cpp @@ -17,6 +17,7 @@ */ //============================================================================== +#include #include #include #include @@ -71,11 +72,39 @@ doTransactionEntry(RPC::JsonContext& context) } else { - jvResult[jss::tx_json] = sttx->getJson(JsonOptions::none); + if (context.apiVersion > 1) + { + jvResult[jss::tx_json] = + sttx->getJson(JsonOptions::disable_API_prior_V2); + jvResult[jss::hash] = to_string(sttx->getTransactionID()); + + if (!lpLedger->open()) + jvResult[jss::ledger_hash] = to_string( + context.ledgerMaster.getHashBySeq(lpLedger->seq())); + + bool const validated = + context.ledgerMaster.isValidated(*lpLedger); + + jvResult[jss::validated] = validated; + if (validated) + { + jvResult[jss::ledger_index] = lpLedger->seq(); + if (auto closeTime = context.ledgerMaster.getCloseTimeBySeq( + lpLedger->seq())) + jvResult[jss::close_time_iso] = + to_string_iso(*closeTime); + } + } + else + jvResult[jss::tx_json] = sttx->getJson(JsonOptions::none); + RPC::insertDeliverMax( jvResult[jss::tx_json], sttx->getTxnType(), context.apiVersion); + + auto const json_meta = + (context.apiVersion > 1 ? jss::meta : jss::metadata); if (stobj) - jvResult[jss::metadata] = stobj->getJson(JsonOptions::none); + jvResult[json_meta] = stobj->getJson(JsonOptions::none); // 'accounts' // 'engine_...' // 'ledger_...' diff --git a/src/ripple/rpc/handlers/Tx.cpp b/src/ripple/rpc/handlers/Tx.cpp index 92d0e4dd673..0237fef22ac 100644 --- a/src/ripple/rpc/handlers/Tx.cpp +++ b/src/ripple/rpc/handlers/Tx.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include #include + #include #include @@ -55,6 +57,8 @@ struct TxResult std::variant, Blob> meta; bool validated = false; std::optional ctid; + std::optional closeTime; + std::optional ledgerHash; TxSearched searchedAll; }; @@ -140,6 +144,13 @@ doTxPostgres(RPC::Context& context, TxArgs const& args) *(args.hash), res.txn->getLedger(), *meta); } res.validated = true; + + auto const ledgerInfo = + context.app.getRelationalDatabase().getLedgerInfoByIndex( + locator.getLedgerSequence()); + res.closeTime = ledgerInfo->closeTime; + res.ledgerHash = ledgerInfo->hash; + return {res, rpcSUCCESS}; } else @@ -257,6 +268,9 @@ doTxHelp(RPC::Context& context, TxArgs args) std::shared_ptr ledger = context.ledgerMaster.getLedgerBySeq(txn->getLedger()); + if (ledger && !ledger->open()) + result.ledgerHash = ledger->info().hash; + if (ledger && meta) { if (args.binary) @@ -269,6 +283,9 @@ doTxHelp(RPC::Context& context, TxArgs args) } result.validated = isValidated( context.ledgerMaster, ledger->info().seq, ledger->info().hash); + if (result.validated) + result.closeTime = + context.ledgerMaster.getCloseTimeBySeq(txn->getLedger()); // compute outgoing CTID uint32_t lgrSeq = ledger->info().seq; @@ -311,17 +328,52 @@ populateJsonResponse( // no errors else if (result.txn) { - response = result.txn->getJson(JsonOptions::include_date, args.binary); auto const& sttx = result.txn->getSTransaction(); - if (!args.binary) - RPC::insertDeliverMax( - response, sttx->getTxnType(), context.apiVersion); + if (context.apiVersion > 1) + { + constexpr auto optionsJson = + JsonOptions::include_date | JsonOptions::disable_API_prior_V2; + if (args.binary) + response[jss::tx_blob] = result.txn->getJson(optionsJson, true); + else + { + response[jss::tx_json] = result.txn->getJson(optionsJson); + RPC::insertDeliverMax( + response[jss::tx_json], + sttx->getTxnType(), + context.apiVersion); + } + + // Note, result.ledgerHash is only set in a closed or validated + // ledger - as seen in `doTxHelp` and `doTxPostgres` + if (result.ledgerHash) + response[jss::ledger_hash] = to_string(*result.ledgerHash); + + response[jss::hash] = to_string(result.txn->getID()); + if (result.validated) + { + response[jss::ledger_index] = result.txn->getLedger(); + if (result.closeTime) + response[jss::close_time_iso] = + to_string_iso(*result.closeTime); + } + } + else + { + response = + result.txn->getJson(JsonOptions::include_date, args.binary); + if (!args.binary) + RPC::insertDeliverMax( + response, sttx->getTxnType(), context.apiVersion); + } // populate binary metadata if (auto blob = std::get_if(&result.meta)) { assert(args.binary); - response[jss::meta] = strHex(makeSlice(*blob)); + auto json_meta = + (context.apiVersion > 1 ? jss::meta_blob : jss::meta); + response[json_meta] = strHex(makeSlice(*blob)); } // populate meta data else if (auto m = std::get_if>(&result.meta)) diff --git a/src/ripple/rpc/handlers/Version.h b/src/ripple/rpc/handlers/Version.h index a9f42b94993..8f33b62f1cf 100644 --- a/src/ripple/rpc/handlers/Version.h +++ b/src/ripple/rpc/handlers/Version.h @@ -46,23 +46,15 @@ class VersionHandler setVersion(obj, apiVersion_, betaEnabled_); } - static char const* - name() - { - return "version"; - } + static constexpr char const* name = "version"; - static Role - role() - { - return Role::USER; - } + static constexpr unsigned minApiVer = RPC::apiMinimumSupportedVersion; - static Condition - condition() - { - return NO_CONDITION; - } + static constexpr unsigned maxApiVer = RPC::apiMaximumValidVersion; + + static constexpr Role role = Role::USER; + + static constexpr Condition condition = NO_CONDITION; private: unsigned int apiVersion_; diff --git a/src/ripple/rpc/impl/Handler.cpp b/src/ripple/rpc/impl/Handler.cpp index b69d2608b0e..d05c3279800 100644 --- a/src/ripple/rpc/impl/Handler.cpp +++ b/src/ripple/rpc/impl/Handler.cpp @@ -17,11 +17,14 @@ */ //============================================================================== +#include #include #include #include #include +#include + namespace ripple { namespace RPC { namespace { @@ -47,6 +50,9 @@ template Status handle(JsonContext& context, Object& object) { + assert( + context.apiVersion >= HandlerImpl::minApiVer && + context.apiVersion <= HandlerImpl::maxApiVer); HandlerImpl handler(context); auto status = handler.check(); @@ -55,7 +61,20 @@ handle(JsonContext& context, Object& object) else handler.writeResult(object); return status; -}; +} + +template +Handler +handlerFrom() +{ + return { + HandlerImpl::name, + &handle, + HandlerImpl::role, + HandlerImpl::condition, + HandlerImpl::minApiVer, + HandlerImpl::maxApiVer}; +} Handler const handlerArray[]{ // Some handlers not specified here are added to the table via addHandler() @@ -110,7 +129,7 @@ Handler const handlerArray[]{ NEEDS_CURRENT_LEDGER}, {"ledger_data", byRef(&doLedgerData), Role::USER, NO_CONDITION}, {"ledger_entry", byRef(&doLedgerEntry), Role::USER, NO_CONDITION}, - {"ledger_header", byRef(&doLedgerHeader), Role::USER, NO_CONDITION}, + {"ledger_header", byRef(&doLedgerHeader), Role::USER, NO_CONDITION, 1, 1}, {"ledger_request", byRef(&doLedgerRequest), Role::ADMIN, NO_CONDITION}, {"log_level", byRef(&doLogLevel), Role::ADMIN, NO_CONDITION}, {"logrotate", byRef(&doLogRotate), Role::ADMIN, NO_CONDITION}, @@ -156,7 +175,7 @@ Handler const handlerArray[]{ NEEDS_CURRENT_LEDGER}, {"transaction_entry", byRef(&doTransactionEntry), Role::USER, NO_CONDITION}, {"tx", byRef(&doTxJson), Role::USER, NEEDS_NETWORK_CONNECTION}, - {"tx_history", byRef(&doTxHistory), Role::USER, NO_CONDITION}, + {"tx_history", byRef(&doTxHistory), Role::USER, NO_CONDITION, 1, 1}, {"tx_reduce_relay", byRef(&doTxReduceRelay), Role::USER, NO_CONDITION}, {"unl_list", byRef(&doUnlList), Role::ADMIN, NO_CONDITION}, {"validation_create", @@ -178,14 +197,42 @@ Handler const handlerArray[]{ class HandlerTable { private: + using handler_table_t = std::multimap; + + // Use with equal_range to enforce that API range of a newly added handler + // does not overlap with API range of an existing handler with same name + [[nodiscard]] bool + overlappingApiVersion( + std::pair range, + unsigned minVer, + unsigned maxVer) + { + assert(minVer <= maxVer); + assert(maxVer <= RPC::apiMaximumValidVersion); + + return std::any_of( + range.first, + range.second, // + [minVer, maxVer](auto const& item) { + return item.second.minApiVer_ <= maxVer && + item.second.maxApiVer_ >= minVer; + }); + } + template explicit HandlerTable(const Handler (&entries)[N]) { - for (std::size_t i = 0; i < N; ++i) + for (auto const& entry : entries) { - auto const& entry = entries[i]; - assert(table_.find(entry.name_) == table_.end()); - table_[entry.name_] = entry; + if (overlappingApiVersion( + table_.equal_range(entry.name_), + entry.minApiVer_, + entry.maxApiVer_)) + LogicError( + std::string("Handler for ") + entry.name_ + + " overlaps with an existing handler"); + + table_.insert({entry.name_, entry}); } // This is where the new-style handlers are added. @@ -201,7 +248,7 @@ class HandlerTable return handlerTable; } - Handler const* + [[nodiscard]] Handler const* getHandler(unsigned version, bool betaEnabled, std::string const& name) const { @@ -209,36 +256,48 @@ class HandlerTable version > (betaEnabled ? RPC::apiBetaVersion : RPC::apiMaximumSupportedVersion)) return nullptr; - auto i = table_.find(name); - return i == table_.end() ? nullptr : &i->second; + + auto const range = table_.equal_range(name); + auto const i = std::find_if( + range.first, range.second, [version](auto const& entry) { + return entry.second.minApiVer_ <= version && + version <= entry.second.maxApiVer_; + }); + + return i == range.second ? nullptr : &i->second; } - std::vector + [[nodiscard]] std::set getHandlerNames() const { - std::vector ret; - ret.reserve(table_.size()); + std::set ret; for (auto const& i : table_) - ret.push_back(i.second.name_); + ret.insert(i.second.name_); + return ret; } private: - std::map table_; + handler_table_t table_; template void addHandler() { - assert(table_.find(HandlerImpl::name()) == table_.end()); + static_assert(HandlerImpl::minApiVer <= HandlerImpl::maxApiVer); + static_assert(HandlerImpl::maxApiVer <= RPC::apiMaximumValidVersion); + static_assert( + RPC::apiMinimumSupportedVersion <= HandlerImpl::minApiVer); - Handler h; - h.name_ = HandlerImpl::name(); - h.valueMethod_ = &handle; - h.role_ = HandlerImpl::role(); - h.condition_ = HandlerImpl::condition(); + if (overlappingApiVersion( + table_.equal_range(HandlerImpl::name), + HandlerImpl::minApiVer, + HandlerImpl::maxApiVer)) + LogicError( + std::string("Handler for ") + HandlerImpl::name + + " overlaps with an existing handler"); - table_[HandlerImpl::name()] = h; + table_.insert({HandlerImpl::name, handlerFrom()}); } }; @@ -250,11 +309,11 @@ getHandler(unsigned version, bool betaEnabled, std::string const& name) return HandlerTable::instance().getHandler(version, betaEnabled, name); } -std::vector +std::set getHandlerNames() { return HandlerTable::instance().getHandlerNames(); -}; +} } // namespace RPC } // namespace ripple diff --git a/src/ripple/rpc/impl/Handler.h b/src/ripple/rpc/impl/Handler.h index e2188ef51e7..28d1ee60c85 100644 --- a/src/ripple/rpc/impl/Handler.h +++ b/src/ripple/rpc/impl/Handler.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -52,6 +53,9 @@ struct Handler Method valueMethod_; Role role_; RPC::Condition condition_; + + unsigned minApiVer_ = apiMinimumSupportedVersion; + unsigned maxApiVer_ = apiMaximumValidVersion; }; Handler const* @@ -70,7 +74,7 @@ makeObjectValue( } /** Return names of all methods. */ -std::vector +std::set getHandlerNames(); template diff --git a/src/ripple/rpc/impl/RPCHelpers.cpp b/src/ripple/rpc/impl/RPCHelpers.cpp index a9cc0f9fffe..672095fe950 100644 --- a/src/ripple/rpc/impl/RPCHelpers.cpp +++ b/src/ripple/rpc/impl/RPCHelpers.cpp @@ -600,59 +600,6 @@ getLedger<>( template Status getLedger<>(std::shared_ptr&, uint256 const&, Context&); -bool -isValidated( - LedgerMaster& ledgerMaster, - ReadView const& ledger, - Application& app) -{ - if (app.config().reporting()) - return true; - - if (ledger.open()) - return false; - - if (ledger.info().validated) - return true; - - auto seq = ledger.info().seq; - try - { - // Use the skip list in the last validated ledger to see if ledger - // comes before the last validated ledger (and thus has been - // validated). - auto hash = - ledgerMaster.walkHashBySeq(seq, InboundLedger::Reason::GENERIC); - - if (!hash || ledger.info().hash != *hash) - { - // This ledger's hash is not the hash of the validated ledger - if (hash) - { - assert(hash->isNonZero()); - uint256 valHash = - app.getRelationalDatabase().getHashByIndex(seq); - if (valHash == ledger.info().hash) - { - // SQL database doesn't match ledger chain - ledgerMaster.clearLedger(seq); - } - } - return false; - } - } - catch (SHAMapMissingNode const& mn) - { - auto stream = app.journal("RPCHandler").warn(); - JLOG(stream) << "Ledger #" << seq << ": " << mn.what(); - return false; - } - - // Mark ledger as validated to save time if we see it again. - ledger.info().validated = true; - return true; -} - // The previous version of the lookupLedger command would accept the // "ledger_index" argument as a string and silently treat it as a request to // return the current ledger which, while not strictly wrong, could cause a lot @@ -693,8 +640,7 @@ lookupLedger( result[jss::ledger_current_index] = info.seq; } - result[jss::validated] = - isValidated(context.ledgerMaster, *ledger, context.app); + result[jss::validated] = context.ledgerMaster.isValidated(*ledger); return Status::OK; } diff --git a/src/ripple/rpc/impl/RPCHelpers.h b/src/ripple/rpc/impl/RPCHelpers.h index eb02e6ea37a..97015f1a35d 100644 --- a/src/ripple/rpc/impl/RPCHelpers.h +++ b/src/ripple/rpc/impl/RPCHelpers.h @@ -168,12 +168,6 @@ ledgerFromSpecifier( org::xrpl::rpc::v1::LedgerSpecifier const& specifier, Context& context); -bool -isValidated( - LedgerMaster& ledgerMaster, - ReadView const& ledger, - Application& app); - hash_set parseAccountIds(Json::Value const& jvArray); @@ -242,10 +236,12 @@ constexpr unsigned int apiVersionIfUnspecified = 1; constexpr unsigned int apiMinimumSupportedVersion = 1; constexpr unsigned int apiMaximumSupportedVersion = 1; constexpr unsigned int apiBetaVersion = 2; +constexpr unsigned int apiMaximumValidVersion = apiBetaVersion; static_assert(apiMinimumSupportedVersion >= apiVersionIfUnspecified); static_assert(apiMaximumSupportedVersion >= apiMinimumSupportedVersion); static_assert(apiBetaVersion >= apiMaximumSupportedVersion); +static_assert(apiMaximumValidVersion >= apiMaximumSupportedVersion); template void diff --git a/src/ripple/rpc/impl/TransactionSign.cpp b/src/ripple/rpc/impl/TransactionSign.cpp index 5dbfa49aef9..48a9c66d81c 100644 --- a/src/ripple/rpc/impl/TransactionSign.cpp +++ b/src/ripple/rpc/impl/TransactionSign.cpp @@ -645,12 +645,20 @@ transactionConstructImpl( } static Json::Value -transactionFormatResultImpl(Transaction::pointer tpTrans) +transactionFormatResultImpl(Transaction::pointer tpTrans, unsigned apiVersion) { Json::Value jvResult; try { - jvResult[jss::tx_json] = tpTrans->getJson(JsonOptions::none); + if (apiVersion > 1) + { + jvResult[jss::tx_json] = + tpTrans->getJson(JsonOptions::disable_API_prior_V2); + jvResult[jss::hash] = to_string(tpTrans->getID()); + } + else + jvResult[jss::tx_json] = tpTrans->getJson(JsonOptions::none); + jvResult[jss::tx_blob] = strHex(tpTrans->getSTransaction()->getSerializer().peekData()); @@ -777,6 +785,7 @@ checkFee( Json::Value transactionSign( Json::Value jvRequest, + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -807,13 +816,14 @@ transactionSign( if (!txn.second) return txn.first; - return transactionFormatResultImpl(txn.second); + return transactionFormatResultImpl(txn.second, apiVersion); } /** Returns a Json::objectValue. */ Json::Value transactionSubmit( Json::Value jvRequest, + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -853,7 +863,7 @@ transactionSubmit( rpcINTERNAL, "Exception occurred during transaction submission."); } - return transactionFormatResultImpl(txn.second); + return transactionFormatResultImpl(txn.second, apiVersion); } namespace detail { @@ -943,6 +953,7 @@ sortAndValidateSigners(STArray& signers, AccountID const& signingForID) Json::Value transactionSignFor( Json::Value jvRequest, + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -1043,13 +1054,14 @@ transactionSignFor( if (!txn.second) return txn.first; - return transactionFormatResultImpl(txn.second); + return transactionFormatResultImpl(txn.second, apiVersion); } /** Returns a Json::objectValue. */ Json::Value transactionSubmitMultiSigned( Json::Value jvRequest, + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -1236,7 +1248,7 @@ transactionSubmitMultiSigned( rpcINTERNAL, "Exception occurred during transaction submission."); } - return transactionFormatResultImpl(txn.second); + return transactionFormatResultImpl(txn.second, apiVersion); } } // namespace RPC diff --git a/src/ripple/rpc/impl/TransactionSign.h b/src/ripple/rpc/impl/TransactionSign.h index a396e65af52..48d2859ccf5 100644 --- a/src/ripple/rpc/impl/TransactionSign.h +++ b/src/ripple/rpc/impl/TransactionSign.h @@ -96,6 +96,7 @@ getProcessTxnFn(NetworkOPs& netOPs) Json::Value transactionSign( Json::Value params, // Passed by value so it can be modified locally. + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -105,6 +106,7 @@ transactionSign( Json::Value transactionSubmit( Json::Value params, // Passed by value so it can be modified locally. + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -116,6 +118,7 @@ transactionSubmit( Json::Value transactionSignFor( Json::Value params, // Passed by value so it can be modified locally. + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, @@ -125,6 +128,7 @@ transactionSignFor( Json::Value transactionSubmitMultiSigned( Json::Value params, // Passed by value so it can be modified locally. + unsigned apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, diff --git a/src/test/app/AMMExtended_test.cpp b/src/test/app/AMMExtended_test.cpp index ad2c1f67805..8c7cbce92f4 100644 --- a/src/test/app/AMMExtended_test.cpp +++ b/src/test/app/AMMExtended_test.cpp @@ -2103,7 +2103,7 @@ struct AMMExtended_test : public jtx::AMMTest false, false, true, - false, + OfferCrossing::no, std::nullopt, smax, flowJournal); diff --git a/src/test/app/AmendmentTable_test.cpp b/src/test/app/AmendmentTable_test.cpp index 64e6a038653..4284190a18a 100644 --- a/src/test/app/AmendmentTable_test.cpp +++ b/src/test/app/AmendmentTable_test.cpp @@ -481,37 +481,30 @@ class AmendmentTable_test final : public beast::unit_test::suite } } - // Make a list of trusted validators. - // Register the validators with AmendmentTable and return the list. std::vector> - makeValidators(int num, std::unique_ptr const& table) + makeValidators(int num) { std::vector> ret; ret.reserve(num); - hash_set trustedValidators; - trustedValidators.reserve(num); for (int i = 0; i < num; ++i) { - auto const& back = - ret.emplace_back(randomKeyPair(KeyType::secp256k1)); - trustedValidators.insert(back.first); + ret.emplace_back(randomKeyPair(KeyType::secp256k1)); } - table->trustChanged(trustedValidators); return ret; } static NetClock::time_point - hourTime(std::chrono::hours h) + weekTime(weeks w) { - return NetClock::time_point{h}; + return NetClock::time_point{w}; } // Execute a pretend consensus round for a flag ledger void doRound( - Rules const& rules, + uint256 const& feat, AmendmentTable& table, - std::chrono::hours hour, + weeks week, std::vector> const& validators, std::vector> const& votes, std::vector& ourVotes, @@ -529,7 +522,7 @@ class AmendmentTable_test final : public beast::unit_test::suite // enabled: In/out enabled amendments // majority: In/our majority amendments (and when they got a majority) - auto const roundTime = hourTime(hour); + auto const roundTime = weekTime(week); // Build validations std::vector> validations; @@ -543,8 +536,7 @@ class AmendmentTable_test final : public beast::unit_test::suite for (auto const& [hash, nVotes] : votes) { - if (rules.enabled(fixAmendmentMajorityCalc) ? nVotes >= i - : nVotes > i) + if (feat == fixAmendmentMajorityCalc ? nVotes >= i : nVotes > i) { // We vote yes on this amendment field.push_back(hash); @@ -568,8 +560,8 @@ class AmendmentTable_test final : public beast::unit_test::suite ourVotes = table.doValidation(enabled); - auto actions = - table.doVoting(rules, roundTime, enabled, majority, validations); + auto actions = table.doVoting( + Rules({feat}), roundTime, enabled, majority, validations); for (auto const& [hash, action] : actions) { // This code assumes other validators do as we do @@ -608,25 +600,24 @@ class AmendmentTable_test final : public beast::unit_test::suite // No vote on unknown amendment void - testNoOnUnknown(FeatureBitset const& feat) + testNoOnUnknown(uint256 const& feat) { testcase("Vote NO on unknown"); auto const testAmendment = amendmentId("TestAmendment"); + auto const validators = makeValidators(10); - test::jtx::Env env{*this, feat}; + test::jtx::Env env{*this}; auto table = makeTable(env, weeks(2), emptyYes_, emptySection_, emptySection_); - auto const validators = makeValidators(10, table); - std::vector> votes; std::vector ourVotes; std::set enabled; majorityAmendments_t majority; doRound( - env.current()->rules(), + feat, *table, weeks{1}, validators, @@ -641,7 +632,7 @@ class AmendmentTable_test final : public beast::unit_test::suite votes.emplace_back(testAmendment, validators.size()); doRound( - env.current()->rules(), + feat, *table, weeks{2}, validators, @@ -652,12 +643,12 @@ class AmendmentTable_test final : public beast::unit_test::suite BEAST_EXPECT(ourVotes.empty()); BEAST_EXPECT(enabled.empty()); - majority[testAmendment] = hourTime(weeks{1}); + majority[testAmendment] = weekTime(weeks{1}); // Note that the simulation code assumes others behave as we do, // so the amendment won't get enabled doRound( - env.current()->rules(), + feat, *table, weeks{5}, validators, @@ -671,13 +662,13 @@ class AmendmentTable_test final : public beast::unit_test::suite // No vote on vetoed amendment void - testNoOnVetoed(FeatureBitset const& feat) + testNoOnVetoed(uint256 const& feat) { testcase("Vote NO on vetoed"); auto const testAmendment = amendmentId("vetoedAmendment"); - test::jtx::Env env{*this, feat}; + test::jtx::Env env{*this}; auto table = makeTable( env, weeks(2), @@ -685,7 +676,7 @@ class AmendmentTable_test final : public beast::unit_test::suite emptySection_, makeSection(testAmendment)); - auto const validators = makeValidators(10, table); + auto const validators = makeValidators(10); std::vector> votes; std::vector ourVotes; @@ -693,7 +684,7 @@ class AmendmentTable_test final : public beast::unit_test::suite majorityAmendments_t majority; doRound( - env.current()->rules(), + feat, *table, weeks{1}, validators, @@ -708,7 +699,7 @@ class AmendmentTable_test final : public beast::unit_test::suite votes.emplace_back(testAmendment, validators.size()); doRound( - env.current()->rules(), + feat, *table, weeks{2}, validators, @@ -719,10 +710,10 @@ class AmendmentTable_test final : public beast::unit_test::suite BEAST_EXPECT(ourVotes.empty()); BEAST_EXPECT(enabled.empty()); - majority[testAmendment] = hourTime(weeks{1}); + majority[testAmendment] = weekTime(weeks{1}); doRound( - env.current()->rules(), + feat, *table, weeks{5}, validators, @@ -736,16 +727,15 @@ class AmendmentTable_test final : public beast::unit_test::suite // Vote on and enable known, not-enabled amendment void - testVoteEnable(FeatureBitset const& feat) + testVoteEnable(uint256 const& feat) { testcase("voteEnable"); - test::jtx::Env env{*this, feat}; + test::jtx::Env env{*this}; auto table = makeTable( env, weeks(2), makeDefaultYes(yes_), emptySection_, emptySection_); - auto const validators = makeValidators(10, table); - + auto const validators = makeValidators(10); std::vector> votes; std::vector ourVotes; std::set enabled; @@ -753,7 +743,7 @@ class AmendmentTable_test final : public beast::unit_test::suite // Week 1: We should vote for all known amendments not enabled doRound( - env.current()->rules(), + feat, *table, weeks{1}, validators, @@ -772,7 +762,7 @@ class AmendmentTable_test final : public beast::unit_test::suite // Week 2: We should recognize a majority doRound( - env.current()->rules(), + feat, *table, weeks{2}, validators, @@ -784,11 +774,11 @@ class AmendmentTable_test final : public beast::unit_test::suite BEAST_EXPECT(enabled.empty()); for (auto const& i : yes_) - BEAST_EXPECT(majority[amendmentId(i)] == hourTime(weeks{2})); + BEAST_EXPECT(majority[amendmentId(i)] == weekTime(weeks{2})); // Week 5: We should enable the amendment doRound( - env.current()->rules(), + feat, *table, weeks{5}, validators, @@ -800,7 +790,7 @@ class AmendmentTable_test final : public beast::unit_test::suite // Week 6: We should remove it from our votes and from having a majority doRound( - env.current()->rules(), + feat, *table, weeks{6}, validators, @@ -816,12 +806,12 @@ class AmendmentTable_test final : public beast::unit_test::suite // Detect majority at 80%, enable later void - testDetectMajority(FeatureBitset const& feat) + testDetectMajority(uint256 const& feat) { testcase("detectMajority"); auto const testAmendment = amendmentId("detectMajority"); - test::jtx::Env env{*this, feat}; + test::jtx::Env env{*this}; auto table = makeTable( env, weeks(2), @@ -829,7 +819,7 @@ class AmendmentTable_test final : public beast::unit_test::suite emptySection_, emptySection_); - auto const validators = makeValidators(16, table); + auto const validators = makeValidators(16); std::set enabled; majorityAmendments_t majority; @@ -843,7 +833,7 @@ class AmendmentTable_test final : public beast::unit_test::suite votes.emplace_back(testAmendment, i); doRound( - env.current()->rules(), + feat, *table, weeks{i}, validators, @@ -885,13 +875,14 @@ class AmendmentTable_test final : public beast::unit_test::suite // Detect loss of majority void - testLostMajority(FeatureBitset const& feat) + testLostMajority(uint256 const& feat) { testcase("lostMajority"); auto const testAmendment = amendmentId("lostMajority"); + auto const validators = makeValidators(16); - test::jtx::Env env{*this, feat}; + test::jtx::Env env{*this}; auto table = makeTable( env, weeks(8), @@ -899,8 +890,6 @@ class AmendmentTable_test final : public beast::unit_test::suite emptySection_, emptySection_); - auto const validators = makeValidators(16, table); - std::set enabled; majorityAmendments_t majority; @@ -912,7 +901,7 @@ class AmendmentTable_test final : public beast::unit_test::suite votes.emplace_back(testAmendment, validators.size()); doRound( - env.current()->rules(), + feat, *table, weeks{1}, validators, @@ -934,7 +923,7 @@ class AmendmentTable_test final : public beast::unit_test::suite votes.emplace_back(testAmendment, validators.size() - i); doRound( - env.current()->rules(), + feat, *table, weeks{i + 1}, validators, @@ -960,258 +949,6 @@ class AmendmentTable_test final : public beast::unit_test::suite } } - // Exercise the UNL changing while voting is in progress. - void - testChangedUNL(FeatureBitset const& feat) - { - // This test doesn't work without fixAmendmentMajorityCalc enabled. - if (!feat[fixAmendmentMajorityCalc]) - return; - - testcase("changedUNL"); - - auto const testAmendment = amendmentId("changedUNL"); - test::jtx::Env env{*this, feat}; - auto table = makeTable( - env, - weeks(8), - makeDefaultYes(testAmendment), - emptySection_, - emptySection_); - - std::vector> validators = - makeValidators(10, table); - - std::set enabled; - majorityAmendments_t majority; - - { - // 10 validators with 2 voting against won't get majority. - std::vector> votes; - std::vector ourVotes; - - votes.emplace_back(testAmendment, validators.size() - 2); - - doRound( - env.current()->rules(), - *table, - weeks{1}, - validators, - votes, - ourVotes, - enabled, - majority); - - BEAST_EXPECT(enabled.empty()); - BEAST_EXPECT(majority.empty()); - } - - // Add one new validator to the UNL. - validators.emplace_back(randomKeyPair(KeyType::secp256k1)); - - // A lambda that updates the AmendmentTable with the latest - // trusted validators. - auto callTrustChanged = - [](std::vector> const& validators, - std::unique_ptr const& table) { - // We need a hash_set to pass to trustChanged. - hash_set trustedValidators; - trustedValidators.reserve(validators.size()); - std::for_each( - validators.begin(), - validators.end(), - [&trustedValidators](auto const& val) { - trustedValidators.insert(val.first); - }); - - // Tell the AmendmentTable that the UNL changed. - table->trustChanged(trustedValidators); - }; - - // Tell the table that there's been a change in trusted validators. - callTrustChanged(validators, table); - - { - // 11 validators with 2 voting against gains majority. - std::vector> votes; - std::vector ourVotes; - - votes.emplace_back(testAmendment, validators.size() - 2); - - doRound( - env.current()->rules(), - *table, - weeks{2}, - validators, - votes, - ourVotes, - enabled, - majority); - - BEAST_EXPECT(enabled.empty()); - BEAST_EXPECT(!majority.empty()); - } - { - // One of the validators goes flaky and doesn't send validations - // (without the UNL changing) so the amendment loses majority. - std::pair const savedValidator = - validators.front(); - validators.erase(validators.begin()); - - std::vector> votes; - std::vector ourVotes; - - votes.emplace_back(testAmendment, validators.size() - 2); - - doRound( - env.current()->rules(), - *table, - weeks{3}, - validators, - votes, - ourVotes, - enabled, - majority); - - BEAST_EXPECT(enabled.empty()); - BEAST_EXPECT(majority.empty()); - - // Simulate the validator re-syncing to the network by adding it - // back to the validators vector - validators.insert(validators.begin(), savedValidator); - - votes.front().second = validators.size() - 2; - - doRound( - env.current()->rules(), - *table, - weeks{4}, - validators, - votes, - ourVotes, - enabled, - majority); - - BEAST_EXPECT(enabled.empty()); - BEAST_EXPECT(!majority.empty()); - - // Finally, remove one validator from the UNL and see that majority - // is lost. - validators.erase(validators.begin()); - - // Tell the table that there's been a change in trusted validators. - callTrustChanged(validators, table); - - votes.front().second = validators.size() - 2; - - doRound( - env.current()->rules(), - *table, - weeks{5}, - validators, - votes, - ourVotes, - enabled, - majority); - - BEAST_EXPECT(enabled.empty()); - BEAST_EXPECT(majority.empty()); - } - } - - // Exercise a validator losing connectivity and then regaining it after - // extended delays. Depending on how long that delay is an amendment - // either will or will not go live. - void - testValidatorFlapping(FeatureBitset const& feat) - { - // This test doesn't work without fixAmendmentMajorityCalc enabled. - if (!feat[fixAmendmentMajorityCalc]) - return; - - testcase("validatorFlapping"); - - // We run a test where a validator flaps on and off every 23 hours - // and another one one where it flaps on and off every 25 hours. - // - // Since the local validator vote record expires after 24 hours, - // with 23 hour flapping the amendment will go live. But with 25 - // hour flapping the amendment will not go live. - for (int flapRateHours : {23, 25}) - { - test::jtx::Env env{*this, feat}; - auto const testAmendment = amendmentId("validatorFlapping"); - auto table = makeTable( - env, - weeks(1), - makeDefaultYes(testAmendment), - emptySection_, - emptySection_); - - // Make two lists of validators, one with a missing validator, to - // make it easy to simulate validator flapping. - auto const allValidators = makeValidators(11, table); - decltype(allValidators) const mostValidators( - allValidators.begin() + 1, allValidators.end()); - BEAST_EXPECT(allValidators.size() == mostValidators.size() + 1); - - std::set enabled; - majorityAmendments_t majority; - - std::vector> votes; - std::vector ourVotes; - - votes.emplace_back(testAmendment, allValidators.size() - 2); - - int delay = flapRateHours; - // Loop for 1 week plus a day. - for (int hour = 1; hour < (24 * 8); ++hour) - { - decltype(allValidators) const& thisHoursValidators = - (delay < flapRateHours) ? mostValidators : allValidators; - delay = delay == flapRateHours ? 0 : delay + 1; - - votes.front().second = thisHoursValidators.size() - 2; - - using namespace std::chrono; - doRound( - env.current()->rules(), - *table, - hours(hour), - thisHoursValidators, - votes, - ourVotes, - enabled, - majority); - - if (hour <= (24 * 7) || flapRateHours > 24) - { - // The amendment should not be enabled under any - // circumstance until one week has elapsed. - BEAST_EXPECT(enabled.empty()); - - // If flapping is less than 24 hours, there should be - // no flapping. Otherwise we should only have majority - // if allValidators vote -- which means there are no - // missing validators. - bool const expectMajority = (delay <= 24) - ? true - : &thisHoursValidators == &allValidators; - BEAST_EXPECT(majority.empty() != expectMajority); - } - else - { - // We're... - // o Past one week, and - // o AmendmentFlapping was less than 24 hours. - // The amendment should be enabled. - BEAST_EXPECT(!enabled.empty()); - BEAST_EXPECT(majority.empty()); - } - } - } - } - void testHasUnsupported() { @@ -1256,30 +993,25 @@ class AmendmentTable_test final : public beast::unit_test::suite } void - testFeature(FeatureBitset const& feat) + testFeature(uint256 const& feat) { testNoOnUnknown(feat); testNoOnVetoed(feat); testVoteEnable(feat); testDetectMajority(feat); testLostMajority(feat); - testChangedUNL(feat); - testValidatorFlapping(feat); } void run() override { - FeatureBitset const all{test::jtx::supported_amendments()}; - FeatureBitset const fixMajorityCalc{fixAmendmentMajorityCalc}; - testConstruct(); testGet(); testBadConfig(); testEnableVeto(); testHasUnsupported(); - testFeature(all - fixMajorityCalc); - testFeature(all); + testFeature({}); + testFeature(fixAmendmentMajorityCalc); } }; diff --git a/src/test/app/Flow_test.cpp b/src/test/app/Flow_test.cpp index 131cad6f042..920f7a6e058 100644 --- a/src/test/app/Flow_test.cpp +++ b/src/test/app/Flow_test.cpp @@ -473,7 +473,7 @@ struct Flow_test : public beast::unit_test::suite false, false, true, - false, + OfferCrossing::no, std::nullopt, smax, flowJournal); diff --git a/src/test/app/Offer_test.cpp b/src/test/app/Offer_test.cpp index fbf9cc890dc..25ca5a4b2dc 100644 --- a/src/test/app/Offer_test.cpp +++ b/src/test/app/Offer_test.cpp @@ -5081,6 +5081,196 @@ class Offer0_test : public beast::unit_test::suite pass(); } + void + testFillOrKill(FeatureBitset features) + { + testcase("fixFillOrKill"); + using namespace jtx; + Env env(*this, features); + Account const issuer("issuer"); + Account const maker("maker"); + Account const taker("taker"); + auto const USD = issuer["USD"]; + auto const EUR = issuer["EUR"]; + + env.fund(XRP(1'000), issuer); + env.fund(XRP(1'000), maker, taker); + env.close(); + + env.trust(USD(1'000), maker, taker); + env.trust(EUR(1'000), maker, taker); + env.close(); + + env(pay(issuer, maker, USD(1'000))); + env(pay(issuer, taker, USD(1'000))); + env(pay(issuer, maker, EUR(1'000))); + env.close(); + + auto makerUSDBalance = env.balance(maker, USD).value(); + auto takerUSDBalance = env.balance(taker, USD).value(); + auto makerEURBalance = env.balance(maker, EUR).value(); + auto takerEURBalance = env.balance(taker, EUR).value(); + auto makerXRPBalance = env.balance(maker, XRP).value(); + auto takerXRPBalance = env.balance(taker, XRP).value(); + + // tfFillOrKill, TakerPays must be filled + { + TER const err = + features[fixFillOrKill] || !features[featureFlowCross] + ? TER(tesSUCCESS) + : tecKILLED; + + env(offer(maker, XRP(100), USD(100))); + env.close(); + + env(offer(taker, USD(100), XRP(101)), + txflags(tfFillOrKill), + ter(err)); + env.close(); + + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + if (err == tesSUCCESS) + { + makerUSDBalance -= USD(100); + takerUSDBalance += USD(100); + makerXRPBalance += XRP(100).value(); + takerXRPBalance -= XRP(100).value(); + } + BEAST_EXPECT(expectOffers(env, taker, 0)); + + env(offer(maker, USD(100), XRP(100))); + env.close(); + + env(offer(taker, XRP(100), USD(101)), + txflags(tfFillOrKill), + ter(err)); + env.close(); + + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + if (err == tesSUCCESS) + { + makerUSDBalance += USD(100); + takerUSDBalance -= USD(100); + makerXRPBalance -= XRP(100).value(); + takerXRPBalance += XRP(100).value(); + } + BEAST_EXPECT(expectOffers(env, taker, 0)); + + env(offer(maker, USD(100), EUR(100))); + env.close(); + + env(offer(taker, EUR(100), USD(101)), + txflags(tfFillOrKill), + ter(err)); + env.close(); + + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + if (err == tesSUCCESS) + { + makerUSDBalance += USD(100); + takerUSDBalance -= USD(100); + makerEURBalance -= EUR(100); + takerEURBalance += EUR(100); + } + BEAST_EXPECT(expectOffers(env, taker, 0)); + } + + // tfFillOrKill + tfSell, TakerGets must be filled + { + env(offer(maker, XRP(101), USD(101))); + env.close(); + + env(offer(taker, USD(100), XRP(101)), + txflags(tfFillOrKill | tfSell)); + env.close(); + + makerUSDBalance -= USD(101); + takerUSDBalance += USD(101); + makerXRPBalance += XRP(101).value() - txfee(env, 1); + takerXRPBalance -= XRP(101).value() + txfee(env, 1); + BEAST_EXPECT(expectOffers(env, taker, 0)); + + env(offer(maker, USD(101), XRP(101))); + env.close(); + + env(offer(taker, XRP(100), USD(101)), + txflags(tfFillOrKill | tfSell)); + env.close(); + + makerUSDBalance += USD(101); + takerUSDBalance -= USD(101); + makerXRPBalance -= XRP(101).value() + txfee(env, 1); + takerXRPBalance += XRP(101).value() - txfee(env, 1); + BEAST_EXPECT(expectOffers(env, taker, 0)); + + env(offer(maker, USD(101), EUR(101))); + env.close(); + + env(offer(taker, EUR(100), USD(101)), + txflags(tfFillOrKill | tfSell)); + env.close(); + + makerUSDBalance += USD(101); + takerUSDBalance -= USD(101); + makerEURBalance -= EUR(101); + takerEURBalance += EUR(101); + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + BEAST_EXPECT(expectOffers(env, taker, 0)); + } + + // Fail regardless of fixFillOrKill amendment + for (auto const flags : {tfFillOrKill, tfFillOrKill + tfSell}) + { + env(offer(maker, XRP(100), USD(100))); + env.close(); + + env(offer(taker, USD(100), XRP(99)), + txflags(flags), + ter(tecKILLED)); + env.close(); + + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + BEAST_EXPECT(expectOffers(env, taker, 0)); + + env(offer(maker, USD(100), XRP(100))); + env.close(); + + env(offer(taker, XRP(100), USD(99)), + txflags(flags), + ter(tecKILLED)); + env.close(); + + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + BEAST_EXPECT(expectOffers(env, taker, 0)); + + env(offer(maker, USD(100), EUR(100))); + env.close(); + + env(offer(taker, EUR(100), USD(99)), + txflags(flags), + ter(tecKILLED)); + env.close(); + + makerXRPBalance -= txfee(env, 1); + takerXRPBalance -= txfee(env, 1); + BEAST_EXPECT(expectOffers(env, taker, 0)); + } + + BEAST_EXPECT( + env.balance(maker, USD) == makerUSDBalance && + env.balance(taker, USD) == takerUSDBalance && + env.balance(maker, EUR) == makerEURBalance && + env.balance(taker, EUR) == takerEURBalance && + env.balance(maker, XRP) == makerXRPBalance && + env.balance(taker, XRP) == takerXRPBalance); + } + void testAll(FeatureBitset features) { @@ -5142,6 +5332,7 @@ class Offer0_test : public beast::unit_test::suite testTicketCancelOffer(features); testRmSmallIncreasedQOffersXRP(features); testRmSmallIncreasedQOffersIOU(features); + testFillOrKill(features); } void @@ -5155,12 +5346,14 @@ class Offer0_test : public beast::unit_test::suite fixRmSmallIncreasedQOffers}; static FeatureBitset const immediateOfferKilled{ featureImmediateOfferKilled}; + FeatureBitset const fillOrKill{fixFillOrKill}; - static std::array const feats{ + static std::array const feats{ all - takerDryOffer - immediateOfferKilled, all - flowCross - takerDryOffer - immediateOfferKilled, all - flowCross - immediateOfferKilled, - all - rmSmallIncreasedQOffers - immediateOfferKilled, + all - rmSmallIncreasedQOffers - immediateOfferKilled - fillOrKill, + all - fillOrKill, all}; if (BEAST_EXPECT(instance < feats.size())) @@ -5210,7 +5403,16 @@ class Offer4_test : public Offer0_test void run() override { - Offer0_test::run(4, true); + Offer0_test::run(4); + } +}; + +class Offer5_test : public Offer0_test +{ + void + run() override + { + Offer0_test::run(5, true); } }; @@ -5225,10 +5427,12 @@ class Offer_manual_test : public Offer0_test FeatureBitset const f1513{fix1513}; FeatureBitset const immediateOfferKilled{featureImmediateOfferKilled}; FeatureBitset const takerDryOffer{fixTakerDryOfferRemoval}; + FeatureBitset const fillOrKill{fixFillOrKill}; testAll(all - flowCross - f1513 - immediateOfferKilled); testAll(all - flowCross - immediateOfferKilled); - testAll(all - immediateOfferKilled); + testAll(all - immediateOfferKilled - fillOrKill); + testAll(all - fillOrKill); testAll(all); testAll(all - flowCross - takerDryOffer); @@ -5240,6 +5444,7 @@ BEAST_DEFINE_TESTSUITE_PRIO(Offer1, tx, ripple, 4); BEAST_DEFINE_TESTSUITE_PRIO(Offer2, tx, ripple, 4); BEAST_DEFINE_TESTSUITE_PRIO(Offer3, tx, ripple, 4); BEAST_DEFINE_TESTSUITE_PRIO(Offer4, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(Offer5, tx, ripple, 4); BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(Offer_manual, tx, ripple, 20); } // namespace test diff --git a/src/test/app/PayStrand_test.cpp b/src/test/app/PayStrand_test.cpp index a70ab099745..55c15e54fc0 100644 --- a/src/test/app/PayStrand_test.cpp +++ b/src/test/app/PayStrand_test.cpp @@ -656,7 +656,7 @@ struct PayStrand_test : public beast::unit_test::suite sendMaxIssue, path, true, - false, + OfferCrossing::no, ammContext, env.app().logs().journal("Flow")); BEAST_EXPECT(ter == expTer); @@ -684,7 +684,7 @@ struct PayStrand_test : public beast::unit_test::suite /*sendMaxIssue*/ EUR.issue(), path, true, - false, + OfferCrossing::no, ammContext, env.app().logs().journal("Flow")); (void)_; @@ -701,7 +701,7 @@ struct PayStrand_test : public beast::unit_test::suite /*sendMaxIssue*/ EUR.issue(), path, true, - false, + OfferCrossing::no, ammContext, env.app().logs().journal("Flow")); (void)_; @@ -821,7 +821,7 @@ struct PayStrand_test : public beast::unit_test::suite USD.issue(), STPath(), true, - false, + OfferCrossing::no, ammContext, flowJournal); BEAST_EXPECT(r.first == temBAD_PATH); @@ -837,7 +837,7 @@ struct PayStrand_test : public beast::unit_test::suite std::nullopt, STPath(), true, - false, + OfferCrossing::no, ammContext, flowJournal); BEAST_EXPECT(r.first == temBAD_PATH); @@ -853,7 +853,7 @@ struct PayStrand_test : public beast::unit_test::suite std::nullopt, STPath(), true, - false, + OfferCrossing::no, ammContext, flowJournal); BEAST_EXPECT(r.first == temBAD_PATH); @@ -990,7 +990,7 @@ struct PayStrand_test : public beast::unit_test::suite std::nullopt, STPath(), true, - false, + OfferCrossing::no, ammContext, env.app().logs().journal("Flow")); BEAST_EXPECT(ter == tesSUCCESS); @@ -1017,7 +1017,7 @@ struct PayStrand_test : public beast::unit_test::suite USD.issue(), path, false, - false, + OfferCrossing::no, ammContext, env.app().logs().journal("Flow")); BEAST_EXPECT(ter == tesSUCCESS); diff --git a/src/test/app/TheoreticalQuality_test.cpp b/src/test/app/TheoreticalQuality_test.cpp index 5da26512deb..b76ea723542 100644 --- a/src/test/app/TheoreticalQuality_test.cpp +++ b/src/test/app/TheoreticalQuality_test.cpp @@ -266,7 +266,7 @@ class TheoreticalQuality_test : public beast::unit_test::suite rcp.paths, /*defaultPaths*/ rcp.paths.empty(), sb.rules().enabled(featureOwnerPaysFee), - /*offerCrossing*/ false, + OfferCrossing::no, ammContext, dummyJ); diff --git a/src/test/app/XChain_test.cpp b/src/test/app/XChain_test.cpp index 9b8aaf397dc..e6e193adf10 100644 --- a/src/test/app/XChain_test.cpp +++ b/src/test/app/XChain_test.cpp @@ -4209,6 +4209,73 @@ struct XChain_test : public beast::unit_test::suite, } } + void + testBadPublicKey() + { + using namespace jtx; + + testcase("Bad attestations"); + { + // Create a bridge and add an attestation with a bad public key + XEnv scEnv(*this, true); + std::uint32_t const claimID = 1; + std::optional dst{scBob}; + auto const amt = XRP(1000); + scEnv.tx(create_bridge(Account::master, jvb)) + .tx(jtx::signers(Account::master, quorum, signers)) + .close(); + scEnv.tx(xchain_create_claim_id(scAlice, jvb, reward, mcAlice)) + .close(); + auto jvAtt = claim_attestation( + scAttester, + jvb, + mcAlice, + amt, + payees[UT_XCHAIN_DEFAULT_QUORUM], + true, + claimID, + dst, + signers[UT_XCHAIN_DEFAULT_QUORUM]); + { + // Change to an invalid keytype + auto k = jvAtt["PublicKey"].asString(); + k.at(1) = '9'; + jvAtt["PublicKey"] = k; + } + scEnv.tx(jvAtt, ter(temMALFORMED)).close(); + } + { + // Create a bridge and add an create account attestation with a bad + // public key + XEnv scEnv(*this, true); + std::uint32_t const createCount = 1; + Account dst{scBob}; + auto const amt = XRP(1000); + auto const rewardAmt = XRP(1); + scEnv.tx(create_bridge(Account::master, jvb)) + .tx(jtx::signers(Account::master, quorum, signers)) + .close(); + auto jvAtt = create_account_attestation( + scAttester, + jvb, + mcAlice, + amt, + rewardAmt, + payees[UT_XCHAIN_DEFAULT_QUORUM], + true, + createCount, + dst, + signers[UT_XCHAIN_DEFAULT_QUORUM]); + { + // Change to an invalid keytype + auto k = jvAtt["PublicKey"].asString(); + k.at(1) = '9'; + jvAtt["PublicKey"] = k; + } + scEnv.tx(jvAtt, ter(temMALFORMED)).close(); + } + } + void run() override { @@ -4226,6 +4293,7 @@ struct XChain_test : public beast::unit_test::suite, testXChainCreateAccount(); testFeeDipsIntoReserve(); testXChainDeleteDoor(); + testBadPublicKey(); } }; diff --git a/src/test/basics/PerfLog_test.cpp b/src/test/basics/PerfLog_test.cpp index 79944e0ed71..f0a6645195b 100644 --- a/src/test/basics/PerfLog_test.cpp +++ b/src/test/basics/PerfLog_test.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -309,7 +310,8 @@ class PerfLog_test : public beast::unit_test::suite // Get the all the labels we can use for RPC interfaces without // causing an assert. - std::vector labels{ripple::RPC::getHandlerNames()}; + std::vector labels = + test::jtx::make_vector(ripple::RPC::getHandlerNames()); std::shuffle(labels.begin(), labels.end(), default_prng()); // Get two IDs to associate with each label. Errors tend to happen at diff --git a/src/test/jtx/TestHelpers.h b/src/test/jtx/TestHelpers.h index ff681ffa50b..b065cec470a 100644 --- a/src/test/jtx/TestHelpers.h +++ b/src/test/jtx/TestHelpers.h @@ -27,10 +27,27 @@ #include #include +#include + namespace ripple { namespace test { namespace jtx { +// TODO We only need this long "requires" clause as polyfill, for C++20 +// implementations which are missing header. Replace with +// `std::ranges::range`, and accordingly use std::ranges::begin/end +// when we have moved to better compilers. +template +auto +make_vector(Input const& input) requires requires(Input& v) +{ + std::begin(v); + std::end(v); +} +{ + return std::vector(std::begin(input), std::end(input)); +} + // Functions used in debugging Json::Value getAccountOffers(Env& env, AccountID const& acct, bool current = false); diff --git a/src/test/jtx/impl/Env.cpp b/src/test/jtx/impl/Env.cpp index 5e1aaa166f0..3467a42cbbb 100644 --- a/src/test/jtx/impl/Env.cpp +++ b/src/test/jtx/impl/Env.cpp @@ -463,7 +463,13 @@ Env::do_rpc( std::vector const& args, std::unordered_map const& headers) { - return rpcClient(args, app().config(), app().logs(), headers).second; + return rpcClient( + args, + app().config(), + app().logs(), + RPC::apiMaximumSupportedVersion, + headers) + .second; } void diff --git a/src/test/jtx/impl/utility.cpp b/src/test/jtx/impl/utility.cpp index a2acdd87b4d..196fd9258d7 100644 --- a/src/test/jtx/impl/utility.cpp +++ b/src/test/jtx/impl/utility.cpp @@ -18,6 +18,8 @@ //============================================================================== #include +#include +#include #include #include #include @@ -73,6 +75,38 @@ fill_seq(Json::Value& jv, ReadView const& view) jv[jss::Sequence] = ar->getFieldU32(sfSequence); } +Json::Value +cmdToJSONRPC( + std::vector const& args, + beast::Journal j, + unsigned int apiVersion) +{ + Json::Value jv = Json::Value(Json::objectValue); + auto const paramsObj = rpcCmdToJson(args, jv, apiVersion, j); + + // Re-use jv to return our formatted result. + jv.clear(); + + // Allow parser to rewrite method. + jv[jss::method] = paramsObj.isMember(jss::method) + ? paramsObj[jss::method].asString() + : args[0]; + + // If paramsObj is not empty, put it in a [params] array. + if (paramsObj.begin() != paramsObj.end()) + { + auto& paramsArray = Json::setArray(jv, jss::params); + paramsArray.append(paramsObj); + } + if (paramsObj.isMember(jss::jsonrpc)) + jv[jss::jsonrpc] = paramsObj[jss::jsonrpc]; + if (paramsObj.isMember(jss::ripplerpc)) + jv[jss::ripplerpc] = paramsObj[jss::ripplerpc]; + if (paramsObj.isMember(jss::id)) + jv[jss::id] = paramsObj[jss::id]; + return jv; +} + } // namespace jtx } // namespace test } // namespace ripple diff --git a/src/test/jtx/utility.h b/src/test/jtx/utility.h index cb013a68435..7bc9fbaa817 100644 --- a/src/test/jtx/utility.h +++ b/src/test/jtx/utility.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,13 @@ fill_fee(Json::Value& jv, ReadView const& view); void fill_seq(Json::Value& jv, ReadView const& view); +/** Given a rippled unit test rpc command, return the corresponding JSON. */ +Json::Value +cmdToJSONRPC( + std::vector const& args, + beast::Journal j, + unsigned int apiVersion = RPC::apiMaximumSupportedVersion); + } // namespace jtx } // namespace test } // namespace ripple diff --git a/src/test/rpc/AccountTx_test.cpp b/src/test/rpc/AccountTx_test.cpp index 8c583ee1254..3834d623dca 100644 --- a/src/test/rpc/AccountTx_test.cpp +++ b/src/test/rpc/AccountTx_test.cpp @@ -120,22 +120,56 @@ class AccountTx_test : public beast::unit_test::suite // All other ledgers have no txs auto hasTxs = [apiVersion](Json::Value const& j) { - return j.isMember(jss::result) && - (j[jss::result][jss::status] == "success") && - (j[jss::result][jss::transactions].size() == 2) && - (j[jss::result][jss::transactions][0u][jss::tx] - [jss::TransactionType] == jss::AccountSet) && - (j[jss::result][jss::transactions][1u][jss::tx] - [jss::TransactionType] == jss::Payment) && - (j[jss::result][jss::transactions][1u][jss::tx] - [jss::DeliverMax] == "10000000010") && - ((apiVersion > 1 && - !j[jss::result][jss::transactions][1u][jss::tx].isMember( - jss::Amount)) || - (apiVersion <= 1 && - j[jss::result][jss::transactions][1u][jss::tx][jss::Amount] == - j[jss::result][jss::transactions][1u][jss::tx] - [jss::DeliverMax])); + switch (apiVersion) + { + case 1: + return j.isMember(jss::result) && + (j[jss::result][jss::status] == "success") && + (j[jss::result][jss::transactions].size() == 2) && + (j[jss::result][jss::transactions][0u][jss::tx] + [jss::TransactionType] == jss::AccountSet) && + (j[jss::result][jss::transactions][1u][jss::tx] + [jss::TransactionType] == jss::Payment) && + (j[jss::result][jss::transactions][1u][jss::tx] + [jss::DeliverMax] == "10000000010") && + (j[jss::result][jss::transactions][1u][jss::tx] + [jss::Amount] == + j[jss::result][jss::transactions][1u][jss::tx] + [jss::DeliverMax]); + case 2: + if (j.isMember(jss::result) && + (j[jss::result][jss::status] == "success") && + (j[jss::result][jss::transactions].size() == 2) && + (j[jss::result][jss::transactions][0u][jss::tx_json] + [jss::TransactionType] == jss::AccountSet)) + { + auto const& payment = + j[jss::result][jss::transactions][1u]; + + return (payment.isMember(jss::tx_json)) && + (payment[jss::tx_json][jss::TransactionType] == + jss::Payment) && + (payment[jss::tx_json][jss::DeliverMax] == + "10000000010") && + (!payment[jss::tx_json].isMember(jss::Amount)) && + (!payment[jss::tx_json].isMember(jss::hash)) && + (payment[jss::hash] == + "9F3085D85F472D1CC29627F260DF68EDE59D42D1D0C33E345" + "ECF0D4CE981D0A8") && + (payment[jss::validated] == true) && + (payment[jss::ledger_index] == 3) && + (payment[jss::ledger_hash] == + "5476DCD816EA04CBBA57D47BBF1FC58A5217CC93A5ADD79CB" + "580A5AFDD727E33") && + (payment[jss::close_time_iso] == + "2000-01-01T00:00:10Z"); + } + else + return false; + + default: + return false; + } }; auto noTxs = [](Json::Value const& j) { diff --git a/src/test/rpc/Handler_test.cpp b/src/test/rpc/Handler_test.cpp new file mode 100644 index 00000000000..5160a68aac2 --- /dev/null +++ b/src/test/rpc/Handler_test.cpp @@ -0,0 +1,132 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace ripple::test { + +// NOTE: there should be no need for this function; +// `std::cout << some_duration` should just work if built with a compliant +// C++20 compiler. Sadly, we are not using one, as of today +// TODO: remove this operator<< overload when we bump compiler version +std::ostream& +operator<<(std::ostream& os, std::chrono::nanoseconds ns) +{ + return (os << ns.count() << "ns"); +} + +// NOTE This is a rather naive effort at a microbenchmark. Ideally we want +// Google Benchmark, or something similar. Also, this actually does not belong +// to unit tests, as it makes little sense to run it in conditions very +// dissimilar to how rippled will normally work. +// TODO as https://github.com/XRPLF/rippled/issues/4765 + +class Handler_test : public beast::unit_test::suite +{ + auto + time(std::size_t n, auto f, auto prng) -> auto + { + using clock = std::chrono::steady_clock; + assert(n > 0); + double sum = 0; + double sum_squared = 0; + std::size_t j = 0; + while (j < n) + { + // Generate 100 inputs upfront, separated from the inner loop + std::array inputs = {}; + for (auto& i : inputs) + { + i = prng(); + } + + // Take 100 samples, then sort and throw away 35 from each end, + // using only middle 30. This helps to reduce measurement noise. + std::array samples = {}; + for (std::size_t k = 0; k < 100; ++k) + { + auto start = std::chrono::steady_clock::now(); + f(inputs[k]); + samples[k] = (std::chrono::steady_clock::now() - start).count(); + } + + std::sort(samples.begin(), samples.end()); + for (std::size_t k = 35; k < 65; ++k) + { + j += 1; + sum += samples[k]; + sum_squared += (samples[k] * samples[k]); + } + } + + const double mean_squared = (sum * sum) / (j * j); + return std::make_tuple( + clock::duration{static_cast(sum / j)}, + clock::duration{ + static_cast(std::sqrt((sum_squared / j) - mean_squared))}, + j); + } + + void + reportLookupPerformance() + { + testcase("Handler lookup performance"); + + std::random_device dev; + std::ranlux48 prng(dev()); + + std::vector names = + test::jtx::make_vector(ripple::RPC::getHandlerNames()); + + std::uniform_int_distribution distr{0, names.size() - 1}; + + std::size_t dummy = 0; + auto const [mean, stdev, n] = time( + 1'000'000, + [&](std::size_t i) { + auto const d = RPC::getHandler(1, false, names[i]); + dummy = dummy + i + (int)d->role_; + }, + [&]() -> std::size_t { return distr(prng); }); + + std::cout << "mean=" << mean << " stdev=" << stdev << " N=" << n + << '\n'; + + BEAST_EXPECT(dummy != 0); + } + +public: + void + run() override + { + reportLookupPerformance(); + } +}; + +BEAST_DEFINE_TESTSUITE_MANUAL(Handler, test, ripple); + +} // namespace ripple::test diff --git a/src/test/rpc/JSONRPC_test.cpp b/src/test/rpc/JSONRPC_test.cpp index 5d4c09ef8d1..1e8ce554be3 100644 --- a/src/test/rpc/JSONRPC_test.cpp +++ b/src/test/rpc/JSONRPC_test.cpp @@ -2536,17 +2536,19 @@ class JSONRPC_test : public beast::unit_test::suite // A list of all the functions we want to test. using signFunc = Json::Value (*)( Json::Value params, + unsigned int apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, - Application & app); + Application& app); using submitFunc = Json::Value (*)( Json::Value params, + unsigned int apiVersion, NetworkOPs::FailHard failType, Role role, std::chrono::seconds validatedLedgerAge, - Application & app, + Application& app, ProcessTransactionFn const& processTransaction, RPC::SubmitSync sync); @@ -2586,6 +2588,7 @@ class JSONRPC_test : public beast::unit_test::suite assert(get<1>(testFunc) == nullptr); result = signFn( req, + 1, NetworkOPs::FailHard::yes, testRole, 1s, @@ -2597,6 +2600,7 @@ class JSONRPC_test : public beast::unit_test::suite assert(submitFn != nullptr); result = submitFn( req, + 1, NetworkOPs::FailHard::yes, testRole, 1s, diff --git a/src/test/rpc/LedgerHeader_test.cpp b/src/test/rpc/LedgerHeader_test.cpp new file mode 100644 index 00000000000..d6c0652d5a2 --- /dev/null +++ b/src/test/rpc/LedgerHeader_test.cpp @@ -0,0 +1,91 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +namespace ripple { + +class LedgerHeader_test : public beast::unit_test::suite +{ + void + testSimpleCurrent() + { + testcase("Current ledger"); + using namespace test::jtx; + Env env{*this, envconfig(no_admin)}; + + Json::Value params{Json::objectValue}; + params[jss::api_version] = 1; + params[jss::ledger_index] = "current"; + auto const result = + env.client().invoke("ledger_header", params)[jss::result]; + BEAST_EXPECT(result[jss::status] == "success"); + BEAST_EXPECT(result.isMember("ledger")); + BEAST_EXPECT(result[jss::ledger][jss::closed] == false); + BEAST_EXPECT(result[jss::validated] == false); + } + + void + testSimpleValidated() + { + testcase("Validated ledger"); + using namespace test::jtx; + Env env{*this, envconfig(no_admin)}; + + Json::Value params{Json::objectValue}; + params[jss::api_version] = 1; + params[jss::ledger_index] = "validated"; + auto const result = + env.client().invoke("ledger_header", params)[jss::result]; + BEAST_EXPECT(result[jss::status] == "success"); + BEAST_EXPECT(result.isMember("ledger")); + BEAST_EXPECT(result[jss::ledger][jss::closed] == true); + BEAST_EXPECT(result[jss::validated] == true); + } + + void + testCommandRetired() + { + testcase("Command retired from API v2"); + using namespace test::jtx; + Env env{*this, envconfig(no_admin)}; + + Json::Value params{Json::objectValue}; + params[jss::api_version] = 2; + auto const result = + env.client().invoke("ledger_header", params)[jss::result]; + BEAST_EXPECT(result[jss::error] == "unknownCmd"); + BEAST_EXPECT(result[jss::status] == "error"); + } + +public: + void + run() override + { + testSimpleCurrent(); + testSimpleValidated(); + testCommandRetired(); + } +}; + +BEAST_DEFINE_TESTSUITE(LedgerHeader, rpc, ripple); + +} // namespace ripple diff --git a/src/test/rpc/RPCCall_test.cpp b/src/test/rpc/RPCCall_test.cpp index 5385ba768e6..ce270184a26 100644 --- a/src/test/rpc/RPCCall_test.cpp +++ b/src/test/rpc/RPCCall_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -6442,7 +6443,7 @@ class RPCCall_test : public beast::unit_test::suite Json::Value got; try { - got = cmdLineToJSONRPC(args, env.journal); + got = jtx::cmdToJSONRPC(args, env.journal); } catch (std::bad_cast const&) { diff --git a/src/test/rpc/Subscribe_test.cpp b/src/test/rpc/Subscribe_test.cpp index 7725390f6b6..ea815eb27ed 100644 --- a/src/test/rpc/Subscribe_test.cpp +++ b/src/test/rpc/Subscribe_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include "ripple/json/json_value.h" #include #include #include @@ -307,6 +308,85 @@ class Subscribe_test : public beast::unit_test::suite BEAST_EXPECT(jv[jss::status] == "success"); } + void + testTransactions_APIv2() + { + testcase("transactions API version 2"); + + using namespace std::chrono_literals; + using namespace jtx; + Env env(*this); + auto wsc = makeWSClient(env.app().config()); + Json::Value stream{Json::objectValue}; + + { + // RPC subscribe to transactions stream + stream[jss::api_version] = 2; + stream[jss::streams] = Json::arrayValue; + stream[jss::streams].append("transactions"); + auto jv = wsc->invoke("subscribe", stream); + if (wsc->version() == 2) + { + BEAST_EXPECT( + jv.isMember(jss::jsonrpc) && jv[jss::jsonrpc] == "2.0"); + BEAST_EXPECT( + jv.isMember(jss::ripplerpc) && jv[jss::ripplerpc] == "2.0"); + BEAST_EXPECT(jv.isMember(jss::id) && jv[jss::id] == 5); + } + BEAST_EXPECT(jv[jss::status] == "success"); + } + + { + env.fund(XRP(10000), "alice"); + env.close(); + + // Check stream update for payment transaction + BEAST_EXPECT(wsc->findMsg(5s, [&](auto const& jv) { + return jv[jss::meta]["AffectedNodes"][1u]["CreatedNode"] + ["NewFields"][jss::Account] // + == Account("alice").human() && + jv[jss::close_time_iso] // + == "2000-01-01T00:00:10Z" && + jv[jss::validated] == true && // + jv[jss::ledger_hash] == + "0F1A9E0C109ADEF6DA2BDE19217C12BBEC57174CBDBD212B0EBDC1CEDB" + "853185" && // + !jv[jss::inLedger] && + jv[jss::ledger_index] == 3 && // + jv[jss::tx_json][jss::TransactionType] // + == jss::Payment && + jv[jss::tx_json][jss::DeliverMax] // + == "10000000010" && + !jv[jss::tx_json].isMember(jss::Amount) && + jv[jss::tx_json][jss::Fee] // + == "10" && + jv[jss::tx_json][jss::Sequence] // + == 1; + })); + + // Check stream update for accountset transaction + BEAST_EXPECT(wsc->findMsg(5s, [&](auto const& jv) { + return jv[jss::meta]["AffectedNodes"][0u]["ModifiedNode"] + ["FinalFields"][jss::Account] == + Account("alice").human(); + })); + } + + { + // RPC unsubscribe + auto jv = wsc->invoke("unsubscribe", stream); + if (wsc->version() == 2) + { + BEAST_EXPECT( + jv.isMember(jss::jsonrpc) && jv[jss::jsonrpc] == "2.0"); + BEAST_EXPECT( + jv.isMember(jss::ripplerpc) && jv[jss::ripplerpc] == "2.0"); + BEAST_EXPECT(jv.isMember(jss::id) && jv[jss::id] == 5); + } + BEAST_EXPECT(jv[jss::status] == "success"); + } + } + void testManifests() { @@ -1223,6 +1303,7 @@ class Subscribe_test : public beast::unit_test::suite testServer(); testLedger(); testTransactions(); + testTransactions_APIv2(); testManifests(); testValidations(all - xrpFees); testValidations(all); diff --git a/src/test/rpc/TransactionEntry_test.cpp b/src/test/rpc/TransactionEntry_test.cpp index 60225f4621d..ae988343a6c 100644 --- a/src/test/rpc/TransactionEntry_test.cpp +++ b/src/test/rpc/TransactionEntry_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -143,21 +144,23 @@ class TransactionEntry_test : public beast::unit_test::suite } void - testRequest() + testRequest(unsigned apiVersion) { - testcase("Basic request"); + testcase("Basic request API version " + std::to_string(apiVersion)); using namespace test::jtx; Env env{*this}; - auto check_tx = [this, &env]( + auto check_tx = [this, &env, apiVersion]( int index, std::string const txhash, - std::string const expected_json = "") { + std::string const expected_json = "", + std::string const close_time_iso = "") { // first request using ledger_index to lookup - Json::Value const resIndex{[&env, index, &txhash]() { + Json::Value const resIndex{[&env, index, &txhash, apiVersion]() { Json::Value params{Json::objectValue}; params[jss::ledger_index] = index; params[jss::tx_hash] = txhash; + params[jss::api_version] = apiVersion; return env.client().invoke( "transaction_entry", params)[jss::result]; }()}; @@ -165,7 +168,19 @@ class TransactionEntry_test : public beast::unit_test::suite if (!BEAST_EXPECTS(resIndex.isMember(jss::tx_json), txhash)) return; - BEAST_EXPECT(resIndex[jss::tx_json][jss::hash] == txhash); + if (apiVersion > 1) + { + BEAST_EXPECT(resIndex[jss::hash] == txhash); + BEAST_EXPECT(resIndex[jss::validated] == true); + BEAST_EXPECT(!resIndex[jss::tx_json].isMember(jss::Amount)); + + if (BEAST_EXPECT(!close_time_iso.empty())) + BEAST_EXPECT( + resIndex[jss::close_time_iso] == close_time_iso); + } + else + BEAST_EXPECT(resIndex[jss::tx_json][jss::hash] == txhash); + if (!expected_json.empty()) { Json::Value expected; @@ -198,12 +213,14 @@ class TransactionEntry_test : public beast::unit_test::suite Json::Value params{Json::objectValue}; params[jss::ledger_hash] = resIndex[jss::ledger_hash]; params[jss::tx_hash] = txhash; + params[jss::api_version] = apiVersion; Json::Value const resHash = env.client().invoke( "transaction_entry", params)[jss::result]; BEAST_EXPECT(resHash == resIndex); } // Use the command line form with the index. + if (apiVersion == RPC::apiMaximumSupportedVersion) { Json::Value const clIndex{env.rpc( "transaction_entry", txhash, std::to_string(index))}; @@ -211,6 +228,7 @@ class TransactionEntry_test : public beast::unit_test::suite } // Use the command line form with the ledger_hash. + if (apiVersion == RPC::apiMaximumSupportedVersion) { Json::Value const clHash{env.rpc( "transaction_entry", @@ -235,7 +253,10 @@ class TransactionEntry_test : public beast::unit_test::suite // these are actually AccountSet txs because fund does two txs and // env.tx only reports the last one - check_tx(env.closed()->seq(), fund_1_tx, R"( + check_tx( + env.closed()->seq(), + fund_1_tx, + R"( { "Account" : "r4nmQNH4Fhjfh6cHDbvVSsBv7KySbj4cBf", "Fee" : "10", @@ -244,10 +265,13 @@ class TransactionEntry_test : public beast::unit_test::suite "SigningPubKey" : "0324CAAFA2212D2AEAB9D42D481535614AED486293E1FB1380FF070C3DD7FB4264", "TransactionType" : "AccountSet", "TxnSignature" : "3044022007B35E3B99460534FF6BC3A66FBBA03591C355CC38E38588968E87CCD01BE229022071A443026DE45041B55ABB1CC76812A87EA701E475BBB7E165513B4B242D3474", - "hash" : "F4E9DF90D829A9E8B423FF68C34413E240D8D8BB0EFD080DF08114ED398E2506" } -)"); - check_tx(env.closed()->seq(), fund_2_tx, R"( +)", + "2000-01-01T00:00:10Z"); + check_tx( + env.closed()->seq(), + fund_2_tx, + R"( { "Account" : "rGpeQzUWFu4fMhJHZ1Via5aqFC3A5twZUD", "Fee" : "10", @@ -256,9 +280,9 @@ class TransactionEntry_test : public beast::unit_test::suite "SigningPubKey" : "03CFF28E067A2CCE6CC5A598C0B845CBD3F30A7863BE9C0DD55F4960EFABCCF4D0", "TransactionType" : "AccountSet", "TxnSignature" : "3045022100C8857FC0759A2AC0D2F320684691A66EAD252EAED9EF88C79791BC58BFCC9D860220421722286487DD0ED6BBA626CE6FCBDD14289F7F4726870C3465A4054C2702D7", - "hash" : "6853CD8226A05068C951CB1F54889FF4E40C5B440DC1C5BA38F114C4E0B1E705" } -)"); +)", + "2000-01-01T00:00:10Z"); env.trust(A2["USD"](1000), A1); // the trust tx is actually a payment since the trust method @@ -272,7 +296,10 @@ class TransactionEntry_test : public beast::unit_test::suite boost::lexical_cast(env.tx()->getTransactionID()); env.close(); - check_tx(env.closed()->seq(), trust_tx, R"( + check_tx( + env.closed()->seq(), + trust_tx, + R"( { "Account" : "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", "DeliverMax" : "10", @@ -283,9 +310,9 @@ class TransactionEntry_test : public beast::unit_test::suite "SigningPubKey" : "0330E7FC9D56BB25D6893BA3F317AE5BCF33B3291BD63DB32654A313222F7FD020", "TransactionType" : "Payment", "TxnSignature" : "3044022033D9EBF7F02950AF2F6B13C07AEE641C8FEBDD540A338FCB9027A965A4AED35B02206E4E227DCC226A3456C0FEF953449D21645A24EB63CA0BB7C5B62470147FD1D1", - "hash" : "C992D97D88FF444A1AB0C06B27557EC54B7F7DA28254778E60238BEA88E0C101" } -)"); +)", + "2000-01-01T00:00:20Z"); check_tx( env.closed()->seq(), @@ -306,9 +333,9 @@ class TransactionEntry_test : public beast::unit_test::suite "SigningPubKey" : "03CFF28E067A2CCE6CC5A598C0B845CBD3F30A7863BE9C0DD55F4960EFABCCF4D0", "TransactionType" : "Payment", "TxnSignature" : "30450221008A722B7F16EDB2348886E88ED4EC682AE9973CC1EE0FF37C93BB2CEC821D3EDF022059E464472031BA5E0D88A93E944B6A8B8DB3E1D5E5D1399A805F615789DB0BED", - "hash" : "988046D484ACE9F5F6A8C792D89C6EA2DB307B5DDA9864AEBA88E6782ABD0865" } -)"); +)", + "2000-01-01T00:00:20Z"); env(offer(A2, XRP(100), A2["USD"](1))); auto offer_tx = @@ -333,9 +360,9 @@ class TransactionEntry_test : public beast::unit_test::suite "TakerPays" : "100000000", "TransactionType" : "OfferCreate", "TxnSignature" : "304502210093FC93ACB77B4E3DE3315441BD010096734859080C1797AB735EB47EBD541BD102205020BB1A7C3B4141279EE4C287C13671E2450EA78914EFD0C6DB2A18344CD4F2", - "hash" : "5FCC1A27A7664F82A0CC4BE5766FBBB7C560D52B93AA7B550CD33B27AEC7EFFB" } -)"); +)", + "2000-01-01T00:00:30Z"); } public: @@ -343,7 +370,8 @@ class TransactionEntry_test : public beast::unit_test::suite run() override { testBadInput(); - testRequest(); + testRequest(1); + testRequest(2); } }; diff --git a/src/test/rpc/TransactionHistory_test.cpp b/src/test/rpc/TransactionHistory_test.cpp index 3f9b5792744..862eaaee507 100644 --- a/src/test/rpc/TransactionHistory_test.cpp +++ b/src/test/rpc/TransactionHistory_test.cpp @@ -54,6 +54,21 @@ class TransactionHistory_test : public beast::unit_test::suite } } + void + testCommandRetired() + { + testcase("Command retired from API v2"); + using namespace test::jtx; + Env env{*this, envconfig(no_admin)}; + + Json::Value params{Json::objectValue}; + params[jss::api_version] = 2; + auto const result = + env.client().invoke("tx_history", params)[jss::result]; + BEAST_EXPECT(result[jss::error] == "unknownCmd"); + BEAST_EXPECT(result[jss::status] == "error"); + } + void testRequest() { @@ -148,6 +163,7 @@ class TransactionHistory_test : public beast::unit_test::suite { testBadInput(); testRequest(); + testCommandRetired(); } }; diff --git a/src/test/rpc/Transaction_test.cpp b/src/test/rpc/Transaction_test.cpp index c16d7bbd004..36de520187b 100644 --- a/src/test/rpc/Transaction_test.cpp +++ b/src/test/rpc/Transaction_test.cpp @@ -694,15 +694,13 @@ class Transaction_test : public beast::unit_test::suite } void - testRequest(FeatureBitset features) + testRequest(FeatureBitset features, unsigned apiVersion) { - testcase("Test Request"); + testcase("Test Request API version " + std::to_string(apiVersion)); using namespace test::jtx; using std::to_string; - const char* COMMAND = jss::tx.c_str(); - Env env{*this}; Account const alice{"alice"}; Account const alie{"alie"}; @@ -725,18 +723,47 @@ class Transaction_test : public beast::unit_test::suite Json::Value expected = txn->getJson(JsonOptions::none); expected[jss::DeliverMax] = expected[jss::Amount]; + if (apiVersion > 1) + { + expected.removeMember(jss::hash); + expected.removeMember(jss::Amount); + } + + Json::Value const result = {[&env, txn, apiVersion]() { + Json::Value params{Json::objectValue}; + params[jss::transaction] = to_string(txn->getTransactionID()); + params[jss::binary] = false; + params[jss::api_version] = apiVersion; + return env.client().invoke("tx", params); + }()}; - auto const result = - env.rpc(COMMAND, to_string(txn->getTransactionID())); BEAST_EXPECT(result[jss::result][jss::status] == jss::success); + if (apiVersion > 1) + { + BEAST_EXPECT( + result[jss::result][jss::close_time_iso] == + "2000-01-01T00:00:20Z"); + BEAST_EXPECT( + result[jss::result][jss::hash] == + to_string(txn->getTransactionID())); + BEAST_EXPECT(result[jss::result][jss::validated] == true); + BEAST_EXPECT(result[jss::result][jss::ledger_index] == 4); + BEAST_EXPECT( + result[jss::result][jss::ledger_hash] == + "B41882E20F0EC6228417D28B9AE0F33833645D35F6799DFB782AC97FC4BB51" + "D2"); + } for (auto memberIt = expected.begin(); memberIt != expected.end(); memberIt++) { std::string const name = memberIt.memberName(); - if (BEAST_EXPECT(result[jss::result].isMember(name))) + auto const& result_transaction = + (apiVersion > 1 ? result[jss::result][jss::tx_json] + : result[jss::result]); + if (BEAST_EXPECT(result_transaction.isMember(name))) { - auto const received = result[jss::result][name]; + auto const received = result_transaction[name]; BEAST_EXPECTS( received == *memberIt, "Transaction contains \n\"" + name + "\": " // @@ -763,7 +790,8 @@ class Transaction_test : public beast::unit_test::suite testRangeCTIDRequest(features); testCTIDValidation(features); testCTIDRPC(features); - testRequest(features); + testRequest(features, 1); + testRequest(features, 2); } };