diff --git a/.github/workflows/doxygen.yml b/.github/workflows/doxygen.yml new file mode 100644 index 00000000000..7a21548bf0d --- /dev/null +++ b/.github/workflows/doxygen.yml @@ -0,0 +1,23 @@ +name: Build and publish Doxygen documentation +on: + push: + branches: + - develop + +jobs: + job: + runs-on: ubuntu-18.04 + steps: + - name: checkout + uses: actions/checkout@v2 + - name: set OUTPUT_DIRECTORY + run: echo 'OUTPUT_DIRECTORY = html' >> docs/Doxyfile + - name: build + uses: mattnotmitt/doxygen-action@v1 + with: + doxyfile-path: 'docs/Doxyfile' + - name: publish + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./html diff --git a/Builds/CMake/RippleConfig.cmake b/Builds/CMake/RippleConfig.cmake index 1091e741e74..1dc75cb9093 100644 --- a/Builds/CMake/RippleConfig.cmake +++ b/Builds/CMake/RippleConfig.cmake @@ -21,7 +21,6 @@ find_dependency (Boost 1.70 filesystem program_options regex - serialization system thread) #[=========================================================[ diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 35a015d5464..56502aa6690 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -487,6 +487,7 @@ target_sources (rippled PRIVATE main sources: subdir: net #]===============================] + src/ripple/net/impl/DatabaseDownloader.cpp src/ripple/net/impl/HTTPClient.cpp src/ripple/net/impl/InfoSub.cpp src/ripple/net/impl/RPCCall.cpp @@ -513,6 +514,7 @@ target_sources (rippled PRIVATE src/ripple/nodestore/impl/ManagerImp.cpp src/ripple/nodestore/impl/NodeObject.cpp src/ripple/nodestore/impl/Shard.cpp + src/ripple/nodestore/impl/TaskQueue.cpp #[===============================[ main sources: subdir: overlay @@ -853,6 +855,7 @@ target_sources (rippled PRIVATE src/test/overlay/ProtocolVersion_test.cpp src/test/overlay/cluster_test.cpp src/test/overlay/short_read_test.cpp + src/test/overlay/compression_test.cpp #[===============================[ test sources: subdir: peerfinder @@ -917,6 +920,7 @@ target_sources (rippled PRIVATE src/test/rpc/RPCOverload_test.cpp src/test/rpc/RobustTransaction_test.cpp src/test/rpc/ServerInfo_test.cpp + src/test/rpc/ShardArchiveHandler_test.cpp src/test/rpc/Status_test.cpp src/test/rpc/Submit_test.cpp src/test/rpc/Subscribe_test.cpp @@ -961,7 +965,8 @@ endif () if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.16) # any files that don't play well with unity should be added here set_source_files_properties( - # this one seems to produce conflicts in beast teardown template methods: + # these two seem to produce conflicts in beast teardown template methods src/test/rpc/ValidatorRPC_test.cpp + src/test/rpc/ShardArchiveHandler_test.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE) endif () diff --git a/Builds/CMake/RippledDocs.cmake b/Builds/CMake/RippledDocs.cmake index be964a56049..b168b8d0154 100644 --- a/Builds/CMake/RippledDocs.cmake +++ b/Builds/CMake/RippledDocs.cmake @@ -3,29 +3,82 @@ #]===================================================================] find_package (Doxygen) -if (TARGET Doxygen::doxygen) - set (doc_srcs docs/source.dox) - file (GLOB_RECURSE other_docs docs/*.md) - list (APPEND doc_srcs "${other_docs}") - # read the source config and make a modified one - # that points the output files to our build directory - file (READ "${CMAKE_CURRENT_SOURCE_DIR}/docs/source.dox" dox_content) - string (REGEX REPLACE "[\t ]*OUTPUT_DIRECTORY[\t ]*=(.*)" - "OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}\n\\1" - new_config "${dox_content}") - file (WRITE "${CMAKE_BINARY_DIR}/source.dox" "${new_config}") - add_custom_target (docs - COMMAND "${DOXYGEN_EXECUTABLE}" "${CMAKE_BINARY_DIR}/source.dox" - BYPRODUCTS "${CMAKE_BINARY_DIR}/html_doc/index.html" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/docs" - SOURCES "${doc_srcs}") - if (is_multiconfig) - set_property ( - SOURCE ${doc_srcs} - APPEND - PROPERTY HEADER_FILE_ONLY - true) - endif () -else () +if (NOT TARGET Doxygen::doxygen) message (STATUS "doxygen executable not found -- skipping docs target") + return () +endif () + +set (doxygen_output_directory "${CMAKE_BINARY_DIR}/docs") +set (doxygen_include_path "${CMAKE_SOURCE_DIR}/src") +set (doxygen_index_file "${doxygen_output_directory}/html/index.html") +set (doxyfile "${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile") + +file (GLOB_RECURSE doxygen_input + docs/*.md + src/ripple/*.h + src/ripple/*.cpp + src/ripple/*.md + src/test/*.h + src/test/*.md + Builds/*/README.md) +list (APPEND doxygen_input + README.md + RELEASENOTES.md + src/README.md) +set (dependencies "${doxygen_input}" "${doxyfile}") + +function (verbose_find_path variable name) + # find_path sets a CACHE variable, so don't try using a "local" variable. + find_path (${variable} "${name}" ${ARGN}) + if (NOT ${variable}) + message (WARNING "could not find ${name}") + else () + message (STATUS "found ${name}: ${${variable}}/${name}") + endif () +endfunction () + +verbose_find_path (doxygen_plantuml_jar_path plantuml.jar PATH_SUFFIXES share/plantuml) +verbose_find_path (doxygen_dot_path dot) + +# https://en.cppreference.com/w/Cppreference:Archives +# https://stackoverflow.com/questions/60822559/how-to-move-a-file-download-from-configure-step-to-build-step +set (download_script "${CMAKE_BINARY_DIR}/docs/download-cppreference.cmake") +file (WRITE + "${download_script}" + "file (DOWNLOAD \ + http://upload.cppreference.com/mwiki/images/b/b2/html_book_20190607.zip \ + ${CMAKE_BINARY_DIR}/docs/cppreference.zip \ + EXPECTED_HASH MD5=82b3a612d7d35a83e3cb1195a63689ab \ + )\n \ + execute_process ( \ + COMMAND \"${CMAKE_COMMAND}\" -E tar -xf cppreference.zip \ + )\n" +) +set (tagfile "${CMAKE_BINARY_DIR}/docs/cppreference-doxygen-web.tag.xml") +add_custom_command ( + OUTPUT "${tagfile}" + COMMAND "${CMAKE_COMMAND}" -P "${download_script}" + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/docs" +) +set (doxygen_tagfiles "${tagfile}=http://en.cppreference.com/w/") + +add_custom_command ( + OUTPUT "${doxygen_index_file}" + COMMAND "${CMAKE_COMMAND}" -E env + "DOXYGEN_OUTPUT_DIRECTORY=${doxygen_output_directory}" + "DOXYGEN_INCLUDE_PATH=${doxygen_include_path}" + "DOXYGEN_TAGFILES=${doxygen_tagfiles}" + "DOXYGEN_PLANTUML_JAR_PATH=${doxygen_plantuml_jar_path}" + "DOXYGEN_DOT_PATH=${doxygen_dot_path}" + "${DOXYGEN_EXECUTABLE}" "${doxyfile}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + DEPENDS "${dependencies}" "${tagfile}") +add_custom_target (docs + DEPENDS "${doxygen_index_file}" + SOURCES "${dependencies}") +if (is_multiconfig) + set_property ( + SOURCE ${dependencies} + APPEND PROPERTY + HEADER_FILE_ONLY true) endif () diff --git a/Builds/CMake/RippledInterface.cmake b/Builds/CMake/RippledInterface.cmake index c28896087ee..e0c0c1e5c0f 100644 --- a/Builds/CMake/RippledInterface.cmake +++ b/Builds/CMake/RippledInterface.cmake @@ -21,9 +21,7 @@ target_compile_definitions (opts > $<$:BEAST_NO_UNIT_TEST_INLINE=1> $<$:BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES=1> - $<$:RIPPLE_SINGLE_IO_SERVICE_THREAD=1> - # doesn't currently compile ? : - $<$:RIPPLE_VERIFY_NODEOBJECT_KEYS=1>) + $<$:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>) target_compile_options (opts INTERFACE $<$,$>:-Wsuggest-override> diff --git a/Builds/CMake/RippledSettings.cmake b/Builds/CMake/RippledSettings.cmake index 0fe3354f395..cd17d86552a 100644 --- a/Builds/CMake/RippledSettings.cmake +++ b/Builds/CMake/RippledSettings.cmake @@ -100,12 +100,6 @@ option (have_package_container option (beast_no_unit_test_inline "Prevents unit test definitions from being inserted into global table" OFF) -# NOTE - THIS OPTION CURRENTLY DOES NOT COMPILE : -# TODO: fix or remove -option (verify_nodeobject_keys - "This verifies that the hash of node objects matches the payload. \ - This check is expensive - use with caution." - OFF) option (single_io_service_thread "Restricts the number of threads calling io_service::run to one. \ This can be useful when debugging." diff --git a/Builds/CMake/deps/Boost.cmake b/Builds/CMake/deps/Boost.cmake index e3e8d92d85e..bdff36909cc 100644 --- a/Builds/CMake/deps/Boost.cmake +++ b/Builds/CMake/deps/Boost.cmake @@ -47,7 +47,6 @@ find_package (Boost 1.70 REQUIRED filesystem program_options regex - serialization system thread) @@ -69,7 +68,6 @@ target_link_libraries (ripple_boost Boost::filesystem Boost::program_options Boost::regex - Boost::serialization Boost::system Boost::thread) if (Boost_COMPILER) diff --git a/Builds/containers/gitlab-ci/pkgbuild.yml b/Builds/containers/gitlab-ci/pkgbuild.yml index 555b9d00333..c431ee665b7 100644 --- a/Builds/containers/gitlab-ci/pkgbuild.yml +++ b/Builds/containers/gitlab-ci/pkgbuild.yml @@ -19,7 +19,7 @@ variables: DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}" ARTIFACTORY_HOST: "artifactory.ops.ripple.com" ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555" - GIT_SIGN_PUBKEYS_URL: "https://gitlab.ops.ripple.com/snippets/11/raw" + GIT_SIGN_PUBKEYS_URL: "https://gitlab.ops.ripple.com/xrpledger/rippled-packages/snippets/49/raw" PUBLIC_REPO_ROOT: "https://repos.ripple.com/repos" # also need to define this variable ONLY for the primary # build/publish pipeline on the mainline repo: diff --git a/Builds/containers/shared/install_boost.sh b/Builds/containers/shared/install_boost.sh index 51d6524d785..ea26220e627 100755 --- a/Builds/containers/shared/install_boost.sh +++ b/Builds/containers/shared/install_boost.sh @@ -39,7 +39,6 @@ else BLDARGS+=(--with-filesystem) BLDARGS+=(--with-program_options) BLDARGS+=(--with-regex) - BLDARGS+=(--with-serialization) BLDARGS+=(--with-system) BLDARGS+=(--with-atomic) BLDARGS+=(--with-thread) diff --git a/bin/ci/ubuntu/build-and-test.sh b/bin/ci/ubuntu/build-and-test.sh index 41f433ef1ae..7ffad801dd1 100755 --- a/bin/ci/ubuntu/build-and-test.sh +++ b/bin/ci/ubuntu/build-and-test.sh @@ -105,7 +105,7 @@ ${time} eval cmake --build . ${BUILDARGS} -- ${BUILDTOOLARGS} if [[ ${TARGET} == "docs" ]]; then ## mimic the standard test output for docs build ## to make controlling processes like jenkins happy - if [ -f html_doc/index.html ]; then + if [ -f docs/html/index.html ]; then echo "1 case, 1 test total, 0 failures" else echo "1 case, 1 test total, 1 failures" diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 43982b9a951..3c856b40fd8 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -746,6 +746,7 @@ # # main -> 0 # testnet -> 1 +# devnet -> 2 # # If this value is not specified the server is not explicitly configured # to track a particular network. diff --git a/docs/Dockerfile b/docs/Dockerfile index dd33d95e647..d716ca21315 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -26,6 +26,7 @@ RUN rm -rf /tmp/doxygen-1.8.14 RUN mkdir -p /opt/plantuml RUN wget -O /opt/plantuml/plantuml.jar http://sourceforge.net/projects/plantuml/files/plantuml.jar/download -ENV PLANTUML_JAR=/opt/plantuml/plantuml.jar +ENV DOXYGEN_PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar -CMD cd /opt/rippled/docs && doxygen source.dox +ENV DOXYGEN_OUTPUT_DIRECTORY=html +CMD cd /opt/rippled && doxygen docs/Doxyfile diff --git a/docs/source.dox b/docs/Doxyfile similarity index 77% rename from docs/source.dox rename to docs/Doxyfile index fb2fa12ff6e..48a0b5d1e1a 100644 --- a/docs/source.dox +++ b/docs/Doxyfile @@ -4,10 +4,10 @@ DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = "rippled" PROJECT_NUMBER = -PROJECT_BRIEF = C++ Library +PROJECT_BRIEF = PROJECT_LOGO = -PROJECT_LOGO = images/LogoForDocumentation.png -OUTPUT_DIRECTORY = +PROJECT_LOGO = +OUTPUT_DIRECTORY = $(DOXYGEN_OUTPUT_DIRECTORY) CREATE_SUBDIRS = NO ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English @@ -17,7 +17,7 @@ ABBREVIATE_BRIEF = ALWAYS_DETAILED_SEC = NO INLINE_INHERITED_MEMB = YES FULL_PATH_NAMES = NO -STRIP_FROM_PATH = ../src/ +STRIP_FROM_PATH = src/ STRIP_FROM_INC_PATH = SHORT_NAMES = NO JAVADOC_AUTOBRIEF = YES @@ -27,7 +27,6 @@ INHERIT_DOCS = YES SEPARATE_MEMBER_PAGES = NO TAB_SIZE = 4 ALIASES = -TCL_SUBST = OPTIMIZE_OUTPUT_FOR_C = NO OPTIMIZE_OUTPUT_JAVA = NO OPTIMIZE_FOR_FORTRAN = NO @@ -35,7 +34,7 @@ OPTIMIZE_OUTPUT_VHDL = NO EXTENSION_MAPPING = MARKDOWN_SUPPORT = YES AUTOLINK_SUPPORT = YES -BUILTIN_STL_SUPPORT = NO +BUILTIN_STL_SUPPORT = YES CPP_CLI_SUPPORT = NO SIP_SUPPORT = NO IDL_PROPERTY_SUPPORT = YES @@ -83,7 +82,7 @@ ENABLED_SECTIONS = MAX_INITIALIZER_LINES = 30 SHOW_USED_FILES = NO SHOW_FILES = NO -SHOW_NAMESPACES = NO +SHOW_NAMESPACES = YES FILE_VERSION_FILTER = LAYOUT_FILE = CITE_BIB_FILES = @@ -104,71 +103,17 @@ WARN_LOGFILE = # Configuration options related to the input files #--------------------------------------------------------------------------- INPUT = \ -\ - ../src/ripple/app/misc/TxQ.h \ - ../src/ripple/app/tx/apply.h \ - ../src/ripple/app/tx/applySteps.h \ - ../src/ripple/protocol/STObject.h \ - ../src/ripple/protocol/JsonFields.h \ - ../src/test/jtx/AbstractClient.h \ - ../src/test/jtx/JSONRPCClient.h \ - ../src/test/jtx/WSClient.h \ - ../src/ripple/consensus/Consensus.h \ - ../src/ripple/consensus/ConsensusProposal.h \ - ../src/ripple/consensus/ConsensusTypes.h \ - ../src/ripple/consensus/DisputedTx.h \ - ../src/ripple/consensus/LedgerTiming.h \ - ../src/ripple/consensus/LedgerTrie.h \ - ../src/ripple/consensus/Validations.h \ - ../src/ripple/consensus/ConsensusParms.h \ - ../src/ripple/app/consensus/RCLCxTx.h \ - ../src/ripple/app/consensus/RCLCxLedger.h \ - ../src/ripple/app/consensus/RCLConsensus.h \ - ../src/ripple/app/consensus/RCLCxPeerPos.h \ - ../src/ripple/app/tx/apply.h \ - ../src/ripple/app/tx/applySteps.h \ - ../src/ripple/app/tx/impl/InvariantCheck.h \ - ../src/ripple/app/consensus/RCLValidations.h \ - ../src/README.md \ - ../src/ripple/README.md \ - ../README.md \ - ../RELEASENOTES.md \ - ../docs/CodingStyle.md \ - ../docs/CheatSheet.md \ - ../docs/README.md \ - ../docs/sample_chart.doc \ - ../docs/HeapProfiling.md \ - ../docs/Docker.md \ - ../docs/consensus.md \ - ../Builds/macos/README.md \ - ../Builds/linux/README.md \ - ../Builds/VisualStudio2017/README.md \ - ../src/ripple/consensus/README.md \ - ../src/ripple/app/consensus/README.md \ - ../src/test/csf/README.md \ - ../src/ripple/basics/README.md \ - ../src/ripple/crypto/README.md \ - ../src/ripple/peerfinder/README.md \ - ../src/ripple/app/misc/README.md \ - ../src/ripple/app/misc/FeeEscalation.md \ - ../src/ripple/app/ledger/README.md \ - ../src/ripple/app/paths/README.md \ - ../src/ripple/app/tx/README.md \ - ../src/ripple/proto/README.md \ - ../src/ripple/shamap/README.md \ - ../src/ripple/protocol/README.md \ - ../src/ripple/json/README.md \ - ../src/ripple/json/TODO.md \ - ../src/ripple/resource/README.md \ - ../src/ripple/rpc/README.md \ - ../src/ripple/overlay/README.md \ - ../src/ripple/nodestore/README.md \ - ../src/ripple/nodestore/Benchmarks.md \ + docs \ + src/ripple \ + src/test \ + src/README.md \ + README.md \ + RELEASENOTES.md \ INPUT_ENCODING = UTF-8 -FILE_PATTERNS = -RECURSIVE = NO +FILE_PATTERNS = *.h *.cpp *.md +RECURSIVE = YES EXCLUDE = EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = @@ -177,20 +122,20 @@ EXAMPLE_PATH = EXAMPLE_PATTERNS = EXAMPLE_RECURSIVE = NO IMAGE_PATH = \ - ./images/ \ - ./images/consensus/ \ - ../src/test/csf/ \ + docs/images/ \ + docs/images/consensus/ \ + src/test/csf/ \ INPUT_FILTER = FILTER_PATTERNS = FILTER_SOURCE_FILES = NO FILTER_SOURCE_PATTERNS = -USE_MDFILE_AS_MAINPAGE = ../src/README.md +USE_MDFILE_AS_MAINPAGE = src/README.md #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- -SOURCE_BROWSER = NO +SOURCE_BROWSER = YES INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES REFERENCED_BY_RELATION = NO @@ -213,7 +158,7 @@ IGNORE_PREFIX = # Configuration options related to the HTML output #--------------------------------------------------------------------------- GENERATE_HTML = YES -HTML_OUTPUT = html_doc +HTML_OUTPUT = html HTML_FILE_EXTENSION = .html HTML_HEADER = HTML_FOOTER = @@ -273,7 +218,7 @@ EXTRA_SEARCH_MAPPINGS = #--------------------------------------------------------------------------- GENERATE_LATEX = NO LATEX_OUTPUT = latex -LATEX_CMD_NAME = latex +LATEX_CMD_NAME = MAKEINDEX_CMD_NAME = makeindex COMPACT_LATEX = NO PAPER_TYPE = a4 @@ -314,7 +259,7 @@ MAN_LINKS = NO # Configuration options related to the XML output #--------------------------------------------------------------------------- GENERATE_XML = NO -XML_OUTPUT = temp +XML_OUTPUT = xml XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- @@ -340,7 +285,7 @@ ENABLE_PREPROCESSING = YES MACRO_EXPANSION = YES EXPAND_ONLY_PREDEF = YES SEARCH_INCLUDES = YES -INCLUDE_PATH = ../ +INCLUDE_PATH = $(DOXYGEN_INCLUDE_PATH) INCLUDE_FILE_PATTERNS = PREDEFINED = DOXYGEN \ GENERATING_DOCS \ @@ -353,21 +298,20 @@ SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration options related to external references #--------------------------------------------------------------------------- -TAGFILES = +TAGFILES = $(DOXYGEN_TAGFILES) GENERATE_TAGFILE = ALLEXTERNALS = NO EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES -PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- CLASS_DIAGRAMS = NO -MSCGEN_PATH = DIA_PATH = HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = NO +HAVE_DOT = YES +# DOT_NUM_THREADS = 0 means 1 for every processor. DOT_NUM_THREADS = 0 DOT_FONTNAME = Helvetica DOT_FONTSIZE = 10 @@ -386,11 +330,11 @@ GRAPHICAL_HIERARCHY = YES DIRECTORY_GRAPH = YES DOT_IMAGE_FORMAT = png INTERACTIVE_SVG = NO -DOT_PATH = +DOT_PATH = $(DOXYGEN_DOT_PATH) DOTFILE_DIRS = MSCFILE_DIRS = DIAFILE_DIRS = -PLANTUML_JAR_PATH = $(PLANTUML_JAR) +PLANTUML_JAR_PATH = $(DOXYGEN_PLANTUML_JAR_PATH) PLANTUML_INCLUDE_PATH = DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 diff --git a/docs/README.md b/docs/README.md index 04e25775273..3345ee828f3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,11 +1,5 @@ # Building documentation -## Specifying Files - -To specify the source files for which to build documentation, modify `INPUT` -and its related fields in `docs/source.dox`. Note that the `INPUT` paths are -relative to the `docs/` directory. - ## Install Dependencies ### Windows @@ -30,32 +24,38 @@ Install these dependencies: ### [Optional] Install Plantuml (all platforms) -Doxygen supports the optional use of [plantuml](http://plantuml.com) to +Doxygen supports the optional use of [plantuml](http://plantuml.com) to generate diagrams from `@startuml` sections. We don't currently rely on this functionality for docs, so it's largely optional. Requirements: 1. Download/install a functioning java runtime, if you don't already have one. 2. Download [plantuml](http://plantuml.com) from [here](http://sourceforge.net/projects/plantuml/files/plantuml.jar/download). - Set a system environment variable named `PLANTUML_JAR` with a value of the fullpath - to the file system location of the `plantuml.jar` file you downloaded. + Set a system environment variable named `DOXYGEN_PLANTUML_JAR_PATH` to + the absolute path of the `plantuml.jar` file you downloaded. + + +## Configure + +You should set these environment variables: -## Do it +- `DOXYGEN_OUTPUT_DIRECTORY` +- `DOXYGEN_PLANTUML_JAR_PATH` -### all platforms +## Build From the rippled root folder: + ``` -cd docs -mkdir -p html_doc -doxygen source.dox +doxygen docs/Doxyfile ``` -The output will be in `docs/html_doc`. + +The output will be wherever you chose for `DOXYGEN_OUTPUT_DIRECTORY`. ## Docker (applicable to all platforms) - + Instead of installing the doxygen tools locally, you can use the provided `Dockerfile` to create an ubuntu based image for running the tools: @@ -72,5 +72,4 @@ Then to run the image, from the rippled root folder: sudo docker run -v $PWD:/opt/rippled --rm rippled-docs ``` -The output will be in `docs/html_doc`. - +The output will be in `html`. diff --git a/src/ripple/app/ledger/Ledger.cpp b/src/ripple/app/ledger/Ledger.cpp index 6125149796f..f72d6d73472 100644 --- a/src/ripple/app/ledger/Ledger.cpp +++ b/src/ripple/app/ledger/Ledger.cpp @@ -835,7 +835,7 @@ static bool saveValidatedLedger ( if (! aLedger) { aLedger = std::make_shared(ledger, app.accountIDCache(), app.logs()); - app.getAcceptedLedgerCache().canonicalize(ledger->info().hash, aLedger); + app.getAcceptedLedgerCache().canonicalize_replace_client(ledger->info().hash, aLedger); } } catch (std::exception const&) diff --git a/src/ripple/app/ledger/LedgerHistory.cpp b/src/ripple/app/ledger/LedgerHistory.cpp index 95c35166b14..aa07e8ad40b 100644 --- a/src/ripple/app/ledger/LedgerHistory.cpp +++ b/src/ripple/app/ledger/LedgerHistory.cpp @@ -62,8 +62,8 @@ LedgerHistory::insert( std::unique_lock sl (m_ledgers_by_hash.peekMutex ()); - const bool alreadyHad = m_ledgers_by_hash.canonicalize ( - ledger->info().hash, ledger, true); + const bool alreadyHad = m_ledgers_by_hash.canonicalize_replace_cache( + ledger->info().hash, ledger); if (validated) mLedgersByIndex[ledger->info().seq] = ledger->info().hash; @@ -108,7 +108,7 @@ LedgerHistory::getLedgerBySeq (LedgerIndex index) std::unique_lock sl (m_ledgers_by_hash.peekMutex ()); assert (ret->isImmutable ()); - m_ledgers_by_hash.canonicalize (ret->info().hash, ret); + m_ledgers_by_hash.canonicalize_replace_client(ret->info().hash, ret); mLedgersByIndex[ret->info().seq] = ret->info().hash; return (ret->info().seq == index) ? ret : nullptr; } @@ -133,7 +133,7 @@ LedgerHistory::getLedgerByHash (LedgerHash const& hash) assert (ret->isImmutable ()); assert (ret->info().hash == hash); - m_ledgers_by_hash.canonicalize (ret->info().hash, ret); + m_ledgers_by_hash.canonicalize_replace_client(ret->info().hash, ret); assert (ret->info().hash == hash); return ret; @@ -432,7 +432,7 @@ void LedgerHistory::builtLedger ( m_consensus_validated.peekMutex()); auto entry = std::make_shared(); - m_consensus_validated.canonicalize(index, entry, false); + m_consensus_validated.canonicalize_replace_client(index, entry); if (entry->validated && ! entry->built) { @@ -472,7 +472,7 @@ void LedgerHistory::validatedLedger ( m_consensus_validated.peekMutex()); auto entry = std::make_shared(); - m_consensus_validated.canonicalize(index, entry, false); + m_consensus_validated.canonicalize_replace_client(index, entry); if (entry->built && ! entry->validated) { diff --git a/src/ripple/app/ledger/LedgerMaster.h b/src/ripple/app/ledger/LedgerMaster.h index fa9d428a154..0eb303207f3 100644 --- a/src/ripple/app/ledger/LedgerMaster.h +++ b/src/ripple/app/ledger/LedgerMaster.h @@ -39,6 +39,7 @@ #include #include #include +#include #include @@ -280,11 +281,6 @@ class LedgerMaster // Try to publish ledgers, acquire missing ledgers. Always called with // m_mutex locked. The passed lock is a reminder to callers. void doAdvance(std::unique_lock&); - bool shouldAcquire( - std::uint32_t const currentLedger, - std::uint32_t const ledgerHistory, - std::uint32_t const ledgerHistoryIndex, - std::uint32_t const candidateLedger) const; std::vector> findNewLedgersToPublish(std::unique_lock&); @@ -295,7 +291,6 @@ class LedgerMaster // The passed lock is a reminder to callers. bool newPFWork(const char *name, std::unique_lock&); -private: Application& app_; beast::Journal m_journal; diff --git a/src/ripple/app/ledger/impl/InboundLedger.cpp b/src/ripple/app/ledger/impl/InboundLedger.cpp index 266695aaf56..db5465593dc 100644 --- a/src/ripple/app/ledger/impl/InboundLedger.cpp +++ b/src/ripple/app/ledger/impl/InboundLedger.cpp @@ -110,7 +110,7 @@ InboundLedger::init(ScopedLockType& collectionLock) if (mFailed) return; } - else if (shardStore && mSeq >= shardStore->earliestSeq()) + else if (shardStore && mSeq >= shardStore->earliestLedgerSeq()) { if (auto l = shardStore->fetchLedger(mHash, mSeq)) { diff --git a/src/ripple/app/ledger/impl/InboundLedgers.cpp b/src/ripple/app/ledger/impl/InboundLedgers.cpp index 589dfc3d79f..cba1301afb9 100644 --- a/src/ripple/app/ledger/impl/InboundLedgers.cpp +++ b/src/ripple/app/ledger/impl/InboundLedgers.cpp @@ -64,6 +64,7 @@ class InboundLedgersImp { } + /** @callgraph */ std::shared_ptr acquire(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason reason) override @@ -106,7 +107,7 @@ class InboundLedgersImp if (reason == InboundLedger::Reason::HISTORY) { if (inbound->getLedger()->stateMap().family().isShardBacked()) - app_.getNodeStore().copyLedger(inbound->getLedger()); + app_.getNodeStore().storeLedger(inbound->getLedger()); } else if (reason == InboundLedger::Reason::SHARD) { @@ -120,7 +121,7 @@ class InboundLedgersImp if (inbound->getLedger()->stateMap().family().isShardBacked()) shardStore->setStored(inbound->getLedger()); else - shardStore->copyLedger(inbound->getLedger()); + shardStore->storeLedger(inbound->getLedger()); } return inbound->getLedger(); } diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp index 20f52b4d2cf..3b64f8b38d4 100644 --- a/src/ripple/app/ledger/impl/LedgerMaster.cpp +++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include #include #include +#include #include #include @@ -140,6 +142,57 @@ static constexpr std::chrono::minutes MAX_LEDGER_AGE_ACQUIRE {1}; // Don't acquire history if write load is too high static constexpr int MAX_WRITE_LOAD_ACQUIRE {8192}; +// Helper function for LedgerMaster::doAdvance() +// Returns the minimum ledger sequence in SQL database, if any. +static boost::optional +minSqlSeq(Application& app) +{ + boost::optional seq; + auto db = app.getLedgerDB().checkoutDb(); + *db << "SELECT MIN(LedgerSeq) FROM Ledgers", soci::into(seq); + return seq; +} + +// Helper function for LedgerMaster::doAdvance() +// Return true if candidateLedger should be fetched from the network. +static bool +shouldAcquire ( + std::uint32_t const currentLedger, + std::uint32_t const ledgerHistory, + boost::optional minSeq, + std::uint32_t const lastRotated, + std::uint32_t const candidateLedger, + beast::Journal j) +{ + bool ret = [&]() + { + // Fetch ledger if it may be the current ledger + if (candidateLedger >= currentLedger) + return true; + + // Or if it is within our configured history range: + if (currentLedger - candidateLedger <= ledgerHistory) + return true; + + // Or it's greater than or equal to both: + // - the minimum persisted ledger or the maximum possible + // sequence value, if no persisted ledger, and + // - minimum ledger that will be persisted as of the next online + // deletion interval, or 1 if online deletion is disabled. + return + candidateLedger >= std::max( + minSeq.value_or(std::numeric_limits::max()), + lastRotated + 1); + }(); + + JLOG (j.trace()) + << "Missing ledger " + << candidateLedger + << (ret ? " should" : " should NOT") + << " be acquired"; + return ret; +} + LedgerMaster::LedgerMaster (Application& app, Stopwatch& stopwatch, Stoppable& parent, beast::insight::Collector::ptr const& collector, beast::Journal journal) @@ -1697,31 +1750,6 @@ LedgerMaster::releaseReplay () return std::move (replayData); } -bool -LedgerMaster::shouldAcquire ( - std::uint32_t const currentLedger, - std::uint32_t const ledgerHistory, - std::uint32_t const ledgerHistoryIndex, - std::uint32_t const candidateLedger) const -{ - - // Fetch ledger if it might be the current ledger, - // is requested by the advisory delete setting, or - // is within our configured history range - - bool ret (candidateLedger >= currentLedger || - ((ledgerHistoryIndex > 0) && - (candidateLedger > ledgerHistoryIndex)) || - (currentLedger - candidateLedger) <= ledgerHistory); - - JLOG (m_journal.trace()) - << "Missing ledger " - << candidateLedger - << (ret ? " should" : " should NOT") - << " be acquired"; - return ret; -} - void LedgerMaster::fetchForHistory( std::uint32_t missing, @@ -1742,7 +1770,7 @@ LedgerMaster::fetchForHistory( *hash, missing, reason); if (!ledger && missing != fetch_seq_ && - missing > app_.getNodeStore().earliestSeq()) + missing > app_.getNodeStore().earliestLedgerSeq()) { JLOG(m_journal.trace()) << "fetchForHistory want fetch pack " << missing; @@ -1771,7 +1799,7 @@ LedgerMaster::fetchForHistory( mShardLedger = ledger; } if (!ledger->stateMap().family().isShardBacked()) - app_.getShardStore()->copyLedger(ledger); + app_.getShardStore()->storeLedger(ledger); } else { @@ -1807,7 +1835,7 @@ LedgerMaster::fetchForHistory( else // Do not fetch ledger sequences lower // than the earliest ledger sequence - fetchSz = app_.getNodeStore().earliestSeq(); + fetchSz = app_.getNodeStore().earliestLedgerSeq(); fetchSz = missing >= fetchSz ? std::min(ledger_fetch_size_, (missing - fetchSz) + 1) : 0; try @@ -1867,7 +1895,7 @@ void LedgerMaster::doAdvance (std::unique_lock& sl) std::lock_guard sll(mCompleteLock); missing = prevMissing(mCompleteLedgers, mPubLedger->info().seq, - app_.getNodeStore().earliestSeq()); + app_.getNodeStore().earliestLedgerSeq()); } if (missing) { @@ -1875,7 +1903,9 @@ void LedgerMaster::doAdvance (std::unique_lock& sl) "tryAdvance discovered missing " << *missing; if ((mFillInProgress == 0 || *missing > mFillInProgress) && shouldAcquire(mValidLedgerSeq, ledger_history_, - app_.getSHAMapStore().getCanDelete(), *missing)) + minSqlSeq(app_), + app_.getSHAMapStore().getLastRotated(), *missing, + m_journal)) { JLOG(m_journal.trace()) << "advanceThread should acquire"; @@ -1946,7 +1976,7 @@ LedgerMaster::addFetchPack ( uint256 const& hash, std::shared_ptr< Blob >& data) { - fetch_packs_.canonicalize (hash, data); + fetch_packs_.canonicalize_replace_client(hash, data); } boost::optional diff --git a/src/ripple/app/ledger/impl/TransactionMaster.cpp b/src/ripple/app/ledger/impl/TransactionMaster.cpp index d6f7ab425b4..0bfc2500707 100644 --- a/src/ripple/app/ledger/impl/TransactionMaster.cpp +++ b/src/ripple/app/ledger/impl/TransactionMaster.cpp @@ -62,7 +62,7 @@ TransactionMaster::fetch (uint256 const& txnID, error_code_i& ec) if (!txn) return txn; - mCache.canonicalize (txnID, txn); + mCache.canonicalize_replace_client(txnID, txn); return txn; } @@ -82,7 +82,7 @@ TransactionMaster::fetch (uint256 const& txnID, ClosedInterval const& txnID, mApp, range, ec); if (v.which () == 0 && boost::get (v)) - mCache.canonicalize (txnID, boost::get (v)); + mCache.canonicalize_replace_client(txnID, boost::get (v)); return v; } @@ -127,7 +127,7 @@ TransactionMaster::canonicalize(std::shared_ptr* pTransaction) { auto txn = *pTransaction; // VFALCO NOTE canonicalize can change the value of txn! - mCache.canonicalize(tid, txn); + mCache.canonicalize_replace_client(tid, txn); *pTransaction = txn; } } diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index fcb0961ade0..d8f223d2e15 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -64,6 +64,7 @@ #include #include #include +#include #include #include @@ -345,9 +346,9 @@ class ApplicationImp // These are Stoppable-related std::unique_ptr m_jobQueue; std::unique_ptr m_nodeStore; - std::unique_ptr shardStore_; detail::AppFamily family_; - std::unique_ptr sFamily_; + std::unique_ptr shardStore_; + std::unique_ptr shardFamily_; // VFALCO TODO Make OrderBookDB abstract OrderBookDB m_orderBookDB; std::unique_ptr m_pathRequests; @@ -463,18 +464,18 @@ class ApplicationImp m_collectorManager->group ("jobq"), m_nodeStoreScheduler, logs_->journal("JobQueue"), *logs_, *perfLog_)) - , m_nodeStore(m_shaMapStore->makeNodeStore("NodeStore.main", 4)) + , m_nodeStore (m_shaMapStore->makeNodeStore ("NodeStore.main", 4)) + + , family_ (*this, *m_nodeStore, *m_collectorManager) // The shard store is optional and make_ShardStore can return null. - , shardStore_(make_ShardStore( + , shardStore_ (make_ShardStore ( *this, *m_jobQueue, m_nodeStoreScheduler, 4, logs_->journal("ShardStore"))) - , family_ (*this, *m_nodeStore, *m_collectorManager) - , m_orderBookDB (*this, *m_jobQueue) , m_pathRequests (std::make_unique ( @@ -558,14 +559,6 @@ class ApplicationImp logs_->journal("Application"), std::chrono::milliseconds (100), get_io_service()) , grpcServer_(std::make_unique(*this)) { - if (shardStore_) - { - sFamily_ = std::make_unique( - *this, - *shardStore_, - *m_collectorManager); - } - add (m_resourceManager.get ()); // @@ -626,7 +619,7 @@ class ApplicationImp Family* shardFamily() override { - return sFamily_.get(); + return shardFamily_.get(); } TimeKeeper& @@ -943,7 +936,7 @@ class ApplicationImp } bool - initNodeStoreDBs() + initNodeStore() { if (config_->doImport) { @@ -961,12 +954,12 @@ class ApplicationImp JLOG(j.warn()) << "Starting node import from '" << source->getName() << - "' to '" << getNodeStore().getName() << "'."; + "' to '" << m_nodeStore->getName() << "'."; using namespace std::chrono; auto const start = steady_clock::now(); - getNodeStore().import(*source); + m_nodeStore->import(*source); auto const elapsed = duration_cast (steady_clock::now() - start); @@ -990,14 +983,6 @@ class ApplicationImp family().treecache().setTargetAge( seconds{config_->getValueFor(SizedItem::treeCacheAge)}); - if (sFamily_) - { - sFamily_->treecache().setTargetSize( - config_->getValueFor(SizedItem::treeCacheSize)); - sFamily_->treecache().setTargetAge( - seconds{config_->getValueFor(SizedItem::treeCacheAge)}); - } - return true; } @@ -1252,8 +1237,8 @@ class ApplicationImp // have listeners register for "onSweep ()" notification. family().fullbelow().sweep(); - if (sFamily_) - sFamily_->fullbelow().sweep(); + if (shardFamily_) + shardFamily_->fullbelow().sweep(); getMasterTransaction().sweep(); getNodeStore().sweep(); if (shardStore_) @@ -1264,8 +1249,8 @@ class ApplicationImp getInboundLedgers().sweep(); m_acceptedLedgerCache.sweep(); family().treecache().sweep(); - if (sFamily_) - sFamily_->treecache().sweep(); + if (shardFamily_) + shardFamily_->treecache().sweep(); cachedSLEs_.expire(); // Set timer to do another sweep later. @@ -1350,9 +1335,26 @@ bool ApplicationImp::setup() if (!config_->standalone()) timeKeeper_->run(config_->SNTP_SERVERS); - if (!initSQLiteDBs() || !initNodeStoreDBs()) + if (!initSQLiteDBs() || !initNodeStore()) return false; + if (shardStore_) + { + shardFamily_ = std::make_unique( + *this, + *shardStore_, + *m_collectorManager); + + using namespace std::chrono; + shardFamily_->treecache().setTargetSize( + config_->getValueFor(SizedItem::treeCacheSize)); + shardFamily_->treecache().setTargetAge( + seconds{config_->getValueFor(SizedItem::treeCacheAge)}); + + if (!shardStore_->init()) + return false; + } + if (!peerReservations_->load(getWalletDB())) { JLOG(m_journal.fatal()) << "Cannot find peer reservations!"; @@ -1609,6 +1611,53 @@ bool ApplicationImp::setup() } } + if (shardStore_) + { + using namespace boost::filesystem; + + auto stateDb( + RPC::ShardArchiveHandler::getDownloadDirectory(*config_) + / stateDBName); + + try + { + if (exists(stateDb) && + is_regular_file(stateDb) && + !RPC::ShardArchiveHandler::hasInstance()) + { + auto handler = RPC::ShardArchiveHandler::recoverInstance( + *this, + *m_jobQueue); + + assert(handler); + + if (!handler->initFromDB()) + { + JLOG(m_journal.fatal()) + << "Failed to initialize ShardArchiveHandler."; + + return false; + } + + if (!handler->start()) + { + JLOG(m_journal.fatal()) + << "Failed to start ShardArchiveHandler."; + + return false; + } + } + } + catch(std::exception const& e) + { + JLOG(m_journal.fatal()) + << "Exception when starting ShardArchiveHandler from " + "state database: " << e.what(); + + return false; + } + } + return true; } diff --git a/src/ripple/app/main/DBInit.h b/src/ripple/app/main/DBInit.h index b632d168bf3..69d9ccf4543 100644 --- a/src/ripple/app/main/DBInit.h +++ b/src/ripple/app/main/DBInit.h @@ -27,16 +27,16 @@ namespace ripple { //////////////////////////////////////////////////////////////////////////////// // Ledger database holds ledgers and ledger confirmations -static constexpr auto LgrDBName {"ledger.db"}; +inline constexpr auto LgrDBName {"ledger.db"}; -static constexpr +inline constexpr std::array LgrDBPragma {{ "PRAGMA synchronous=NORMAL;", "PRAGMA journal_mode=WAL;", "PRAGMA journal_size_limit=1582080;" }}; -static constexpr +inline constexpr std::array LgrDBInit {{ "BEGIN TRANSACTION;", @@ -63,9 +63,9 @@ std::array LgrDBInit {{ //////////////////////////////////////////////////////////////////////////////// // Transaction database holds transactions and public keys -static constexpr auto TxDBName {"transaction.db"}; +inline constexpr auto TxDBName {"transaction.db"}; -static constexpr +inline constexpr #if (ULONG_MAX > UINT_MAX) && !defined (NO_SQLITE_MMAP) std::array TxDBPragma {{ #else @@ -81,7 +81,7 @@ static constexpr #endif }}; -static constexpr +inline constexpr std::array TxDBInit {{ "BEGIN TRANSACTION;", @@ -116,18 +116,39 @@ std::array TxDBInit {{ //////////////////////////////////////////////////////////////////////////////// +// Temporary database used with an incomplete shard that is being acquired +inline constexpr auto AcquireShardDBName {"acquire.db"}; + +inline constexpr +std::array AcquireShardDBPragma {{ + "PRAGMA synchronous=NORMAL;", + "PRAGMA journal_mode=WAL;", + "PRAGMA journal_size_limit=1582080;" +}}; + +inline constexpr +std::array AcquireShardDBInit {{ + "CREATE TABLE IF NOT EXISTS Shard ( \ + ShardIndex INTEGER PRIMARY KEY, \ + LastLedgerHash CHARACTER(64), \ + StoredLedgerSeqs BLOB \ + );" +}}; + +//////////////////////////////////////////////////////////////////////////////// + // Pragma for Ledger and Transaction databases with complete shards -static constexpr -std::array CompleteShardDBPragma {{ +inline constexpr +std::array CompleteShardDBPragma{{ "PRAGMA synchronous=OFF;", "PRAGMA journal_mode=OFF;" }}; //////////////////////////////////////////////////////////////////////////////// -static constexpr auto WalletDBName {"wallet.db"}; +inline constexpr auto WalletDBName {"wallet.db"}; -static constexpr +inline constexpr std::array WalletDBInit {{ "BEGIN TRANSACTION;", @@ -160,6 +181,45 @@ std::array WalletDBInit {{ "END TRANSACTION;" }}; +//////////////////////////////////////////////////////////////////////////////// + +static constexpr auto stateDBName {"state.db"}; + +static constexpr +std::array DownloaderDBPragma +{{ + "PRAGMA synchronous=FULL;", + "PRAGMA journal_mode=DELETE;" +}}; + +static constexpr +std::array ShardArchiveHandlerDBInit +{{ + "BEGIN TRANSACTION;", + + "CREATE TABLE IF NOT EXISTS State ( \ + ShardIndex INTEGER PRIMARY KEY, \ + URL TEXT \ + );", + + "END TRANSACTION;" +}}; + +static constexpr +std::array DatabaseBodyDBInit +{{ + "BEGIN TRANSACTION;", + + "CREATE TABLE IF NOT EXISTS download ( \ + Path TEXT, \ + Data BLOB, \ + Size BIGINT UNSIGNED, \ + Part BIGINT UNSIGNED PRIMARY KEY \ + );", + + "END TRANSACTION;" +}}; + } // ripple #endif diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 21267db99d5..cfc824915d0 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -142,7 +142,7 @@ void printHelp (const po::options_description& desc) " connect []\n" " consensus_info\n" " deposit_authorized []\n" - " download_shard [[ ]] \n" + " download_shard [[ ]]\n" " feature [ [accept|reject]]\n" " fetch_info [clear]\n" " gateway_balances [] [ [ ]]\n" diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 4a067d5fde9..b0ca0130900 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -2665,7 +2665,7 @@ void NetworkOPsImp::pubLedger ( { alpAccepted = std::make_shared ( lpAccepted, app_.accountIDCache(), app_.logs()); - app_.getAcceptedLedgerCache().canonicalize ( + app_.getAcceptedLedgerCache().canonicalize_replace_client( lpAccepted->info().hash, alpAccepted); } diff --git a/src/ripple/app/misc/SHAMapStore.h b/src/ripple/app/misc/SHAMapStore.h index b41e9c4ca17..0814b98df57 100644 --- a/src/ripple/app/misc/SHAMapStore.h +++ b/src/ripple/app/misc/SHAMapStore.h @@ -56,7 +56,9 @@ class SHAMapStore /** Whether advisory delete is enabled. */ virtual bool advisoryDelete() const = 0; - /** Last ledger which was copied during rotation of backends. */ + /** Maximum ledger that has been deleted, or will be deleted if + * currently in the act of online deletion. + */ virtual LedgerIndex getLastRotated() = 0; /** Highest ledger that may be deleted. */ diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index 7e32ff0f890..5c2644e6353 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -322,7 +322,7 @@ void SHAMapStoreImp::run() { beast::setCurrentThreadName ("SHAMapStore"); - LedgerIndex lastRotated = state_db_.getState().lastRotated; + lastRotated_ = state_db_.getState().lastRotated; netOPs_ = &app_.getOPs(); ledgerMaster_ = &app_.getLedgerMaster(); fullBelowCache_ = &app_.family().fullbelow(); @@ -333,7 +333,7 @@ SHAMapStoreImp::run() if (advisoryDelete_) canDelete_ = state_db_.getCanDelete (); - while (1) + while (true) { healthy_ = true; std::shared_ptr validatedLedger; @@ -356,20 +356,20 @@ SHAMapStoreImp::run() continue; } - LedgerIndex validatedSeq = validatedLedger->info().seq; - if (!lastRotated) + LedgerIndex const validatedSeq = validatedLedger->info().seq; + if (!lastRotated_) { - lastRotated = validatedSeq; - state_db_.setLastRotated (lastRotated); + lastRotated_ = validatedSeq; + state_db_.setLastRotated (lastRotated_); } - // will delete up to (not including) lastRotated) - if (validatedSeq >= lastRotated + deleteInterval_ - && canDelete_ >= lastRotated - 1) + // will delete up to (not including) lastRotated_ + if (validatedSeq >= lastRotated_ + deleteInterval_ + && canDelete_ >= lastRotated_ - 1) { JLOG(journal_.warn()) << "rotating validatedSeq " << validatedSeq - << " lastRotated " << lastRotated << " deleteInterval " - << deleteInterval_ << " canDelete_ " << canDelete_; + << " lastRotated_ " << lastRotated_ << " deleteInterval " + << deleteInterval_ << " canDelete_ " << canDelete_; switch (health()) { @@ -383,7 +383,7 @@ SHAMapStoreImp::run() ; } - clearPrior (lastRotated); + clearPrior (lastRotated_); switch (health()) { case Health::stopping: @@ -448,16 +448,17 @@ SHAMapStoreImp::run() std::string nextArchiveDir = dbRotating_->getWritableBackend()->getName(); - lastRotated = validatedSeq; - std::unique_ptr oldBackend; + lastRotated_ = validatedSeq; + std::shared_ptr oldBackend; { std::lock_guard lock (dbRotating_->peekMutex()); state_db_.setState (SavedState {newBackend->getName(), - nextArchiveDir, lastRotated}); + nextArchiveDir, lastRotated_}); clearCaches (validatedSeq); oldBackend = dbRotating_->rotateBackends( - std::move(newBackend)); + std::move(newBackend), + lock); } JLOG(journal_.warn()) << "finished rotation " << validatedSeq; diff --git a/src/ripple/app/misc/SHAMapStoreImp.h b/src/ripple/app/misc/SHAMapStoreImp.h index ea2579dd005..5c8454b356c 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.h +++ b/src/ripple/app/misc/SHAMapStoreImp.h @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -84,6 +85,13 @@ class SHAMapStoreImp : public SHAMapStore static std::uint32_t const minimumDeletionInterval_ = 256; // minimum # of ledgers required for standalone mode. static std::uint32_t const minimumDeletionIntervalSA_ = 8; + // Ledger sequence at which the last deletion interval was triggered, + // or the current validated sequence as of first use + // if there have been no prior deletions. Deletion occurs up to (but + // not including) this value. All ledgers past this value are accumulated + // until the next online deletion. This value is persisted to SQLite + // nearly immediately after modification. + std::atomic lastRotated_ {}; NodeStore::Scheduler& scheduler_; beast::Journal const journal_; @@ -159,7 +167,7 @@ class SHAMapStoreImp : public SHAMapStore LedgerIndex getLastRotated() override { - return state_db_.getState().lastRotated; + return lastRotated_; } // All ledgers before and including this are unprotected diff --git a/src/ripple/app/paths/impl/XRPEndpointStep.cpp b/src/ripple/app/paths/impl/XRPEndpointStep.cpp index 2c6b91680fe..d30bd1887f7 100644 --- a/src/ripple/app/paths/impl/XRPEndpointStep.cpp +++ b/src/ripple/app/paths/impl/XRPEndpointStep.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -359,6 +360,18 @@ XRPEndpointStep::check (StrandContext const& ctx) const if (ter != tesSUCCESS) return ter; + if (ctx.view.rules().enabled(fix1781)) + { + auto const issuesIndex = isLast_ ? 0 : 1; + if (!ctx.seenDirectIssues[issuesIndex].insert(xrpIssue()).second) + { + JLOG(j_.debug()) + << "XRPEndpointStep: loop detected: Index: " << ctx.strandSize + << ' ' << *this; + return temBAD_PATH_LOOP; + } + } + return tesSUCCESS; } diff --git a/src/ripple/basics/CompressionAlgorithms.h b/src/ripple/basics/CompressionAlgorithms.h new file mode 100644 index 00000000000..3cd67c753d8 --- /dev/null +++ b/src/ripple/basics/CompressionAlgorithms.h @@ -0,0 +1,153 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLED_COMPRESSIONALGORITHMS_H_INCLUDED +#define RIPPLED_COMPRESSIONALGORITHMS_H_INCLUDED + +#include +#include +#include + +namespace ripple { + +namespace compression_algorithms { + +/** Convenience wrapper for Throw + * @param message Message to log/throw + */ +inline void doThrow(const char *message) +{ + Throw(message); +} + +/** LZ4 block compression. + * @tparam BufferFactory Callable object or lambda. + * Takes the requested buffer size and returns allocated buffer pointer. + * @param in Data to compress + * @param inSize Size of the data + * @param bf Compressed buffer allocator + * @return Size of compressed data, or zero if failed to compress + */ +template +std::size_t +lz4Compress(void const* in, + std::size_t inSize, BufferFactory&& bf) +{ + if (inSize > UINT32_MAX) + doThrow("lz4 compress: invalid size"); + + auto const outCapacity = LZ4_compressBound(inSize); + + // Request the caller to allocate and return the buffer to hold compressed data + auto compressed = bf(outCapacity); + + auto compressedSize = LZ4_compress_default( + reinterpret_cast(in), + reinterpret_cast(compressed), + inSize, + outCapacity); + if (compressedSize == 0) + doThrow("lz4 compress: failed"); + + return compressedSize; +} + +/** + * @param in Compressed data + * @param inSize Size of compressed data + * @param decompressed Buffer to hold decompressed data + * @param decompressedSize Size of the decompressed buffer + * @return size of the decompressed data + */ +inline +std::size_t +lz4Decompress(std::uint8_t const* in, std::size_t inSize, + std::uint8_t* decompressed, std::size_t decompressedSize) +{ + auto ret = LZ4_decompress_safe(reinterpret_cast(in), + reinterpret_cast(decompressed), inSize, decompressedSize); + + if (ret <= 0 || ret != decompressedSize) + doThrow("lz4 decompress: failed"); + + return decompressedSize; +} + +/** LZ4 block decompression. + * @tparam InputStream ZeroCopyInputStream + * @param in Input source stream + * @param inSize Size of compressed data + * @param decompressed Buffer to hold decompressed data + * @param decompressedSize Size of the decompressed buffer + * @return size of the decompressed data + */ +template +std::size_t +lz4Decompress(InputStream& in, std::size_t inSize, + std::uint8_t* decompressed, std::size_t decompressedSize) +{ + std::vector compressed; + std::uint8_t const* chunk = nullptr; + int chunkSize = 0; + int copiedInSize = 0; + auto const currentBytes = in.ByteCount(); + + // Use the first chunk if it is >= inSize bytes of the compressed message. + // Otherwise copy inSize bytes of chunks into compressed buffer and + // use the buffer to decompress. + while (in.Next(reinterpret_cast(&chunk), &chunkSize)) + { + if (copiedInSize == 0) + { + if (chunkSize >= inSize) + { + copiedInSize = inSize; + break; + } + compressed.resize(inSize); + } + + chunkSize = chunkSize < (inSize - copiedInSize) ? chunkSize : (inSize - copiedInSize); + + std::copy(chunk, chunk + chunkSize, compressed.data() + copiedInSize); + + copiedInSize += chunkSize; + + if (copiedInSize == inSize) + { + chunk = compressed.data(); + break; + } + } + + // Put back unused bytes + if (in.ByteCount() > (currentBytes + copiedInSize)) + in.BackUp(in.ByteCount() - currentBytes - copiedInSize); + + if ((copiedInSize == 0 && chunkSize < inSize) || (copiedInSize > 0 && copiedInSize != inSize)) + doThrow("lz4 decompress: insufficient input size"); + + return lz4Decompress(chunk, inSize, decompressed, decompressedSize); +} + +} // compression + +} // ripple + +#endif //RIPPLED_COMPRESSIONALGORITHMS_H_INCLUDED diff --git a/src/ripple/basics/RangeSet.h b/src/ripple/basics/RangeSet.h index 13f58c94ad9..4e00a4627ed 100644 --- a/src/ripple/basics/RangeSet.h +++ b/src/ripple/basics/RangeSet.h @@ -20,11 +20,14 @@ #ifndef RIPPLE_BASICS_RANGESET_H_INCLUDED #define RIPPLE_BASICS_RANGESET_H_INCLUDED -#include -#include +#include + +#include #include #include -#include +#include + +#include namespace ripple { @@ -86,8 +89,8 @@ std::string to_string(ClosedInterval const & ci) /** Convert the given RangeSet to a styled string. - The styled string represention is the set of disjoint intervals joined by - commas. The string "empty" is returned if the set is empty. + The styled string representation is the set of disjoint intervals joined + by commas. The string "empty" is returned if the set is empty. @param rs The rangeset to convert @return The styled string @@ -109,6 +112,67 @@ std::string to_string(RangeSet const & rs) return res; } +/** Convert the given styled string to a RangeSet. + + The styled string representation is the set + of disjoint intervals joined by commas. + + @param rs The set to be populated + @param s The styled string to convert + @return True on successfully converting styled string +*/ +template +bool +from_string(RangeSet& rs, std::string const& s) +{ + std::vector intervals; + std::vector tokens; + bool result {true}; + + boost::split(tokens, s, boost::algorithm::is_any_of(",")); + for (auto const& t : tokens) + { + boost::split(intervals, t, boost::algorithm::is_any_of("-")); + switch (intervals.size()) + { + case 1: + { + T front; + if (!beast::lexicalCastChecked(front, intervals.front())) + result = false; + else + rs.insert(front); + break; + } + case 2: + { + T front; + if (!beast::lexicalCastChecked(front, intervals.front())) + result = false; + else + { + T back; + if (!beast::lexicalCastChecked(back, intervals.back())) + result = false; + else + rs.insert(range(front, back)); + } + break; + } + default: + result = false; + } + + if (!result) + break; + intervals.clear(); + } + + if (!result) + rs.clear(); + return result; +} + /** Find the largest value not in the set that is less than a given value. @param rs The set of interest @@ -129,75 +193,8 @@ prevMissing(RangeSet const & rs, T t, T minVal = 0) return boost::none; return boost::icl::last(tgt); } -} // namespace ripple - - -// The boost serialization documents recommended putting free-function helpers -// in the boost serialization namespace -namespace boost { -namespace serialization { -template -void -save(Archive& ar, - ripple::ClosedInterval const& ci, - const unsigned int version) -{ - auto l = ci.lower(); - auto u = ci.upper(); - ar << l << u; -} - -template -void -load(Archive& ar, ripple::ClosedInterval& ci, const unsigned int version) -{ - T low, up; - ar >> low >> up; - ci = ripple::ClosedInterval{low, up}; -} - -template -void -serialize(Archive& ar, - ripple::ClosedInterval& ci, - const unsigned int version) -{ - split_free(ar, ci, version); -} - -template -void -save(Archive& ar, ripple::RangeSet const& rs, const unsigned int version) -{ - auto s = rs.iterative_size(); - ar << s; - for (auto const& r : rs) - ar << r; -} - -template -void -load(Archive& ar, ripple::RangeSet& rs, const unsigned int version) -{ - rs.clear(); - std::size_t intervals; - ar >> intervals; - for (std::size_t i = 0; i < intervals; ++i) - { - ripple::ClosedInterval ci; - ar >> ci; - rs.insert(ci); - } -} +} // namespace ripple -template -void -serialize(Archive& ar, ripple::RangeSet& rs, const unsigned int version) -{ - split_free(ar, rs, version); -} -} // serialization -} // boost #endif diff --git a/src/ripple/basics/TaggedCache.h b/src/ripple/basics/TaggedCache.h index 3ed6bd1c3bb..4898e9831d3 100644 --- a/src/ripple/basics/TaggedCache.h +++ b/src/ripple/basics/TaggedCache.h @@ -31,9 +31,6 @@ namespace ripple { -// VFALCO NOTE Deprecated -struct TaggedCacheLog; - /** Map/cache combination. This class implements a cache and a map. The cache keeps objects alive in the map. The map allows multiple code paths that reference objects @@ -291,7 +288,14 @@ class TaggedCache @return `true` If the key already existed. */ - bool canonicalize (const key_type& key, std::shared_ptr& data, bool replace = false) +private: + + template + bool + canonicalize( + const key_type& key, + std::conditional_t const, std::shared_ptr>& data + ) { // Return canonical value, store if needed, refresh in cache // Return values: true=we had the data already @@ -313,7 +317,7 @@ class TaggedCache if (entry.isCached ()) { - if (replace) + if constexpr (replace) { entry.ptr = data; entry.weak_ptr = data; @@ -330,7 +334,7 @@ class TaggedCache if (cachedData) { - if (replace) + if constexpr (replace) { entry.ptr = data; entry.weak_ptr = data; @@ -352,6 +356,17 @@ class TaggedCache return false; } +public: + bool canonicalize_replace_cache(const key_type& key, std::shared_ptr const& data) + { + return canonicalize(key, data); + } + + bool canonicalize_replace_client(const key_type& key, std::shared_ptr& data) + { + return canonicalize(key, data); + } + std::shared_ptr fetch (const key_type& key) { // fetch us a shared pointer to the stored data object @@ -396,7 +411,7 @@ class TaggedCache { mapped_ptr p (std::make_shared ( std::cref (value))); - return canonicalize (key, p); + return canonicalize_replace_client(key, p); } // VFALCO NOTE It looks like this returns a copy of the data in diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index e1200c63cef..5231f169468 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -171,6 +171,9 @@ class Config : public BasicConfig std::string SSL_VERIFY_FILE; std::string SSL_VERIFY_DIR; + // Compression + bool COMPRESSION = false; + // Thread pool configuration std::size_t WORKERS = 0; diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index e5f1a3f490e..653b9c404e4 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -37,6 +37,7 @@ struct ConfigSection // VFALCO TODO Rename and replace these macros with variables. #define SECTION_AMENDMENTS "amendments" #define SECTION_CLUSTER_NODES "cluster_nodes" +#define SECTION_COMPRESSION "compression" #define SECTION_DEBUG_LOGFILE "debug_logfile" #define SECTION_ELB_SUPPORT "elb_support" #define SECTION_FEE_DEFAULT "fee_default" diff --git a/src/ripple/core/DatabaseCon.h b/src/ripple/core/DatabaseCon.h index b4dbf6b6d9c..0090df52b0e 100644 --- a/src/ripple/core/DatabaseCon.h +++ b/src/ripple/core/DatabaseCon.h @@ -102,18 +102,17 @@ class DatabaseCon boost::filesystem::path pPath = useTempFiles ? "" : (setup.dataDir / DBName); - open(session_, "sqlite", pPath.string()); + init(pPath, pragma, initSQL); + } - for (auto const& p : pragma) - { - soci::statement st = session_.prepare << p; - st.execute(true); - } - for (auto const& sql : initSQL) - { - soci::statement st = session_.prepare << sql; - st.execute(true); - } + template + DatabaseCon( + boost::filesystem::path const& dataDir, + std::string const& DBName, + std::array const& pragma, + std::array const& initSQL) + { + init((dataDir / DBName), pragma, initSQL); } soci::session& getSession() @@ -129,6 +128,27 @@ class DatabaseCon void setupCheckpointing (JobQueue*, Logs&); private: + + template + void + init(boost::filesystem::path const& pPath, + std::array const& pragma, + std::array const& initSQL) + { + open(session_, "sqlite", pPath.string()); + + for (auto const& p : pragma) + { + soci::statement st = session_.prepare << p; + st.execute(true); + } + for (auto const& sql : initSQL) + { + soci::statement st = session_.prepare << sql; + st.execute(true); + } + } + LockedSociSession::mutex lock_; soci::session session_; diff --git a/src/ripple/core/Stoppable.h b/src/ripple/core/Stoppable.h index cca36e013a4..4d795147f81 100644 --- a/src/ripple/core/Stoppable.h +++ b/src/ripple/core/Stoppable.h @@ -186,13 +186,13 @@ class RootStoppable; | JobQueue | - +-----------+-----------+-----------+-----------+----+--------+ - | | | | | | - | NetworkOPs | InboundLedgers | OrderbookDB - | | | - Overlay InboundTransactions LedgerMaster - | | - PeerFinder LedgerCleaner + +--------+-----------+-----------+-----------+-------+---+----------+ + | | | | | | | + | NetworkOPs | InboundLedgers | OrderbookDB | + | | | | + Overlay InboundTransactions LedgerMaster Database + | | | + PeerFinder LedgerCleaner TaskQueue @endcode */ diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index b49bcebffb9..36732800616 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -454,6 +454,9 @@ void Config::loadFromString (std::string const& fileContents) if (getSingleSection (secConfig, SECTION_WORKERS, strTemp, j_)) WORKERS = beast::lexicalCastThrow (strTemp); + if (getSingleSection (secConfig, SECTION_COMPRESSION, strTemp, j_)) + COMPRESSION = beast::lexicalCastThrow (strTemp); + // Do not load trusted validator configuration for standalone mode if (! RUN_STANDALONE) { diff --git a/src/ripple/core/impl/JobQueue.cpp b/src/ripple/core/impl/JobQueue.cpp index c418bc67a03..3cf796f06e1 100644 --- a/src/ripple/core/impl/JobQueue.cpp +++ b/src/ripple/core/impl/JobQueue.cpp @@ -31,7 +31,7 @@ JobQueue::JobQueue (beast::insight::Collector::ptr const& collector, , m_lastJob (0) , m_invalidJobData (JobTypes::instance().getInvalid (), collector, logs) , m_processCount (0) - , m_workers (*this, perfLog, "JobQueue", 0) + , m_workers (*this, &perfLog, "JobQueue", 0) , m_cancelCallback (std::bind (&Stoppable::isStopping, this)) , perfLog_ (perfLog) , m_collector (collector) diff --git a/src/ripple/core/impl/Workers.cpp b/src/ripple/core/impl/Workers.cpp index ca456de5728..f04f94e4b84 100644 --- a/src/ripple/core/impl/Workers.cpp +++ b/src/ripple/core/impl/Workers.cpp @@ -26,7 +26,7 @@ namespace ripple { Workers::Workers ( Callback& callback, - perf::PerfLog& perfLog, + perf::PerfLog* perfLog, std::string const& threadNames, int numberOfThreads) : m_callback (callback) @@ -63,7 +63,8 @@ void Workers::setNumberOfThreads (int numberOfThreads) static int instance {0}; if (m_numberOfThreads != numberOfThreads) { - perfLog_.resizeJobs(numberOfThreads); + if (perfLog_) + perfLog_->resizeJobs(numberOfThreads); if (numberOfThreads > m_numberOfThreads) { diff --git a/src/ripple/core/impl/Workers.h b/src/ripple/core/impl/Workers.h index 9721ae9e6e2..3a811ce899c 100644 --- a/src/ripple/core/impl/Workers.h +++ b/src/ripple/core/impl/Workers.h @@ -69,7 +69,7 @@ class Workers @param threadNames The name given to each created worker thread. */ explicit Workers (Callback& callback, - perf::PerfLog& perfLog, + perf::PerfLog* perfLog, std::string const& threadNames = "Worker", int numberOfThreads = static_cast(std::thread::hardware_concurrency())); @@ -166,7 +166,7 @@ class Workers private: Callback& m_callback; - perf::PerfLog& perfLog_; + perf::PerfLog* perfLog_; std::string m_threadNames; // The name to give each thread std::condition_variable m_cv; // signaled when all threads paused std::mutex m_mut; diff --git a/src/ripple/core/impl/semaphore.h b/src/ripple/core/impl/semaphore.h index 4b417525166..392a0d48718 100644 --- a/src/ripple/core/impl/semaphore.h +++ b/src/ripple/core/impl/semaphore.h @@ -29,8 +29,6 @@ template class basic_semaphore { private: - using scoped_lock = std::unique_lock ; - Mutex m_mutex; CondVar m_cond; std::size_t m_count; @@ -49,7 +47,7 @@ class basic_semaphore /** Increment the count and unblock one waiting thread. */ void notify () { - scoped_lock lock (m_mutex); + std::lock_guard lock{m_mutex}; ++m_count; m_cond.notify_one (); } @@ -57,7 +55,7 @@ class basic_semaphore /** Block until notify is called. */ void wait () { - scoped_lock lock (m_mutex); + std::unique_lock lock{m_mutex}; while (m_count == 0) m_cond.wait (lock); --m_count; @@ -68,7 +66,7 @@ class basic_semaphore */ bool try_wait () { - scoped_lock lock (m_mutex); + std::lock_guard lock{m_mutex}; if (m_count == 0) return false; --m_count; diff --git a/src/ripple/net/DatabaseBody.h b/src/ripple/net/DatabaseBody.h new file mode 100644 index 00000000000..c616aeb9b7f --- /dev/null +++ b/src/ripple/net/DatabaseBody.h @@ -0,0 +1,170 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_NET_DATABASEBODY_H +#define RIPPLE_NET_DATABASEBODY_H + +#include +#include +#include +#include +#include + +namespace ripple { + +struct DatabaseBody +{ + // Algorithm for storing buffers when parsing. + class reader; + + // The type of the @ref message::body member. + class value_type; + + /** Returns the size of the body + + @param body The database body to use + */ + static std::uint64_t + size(value_type const& body); +}; + +class DatabaseBody::value_type +{ + // This body container holds a connection to the + // database, and also caches the size when set. + + friend class reader; + friend struct DatabaseBody; + + // The cached file size + std::uint64_t file_size_ = 0; + boost::filesystem::path path_; + std::unique_ptr conn_; + std::string batch_; + std::shared_ptr strand_; + std::mutex m_; + std::condition_variable c_; + uint64_t handler_count_ = 0; + uint64_t part_ = 0; + bool closing_ = false; + +public: + /// Destructor + ~value_type() = default; + + /// Constructor + value_type() = default; + + /// Returns `true` if the file is open + bool + is_open() const + { + return bool{conn_}; + } + + /// Returns the size of the file if open + std::uint64_t + size() const + { + return file_size_; + } + + /// Close the file if open + void + close(); + + /** Open a file at the given path with the specified mode + + @param path The utf-8 encoded path to the file + + @param mode The file mode to use + + @param ec Set to the error, if any occurred + */ + void + open( + boost::filesystem::path path, + Config const& config, + boost::asio::io_service& io_service, + boost::system::error_code& ec); +}; + +/** Algorithm for storing buffers when parsing. + + Objects of this type are created during parsing + to store incoming buffers representing the body. +*/ +class DatabaseBody::reader +{ + value_type& body_; // The body we are writing to + + static const uint32_t FLUSH_SIZE = 50000000; + static const uint8_t MAX_HANDLERS = 3; + static const uint16_t MAX_ROW_SIZE_PAD = 500; + +public: + // Constructor. + // + // This is called after the header is parsed and + // indicates that a non-zero sized body may be present. + // `h` holds the received message headers. + // `b` is an instance of `DatabaseBody`. + // + template + explicit reader( + boost::beast::http::header& h, + value_type& b); + + // Initializer + // + // This is called before the body is parsed and + // gives the reader a chance to do something that might + // need to return an error code. It informs us of + // the payload size (`content_length`) which we can + // optionally use for optimization. + // + void + init(boost::optional const&, boost::system::error_code& ec); + + // This function is called one or more times to store + // buffer sequences corresponding to the incoming body. + // + template + std::size_t + put(ConstBufferSequence const& buffers, boost::system::error_code& ec); + + void + do_put(std::string data); + + // This function is called when writing is complete. + // It is an opportunity to perform any final actions + // which might fail, in order to return an error code. + // Operations that might fail should not be attempted in + // destructors, since an exception thrown from there + // would terminate the program. + // + void + finish(boost::system::error_code& ec); +}; + +} // namespace ripple + +#include + +#endif // RIPPLE_NET_DATABASEBODY_H diff --git a/src/ripple/net/DatabaseDownloader.h b/src/ripple/net/DatabaseDownloader.h new file mode 100644 index 00000000000..ec86242c349 --- /dev/null +++ b/src/ripple/net/DatabaseDownloader.h @@ -0,0 +1,60 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_NET_DATABASEDOWNLOADER_H +#define RIPPLE_NET_DATABASEDOWNLOADER_H + +#include +#include + +namespace ripple { + +class DatabaseDownloader : public SSLHTTPDownloader +{ +public: + DatabaseDownloader( + boost::asio::io_service& io_service, + beast::Journal j, + Config const& config); + +private: + static const uint8_t MAX_PATH_LEN = std::numeric_limits::max(); + + std::shared_ptr + getParser( + boost::filesystem::path dstPath, + std::function complete, + boost::system::error_code& ec) override; + + bool + checkPath(boost::filesystem::path const& dstPath) override; + + void + closeBody(std::shared_ptr p) override; + + uint64_t + size(std::shared_ptr p) override; + + Config const& config_; + boost::asio::io_service& io_service_; +}; + +} // namespace ripple + +#endif // RIPPLE_NET_DATABASEDOWNLOADER_H diff --git a/src/ripple/net/SSLHTTPDownloader.h b/src/ripple/net/SSLHTTPDownloader.h index 893043ce49f..b70383373cb 100644 --- a/src/ripple/net/SSLHTTPDownloader.h +++ b/src/ripple/net/SSLHTTPDownloader.h @@ -50,7 +50,8 @@ class SSLHTTPDownloader SSLHTTPDownloader( boost::asio::io_service& io_service, beast::Journal j, - Config const& config); + Config const& config, + bool isPaused = false); bool download( @@ -61,13 +62,36 @@ class SSLHTTPDownloader boost::filesystem::path const& dstPath, std::function complete); + void + onStop(); + + virtual + ~SSLHTTPDownloader() = default; + +protected: + + using parser = boost::beast::http::basic_parser; + + beast::Journal const j_; + + bool + fail( + boost::filesystem::path dstPath, + std::function const& complete, + boost::system::error_code const& ec, + std::string const& errMsg, + std::shared_ptr parser = nullptr); + private: HTTPClientSSLContext ssl_ctx_; boost::asio::io_service::strand strand_; boost::optional< boost::asio::ssl::stream> stream_; boost::beast::flat_buffer read_buf_; - beast::Journal const j_; + std::atomic isStopped_; + bool sessionActive_; + std::mutex m_; + std::condition_variable c_; void do_session( @@ -79,12 +103,25 @@ class SSLHTTPDownloader std::function complete, boost::asio::yield_context yield); - void - fail( + virtual + std::shared_ptr + getParser( boost::filesystem::path dstPath, - std::function const& complete, - boost::system::error_code const& ec, - std::string const& errMsg); + std::function complete, + boost::system::error_code & ec) = 0; + + virtual + bool + checkPath( + boost::filesystem::path const& dstPath) = 0; + + virtual + void + closeBody(std::shared_ptr p) = 0; + + virtual + uint64_t + size(std::shared_ptr p) = 0; }; } // ripple diff --git a/src/ripple/net/ShardDownloader.md b/src/ripple/net/ShardDownloader.md new file mode 100644 index 00000000000..9ef643a964b --- /dev/null +++ b/src/ripple/net/ShardDownloader.md @@ -0,0 +1,263 @@ +# Shard Downloader Process + +## Overview + +This document describes mechanics of the `SSLHTTPDownloader`, a class that performs the task of downloading shards from remote web servers via +SSL HTTP. The downloader utilizes a strand (`boost::asio::io_service::strand`) to ensure that downloads are never executed concurrently. Hence, if a download is in progress when another download is initiated, the second download will be queued and invoked only when the first download is completed. + +## New Features + +The downloader has been recently (March 2020) been modified to provide some key features: + +- The ability to stop downloads during a graceful shutdown. +- The ability to resume partial downloads after a crash or shutdown. +- *(Deferred) The ability to download from multiple servers to a single file.* + +## Classes + +Much of the shard downloading process concerns the following classes: + +- `SSLHTTPDownloader` + + This is a generic class designed for serially executing downloads via HTTP SSL. + +- `ShardArchiveHandler` + + This class uses the `SSLHTTPDownloader` to fetch shards from remote web servers. Additionally, the archive handler performs sanity checks on the downloaded files and imports the validated files into the local shard store. + + The `ShardArchiveHandler` exposes a simple public interface: + + ```C++ + /** Add an archive to be downloaded and imported. + @param shardIndex the index of the shard to be imported. + @param url the location of the archive. + @return `true` if successfully added. + @note Returns false if called while downloading. + */ + bool + add(std::uint32_t shardIndex, std::pair&& url); + + /** Starts downloading and importing archives. */ + bool + start(); + ``` + + When a client submits a `download_shard` command via the RPC interface, each of the requested files is registered with the handler via the `add` method. After all the files have been registered, the handler's `start` method is invoked, which in turn creates an instance of the `SSLHTTPDownloader` and begins the first download. When the download is completed, the downloader invokes the handler's `complete` method, which will initiate the download of the next file, or simply return if there are no more downloads to process. When `complete` is invoked with no remaining files to be downloaded, the handler and downloader are not destroyed automatically, but persist for the duration of the application. + +Additionally, we leverage a novel class to provide customized parsing for downloaded files: + +- `DatabaseBody` + + This class will define a custom message body type, allowing an `http::response_parser` to write to a SQLite database rather than to a flat file. This class is discussed in further detail in the Recovery section. + +## Execution Concept + +This section describes in greater detail how the key features of the downloader are implemented in C++ using the `boost::asio` framework. + +##### Member Variables: + +The variables shown here are members of the `SSLHTTPDownloader` class and +will be used in the following code examples. + +```c++ +using boost::asio::ssl::stream; +using boost::asio::ip::tcp::socket; + +stream stream_; +std::condition_variable c_; +std::atomic isStopped_; +``` + +### Graceful Shutdowns + +##### Thread 1: + +A graceful shutdown begins when the `onStop()` method of the `ShardArchiveHandler` is invoked: + +```c++ +void +ShardArchiveHandler::onStop() +{ + std::lock_guard lock(m_); + + if (downloader_) + { + downloader_->onStop(); + downloader_.reset(); + } + + stopped(); +} +``` + +Inside of `SSLHTTPDownloader::onStop()`, if a download is currently in progress, the `isStopped_` member variable is set and the thread waits for the download to stop: + +```c++ +void +SSLHTTPDownloader::onStop() +{ + std::unique_lock lock(m_); + + isStopped_ = true; + + if(sessionActive_) + { + // Wait for the handler to exit. + c_.wait(lock, + [this]() + { + return !sessionActive_; + }); + } +} +``` + +##### Thread 2: + +The graceful shutdown is realized when the thread executing the download polls `isStopped_` after this variable has been set to `true`. Polling only occurs while the file is being downloaded, in between calls to `async_read_some()`. The stop takes effect when the socket is closed and the handler function ( `do_session()` ) is exited. + +```c++ +void SSLHTTPDownloader::do_session() +{ + + // (Connection initialization logic) + + . + . + . + + // (In between calls to async_read_some): + if(isStopped_.load()) + { + close(p); + return exit(); + } + + . + . + . + + break; +} +``` + +### Recovery + +Persisting the current state of both the archive handler and the downloader is achieved by leveraging a SQLite database rather than flat files, as the database protects against data corruption that could result from a system crash. + +##### ShardArchiveHandler + +Although `SSLHTTPDownloader` is a generic class that could be used to download a variety of file types, currently it is used exclusively by the `ShardArchiveHandler` to download shards. In order to provide resilience, the `ShardArchiveHandler` will utilize a SQLite database to preserve its current state whenever there are active, paused, or queued downloads. The `shard_db` section in the configuration file allows users to specify the location of the database to use for this purpose. + +###### SQLite Table Format + +| Index | URL | +|:-----:|:-----------------------------------:| +| 1 | https://example.com/1.tar.lz4 | +| 2 | https://example.com/2.tar.lz4 | +| 5 | https://example.com/5.tar.lz4 | + +##### SSLHTTPDownloader + +While the archive handler maintains a list of all partial and queued downloads, the `SSLHTTPDownloader` stores the raw bytes of the file currently being downloaded. The partially downloaded file will be represented as one or more `BLOB` entries in a SQLite database. As the maximum size of a `BLOB` entry is currently limited to roughly 2.1 GB, a 5 GB shard file for instance will occupy three database entries upon completion. + +###### SQLite Table Format + +Since downloads execute serially by design, the entries in this table always correspond to the content of a single file. + +| Bytes | Size | Part | +|:------:|:----------:|:----:| +| 0x... | 2147483647 | 0 | +| 0x... | 2147483647 | 1 | +| 0x... | 705032706 | 2 | + +##### Config File Entry +The `download_path` field of the `shard_db` entry will be used to determine where to store the recovery database. If this field is omitted, the `path` field will be used instead. + +```dosini +# This is the persistent datastore for shards. It is important for the health +# of the ripple network that rippled operators shard as much as practical. +# NuDB requires SSD storage. Helpful information can be found here +# https://ripple.com/build/history-sharding +[shard_db] +type=NuDB +path=/var/lib/rippled/db/shards/nudb +download_path=/var/lib/rippled/db/shards/ +max_size_gb=50 +``` + +##### Resuming Partial Downloads +When resuming downloads after a crash or other interruption, the `SSLHTTPDownloader` will utilize the `range` field of the HTTP header to download only the remainder of the partially downloaded file. + +```C++ +auto downloaded = getPartialFileSize(); +auto total = getTotalFileSize(); + +http::request req {http::verb::head, + target, + version}; + +if (downloaded < total) +{ + // If we already download 1000 bytes to the partial file, + // the range header will look like: + // Range: "bytes=1000-" + req.set(http::field::range, "bytes=" + to_string(downloaded) + "-"); +} +else if(downloaded == total) +{ + // Download is already complete. (Interruption Must + // have occurred after file was downloaded but before + // the state file was updated.) +} +else +{ + // The size of the partially downloaded file exceeds + // the total download size. Error condition. Handle + // appropriately. +} +``` + +##### DatabaseBody + +Previously, the `SSLHTTPDownloader` leveraged an `http::response_parser` instantiated with an `http::file_body`. The `file_body` class declares a nested type, `reader`, which does the task of writing HTTP message payloads (constituting a requested file) to the filesystem. In order for the `http::response_parser` to interface with the database, we implement a custom body type that declares a nested `reader` type which has been outfitted to persist octects received from the remote host to a local SQLite database. The code snippet below illustrates the customization points available to user-defined body types: + +```C++ +/// Defines a Body type +struct body +{ + /// This determines the return type of the `message::body` member function + using value_type = ...; + + /// An optional function, returns the body's payload size (which may be zero) + static + std::uint64_t + size(value_type const& v); + + /// The algorithm used for extracting buffers + class reader; + + /// The algorithm used for inserting buffers + class writer; +} + +``` + +The method invoked to write data to the filesystem (or SQLite database in our case) has the following signature: + +```C++ +std::size_t +body::reader::put(ConstBufferSequence const& buffers, error_code& ec); +``` + +## Sequence Diagram + +This sequence diagram demonstrates a scenario wherein the `ShardArchiveHandler` leverages the state persisted in the database to recover from a crash and resume the scheduled downloads. + +![alt_text](./images/interrupt_sequence.png "Resuming downloads post abort") + +## State Diagram + +This diagram illustrates the various states of the Shard Downloader module. + +![alt_text](./images/states.png "Shard Downloader states") diff --git a/src/ripple/net/images/interrupt_sequence.png b/src/ripple/net/images/interrupt_sequence.png new file mode 100644 index 00000000000..87cc3c8d796 Binary files /dev/null and b/src/ripple/net/images/interrupt_sequence.png differ diff --git a/src/ripple/net/images/states.png b/src/ripple/net/images/states.png new file mode 100644 index 00000000000..d982955ddf2 Binary files /dev/null and b/src/ripple/net/images/states.png differ diff --git a/src/ripple/net/impl/DatabaseBody.ipp b/src/ripple/net/impl/DatabaseBody.ipp new file mode 100644 index 00000000000..d5b854f4bed --- /dev/null +++ b/src/ripple/net/impl/DatabaseBody.ipp @@ -0,0 +1,312 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +namespace ripple { + +inline void +DatabaseBody::value_type::close() +{ + { + std::unique_lock lock(m_); + + // Stop all scheduled and currently + // executing handlers before closing. + if (handler_count_) + { + closing_ = true; + + auto predicate = [&] { return !handler_count_; }; + c_.wait(lock, predicate); + } + + conn_.reset(); + } +} + +inline void +DatabaseBody::value_type::open( + boost::filesystem::path path, + Config const& config, + boost::asio::io_service& io_service, + boost::system::error_code& ec) +{ + strand_.reset(new boost::asio::io_service::strand(io_service)); + + auto setup = setup_DatabaseCon(config); + setup.dataDir = path.parent_path(); + + conn_ = std::make_unique( + setup, "Download", DownloaderDBPragma, DatabaseBodyDBInit); + + path_ = path; + + auto db = conn_->checkoutDb(); + + boost::optional pathFromDb; + + *db << "SELECT Path FROM Download WHERE Part=0;", soci::into(pathFromDb); + + // Try to reuse preexisting + // database. + if (pathFromDb) + { + // Can't resuse - database was + // from a different file download. + if (pathFromDb != path.string()) + { + *db << "DROP TABLE Download;"; + } + + // Continuing a file download. + else + { + boost::optional size; + + *db << "SELECT SUM(LENGTH(Data)) FROM Download;", soci::into(size); + + if (size) + file_size_ = size.get(); + } + } +} + +// This is called from message::payload_size +inline std::uint64_t +DatabaseBody::size(value_type const& body) +{ + // Forward the call to the body + return body.size(); +} + +// We don't do much in the reader constructor since the +// database is already open. +// +template +DatabaseBody::reader::reader( + boost::beast::http::header&, + value_type& body) + : body_(body) +{ +} + +// We don't do anything with content_length but a sophisticated +// application might check available space on the device +// to see if there is enough room to store the body. +inline void +DatabaseBody::reader::init( + boost::optional const& /*content_length*/, + boost::system::error_code& ec) +{ + // The connection must already be available for writing + assert(body_.conn_); + + // The error_code specification requires that we + // either set the error to some value, or set it + // to indicate no error. + // + // We don't do anything fancy so set "no error" + ec = {}; +} + +// This will get called one or more times with body buffers +// +template +std::size_t +DatabaseBody::reader::put( + ConstBufferSequence const& buffers, + boost::system::error_code& ec) +{ + // This function must return the total number of + // bytes transferred from the input buffers. + std::size_t nwritten = 0; + + // Loop over all the buffers in the sequence, + // and write each one to the database. + for (auto it = buffer_sequence_begin(buffers); + it != buffer_sequence_end(buffers); + ++it) + { + boost::asio::const_buffer buffer = *it; + + body_.batch_.append( + static_cast(buffer.data()), buffer.size()); + + // Write this buffer to the database + if (body_.batch_.size() > FLUSH_SIZE) + { + bool post = true; + + { + std::lock_guard lock(body_.m_); + + if (body_.handler_count_ >= MAX_HANDLERS) + post = false; + else + ++body_.handler_count_; + } + + if (post) + { + body_.strand_->post( + [data = body_.batch_, this] { this->do_put(data); }); + + body_.batch_.clear(); + } + } + + nwritten += it->size(); + } + + // Indicate success + // This is required by the error_code specification + ec = {}; + + return nwritten; +} + +inline void +DatabaseBody::reader::do_put(std::string data) +{ + using namespace boost::asio; + + { + std::unique_lock lock(body_.m_); + + // The download is being halted. + if (body_.closing_) + { + if (--body_.handler_count_ == 0) + { + lock.unlock(); + body_.c_.notify_one(); + } + + return; + } + } + + auto path = body_.path_.string(); + uint64_t rowSize; + soci::indicator rti; + + uint64_t remainingInRow; + + auto db = body_.conn_->checkoutDb(); + + auto be = dynamic_cast(db->get_backend()); + BOOST_ASSERT(be); + + // This limits how large we can make the blob + // in each row. Also subtract a pad value to + // account for the other values in the row. + auto const blobMaxSize = + sqlite_api::sqlite3_limit(be->conn_, SQLITE_LIMIT_LENGTH, -1) - + MAX_ROW_SIZE_PAD; + + auto rowInit = [&] { + *db << "INSERT INTO Download VALUES (:path, zeroblob(0), 0, :part)", + soci::use(path), soci::use(body_.part_); + + remainingInRow = blobMaxSize; + rowSize = 0; + }; + + *db << "SELECT Path,Size,Part FROM Download ORDER BY Part DESC " + "LIMIT 1", + soci::into(path), soci::into(rowSize), soci::into(body_.part_, rti); + + if (!db->got_data()) + rowInit(); + else + remainingInRow = blobMaxSize - rowSize; + + auto insert = [&db, &rowSize, &part = body_.part_, &fs = body_.file_size_]( + auto const& data) { + uint64_t updatedSize = rowSize + data.size(); + + *db << "UPDATE Download SET Data = CAST(Data || :data AS blob), " + "Size = :size WHERE Part = :part;", + soci::use(data), soci::use(updatedSize), soci::use(part); + + fs += data.size(); + }; + + while (remainingInRow < data.size()) + { + if (remainingInRow) + { + insert(data.substr(0, remainingInRow)); + data.erase(0, remainingInRow); + } + + ++body_.part_; + rowInit(); + } + + insert(data); + + bool const notify = [this] { + std::lock_guard lock(body_.m_); + return --body_.handler_count_ == 0; + }(); + + if (notify) + body_.c_.notify_one(); +} + +// Called after writing is done when there's no error. +inline void +DatabaseBody::reader::finish(boost::system::error_code& ec) +{ + { + std::unique_lock lock(body_.m_); + + // Wait for scheduled DB writes + // to complete. + if (body_.handler_count_) + { + auto predicate = [&] { return !body_.handler_count_; }; + body_.c_.wait(lock, predicate); + } + } + + auto db = body_.conn_->checkoutDb(); + + soci::rowset rs = + (db->prepare << "SELECT Data FROM Download ORDER BY PART ASC;"); + + std::ofstream fout; + fout.open(body_.path_.string(), std::ios::binary | std::ios::out); + + // iteration through the resultset: + for (auto it = rs.begin(); it != rs.end(); ++it) + fout.write(it->data(), it->size()); + + // Flush any pending data that hasn't + // been been written to the DB. + if (body_.batch_.size()) + { + fout.write(body_.batch_.data(), body_.batch_.size()); + body_.batch_.clear(); + } + + fout.close(); +} + +} // namespace ripple diff --git a/src/ripple/net/impl/DatabaseDownloader.cpp b/src/ripple/net/impl/DatabaseDownloader.cpp new file mode 100644 index 00000000000..46f82d6a757 --- /dev/null +++ b/src/ripple/net/impl/DatabaseDownloader.cpp @@ -0,0 +1,88 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +namespace ripple +{ + +DatabaseDownloader::DatabaseDownloader( + boost::asio::io_service & io_service, + beast::Journal j, + Config const & config) + : SSLHTTPDownloader(io_service, j, config) + , config_(config) + , io_service_(io_service) +{ +} + +auto +DatabaseDownloader::getParser(boost::filesystem::path dstPath, + std::function complete, + boost::system::error_code & ec) -> std::shared_ptr +{ + using namespace boost::beast; + + auto p = std::make_shared>(); + p->body_limit(std::numeric_limits::max()); + p->get().body().open( + dstPath, + config_, + io_service_, + ec); + if(ec) + { + p->get().body().close(); + fail(dstPath, complete, ec, "open"); + } + + return p; +} + +bool +DatabaseDownloader::checkPath(boost::filesystem::path const & dstPath) +{ + return dstPath.string().size() <= MAX_PATH_LEN; +} + +void +DatabaseDownloader::closeBody(std::shared_ptr p) +{ + using namespace boost::beast; + + auto databaseBodyParser = std::dynamic_pointer_cast< + http::response_parser>(p); + assert(databaseBodyParser); + + databaseBodyParser->get().body().close(); +} + +uint64_t +DatabaseDownloader::size(std::shared_ptr p) +{ + using namespace boost::beast; + + auto databaseBodyParser = std::dynamic_pointer_cast< + http::response_parser>(p); + assert(databaseBodyParser); + + return databaseBodyParser->get().body().size(); +} + +} // ripple diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index 64f2941653f..bca8fefb5b7 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -186,7 +186,6 @@ class RPCParser ++i; else if (!boost::iequals(jvParams[--sz].asString(), "novalidate")) return rpcError(rpcINVALID_PARAMS); - jvResult[jss::validate] = false; } // Create the 'shards' array diff --git a/src/ripple/net/impl/SSLHTTPDownloader.cpp b/src/ripple/net/impl/SSLHTTPDownloader.cpp index bf8f9962b9b..3bf0622fab2 100644 --- a/src/ripple/net/impl/SSLHTTPDownloader.cpp +++ b/src/ripple/net/impl/SSLHTTPDownloader.cpp @@ -25,10 +25,13 @@ namespace ripple { SSLHTTPDownloader::SSLHTTPDownloader( boost::asio::io_service& io_service, beast::Journal j, - Config const& config) - : ssl_ctx_(config, j, boost::asio::ssl::context::tlsv12_client) + Config const& config, + bool isPaused) + : j_(j) + , ssl_ctx_(config, j, boost::asio::ssl::context::tlsv12_client) , strand_(io_service) - , j_(j) + , isStopped_(false) + , sessionActive_(false) { } @@ -41,21 +44,8 @@ SSLHTTPDownloader::download( boost::filesystem::path const& dstPath, std::function complete) { - try - { - if (exists(dstPath)) - { - JLOG(j_.error()) << - "Destination file exists"; - return false; - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << - "exception: " << e.what(); + if (!checkPath(dstPath)) return false; - } if (!strand_.running_in_this_thread()) strand_.post( @@ -84,6 +74,24 @@ SSLHTTPDownloader::download( return true; } +void +SSLHTTPDownloader::onStop() +{ + std::unique_lock lock(m_); + + isStopped_ = true; + + if(sessionActive_) + { + // Wait for the handler to exit. + c_.wait(lock, + [this]() + { + return !sessionActive_; + }); + } +} + void SSLHTTPDownloader::do_session( std::string const host, @@ -98,126 +106,231 @@ SSLHTTPDownloader::do_session( using namespace boost::beast; boost::system::error_code ec; - ip::tcp::resolver resolver {strand_.context()}; - auto const results = resolver.async_resolve(host, port, yield[ec]); - if (ec) - return fail(dstPath, complete, ec, "async_resolve"); + bool skip = false; - try - { - stream_.emplace(strand_.context(), ssl_ctx_.context()); - } - catch (std::exception const& e) + ////////////////////////////////////////////// + // Define lambdas for encapsulating download + // operations: + auto connect = [&](std::shared_ptr parser) { - return fail(dstPath, complete, ec, - std::string("exception: ") + e.what()); - } + uint64_t const rangeStart = size(parser); - ec = ssl_ctx_.preConnectVerify(*stream_, host); - if (ec) - return fail(dstPath, complete, ec, "preConnectVerify"); + ip::tcp::resolver resolver {strand_.context()}; + auto const results = resolver.async_resolve(host, port, yield[ec]); + if (ec) + return fail(dstPath, complete, ec, "async_resolve", parser); - boost::asio::async_connect( - stream_->next_layer(), results.begin(), results.end(), yield[ec]); - if (ec) - return fail(dstPath, complete, ec, "async_connect"); + try + { + stream_.emplace(strand_.context(), ssl_ctx_.context()); + } + catch (std::exception const& e) + { + return fail(dstPath, complete, ec, + std::string("exception: ") + e.what(), parser); + } - ec = ssl_ctx_.postConnectVerify(*stream_, host); - if (ec) - return fail(dstPath, complete, ec, "postConnectVerify"); + ec = ssl_ctx_.preConnectVerify(*stream_, host); + if (ec) + return fail(dstPath, complete, ec, "preConnectVerify", parser); - stream_->async_handshake(ssl::stream_base::client, yield[ec]); - if (ec) - return fail(dstPath, complete, ec, "async_handshake"); + boost::asio::async_connect( + stream_->next_layer(), results.begin(), results.end(), yield[ec]); + if (ec) + return fail(dstPath, complete, ec, "async_connect", parser); - // Set up an HTTP HEAD request message to find the file size - http::request req {http::verb::head, target, version}; - req.set(http::field::host, host); - req.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); + ec = ssl_ctx_.postConnectVerify(*stream_, host); + if (ec) + return fail(dstPath, complete, ec, "postConnectVerify", parser); - http::async_write(*stream_, req, yield[ec]); - if(ec) - return fail(dstPath, complete, ec, "async_write"); + stream_->async_handshake(ssl::stream_base::client, yield[ec]); + if (ec) + return fail(dstPath, complete, ec, "async_handshake", parser); - { - // Check if available storage for file size - http::response_parser p; - p.skip(true); - http::async_read(*stream_, read_buf_, p, yield[ec]); + // Set up an HTTP HEAD request message to find the file size + http::request req {http::verb::head, target, version}; + req.set(http::field::host, host); + req.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); + + // Requesting a portion of the file + if (rangeStart) + { + req.set(http::field::range, + (boost::format("bytes=%llu-") % rangeStart).str()); + } + + http::async_write(*stream_, req, yield[ec]); if(ec) - return fail(dstPath, complete, ec, "async_read"); - if (auto len = p.content_length()) + return fail(dstPath, complete, ec, "async_write", parser); + { - try + // Check if available storage for file size + http::response_parser p; + p.skip(true); + http::async_read(*stream_, read_buf_, p, yield[ec]); + if(ec) + return fail(dstPath, complete, ec, "async_read", parser); + + // Range request was rejected + if(p.get().result() == http::status::range_not_satisfiable) { - if (*len > space(dstPath.parent_path()).available) + req.erase(http::field::range); + + http::async_write(*stream_, req, yield[ec]); + if(ec) + return fail(dstPath, complete, ec, + "async_write_range_verify", parser); + + http::response_parser p; + p.skip(true); + + http::async_read(*stream_, read_buf_, p, yield[ec]); + if(ec) + return fail(dstPath, complete, ec, + "async_read_range_verify", parser); + + // The entire file is downloaded already. + if(p.content_length() == rangeStart) + skip = true; + else + return fail(dstPath, complete, ec, + "range_not_satisfiable", parser); + } + else if (rangeStart && + p.get().result() != http::status::partial_content) + { + ec.assign(boost::system::errc::not_supported, + boost::system::generic_category()); + + return fail(dstPath, complete, ec, + "Range request ignored", parser); + } + else if (auto len = p.content_length()) + { + try + { + if (*len > space(dstPath.parent_path()).available) + { + return fail(dstPath, complete, ec, + "Insufficient disk space for download", parser); + } + } + catch (std::exception const& e) { return fail(dstPath, complete, ec, - "Insufficient disk space for download"); + std::string("exception: ") + e.what(), parser); } } - catch (std::exception const& e) + } + + if(!skip) + { + // Set up an HTTP GET request message to download the file + req.method(http::verb::get); + + if (rangeStart) { - return fail(dstPath, complete, ec, - std::string("exception: ") + e.what()); + req.set(http::field::range, + (boost::format("bytes=%llu-") % rangeStart).str()); } } - } - // Set up an HTTP GET request message to download the file - req.method(http::verb::get); - http::async_write(*stream_, req, yield[ec]); - if(ec) - return fail(dstPath, complete, ec, "async_write"); + http::async_write(*stream_, req, yield[ec]); + if(ec) + return fail(dstPath, complete, ec, "async_write", parser); + + return true; + }; - // Download the file - http::response_parser p; - p.body_limit(std::numeric_limits::max()); - p.get().body().open( - dstPath.string().c_str(), - boost::beast::file_mode::write, - ec); - if (ec) + auto close = [&](auto p) { - p.get().body().close(); - return fail(dstPath, complete, ec, "open"); - } + closeBody(p); - http::async_read(*stream_, read_buf_, p, yield[ec]); - if (ec) + // Gracefully close the stream + stream_->async_shutdown(yield[ec]); + if (ec == boost::asio::error::eof) + ec.assign(0, ec.category()); + if (ec) + { + // Most web servers don't bother with performing + // the SSL shutdown handshake, for speed. + JLOG(j_.trace()) << + "async_shutdown: " << ec.message(); + } + // The socket cannot be reused + stream_ = boost::none; + }; + + auto getParser = [&] { - p.get().body().close(); - return fail(dstPath, complete, ec, "async_read"); + auto p = this->getParser(dstPath, complete, ec); + if (ec) + fail(dstPath, complete, ec, "getParser", p); + + return p; + }; + + // When the downloader is being stopped + // because the server is shutting down, + // this method notifies a 'Stoppable' + // object that the session has ended. + auto exit = [this]() + { + std::lock_guard lock(m_); + sessionActive_ = false; + c_.notify_one(); + }; + + // end lambdas + //////////////////////////////////////////////////////////// + + { + std::lock_guard lock(m_); + sessionActive_ = true; } - p.get().body().close(); - // Gracefully close the stream - stream_->async_shutdown(yield[ec]); - if (ec == boost::asio::error::eof) - ec.assign(0, ec.category()); + if(isStopped_.load()) + return exit(); + + auto p = getParser(); if (ec) + return exit(); + + if (!connect(p) || ec) + return exit(); + + if(skip) + p->skip(true); + + // Download the file + while (!p->is_done()) { - // Most web servers don't bother with performing - // the SSL shutdown handshake, for speed. - JLOG(j_.trace()) << - "async_shutdown: " << ec.message(); + if(isStopped_.load()) + { + close(p); + return exit(); + } + + http::async_read_some(*stream_, read_buf_, *p, yield[ec]); } - // The socket cannot be reused - stream_ = boost::none; JLOG(j_.trace()) << "download completed: " << dstPath.string(); + close(p); + exit(); + // Notify the completion handler complete(std::move(dstPath)); } -void +bool SSLHTTPDownloader::fail( boost::filesystem::path dstPath, std::function const& complete, boost::system::error_code const& ec, - std::string const& errMsg) + std::string const& errMsg, + std::shared_ptr parser) { if (!ec) { @@ -230,18 +343,21 @@ SSLHTTPDownloader::fail( errMsg << ": " << ec.message(); } + if (parser) + closeBody(parser); + try { remove(dstPath); } catch (std::exception const& e) { - JLOG(j_.error()) << - "exception: " << e.what(); + JLOG(j_.error()) << "exception: " << e.what() + << " in function: " << __func__; } complete(std::move(dstPath)); -} - + return false; +} }// ripple diff --git a/src/ripple/net/uml/interrupt_sequence.pu b/src/ripple/net/uml/interrupt_sequence.pu new file mode 100644 index 00000000000..ba046d084f8 --- /dev/null +++ b/src/ripple/net/uml/interrupt_sequence.pu @@ -0,0 +1,233 @@ +@startuml + + +skinparam shadowing false + +/' +skinparam sequence { + ArrowColor #e1e4e8 + ActorBorderColor #e1e4e8 + DatabaseBorderColor #e1e4e8 + LifeLineBorderColor Black + LifeLineBackgroundColor #d3d6d9 + + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + + NoteBackgroundColor #6a737d + + ActorBackgroundColor #f6f8fa + ActorFontColor #6a737d + ActorFontSize 17 + ActorFontName Aapex + + EntityBackgroundColor #f6f8fa + EntityFontColor #6a737d + EntityFontSize 17 + EntityFontName Aapex + + DatabaseBackgroundColor #f6f8fa + DatabaseFontColor #6a737d + DatabaseFontSize 17 + DatabaseFontName Aapex + + CollectionsBackgroundColor #f6f8fa + ActorFontColor #6a737d + ActorFontSize 17 + ActorFontName Aapex +} + +skinparam note { + BackgroundColor #fafbfc + BorderColor #e1e4e8 +} +'/ + +'skinparam monochrome true + +actor Client as c +entity RippleNode as rn +entity ShardArchiveHandler as sa +entity SSLHTTPDownloader as d +database Database as db +collections Fileserver as s + +c -> rn: Launch RippleNode +activate rn + +c -> rn: Issue download request + +note right of c + **Download Request:** + + { + "method": "download_shard", + "params": + [ + { + "shards": + [ + {"index": 1, "url": "https://example.com/1.tar.lz4"}, + {"index": 2, "url": "https://example.com/2.tar.lz4"}, + {"index": 5, "url": "https://example.com/5.tar.lz4"} + ] + } + ] + } +end note + +rn -> sa: Create instance of Handler +activate sa + +rn -> sa: Add three downloads +sa -> sa: Validate requested downloads + +rn -> sa: Initiate Downloads +sa -> rn: ACK: Initiating +rn -> c: Initiating requested downloads + +sa -> db: Save state to the database\n(Processing three downloads) + +note right of db + + **ArchiveHandler State (SQLite Table):** + + | Index | URL | + | 1 | https://example.com/1.tar.lz4 | + | 2 | https://example.com/2.tar.lz4 | + | 5 | https://example.com/5.tar.lz4 | + +end note + +sa -> d: Create instance of Downloader +activate d + +group Download 1 + + note over sa + **Download 1:** + + This encapsulates the download of the first file + at URL "https://example.com/1.tar.lz4". + + end note + + sa -> d: Start download + + d -> s: Connect and request file + s -> d: Send file + d -> sa: Invoke completion handler + +end + +sa -> sa: Import and validate shard + +sa -> db: Update persisted state\n(Remove download) + +note right of db + **ArchiveHandler State:** + + | Index | URL | + | 2 | https://example.com/2.tar.lz4 | + | 5 | https://example.com/5.tar.lz4 | + +end note + +group Download 2 + + sa -> d: Start download + + d -> s: Connect and request file + +end + +rn -> rn: **RippleNode crashes** + +deactivate sa +deactivate rn +deactivate d + +c -> rn: Restart RippleNode +activate rn + +rn -> db: Detect non-empty state database + +rn -> sa: Create instance of Handler +activate sa + +sa -> db: Load state + +note right of db + **ArchiveHandler State:** + + | Index | URL | + | 2 | https://example.com/2.tar.lz4 | + | 5 | https://example.com/5.tar.lz4 | + +end note + +sa -> d: Create instance of Downloader +activate d + +sa -> sa: Resume Download 2 + +group Download 2 + + sa -> d: Start download + + d -> s: Connect and request file + s -> d: Send file + d -> sa: Invoke completion handler + +end + +sa -> sa: Import and validate shard + +sa -> db: Update persisted state \n(Remove download) + +note right of db + **ArchiveHandler State:** + + | Index | URL | + | 5 | https://example.com/5.tar.lz4 | + +end note + +group Download 3 + + sa -> d: Start download + + d -> s: Connect and request file + s -> d: Send file + d -> sa: Invoke completion handler + +end + +sa -> sa: Import and validate shard + +sa -> db: Update persisted state \n(Remove download) + +note right of db + **ArchiveHandler State:** + + ***empty*** + +end note + +sa -> db: Remove empty database + +sa -> sa: Automatically destroyed +deactivate sa + +d -> d: Destroyed via reference\ncounting +deactivate d + +c -> rn: Poll RippleNode to verify successfull\nimport of all requested shards. +c -> rn: Shutdown RippleNode + +deactivate rn + +@enduml diff --git a/src/ripple/net/uml/states.pu b/src/ripple/net/uml/states.pu new file mode 100644 index 00000000000..b5db8ee48f4 --- /dev/null +++ b/src/ripple/net/uml/states.pu @@ -0,0 +1,69 @@ +@startuml + +state "Updating Database" as UD4 { + UD4: Update the database to reflect + UD4: the current state. +} +state "Initiating Download" as ID { + ID: Omit the range header to download + ID: the entire file. +} + +state "Evaluate Database" as ED { + ED: Determine the current state + ED: based on the contents of the + ED: database from a previous run. +} + +state "Remove Database" as RD { + RD: The database is destroyed when + RD: empty. +} + +state "Download in Progress" as DP + +state "Download Completed" as DC { + + state "Updating Database" as UD { + UD: Update the database to reflect + UD: the current state. + } + + state "Queue Check" as QC { + QC: Check the queue for any reamining + QC: downloads. + } + + [*] --> UD + UD --> QC +} + +state "Check Resume" as CR { + CR: Determine whether we're resuming + CR: a previous download or starting a + CR: new one. +} + +state "Resuming Download" as IPD { + IPD: Set the range header in the + IPD: HTTP request as needed. +} + +[*] --> ED : State DB is present at\nnode launch +ED --> RD : State DB is empty +ED --> CR : There are downloads queued +RD --> [*] + +[*] --> UD4 : Client invokes <>\ncommand +UD4 --> ID : Database updated +ID --> DP : Download started +DP --> DC : Download completed +DC --> ID : There **are** additional downloads\nqueued +DP --> [*] : A graceful shutdown is\nin progress +DC --> RD : There **are no** additional\ndownloads queued + +CR --> IPD : Resuming an interrupted\ndownload +IPD --> DP: Download started +CR --> ID : Initiating a new\ndownload + +@enduml diff --git a/src/ripple/nodestore/Database.h b/src/ripple/nodestore/Database.h index 1dde5f4df47..3d91d45a020 100644 --- a/src/ripple/nodestore/Database.h +++ b/src/ripple/nodestore/Database.h @@ -149,7 +149,7 @@ class Database : public Stoppable */ virtual bool - copyLedger(std::shared_ptr const& ledger) = 0; + storeLedger(std::shared_ptr const& srcLedger) = 0; /** Wait for all currently pending async reads to complete. */ @@ -211,12 +211,15 @@ class Database : public Stoppable void onStop() override; + void + onChildrenStopped() override; + /** @return The earliest ledger sequence allowed */ std::uint32_t - earliestSeq() const + earliestLedgerSeq() const { - return earliestSeq_; + return earliestLedgerSeq_; } protected: @@ -234,14 +237,17 @@ class Database : public Stoppable storeSz_ += sz; } + // Called by the public asyncFetch function void asyncFetch(uint256 const& hash, std::uint32_t seq, std::shared_ptr> const& pCache, std::shared_ptr> const& nCache); + // Called by the public fetch function std::shared_ptr - fetchInternal(uint256 const& hash, Backend& srcBackend); + fetchInternal(uint256 const& hash, std::shared_ptr backend); + // Called by the public import function void importInternal(Backend& dstBackend, Database& srcDB); @@ -250,11 +256,14 @@ class Database : public Stoppable TaggedCache& pCache, KeyCache& nCache, bool isAsync); + // Called by the public storeLedger function bool - copyLedger(Backend& dstBackend, Ledger const& srcLedger, - std::shared_ptr> const& pCache, - std::shared_ptr> const& nCache, - std::shared_ptr const& srcNext); + storeLedger( + Ledger const& srcLedger, + std::shared_ptr dstBackend, + std::shared_ptr> dstPCache, + std::shared_ptr> dstNCache, + std::shared_ptr next); private: std::atomic storeCount_ {0}; @@ -283,7 +292,7 @@ class Database : public Stoppable // The default is 32570 to match the XRP ledger network's earliest // allowed sequence. Alternate networks may set this value. - std::uint32_t const earliestSeq_; + std::uint32_t const earliestLedgerSeq_; virtual std::shared_ptr diff --git a/src/ripple/nodestore/DatabaseRotating.h b/src/ripple/nodestore/DatabaseRotating.h index 75606be187e..b44c6849c23 100644 --- a/src/ripple/nodestore/DatabaseRotating.h +++ b/src/ripple/nodestore/DatabaseRotating.h @@ -50,12 +50,14 @@ class DatabaseRotating : public Database virtual std::mutex& peekMutex() const = 0; virtual - std::unique_ptr const& + std::shared_ptr const& getWritableBackend() const = 0; virtual - std::unique_ptr - rotateBackends(std::unique_ptr newBackend) = 0; + std::shared_ptr + rotateBackends( + std::shared_ptr newBackend, + std::lock_guard const&) = 0; }; } diff --git a/src/ripple/nodestore/DatabaseShard.h b/src/ripple/nodestore/DatabaseShard.h index a6bd7dd283f..0e86664ca8c 100644 --- a/src/ripple/nodestore/DatabaseShard.h +++ b/src/ripple/nodestore/DatabaseShard.h @@ -109,14 +109,14 @@ class DatabaseShard : public Database @param shardIndex Shard index to import @param srcDir The directory to import from - @param validate If true validate shard ledger data @return true If the shard was successfully imported @implNote if successful, srcDir is moved to the database directory */ virtual bool - importShard(std::uint32_t shardIndex, - boost::filesystem::path const& srcDir, bool validate) = 0; + importShard( + std::uint32_t shardIndex, + boost::filesystem::path const& srcDir) = 0; /** Fetch a ledger from the shard store @@ -137,15 +137,6 @@ class DatabaseShard : public Database void setStored(std::shared_ptr const& ledger) = 0; - /** Query if a ledger with the given sequence is stored - - @param seq The ledger sequence to check if stored - @return `true` if the ledger is stored - */ - virtual - bool - contains(std::uint32_t seq) = 0; - /** Query which complete shards are stored @return the indexes of complete shards diff --git a/src/ripple/nodestore/NodeObject.h b/src/ripple/nodestore/NodeObject.h index caacaae3f96..90438deb3e4 100644 --- a/src/ripple/nodestore/NodeObject.h +++ b/src/ripple/nodestore/NodeObject.h @@ -95,9 +95,9 @@ class NodeObject : public CountedObject Blob const& getData () const; private: - NodeObjectType mType; - uint256 mHash; - Blob mData; + NodeObjectType const mType; + uint256 const mHash; + Blob const mData; }; } diff --git a/src/ripple/nodestore/impl/Database.cpp b/src/ripple/nodestore/impl/Database.cpp index f6a3c3785b0..37f58a21204 100644 --- a/src/ripple/nodestore/impl/Database.cpp +++ b/src/ripple/nodestore/impl/Database.cpp @@ -36,12 +36,12 @@ Database::Database( : Stoppable(name, parent.getRoot()) , j_(journal) , scheduler_(scheduler) - , earliestSeq_(get( + , earliestLedgerSeq_(get( config, "earliest_seq", XRP_LEDGER_EARLIEST_SEQ)) { - if (earliestSeq_ < 1) + if (earliestLedgerSeq_ < 1) Throw("Invalid earliest_seq"); while (readThreads-- > 0) @@ -83,6 +83,11 @@ Database::onStop() // After stop time we can no longer use the JobQueue for background // reads. Join the background read threads. stopThreads(); +} + +void +Database::onChildrenStopped() +{ stopped(); } @@ -115,13 +120,13 @@ Database::asyncFetch(uint256 const& hash, std::uint32_t seq, } std::shared_ptr -Database::fetchInternal(uint256 const& hash, Backend& srcBackend) +Database::fetchInternal(uint256 const& hash, std::shared_ptr backend) { std::shared_ptr nObj; Status status; try { - status = srcBackend.fetch(hash.begin(), &nObj); + status = backend->fetch(hash.begin(), &nObj); } catch (std::exception const& e) { @@ -211,7 +216,7 @@ Database::doFetch(uint256 const& hash, std::uint32_t seq, else { // Ensure all threads get the same object - pCache.canonicalize(hash, nObj); + pCache.canonicalize_replace_client(hash, nObj); // Since this was a 'hard' fetch, we will log it. JLOG(j_.trace()) << @@ -226,12 +231,14 @@ Database::doFetch(uint256 const& hash, std::uint32_t seq, } bool -Database::copyLedger(Backend& dstBackend, Ledger const& srcLedger, - std::shared_ptr> const& pCache, - std::shared_ptr> const& nCache, - std::shared_ptr const& srcNext) +Database::storeLedger( + Ledger const& srcLedger, + std::shared_ptr dstBackend, + std::shared_ptr> dstPCache, + std::shared_ptr> dstNCache, + std::shared_ptr next) { - assert(static_cast(pCache) == static_cast(nCache)); + assert(static_cast(dstPCache) == static_cast(dstNCache)); if (srcLedger.info().hash.isZero() || srcLedger.info().accountHash.isZero()) { @@ -254,48 +261,42 @@ Database::copyLedger(Backend& dstBackend, Ledger const& srcLedger, Batch batch; batch.reserve(batchWritePreallocationSize); auto storeBatch = [&]() { -#if RIPPLE_VERIFY_NODEOBJECT_KEYS - for (auto& nObj : batch) + if (dstPCache && dstNCache) { - assert(nObj->getHash() == - sha512Hash(makeSlice(nObj->getData()))); - if (pCache && nCache) - { - pCache->canonicalize(nObj->getHash(), nObj, true); - nCache->erase(nObj->getHash()); - storeStats(nObj->getData().size()); - } - } -#else - if (pCache && nCache) for (auto& nObj : batch) { - pCache->canonicalize(nObj->getHash(), nObj, true); - nCache->erase(nObj->getHash()); + dstPCache->canonicalize_replace_cache(nObj->getHash(), nObj); + dstNCache->erase(nObj->getHash()); storeStats(nObj->getData().size()); } -#endif - dstBackend.storeBatch(batch); + } + dstBackend->storeBatch(batch); batch.clear(); batch.reserve(batchWritePreallocationSize); }; bool error = false; - auto f = [&](SHAMapAbstractNode& node) { + auto visit = [&](SHAMapAbstractNode& node) + { if (auto nObj = srcDB.fetch( node.getNodeHash().as_uint256(), srcLedger.info().seq)) { batch.emplace_back(std::move(nObj)); - if (batch.size() >= batchWritePreallocationSize) - storeBatch(); + if (batch.size() < batchWritePreallocationSize) + return true; + + storeBatch(); + + if (!isStopping()) + return true; } - else - error = true; - return !error; + + error = true; + return false; }; // Store ledger header { - Serializer s(1024); + Serializer s(sizeof(std::uint32_t) + sizeof(LedgerInfo)); s.add32(HashPrefix::ledgerMaster); addRaw(srcLedger.info(), s); auto nObj = NodeObject::createObject(hotLEDGER, @@ -313,14 +314,14 @@ Database::copyLedger(Backend& dstBackend, Ledger const& srcLedger, " state map invalid"; return false; } - if (srcNext && srcNext->info().parentHash == srcLedger.info().hash) + if (next && next->info().parentHash == srcLedger.info().hash) { - auto have = srcNext->stateMap().snapShot(false); + auto have = next->stateMap().snapShot(false); srcLedger.stateMap().snapShot( - false)->visitDifferences(&(*have), f); + false)->visitDifferences(&(*have), visit); } else - srcLedger.stateMap().snapShot(false)->visitNodes(f); + srcLedger.stateMap().snapShot(false)->visitNodes(visit); if (error) return false; } @@ -335,7 +336,7 @@ Database::copyLedger(Backend& dstBackend, Ledger const& srcLedger, " transaction map invalid"; return false; } - srcLedger.txMap().snapShot(false)->visitNodes(f); + srcLedger.txMap().snapShot(false)->visitNodes(visit); if (error) return false; } diff --git a/src/ripple/nodestore/impl/DatabaseNodeImp.cpp b/src/ripple/nodestore/impl/DatabaseNodeImp.cpp index 6c3c3e3a545..e0879a617cb 100644 --- a/src/ripple/nodestore/impl/DatabaseNodeImp.cpp +++ b/src/ripple/nodestore/impl/DatabaseNodeImp.cpp @@ -28,11 +28,8 @@ void DatabaseNodeImp::store(NodeObjectType type, Blob&& data, uint256 const& hash, std::uint32_t seq) { -#if RIPPLE_VERIFY_NODEOBJECT_KEYS - assert(hash == sha512Hash(makeSlice(data))); -#endif auto nObj = NodeObject::createObject(type, std::move(data), hash); - pCache_->canonicalize(hash, nObj, true); + pCache_->canonicalize_replace_cache(hash, nObj); backend_->store(nObj); nCache_->erase(hash); storeStats(nObj->getData().size()); diff --git a/src/ripple/nodestore/impl/DatabaseNodeImp.h b/src/ripple/nodestore/impl/DatabaseNodeImp.h index 4e62f9ace43..7543434e227 100644 --- a/src/ripple/nodestore/impl/DatabaseNodeImp.h +++ b/src/ripple/nodestore/impl/DatabaseNodeImp.h @@ -38,7 +38,7 @@ class DatabaseNodeImp : public Database Scheduler& scheduler, int readThreads, Stoppable& parent, - std::unique_ptr backend, + std::shared_ptr backend, Section const& config, beast::Journal j) : Database(name, parent, scheduler, readThreads, config, j) @@ -91,10 +91,10 @@ class DatabaseNodeImp : public Database std::shared_ptr& object) override; bool - copyLedger(std::shared_ptr const& ledger) override + storeLedger(std::shared_ptr const& srcLedger) override { - return Database::copyLedger( - *backend_, *ledger, pCache_, nCache_, nullptr); + return Database::storeLedger( + *srcLedger, backend_, pCache_, nCache_, nullptr); } int @@ -123,12 +123,12 @@ class DatabaseNodeImp : public Database std::shared_ptr> nCache_; // Persistent key/value storage - std::unique_ptr backend_; + std::shared_ptr backend_; std::shared_ptr fetchFrom(uint256 const& hash, std::uint32_t seq) override { - return fetchInternal(hash, *backend_); + return fetchInternal(hash, backend_); } void diff --git a/src/ripple/nodestore/impl/DatabaseRotatingImp.cpp b/src/ripple/nodestore/impl/DatabaseRotatingImp.cpp index 76b2b4ec59d..00a9365cc73 100644 --- a/src/ripple/nodestore/impl/DatabaseRotatingImp.cpp +++ b/src/ripple/nodestore/impl/DatabaseRotatingImp.cpp @@ -29,8 +29,8 @@ DatabaseRotatingImp::DatabaseRotatingImp( Scheduler& scheduler, int readThreads, Stoppable& parent, - std::unique_ptr writableBackend, - std::unique_ptr archiveBackend, + std::shared_ptr writableBackend, + std::shared_ptr archiveBackend, Section const& config, beast::Journal j) : DatabaseRotating(name, parent, scheduler, readThreads, config, j) @@ -48,10 +48,10 @@ DatabaseRotatingImp::DatabaseRotatingImp( setParent(parent); } -// Make sure to call it already locked! -std::unique_ptr +std::shared_ptr DatabaseRotatingImp::rotateBackends( - std::unique_ptr newBackend) + std::shared_ptr newBackend, + std::lock_guard const&) { auto oldBackend {std::move(archiveBackend_)}; archiveBackend_ = std::move(writableBackend_); @@ -63,11 +63,8 @@ void DatabaseRotatingImp::store(NodeObjectType type, Blob&& data, uint256 const& hash, std::uint32_t seq) { -#if RIPPLE_VERIFY_NODEOBJECT_KEYS - assert(hash == sha512Hash(makeSlice(data))); -#endif auto nObj = NodeObject::createObject(type, std::move(data), hash); - pCache_->canonicalize(hash, nObj, true); + pCache_->canonicalize_replace_cache(hash, nObj); getWritableBackend()->store(nObj); nCache_->erase(hash); storeStats(nObj->getData().size()); @@ -106,10 +103,10 @@ std::shared_ptr DatabaseRotatingImp::fetchFrom(uint256 const& hash, std::uint32_t seq) { Backends b = getBackends(); - auto nObj = fetchInternal(hash, *b.writableBackend); + auto nObj = fetchInternal(hash, b.writableBackend); if (! nObj) { - nObj = fetchInternal(hash, *b.archiveBackend); + nObj = fetchInternal(hash, b.archiveBackend); if (nObj) { getWritableBackend()->store(nObj); diff --git a/src/ripple/nodestore/impl/DatabaseRotatingImp.h b/src/ripple/nodestore/impl/DatabaseRotatingImp.h index e925de5d89d..4cdf6396f87 100644 --- a/src/ripple/nodestore/impl/DatabaseRotatingImp.h +++ b/src/ripple/nodestore/impl/DatabaseRotatingImp.h @@ -37,8 +37,8 @@ class DatabaseRotatingImp : public DatabaseRotating Scheduler& scheduler, int readThreads, Stoppable& parent, - std::unique_ptr writableBackend, - std::unique_ptr archiveBackend, + std::shared_ptr writableBackend, + std::shared_ptr archiveBackend, Section const& config, beast::Journal j); @@ -48,15 +48,17 @@ class DatabaseRotatingImp : public DatabaseRotating stopThreads(); } - std::unique_ptr const& + std::shared_ptr const& getWritableBackend() const override { std::lock_guard lock (rotateMutex_); return writableBackend_; } - std::unique_ptr - rotateBackends(std::unique_ptr newBackend) override; + std::shared_ptr + rotateBackends( + std::shared_ptr newBackend, + std::lock_guard const&) override; std::mutex& peekMutex() const override { @@ -92,10 +94,10 @@ class DatabaseRotatingImp : public DatabaseRotating std::shared_ptr& object) override; bool - copyLedger(std::shared_ptr const& ledger) override + storeLedger(std::shared_ptr const& srcLedger) override { - return Database::copyLedger( - *getWritableBackend(), *ledger, pCache_, nCache_, nullptr); + return Database::storeLedger( + *srcLedger, getWritableBackend(), pCache_, nCache_, nullptr); } int @@ -126,13 +128,13 @@ class DatabaseRotatingImp : public DatabaseRotating // Negative cache std::shared_ptr> nCache_; - std::unique_ptr writableBackend_; - std::unique_ptr archiveBackend_; + std::shared_ptr writableBackend_; + std::shared_ptr archiveBackend_; mutable std::mutex rotateMutex_; struct Backends { - std::unique_ptr const& writableBackend; - std::unique_ptr const& archiveBackend; + std::shared_ptr const& writableBackend; + std::shared_ptr const& archiveBackend; }; Backends getBackends() const diff --git a/src/ripple/nodestore/impl/DatabaseShardImp.cpp b/src/ripple/nodestore/impl/DatabaseShardImp.cpp index 0d28c9cb40e..eaeabc19cef 100644 --- a/src/ripple/nodestore/impl/DatabaseShardImp.cpp +++ b/src/ripple/nodestore/impl/DatabaseShardImp.cpp @@ -20,12 +20,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include #include @@ -50,209 +50,174 @@ DatabaseShardImp::DatabaseShardImp( app.config().section(ConfigSection::shardDatabase()), j) , app_(app) - , earliestShardIndex_(seqToShardIndex(earliestSeq())) + , parent_(parent) + , taskQueue_(std::make_unique(*this)) + , earliestShardIndex_(seqToShardIndex(earliestLedgerSeq())) , avgShardFileSz_(ledgersPerShard_ * kilobytes(192)) { } - DatabaseShardImp::~DatabaseShardImp() { - // Stop threads before data members are destroyed - stopThreads(); - - // Close backend databases before destroying the context - std::lock_guard lock(m_); - complete_.clear(); - if (incomplete_) - incomplete_.reset(); - preShards_.clear(); - ctx_.reset(); + onStop(); } bool DatabaseShardImp::init() { - using namespace boost::filesystem; - - std::lock_guard lock(m_); - auto fail = [j = j_](std::string const& msg) { - JLOG(j.error()) << - "[" << ConfigSection::shardDatabase() << "] " << msg; - return false; - }; - - if (init_) - return fail("already initialized"); + std::lock_guard lock(mutex_); + if (init_) + { + JLOG(j_.error()) << "already initialized"; + return false; + } - Config const& config {app_.config()}; - Section const& section {config.section(ConfigSection::shardDatabase())}; - if (section.empty()) - return fail("missing configuration"); + if (!initConfig(lock)) + { + JLOG(j_.error()) << "invalid configuration file settings"; + return false; + } - { - // Node and shard stores must use same earliest ledger sequence - std::uint32_t seq; - if (get_if_exists( - config.section(ConfigSection::nodeDatabase()), - "earliest_seq", - seq)) + try { - std::uint32_t seq2; - if (get_if_exists(section, "earliest_seq", seq2) && - seq != seq2) + using namespace boost::filesystem; + if (exists(dir_)) { - return fail("and [" + ConfigSection::shardDatabase() + - "] both define 'earliest_seq'"); + if (!is_directory(dir_)) + { + JLOG(j_.error()) << "'path' must be a directory"; + return false; + } } - } - } - - if (!get_if_exists(section, "path", dir_)) - return fail("'path' missing"); + else + create_directories(dir_); - if (boost::filesystem::exists(dir_)) - { - if (!boost::filesystem::is_directory(dir_)) - return fail("'path' must be a directory"); - } - else - boost::filesystem::create_directories(dir_); + ctx_ = std::make_unique(); + ctx_->start(); - { - std::uint64_t sz; - if (!get_if_exists(section, "max_size_gb", sz)) - return fail("'max_size_gb' missing"); + // Find shards + for (auto const& d : directory_iterator(dir_)) + { + if (!is_directory(d)) + continue; - if ((sz << 30) < sz) - return fail("'max_size_gb' overflow"); + // Check shard directory name is numeric + auto dirName = d.path().stem().string(); + if (!std::all_of( + dirName.begin(), + dirName.end(), + [](auto c) { + return ::isdigit(static_cast(c)); + })) + { + continue; + } - // Minimum storage space required (in gigabytes) - if (sz < 10) - return fail("'max_size_gb' must be at least 10"); + auto const shardIndex {std::stoul(dirName)}; + if (shardIndex < earliestShardIndex()) + { + JLOG(j_.error()) << + "shard " << shardIndex << + " comes before earliest shard index " << + earliestShardIndex(); + return false; + } - // Convert to bytes - maxFileSz_ = sz << 30; - } + auto const shardDir {dir_ / std::to_string(shardIndex)}; - if (section.exists("ledgers_per_shard")) - { - // To be set only in standalone for testing - if (!config.standalone()) - return fail("'ledgers_per_shard' only honored in stand alone"); + // Check if a previous import failed + if (is_regular_file(shardDir / importMarker_)) + { + JLOG(j_.warn()) << + "shard " << shardIndex << + " previously failed import, removing"; + remove_all(shardDir); + continue; + } - ledgersPerShard_ = get(section, "ledgers_per_shard"); - if (ledgersPerShard_ == 0 || ledgersPerShard_ % 256 != 0) - return fail("'ledgers_per_shard' must be a multiple of 256"); - } + auto shard {std::make_unique( + app_, + *this, + shardIndex, + j_)}; + if (!shard->open(scheduler_, *ctx_)) + { + if (!shard->isLegacy()) + return false; + + // Remove legacy shard + JLOG(j_.warn()) << + "shard " << shardIndex << + " incompatible legacy shard, removing"; + remove_all(shardDir); + continue; + } - // NuDB is the default and only supported permanent storage backend - // "Memory" and "none" types are supported for tests - backendName_ = get(section, "type", "nudb"); - if (!boost::iequals(backendName_, "NuDB") && - !boost::iequals(backendName_, "Memory") && - !boost::iequals(backendName_, "none")) - { - return fail("'type' value unsupported"); - } + if (shard->isFinal()) + { + shards_.emplace( + shardIndex, + ShardInfo(std::move(shard), ShardInfo::State::final)); + } + else if (shard->isBackendComplete()) + { + auto const result {shards_.emplace( + shardIndex, + ShardInfo(std::move(shard), ShardInfo::State::none))}; + finalizeShard(result.first->second, true, lock); + } + else + { + if (acquireIndex_ != 0) + { + JLOG(j_.error()) << + "more than one shard being acquired"; + return false; + } - // Check if backend uses permanent storage - if (auto factory = Manager::instance().find(backendName_)) - { - auto backend {factory->createInstance( - NodeObject::keyBytes, section, scheduler_, j_)}; - backed_ = backend->backed(); - if (!backed_) - { - setFileStats(lock); - init_ = true; - return true; + shards_.emplace( + shardIndex, + ShardInfo(std::move(shard), ShardInfo::State::acquire)); + acquireIndex_ = shardIndex; + } + } } - } - else - return fail(backendName_ + " backend unsupported"); - - try - { - ctx_ = std::make_unique(); - ctx_->start(); - - // Find shards - for (auto const& d : directory_iterator(dir_)) + catch (std::exception const& e) { - if (!is_directory(d)) - continue; - - // Validate shard directory name is numeric - auto dirName = d.path().stem().string(); - if (!std::all_of( - dirName.begin(), - dirName.end(), - [](auto c) { - return ::isdigit(static_cast(c)); - })) - { - continue; - } - - auto const shardIndex {std::stoul(dirName)}; - if (shardIndex < earliestShardIndex()) - { - return fail("shard " + std::to_string(shardIndex) + - " comes before earliest shard index " + - std::to_string(earliestShardIndex())); - } - - // Check if a previous import failed - if (is_regular_file( - dir_ / std::to_string(shardIndex) / importMarker_)) - { - JLOG(j_.warn()) << - "shard " << shardIndex << - " previously failed import, removing"; - remove_all(dir_ / std::to_string(shardIndex)); - continue; - } - - auto shard {std::make_unique(app_, *this, shardIndex, j_)}; - if (!shard->open(scheduler_, *ctx_)) - return false; - - if (shard->complete()) - complete_.emplace(shard->index(), std::move(shard)); - else - { - if (incomplete_) - return fail("more than one control file found"); - incomplete_ = std::move(shard); - } + JLOG(j_.error()) << + "exception " << e.what() << " in function " << __func__; } - } - catch (std::exception const& e) - { - return fail(std::string("exception ") + - e.what() + " in function " + __func__); + + updateStatus(lock); + setParent(parent_); + init_ = true; } - setFileStats(lock); - updateStatus(lock); - init_ = true; + setFileStats(); return true; } boost::optional DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq) { - std::lock_guard lock(m_); - assert(init_); + boost::optional shardIndex; - if (incomplete_) - return incomplete_->prepare(); - if (!canAdd_) - return boost::none; - if (backed_) { + std::lock_guard lock(mutex_); + assert(init_); + + if (acquireIndex_ != 0) + { + if (auto it {shards_.find(acquireIndex_)}; it != shards_.end()) + return it->second.shard->prepare(); + assert(false); + return boost::none; + } + + if (!canAdd_) + return boost::none; + // Check available storage space if (fileSz_ + avgShardFileSz_ > maxFileSz_) { @@ -266,39 +231,45 @@ DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq) canAdd_ = false; return boost::none; } + + shardIndex = findAcquireIndex(validLedgerSeq, lock); } - auto const shardIndex {findShardIndexToAdd(validLedgerSeq, lock)}; if (!shardIndex) { JLOG(j_.debug()) << "no new shards to add"; - canAdd_ = false; + { + std::lock_guard lock(mutex_); + canAdd_ = false; + } return boost::none; } - // With every new shard, clear family caches - app_.shardFamily()->reset(); - incomplete_ = std::make_unique(app_, *this, *shardIndex, j_); - if (!incomplete_->open(scheduler_, *ctx_)) - { - incomplete_.reset(); + auto shard {std::make_unique(app_, *this, *shardIndex, j_)}; + if (!shard->open(scheduler_, *ctx_)) return boost::none; - } - return incomplete_->prepare(); + auto const seq {shard->prepare()}; + { + std::lock_guard lock(mutex_); + shards_.emplace( + *shardIndex, + ShardInfo(std::move(shard), ShardInfo::State::acquire)); + acquireIndex_ = *shardIndex; + } + return seq; } bool DatabaseShardImp::prepareShard(std::uint32_t shardIndex) { - std::lock_guard lock(m_); - assert(init_); - auto fail = [j = j_, shardIndex](std::string const& msg) { JLOG(j.error()) << "shard " << shardIndex << " " << msg; return false; }; + std::lock_guard lock(mutex_); + assert(init_); if (!canAdd_) return fail("cannot be stored at this time"); @@ -314,7 +285,7 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex) auto seqCheck = [&](std::uint32_t seq) { // seq will be greater than zero if valid - if (seq > earliestSeq() && shardIndex >= seqToShardIndex(seq)) + if (seq > earliestLedgerSeq() && shardIndex >= seqToShardIndex(seq)) return fail("has an invalid index"); return true; }; @@ -324,50 +295,35 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex) return false; } - if (complete_.find(shardIndex) != complete_.end()) - { - JLOG(j_.debug()) << "shard " << shardIndex << " is already stored"; - return false; - } - if (incomplete_ && incomplete_->index() == shardIndex) - { - JLOG(j_.debug()) << "shard " << shardIndex << " is being acquired"; - return false; - } - if (preShards_.find(shardIndex) != preShards_.end()) + if (shards_.find(shardIndex) != shards_.end()) { JLOG(j_.debug()) << - "shard " << shardIndex << " is already prepared for import"; + "shard " << shardIndex << + " is already stored or queued for import"; return false; } - // Check limit and space requirements - if (backed_) - { - std::uint64_t const sz { - (preShards_.size() + 1 + (incomplete_ ? 1 : 0)) * avgShardFileSz_}; - if (fileSz_ + sz > maxFileSz_) - { - JLOG(j_.debug()) << - "shard " << shardIndex << " exceeds the maximum storage size"; - return false; - } - if (sz > available()) - return fail("insufficient storage space available"); - } + // Check available storage space + if (fileSz_ + avgShardFileSz_ > maxFileSz_) + return fail("maximum storage size reached"); + if (avgShardFileSz_ > available()) + return fail("insufficient storage space available"); - // Add to shards prepared - preShards_.emplace(shardIndex, nullptr); + shards_.emplace(shardIndex, ShardInfo(nullptr, ShardInfo::State::import)); return true; } void DatabaseShardImp::removePreShard(std::uint32_t shardIndex) { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - preShards_.erase(shardIndex); + if (auto const it {shards_.find(shardIndex)}; + it != shards_.end() && it->second.state == ShardInfo::State::import) + { + shards_.erase(it); + } } std::string @@ -375,27 +331,32 @@ DatabaseShardImp::getPreShards() { RangeSet rs; { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - if (preShards_.empty()) - return {}; - for (auto const& ps : preShards_) - rs.insert(ps.first); + for (auto const& e : shards_) + if (e.second.state == ShardInfo::State::import) + rs.insert(e.first); } + + if (rs.empty()) + return {}; + return to_string(rs); }; bool -DatabaseShardImp::importShard(std::uint32_t shardIndex, - boost::filesystem::path const& srcDir, bool validate) +DatabaseShardImp::importShard( + std::uint32_t shardIndex, + boost::filesystem::path const& srcDir) { using namespace boost::filesystem; try { if (!is_directory(srcDir) || is_empty(srcDir)) { - JLOG(j_.error()) << "invalid source directory " << srcDir.string(); + JLOG(j_.error()) << + "invalid source directory " << srcDir.string(); return false; } } @@ -406,7 +367,7 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex, return false; } - auto move = [&](path const& src, path const& dst) + auto renameDir = [&](path const& src, path const& dst) { try { @@ -421,86 +382,88 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex, return true; }; - std::unique_lock lock(m_); - assert(init_); - - // Check shard is prepared - auto it {preShards_.find(shardIndex)}; - if(it == preShards_.end()) + path dstDir; { - JLOG(j_.error()) << "shard " << shardIndex << " is an invalid index"; - return false; + std::lock_guard lock(mutex_); + assert(init_); + + // Check shard is prepared + if (auto const it {shards_.find(shardIndex)}; + it == shards_.end() || + it->second.shard || + it->second.state != ShardInfo::State::import) + { + JLOG(j_.error()) << + "shard " << shardIndex << " failed to import"; + return false; + } + + dstDir = dir_ / std::to_string(shardIndex); } - // Move source directory to the shard database directory - auto const dstDir {dir_ / std::to_string(shardIndex)}; - if (!move(srcDir, dstDir)) + // Rename source directory to the shard database directory + if (!renameDir(srcDir, dstDir)) return false; // Create the new shard auto shard {std::make_unique(app_, *this, shardIndex, j_)}; - auto fail = [&](std::string const& msg) + if (!shard->open(scheduler_, *ctx_) || !shard->isBackendComplete()) { - if (!msg.empty()) - { - JLOG(j_.error()) << "shard " << shardIndex << " " << msg; - } + JLOG(j_.error()) << + "shard " << shardIndex << " failed to import"; shard.reset(); - move(dstDir, srcDir); + renameDir(dstDir, srcDir); return false; - }; - - if (!shard->open(scheduler_, *ctx_)) - return fail({}); - if (!shard->complete()) - return fail("is incomplete"); - - try - { - // Verify database integrity - shard->getBackend()->verify(); - } - catch (std::exception const& e) - { - return fail(std::string("exception ") + - e.what() + " in function " + __func__); } - // Validate shard ledgers - if (validate) + std::lock_guard lock(mutex_); + auto const it {shards_.find(shardIndex)}; + if (it == shards_.end() || + it->second.shard || + it->second.state != ShardInfo::State::import) { - // Shard validation requires releasing the lock - // so the database can fetch data from it - it->second = shard.get(); - lock.unlock(); - auto const valid {shard->validate()}; - lock.lock(); - if (!valid) - { - it = preShards_.find(shardIndex); - if(it != preShards_.end()) - it->second = nullptr; - return fail("failed validation"); - } + JLOG(j_.error()) << + "shard " << shardIndex << " failed to import"; + return false; } - // Add the shard - complete_.emplace(shardIndex, std::move(shard)); - preShards_.erase(shardIndex); - - std::lock_guard lockg(*lock.release(), std::adopt_lock); - setFileStats(lockg); - updateStatus(lockg); + it->second.shard = std::move(shard); + finalizeShard(it->second, true, lock); return true; } std::shared_ptr DatabaseShardImp::fetchLedger(uint256 const& hash, std::uint32_t seq) { - if (!contains(seq)) - return {}; + auto const shardIndex {seqToShardIndex(seq)}; + { + ShardInfo shardInfo; + { + std::lock_guard lock(mutex_); + assert(init_); + + if (auto const it {shards_.find(shardIndex)}; it != shards_.end()) + shardInfo = it->second; + else + return {}; + } + + // Check if the ledger is stored in a final shard + // or in the shard being acquired + switch (shardInfo.state) + { + case ShardInfo::State::final: + break; + case ShardInfo::State::acquire: + if (shardInfo.shard->containsLedger(seq)) + break; + [[fallthrough]]; + default: + return {}; + } + } - auto nObj = fetch(hash, seq); + auto nObj {fetch(hash, seq)}; if (!nObj) return {}; @@ -549,69 +512,63 @@ DatabaseShardImp::fetchLedger(uint256 const& hash, std::uint32_t seq) void DatabaseShardImp::setStored(std::shared_ptr const& ledger) { - auto const shardIndex {seqToShardIndex(ledger->info().seq)}; - auto fail = [j = j_, shardIndex](std::string const& msg) - { - JLOG(j.error()) << "shard " << shardIndex << " " << msg; - }; - if (ledger->info().hash.isZero()) { - return fail("encountered a zero ledger hash on sequence " + - std::to_string(ledger->info().seq)); + JLOG(j_.error()) << + "zero ledger hash for ledger sequence " << ledger->info().seq; + return; } if (ledger->info().accountHash.isZero()) { - return fail("encountered a zero account hash on sequence " + - std::to_string(ledger->info().seq)); + JLOG(j_.error()) << + "zero account hash for ledger sequence " << ledger->info().seq; + return; } - - std::lock_guard lock(m_); - assert(init_); - - if (!incomplete_ || shardIndex != incomplete_->index()) + if (ledger->stateMap().getHash().isNonZero() && + !ledger->stateMap().isValid()) { - return fail("ledger sequence " + std::to_string(ledger->info().seq) + - " is not being acquired"); - } - if (!incomplete_->setStored(ledger)) + JLOG(j_.error()) << + "invalid state map for ledger sequence " << ledger->info().seq; return; - if (incomplete_->complete()) + } + if (ledger->info().txHash.isNonZero() && !ledger->txMap().isValid()) { - complete_.emplace(incomplete_->index(), std::move(incomplete_)); - incomplete_.reset(); - updateStatus(lock); - - // Update peers with new shard index - protocol::TMPeerShardInfo message; - PublicKey const& publicKey {app_.nodeIdentity().first}; - message.set_nodepubkey(publicKey.data(), publicKey.size()); - message.set_shardindexes(std::to_string(shardIndex)); - app_.overlay().foreach(send_always( - std::make_shared(message, protocol::mtPEER_SHARD_INFO))); + JLOG(j_.error()) << + "invalid transaction map for ledger sequence " << + ledger->info().seq; + return; } - setFileStats(lock); -} + auto const shardIndex {seqToShardIndex(ledger->info().seq)}; + std::shared_ptr shard; + { + std::lock_guard lock(mutex_); + assert(init_); -bool -DatabaseShardImp::contains(std::uint32_t seq) -{ - auto const shardIndex {seqToShardIndex(seq)}; - std::lock_guard lock(m_); - assert(init_); + if (shardIndex != acquireIndex_) + { + JLOG(j_.trace()) << + "shard " << shardIndex << " is not being acquired"; + return; + } - if (complete_.find(shardIndex) != complete_.end()) - return true; - if (incomplete_ && incomplete_->index() == shardIndex) - return incomplete_->contains(seq); - return false; + if (auto const it {shards_.find(shardIndex)}; it != shards_.end()) + shard = it->second.shard; + else + { + JLOG(j_.error()) << + "shard " << shardIndex << " is not being acquired"; + return; + } + } + + storeLedgerInShard(shard, ledger); } std::string DatabaseShardImp::getCompleteShards() { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); return status_; @@ -620,36 +577,53 @@ DatabaseShardImp::getCompleteShards() void DatabaseShardImp::validate() { - std::vector> completeShards; + std::vector> shards; { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - if (complete_.empty()) - { - JLOG(j_.error()) << "no shards found to validate"; + // Only shards with a state of final should be validated + for (auto& e : shards_) + if (e.second.state == ShardInfo::State::final) + shards.push_back(e.second.shard); + + if (shards.empty()) return; - } JLOG(j_.debug()) << "Validating shards " << status_; - - completeShards.reserve(complete_.size()); - for (auto const& shard : complete_) - completeShards.push_back(shard.second); } - // Verify each complete stored shard - for (auto const& shard : completeShards) - shard->validate(); + for (auto const& e : shards) + { + if (auto shard {e.lock()}; shard) + shard->finalize(true); + } app_.shardFamily()->reset(); } +void +DatabaseShardImp::onStop() +{ + // Stop read threads in base before data members are destroyed + stopThreads(); + + std::lock_guard lock(mutex_); + if (shards_.empty()) + return; + + // Notify shards to stop + for (auto const& e : shards_) + if (e.second.shard) + e.second.shard->stop(); + shards_.clear(); +} + void DatabaseShardImp::import(Database& source) { { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); // Only the application local node store can be imported @@ -669,7 +643,7 @@ DatabaseShardImp::import(Database& source) std::shared_ptr ledger; std::uint32_t seq; std::tie(ledger, seq, std::ignore) = loadLedgerHelper( - "WHERE LedgerSeq >= " + std::to_string(earliestSeq()) + + "WHERE LedgerSeq >= " + std::to_string(earliestLedgerSeq()) + " order by LedgerSeq " + (ascendSort ? "asc" : "desc") + " limit 1", app_, false); if (!ledger || seq == 0) @@ -729,10 +703,11 @@ DatabaseShardImp::import(Database& source) } // Skip if already stored - if (complete_.find(shardIndex) != complete_.end() || - (incomplete_ && incomplete_->index() == shardIndex)) + if (shardIndex == acquireIndex_ || + shards_.find(shardIndex) != shards_.end()) { - JLOG(j_.debug()) << "shard " << shardIndex << " already exists"; + JLOG(j_.debug()) << + "shard " << shardIndex << " already exists"; continue; } @@ -743,7 +718,7 @@ DatabaseShardImp::import(Database& source) std::max(firstSeq, lastLedgerSeq(shardIndex))}; auto const numLedgers {shardIndex == earliestShardIndex() ? lastSeq - firstSeq + 1 : ledgersPerShard_}; - auto ledgerHashes{getHashesByIndex(firstSeq, lastSeq, app_)}; + auto ledgerHashes {getHashesByIndex(firstSeq, lastSeq, app_)}; if (ledgerHashes.size() != numLedgers) continue; @@ -768,126 +743,178 @@ DatabaseShardImp::import(Database& source) auto const shardDir {dir_ / std::to_string(shardIndex)}; auto shard {std::make_unique(app_, *this, shardIndex, j_)}; if (!shard->open(scheduler_, *ctx_)) - { - shard.reset(); continue; - } // Create a marker file to signify an import in progress auto const markerFile {shardDir / importMarker_}; - std::ofstream ofs {markerFile.string()}; - if (!ofs.is_open()) { - JLOG(j_.error()) << - "shard " << shardIndex << - " is unable to create temp marker file"; - shard.reset(); - removeAll(shardDir, j_); - continue; + std::ofstream ofs {markerFile.string()}; + if (!ofs.is_open()) + { + JLOG(j_.error()) << + "shard " << shardIndex << + " is unable to create temp marker file"; + remove_all(shardDir); + continue; + } + ofs.close(); } - ofs.close(); // Copy the ledgers from node store + std::shared_ptr recentStored; + boost::optional lastLedgerHash; + while (auto seq = shard->prepare()) { - auto ledger = loadByIndex(*seq, app_, false); - if (!ledger || ledger->info().seq != seq || - !Database::copyLedger(*shard->getBackend(), *ledger, - nullptr, nullptr, shard->lastStored())) + auto ledger {loadByIndex(*seq, app_, false)}; + if (!ledger || ledger->info().seq != seq) break; - if (!shard->setStored(ledger)) - break; - if (shard->complete()) + if (!Database::storeLedger( + *ledger, + shard->getBackend(), + nullptr, + nullptr, + recentStored)) { - JLOG(j_.debug()) << - "shard " << shardIndex << " was successfully imported"; - removeAll(markerFile, j_); break; } + + if (!shard->store(ledger)) + break; + + if (!lastLedgerHash && seq == lastLedgerSeq(shardIndex)) + lastLedgerHash = ledger->info().hash; + + recentStored = ledger; } - if (!shard->complete()) + using namespace boost::filesystem; + if (lastLedgerHash && shard->isBackendComplete()) + { + // Store shard final key + Serializer s; + s.add32(Shard::version); + s.add32(firstLedgerSeq(shardIndex)); + s.add32(lastLedgerSeq(shardIndex)); + s.add256(*lastLedgerHash); + auto nObj {NodeObject::createObject( + hotUNKNOWN, + std::move(s.modData()), + Shard::finalKey)}; + + try + { + shard->getBackend()->store(nObj); + + // The import process is complete and the + // marker file is no longer required + remove_all(markerFile); + + JLOG(j_.debug()) << + "shard " << shardIndex << + " was successfully imported"; + + auto const result {shards_.emplace( + shardIndex, + ShardInfo(std::move(shard), ShardInfo::State::none))}; + finalizeShard(result.first->second, true, lock); + } + catch (std::exception const& e) + { + JLOG(j_.error()) << + "exception " << e.what() << + " in function " << __func__; + remove_all(shardDir); + } + } + else { JLOG(j_.error()) << "shard " << shardIndex << " failed to import"; - shard.reset(); - removeAll(shardDir, j_); + remove_all(shardDir); } - else - setFileStats(lock); } - // Re initialize the shard store - init_ = false; - complete_.clear(); - incomplete_.reset(); + updateStatus(lock); } - if (!init()) - Throw("import: failed to initialize"); + setFileStats(); } std::int32_t DatabaseShardImp::getWriteLoad() const { - std::int32_t wl {0}; + std::shared_ptr shard; { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - for (auto const& e : complete_) - wl += e.second->getBackend()->getWriteLoad(); - if (incomplete_) - wl += incomplete_->getBackend()->getWriteLoad(); + if (auto const it {shards_.find(acquireIndex_)}; it != shards_.end()) + shard = it->second.shard; + else + return 0; } - return wl; + + return shard->getBackend()->getWriteLoad(); } void -DatabaseShardImp::store(NodeObjectType type, - Blob&& data, uint256 const& hash, std::uint32_t seq) +DatabaseShardImp::store( + NodeObjectType type, + Blob&& data, + uint256 const& hash, + std::uint32_t seq) { -#if RIPPLE_VERIFY_NODEOBJECT_KEYS - assert(hash == sha512Hash(makeSlice(data))); -#endif - std::shared_ptr nObj; auto const shardIndex {seqToShardIndex(seq)}; + std::shared_ptr shard; { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - if (!incomplete_ || shardIndex != incomplete_->index()) + if (shardIndex != acquireIndex_) + { + JLOG(j_.trace()) << + "shard " << shardIndex << " is not being acquired"; + return; + } + + if (auto const it {shards_.find(shardIndex)}; it != shards_.end()) + shard = it->second.shard; + else { - JLOG(j_.warn()) << - "shard " << shardIndex << - " ledger sequence " << seq << - " is not being acquired"; + JLOG(j_.error()) << + "shard " << shardIndex << " is not being acquired"; return; } - nObj = NodeObject::createObject( - type, std::move(data), hash); - incomplete_->pCache()->canonicalize(hash, nObj, true); - incomplete_->getBackend()->store(nObj); - incomplete_->nCache()->erase(hash); } + + auto [backend, pCache, nCache] = shard->getBackendAll(); + auto nObj {NodeObject::createObject(type, std::move(data), hash)}; + + pCache->canonicalize_replace_cache(hash, nObj); + backend->store(nObj); + nCache->erase(hash); + storeStats(nObj->getData().size()); } std::shared_ptr DatabaseShardImp::fetch(uint256 const& hash, std::uint32_t seq) { - auto cache {selectCache(seq)}; + auto cache {getCache(seq)}; if (cache.first) return doFetch(hash, seq, *cache.first, *cache.second, false); return {}; } bool -DatabaseShardImp::asyncFetch(uint256 const& hash, - std::uint32_t seq, std::shared_ptr& object) +DatabaseShardImp::asyncFetch( + uint256 const& hash, + std::uint32_t seq, + std::shared_ptr& object) { - auto cache {selectCache(seq)}; + auto cache {getCache(seq)}; if (cache.first) { // See if the object is in cache @@ -901,125 +928,227 @@ DatabaseShardImp::asyncFetch(uint256 const& hash, } bool -DatabaseShardImp::copyLedger(std::shared_ptr const& ledger) +DatabaseShardImp::storeLedger(std::shared_ptr const& srcLedger) { - auto const shardIndex {seqToShardIndex(ledger->info().seq)}; - std::lock_guard lock(m_); - assert(init_); - - if (!incomplete_ || shardIndex != incomplete_->index()) + auto const seq {srcLedger->info().seq}; + auto const shardIndex {seqToShardIndex(seq)}; + std::shared_ptr shard; { - JLOG(j_.warn()) << - "shard " << shardIndex << - " source ledger sequence " << ledger->info().seq << - " is not being acquired"; - return false; + std::lock_guard lock(mutex_); + assert(init_); + + if (shardIndex != acquireIndex_) + { + JLOG(j_.trace()) << + "shard " << shardIndex << " is not being acquired"; + return false; + } + + if (auto const it {shards_.find(shardIndex)}; it != shards_.end()) + shard = it->second.shard; + else + { + JLOG(j_.error()) << + "shard " << shardIndex << " is not being acquired"; + return false; + } } - if (!Database::copyLedger(*incomplete_->getBackend(), *ledger, - incomplete_->pCache(), incomplete_->nCache(), - incomplete_->lastStored())) + if (shard->containsLedger(seq)) { + JLOG(j_.trace()) << + "shard " << shardIndex << " ledger already stored"; return false; } - if (!incomplete_->setStored(ledger)) - return false; - if (incomplete_->complete()) { - complete_.emplace(incomplete_->index(), std::move(incomplete_)); - incomplete_.reset(); - updateStatus(lock); + auto [backend, pCache, nCache] = shard->getBackendAll(); + if (!Database::storeLedger( + *srcLedger, + backend, + pCache, + nCache, + nullptr)) + { + return false; + } } - setFileStats(lock); - return true; + return storeLedgerInShard(shard, srcLedger); } int DatabaseShardImp::getDesiredAsyncReadCount(std::uint32_t seq) { auto const shardIndex {seqToShardIndex(seq)}; + std::shared_ptr shard; { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - auto it = complete_.find(shardIndex); - if (it != complete_.end()) - return it->second->pCache()->getTargetSize() / asyncDivider; - if (incomplete_ && incomplete_->index() == shardIndex) - return incomplete_->pCache()->getTargetSize() / asyncDivider; + if (auto const it {shards_.find(shardIndex)}; + it != shards_.end() && + (it->second.state == ShardInfo::State::final || + it->second.state == ShardInfo::State::acquire)) + { + shard = it->second.shard; + } + else + return 0; } - return cacheTargetSize / asyncDivider; + + return shard->pCache()->getTargetSize() / asyncDivider; } float DatabaseShardImp::getCacheHitRate() { - float sz, f {0}; + std::shared_ptr shard; { - std::lock_guard lock(m_); + std::lock_guard lock(mutex_); assert(init_); - sz = complete_.size(); - for (auto const& e : complete_) - f += e.second->pCache()->getHitRate(); - if (incomplete_) - { - f += incomplete_->pCache()->getHitRate(); - ++sz; - } + if (auto const it {shards_.find(acquireIndex_)}; it != shards_.end()) + shard = it->second.shard; + else + return 0; } - return f / std::max(1.0f, sz); + + return shard->pCache()->getHitRate(); } void DatabaseShardImp::sweep() { - std::lock_guard lock(m_); - assert(init_); + std::vector> shards; + { + std::lock_guard lock(mutex_); + assert(init_); - for (auto const& e : complete_) - e.second->sweep(); + for (auto const& e : shards_) + if (e.second.state == ShardInfo::State::final || + e.second.state == ShardInfo::State::acquire) + { + shards.push_back(e.second.shard); + } + } - if (incomplete_) - incomplete_->sweep(); + for (auto const& e : shards) + { + if (auto shard {e.lock()}; shard) + shard->sweep(); + } } -std::shared_ptr -DatabaseShardImp::fetchFrom(uint256 const& hash, std::uint32_t seq) +bool +DatabaseShardImp::initConfig(std::lock_guard&) { - auto const shardIndex {seqToShardIndex(seq)}; - std::unique_lock lock(m_); - assert(init_); + auto fail = [j = j_](std::string const& msg) + { + JLOG(j.error()) << + "[" << ConfigSection::shardDatabase() << "] " << msg; + return false; + }; + + Config const& config {app_.config()}; + Section const& section {config.section(ConfigSection::shardDatabase())}; + { - auto it = complete_.find(shardIndex); - if (it != complete_.end()) + // The earliest ledger sequence defaults to XRP_LEDGER_EARLIEST_SEQ. + // A custom earliest ledger sequence can be set through the + // configuration file using the 'earliest_seq' field under the + // 'node_db' and 'shard_db' stanzas. If specified, this field must + // have a value greater than zero and be equally assigned in + // both stanzas. + + std::uint32_t shardDBEarliestSeq {0}; + get_if_exists( + section, + "earliest_seq", + shardDBEarliestSeq); + + std::uint32_t nodeDBEarliestSeq {0}; + get_if_exists( + config.section(ConfigSection::nodeDatabase()), + "earliest_seq", + nodeDBEarliestSeq); + + if (shardDBEarliestSeq != nodeDBEarliestSeq) { - lock.unlock(); - return fetchInternal(hash, *it->second->getBackend()); + return fail("and [" + ConfigSection::nodeDatabase() + + "] define different 'earliest_seq' values"); } } - if (incomplete_ && incomplete_->index() == shardIndex) + + using namespace boost::filesystem; + if (!get_if_exists(section, "path", dir_)) + return fail("'path' missing"); + { - lock.unlock(); - return fetchInternal(hash, *incomplete_->getBackend()); + std::uint64_t sz; + if (!get_if_exists(section, "max_size_gb", sz)) + return fail("'max_size_gb' missing"); + + if ((sz << 30) < sz) + return fail("'max_size_gb' overflow"); + + // Minimum storage space required (in gigabytes) + if (sz < 10) + return fail("'max_size_gb' must be at least 10"); + + // Convert to bytes + maxFileSz_ = sz << 30; } - // Used to validate import shards - auto it = preShards_.find(shardIndex); - if (it != preShards_.end() && it->second) + if (section.exists("ledgers_per_shard")) { - lock.unlock(); - return fetchInternal(hash, *it->second->getBackend()); + // To be set only in standalone for testing + if (!config.standalone()) + return fail("'ledgers_per_shard' only honored in stand alone"); + + ledgersPerShard_ = get(section, "ledgers_per_shard"); + if (ledgersPerShard_ == 0 || ledgersPerShard_ % 256 != 0) + return fail("'ledgers_per_shard' must be a multiple of 256"); } - return {}; + + // NuDB is the default and only supported permanent storage backend + backendName_ = get(section, "type", "nudb"); + if (!boost::iequals(backendName_, "NuDB")) + return fail("'type' value unsupported"); + + return true; +} + +std::shared_ptr +DatabaseShardImp::fetchFrom(uint256 const& hash, std::uint32_t seq) +{ + auto const shardIndex {seqToShardIndex(seq)}; + std::shared_ptr shard; + { + std::lock_guard lock(mutex_); + assert(init_); + + if (auto const it {shards_.find(shardIndex)}; + it != shards_.end() && + it->second.shard) + { + shard = it->second.shard; + } + else + return {}; + } + + return fetchInternal(hash, shard->getBackend()); } boost::optional -DatabaseShardImp::findShardIndexToAdd( - std::uint32_t validLedgerSeq, std::lock_guard&) +DatabaseShardImp::findAcquireIndex( + std::uint32_t validLedgerSeq, + std::lock_guard&) { + if (validLedgerSeq < earliestLedgerSeq()) + return boost::none; + auto const maxShardIndex {[this, validLedgerSeq]() { auto shardIndex {seqToShardIndex(validLedgerSeq)}; @@ -1027,31 +1156,26 @@ DatabaseShardImp::findShardIndexToAdd( --shardIndex; return shardIndex; }()}; - auto const numShards {complete_.size() + - (incomplete_ ? 1 : 0) + preShards_.size()}; + auto const maxNumShards {maxShardIndex - earliestShardIndex() + 1}; // Check if the shard store has all shards - if (numShards >= maxShardIndex) + if (shards_.size() >= maxNumShards) return boost::none; if (maxShardIndex < 1024 || - static_cast(numShards) / maxShardIndex > 0.5f) + static_cast(shards_.size()) / maxNumShards > 0.5f) { // Small or mostly full index space to sample // Find the available indexes and select one at random std::vector available; - available.reserve(maxShardIndex - numShards + 1); + available.reserve(maxNumShards - shards_.size()); for (auto shardIndex = earliestShardIndex(); shardIndex <= maxShardIndex; ++shardIndex) { - if (complete_.find(shardIndex) == complete_.end() && - (!incomplete_ || incomplete_->index() != shardIndex) && - preShards_.find(shardIndex) == preShards_.end()) - { + if (shards_.find(shardIndex) == shards_.end()) available.push_back(shardIndex); - } } if (available.empty()) @@ -1070,12 +1194,8 @@ DatabaseShardImp::findShardIndexToAdd( for (int i = 0; i < 40; ++i) { auto const shardIndex {rand_int(earliestShardIndex(), maxShardIndex)}; - if (complete_.find(shardIndex) == complete_.end() && - (!incomplete_ || incomplete_->index() != shardIndex) && - preShards_.find(shardIndex) == preShards_.end()) - { + if (shards_.find(shardIndex) == shards_.end()) return shardIndex; - } } assert(false); @@ -1083,33 +1203,130 @@ DatabaseShardImp::findShardIndexToAdd( } void -DatabaseShardImp::setFileStats(std::lock_guard&) +DatabaseShardImp::finalizeShard( + ShardInfo& shardInfo, + bool writeSQLite, + std::lock_guard&) { - fileSz_ = 0; - fdRequired_ = 0; - if (!complete_.empty()) + assert(shardInfo.shard); + assert(shardInfo.shard->index() != acquireIndex_); + assert(shardInfo.shard->isBackendComplete()); + assert(shardInfo.state != ShardInfo::State::finalize); + + auto const shardIndex {shardInfo.shard->index()}; + + shardInfo.state = ShardInfo::State::finalize; + taskQueue_->addTask([this, shardIndex, writeSQLite]() { - for (auto const& e : complete_) + if (isStopping()) + return; + + std::shared_ptr shard; { - fileSz_ += e.second->fileSize(); - fdRequired_ += e.second->fdRequired(); + std::lock_guard lock(mutex_); + if (auto const it {shards_.find(shardIndex)}; it != shards_.end()) + shard = it->second.shard; + else + { + JLOG(j_.error()) << + "Unable to finalize shard " << shardIndex; + return; + } } - avgShardFileSz_ = fileSz_ / complete_.size(); - } - else - avgShardFileSz_ = 0; - if (incomplete_) + if (!shard->finalize(writeSQLite)) + { + if (isStopping()) + return; + + // Bad shard, remove it + { + std::lock_guard lock(mutex_); + shards_.erase(shardIndex); + updateStatus(lock); + + using namespace boost::filesystem; + path const dir {shard->getDir()}; + shard.reset(); + try + { + remove_all(dir); + } + catch (std::exception const& e) + { + JLOG(j_.error()) << + "exception " << e.what() << " in function " << __func__; + } + } + + setFileStats(); + return; + } + + if (isStopping()) + return; + + { + std::lock_guard lock(mutex_); + auto const it {shards_.find(shardIndex)}; + if (it == shards_.end()) + return; + it->second.state = ShardInfo::State::final; + updateStatus(lock); + } + + setFileStats(); + + // Update peers with new shard index + if (!app_.config().standalone() && + app_.getOPs().getOperatingMode() != OperatingMode::DISCONNECTED) + { + protocol::TMPeerShardInfo message; + PublicKey const& publicKey {app_.nodeIdentity().first}; + message.set_nodepubkey(publicKey.data(), publicKey.size()); + message.set_shardindexes(std::to_string(shardIndex)); + app_.overlay().foreach(send_always( + std::make_shared( + message, + protocol::mtPEER_SHARD_INFO))); + } + }); +} + +void +DatabaseShardImp::setFileStats() +{ + std::vector> shards; { - fileSz_ += incomplete_->fileSize(); - fdRequired_ += incomplete_->fdRequired(); + std::lock_guard lock(mutex_); + assert(init_); + + if (shards_.empty()) + return; + + for (auto const& e : shards_) + if (e.second.shard) + shards.push_back(e.second.shard); } - if (!backed_) - return; + std::uint64_t sumSz {0}; + std::uint32_t sumFd {0}; + std::uint32_t numShards {0}; + for (auto const& e : shards) + { + if (auto shard {e.lock()}; shard) + { + auto[sz, fd] = shard->fileInfo(); + sumSz += sz; + sumFd += fd; + ++numShards; + } + } - // Require at least 15 file descriptors - fdRequired_ = std::max(fdRequired_, 15); + std::lock_guard lock(mutex_); + fileSz_ = sumSz; + fdRequired_ = sumFd; + avgShardFileSz_ = fileSz_ / numShards; if (fileSz_ >= maxFileSz_) { @@ -1126,11 +1343,12 @@ DatabaseShardImp::setFileStats(std::lock_guard&) void DatabaseShardImp::updateStatus(std::lock_guard&) { - if (!complete_.empty()) + if (!shards_.empty()) { RangeSet rs; - for (auto const& e : complete_) - rs.insert(e.second->index()); + for (auto const& e : shards_) + if (e.second.state == ShardInfo::State::final) + rs.insert(e.second.shard->index()); status_ = to_string(rs); } else @@ -1138,32 +1356,28 @@ DatabaseShardImp::updateStatus(std::lock_guard&) } std::pair, std::shared_ptr> -DatabaseShardImp::selectCache(std::uint32_t seq) +DatabaseShardImp::getCache(std::uint32_t seq) { auto const shardIndex {seqToShardIndex(seq)}; - std::lock_guard lock(m_); - assert(init_); - + std::shared_ptr shard; { - auto it = complete_.find(shardIndex); - if (it != complete_.end()) + std::lock_guard lock(mutex_); + assert(init_); + + if (auto const it {shards_.find(shardIndex)}; + it != shards_.end() && it->second.shard) { - return std::make_pair(it->second->pCache(), - it->second->nCache()); + shard = it->second.shard; } + else + return {}; } - if (incomplete_ && incomplete_->index() == shardIndex) - { - return std::make_pair(incomplete_->pCache(), - incomplete_->nCache()); - } + std::shared_ptr pCache; + std::shared_ptr nCache; + std::tie(std::ignore, pCache, nCache) = shard->getBackendAll(); - // Used to validate import shards - auto it = preShards_.find(shardIndex); - if (it != preShards_.end() && it->second) - return std::make_pair(it->second->pCache(), it->second->nCache()); - return {}; + return std::make_pair(pCache, nCache); } std::uint64_t @@ -1175,12 +1389,70 @@ DatabaseShardImp::available() const } catch (std::exception const& e) { - JLOG(j_.error()) << "exception " << e.what() << - " in function " << __func__; + JLOG(j_.error()) << + "exception " << e.what() << " in function " << __func__; return 0; } } +bool +DatabaseShardImp::storeLedgerInShard( + std::shared_ptr& shard, + std::shared_ptr const& ledger) +{ + bool result {true}; + + if (!shard->store(ledger)) + { + // Shard may be corrupt, remove it + std::lock_guard lock(mutex_); + + shards_.erase(shard->index()); + if (shard->index() == acquireIndex_) + acquireIndex_ = 0; + + updateStatus(lock); + + using namespace boost::filesystem; + path const dir {shard->getDir()}; + shard.reset(); + try + { + remove_all(dir); + } + catch (std::exception const& e) + { + JLOG(j_.error()) << + "exception " << e.what() << " in function " << __func__; + } + + result = false; + } + else if (shard->isBackendComplete()) + { + std::lock_guard lock(mutex_); + + if (auto const it {shards_.find(shard->index())}; + it != shards_.end()) + { + if (shard->index() == acquireIndex_) + acquireIndex_ = 0; + + if (it->second.state != ShardInfo::State::finalize) + finalizeShard(it->second, false, lock); + } + else + { + JLOG(j_.debug()) << + "shard " << shard->index() << + " is no longer being acquired"; + } + } + + setFileStats(); + return result; +} + //------------------------------------------------------------------------------ std::unique_ptr @@ -1197,19 +1469,13 @@ make_ShardStore( if (section.empty()) return nullptr; - auto shardStore = std::make_unique( + return std::make_unique( app, parent, "ShardStore", scheduler, readThreads, j); - if (shardStore->init()) - shardStore->setParent(parent); - else - shardStore.reset(); - - return shardStore; } } // NodeStore diff --git a/src/ripple/nodestore/impl/DatabaseShardImp.h b/src/ripple/nodestore/impl/DatabaseShardImp.h index 9d90e4d5baa..21a117bdab1 100644 --- a/src/ripple/nodestore/impl/DatabaseShardImp.h +++ b/src/ripple/nodestore/impl/DatabaseShardImp.h @@ -22,6 +22,7 @@ #include #include +#include namespace ripple { namespace NodeStore { @@ -61,8 +62,9 @@ class DatabaseShardImp : public DatabaseShard getPreShards() override; bool - importShard(std::uint32_t shardIndex, - boost::filesystem::path const& srcDir, bool validate) override; + importShard( + std::uint32_t shardIndex, + boost::filesystem::path const& srcDir) override; std::shared_ptr fetchLedger(uint256 const& hash, std::uint32_t seq) override; @@ -70,9 +72,6 @@ class DatabaseShardImp : public DatabaseShard void setStored(std::shared_ptr const& ledger) override; - bool - contains(std::uint32_t seq) override; - std::string getCompleteShards() override; @@ -94,7 +93,7 @@ class DatabaseShardImp : public DatabaseShard std::uint32_t seqToShardIndex(std::uint32_t seq) const override { - assert(seq >= earliestSeq()); + assert(seq >= earliestLedgerSeq()); return NodeStore::seqToShardIndex(seq, ledgersPerShard_); } @@ -103,7 +102,7 @@ class DatabaseShardImp : public DatabaseShard { assert(shardIndex >= earliestShardIndex_); if (shardIndex <= earliestShardIndex_) - return earliestSeq(); + return earliestLedgerSeq(); return 1 + (shardIndex * ledgersPerShard_); } @@ -126,6 +125,9 @@ class DatabaseShardImp : public DatabaseShard return backendName_; } + void + onStop() override; + /** Import the application local node store @param source The application node store. @@ -137,18 +139,23 @@ class DatabaseShardImp : public DatabaseShard getWriteLoad() const override; void - store(NodeObjectType type, Blob&& data, - uint256 const& hash, std::uint32_t seq) override; + store( + NodeObjectType type, + Blob&& data, + uint256 const& hash, + std::uint32_t seq) override; std::shared_ptr fetch(uint256 const& hash, std::uint32_t seq) override; bool - asyncFetch(uint256 const& hash, std::uint32_t seq, + asyncFetch( + uint256 const& hash, + std::uint32_t seq, std::shared_ptr& object) override; bool - copyLedger(std::shared_ptr const& ledger) override; + storeLedger(std::shared_ptr const& srcLedger) override; int getDesiredAsyncReadCount(std::uint32_t seq) override; @@ -163,21 +170,43 @@ class DatabaseShardImp : public DatabaseShard sweep() override; private: + struct ShardInfo + { + enum class State + { + none, + final, // Immutable, complete and validated + acquire, // Being acquired + import, // Being imported + finalize // Being finalized + }; + + ShardInfo() = default; + ShardInfo(std::shared_ptr shard_, State state_) + : shard(std::move(shard_)) + , state(state_) + {} + + std::shared_ptr shard; + State state {State::none}; + }; + Application& app_; - mutable std::mutex m_; + Stoppable& parent_; + mutable std::mutex mutex_; bool init_ {false}; // The context shared with all shard backend databases std::unique_ptr ctx_; - // Complete shards - std::map> complete_; + // Queue of background tasks to be performed + std::unique_ptr taskQueue_; - // A shard being acquired from the peer network - std::unique_ptr incomplete_; + // Shards held by this server + std::map shards_; - // Shards prepared for import - std::map preShards_; + // Shard index being acquired from the peer network + std::uint32_t acquireIndex_ {0}; // The shard store root directory boost::filesystem::path dir_; @@ -188,9 +217,6 @@ class DatabaseShardImp : public DatabaseShard // Complete shard indexes std::string status_; - // If backend type uses permanent storage - bool backed_; - // The name associated with the backend used with the shard store std::string backendName_; @@ -214,6 +240,11 @@ class DatabaseShardImp : public DatabaseShard // File name used to mark shards being imported from node store static constexpr auto importMarker_ = "import"; + // Initialize settings from the configuration file + // Lock must be held + bool + initConfig(std::lock_guard&); + std::shared_ptr fetchFrom(uint256 const& hash, std::uint32_t seq) override; @@ -223,17 +254,25 @@ class DatabaseShardImp : public DatabaseShard Throw("Shard store import not supported"); } - // Finds a random shard index that is not stored + // Randomly select a shard index not stored // Lock must be held boost::optional - findShardIndexToAdd( + findAcquireIndex( std::uint32_t validLedgerSeq, std::lock_guard&); - // Set storage and file descriptor usage stats + // Queue a task to finalize a shard by validating its databases // Lock must be held void - setFileStats(std::lock_guard&); + finalizeShard( + ShardInfo& shardInfo, + bool writeSQLite, + std::lock_guard&); + + // Set storage and file descriptor usage stats + // Lock must NOT be held + void + setFileStats(); // Update status string // Lock must be held @@ -241,11 +280,16 @@ class DatabaseShardImp : public DatabaseShard updateStatus(std::lock_guard&); std::pair, std::shared_ptr> - selectCache(std::uint32_t seq); + getCache(std::uint32_t seq); // Returns available storage space std::uint64_t available() const; + + bool + storeLedgerInShard( + std::shared_ptr& shard, + std::shared_ptr const& ledger); }; } // NodeStore diff --git a/src/ripple/nodestore/impl/NodeObject.cpp b/src/ripple/nodestore/impl/NodeObject.cpp index 91a8459263e..682b3b3b4de 100644 --- a/src/ripple/nodestore/impl/NodeObject.cpp +++ b/src/ripple/nodestore/impl/NodeObject.cpp @@ -31,8 +31,8 @@ NodeObject::NodeObject ( PrivateAccess) : mType (type) , mHash (hash) + , mData (std::move(data)) { - mData = std::move (data); } std::shared_ptr diff --git a/src/ripple/nodestore/impl/Shard.cpp b/src/ripple/nodestore/impl/Shard.cpp index 2b685a661c7..086e4e4c37c 100644 --- a/src/ripple/nodestore/impl/Shard.cpp +++ b/src/ripple/nodestore/impl/Shard.cpp @@ -24,17 +24,16 @@ #include #include #include +#include #include -#include -#include #include -#include - namespace ripple { namespace NodeStore { +uint256 const Shard::finalKey {0}; + Shard::Shard( Application& app, DatabaseShard const& db, @@ -47,7 +46,6 @@ Shard::Shard( , maxLedgers_(index == db.earliestShardIndex() ? lastSeq_ - firstSeq_ + 1 : db.ledgersPerShard()) , dir_(db.getRootDir() / std::to_string(index_)) - , control_(dir_ / controlFileName) , j_(j) { if (index_ < db.earliestShardIndex()) @@ -57,96 +55,149 @@ Shard::Shard( bool Shard::open(Scheduler& scheduler, nudb::context& ctx) { - using namespace boost::filesystem; - std::lock_guard lock(mutex_); + std::lock_guard lock {mutex_}; assert(!backend_); Config const& config {app_.config()}; - Section section {config.section(ConfigSection::shardDatabase())}; - std::string const type (get(section, "type", "nudb")); - auto factory {Manager::instance().find(type)}; - if (!factory) { - JLOG(j_.error()) << - "shard " << index_ << - " failed to create backend type " << type; - return false; - } + Section section {config.section(ConfigSection::shardDatabase())}; + std::string const type {get(section, "type", "nudb")}; + auto factory {Manager::instance().find(type)}; + if (!factory) + { + JLOG(j_.error()) << + "shard " << index_ << + " failed to create backend type " << type; + return false; + } - section.set("path", dir_.string()); - backend_ = factory->createInstance( - NodeObject::keyBytes, section, scheduler, ctx, j_); + section.set("path", dir_.string()); + backend_ = factory->createInstance( + NodeObject::keyBytes, section, scheduler, ctx, j_); + } - auto const preexist {exists(dir_)}; - auto fail = [this, preexist](std::string const& msg) + using namespace boost::filesystem; + auto preexist {false}; + auto fail = [this, &preexist](std::string const& msg) { pCache_.reset(); nCache_.reset(); backend_.reset(); lgrSQLiteDB_.reset(); txSQLiteDB_.reset(); - storedSeqs_.clear(); - lastStored_.reset(); + acquireInfo_.reset(); if (!preexist) - removeAll(dir_, j_); + remove_all(dir_); if (!msg.empty()) { - JLOG(j_.error()) << - "shard " << index_ << " " << msg; + JLOG(j_.fatal()) << "shard " << index_ << " " << msg; } return false; }; + auto createAcquireInfo = [this, &config]() + { + acquireInfo_ = std::make_unique(); + + DatabaseCon::Setup setup; + setup.startUp = config.START_UP; + setup.standAlone = config.standalone(); + setup.dataDir = dir_; + + acquireInfo_->SQLiteDB = std::make_unique( + setup, + AcquireShardDBName, + AcquireShardDBPragma, + AcquireShardDBInit); + acquireInfo_->SQLiteDB->setupCheckpointing( + &app_.getJobQueue(), + app_.logs()); + }; + try { - // Open/Create the NuDB key/value store for node objects + // Open or create the NuDB key/value store + preexist = exists(dir_); backend_->open(!preexist); - if (!backend_->backed()) - return true; - if (!preexist) { - // New shard, create a control file - if (!saveControl(lock)) - return fail({}); + // A new shard + createAcquireInfo(); + acquireInfo_->SQLiteDB->getSession() << + "INSERT INTO Shard (ShardIndex) " + "VALUES (:shardIndex);" + , soci::use(index_); } - else if (is_regular_file(control_)) + else if (exists(dir_ / AcquireShardDBName)) { - // Incomplete shard, inspect control file - std::ifstream ifs(control_.string()); - if (!ifs.is_open()) - return fail("failed to open control file"); - - boost::archive::text_iarchive ar(ifs); - ar & storedSeqs_; - if (!storedSeqs_.empty()) + // An incomplete shard, being acquired + createAcquireInfo(); + + auto& session {acquireInfo_->SQLiteDB->getSession()}; + boost::optional index; + soci::blob sociBlob(session); + soci::indicator blobPresent; + + session << + "SELECT ShardIndex, StoredLedgerSeqs " + "FROM Shard " + "WHERE ShardIndex = :index;" + , soci::into(index) + , soci::into(sociBlob, blobPresent) + , soci::use(index_); + + if (!index || index != index_) + return fail("invalid acquire SQLite database"); + + if (blobPresent == soci::i_ok) { - if (boost::icl::first(storedSeqs_) < firstSeq_ || - boost::icl::last(storedSeqs_) > lastSeq_) + std::string s; + auto& storedSeqs {acquireInfo_->storedSeqs}; + if (convert(sociBlob, s); !from_string(storedSeqs, s)) + return fail("invalid StoredLedgerSeqs"); + + if (boost::icl::first(storedSeqs) < firstSeq_ || + boost::icl::last(storedSeqs) > lastSeq_) { - return fail("has an invalid control file"); + return fail("invalid StoredLedgerSeqs"); } - if (boost::icl::length(storedSeqs_) >= maxLedgers_) + if (boost::icl::length(storedSeqs) == maxLedgers_) { - JLOG(j_.warn()) << - "shard " << index_ << - " has a control file for complete shard"; - setComplete(lock); + // All ledgers have been acquired, shard is complete + acquireInfo_.reset(); + backendComplete_ = true; } } } else - setComplete(lock); - - if (!complete_) { - setCache(lock); - if (!initSQLite(lock) ||!setFileStats(lock)) - return fail({}); + // A finalized shard or has all ledgers stored in the backend + std::shared_ptr nObj; + if (backend_->fetch(finalKey.data(), &nObj) != Status::ok) + { + legacy_ = true; + return fail("incompatible, missing backend final key"); + } + + // Check final key's value + SerialIter sIt(nObj->getData().data(), nObj->getData().size()); + if (sIt.get32() != version) + return fail("invalid version"); + + if (sIt.get32() != firstSeq_ || sIt.get32() != lastSeq_) + return fail("out of range ledger sequences"); + + if (sIt.get256().isZero()) + return fail("invalid last ledger hash"); + + if (exists(dir_ / LgrDBName) && exists(dir_ / TxDBName)) + final_ = true; + + backendComplete_ = true; } } catch (std::exception const& e) @@ -155,66 +206,104 @@ Shard::open(Scheduler& scheduler, nudb::context& ctx) e.what() + " in function " + __func__); } + setBackendCache(lock); + if (!initSQLite(lock)) + return fail({}); + + setFileStats(lock); return true; } +boost::optional +Shard::prepare() +{ + std::lock_guard lock(mutex_); + assert(backend_); + + if (backendComplete_) + { + JLOG(j_.warn()) << + "shard " << index_ << + " prepare called when shard is complete"; + return {}; + } + + assert(acquireInfo_); + auto const& storedSeqs {acquireInfo_->storedSeqs}; + if (storedSeqs.empty()) + return lastSeq_; + return prevMissing(storedSeqs, 1 + lastSeq_, firstSeq_); +} + bool -Shard::setStored(std::shared_ptr const& ledger) +Shard::store(std::shared_ptr const& ledger) { + auto const seq {ledger->info().seq}; + if (seq < firstSeq_ || seq > lastSeq_) + { + JLOG(j_.error()) << + "shard " << index_ << + " invalid ledger sequence " << seq; + return false; + } + std::lock_guard lock(mutex_); - assert(backend_ && !complete_); + assert(backend_); - if (boost::icl::contains(storedSeqs_, ledger->info().seq)) + if (backendComplete_) { JLOG(j_.debug()) << "shard " << index_ << - " has ledger sequence " << ledger->info().seq << " already stored"; - return false; + " ledger sequence " << seq << " already stored"; + return true; + } + + assert(acquireInfo_); + auto& storedSeqs {acquireInfo_->storedSeqs}; + if (boost::icl::contains(storedSeqs, seq)) + { + JLOG(j_.debug()) << + "shard " << index_ << + " ledger sequence " << seq << " already stored"; + return true; } + // storeSQLite looks at storedSeqs so insert before the call + storedSeqs.insert(seq); - if (!setSQLiteStored(ledger, lock)) + if (!storeSQLite(ledger, lock)) return false; - // Check if the shard is complete - if (boost::icl::length(storedSeqs_) >= maxLedgers_ - 1) - setComplete(lock); - else + if (boost::icl::length(storedSeqs) >= maxLedgers_) { - storedSeqs_.insert(ledger->info().seq); - if (backend_->backed() && !saveControl(lock)) + if (!initSQLite(lock)) return false; + + acquireInfo_.reset(); + backendComplete_ = true; + setBackendCache(lock); } JLOG(j_.debug()) << "shard " << index_ << - " stored ledger sequence " << ledger->info().seq << - (complete_ ? " and is complete" : ""); + " stored ledger sequence " << seq << + (backendComplete_ ? " . All ledgers stored" : ""); - lastStored_ = ledger; + setFileStats(lock); return true; } -boost::optional -Shard::prepare() -{ - std::lock_guard lock(mutex_); - assert(backend_ && !complete_); - - if (storedSeqs_.empty()) - return lastSeq_; - return prevMissing(storedSeqs_, 1 + lastSeq_, firstSeq_); -} - bool -Shard::contains(std::uint32_t seq) const +Shard::containsLedger(std::uint32_t seq) const { if (seq < firstSeq_ || seq > lastSeq_) return false; std::lock_guard lock(mutex_); - assert(backend_); + if (backendComplete_) + return true; - return complete_ || boost::icl::contains(storedSeqs_, seq); + assert(acquireInfo_); + return boost::icl::contains(acquireInfo_->storedSeqs, seq); } void @@ -227,7 +316,19 @@ Shard::sweep() nCache_->sweep(); } -std::shared_ptr const& +std::tuple< + std::shared_ptr, + std::shared_ptr, + std::shared_ptr> +Shard::getBackendAll() const +{ + std::lock_guard lock(mutex_); + assert(backend_); + + return {backend_, pCache_, nCache_}; +} + +std::shared_ptr Shard::getBackend() const { std::lock_guard lock(mutex_); @@ -237,12 +338,10 @@ Shard::getBackend() const } bool -Shard::complete() const +Shard::isBackendComplete() const { std::lock_guard lock(mutex_); - assert(backend_); - - return complete_; + return backendComplete_; } std::shared_ptr @@ -263,94 +362,160 @@ Shard::nCache() const return nCache_; } -std::uint64_t -Shard::fileSize() const +std::pair +Shard::fileInfo() const { std::lock_guard lock(mutex_); - assert(backend_); - - return fileSz_; + return {fileSz_, fdRequired_}; } -std::uint32_t -Shard::fdRequired() const +bool +Shard::isFinal() const { std::lock_guard lock(mutex_); - assert(backend_); - - return fdRequired_; + return final_; } -std::shared_ptr -Shard::lastStored() const +bool +Shard::isLegacy() const { std::lock_guard lock(mutex_); - assert(backend_); - - return lastStored_; + return legacy_; } bool -Shard::validate() const +Shard::finalize(const bool writeSQLite) { - uint256 hash; + assert(backend_); + + if (stop_) + return false; + + uint256 hash {0}; std::uint32_t seq {0}; auto fail = [j = j_, index = index_, &hash, &seq](std::string const& msg) { - JLOG(j.error()) << + JLOG(j.fatal()) << "shard " << index << ". " << msg << (hash.isZero() ? "" : ". Ledger hash " + to_string(hash)) << (seq == 0 ? "" : ". Ledger sequence " + std::to_string(seq)); return false; }; - std::shared_ptr ledger; - // Find the hash of the last ledger in this shard + try { - std::tie(ledger, seq, hash) = loadLedgerHelper( - "WHERE LedgerSeq >= " + std::to_string(lastSeq_) + - " order by LedgerSeq desc limit 1", app_, false); - if (!ledger) - return fail("Unable to validate due to lacking lookup data"); + std::unique_lock lock(mutex_); + if (!backendComplete_) + return fail("incomplete"); + + /* + TODO MP + A lock is required when calling the NuDB verify function. Because + this can be a time consuming process, the server may desync. + Until this function is modified to work on an open database, we + are unable to use it from rippled. + + // Verify backend integrity + backend_->verify(); + */ + + // Check if a final key has been stored + lock.unlock(); + if (std::shared_ptr nObj; + backend_->fetch(finalKey.data(), &nObj) == Status::ok) + { + // Check final key's value + SerialIter sIt(nObj->getData().data(), nObj->getData().size()); + if (sIt.get32() != version) + return fail("invalid version"); - if (seq != lastSeq_) + if (sIt.get32() != firstSeq_ || sIt.get32() != lastSeq_) + return fail("out of range ledger sequences"); + + if (hash = sIt.get256(); hash.isZero()) + return fail("invalid last ledger hash"); + } + else { - boost::optional h; + // In the absence of a final key, an acquire SQLite database + // must be present in order to validate the shard + lock.lock(); + if (!acquireInfo_) + return fail("missing acquire SQLite database"); + + auto& session {acquireInfo_->SQLiteDB->getSession()}; + boost::optional index; + boost::optional sHash; + soci::blob sociBlob(session); + soci::indicator blobPresent; + session << + "SELECT ShardIndex, LastLedgerHash, StoredLedgerSeqs " + "FROM Shard " + "WHERE ShardIndex = :index;" + , soci::into(index) + , soci::into(sHash) + , soci::into(sociBlob, blobPresent) + , soci::use(index_); - ledger->setImmutable(app_.config()); - try - { - h = hashOfSeq(*ledger, lastSeq_, j_); - } - catch (std::exception const& e) + lock.unlock(); + if (!index || index != index_) + return fail("missing or invalid ShardIndex"); + + if (!sHash) + return fail("missing LastLedgerHash"); + + if (hash.SetHexExact(*sHash); hash.isZero()) + return fail("invalid LastLedgerHash"); + + if (blobPresent != soci::i_ok) + return fail("missing StoredLedgerSeqs"); + + std::string s; + convert(sociBlob, s); + + lock.lock(); + + auto& storedSeqs {acquireInfo_->storedSeqs}; + if (!from_string(storedSeqs, s) || + boost::icl::first(storedSeqs) != firstSeq_ || + boost::icl::last(storedSeqs) != lastSeq_ || + storedSeqs.size() != maxLedgers_) { - return fail(std::string("exception ") + - e.what() + " in function " + __func__); + return fail("invalid StoredLedgerSeqs"); } - - if (!h) - return fail("Missing hash for last ledger sequence"); - hash = *h; - seq = lastSeq_; } } + catch (std::exception const& e) + { + return fail(std::string("exception ") + + e.what() + " in function " + __func__); + } - // Validate every ledger stored in this shard + // Validate every ledger stored in the backend + std::shared_ptr ledger; std::shared_ptr next; + auto const lastLedgerHash {hash}; + + // Start with the last ledger in the shard and walk backwards from + // child to parent until we reach the first ledger + seq = lastSeq_; while (seq >= firstSeq_) { + if (stop_) + return false; + auto nObj = valFetch(hash); if (!nObj) - return fail("Invalid ledger"); + return fail("invalid ledger"); ledger = std::make_shared( InboundLedger::deserializeHeader(makeSlice(nObj->getData()), true), app_.config(), *app_.shardFamily()); if (ledger->info().seq != seq) - return fail("Invalid ledger header sequence"); + return fail("invalid ledger sequence"); if (ledger->info().hash != hash) - return fail("Invalid ledger header hash"); + return fail("invalid ledger hash"); ledger->stateMap().setLedgerSeq(seq); ledger->txMap().setLedgerSeq(seq); @@ -358,96 +523,138 @@ Shard::validate() const if (!ledger->stateMap().fetchRoot( SHAMapHash {ledger->info().accountHash}, nullptr)) { - return fail("Missing root STATE node"); + return fail("missing root STATE node"); } if (ledger->info().txHash.isNonZero() && !ledger->txMap().fetchRoot( SHAMapHash {ledger->info().txHash}, nullptr)) { - return fail("Missing root TXN node"); + return fail("missing root TXN node"); } if (!valLedger(ledger, next)) - return false; + return fail("failed to validate ledger"); + + if (writeSQLite) + { + std::lock_guard lock(mutex_); + if (!storeSQLite(ledger, lock)) + return fail("failed storing to SQLite databases"); + } hash = ledger->info().parentHash; --seq; next = ledger; } - { - std::lock_guard lock(mutex_); - pCache_->reset(); - nCache_->reset(); - } - JLOG(j_.debug()) << "shard " << index_ << " is valid"; - return true; -} -bool -Shard::setComplete(std::lock_guard const& lock) -{ - // Remove the control file if one exists + /* + TODO MP + SQLite VACUUM blocks all database access while processing. + Depending on the file size, that can take a while. Until we find + a non-blocking way of doing this, we cannot enable vacuum as + it can desync a server. + try { - using namespace boost::filesystem; - if (is_regular_file(control_)) - remove_all(control_); + // VACUUM the SQLite databases + auto const tmpDir {dir_ / "tmp_vacuum"}; + create_directory(tmpDir); + auto vacuum = [&tmpDir](std::unique_ptr& sqliteDB) + { + auto& session {sqliteDB->getSession()}; + session << "PRAGMA synchronous=OFF;"; + session << "PRAGMA journal_mode=OFF;"; + session << "PRAGMA temp_store_directory='" << + tmpDir.string() << "';"; + session << "VACUUM;"; + }; + vacuum(lgrSQLiteDB_); + vacuum(txSQLiteDB_); + remove_all(tmpDir); } catch (std::exception const& e) { - JLOG(j_.error()) << - "shard " << index_ << - " exception " << e.what() << - " in function " << __func__; - return false; + return fail(std::string("exception ") + + e.what() + " in function " + __func__); } + */ + + // Store final key's value, may already be stored + Serializer s; + s.add32(version); + s.add32(firstSeq_); + s.add32(lastSeq_); + s.add256(lastLedgerHash); + auto nObj {NodeObject::createObject( + hotUNKNOWN, + std::move(s.modData()), + finalKey)}; + try + { + backend_->store(nObj); + + std::lock_guard lock(mutex_); + final_ = true; + + // Remove the acquire SQLite database if present + if (acquireInfo_) + acquireInfo_.reset(); + remove_all(dir_ / AcquireShardDBName); - storedSeqs_.clear(); - complete_ = true; + if (!initSQLite(lock)) + return fail("failed to initialize SQLite databases"); - setCache(lock); - return initSQLite(lock) && setFileStats(lock); + setFileStats(lock); + } + catch (std::exception const& e) + { + return fail(std::string("exception ") + + e.what() + " in function " + __func__); + } + + return true; } void -Shard::setCache(std::lock_guard const&) +Shard::setBackendCache(std::lock_guard const&) { - // complete shards use the smallest cache and + // Complete shards use the smallest cache and // fastest expiration to reduce memory consumption. - // The incomplete shard is set according to configuration. + // An incomplete shard is set according to configuration. + + Config const& config {app_.config()}; if (!pCache_) { auto const name {"shard " + std::to_string(index_)}; - auto const sz = app_.config().getValueFor(SizedItem::nodeCacheSize, - complete_ ? boost::optional(0) : boost::none); - auto const age = std::chrono::seconds{ - app_.config().getValueFor(SizedItem::nodeCacheAge, - complete_ ? boost::optional(0) : boost::none)}; + auto const sz {config.getValueFor( + SizedItem::nodeCacheSize, + backendComplete_ ? boost::optional(0) : boost::none)}; + auto const age {std::chrono::seconds{config.getValueFor( + SizedItem::nodeCacheAge, + backendComplete_ ? boost::optional(0) : boost::none)}}; pCache_ = std::make_shared(name, sz, age, stopwatch(), j_); nCache_ = std::make_shared(name, stopwatch(), sz, age); } else { - auto const sz = app_.config().getValueFor( - SizedItem::nodeCacheSize, 0); + auto const sz {config.getValueFor(SizedItem::nodeCacheSize, 0)}; pCache_->setTargetSize(sz); nCache_->setTargetSize(sz); - auto const age = std::chrono::seconds{ - app_.config().getValueFor( - SizedItem::nodeCacheAge, 0)}; + auto const age {std::chrono::seconds{ + config.getValueFor(SizedItem::nodeCacheAge, 0)}}; pCache_->setTargetAge(age); nCache_->setTargetAge(age); } } bool -Shard::initSQLite(std::lock_guard const&) +Shard::initSQLite(std::lock_guard const&) { Config const& config {app_.config()}; DatabaseCon::Setup setup; @@ -457,61 +664,40 @@ Shard::initSQLite(std::lock_guard const&) try { - if (complete_) - { - // Remove WAL files if they exist - using namespace boost::filesystem; - for (auto const& d : directory_iterator(dir_)) - { - if (is_regular_file(d) && - boost::iends_with(extension(d), "-wal")) - { - // Closing the session forces a checkpoint - if (!lgrSQLiteDB_) - { - lgrSQLiteDB_ = std::make_unique ( - setup, - LgrDBName, - LgrDBPragma, - LgrDBInit); - } - lgrSQLiteDB_->getSession().close(); + if (lgrSQLiteDB_) + lgrSQLiteDB_.reset(); - if (!txSQLiteDB_) - { - txSQLiteDB_ = std::make_unique ( - setup, - TxDBName, - TxDBPragma, - TxDBInit); - } - txSQLiteDB_->getSession().close(); - break; - } - } + if (txSQLiteDB_) + txSQLiteDB_.reset(); - lgrSQLiteDB_ = std::make_unique ( + if (backendComplete_) + { + lgrSQLiteDB_ = std::make_unique( setup, LgrDBName, CompleteShardDBPragma, LgrDBInit); lgrSQLiteDB_->getSession() << boost::str(boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::lgrDBCache, boost::none))); + kilobytes(config.getValueFor( + SizedItem::lgrDBCache, + boost::none))); - txSQLiteDB_ = std::make_unique ( + txSQLiteDB_ = std::make_unique( setup, TxDBName, CompleteShardDBPragma, TxDBInit); txSQLiteDB_->getSession() << boost::str(boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::txnDBCache, boost::none))); + kilobytes(config.getValueFor( + SizedItem::txnDBCache, + boost::none))); } else { // The incomplete shard uses a Write Ahead Log for performance - lgrSQLiteDB_ = std::make_unique ( + lgrSQLiteDB_ = std::make_unique( setup, LgrDBName, LgrDBPragma, @@ -521,7 +707,7 @@ Shard::initSQLite(std::lock_guard const&) kilobytes(config.getValueFor(SizedItem::lgrDBCache))); lgrSQLiteDB_->setupCheckpointing(&app_.getJobQueue(), app_.logs()); - txSQLiteDB_ = std::make_unique ( + txSQLiteDB_ = std::make_unique( setup, TxDBName, TxDBPragma, @@ -534,7 +720,7 @@ Shard::initSQLite(std::lock_guard const&) } catch (std::exception const& e) { - JLOG(j_.error()) << + JLOG(j_.fatal()) << "shard " << index_ << " exception " << e.what() << " in function " << __func__; @@ -544,25 +730,29 @@ Shard::initSQLite(std::lock_guard const&) } bool -Shard::setSQLiteStored( +Shard::storeSQLite( std::shared_ptr const& ledger, - std::lock_guard const&) + std::lock_guard const&) { + if (stop_) + return false; + auto const seq {ledger->info().seq}; - assert(backend_ && !complete_); - assert(!boost::icl::contains(storedSeqs_, seq)); try { + // Update the transactions database { auto& session {txSQLiteDB_->getSession()}; soci::transaction tr(session); session << - "DELETE FROM Transactions WHERE LedgerSeq = :seq;" + "DELETE FROM Transactions " + "WHERE LedgerSeq = :seq;" , soci::use(seq); session << - "DELETE FROM AccountTransactions WHERE LedgerSeq = :seq;" + "DELETE FROM AccountTransactions " + "WHERE LedgerSeq = :seq;" , soci::use(seq); if (ledger->info().txHash.isNonZero()) @@ -579,24 +769,29 @@ Shard::setSQLiteStored( for (auto const& item : ledger->txs) { + if (stop_) + return false; + auto const txID {item.first->getTransactionID()}; auto const sTxID {to_string(txID)}; auto const txMeta {std::make_shared( txID, ledger->seq(), *item.second)}; session << - "DELETE FROM AccountTransactions WHERE TransID = :txID;" + "DELETE FROM AccountTransactions " + "WHERE TransID = :txID;" , soci::use(sTxID); auto const& accounts = txMeta->getAffectedAccounts(j_); if (!accounts.empty()) { - auto const s(boost::str(boost::format( + auto const sTxnSeq {std::to_string(txMeta->getIndex())}; + auto const s {boost::str(boost::format( "('%s','%s',%s,%s)") % sTxID % "%s" % sSeq - % std::to_string(txMeta->getIndex()))); + % sTxnSeq)}; std::string sql; sql.reserve((accounts.size() + 1) * 128); sql = "INSERT INTO AccountTransactions " @@ -638,37 +833,81 @@ Shard::setSQLiteStored( tr.commit (); } - auto& session {lgrSQLiteDB_->getSession()}; - soci::transaction tr(session); - - session << - "DELETE FROM Ledgers WHERE LedgerSeq = :seq;" - , soci::use(seq); - session << - "INSERT OR REPLACE INTO Ledgers (" - "LedgerHash, LedgerSeq, PrevHash, TotalCoins, ClosingTime," - "PrevClosingTime, CloseTimeRes, CloseFlags, AccountSetHash," - "TransSetHash)" - "VALUES (" - ":ledgerHash, :ledgerSeq, :prevHash, :totalCoins, :closingTime," - ":prevClosingTime, :closeTimeRes, :closeFlags, :accountSetHash," - ":transSetHash);", - soci::use(to_string(ledger->info().hash)), - soci::use(seq), - soci::use(to_string(ledger->info().parentHash)), - soci::use(to_string(ledger->info().drops)), - soci::use(ledger->info().closeTime.time_since_epoch().count()), - soci::use(ledger->info().parentCloseTime.time_since_epoch().count()), - soci::use(ledger->info().closeTimeResolution.count()), - soci::use(ledger->info().closeFlags), - soci::use(to_string(ledger->info().accountHash)), - soci::use(to_string(ledger->info().txHash)); - - tr.commit(); + auto const sHash {to_string(ledger->info().hash)}; + + // Update the ledger database + { + auto& session {lgrSQLiteDB_->getSession()}; + soci::transaction tr(session); + + auto const sParentHash {to_string(ledger->info().parentHash)}; + auto const sDrops {to_string(ledger->info().drops)}; + auto const sAccountHash {to_string(ledger->info().accountHash)}; + auto const sTxHash {to_string(ledger->info().txHash)}; + + session << + "DELETE FROM Ledgers " + "WHERE LedgerSeq = :seq;" + , soci::use(seq); + session << + "INSERT OR REPLACE INTO Ledgers (" + "LedgerHash, LedgerSeq, PrevHash, TotalCoins, ClosingTime," + "PrevClosingTime, CloseTimeRes, CloseFlags, AccountSetHash," + "TransSetHash)" + "VALUES (" + ":ledgerHash, :ledgerSeq, :prevHash, :totalCoins," + ":closingTime, :prevClosingTime, :closeTimeRes," + ":closeFlags, :accountSetHash, :transSetHash);", + soci::use(sHash), + soci::use(seq), + soci::use(sParentHash), + soci::use(sDrops), + soci::use(ledger->info().closeTime.time_since_epoch().count()), + soci::use( + ledger->info().parentCloseTime.time_since_epoch().count()), + soci::use(ledger->info().closeTimeResolution.count()), + soci::use(ledger->info().closeFlags), + soci::use(sAccountHash), + soci::use(sTxHash); + + tr.commit(); + } + + // Update the acquire database if present + if (acquireInfo_) + { + auto& session {acquireInfo_->SQLiteDB->getSession()}; + soci::blob sociBlob(session); + + if (!acquireInfo_->storedSeqs.empty()) + convert(to_string(acquireInfo_->storedSeqs), sociBlob); + + if (ledger->info().seq == lastSeq_) + { + // Store shard's last ledger hash + session << + "UPDATE Shard " + "SET LastLedgerHash = :lastLedgerHash," + "StoredLedgerSeqs = :storedLedgerSeqs " + "WHERE ShardIndex = :shardIndex;" + , soci::use(sHash) + , soci::use(sociBlob) + , soci::use(index_); + } + else + { + session << + "UPDATE Shard " + "SET StoredLedgerSeqs = :storedLedgerSeqs " + "WHERE ShardIndex = :shardIndex;" + , soci::use(sociBlob) + , soci::use(index_); + } + } } catch (std::exception const& e) { - JLOG(j_.error()) << + JLOG(j_.fatal()) << "shard " << index_ << " exception " << e.what() << " in function " << __func__; @@ -677,51 +916,30 @@ Shard::setSQLiteStored( return true; } -bool -Shard::setFileStats(std::lock_guard const&) +void +Shard::setFileStats(std::lock_guard const&) { fileSz_ = 0; fdRequired_ = 0; - if (backend_->backed()) + try { - try + using namespace boost::filesystem; + for (auto const& d : directory_iterator(dir_)) { - using namespace boost::filesystem; - for (auto const& d : directory_iterator(dir_)) + if (is_regular_file(d)) { - if (is_regular_file(d)) - { - fileSz_ += file_size(d); - ++fdRequired_; - } + fileSz_ += file_size(d); + ++fdRequired_; } } - catch (std::exception const& e) - { - JLOG(j_.error()) << - "shard " << index_ << - " exception " << e.what() << - " in function " << __func__; - return false; - } } - return true; -} - -bool -Shard::saveControl(std::lock_guard const&) -{ - std::ofstream ofs {control_.string(), std::ios::trunc}; - if (!ofs.is_open()) + catch (std::exception const& e) { - JLOG(j_.fatal()) << - "shard " << index_ << " is unable to save control file"; - return false; + JLOG(j_.error()) << + "shard " << index_ << + " exception " << e.what() << + " in function " << __func__; } - - boost::archive::text_oarchive ar(ofs); - ar & storedSeqs_; - return true; } bool @@ -731,25 +949,27 @@ Shard::valLedger( { auto fail = [j = j_, index = index_, &ledger](std::string const& msg) { - JLOG(j.error()) << + JLOG(j.fatal()) << "shard " << index << ". " << msg << (ledger->info().hash.isZero() ? - "" : ". Ledger header hash " + + "" : ". Ledger hash " + to_string(ledger->info().hash)) << (ledger->info().seq == 0 ? - "" : ". Ledger header sequence " + + "" : ". Ledger sequence " + std::to_string(ledger->info().seq)); return false; }; if (ledger->info().hash.isZero()) - return fail("Invalid ledger header hash"); + return fail("Invalid ledger hash"); if (ledger->info().accountHash.isZero()) - return fail("Invalid ledger header account hash"); + return fail("Invalid ledger account hash"); bool error {false}; auto visit = [this, &error](SHAMapAbstractNode& node) { + if (stop_) + return false; if (!valFetch(node.getNodeHash().as_uint256())) error = true; return !error; @@ -773,6 +993,8 @@ Shard::valLedger( return fail(std::string("exception ") + e.what() + " in function " + __func__); } + if (stop_) + return false; if (error) return fail("Invalid state map"); } @@ -792,11 +1014,14 @@ Shard::valLedger( return fail(std::string("exception ") + e.what() + " in function " + __func__); } + if (stop_) + return false; if (error) return fail("Invalid transaction map"); } + return true; -}; +} std::shared_ptr Shard::valFetch(uint256 const& hash) const @@ -804,25 +1029,22 @@ Shard::valFetch(uint256 const& hash) const std::shared_ptr nObj; auto fail = [j = j_, index = index_, &hash, &nObj](std::string const& msg) { - JLOG(j.error()) << + JLOG(j.fatal()) << "shard " << index << ". " << msg << ". Node object hash " << to_string(hash); nObj.reset(); return nObj; }; - Status status; try { - { - std::lock_guard lock(mutex_); - status = backend_->fetch(hash.begin(), &nObj); - } - - switch (status) + switch (backend_->fetch(hash.data(), &nObj)) { case ok: - break; + // This verifies that the hash of node object matches the payload + if (nObj->getHash() != sha512Half(makeSlice(nObj->getData()))) + return fail("Node object hash does not match payload"); + return nObj; case notFound: return fail("Missing node object"); case dataCorrupt: @@ -836,7 +1058,6 @@ Shard::valFetch(uint256 const& hash) const return fail(std::string("exception ") + e.what() + " in function " + __func__); } - return nObj; } } // NodeStore diff --git a/src/ripple/nodestore/impl/Shard.h b/src/ripple/nodestore/impl/Shard.h index 687de6c20c8..a75362ba05e 100644 --- a/src/ripple/nodestore/impl/Shard.h +++ b/src/ripple/nodestore/impl/Shard.h @@ -30,29 +30,12 @@ #include #include +#include +#include + namespace ripple { namespace NodeStore { -// Removes a path in its entirety -inline static -bool -removeAll( - boost::filesystem::path const& path, - beast::Journal const& j) -{ - try - { - boost::filesystem::remove_all(path); - } - catch (std::exception const& e) - { - JLOG(j.error()) << - "exception: " << e.what(); - return false; - } - return true; -} - using PCache = TaggedCache; using NCache = KeyCache; class DatabaseShard; @@ -65,7 +48,7 @@ class DatabaseShard; Public functions can be called concurrently from any thread. */ -class Shard +class Shard final { public: Shard( @@ -77,29 +60,37 @@ class Shard bool open(Scheduler& scheduler, nudb::context& ctx); - bool - setStored(std::shared_ptr const& ledger); - boost::optional prepare(); bool - contains(std::uint32_t seq) const; + store(std::shared_ptr const& ledger); + + bool + containsLedger(std::uint32_t seq) const; void sweep(); std::uint32_t - index() const - { - return index_; - } + index() const {return index_;} + + boost::filesystem::path const& + getDir() const {return dir_;} + + std::tuple< + std::shared_ptr, + std::shared_ptr, + std::shared_ptr> + getBackendAll() const; - std::shared_ptr const& + std::shared_ptr getBackend() const; + /** Returns `true` if all shard ledgers have been stored in the backend + */ bool - complete() const; + isBackendComplete() const; std::shared_ptr pCache() const; @@ -107,36 +98,65 @@ class Shard std::shared_ptr nCache() const; - std::uint64_t - fileSize() const; + /** Returns a pair where the first item describes the storage space + utilized and the second item is the number of file descriptors required. + */ + std::pair + fileInfo() const; - std::uint32_t - fdRequired() const; + /** Returns `true` if the shard is complete, validated, and immutable. + */ + bool + isFinal() const; + + /** Returns `true` if the shard is older, without final key data + */ + bool + isLegacy() const; - std::shared_ptr - lastStored() const; + /** Finalize shard by walking its ledgers and verifying each Merkle tree. + @param writeSQLite If true, SQLite entries will be rewritten using + verified backend data. + */ bool - validate() const; + finalize(const bool writeSQLite); + + void + stop() {stop_ = true;} + + // Current shard version + static constexpr std::uint32_t version {2}; + + // The finalKey is a hard coded value of zero. It is used to store + // finalizing shard data to the backend. The data contains a version, + // last ledger's hash, and the first and last ledger sequences. + static uint256 const finalKey; private: - static constexpr auto controlFileName = "control.txt"; + struct AcquireInfo + { + // SQLite database to track information about what has been acquired + std::unique_ptr SQLiteDB; + + // Tracks the sequences of ledgers acquired and stored in the backend + RangeSet storedSeqs; + }; Application& app_; - mutable std::mutex mutex_; + mutable std::recursive_mutex mutex_; // Shard Index std::uint32_t const index_; - // First ledger sequence in this shard + // First ledger sequence in the shard std::uint32_t const firstSeq_; - // Last ledger sequence in this shard + // Last ledger sequence in the shard std::uint32_t const lastSeq_; - // The maximum number of ledgers this shard can store - // The earliest shard may store less ledgers than - // subsequent shards + // The maximum number of ledgers the shard can store + // The earliest shard may store fewer ledgers than subsequent shards std::uint32_t const maxLedgers_; // Database positive cache @@ -148,14 +168,11 @@ class Shard // Path to database files boost::filesystem::path const dir_; - // Path to control file - boost::filesystem::path const control_; - // Storage space utilized by the shard - std::uint64_t fileSz_; + std::uint64_t fileSz_ {0}; // Number of file descriptors required by the shard - std::uint32_t fdRequired_; + std::uint32_t fdRequired_ {0}; // NuDB key/value store for node objects std::shared_ptr backend_; @@ -166,58 +183,54 @@ class Shard // Transaction SQLite database used for indexes std::unique_ptr txSQLiteDB_; + // Tracking information used only when acquiring a shard from the network. + // If the shard is complete, this member will be null. + std::unique_ptr acquireInfo_; + beast::Journal const j_; - // True if shard has its entire ledger range stored - bool complete_ {false}; + // True if backend has stored all ledgers pertaining to the shard + bool backendComplete_ {false}; - // Sequences of ledgers stored with an incomplete shard - RangeSet storedSeqs_; + // Older shard without an acquire database or final key + // Eventually there will be no need for this and should be removed + bool legacy_ {false}; - // Used as an optimization for visitDifferences - std::shared_ptr lastStored_; + // True if the backend has a final key stored + bool final_ {false}; - // Marks shard immutable - // Lock over mutex_ required - bool - setComplete(std::lock_guard const& lock); + // Determines if the shard needs to stop processing for shutdown + std::atomic stop_ {false}; // Set the backend cache // Lock over mutex_ required void - setCache(std::lock_guard const& lock); + setBackendCache(std::lock_guard const& lock); // Open/Create SQLite databases // Lock over mutex_ required bool - initSQLite(std::lock_guard const& lock); + initSQLite(std::lock_guard const& lock); - // Write SQLite entries for a ledger stored in this shard's backend + // Write SQLite entries for this ledger // Lock over mutex_ required bool - setSQLiteStored( + storeSQLite( std::shared_ptr const& ledger, - std::lock_guard const& lock); + std::lock_guard const& lock); // Set storage and file descriptor usage stats // Lock over mutex_ required - bool - setFileStats(std::lock_guard const& lock); - - // Save the control file for an incomplete shard - // Lock over mutex_ required - bool - saveControl(std::lock_guard const& lock); + void + setFileStats(std::lock_guard const& lock); - // Validate this ledger by walking its SHAMaps - // and verifying each merkle tree + // Validate this ledger by walking its SHAMaps and verifying Merkle trees bool valLedger( std::shared_ptr const& ledger, std::shared_ptr const& next) const; - // Fetches from the backend and will log - // errors based on status codes + // Fetches from backend and log errors based on status codes std::shared_ptr valFetch(uint256 const& hash) const; }; diff --git a/src/ripple/nodestore/impl/TaskQueue.cpp b/src/ripple/nodestore/impl/TaskQueue.cpp new file mode 100644 index 00000000000..1ee718679f3 --- /dev/null +++ b/src/ripple/nodestore/impl/TaskQueue.cpp @@ -0,0 +1,66 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2019 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include + +namespace ripple { +namespace NodeStore { + +TaskQueue::TaskQueue(Stoppable& parent) + : Stoppable("TaskQueue", parent) + , workers_(*this, nullptr, "Shard store taskQueue", 1) +{ +} + +void +TaskQueue::onStop() +{ + workers_.pauseAllThreadsAndWait(); + stopped(); +} + +void +TaskQueue::addTask(std::function task) +{ + std::lock_guard lock {mutex_}; + + tasks_.emplace(std::move(task)); + workers_.addTask(); +} + +void +TaskQueue::processTask(int instance) +{ + std::function task; + + { + std::lock_guard lock {mutex_}; + assert(!tasks_.empty()); + + task = std::move(tasks_.front()); + tasks_.pop(); + } + + task(); +} + +} // NodeStore +} // ripple diff --git a/src/ripple/nodestore/impl/TaskQueue.h b/src/ripple/nodestore/impl/TaskQueue.h new file mode 100644 index 00000000000..ab53ea090ed --- /dev/null +++ b/src/ripple/nodestore/impl/TaskQueue.h @@ -0,0 +1,62 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2019 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_TASKQUEUE_H_INCLUDED +#define RIPPLE_NODESTORE_TASKQUEUE_H_INCLUDED + +#include +#include + +#include +#include + +namespace ripple { +namespace NodeStore { + +class TaskQueue + : public Stoppable + , private Workers::Callback +{ +public: + explicit + TaskQueue(Stoppable& parent); + + void + onStop() override; + + /** Adds a task to the queue + + @param task std::function with signature void() + */ + void + addTask(std::function task); + +private: + std::mutex mutex_; + Workers workers_; + std::queue> tasks_; + + void + processTask(int instance) override; +}; + +} // NodeStore +} // ripple + +#endif diff --git a/src/ripple/overlay/Compression.h b/src/ripple/overlay/Compression.h new file mode 100644 index 00000000000..efec6b524e0 --- /dev/null +++ b/src/ripple/overlay/Compression.h @@ -0,0 +1,103 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLED_COMPRESSION_H_INCLUDED +#define RIPPLED_COMPRESSION_H_INCLUDED + +#include +#include +#include + +namespace ripple { + +namespace compression { + +std::size_t constexpr headerBytes = 6; +std::size_t constexpr headerBytesCompressed = 10; + +enum class Algorithm : std::uint8_t { + None = 0x00, + LZ4 = 0x01 +}; + +enum class Compressed : std::uint8_t { + On, + Off +}; + +/** Decompress input stream. + * @tparam InputStream ZeroCopyInputStream + * @param in Input source stream + * @param inSize Size of compressed data + * @param decompressed Buffer to hold decompressed message + * @param algorithm Compression algorithm type + * @return Size of decompressed data or zero if failed to decompress + */ +template +std::size_t +decompress(InputStream& in, std::size_t inSize, std::uint8_t* decompressed, + std::size_t decompressedSize, Algorithm algorithm = Algorithm::LZ4) { + try + { + if (algorithm == Algorithm::LZ4) + return ripple::compression_algorithms::lz4Decompress(in, inSize, + decompressed, decompressedSize); + else + { + JLOG(debugLog().warn()) << "decompress: invalid compression algorithm " + << static_cast(algorithm); + assert(0); + } + } + catch (...) {} + return 0; +} + +/** Compress input data. + * @tparam BufferFactory Callable object or lambda. + * Takes the requested buffer size and returns allocated buffer pointer. + * @param in Data to compress + * @param inSize Size of the data + * @param bf Compressed buffer allocator + * @param algorithm Compression algorithm type + * @return Size of compressed data, or zero if failed to compress + */ +template +std::size_t +compress(void const* in, + std::size_t inSize, BufferFactory&& bf, Algorithm algorithm = Algorithm::LZ4) { + try + { + if (algorithm == Algorithm::LZ4) + return ripple::compression_algorithms::lz4Compress(in, inSize, std::forward(bf)); + else + { + JLOG(debugLog().warn()) << "compress: invalid compression algorithm" + << static_cast(algorithm); + assert(0); + } + } + catch (...) {} + return 0; +} +} // compression + +} // ripple + +#endif //RIPPLED_COMPRESSION_H_INCLUDED diff --git a/src/ripple/overlay/Message.h b/src/ripple/overlay/Message.h index e186272b3bf..b3ae036a500 100644 --- a/src/ripple/overlay/Message.h +++ b/src/ripple/overlay/Message.h @@ -20,6 +20,7 @@ #ifndef RIPPLE_OVERLAY_MESSAGE_H_INCLUDED #define RIPPLE_OVERLAY_MESSAGE_H_INCLUDED +#include #include #include #include @@ -47,27 +48,61 @@ namespace ripple { class Message : public std::enable_shared_from_this { + using Compressed = compression::Compressed; + using Algorithm = compression::Algorithm; public: + /** Constructor + * @param message Protocol message to serialize + * @param type Protocol message type + */ Message (::google::protobuf::Message const& message, int type); -public: - /** Retrieve the packed message data. */ + /** Retrieve the packed message data. If compressed message is requested but the message + * is not compressible then the uncompressed buffer is returned. + * @param compressed Request compressed (Compress::On) or + * uncompressed (Compress::Off) payload buffer + * @return Payload buffer + */ std::vector const& - getBuffer () const - { - return mBuffer; - } + getBuffer (Compressed tryCompressed); /** Get the traffic category */ std::size_t getCategory () const { - return mCategory; + return category_; } private: - std::vector mBuffer; - std::size_t mCategory; + std::vector buffer_; + std::vector bufferCompressed_; + std::size_t category_; + std::once_flag once_flag_; + + /** Set the payload header + * @param in Pointer to the payload + * @param payloadBytes Size of the payload excluding the header size + * @param type Protocol message type + * @param comprAlgorithm Compression algorithm used in compression, + * currently LZ4 only. If None then the message is uncompressed. + * @param uncompressedBytes Size of the uncompressed message + */ + void setHeader(std::uint8_t* in, std::uint32_t payloadBytes, int type, + Algorithm comprAlgorithm, std::uint32_t uncompressedBytes); + + /** Try to compress the payload. + * Can be called concurrently by multiple peers but is compressed once. + * If the message is not compressible then the serialized buffer_ is used. + */ + void compress(); + + /** Get the message type from the payload header. + * First four bytes are the compression/algorithm flag and the payload size. + * Next two bytes are the message type + * @param in Payload header pointer + * @return Message type + */ + int getType(std::uint8_t const* in) const; }; } diff --git a/src/ripple/overlay/impl/ConnectAttempt.cpp b/src/ripple/overlay/impl/ConnectAttempt.cpp index 2c79e7ccc24..4cd38715fcc 100644 --- a/src/ripple/overlay/impl/ConnectAttempt.cpp +++ b/src/ripple/overlay/impl/ConnectAttempt.cpp @@ -197,7 +197,7 @@ ConnectAttempt::onHandshake (error_code ec) if (! sharedValue) return close(); // makeSharedValue logs - req_ = makeRequest(!overlay_.peerFinder().config().peerPrivate); + req_ = makeRequest(!overlay_.peerFinder().config().peerPrivate, app_.config().COMPRESSION); buildHandshake(req_, *sharedValue, overlay_.setup().networkID, overlay_.setup().public_ip, remote_endpoint_.address(), app_); @@ -264,7 +264,7 @@ ConnectAttempt::onShutdown (error_code ec) //-------------------------------------------------------------------------- auto -ConnectAttempt::makeRequest (bool crawl) -> request_type +ConnectAttempt::makeRequest (bool crawl, bool compressionEnabled) -> request_type { request_type m; m.method(boost::beast::http::verb::get); @@ -275,6 +275,8 @@ ConnectAttempt::makeRequest (bool crawl) -> request_type m.insert ("Connection", "Upgrade"); m.insert ("Connect-As", "Peer"); m.insert ("Crawl", crawl ? "public" : "private"); + if (compressionEnabled) + m.insert("X-Offer-Compression", "lz4"); return m; } diff --git a/src/ripple/overlay/impl/ConnectAttempt.h b/src/ripple/overlay/impl/ConnectAttempt.h index 601207de13d..5464ae8cdfd 100644 --- a/src/ripple/overlay/impl/ConnectAttempt.h +++ b/src/ripple/overlay/impl/ConnectAttempt.h @@ -93,7 +93,7 @@ class ConnectAttempt static request_type - makeRequest (bool crawl); + makeRequest (bool crawl, bool compressionEnabled); void processResponse(); diff --git a/src/ripple/overlay/impl/Message.cpp b/src/ripple/overlay/impl/Message.cpp index acf201e49e9..edac6b41248 100644 --- a/src/ripple/overlay/impl/Message.cpp +++ b/src/ripple/overlay/impl/Message.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include #include @@ -25,8 +24,9 @@ namespace ripple { Message::Message (::google::protobuf::Message const& message, int type) - : mCategory(TrafficCount::categorize(message, type, false)) + : category_(TrafficCount::categorize(message, type, false)) { + using namespace ripple::compression; #if defined(GOOGLE_PROTOBUF_VERSION) && (GOOGLE_PROTOBUF_VERSION >= 3011000) auto const messageBytes = message.ByteSizeLong (); @@ -36,23 +36,129 @@ Message::Message (::google::protobuf::Message const& message, int type) assert (messageBytes != 0); - /** Number of bytes in a message header. */ - std::size_t constexpr headerBytes = 6; + buffer_.resize (headerBytes + messageBytes); - mBuffer.resize (headerBytes + messageBytes); + setHeader(buffer_.data(), messageBytes, type, Algorithm::None, 0); - auto ptr = mBuffer.data(); + if (messageBytes != 0) + message.SerializeToArray(buffer_.data() + headerBytes, messageBytes); +} + +void +Message::compress() +{ + using namespace ripple::compression; + auto const messageBytes = buffer_.size () - headerBytes; - *ptr++ = static_cast((messageBytes >> 24) & 0xFF); - *ptr++ = static_cast((messageBytes >> 16) & 0xFF); - *ptr++ = static_cast((messageBytes >> 8) & 0xFF); - *ptr++ = static_cast(messageBytes & 0xFF); + auto type = getType(buffer_.data()); - *ptr++ = static_cast((type >> 8) & 0xFF); - *ptr++ = static_cast (type & 0xFF); + bool const compressible = [&]{ + if (messageBytes <= 70) + return false; + switch(type) + { + case protocol::mtMANIFESTS: + case protocol::mtENDPOINTS: + case protocol::mtTRANSACTION: + case protocol::mtGET_LEDGER: + case protocol::mtLEDGER_DATA: + case protocol::mtGET_OBJECTS: + case protocol::mtVALIDATORLIST: + return true; + case protocol::mtPING: + case protocol::mtCLUSTER: + case protocol::mtPROPOSE_LEDGER: + case protocol::mtSTATUS_CHANGE: + case protocol::mtHAVE_SET: + case protocol::mtVALIDATION: + case protocol::mtGET_SHARD_INFO: + case protocol::mtSHARD_INFO: + case protocol::mtGET_PEER_SHARD_INFO: + case protocol::mtPEER_SHARD_INFO: + break; + } + return false; + }(); - if (messageBytes != 0) - message.SerializeToArray(ptr, messageBytes); + if (compressible) + { + auto payload = static_cast(buffer_.data() + headerBytes); + + auto compressedSize = ripple::compression::compress( + payload, + messageBytes, + [&](std::size_t inSize) { // size of required compressed buffer + bufferCompressed_.resize(inSize + headerBytesCompressed); + return (bufferCompressed_.data() + headerBytesCompressed); + }); + + if (compressedSize < (messageBytes - (headerBytesCompressed - headerBytes))) + { + bufferCompressed_.resize(headerBytesCompressed + compressedSize); + setHeader(bufferCompressed_.data(), compressedSize, type, Algorithm::LZ4, messageBytes); + } + else + bufferCompressed_.resize(0); + } +} + +/** Set payload header + * Uncompressed message header + * 47-42 Set to 0 + * 41-16 Payload size + * 15-0 Message Type + * Compressed message header + * 79 Set to 0, indicates the message is compressed + * 78-76 Compression algorithm, value 1-7. Set to 1 to indicate LZ4 compression + * 75-74 Set to 0 + * 73-48 Payload size + * 47-32 Message Type + * 31-0 Uncompressed message size +*/ +void +Message::setHeader(std::uint8_t* in, std::uint32_t payloadBytes, int type, + Algorithm comprAlgorithm, std::uint32_t uncompressedBytes) +{ + auto h = in; + + auto pack = [](std::uint8_t*& in, std::uint32_t size) { + *in++ = static_cast((size >> 24) & 0x0F); // leftmost 4 are compression bits + *in++ = static_cast((size >> 16) & 0xFF); + *in++ = static_cast((size >> 8) & 0xFF); + *in++ = static_cast(size & 0xFF); + }; + + pack(in, payloadBytes); + + *in++ = static_cast((type >> 8) & 0xFF); + *in++ = static_cast (type & 0xFF); + + if (comprAlgorithm != Algorithm::None) + { + pack(in, uncompressedBytes); + *h |= 0x80 | (static_cast(comprAlgorithm) << 4); + } +} + +std::vector const& +Message::getBuffer (Compressed tryCompressed) +{ + if (tryCompressed == Compressed::Off) + return buffer_; + + std::call_once(once_flag_, &Message::compress, this); + + if (bufferCompressed_.size() > 0) + return bufferCompressed_; + else + return buffer_; +} + +int +Message::getType(std::uint8_t const* in) const +{ + int type = (static_cast(*(in + 4)) << 8) + *(in + 5); + return type; } } diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index 122e3dd197c..0733bfe99ad 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -86,6 +86,7 @@ PeerImp::PeerImp (Application& app, id_t id, , slot_ (slot) , request_(std::move(request)) , headers_(request_) + , compressionEnabled_(headers_["X-Offer-Compression"] == "lz4" ? Compressed::On : Compressed::Off) { } @@ -219,7 +220,7 @@ PeerImp::send (std::shared_ptr const& m) overlay_.reportTraffic ( safe_cast(m->getCategory()), - false, static_cast(m->getBuffer().size())); + false, static_cast(m->getBuffer(compressionEnabled_).size())); auto sendq_size = send_queue_.size(); @@ -246,7 +247,7 @@ PeerImp::send (std::shared_ptr const& m) boost::asio::async_write( stream_, - boost::asio::buffer(send_queue_.front()->getBuffer()), + boost::asio::buffer(send_queue_.front()->getBuffer(compressionEnabled_)), bind_executor( strand_, std::bind( @@ -431,7 +432,7 @@ PeerImp::hasLedger (uint256 const& hash, std::uint32_t seq) const return true; } - return seq >= app_.getNodeStore().earliestSeq() && + return seq >= app_.getNodeStore().earliestLedgerSeq() && hasShard(NodeStore::seqToShardIndex(seq)); } @@ -757,6 +758,8 @@ PeerImp::makeResponse (bool crawl, resp.insert("Connect-As", "Peer"); resp.insert("Server", BuildInfo::getFullVersionString()); resp.insert("Crawl", crawl ? "public" : "private"); + if (req["X-Offer-Compression"] == "lz4" && app_.config().COMPRESSION) + resp.insert("X-Offer-Compression", "lz4"); buildHandshake(resp, sharedValue, overlay_.setup().networkID, overlay_.setup().public_ip, remote_ip, app_); @@ -945,7 +948,7 @@ PeerImp::onWriteMessage (error_code ec, std::size_t bytes_transferred) // Timeout on writes only return boost::asio::async_write( stream_, - boost::asio::buffer(send_queue_.front()->getBuffer()), + boost::asio::buffer(send_queue_.front()->getBuffer(compressionEnabled_)), bind_executor( strand_, std::bind( @@ -1259,6 +1262,9 @@ PeerImp::onMessage(std::shared_ptr const& m) // Parse the shard indexes received in the shard info RangeSet shardIndexes; { + if (!from_string(shardIndexes, m->shardindexes())) + return badData("Invalid shard indexes"); + std::uint32_t earliestShard; boost::optional latestShard; { @@ -1267,70 +1273,23 @@ PeerImp::onMessage(std::shared_ptr const& m) if (auto shardStore = app_.getShardStore()) { earliestShard = shardStore->earliestShardIndex(); - if (curLedgerSeq >= shardStore->earliestSeq()) + if (curLedgerSeq >= shardStore->earliestLedgerSeq()) latestShard = shardStore->seqToShardIndex(curLedgerSeq); } else { - auto const earliestSeq {app_.getNodeStore().earliestSeq()}; - earliestShard = NodeStore::seqToShardIndex(earliestSeq); - if (curLedgerSeq >= earliestSeq) + auto const earliestLedgerSeq { + app_.getNodeStore().earliestLedgerSeq()}; + earliestShard = NodeStore::seqToShardIndex(earliestLedgerSeq); + if (curLedgerSeq >= earliestLedgerSeq) latestShard = NodeStore::seqToShardIndex(curLedgerSeq); } } - auto getIndex = [this, &earliestShard, &latestShard] - (std::string const& s) -> boost::optional + if (boost::icl::first(shardIndexes) < earliestShard || + (latestShard && boost::icl::last(shardIndexes) > latestShard)) { - std::uint32_t shardIndex; - if (!beast::lexicalCastChecked(shardIndex, s)) - { - fee_ = Resource::feeBadData; - return boost::none; - } - if (shardIndex < earliestShard || - (latestShard && shardIndex > latestShard)) - { - fee_ = Resource::feeBadData; - JLOG(p_journal_.error()) << - "Invalid shard index " << shardIndex; - return boost::none; - } - return shardIndex; - }; - - std::vector tokens; - boost::split(tokens, m->shardindexes(), - boost::algorithm::is_any_of(",")); - std::vector indexes; - for (auto const& t : tokens) - { - indexes.clear(); - boost::split(indexes, t, boost::algorithm::is_any_of("-")); - switch (indexes.size()) - { - case 1: - { - auto const first {getIndex(indexes.front())}; - if (!first) - return; - shardIndexes.insert(*first); - break; - } - case 2: - { - auto const first {getIndex(indexes.front())}; - if (!first) - return; - auto const second {getIndex(indexes.back())}; - if (!second) - return; - shardIndexes.insert(range(*first, *second)); - break; - } - default: - return badData("Invalid shard indexes"); - } + return badData("Invalid shard indexes"); } } @@ -1340,7 +1299,7 @@ PeerImp::onMessage(std::shared_ptr const& m) { if (m->endpoint() != "0") { - auto result = + auto result = beast::IP::Endpoint::from_string_checked(m->endpoint()); if (!result) return badData("Invalid incoming endpoint: " + m->endpoint()); @@ -2268,7 +2227,7 @@ PeerImp::onMessage (std::shared_ptr const& m) { if (auto shardStore = app_.getShardStore()) { - if (seq >= shardStore->earliestSeq()) + if (seq >= shardStore->earliestLedgerSeq()) hObj = shardStore->fetch(hash, seq); } } @@ -2714,7 +2673,7 @@ PeerImp::getLedger (std::shared_ptr const& m) if (auto shardStore = app_.getShardStore()) { auto seq = packet.ledgerseq(); - if (seq >= shardStore->earliestSeq()) + if (seq >= shardStore->earliestLedgerSeq()) ledger = shardStore->fetchLedger(ledgerhash, seq); } } diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index 559acb26bc0..0d484517160 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -99,6 +99,7 @@ class PeerImp using address_type = boost::asio::ip::address; using endpoint_type = boost::asio::ip::tcp::endpoint; using waitable_timer = boost::asio::basic_waitable_timer; + using Compressed = compression::Compressed; Application& app_; id_t const id_; @@ -201,6 +202,8 @@ class PeerImp std::mutex mutable shardInfoMutex_; hash_map shardInfo_; + Compressed compressionEnabled_ = Compressed::Off; + friend class OverlayImpl; class Metrics { @@ -600,6 +603,9 @@ PeerImp::PeerImp (Application& app, std::unique_ptr&& stream_ptr, , slot_ (std::move(slot)) , response_(std::move(response)) , headers_(response_) + , compressionEnabled_( + headers_["X-Offer-Compression"] == "lz4" && app_.config().COMPRESSION + ? Compressed::On : Compressed::Off) { read_buffer_.commit (boost::asio::buffer_copy(read_buffer_.prepare( boost::asio::buffer_size(buffers)), buffers)); diff --git a/src/ripple/overlay/impl/ProtocolMessage.h b/src/ripple/overlay/impl/ProtocolMessage.h index 8bc8ef77bb9..be4a93fbb4a 100644 --- a/src/ripple/overlay/impl/ProtocolMessage.h +++ b/src/ripple/overlay/impl/ProtocolMessage.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -81,36 +82,66 @@ struct MessageHeader /** The size of the payload on the wire. */ std::uint32_t payload_wire_size = 0; + /** Uncompressed message size if the message is compressed. */ + std::uint32_t uncompressed_size = 0; + /** The type of the message. */ std::uint16_t message_type = 0; + + /** Indicates which compression algorithm the payload is compressed with. + * Currenly only lz4 is supported. If None then the message is not compressed. + */ + compression::Algorithm algorithm = compression::Algorithm::None; }; +template +auto +buffersBegin(BufferSequence const &bufs) +{ + return boost::asio::buffers_iterator::begin(bufs); +} + template boost::optional parseMessageHeader( BufferSequence const& bufs, std::size_t size) { - auto iter = boost::asio::buffers_iterator::begin(bufs); + using namespace ripple::compression; + auto iter = buffersBegin(bufs); MessageHeader hdr; + auto const compressed = (*iter & 0x80) == 0x80; - // Version 1 header: uncompressed payload. - // The top six bits of the first byte are 0. - if ((*iter & 0xFC) == 0) + // Check valid header + if ((*iter & 0xFC) == 0 || compressed) { - hdr.header_size = 6; + hdr.header_size = compressed ? headerBytesCompressed : headerBytes; if (size < hdr.header_size) return {}; + if (compressed) + { + uint8_t algorithm = (*iter & 0x70) >> 4; + if (algorithm != static_cast(compression::Algorithm::LZ4)) + return {}; + hdr.algorithm = compression::Algorithm::LZ4; + } + for (int i = 0; i != 4; ++i) hdr.payload_wire_size = (hdr.payload_wire_size << 8) + *iter++; + // clear the compression bits + hdr.payload_wire_size &= 0x03FFFFFF; hdr.total_wire_size = hdr.header_size + hdr.payload_wire_size; for (int i = 0; i != 2; ++i) hdr.message_type = (hdr.message_type << 8) + *iter++; + if (compressed) + for (int i = 0; i != 4; ++i) + hdr.uncompressed_size = (hdr.uncompressed_size << 8) + *iter++; + return hdr; } @@ -130,7 +161,22 @@ invoke ( ZeroCopyInputStream stream(buffers); stream.Skip(header.header_size); - if (! m->ParseFromZeroCopyStream(&stream)) + if (header.algorithm != compression::Algorithm::None) + { + std::vector payload; + payload.resize(header.uncompressed_size); + + auto payloadSize = ripple::compression::decompress( + stream, + header.payload_wire_size, + payload.data(), + header.uncompressed_size, + header.algorithm); + + if (payloadSize == 0 || !m->ParseFromArray(payload.data(), payloadSize)) + return false; + } + else if (!m->ParseFromZeroCopyStream(&stream)) return false; handler.onMessageBegin (header.message_type, m, header.payload_wire_size); diff --git a/src/ripple/protocol/Feature.h b/src/ripple/protocol/Feature.h index d11a7e3e649..3a7ee97862d 100644 --- a/src/ripple/protocol/Feature.h +++ b/src/ripple/protocol/Feature.h @@ -111,6 +111,7 @@ class FeatureCollections // fixQualityUpperBound should be activated before FlowCross "fixQualityUpperBound", "RequireFullyCanonicalSig", + "fix1781", // XRPEndpointSteps should be included in the circular payment check }; std::vector features; @@ -399,6 +400,7 @@ extern uint256 const fixPayChanRecipientOwnerDir; extern uint256 const featureDeletableAccounts; extern uint256 const fixQualityUpperBound; extern uint256 const featureRequireFullyCanonicalSig; +extern uint256 const fix1781; } // ripple diff --git a/src/ripple/protocol/impl/BuildInfo.cpp b/src/ripple/protocol/impl/BuildInfo.cpp index 4872671e8d3..57f0e401341 100644 --- a/src/ripple/protocol/impl/BuildInfo.cpp +++ b/src/ripple/protocol/impl/BuildInfo.cpp @@ -31,7 +31,7 @@ namespace BuildInfo { // The build version number. You must edit this for each release // and follow the format described at http://semver.org/ //------------------------------------------------------------------------------ -char const* const versionString = "1.5.0" +char const* const versionString = "1.6.0-b1" #if defined(DEBUG) || defined(SANITIZER) "+" diff --git a/src/ripple/protocol/impl/Feature.cpp b/src/ripple/protocol/impl/Feature.cpp index db96583a058..bac6e2e8f33 100644 --- a/src/ripple/protocol/impl/Feature.cpp +++ b/src/ripple/protocol/impl/Feature.cpp @@ -130,6 +130,7 @@ detail::supportedAmendments () "DeletableAccounts", "fixQualityUpperBound", "RequireFullyCanonicalSig", + "fix1781", }; return supported; } @@ -189,5 +190,6 @@ uint256 const fixPayChanRecipientOwnerDir = *getRegisteredFeature("fixPayChanRec uint256 const featureDeletableAccounts = *getRegisteredFeature("DeletableAccounts"); uint256 const fixQualityUpperBound = *getRegisteredFeature("fixQualityUpperBound"); uint256 const featureRequireFullyCanonicalSig = *getRegisteredFeature("RequireFullyCanonicalSig"); +uint256 const fix1781 = *getRegisteredFeature("fix1781"); } // ripple diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index e14b15e28de..46608ed4d5a 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -554,7 +554,6 @@ JSS ( url_password ); // in: Subscribe JSS ( url_username ); // in: Subscribe JSS ( urlgravatar ); // JSS ( username ); // in: Subscribe -JSS ( validate ); // in: DownloadShard JSS ( validated ); // out: NetworkOPs, RPCHelpers, AccountTx* // Tx JSS ( validator_list_expires ); // out: NetworkOps, ValidatorList diff --git a/src/ripple/rpc/ShardArchiveHandler.h b/src/ripple/rpc/ShardArchiveHandler.h index ed7bc2a5cea..73ec512adaf 100644 --- a/src/ripple/rpc/ShardArchiveHandler.h +++ b/src/ripple/rpc/ShardArchiveHandler.h @@ -23,28 +23,76 @@ #include #include #include -#include +#include #include #include namespace ripple { +namespace test { class ShardArchiveHandler_test; } namespace RPC { /** Handles the download and import one or more shard archives. */ class ShardArchiveHandler - : public std::enable_shared_from_this + : public Stoppable + , public std::enable_shared_from_this { public: + + using pointer = std::shared_ptr; + friend class test::ShardArchiveHandler_test; + + static + boost::filesystem::path + getDownloadDirectory(Config const& config); + + static + pointer + getInstance(); + + static + pointer + getInstance(Application& app, Stoppable& parent); + + static + pointer + recoverInstance(Application& app, Stoppable& parent); + + static + bool + hasInstance(); + + bool + init(); + + bool + initFromDB(); + + ~ShardArchiveHandler() = default; + + bool + add(std::uint32_t shardIndex, std::pair&& url); + + /** Starts downloading and importing archives. */ + bool + start(); + + void + release(); + +private: + ShardArchiveHandler() = delete; ShardArchiveHandler(ShardArchiveHandler const&) = delete; ShardArchiveHandler& operator= (ShardArchiveHandler&&) = delete; ShardArchiveHandler& operator= (ShardArchiveHandler const&) = delete; - /** @param validate if shard data should be verified with network. */ - ShardArchiveHandler(Application& app, bool validate); + ShardArchiveHandler( + Application& app, + Stoppable& parent, + bool recovery = false); - ~ShardArchiveHandler(); + void onStop () override; /** Add an archive to be downloaded and imported. @param shardIndex the index of the shard to be imported. @@ -53,13 +101,9 @@ class ShardArchiveHandler @note Returns false if called while downloading. */ bool - add(std::uint32_t shardIndex, parsedURL&& url); - - /** Starts downloading and importing archives. */ - bool - start(); + add(std::uint32_t shardIndex, parsedURL&& url, + std::lock_guard const&); -private: // Begins the download and import of the next archive. bool next(std::lock_guard& l); @@ -76,15 +120,21 @@ class ShardArchiveHandler void remove(std::lock_guard&); + void + doRelease(std::lock_guard const&); + + static std::mutex instance_mutex_; + static pointer instance_; + std::mutex mutable m_; Application& app_; - std::shared_ptr downloader_; + beast::Journal const j_; + std::unique_ptr sqliteDB_; + std::shared_ptr downloader_; boost::filesystem::path const downloadDir_; - bool const validate_; boost::asio::basic_waitable_timer timer_; bool process_; std::map archives_; - beast::Journal const j_; }; } // RPC diff --git a/src/ripple/rpc/handlers/DownloadShard.cpp b/src/ripple/rpc/handlers/DownloadShard.cpp index 6b943f665b5..02ff8e6df8e 100644 --- a/src/ripple/rpc/handlers/DownloadShard.cpp +++ b/src/ripple/rpc/handlers/DownloadShard.cpp @@ -34,7 +34,6 @@ namespace ripple { /** RPC command that downloads and import shard archives. { shards: [{index: , url: }] - validate: // optional, default is true } example: @@ -78,7 +77,7 @@ doDownloadShard(RPC::JsonContext& context) // Validate shards static const std::string ext {".tar.lz4"}; - std::map archives; + std::map> archives; for (auto& it : context.params[jss::shards]) { // Validate the index @@ -95,7 +94,8 @@ doDownloadShard(RPC::JsonContext& context) if (!it.isMember(jss::url)) return RPC::missing_field_error(jss::url); parsedURL url; - if (!parseUrl(url, it[jss::url].asString()) || + auto unparsedURL = it[jss::url].asString(); + if (!parseUrl(url, unparsedURL) || url.domain.empty() || url.path.empty()) { return RPC::invalid_field_error(jss::url); @@ -117,27 +117,39 @@ doDownloadShard(RPC::JsonContext& context) } // Check for duplicate indexes - if (!archives.emplace(jv.asUInt(), std::move(url)).second) + if (!archives.emplace(jv.asUInt(), + std::make_pair(std::move(url), unparsedURL)).second) { return RPC::make_param_error("Invalid field '" + std::string(jss::index) + "', duplicate shard ids."); } } - bool validate {true}; - if (context.params.isMember(jss::validate)) + RPC::ShardArchiveHandler::pointer handler; + + try { - if (!context.params[jss::validate].isBool()) - { - return RPC::expected_field_error( - std::string(jss::validate), "a bool"); - } - validate = context.params[jss::validate].asBool(); + handler = RPC::ShardArchiveHandler::hasInstance() ? + RPC::ShardArchiveHandler::getInstance() : + RPC::ShardArchiveHandler::getInstance( + context.app, + context.app.getJobQueue()); + + if(!handler) + return RPC::make_error (rpcINTERNAL, + "Failed to create ShardArchiveHandler."); + + if(!handler->init()) + return RPC::make_error (rpcINTERNAL, + "Failed to initiate ShardArchiveHandler."); + } + catch (std::exception const& e) + { + return RPC::make_error (rpcINTERNAL, + std::string("Failed to start download: ") + + e.what()); } - // Begin downloading. The handler keeps itself alive while downloading. - auto handler { - std::make_shared(context.app, validate)}; for (auto& [index, url] : archives) { if (!handler->add(index, std::move(url))) @@ -147,8 +159,13 @@ doDownloadShard(RPC::JsonContext& context) std::to_string(index) + " exists or being acquired"); } } + + // Begin downloading. if (!handler->start()) + { + handler->release(); return rpcError(rpcINTERNAL); + } std::string s {"Downloading shard"}; preShards = shardStore->getPreShards(); diff --git a/src/ripple/rpc/handlers/Handlers.h b/src/ripple/rpc/handlers/Handlers.h index 805e156402a..edf0acbdf57 100644 --- a/src/ripple/rpc/handlers/Handlers.h +++ b/src/ripple/rpc/handlers/Handlers.h @@ -61,10 +61,12 @@ Json::Value doManifest (RPC::JsonContext&); Json::Value doNoRippleCheck (RPC::JsonContext&); Json::Value doOwnerInfo (RPC::JsonContext&); Json::Value doPathFind (RPC::JsonContext&); +Json::Value doPause (RPC::JsonContext&); Json::Value doPeers (RPC::JsonContext&); Json::Value doPing (RPC::JsonContext&); Json::Value doPrint (RPC::JsonContext&); Json::Value doRandom (RPC::JsonContext&); +Json::Value doResume (RPC::JsonContext&); Json::Value doPeerReservationsAdd (RPC::JsonContext&); Json::Value doPeerReservationsDel (RPC::JsonContext&); Json::Value doPeerReservationsList (RPC::JsonContext&); diff --git a/src/ripple/rpc/impl/ShardArchiveHandler.cpp b/src/ripple/rpc/impl/ShardArchiveHandler.cpp index dc20704c0dc..fda926ef22b 100644 --- a/src/ripple/rpc/impl/ShardArchiveHandler.cpp +++ b/src/ripple/rpc/impl/ShardArchiveHandler.cpp @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #include @@ -31,42 +34,192 @@ namespace RPC { using namespace boost::filesystem; using namespace std::chrono_literals; -ShardArchiveHandler::ShardArchiveHandler(Application& app, bool validate) - : app_(app) - , downloadDir_(get(app_.config().section( - ConfigSection::shardDatabase()), "path", "") + "/download") - , validate_(validate) +std::mutex ShardArchiveHandler::instance_mutex_; +ShardArchiveHandler::pointer ShardArchiveHandler::instance_ = nullptr; + +boost::filesystem::path +ShardArchiveHandler::getDownloadDirectory(Config const& config) +{ + return get(config.section( + ConfigSection::shardDatabase()), "download_path", + get(config.section(ConfigSection::shardDatabase()), + "path", "")) / "download"; +} + +auto +ShardArchiveHandler::getInstance() -> pointer +{ + std::lock_guard lock(instance_mutex_); + + return instance_; +} + +auto +ShardArchiveHandler::getInstance(Application& app, + Stoppable& parent) -> pointer +{ + std::lock_guard lock(instance_mutex_); + assert(!instance_); + + instance_.reset(new ShardArchiveHandler(app, parent)); + + return instance_; +} + +auto +ShardArchiveHandler::recoverInstance(Application& app, Stoppable& parent) -> pointer +{ + std::lock_guard lock(instance_mutex_); + assert(!instance_); + + instance_.reset(new ShardArchiveHandler(app, parent, true)); + + return instance_; +} + +bool +ShardArchiveHandler::hasInstance() +{ + std::lock_guard lock(instance_mutex_); + + return instance_.get() != nullptr; +} + +ShardArchiveHandler::ShardArchiveHandler( + Application& app, + Stoppable& parent, + bool recovery) + : Stoppable("ShardArchiveHandler", parent) + , app_(app) + , j_(app.journal("ShardArchiveHandler")) + , downloadDir_(getDownloadDirectory(app.config())) , timer_(app_.getIOService()) , process_(false) - , j_(app.journal("ShardArchiveHandler")) { assert(app_.getShardStore()); + + if(recovery) + downloader_.reset(new DatabaseDownloader ( + app_.getIOService(), j_, app_.config())); } -ShardArchiveHandler::~ShardArchiveHandler() +bool +ShardArchiveHandler::init() { - std::lock_guard lock(m_); - timer_.cancel(); - for (auto const& ar : archives_) - app_.getShardStore()->removePreShard(ar.first); - archives_.clear(); + try + { + create_directories(downloadDir_); - // Remove temp root download directory + sqliteDB_ = std::make_unique( + downloadDir_ , + stateDBName, + DownloaderDBPragma, + ShardArchiveHandlerDBInit); + } + catch(std::exception const& e) + { + JLOG(j_.error()) << "exception: " << e.what() + << " in function: " << __func__; + + return false; + } + + return true; +} + +bool +ShardArchiveHandler::initFromDB() +{ try { - remove_all(downloadDir_); + using namespace boost::filesystem; + + assert(exists(downloadDir_ / stateDBName) && + is_regular_file(downloadDir_ / stateDBName)); + + sqliteDB_ = std::make_unique( + downloadDir_, + stateDBName, + DownloaderDBPragma, + ShardArchiveHandlerDBInit); + + auto& session{sqliteDB_->getSession()}; + + soci::rowset rs = (session.prepare + << "SELECT * FROM State;"); + + std::lock_guard lock(m_); + + for (auto it = rs.begin(); it != rs.end(); ++it) + { + parsedURL url; + + if (!parseUrl(url, it->get(1))) + { + JLOG(j_.error()) << "Failed to parse url: " + << it->get(1); + + continue; + } + + add(it->get(0), std::move(url), lock); + } + + // Failed to load anything + // from the state database. + if(archives_.empty()) + { + release(); + return false; + } } - catch (std::exception const& e) + catch(std::exception const& e) { - JLOG(j_.error()) << - "exception: " << e.what(); + JLOG(j_.error()) << "exception: " << e.what() + << " in function: " << __func__; + + return false; } + + return true; +} + +void +ShardArchiveHandler::onStop() +{ + std::lock_guard lock(m_); + + if (downloader_) + { + downloader_->onStop(); + downloader_.reset(); + } + + stopped(); } bool -ShardArchiveHandler::add(std::uint32_t shardIndex, parsedURL&& url) +ShardArchiveHandler::add(std::uint32_t shardIndex, + std::pair&& url) +{ + std::lock_guard lock(m_); + + if (!add(shardIndex, std::forward(url.first), lock)) + return false; + + auto& session{sqliteDB_->getSession()}; + + session << "INSERT INTO State VALUES (:index, :url);", + soci::use(shardIndex), + soci::use(url.second); + + return true; +} + +bool +ShardArchiveHandler::add(std::uint32_t shardIndex, parsedURL&& url, + std::lock_guard const&) { - std::lock_guard lock(m_); if (process_) { JLOG(j_.error()) << @@ -77,9 +230,12 @@ ShardArchiveHandler::add(std::uint32_t shardIndex, parsedURL&& url) auto const it {archives_.find(shardIndex)}; if (it != archives_.end()) return url == it->second; + if (!app_.getShardStore()->prepareShard(shardIndex)) return false; + archives_.emplace(shardIndex, std::move(url)); + return true; } @@ -108,16 +264,13 @@ ShardArchiveHandler::start() try { - // Remove if remnant from a crash - remove_all(downloadDir_); - // Create temp root download directory - create_directory(downloadDir_); + create_directories(downloadDir_); if (!downloader_) { // will throw if can't initialize ssl context - downloader_ = std::make_shared( + downloader_ = std::make_shared( app_.getIOService(), j_, app_.config()); } } @@ -131,12 +284,19 @@ ShardArchiveHandler::start() return next(lock); } +void +ShardArchiveHandler::release() +{ + std::lock_guard lock(m_); + doRelease(lock); +} + bool ShardArchiveHandler::next(std::lock_guard& l) { if (archives_.empty()) { - process_ = false; + doRelease(l); return false; } @@ -155,20 +315,28 @@ ShardArchiveHandler::next(std::lock_guard& l) return next(l); } - // Download the archive + // Download the archive. Process in another thread + // to prevent holding up the lock if the downloader + // sleeps. auto const& url {archives_.begin()->second}; - if (!downloader_->download( - url.domain, - std::to_string(url.port.get_value_or(443)), - url.path, - 11, - dstDir / "archive.tar.lz4", - std::bind(&ShardArchiveHandler::complete, - shared_from_this(), std::placeholders::_1))) - { - remove(l); - return next(l); - } + app_.getJobQueue().addJob( + jtCLIENT, "ShardArchiveHandler", + [this, ptr = shared_from_this(), url, dstDir](Job&) + { + if (!downloader_->download( + url.domain, + std::to_string(url.port.get_value_or(443)), + url.path, + 11, + dstDir / "archive.tar.lz4", + std::bind(&ShardArchiveHandler::complete, + ptr, std::placeholders::_1))) + { + std::lock_guard l(m_); + remove(l); + next(l); + } + }); process_ = true; return true; @@ -207,9 +375,9 @@ ShardArchiveHandler::complete(path dstPath) jtCLIENT, "ShardArchiveHandler", [=, dstPath = std::move(dstPath), ptr = shared_from_this()](Job&) { - // If validating and not synced then defer and retry + // If not synced then defer and retry auto const mode {ptr->app_.getOPs().getOperatingMode()}; - if (ptr->validate_ && mode != OperatingMode::FULL) + if (mode != OperatingMode::FULL) { std::lock_guard lock(m_); timer_.expires_from_now(static_cast( @@ -265,7 +433,7 @@ ShardArchiveHandler::process(path const& dstPath) } // Import the shard into the shard store - if (!app_.getShardStore()->importShard(shardIndex, shardDir, validate_)) + if (!app_.getShardStore()->importShard(shardIndex, shardDir)) { JLOG(j_.error()) << "Importing shard " << shardIndex; @@ -283,6 +451,11 @@ ShardArchiveHandler::remove(std::lock_guard&) app_.getShardStore()->removePreShard(shardIndex); archives_.erase(shardIndex); + auto& session{sqliteDB_->getSession()}; + + session << "DELETE FROM State WHERE ShardIndex = :index;", + soci::use(shardIndex); + auto const dstDir {downloadDir_ / std::to_string(shardIndex)}; try { @@ -295,5 +468,37 @@ ShardArchiveHandler::remove(std::lock_guard&) } } +void +ShardArchiveHandler::doRelease(std::lock_guard const&) +{ + process_ = false; + + timer_.cancel(); + for (auto const& ar : archives_) + app_.getShardStore()->removePreShard(ar.first); + archives_.clear(); + + { + auto& session{sqliteDB_->getSession()}; + + session << "DROP TABLE State;"; + } + + sqliteDB_.reset(); + + // Remove temp root download directory + try + { + remove_all(downloadDir_); + } + catch (std::exception const& e) + { + JLOG(j_.error()) << "exception: " << e.what() + << " in function: " << __func__; + } + + downloader_.reset(); +} + } // RPC } // ripple diff --git a/src/ripple/shamap/README.md b/src/ripple/shamap/README.md index e28d3bbfc31..a04a63c5f52 100644 --- a/src/ripple/shamap/README.md +++ b/src/ripple/shamap/README.md @@ -1,189 +1,349 @@ # SHAMap Introduction # -July 2014 +March 2020 -The SHAMap is a Merkle tree (http://en.wikipedia.org/wiki/Merkle_tree). -The SHAMap is also a radix tree of radix 16 +The `SHAMap` is a Merkle tree (http://en.wikipedia.org/wiki/Merkle_tree). +The `SHAMap` is also a radix trie of radix 16 (http://en.wikipedia.org/wiki/Radix_tree). -*We need some kind of sensible summary of the SHAMap here.* +The Merkle trie data structure is important because subtrees and even the entire +tree can be compared with other trees in O(1) time by simply comparing the hashes. +This makes it very efficient to determine if two `SHAMap`s contain the same set of +transactions or account state modifications. -A given SHAMap always stores only one of three kinds of data: +The radix trie property is helpful in that a key (hash) of a transaction +or account state can be used to navigate the trie. + +A `SHAMap` is a trie with two node types: + +1. SHAMapInnerNode +2. SHAMapTreeNode + +Both of these nodes directly inherit from SHAMapAbstractNode which holds data +common to both of the node types. + +All non-leaf nodes have type SHAMapInnerNode. + +All leaf nodes have type SHAMapTreeNode. + +The root node is always a SHAMapInnerNode. + +A given `SHAMap` always stores only one of three kinds of data: * Transactions with metadata * Transactions without metadata, or * Account states. -So all of the leaf nodes of a particular SHAMap will always have a uniform -type. The inner nodes carry no data other than the hash of the nodes -beneath them. +So all of the leaf nodes of a particular `SHAMap` will always have a uniform type. +The inner nodes carry no data other than the hash of the nodes beneath them. +All nodes are owned by shared_ptrs resident in either other nodes, or in case of +the root node, a shared_ptr in the `SHAMap` itself. The use of shared_ptrs +permits more than one `SHAMap` at a time to share ownership of a node. This +occurs (for example), when a copy of a `SHAMap` is made. -## SHAMap Types ## +Copies are made with the `snapShot` function as opposed to the `SHAMap` copy +constructor. See the section on `SHAMap` creation for more details about +`snapShot`. -There are two different ways of building and using a SHAMap: +Sequence numbers are used to further customize the node ownership strategy. See +the section on sequence numbers for details on sequence numbers. - 1. A mutable SHAMap and - 2. An immutable SHAMap +![node diagram](https://user-images.githubusercontent.com/46455409/77350005-1ef12c80-6cf9-11ea-9c8d-56410f442859.png) -The distinction here is not of the classic C++ immutable-means-unchanging -sense. An immutable SHAMap contains *nodes* that are immutable. Also, -once a node has been located in an immutable SHAMap, that node is -guaranteed to persist in that SHAMap for the lifetime of the SHAMap. +## Mutability ## -So, somewhat counter-intuitively, an immutable SHAMap may grow as new nodes -are introduced. But an immutable SHAMap will never get smaller (until it -entirely evaporates when it is destroyed). Nodes, once introduced to the -immutable SHAMap, also never change their location in memory. So nodes in -an immutable SHAMap can be handled using raw pointers (if you're careful). +There are two different ways of building and using a `SHAMap`: -One consequence of this design is that an immutable SHAMap can never be -"trimmed". There is no way to identify unnecessary nodes in an immutable -SHAMap that could be removed. Once a node has been brought into the -in-memory SHAMap, that node stays in memory for the life of the SHAMap. + 1. A mutable `SHAMap` and + 2. An immutable `SHAMap` -Most SHAMaps are immutable, in the sense that they don't modify or remove -their contained nodes. +The distinction here is not of the classic C++ immutable-means-unchanging sense. + An immutable `SHAMap` contains *nodes* that are immutable. Also, once a node has +been located in an immutable `SHAMap`, that node is guaranteed to persist in that +`SHAMap` for the lifetime of the `SHAMap`. -An example where a mutable SHAMap is required is when we want to apply -transactions to the last closed ledger. To do so we'd make a mutable -snapshot of the state tree and then start applying transactions to it. -Because the snapshot is mutable, changes to nodes in the snapshot will not -affect nodes in other SHAMAps. +So, somewhat counter-intuitively, an immutable `SHAMap` may grow as new nodes are +introduced. But an immutable `SHAMap` will never get smaller (until it entirely +evaporates when it is destroyed). Nodes, once introduced to the immutable +`SHAMap`, also never change their location in memory. So nodes in an immutable +`SHAMap` can be handled using raw pointers (if you're careful). -An example using a immutable ledger would be when there's an open ledger -and some piece of code wishes to query the state of the ledger. In this -case we don't wish to change the state of the SHAMap, so we'd use an -immutable snapshot. +One consequence of this design is that an immutable `SHAMap` can never be +"trimmed". There is no way to identify unnecessary nodes in an immutable `SHAMap` +that could be removed. Once a node has been brought into the in-memory `SHAMap`, +that node stays in memory for the life of the `SHAMap`. +Most `SHAMap`s are immutable, in the sense that they don't modify or remove their +contained nodes. -## SHAMap Creation ## +An example where a mutable `SHAMap` is required is when we want to apply +transactions to the last closed ledger. To do so we'd make a mutable snapshot +of the state trie and then start applying transactions to it. Because the +snapshot is mutable, changes to nodes in the snapshot will not affect nodes in +other `SHAMap`s. + +An example using a immutable ledger would be when there's an open ledger and +some piece of code wishes to query the state of the ledger. In this case we +don't wish to change the state of the `SHAMap`, so we'd use an immutable snapshot. -A SHAMap is usually not created from vacuum. Once an initial SHAMap is -constructed, later SHAMaps are usually created by calling -snapShot(bool isMutable) on the original SHAMap(). The returned SHAMap -has the expected characteristics (mutable or immutable) based on the passed -in flag. +## Sequence numbers ## -It is cheaper to make an immutable snapshot of a SHAMap than to make a mutable -snapshot. If the SHAMap snapshot is mutable then any of the nodes that might -be modified must be copied before they are placed in the mutable map. +Both `SHAMap`s and their nodes carry a sequence number. This is simply an +unsigned number that indicates ownership or membership, or a non-membership. +`SHAMap`s sequence numbers normally start out as 1. However when a snap-shot of +a `SHAMap` is made, the copy's sequence number is 1 greater than the original. -## SHAMap Thread Safety ## +The nodes of a `SHAMap` have their own copy of a sequence number. If the `SHAMap` +is mutable, meaning it can change, then all of its nodes must have the +same sequence number as the `SHAMap` itself. This enforces an invariant that none +of the nodes are shared with other `SHAMap`s. -*This description is obsolete and needs to be rewritten.* +When a `SHAMap` needs to have a private copy of a node, not shared by any other +`SHAMap`, it first clones it and then sets the new copy to have a sequence number +equal to the `SHAMap` sequence number. The `unshareNode` is a private utility +which automates the task of first checking if the node is already sharable, and +if so, cloning it and giving it the proper sequence number. An example case +where a private copy is needed is when an inner node needs to have a child +pointer altered. Any modification to a node will require a non-shared node. -SHAMaps can be thread safe, depending on how they are used. The SHAMap -uses a SyncUnorderedMap for its storage. The SyncUnorderedMap has three -thread-safe methods: +When a `SHAMap` decides that it is safe to share a node of its own, it sets the +node's sequence number to 0 (a `SHAMap` never has a sequence number of 0). This +is done for every node in the trie when `SHAMap::walkSubTree` is executed. - * size(), - * canonicalize(), and - * retrieve() +Note that other objects in rippled also have sequence numbers (e.g. ledgers). +The `SHAMap` and node sequence numbers should not be confused with these other +sequence numbers (no relation). -As long as the SHAMap uses only those three interfaces to its storage -(the mTNByID variable [which stands for Tree Node by ID]) the SHAMap is -thread safe. +## SHAMap Creation ## +A `SHAMap` is usually not created from vacuum. Once an initial `SHAMap` is +constructed, later `SHAMap`s are usually created by calling snapShot(bool +isMutable) on the original `SHAMap`. The returned `SHAMap` has the expected +characteristics (mutable or immutable) based on the passed in flag. + +It is cheaper to make an immutable snapshot of a `SHAMap` than to make a mutable +snapshot. If the `SHAMap` snapshot is mutable then sharable nodes must be +copied before they are placed in the mutable map. + +A new `SHAMap` is created with each new ledger round. Transactions not executed +in the previous ledger populate the `SHAMap` for the new ledger. + +## Storing SHAMap data in the database ## + +When consensus is reached, the ledger is closed. As part of this process, the +`SHAMap` is stored to the database by calling `SHAMap::flushDirty`. + +Both `unshare()` and `flushDirty` walk the `SHAMap` by calling +`SHAMap::walkSubTree`. As `unshare()` walks the trie, nodes are not written to +the database, and as `flushDirty` walks the trie nodes are written to the +database. `walkSubTree` visits every node in the trie. This process must ensure +that each node is only owned by this trie, and so "unshares" as it walks each +node (from the root down). This is done in the `preFlushNode` function by +ensuring that the node has a sequence number equal to that of the `SHAMap`. If +the node doesn't, it is cloned. + +For each inner node encountered (starting with the root node), each of the +children are inspected (from 1 to 16). For each child, if it has a non-zero +sequence number (unshareable), the child is first copied. Then if the child is +an inner node, we recurse down to that node's children. Otherwise we've found a +leaf node and that node is written to the database. A count of each leaf node +that is visited is kept. The hash of the data in the leaf node is computed at +this time, and the child is reassigned back into the parent inner node just in +case the COW operation created a new pointer to this leaf node. + +After processing each node, the node is then marked as sharable again by setting +its sequence number to 0. + +After all of an inner node's children are processed, then its hash is updated +and the inner node is written to the database. Then this inner node is assigned +back into it's parent node, again in case the COW operation created a new +pointer to it. ## Walking a SHAMap ## -*We need a good description of why someone would walk a SHAMap and* -*how it works in the code* - +The private function `SHAMap::walkTowardsKey` is a good example of *how* to walk +a `SHAMap`, and the various functions that call `walkTowardsKey` are good examples +of *why* one would want to walk a `SHAMap` (e.g. `SHAMap::findKey`). +`walkTowardsKey` always starts at the root of the `SHAMap` and traverses down +through the inner nodes, looking for a leaf node along a path in the trie +designated by a `uint256`. + +As one walks the trie, one can *optionally* keep a stack of nodes that one has +passed through. This isn't necessary for walking the trie, but many clients +will use the stack after finding the desired node. For example if one is +deleting a node from the trie, the stack is handy for repairing invariants in +the trie after the deletion. + +To assist in walking the trie, `SHAMap::walkTowardsKey` uses a `SHAMapNodeID` +that identifies a node by its path from the root and its depth in the trie. The +path is just a "list" of numbers, each in the range [0 .. 15], depicting which +child was chosen at each node starting from the root. Each choice is represented +by 4 bits, and then packed in sequence into a `uint256` (such that the longest +path possible has 256 / 4 = 64 steps). The high 4 bits of the first byte +identify which child of the root is chosen, the lower 4 bits of the first byte +identify the child of that node, and so on. The `SHAMapNodeID` identifying the +root node has an ID of 0 and a depth of 0. See `SHAMapNodeID::selectBranch` for +details of how a `SHAMapNodeID` selects a "branch" (child) by indexing into its +path with its depth. + +While the current node is an inner node, traversing down the trie from the root +continues, unless the path indicates a child that does not exist. And in this +case, `nullptr` is returned to indicate no leaf node along the given path +exists. Otherwise a leaf node is found and a (non-owning) pointer to it is +returned. At each step, if a stack is requested, a +`pair, SHAMapNodeID>` is pushed onto the stack. + +When a child node is found by `selectBranch`, the traversal to that node +consists of two steps: + +1. Update the `shared_ptr` to the current node. +2. Update the `SHAMapNodeID`. + +The first step consists of several attempts to find the node in various places: + +1. In the trie itself. +2. In the node cache. +3. In the database. + +If the node is not found in the trie, then it is installed into the trie as part +of the traversal process. ## Late-arriving Nodes ## -As we noted earlier, SHAMaps (even immutable ones) may grow. If a SHAMap -is searching for a node and runs into an empty spot in the tree, then the -SHAMap looks to see if the node exists but has not yet been made part of -the map. This operation is performed in the `SHAMap::fetchNodeExternalNT()` -method. The *NT* is this case stands for 'No Throw'. +As we noted earlier, `SHAMap`s (even immutable ones) may grow. If a `SHAMap` is +searching for a node and runs into an empty spot in the trie, then the `SHAMap` +looks to see if the node exists but has not yet been made part of the map. This +operation is performed in the `SHAMap::fetchNodeNT()` method. The *NT* +is this case stands for 'No Throw'. -The `fetchNodeExternalNT()` method goes through three phases: +The `fetchNodeNT()` method goes through three phases: 1. By calling `getCache()` we attempt to locate the missing node in the - TreeNodeCache. The TreeNodeCache is a cache of immutable - SHAMapTreeNodes that are shared across all SHAMaps. + TreeNodeCache. The TreeNodeCache is a cache of immutable SHAMapTreeNodes + that are shared across all `SHAMap`s. - Any SHAMapTreeNode that is immutable has a sequence number of zero. - When a mutable SHAMap is created then its SHAMapTreeNodes are given - non-zero sequence numbers. So the `assert (ret->getSeq() == 0)` - simply confirms that the TreeNodeCache indeed gave us an immutable node. + Any SHAMapTreeNode that is immutable has a sequence number of zero + (sharable). When a mutable `SHAMap` is created then its SHAMapTreeNodes are + given non-zero sequence numbers (unsharable). But all nodes in the + TreeNodeCache are immutable, so if one is found here, its sequence number + will be 0. 2. If the node is not in the TreeNodeCache, we attempt to locate the node - in the historic data stored by the data base. The call to - to `fetch(hash)` does that work for us. + in the historic data stored by the data base. The call to to + `fetchNodeFromDB(hash)` does that work for us. - 3. Finally, if ledgerSeq_ is non-zero and we did't locate the node in the - historic data, then we call a MissingNodeHandler. + 3. Finally if a filter exists, we check if it can supply the node. This is + typically the LedgerMaster which tracks the current ledger and ledgers + in the process of closing. - The non-zero ledgerSeq_ indicates that the SHAMap is a complete map that - belongs to a historic ledger with the given (non-zero) sequence number. - So, if all expected data is always present, the MissingNodeHandler should - never be executed. +## Canonicalize ## - And, since we now know that this SHAMap does not fully represent - the data from that ledger, we set the SHAMap's sequence number to zero. +`canonicalize()` is called every time a node is introduced into the `SHAMap`. -If phase 1 returned a node, then we already know that the node is immutable. -However, if either phase 2 executes successfully, then we need to turn the -returned node into an immutable node. That's handled by the call to -`make_shared` inside the try block. That code is inside -a try block because the `fetchNodeExternalNT` method promises not to throw. -In case the constructor called by `make_shared` throws we don't want to -break our promise. +A call to `canonicalize()` stores the node in the `TreeNodeCache` if it does not +already exist in the `TreeNodeCache`. +The calls to `canonicalize()` make sure that if the resulting node is already in +the `SHAMap`, node `TreeNodeCache` or database, then we don't create duplicates +by favoring the copy already in the `TreeNodeCache`. -## Canonicalize ## +By using `canonicalize()` we manage a thread race condition where two different +threads might both recognize the lack of a SHAMapTreeNode at the same time +(during a fetch). If they both attempt to insert the node into the `SHAMap`, then +`canonicalize` makes sure that the first node in wins and the slower thread +receives back a pointer to the node inserted by the faster thread. Recall +that these two `SHAMap`s will share the same `TreeNodeCache`. + +## TreeNodeCache ## + +The `TreeNodeCache` is a `std::unordered_map` keyed on the hash of the +`SHAMap` node. The stored type consists of `shared_ptr`, +`weak_ptr`, and a time point indicating the most recent +access of this node in the cache. The time point is based on +`std::chrono::steady_clock`. + +The container uses a cryptographically secure hash that is randomly seeded. + +The `TreeNodeCache` also carries with it various data used for statistics +and logging, and a target age for the contained nodes. When the target age +for a node is exceeded, and there are no more references to the node, the +node is removed from the `TreeNodeCache`. + +## FullBelowCache ## + +This cache remembers which trie keys have all of their children resident in a +`SHAMap`. This optimizes the process of acquiring a complete trie. This is used +when creating the missing nodes list. Missing nodes are those nodes that a +`SHAMap` refers to but that are not stored in the local database. + +As a depth-first walk of a `SHAMap` is performed, if an inner node answers true to +`isFullBelow()` then it is known that none of this node's children are missing +nodes, and thus that subtree does not need to be walked. These nodes are stored +in the FullBelowCache. Subsequent walks check the FullBelowCache first when +encountering a node, and ignore that subtree if found. + +## SHAMapAbstractNode ## + +This is a base class for the two concrete node types. It holds the following +common data: + +1. A node type, one of: + a. error + b. inner + c. transaction with no metadata + d. transaction with metadata + e. account state +2. A hash +3. A sequence number + + +## SHAMapInnerNode ## + +SHAMapInnerNode publicly inherits directly from SHAMapAbstractNode. It holds +the following data: + +1. Up to 16 child nodes, each held with a shared_ptr. +2. A hash for each child. +3. A 16-bit bitset with a 1 bit set for each child that exists. +4. Flag to aid online delete and consistency with data on disk. + +## SHAMapTreeNode ## -The calls to `canonicalize()` make sure that if the resulting node is already -in the SHAMap, then we return the node that's already present -- we never -replace a pre-existing node. By using `canonicalize()` we manage a thread -race condition where two different threads might both recognize the lack of a -SHAMapTreeNode at the same time. If they both attempt to insert the node -then `canonicalize` makes sure that the first node in wins and the slower -thread receives back a pointer to the node inserted by the faster thread. +SHAMapTreeNode publicly inherits directly from SHAMapAbstractNode. It holds the +following data: -There's a problem with the current SHAMap design that `canonicalize()` -accommodates. Two different trees can have the exact same node (the same -hash value) with two different IDs. If the TreeNodeCache returns a node -with the same hash but a different ID, then we assume that the ID of the -passed-in node is 'better' than the older ID in the TreeNodeCache. So we -construct a new SHAMapTreeNode by copying the one we found in the -TreeNodeCache, but we give the new node the new ID. Then we replace the -SHAMapTreeNode in the TreeNodeCache with this newly constructed node. +1. A shared_ptr to a const SHAMapItem. -The TreeNodeCache is not subject to the rule that any node must be -resident forever. So it's okay to replace the old node with the new node. +## SHAMapItem ## -The `SHAMap::getCache()` method exhibits the same behavior. +This holds the following data: +1. uint256. The hash of the data. +2. vector. The data (transactions, account info). ## SHAMap Improvements ## -Here's a simple one: the SHAMapTreeNode::mAccessSeq member is currently not -used and could be removed. +Here's a simple one: the SHAMapTreeNode::mAccessSeq member is currently not used +and could be removed. -Here's a more important change. The tree structure is currently embedded -in the SHAMapTreeNodes themselves. It doesn't have to be that way, and -that should be fixed. +Here's a more important change. The trie structure is currently embedded in the +SHAMapTreeNodes themselves. It doesn't have to be that way, and that should be +fixed. -When we navigate the tree (say, like `SHAMap::walkTo()`) we currently -ask each node for information that we could determine locally. We know -the depth because we know how many nodes we have traversed. We know the -ID that we need because that's how we're steering. So we don't need to -store the ID in the node. The next refactor should remove all calls to -`SHAMapTreeNode::GetID()`. +When we navigate the trie (say, like `SHAMap::walkTo()`) we currently ask each +node for information that we could determine locally. We know the depth because +we know how many nodes we have traversed. We know the ID that we need because +that's how we're steering. So we don't need to store the ID in the node. The +next refactor should remove all calls to `SHAMapTreeNode::GetID()`. Then we can remove the NodeID member from SHAMapTreeNode. -Then we can change the SHAMap::mTNBtID member to be mTNByHash. +Then we can change the `SHAMap::mTNBtID` member to be `mTNByHash`. An additional possible refactor would be to have a base type, SHAMapTreeNode, -and derive from that InnerNode and LeafNode types. That would remove -some storage (the array of 16 hashes) from the LeafNodes. That refactor -would also have the effect of simplifying methods like `isLeaf()` and -`hasItem()`. +and derive from that InnerNode and LeafNode types. That would remove some +storage (the array of 16 hashes) from the LeafNodes. That refactor would also +have the effect of simplifying methods like `isLeaf()` and `hasItem()`. diff --git a/src/ripple/shamap/impl/SHAMap.cpp b/src/ripple/shamap/impl/SHAMap.cpp index 72a8b46e579..3b0817e08fd 100644 --- a/src/ripple/shamap/impl/SHAMap.cpp +++ b/src/ripple/shamap/impl/SHAMap.cpp @@ -1072,7 +1072,7 @@ SHAMap::canonicalize(SHAMapHash const& hash, std::shared_ptr assert (node->getSeq() == 0); assert (node->getNodeHash() == hash); - f_.treecache().canonicalize (hash.as_uint256(), node); + f_.treecache().canonicalize_replace_client(hash.as_uint256(), node); } void diff --git a/src/ripple/shamap/impl/SHAMapNodeID.cpp b/src/ripple/shamap/impl/SHAMapNodeID.cpp index a284a1d7b8a..9b3526e90df 100644 --- a/src/ripple/shamap/impl/SHAMapNodeID.cpp +++ b/src/ripple/shamap/impl/SHAMapNodeID.cpp @@ -112,24 +112,6 @@ SHAMapNodeID SHAMapNodeID::getChildNodeID (int m) const // Which branch would contain the specified hash int SHAMapNodeID::selectBranch (uint256 const& hash) const { -#if RIPPLE_VERIFY_NODEOBJECT_KEYS - - if (mDepth >= 64) - { - assert (false); - return -1; - } - - if ((hash & Masks(mDepth)) != mNodeID) - { - std::cerr << "selectBranch(" << getString () << std::endl; - std::cerr << " " << hash << " off branch" << std::endl; - assert (false); - return -1; // does not go under this node - } - -#endif - int branch = * (hash.begin () + (mDepth / 2)); if (mDepth & 1) diff --git a/src/test/app/Flow_test.cpp b/src/test/app/Flow_test.cpp index 9a7637fa471..bfd1460b4d4 100644 --- a/src/test/app/Flow_test.cpp +++ b/src/test/app/Flow_test.cpp @@ -1180,6 +1180,100 @@ struct Flow_test : public beast::unit_test::suite ter(temBAD_PATH)); } + void + testXRPPathLoop() + { + testcase("Circular XRP"); + + using namespace jtx; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const gw = Account("gw"); + auto const USD = gw["USD"]; + auto const EUR = gw["EUR"]; + + for (auto const withFix : {true, false}) + { + auto const feats = [&withFix]() -> FeatureBitset { + if (withFix) + return supported_amendments(); + return supported_amendments() - FeatureBitset{fix1781}; + }(); + { + // Payment path starting with XRP + Env env(*this, feats); + env.fund(XRP(10000), alice, bob, gw); + env.trust(USD(1000), alice, bob); + env.trust(EUR(1000), alice, bob); + env(pay(gw, alice, USD(100))); + env(pay(gw, alice, EUR(100))); + env.close(); + + env(offer(alice, XRP(100), USD(100)), txflags(tfPassive)); + env(offer(alice, USD(100), XRP(100)), txflags(tfPassive)); + env(offer(alice, XRP(100), EUR(100)), txflags(tfPassive)); + env.close(); + + TER const expectedTer = + withFix ? TER{temBAD_PATH_LOOP} : TER{tesSUCCESS}; + env(pay(alice, bob, EUR(1)), + path(~USD, ~XRP, ~EUR), + sendmax(XRP(1)), + txflags(tfNoRippleDirect), + ter(expectedTer)); + } + pass(); + } + { + // Payment path ending with XRP + Env env(*this); + env.fund(XRP(10000), alice, bob, gw); + env.trust(USD(1000), alice, bob); + env.trust(EUR(1000), alice, bob); + env(pay(gw, alice, USD(100))); + env(pay(gw, alice, EUR(100))); + env.close(); + + env(offer(alice, XRP(100), USD(100)), txflags(tfPassive)); + env(offer(alice, EUR(100), XRP(100)), txflags(tfPassive)); + env.close(); + // EUR -> //XRP -> //USD ->XRP + env(pay(alice, bob, XRP(1)), + path(~XRP, ~USD, ~XRP), + sendmax(EUR(1)), + txflags(tfNoRippleDirect), + ter(temBAD_PATH_LOOP)); + } + { + // Payment where loop is formed in the middle of the path, not on an + // endpoint + auto const JPY = gw["JPY"]; + Env env(*this); + env.fund(XRP(10000), alice, bob, gw); + env.close(); + env.trust(USD(1000), alice, bob); + env.trust(EUR(1000), alice, bob); + env.trust(JPY(1000), alice, bob); + env.close(); + env(pay(gw, alice, USD(100))); + env(pay(gw, alice, EUR(100))); + env(pay(gw, alice, JPY(100))); + env.close(); + + env(offer(alice, USD(100), XRP(100)), txflags(tfPassive)); + env(offer(alice, XRP(100), EUR(100)), txflags(tfPassive)); + env(offer(alice, EUR(100), XRP(100)), txflags(tfPassive)); + env(offer(alice, XRP(100), JPY(100)), txflags(tfPassive)); + env.close(); + + env(pay(alice, bob, JPY(1)), + path(~XRP, ~EUR, ~XRP, ~JPY), + sendmax(USD(1)), + txflags(tfNoRippleDirect), + ter(temBAD_PATH_LOOP)); + } + } + void testWithFeats(FeatureBitset features) { using namespace jtx; @@ -1204,6 +1298,7 @@ struct Flow_test : public beast::unit_test::suite void run() override { testLimitQuality(); + testXRPPathLoop(); testRIPD1443(); testRIPD1449(); @@ -1231,7 +1326,7 @@ struct Flow_manual_test : public Flow_test testWithFeats(all - flowCross - f1513); testWithFeats(all - flowCross ); testWithFeats(all - f1513); - testWithFeats(all ); + testWithFeats(all ); testEmptyStrand(all - f1513); testEmptyStrand(all ); diff --git a/src/test/app/PayChan_test.cpp b/src/test/app/PayChan_test.cpp index 22764086da1..5634ac54a69 100644 --- a/src/test/app/PayChan_test.cpp +++ b/src/test/app/PayChan_test.cpp @@ -946,7 +946,7 @@ struct PayChan_test : public beast::unit_test::suite Env env(*this); env.fund(XRP(10000), alice); - for (auto const a : bobs) + for (auto const& a : bobs) { env.fund(XRP(10000), a); env.close(); @@ -956,7 +956,7 @@ struct PayChan_test : public beast::unit_test::suite // create a channel from alice to every bob account auto const settleDelay = 3600s; auto const channelFunds = XRP(1); - for (auto const b : bobs) + for (auto const& b : bobs) { env(create(alice, b, channelFunds, settleDelay, alice.pk())); } @@ -990,7 +990,7 @@ struct PayChan_test : public beast::unit_test::suite auto const bobsB58 = [&bobs]() -> std::set { std::set r; - for (auto const a : bobs) + for (auto const& a : bobs) r.insert(a.human()); return r; }(); diff --git a/src/test/app/PayStrand_test.cpp b/src/test/app/PayStrand_test.cpp index 3b7c4c6f9a4..59f8967e136 100644 --- a/src/test/app/PayStrand_test.cpp +++ b/src/test/app/PayStrand_test.cpp @@ -703,7 +703,7 @@ struct PayStrand_test : public beast::unit_test::suite alice, /*deliver*/ xrpIssue(), /*limitQuality*/ boost::none, - /*sendMaxIssue*/ xrpIssue(), + /*sendMaxIssue*/ EUR.issue(), path, true, false, diff --git a/src/test/basics/RangeSet_test.cpp b/src/test/basics/RangeSet_test.cpp index 2318398bfd3..d46fd4467b7 100644 --- a/src/test/basics/RangeSet_test.cpp +++ b/src/test/basics/RangeSet_test.cpp @@ -19,8 +19,6 @@ #include #include -#include -#include namespace ripple { @@ -78,39 +76,73 @@ class RangeSet_test : public beast::unit_test::suite } void - testSerialization() + testFromString() { + testcase("fromString"); - auto works = [](RangeSet const & orig) - { - std::stringstream ss; - boost::archive::binary_oarchive oa(ss); - oa << orig; - - boost::archive::binary_iarchive ia(ss); - RangeSet deser; - ia >> deser; - - return orig == deser; - }; - - RangeSet rs; - - BEAST_EXPECT(works(rs)); - - rs.insert(3); - BEAST_EXPECT(works(rs)); - - rs.insert(range(7u, 10u)); - BEAST_EXPECT(works(rs)); + RangeSet set; + BEAST_EXPECT(!from_string(set, "")); + BEAST_EXPECT(boost::icl::length(set) == 0); + + BEAST_EXPECT(!from_string(set, "#")); + BEAST_EXPECT(boost::icl::length(set) == 0); + + BEAST_EXPECT(!from_string(set, ",")); + BEAST_EXPECT(boost::icl::length(set) == 0); + + BEAST_EXPECT(!from_string(set, ",-")); + BEAST_EXPECT(boost::icl::length(set) == 0); + + BEAST_EXPECT(!from_string(set, "1,,2")); + BEAST_EXPECT(boost::icl::length(set) == 0); + + set.clear(); + BEAST_EXPECT(from_string(set, "1")); + BEAST_EXPECT(boost::icl::length(set) == 1); + BEAST_EXPECT(boost::icl::first(set) == 1); + + set.clear(); + BEAST_EXPECT(from_string(set, "1,1")); + BEAST_EXPECT(boost::icl::length(set) == 1); + BEAST_EXPECT(boost::icl::first(set) == 1); + + set.clear(); + BEAST_EXPECT(from_string(set, "1-1")); + BEAST_EXPECT(boost::icl::length(set) == 1); + BEAST_EXPECT(boost::icl::first(set) == 1); + + set.clear(); + BEAST_EXPECT(from_string(set, "1,4-6")); + BEAST_EXPECT(boost::icl::length(set) == 4); + BEAST_EXPECT(boost::icl::first(set) == 1); + BEAST_EXPECT(!boost::icl::contains(set, 2)); + BEAST_EXPECT(!boost::icl::contains(set, 3)); + BEAST_EXPECT(boost::icl::contains(set, 4)); + BEAST_EXPECT(boost::icl::contains(set, 5)); + BEAST_EXPECT(boost::icl::last(set) == 6); + + set.clear(); + BEAST_EXPECT(from_string(set, "1-2,4-6")); + BEAST_EXPECT(boost::icl::length(set) == 5); + BEAST_EXPECT(boost::icl::first(set) == 1); + BEAST_EXPECT(boost::icl::contains(set, 2)); + BEAST_EXPECT(boost::icl::contains(set, 4)); + BEAST_EXPECT(boost::icl::last(set) == 6); + + set.clear(); + BEAST_EXPECT(from_string(set, "1-2,6")); + BEAST_EXPECT(boost::icl::length(set) == 3); + BEAST_EXPECT(boost::icl::first(set) == 1); + BEAST_EXPECT(boost::icl::contains(set, 2)); + BEAST_EXPECT(boost::icl::last(set) == 6); } void run() override { testPrevMissing(); testToString(); - testSerialization(); + testFromString(); } }; diff --git a/src/test/basics/TaggedCache_test.cpp b/src/test/basics/TaggedCache_test.cpp index eee82076678..4480aad1e89 100644 --- a/src/test/basics/TaggedCache_test.cpp +++ b/src/test/basics/TaggedCache_test.cpp @@ -103,7 +103,7 @@ class TaggedCache_test : public beast::unit_test::suite { Cache::mapped_ptr const p1 (c.fetch (3)); Cache::mapped_ptr p2 (std::make_shared ("three")); - c.canonicalize (3, p2); + c.canonicalize_replace_client(3, p2); BEAST_EXPECT(p1.get() == p2.get()); } ++clock; @@ -134,7 +134,7 @@ class TaggedCache_test : public beast::unit_test::suite BEAST_EXPECT(c.getTrackSize() == 1); // Canonicalize a new object with the same key Cache::mapped_ptr p2 (std::make_shared ("four")); - BEAST_EXPECT(c.canonicalize (4, p2, false)); + BEAST_EXPECT(c.canonicalize_replace_client(4, p2)); BEAST_EXPECT(c.getCacheSize() == 1); BEAST_EXPECT(c.getTrackSize() == 1); // Make sure we get the original object diff --git a/src/test/core/Workers_test.cpp b/src/test/core/Workers_test.cpp index cf3edcc84be..931f6d2206f 100644 --- a/src/test/core/Workers_test.cpp +++ b/src/test/core/Workers_test.cpp @@ -109,7 +109,7 @@ class Workers_test : public beast::unit_test::suite std::unique_ptr perfLog = std::make_unique(); - Workers w(cb, *perfLog, "Test", tc1); + Workers w(cb, perfLog.get(), "Test", tc1); BEAST_EXPECT(w.getNumberOfThreads() == tc1); auto testForThreadCount = [this, &cb, &w] (int const threadCount) diff --git a/src/test/net/SSLHTTPDownloader_test.cpp b/src/test/net/SSLHTTPDownloader_test.cpp index f8421e7d248..04726137aac 100644 --- a/src/test/net/SSLHTTPDownloader_test.cpp +++ b/src/test/net/SSLHTTPDownloader_test.cpp @@ -17,11 +17,10 @@ */ //============================================================================== -#include +#include #include #include #include -#include #include #include #include @@ -82,15 +81,15 @@ class SSLHTTPDownloader_test : public beast::unit_test::suite beast::Journal journal_; // The SSLHTTPDownloader must be created as shared_ptr // because it uses shared_from_this - std::shared_ptr ptr_; + std::shared_ptr ptr_; Downloader(jtx::Env& env) : journal_ {sink_} - , ptr_ {std::make_shared( + , ptr_ {std::make_shared( env.app().getIOService(), journal_, env.app().config())} {} - SSLHTTPDownloader* operator->() + DatabaseDownloader* operator->() { return ptr_.get(); } @@ -104,6 +103,7 @@ class SSLHTTPDownloader_test : public beast::unit_test::suite (verify ? "Verify" : "No Verify"); using namespace jtx; + ripple::test::detail::FileDirGuard cert { *this, "_cert", "ca.pem", TrustedPublisherServer::ca_cert()}; @@ -152,22 +152,11 @@ class SSLHTTPDownloader_test : public beast::unit_test::suite testFailures() { testcase("Error conditions"); + using namespace jtx; + Env env {*this}; - { - // file exists - Downloader dl {env}; - ripple::test::detail::FileDirGuard const datafile { - *this, "downloads", "data", "file contents"}; - BEAST_EXPECT(!dl->download( - "localhost", - "443", - "", - 11, - datafile.file(), - std::function {std::ref(cb)})); - } { // bad hostname boost::system::error_code ec; diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index ae38fd63e42..12d37580518 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -165,7 +165,7 @@ class Database_test : public TestBase std::unique_ptr db = Manager::instance().make_Database( "test", scheduler, 2, parent, nodeParams, journal_); - BEAST_EXPECT(db->earliestSeq() == XRP_LEDGER_EARLIEST_SEQ); + BEAST_EXPECT(db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ); } // Set an invalid earliest ledger sequence @@ -190,7 +190,7 @@ class Database_test : public TestBase "test", scheduler, 2, parent, nodeParams, journal_); // Verify database uses the earliest ledger sequence setting - BEAST_EXPECT(db->earliestSeq() == 1); + BEAST_EXPECT(db->earliestLedgerSeq() == 1); } diff --git a/src/test/nodestore/TestBase.h b/src/test/nodestore/TestBase.h index 5343931e72b..e1fd0cfe531 100644 --- a/src/test/nodestore/TestBase.h +++ b/src/test/nodestore/TestBase.h @@ -195,7 +195,7 @@ class TestBase : public beast::unit_test::suite db.store (object->getType (), std::move (data), object->getHash (), - db.earliestSeq()); + db.earliestLedgerSeq()); } } diff --git a/src/test/overlay/compression_test.cpp b/src/test/overlay/compression_test.cpp new file mode 100644 index 00000000000..6d3ba5a9aa5 --- /dev/null +++ b/src/test/overlay/compression_test.cpp @@ -0,0 +1,376 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +namespace test { + +using namespace ripple::test; +using namespace ripple::test::jtx; + +static +uint256 +ledgerHash (LedgerInfo const& info) +{ +return ripple::sha512Half( + HashPrefix::ledgerMaster, + std::uint32_t(info.seq), + std::uint64_t(info.drops.drops ()), + info.parentHash, + info.txHash, + info.accountHash, + std::uint32_t(info.parentCloseTime.time_since_epoch().count()), + std::uint32_t(info.closeTime.time_since_epoch().count()), + std::uint8_t(info.closeTimeResolution.count()), + std::uint8_t(info.closeFlags)); +} + +class compression_test : public beast::unit_test::suite { + using Compressed = compression::Compressed; + using Algorithm = compression::Algorithm; +public: + compression_test() {} + + template + void + doTest(std::shared_ptr proto, protocol::MessageType mt, uint16_t nbuffers, const char *msg, + bool log = false) { + + if (log) + printf("=== compress/decompress %s ===\n", msg); + Message m(*proto, mt); + + auto &buffer = m.getBuffer(Compressed::On); + + if (log) + printf("==> compressed, original %d bytes, compressed %d bytes\n", + (int)m.getBuffer(Compressed::Off).size(), + (int)m.getBuffer(Compressed::On).size()); + + boost::beast::multi_buffer buffers; + + + // simulate multi-buffer + auto sz = buffer.size() / nbuffers; + for (int i = 0; i < nbuffers; i++) { + auto start = buffer.begin() + sz * i; + auto end = i < nbuffers - 1 ? (buffer.begin() + sz * (i + 1)) : buffer.end(); + std::vector slice(start, end); + buffers.commit( + boost::asio::buffer_copy(buffers.prepare(slice.size()), boost::asio::buffer(slice))); + } + auto header = ripple::detail::parseMessageHeader(buffers.data(), buffer.size()); + + if (log) + printf("==> parsed header: buffers size %d, compressed %d, algorithm %d, header size %d, payload size %d, buffer size %d\n", + (int)buffers.size(), header->algorithm != Algorithm::None, (int)header->algorithm, + (int)header->header_size, (int)header->payload_wire_size, (int)buffer.size()); + + if (header->algorithm == Algorithm::None) { + if (log) + printf("==> NOT COMPRESSED\n"); + return; + } + + std::vector decompressed; + decompressed.resize(header->uncompressed_size); + + BEAST_EXPECT(header->payload_wire_size == buffer.size() - header->header_size); + + ZeroCopyInputStream stream(buffers.data()); + stream.Skip(header->header_size); + + auto decompressedSize = ripple::compression::decompress(stream, header->payload_wire_size, + decompressed.data(), header->uncompressed_size); + BEAST_EXPECT(decompressedSize == header->uncompressed_size); + auto const proto1 = std::make_shared(); + + BEAST_EXPECT(proto1->ParseFromArray(decompressed.data(), decompressedSize)); + auto uncompressed = m.getBuffer(Compressed::Off); + BEAST_EXPECT(std::equal(uncompressed.begin() + ripple::compression::headerBytes, + uncompressed.end(), + decompressed.begin())); + if (log) + printf("\n"); + } + + std::shared_ptr + buildManifests(int n) { + auto manifests = std::make_shared(); + manifests->mutable_list()->Reserve(n); + for (int i = 0; i < n; i++) { + auto master = randomKeyPair(KeyType::ed25519); + auto signing = randomKeyPair(KeyType::ed25519); + STObject st(sfGeneric); + st[sfSequence] = i; + st[sfPublicKey] = std::get<0>(master); + st[sfSigningPubKey] = std::get<0>(signing); + st[sfDomain] = makeSlice(std::string("example") + std::to_string(i) + std::string(".com")); + sign(st, HashPrefix::manifest, KeyType::ed25519, std::get<1>(master), sfMasterSignature); + sign(st, HashPrefix::manifest, KeyType::ed25519, std::get<1>(signing)); + Serializer s; + st.add(s); + auto *manifest = manifests->add_list(); + manifest->set_stobject(s.data(), s.size()); + } + return manifests; + } + + std::shared_ptr + buildEndpoints(int n) { + auto endpoints = std::make_shared(); + endpoints->mutable_endpoints()->Reserve(n); + for (int i = 0; i < n; i++) { + auto *endpoint = endpoints->add_endpoints(); + endpoint->set_hops(i); + std::string addr = std::string("10.0.1.") + std::to_string(i); + endpoint->mutable_ipv4()->set_ipv4( + boost::endian::native_to_big(boost::asio::ip::address_v4::from_string(addr).to_uint())); + endpoint->mutable_ipv4()->set_ipv4port(i); + } + endpoints->set_version(2); + + return endpoints; + } + + std::shared_ptr + buildTransaction(Logs &logs) { + Env env(*this, envconfig()); + int fund = 10000; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + env.fund(XRP(fund), "alice", "bob"); + env.trust(bob["USD"](fund), alice); + env.close(); + + auto toBinary = [](std::string const &text) { + std::string binary; + for (size_t i = 0; i < text.size(); ++i) { + unsigned int c = charUnHex(text[i]); + c = c << 4; + ++i; + c = c | charUnHex(text[i]); + binary.push_back(c); + } + + return binary; + }; + + std::string usdTxBlob = ""; + auto wsc = makeWSClient(env.app().config()); + { + Json::Value jrequestUsd; + jrequestUsd[jss::secret] = toBase58(generateSeed("bob")); + jrequestUsd[jss::tx_json] = + pay("bob", "alice", bob["USD"](fund / 2)); + Json::Value jreply_usd = wsc->invoke("sign", jrequestUsd); + + usdTxBlob = + toBinary(jreply_usd[jss::result][jss::tx_blob].asString()); + } + + auto transaction = std::make_shared(); + transaction->set_rawtransaction(usdTxBlob); + transaction->set_status(protocol::tsNEW); + auto tk = make_TimeKeeper(logs.journal("TimeKeeper")); + transaction->set_receivetimestamp(tk->now().time_since_epoch().count()); + transaction->set_deferred(true); + + return transaction; + } + + std::shared_ptr + buildGetLedger() { + auto getLedger = std::make_shared(); + getLedger->set_itype(protocol::liTS_CANDIDATE); + getLedger->set_ltype(protocol::TMLedgerType::ltACCEPTED); + uint256 const hash(ripple::sha512Half(123456789)); + getLedger->set_ledgerhash(hash.begin(), hash.size()); + getLedger->set_ledgerseq(123456789); + ripple::SHAMapNodeID sha(hash.data(), hash.size()); + getLedger->add_nodeids(sha.getRawString()); + getLedger->set_requestcookie(123456789); + getLedger->set_querytype(protocol::qtINDIRECT); + getLedger->set_querydepth(3); + return getLedger; + } + + std::shared_ptr + buildLedgerData(uint32_t n, Logs &logs) { + auto ledgerData = std::make_shared(); + uint256 const hash(ripple::sha512Half(12356789)); + ledgerData->set_ledgerhash(hash.data(), hash.size()); + ledgerData->set_ledgerseq(123456789); + ledgerData->set_type(protocol::TMLedgerInfoType::liAS_NODE); + ledgerData->set_requestcookie(123456789); + ledgerData->set_error(protocol::TMReplyError::reNO_LEDGER); + ledgerData->mutable_nodes()->Reserve(n); + uint256 parentHash(0); + for (int i = 0; i < n; i++) { + LedgerInfo info; + auto tk = make_TimeKeeper(logs.journal("TimeKeeper")); + info.seq = i; + info.parentCloseTime = tk->now(); + info.hash = ripple::sha512Half(i); + info.txHash = ripple::sha512Half(i + 1); + info.accountHash = ripple::sha512Half(i + 2); + info.parentHash = parentHash; + info.drops = XRPAmount(10); + info.closeTimeResolution = tk->now().time_since_epoch(); + info.closeTime = tk->now(); + parentHash = ledgerHash(info); + Serializer nData; + ripple::addRaw(info, nData); + ledgerData->add_nodes()->set_nodedata(nData.getDataPtr(), nData.getLength()); + } + + return ledgerData; + } + + std::shared_ptr + buildGetObjectByHash() { + auto getObject = std::make_shared(); + + getObject->set_type(protocol::TMGetObjectByHash_ObjectType::TMGetObjectByHash_ObjectType_otTRANSACTION); + getObject->set_query(true); + getObject->set_seq(123456789); + uint256 hash(ripple::sha512Half(123456789)); + getObject->set_ledgerhash(hash.data(), hash.size()); + getObject->set_fat(true); + for (int i = 0; i < 100; i++) { + uint256 hash(ripple::sha512Half(i)); + auto object = getObject->add_objects(); + object->set_hash(hash.data(), hash.size()); + ripple::SHAMapNodeID sha(hash.data(), hash.size()); + object->set_nodeid(sha.getRawString()); + object->set_index(""); + object->set_data(""); + object->set_ledgerseq(i); + } + return getObject; + } + + std::shared_ptr + buildValidatorList() + { + auto list = std::make_shared(); + + auto master = randomKeyPair(KeyType::ed25519); + auto signing = randomKeyPair(KeyType::ed25519); + STObject st(sfGeneric); + st[sfSequence] = 0; + st[sfPublicKey] = std::get<0>(master); + st[sfSigningPubKey] = std::get<0>(signing); + st[sfDomain] = makeSlice(std::string("example.com")); + sign(st, HashPrefix::manifest, KeyType::ed25519, std::get<1>(master), sfMasterSignature); + sign(st, HashPrefix::manifest, KeyType::ed25519, std::get<1>(signing)); + Serializer s; + st.add(s); + list->set_manifest(s.data(), s.size()); + list->set_version(3); + STObject signature(sfSignature); + ripple::sign(st, HashPrefix::manifest,KeyType::ed25519, std::get<1>(signing)); + Serializer s1; + st.add(s1); + list->set_signature(s1.data(), s1.size()); + list->set_blob(strHex(s.getString())); + return list; + } + + void + testProtocol() { + testcase("Message Compression"); + + auto thresh = beast::severities::Severity::kInfo; + auto logs = std::make_unique(thresh); + + protocol::TMManifests manifests; + protocol::TMEndpoints endpoints; + protocol::TMTransaction transaction; + protocol::TMGetLedger get_ledger; + protocol::TMLedgerData ledger_data; + protocol::TMGetObjectByHash get_object; + protocol::TMValidatorList validator_list; + + // 4.5KB + doTest(buildManifests(20), protocol::mtMANIFESTS, 4, "TMManifests20"); + // 22KB + doTest(buildManifests(100), protocol::mtMANIFESTS, 4, "TMManifests100"); + // 131B + doTest(buildEndpoints(10), protocol::mtENDPOINTS, 4, "TMEndpoints10"); + // 1.3KB + doTest(buildEndpoints(100), protocol::mtENDPOINTS, 4, "TMEndpoints100"); + // 242B + doTest(buildTransaction(*logs), protocol::mtTRANSACTION, 1, "TMTransaction"); + // 87B + doTest(buildGetLedger(), protocol::mtGET_LEDGER, 1, "TMGetLedger"); + // 61KB + doTest(buildLedgerData(500, *logs), protocol::mtLEDGER_DATA, 10, "TMLedgerData500"); + // 122 KB + doTest(buildLedgerData(1000, *logs), protocol::mtLEDGER_DATA, 20, "TMLedgerData1000"); + // 1.2MB + doTest(buildLedgerData(10000, *logs), protocol::mtLEDGER_DATA, 50, "TMLedgerData10000"); + // 12MB + doTest(buildLedgerData(100000, *logs), protocol::mtLEDGER_DATA, 100, "TMLedgerData100000"); + // 61MB + doTest(buildLedgerData(500000, *logs), protocol::mtLEDGER_DATA, 100, "TMLedgerData500000"); + // 7.7KB + doTest(buildGetObjectByHash(), protocol::mtGET_OBJECTS, 4, "TMGetObjectByHash"); + // 895B + doTest(buildValidatorList(), protocol::mtVALIDATORLIST, 4, "TMValidatorList"); + } + + void run() override { + testProtocol(); + } + +}; + +BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(compression, ripple_data, ripple, 20); + +} +} \ No newline at end of file diff --git a/src/test/rpc/AccountCurrencies_test.cpp b/src/test/rpc/AccountCurrencies_test.cpp index 85469d33106..5825d992f40 100644 --- a/src/test/rpc/AccountCurrencies_test.cpp +++ b/src/test/rpc/AccountCurrencies_test.cpp @@ -153,7 +153,7 @@ class AccountCurrencies_test : public beast::unit_test::suite // does not change env(trust(alice, gw["USD"](100), tfSetFreeze)); result = env.rpc ("account_lines", alice.human()); - for (auto const l : result[jss::lines]) + for (auto const& l : result[jss::lines]) BEAST_EXPECT( l[jss::freeze].asBool() == (l[jss::currency] == "USD")); result = env.rpc ("json", "account_currencies", diff --git a/src/test/rpc/RPCCall_test.cpp b/src/test/rpc/RPCCall_test.cpp index f2ca3758241..bc45dd5c6c1 100644 --- a/src/test/rpc/RPCCall_test.cpp +++ b/src/test/rpc/RPCCall_test.cpp @@ -2998,10 +2998,9 @@ static RPCCallTestData const rpcCallTestArray [] = })" }, { - "download_shard: novalidate.", __LINE__, + "download_shard:", __LINE__, { "download_shard", - "novalidate", "20", "url_NotValidated", }, @@ -3016,8 +3015,7 @@ static RPCCallTestData const rpcCallTestArray [] = "index" : 20, "url" : "url_NotValidated" } - ], - "validate" : false + ] } ] })" @@ -3064,10 +3062,9 @@ static RPCCallTestData const rpcCallTestArray [] = })" }, { - "download_shard: novalidate many shards.", __LINE__, + "download_shard: many shards.", __LINE__, { "download_shard", - "novalidate", "2000000", "url_NotValidated0", "2000001", @@ -3106,8 +3103,7 @@ static RPCCallTestData const rpcCallTestArray [] = "index" : 2000004, "url" : "url_NotValidated4" } - ], - "validate" : false + ] } ] })" @@ -3160,8 +3156,7 @@ static RPCCallTestData const rpcCallTestArray [] = "index" : 20, "url" : "url_NotValidated" } - ], - "validate" : false + ] } ] })" diff --git a/src/test/rpc/ShardArchiveHandler_test.cpp b/src/test/rpc/ShardArchiveHandler_test.cpp new file mode 100644 index 00000000000..3515810eabd --- /dev/null +++ b/src/test/rpc/ShardArchiveHandler_test.cpp @@ -0,0 +1,364 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +class ShardArchiveHandler_test : public beast::unit_test::suite +{ + using Downloads = std::vector>; + + TrustedPublisherServer + createServer(jtx::Env& env, bool ssl = true) + { + std::vector list; + list.push_back(TrustedPublisherServer::randomValidator()); + return TrustedPublisherServer{ + env.app().getIOService(), + list, + env.timeKeeper().now() + std::chrono::seconds{3600}, + ssl}; + } + +public: + void + testStateDatabase1() + { + testcase("testStateDatabase1"); + + { + beast::temp_dir tempDir; + + auto c = jtx::envconfig(); + auto& section = c->section(ConfigSection::shardDatabase()); + section.set("path", tempDir.path()); + section.set("max_size_gb", "100"); + c->setupControl(true, true, true); + + jtx::Env env(*this, std::move(c)); + auto handler = RPC::ShardArchiveHandler::getInstance( + env.app(), env.app().getJobQueue()); + BEAST_EXPECT(handler); + + BEAST_EXPECT(handler->init()); + + std::string const rawUrl = "https://foo:443/1.tar.lz4"; + parsedURL url; + + parseUrl(url, rawUrl); + handler->add(1, {url, rawUrl}); + + { + std::lock_guard lock(handler->m_); + + auto& session{handler->sqliteDB_->getSession()}; + + soci::rowset rs = + (session.prepare << "SELECT * FROM State;"); + + uint64_t rowCount = 0; + + for (auto it = rs.begin(); it != rs.end(); ++it, ++rowCount) + { + BEAST_EXPECT(it->get(0) == 1); + BEAST_EXPECT(it->get(1) == rawUrl); + } + + BEAST_EXPECT(rowCount == 1); + } + + handler->release(); + } + + // Destroy the singleton so we start fresh in + // the next testcase. + RPC::ShardArchiveHandler::instance_.reset(); + } + + void + testStateDatabase2() + { + testcase("testStateDatabase2"); + + { + beast::temp_dir tempDir; + + auto c = jtx::envconfig(); + auto& section = c->section(ConfigSection::shardDatabase()); + section.set("path", tempDir.path()); + section.set("max_size_gb", "100"); + c->setupControl(true, true, true); + + jtx::Env env(*this, std::move(c)); + auto handler = RPC::ShardArchiveHandler::getInstance( + env.app(), env.app().getJobQueue()); + BEAST_EXPECT(handler); + + BEAST_EXPECT(handler->init()); + + Downloads const dl = {{1, "https://foo:443/1.tar.lz4"}, + {2, "https://foo:443/2.tar.lz4"}, + {3, "https://foo:443/3.tar.lz4"}}; + + for (auto const& entry : dl) + { + parsedURL url; + parseUrl(url, entry.second); + handler->add(entry.first, {url, entry.second}); + } + + { + std::lock_guard lock(handler->m_); + + auto& session{handler->sqliteDB_->getSession()}; + soci::rowset rs = + (session.prepare << "SELECT * FROM State;"); + + uint64_t pos = 0; + for (auto it = rs.begin(); it != rs.end(); ++it, ++pos) + { + BEAST_EXPECT(it->get(0) == dl[pos].first); + BEAST_EXPECT(it->get(1) == dl[pos].second); + } + + BEAST_EXPECT(pos == dl.size()); + } + + handler->release(); + } + + // Destroy the singleton so we start fresh in + // the next testcase. + RPC::ShardArchiveHandler::instance_.reset(); + } + + void + testStateDatabase3() + { + testcase("testStateDatabase3"); + + { + beast::temp_dir tempDir; + + auto c = jtx::envconfig(); + auto& section = c->section(ConfigSection::shardDatabase()); + section.set("path", tempDir.path()); + section.set("max_size_gb", "100"); + c->setupControl(true, true, true); + + jtx::Env env(*this, std::move(c)); + auto handler = RPC::ShardArchiveHandler::getInstance( + env.app(), env.app().getJobQueue()); + BEAST_EXPECT(handler); + + BEAST_EXPECT(handler->init()); + + auto server = createServer(env); + auto host = server.local_endpoint().address().to_string(); + auto port = std::to_string(server.local_endpoint().port()); + server.stop(); + + Downloads const dl = [&host, &port] { + Downloads ret; + + for (int i = 1; i <= 10; ++i) + { + ret.push_back({i, + (boost::format("https://%s:%d/%d.tar.lz4") % + host % port % i) + .str()}); + } + + return ret; + }(); + + for (auto const& entry : dl) + { + parsedURL url; + parseUrl(url, entry.second); + handler->add(entry.first, {url, entry.second}); + } + + BEAST_EXPECT(handler->start()); + + auto stateDir = RPC::ShardArchiveHandler::getDownloadDirectory( + env.app().config()); + + std::unique_lock lock(handler->m_); + + BEAST_EXPECT( + boost::filesystem::exists(stateDir) || + handler->archives_.empty()); + + while (!handler->archives_.empty()) + { + lock.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + lock.lock(); + } + + BEAST_EXPECT(!boost::filesystem::exists(stateDir)); + } + + // Destroy the singleton so we start fresh in + // the next testcase. + RPC::ShardArchiveHandler::instance_.reset(); + } + + void + testStateDatabase4() + { + testcase("testStateDatabase4"); + + beast::temp_dir tempDir; + + { + auto c = jtx::envconfig(); + auto& section = c->section(ConfigSection::shardDatabase()); + section.set("path", tempDir.path()); + section.set("max_size_gb", "100"); + c->setupControl(true, true, true); + + jtx::Env env(*this, std::move(c)); + auto handler = RPC::ShardArchiveHandler::getInstance( + env.app(), env.app().getJobQueue()); + BEAST_EXPECT(handler); + + BEAST_EXPECT(handler->init()); + + auto server = createServer(env); + auto host = server.local_endpoint().address().to_string(); + auto port = std::to_string(server.local_endpoint().port()); + server.stop(); + + Downloads const dl = [&host, &port] { + Downloads ret; + + for (int i = 1; i <= 10; ++i) + { + ret.push_back({i, + (boost::format("https://%s:%d/%d.tar.lz4") % + host % port % i) + .str()}); + } + + return ret; + }(); + + for (auto const& entry : dl) + { + parsedURL url; + parseUrl(url, entry.second); + handler->add(entry.first, {url, entry.second}); + } + + auto stateDir = RPC::ShardArchiveHandler::getDownloadDirectory( + env.app().config()); + + boost::filesystem::copy_file( + stateDir / stateDBName, + boost::filesystem::path(tempDir.path()) / stateDBName); + + BEAST_EXPECT(handler->start()); + + std::unique_lock lock(handler->m_); + + BEAST_EXPECT( + boost::filesystem::exists(stateDir) || + handler->archives_.empty()); + + while (!handler->archives_.empty()) + { + lock.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + lock.lock(); + } + + BEAST_EXPECT(!boost::filesystem::exists(stateDir)); + + boost::filesystem::create_directory(stateDir); + + boost::filesystem::copy_file( + boost::filesystem::path(tempDir.path()) / stateDBName, + stateDir / stateDBName); + } + + // Destroy the singleton so we start fresh in + // the new scope. + RPC::ShardArchiveHandler::instance_.reset(); + + auto c = jtx::envconfig(); + auto& section = c->section(ConfigSection::shardDatabase()); + section.set("path", tempDir.path()); + section.set("max_size_gb", "100"); + c->setupControl(true, true, true); + + jtx::Env env(*this, std::move(c)); + + while (!RPC::ShardArchiveHandler::hasInstance()) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + BEAST_EXPECT(RPC::ShardArchiveHandler::hasInstance()); + + auto handler = RPC::ShardArchiveHandler::getInstance(); + + auto stateDir = + RPC::ShardArchiveHandler::getDownloadDirectory(env.app().config()); + + std::unique_lock lock(handler->m_); + + BEAST_EXPECT( + boost::filesystem::exists(stateDir) || handler->archives_.empty()); + + while (!handler->archives_.empty()) + { + lock.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + lock.lock(); + } + + BEAST_EXPECT(!boost::filesystem::exists(stateDir)); + } + + void + run() override + { + testStateDatabase1(); + testStateDatabase2(); + testStateDatabase3(); + testStateDatabase4(); + } +}; + +BEAST_DEFINE_TESTSUITE(ShardArchiveHandler, app, ripple); + +} // namespace test +} // namespace ripple