Proposed 1.6.0-b1

.github/workflows/doxygen.yml

@@ -0,0 +1,23 @@
+name: Build and publish Doxygen documentation
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  job:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: set OUTPUT_DIRECTORY
+        run: echo 'OUTPUT_DIRECTORY = html' >> docs/Doxyfile
+      - name: build
+        uses: mattnotmitt/doxygen-action@v1
+        with:
+          doxyfile-path: 'docs/Doxyfile'
+      - name: publish
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./html

Builds/CMake/RippleConfig.cmake

@@ -21,7 +21,6 @@ find_dependency (Boost 1.70
     filesystem
     program_options
     regex
-    serialization
     system
     thread)
 #[=========================================================[

Builds/CMake/RippledCore.cmake

@@ -487,6 +487,7 @@ target_sources (rippled PRIVATE
      main sources:
        subdir: net
   #]===============================]
+  src/ripple/net/impl/DatabaseDownloader.cpp
   src/ripple/net/impl/HTTPClient.cpp
   src/ripple/net/impl/InfoSub.cpp
   src/ripple/net/impl/RPCCall.cpp
@@ -513,6 +514,7 @@ target_sources (rippled PRIVATE
   src/ripple/nodestore/impl/ManagerImp.cpp
   src/ripple/nodestore/impl/NodeObject.cpp
   src/ripple/nodestore/impl/Shard.cpp
+  src/ripple/nodestore/impl/TaskQueue.cpp
   #[===============================[
      main sources:
        subdir: overlay
@@ -853,6 +855,7 @@ target_sources (rippled PRIVATE
   src/test/overlay/ProtocolVersion_test.cpp
   src/test/overlay/cluster_test.cpp
   src/test/overlay/short_read_test.cpp
+  src/test/overlay/compression_test.cpp
   #[===============================[
      test sources:
        subdir: peerfinder
@@ -917,6 +920,7 @@ target_sources (rippled PRIVATE
   src/test/rpc/RPCOverload_test.cpp
   src/test/rpc/RobustTransaction_test.cpp
   src/test/rpc/ServerInfo_test.cpp
+  src/test/rpc/ShardArchiveHandler_test.cpp
   src/test/rpc/Status_test.cpp
   src/test/rpc/Submit_test.cpp
   src/test/rpc/Subscribe_test.cpp
@@ -961,7 +965,8 @@ endif ()
 if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.16)
   # any files that don't play well with unity should be added here
   set_source_files_properties(
-    # this one seems to produce conflicts in beast teardown template methods:
+    # these two seem to produce conflicts in beast teardown template methods
     src/test/rpc/ValidatorRPC_test.cpp
+    src/test/rpc/ShardArchiveHandler_test.cpp
     PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE)
 endif ()

Builds/CMake/RippledDocs.cmake

@@ -3,29 +3,82 @@
 #]===================================================================]
 
 find_package (Doxygen)
-if (TARGET Doxygen::doxygen)
-  set (doc_srcs docs/source.dox)
-  file (GLOB_RECURSE other_docs docs/*.md)
-  list (APPEND doc_srcs "${other_docs}")
-  # read the source config and make a modified one
-  # that points the output files to our build directory
-  file (READ "${CMAKE_CURRENT_SOURCE_DIR}/docs/source.dox" dox_content)
-  string (REGEX REPLACE "[\t ]*OUTPUT_DIRECTORY[\t ]*=(.*)"
-    "OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}\n\\1"
-    new_config "${dox_content}")
-  file (WRITE "${CMAKE_BINARY_DIR}/source.dox" "${new_config}")
-  add_custom_target (docs
-    COMMAND "${DOXYGEN_EXECUTABLE}" "${CMAKE_BINARY_DIR}/source.dox"
-    BYPRODUCTS "${CMAKE_BINARY_DIR}/html_doc/index.html"
-    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/docs"
-    SOURCES "${doc_srcs}")
-  if (is_multiconfig)
-    set_property (
-      SOURCE ${doc_srcs}
-      APPEND
-      PROPERTY HEADER_FILE_ONLY
-      true)
-  endif ()
-else ()
+if (NOT TARGET Doxygen::doxygen)
   message (STATUS "doxygen executable not found -- skipping docs target")
+  return ()
+endif ()
+
+set (doxygen_output_directory "${CMAKE_BINARY_DIR}/docs")
+set (doxygen_include_path "${CMAKE_SOURCE_DIR}/src")
+set (doxygen_index_file "${doxygen_output_directory}/html/index.html")
+set (doxyfile "${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile")
+
+file (GLOB_RECURSE doxygen_input
+  docs/*.md
+  src/ripple/*.h
+  src/ripple/*.cpp
+  src/ripple/*.md
+  src/test/*.h
+  src/test/*.md
+  Builds/*/README.md)
+list (APPEND doxygen_input
+  README.md
+  RELEASENOTES.md
+  src/README.md)
+set (dependencies "${doxygen_input}" "${doxyfile}")
+
+function (verbose_find_path variable name)
+  # find_path sets a CACHE variable, so don't try using a "local" variable.
+  find_path (${variable} "${name}" ${ARGN})
+  if (NOT ${variable})
+    message (WARNING "could not find ${name}")
+  else ()
+    message (STATUS "found ${name}: ${${variable}}/${name}")
+  endif ()
+endfunction ()
+
+verbose_find_path (doxygen_plantuml_jar_path plantuml.jar PATH_SUFFIXES share/plantuml)
+verbose_find_path (doxygen_dot_path dot)
+
+# https://en.cppreference.com/w/Cppreference:Archives
+# https://stackoverflow.com/questions/60822559/how-to-move-a-file-download-from-configure-step-to-build-step
+set (download_script "${CMAKE_BINARY_DIR}/docs/download-cppreference.cmake")
+file (WRITE
+  "${download_script}"
+  "file (DOWNLOAD \
+    http://upload.cppreference.com/mwiki/images/b/b2/html_book_20190607.zip \
+    ${CMAKE_BINARY_DIR}/docs/cppreference.zip \
+    EXPECTED_HASH MD5=82b3a612d7d35a83e3cb1195a63689ab \
+  )\n \
+  execute_process ( \
+    COMMAND \"${CMAKE_COMMAND}\" -E tar -xf cppreference.zip \
+  )\n"
+)
+set (tagfile "${CMAKE_BINARY_DIR}/docs/cppreference-doxygen-web.tag.xml")
+add_custom_command (
+  OUTPUT "${tagfile}"
+  COMMAND "${CMAKE_COMMAND}" -P "${download_script}"
+  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/docs"
+)
+set (doxygen_tagfiles "${tagfile}=http://en.cppreference.com/w/")
+
+add_custom_command (
+  OUTPUT "${doxygen_index_file}"
+  COMMAND "${CMAKE_COMMAND}" -E env
+    "DOXYGEN_OUTPUT_DIRECTORY=${doxygen_output_directory}"
+    "DOXYGEN_INCLUDE_PATH=${doxygen_include_path}"
+    "DOXYGEN_TAGFILES=${doxygen_tagfiles}"
+    "DOXYGEN_PLANTUML_JAR_PATH=${doxygen_plantuml_jar_path}"
+    "DOXYGEN_DOT_PATH=${doxygen_dot_path}"
+    "${DOXYGEN_EXECUTABLE}" "${doxyfile}"
+  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+  DEPENDS "${dependencies}" "${tagfile}")
+add_custom_target (docs
+  DEPENDS "${doxygen_index_file}"
+  SOURCES "${dependencies}")
+if (is_multiconfig)
+  set_property (
+    SOURCE ${dependencies}
+    APPEND PROPERTY
+    HEADER_FILE_ONLY true)
 endif ()

Builds/CMake/RippledInterface.cmake

@@ -21,9 +21,7 @@ target_compile_definitions (opts
     >
     $<$<BOOL:${beast_no_unit_test_inline}>:BEAST_NO_UNIT_TEST_INLINE=1>
     $<$<BOOL:${beast_disable_autolink}>:BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES=1>
-    $<$<BOOL:${single_io_service_thread}>:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>
-    # doesn't currently compile ? :
-    $<$<BOOL:${verify_nodeobject_keys}>:RIPPLE_VERIFY_NODEOBJECT_KEYS=1>)
+    $<$<BOOL:${single_io_service_thread}>:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>)
 target_compile_options (opts
   INTERFACE
     $<$<AND:$<BOOL:${is_gcc}>,$<COMPILE_LANGUAGE:CXX>>:-Wsuggest-override>

Builds/CMake/RippledSettings.cmake

@@ -100,12 +100,6 @@ option (have_package_container
 option (beast_no_unit_test_inline
   "Prevents unit test definitions from being inserted into global table"
   OFF)
-# NOTE - THIS OPTION CURRENTLY DOES NOT COMPILE :
-# TODO: fix or remove
-option (verify_nodeobject_keys
-  "This verifies that the hash of node objects matches the payload. \
-  This check is expensive - use with caution."
-  OFF)
 option (single_io_service_thread
   "Restricts the number of threads calling io_service::run to one. \
   This can be useful when debugging."

Builds/CMake/deps/Boost.cmake

@@ -47,7 +47,6 @@ find_package (Boost 1.70 REQUIRED
     filesystem
     program_options
     regex
-    serialization
     system
     thread)
 
@@ -69,7 +68,6 @@ target_link_libraries (ripple_boost
     Boost::filesystem
     Boost::program_options
     Boost::regex
-    Boost::serialization
     Boost::system
     Boost::thread)
 if (Boost_COMPILER)

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -19,7 +19,7 @@ variables:
   DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
   ARTIFACTORY_HOST: "artifactory.ops.ripple.com"
   ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555"
-  GIT_SIGN_PUBKEYS_URL: "https://gitlab.ops.ripple.com/snippets/11/raw"
+  GIT_SIGN_PUBKEYS_URL: "https://gitlab.ops.ripple.com/xrpledger/rippled-packages/snippets/49/raw"
   PUBLIC_REPO_ROOT: "https://repos.ripple.com/repos"
   # also need to define this variable ONLY for the primary
   # build/publish pipeline on the mainline repo:

Builds/containers/shared/install_boost.sh

@@ -39,7 +39,6 @@ else
     BLDARGS+=(--with-filesystem)
     BLDARGS+=(--with-program_options)
     BLDARGS+=(--with-regex)
-    BLDARGS+=(--with-serialization)
     BLDARGS+=(--with-system)
     BLDARGS+=(--with-atomic)
     BLDARGS+=(--with-thread)

bin/ci/ubuntu/build-and-test.sh

@@ -105,7 +105,7 @@ ${time} eval cmake --build . ${BUILDARGS} -- ${BUILDTOOLARGS}
 if [[ ${TARGET} == "docs" ]]; then
     ## mimic the standard test output for docs build
     ## to make controlling processes like jenkins happy
-    if [ -f html_doc/index.html ]; then
+    if [ -f docs/html/index.html ]; then
         echo "1 case, 1 test total, 0 failures"
     else
         echo "1 case, 1 test total, 1 failures"

cfg/rippled-example.cfg

@@ -746,6 +746,7 @@
 #
 #   main    -> 0
 #   testnet -> 1
+#   devnet  -> 2
 #
 #   If this value is not specified the server is not explicitly configured
 #   to track a particular network.

docs/Dockerfile

@@ -26,6 +26,7 @@ RUN rm -rf /tmp/doxygen-1.8.14
 
 RUN mkdir -p /opt/plantuml
 RUN wget -O /opt/plantuml/plantuml.jar http://sourceforge.net/projects/plantuml/files/plantuml.jar/download
-ENV PLANTUML_JAR=/opt/plantuml/plantuml.jar
+ENV DOXYGEN_PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar
 
-CMD cd /opt/rippled/docs && doxygen source.dox
+ENV DOXYGEN_OUTPUT_DIRECTORY=html
+CMD cd /opt/rippled && doxygen docs/Doxyfile