From 1c59d3dbd2bf5f0f5af10f4770c64b837a6d905f Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 9 Mar 2024 13:36:07 -0600 Subject: [PATCH 01/12] Add recipe with patches for RocksDB 6.29.5 --- external/rocksdb/conandata.yml | 27 ++ external/rocksdb/conanfile.py | 233 ++++++++++++++++++ ...-0001-add-include-cstdint-for-gcc-13.patch | 30 +++ .../6.29.5-0002-exclude-thirdparty.patch | 6 + 4 files changed, 296 insertions(+) create mode 100644 external/rocksdb/conandata.yml create mode 100644 external/rocksdb/conanfile.py create mode 100644 external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch create mode 100644 external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch diff --git a/external/rocksdb/conandata.yml b/external/rocksdb/conandata.yml new file mode 100644 index 00000000000..86b42f79f0f --- /dev/null +++ b/external/rocksdb/conandata.yml @@ -0,0 +1,27 @@ +sources: + "6.29.5": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.29.5.tar.gz" + sha256: "ddbf84791f0980c0bbce3902feb93a2c7006f6f53bfd798926143e31d4d756f0" + "6.27.3": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz" + sha256: "ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58" + "6.20.3": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.20.3.tar.gz" + sha256: "c6502c7aae641b7e20fafa6c2b92273d935d2b7b2707135ebd9a67b092169dca" + "8.8.1": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.8.1.tar.gz" + sha256: "056c7e21ad8ae36b026ac3b94b9d6e0fcc60e1d937fc80330921e4181be5c36e" +patches: + "6.29.5": + - patch_file: "patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch" + patch_description: "Fix build with gcc 13 by including cstdint" + patch_type: "portability" + patch_source: "https://github.com/facebook/rocksdb/pull/11118" + - patch_file: "patches/6.29.5-0002-exclude-thirdparty.patch" + patch_description: "Do not include thirdparty.inc" + patch_type: "portability" + "6.27.3": + - patch_file: "patches/6.27.3-0001-add-include-cstdint-for-gcc-13.patch" + patch_description: "Fix build with gcc 13 by including cstdint" + patch_type: "portability" + patch_source: "https://github.com/facebook/rocksdb/pull/11118" diff --git a/external/rocksdb/conanfile.py b/external/rocksdb/conanfile.py new file mode 100644 index 00000000000..09425b9f863 --- /dev/null +++ b/external/rocksdb/conanfile.py @@ -0,0 +1,233 @@ +import os +import glob +import shutil + +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime +from conan.tools.scm import Version + +required_conan_version = ">=1.53.0" + + +class RocksDBConan(ConanFile): + name = "rocksdb" + homepage = "https://github.com/facebook/rocksdb" + license = ("GPL-2.0-only", "Apache-2.0") + url = "https://github.com/conan-io/conan-center-index" + description = "A library that provides an embeddable, persistent key-value store for fast storage" + topics = ("database", "leveldb", "facebook", "key-value") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "lite": [True, False], + "with_gflags": [True, False], + "with_snappy": [True, False], + "with_lz4": [True, False], + "with_zlib": [True, False], + "with_zstd": [True, False], + "with_tbb": [True, False], + "with_jemalloc": [True, False], + "enable_sse": [False, "sse42", "avx2"], + "use_rtti": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "lite": False, + "with_snappy": False, + "with_lz4": False, + "with_zlib": False, + "with_zstd": False, + "with_gflags": False, + "with_tbb": False, + "with_jemalloc": False, + "enable_sse": False, + "use_rtti": False, + } + + @property + def _min_cppstd(self): + return "11" if Version(self.version) < "8.8.1" else "17" + + @property + def _compilers_minimum_version(self): + return {} if self._min_cppstd == "11" else { + "apple-clang": "10", + "clang": "7", + "gcc": "7", + "msvc": "191", + "Visual Studio": "15", + } + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_tbb + if self.settings.build_type == "Debug": + self.options.use_rtti = True # Rtti are used in asserts for debug mode... + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self, src_folder="src") + + def requirements(self): + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_snappy: + self.requires("snappy/1.1.10") + if self.options.with_lz4: + self.requires("lz4/1.9.4") + if self.options.with_zlib: + self.requires("zlib/[>=1.2.11 <2]") + if self.options.with_zstd: + self.requires("zstd/1.5.5") + if self.options.get_safe("with_tbb"): + self.requires("onetbb/2021.10.0") + if self.options.with_jemalloc: + self.requires("jemalloc/5.3.0") + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) + + minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." + ) + + if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]: + raise ConanInvalidConfiguration("Rocksdb requires 64 bits") + + check_min_vs(self, "191") + + if self.version == "6.20.3" and \ + self.settings.os == "Linux" and \ + self.settings.compiler == "gcc" and \ + Version(self.settings.compiler.version) < "5": + raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522 + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["FAIL_ON_WARNINGS"] = False + tc.variables["WITH_TESTS"] = False + tc.variables["WITH_TOOLS"] = False + tc.variables["WITH_CORE_TOOLS"] = False + tc.variables["WITH_BENCHMARK_TOOLS"] = False + tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False + if is_msvc(self): + tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self) + tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows" + tc.variables["ROCKSDB_LITE"] = self.options.lite + tc.variables["WITH_GFLAGS"] = self.options.with_gflags + tc.variables["WITH_SNAPPY"] = self.options.with_snappy + tc.variables["WITH_LZ4"] = self.options.with_lz4 + tc.variables["WITH_ZLIB"] = self.options.with_zlib + tc.variables["WITH_ZSTD"] = self.options.with_zstd + tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False) + tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc + tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared + tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared + tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared + tc.variables["USE_RTTI"] = self.options.use_rtti + if not bool(self.options.enable_sse): + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + elif self.options.enable_sse == "sse42": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = True + elif self.options.enable_sse == "avx2": + tc.variables["PORTABLE"] = False + tc.variables["FORCE_SSE42"] = False + # not available yet in CCI + tc.variables["WITH_NUMA"] = False + tc.generate() + + deps = CMakeDeps(self) + if self.options.with_jemalloc: + deps.set_property("jemalloc", "cmake_file_name", "JeMalloc") + deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc") + deps.generate() + + def build(self): + apply_conandata_patches(self) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _remove_static_libraries(self): + rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib")) + for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")): + if not lib.endswith(".dll.a"): + os.remove(lib) + + def _remove_cpp_headers(self): + for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")): + if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"): + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + + def package(self): + copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + cmake = CMake(self) + cmake.install() + if self.options.shared: + self._remove_static_libraries() + self._remove_cpp_headers() # Force stable ABI for shared libraries + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb" + self.cpp_info.set_property("cmake_file_name", "RocksDB") + self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}") + # TODO: back to global scope in conan v2 once cmake_find_package* generators removed + self.cpp_info.components["librocksdb"].libs = collect_libs(self) + if self.settings.os == "Windows": + self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"] + if self.options.shared: + self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"] + elif self.settings.os in ["Linux", "FreeBSD"]: + self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"] + if self.options.lite: + self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE") + + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "RocksDB" + self.cpp_info.names["cmake_find_package_multi"] = "RocksDB" + self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target + self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target + self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}") + if self.options.with_gflags: + self.cpp_info.components["librocksdb"].requires.append("gflags::gflags") + if self.options.with_snappy: + self.cpp_info.components["librocksdb"].requires.append("snappy::snappy") + if self.options.with_lz4: + self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") + if self.options.with_zlib: + self.cpp_info.components["librocksdb"].requires.append("zlib::zlib") + if self.options.with_zstd: + self.cpp_info.components["librocksdb"].requires.append("zstd::zstd") + if self.options.get_safe("with_tbb"): + self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_jemalloc: + self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") diff --git a/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch b/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch new file mode 100644 index 00000000000..05725bf2c9a --- /dev/null +++ b/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch @@ -0,0 +1,30 @@ +--- a/include/rocksdb/utilities/checkpoint.h ++++ b/include/rocksdb/utilities/checkpoint.h +@@ -8,6 +8,7 @@ + #pragma once + #ifndef ROCKSDB_LITE + ++#include + #include + #include + #include "rocksdb/status.h" +--- a/table/block_based/data_block_hash_index.h ++++ b/table/block_based/data_block_hash_index.h +@@ -5,6 +5,7 @@ + + #pragma once + ++#include + #include + #include + +--- a/util/string_util.h ++++ b/util/string_util.h +@@ -6,6 +6,7 @@ + + #pragma once + ++#include + #include + #include + #include diff --git a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch new file mode 100644 index 00000000000..50ca9fa0e6d --- /dev/null +++ b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch @@ -0,0 +1,6 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ec59d4491..5d340fab7 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -103 +102,0 @@ if(MSVC) +- include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) From c820a543450bd3aa59460390898b22488ee0e7e6 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 9 Mar 2024 13:41:54 -0600 Subject: [PATCH 02/12] Teach consumers how to build RocksDB --- BUILD.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/BUILD.md b/BUILD.md index 4848524e899..60ee31d42d1 100644 --- a/BUILD.md +++ b/BUILD.md @@ -147,6 +147,13 @@ which allows you to statically link it with GCC, if you want. conan export external/snappy snappy/1.1.10@ ``` +Export our [Conan recipe for RocksDB](./external/rocksdb). +It does not override paths to dependencies when building with Visual Studio. + + ``` + conan export external/rocksdb rocksdb/6.29.5@ + ``` + Export our [Conan recipe for SOCI](./external/soci). It patches their CMake to correctly import its dependencies. From e0d10eed4a188a418bdc703adc92e11280daf051 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 9 Mar 2024 13:42:01 -0600 Subject: [PATCH 03/12] Build RocksDB in CI --- .github/actions/dependencies/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 3147e8774f2..c61f5b07e56 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -13,6 +13,7 @@ runs: shell: bash run: | conan export external/snappy snappy/1.1.10@ + conan export external/rocksdb rocksdb/6.29.5@ conan export external/soci soci/4.0.3@ - name: install dependencies shell: bash From d374f642f43b44f964858b98aadecde8046bd8a2 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 08:40:22 -0500 Subject: [PATCH 04/12] Try to fix RocksDB patch --- .../6.29.5-0002-exclude-thirdparty.patch | 151 +++++++++++++++++- 1 file changed, 149 insertions(+), 2 deletions(-) diff --git a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch index 50ca9fa0e6d..b6097e32eab 100644 --- a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch +++ b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch @@ -1,6 +1,153 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index ec59d4491..5d340fab7 100644 +index ec59d4491..f7e7812b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -103 +102,0 @@ if(MSVC) +@@ -101 +100,0 @@ if(MSVC) +- option(WITH_GFLAGS "build with GFlags" OFF) +@@ -103 +102,6 @@ if(MSVC) - include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) ++endif() ++ ++if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") ++ # FreeBSD has jemalloc as default malloc ++ # but it does not have all the jemalloc files in include/... ++ set(WITH_JEMALLOC ON) +@@ -105,10 +109,4 @@ else() +- if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") +- # FreeBSD has jemalloc as default malloc +- # but it does not have all the jemalloc files in include/... +- set(WITH_JEMALLOC ON) +- else() +- if(WITH_JEMALLOC) +- find_package(JeMalloc REQUIRED) +- add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) +- list(APPEND THIRDPARTY_LIBS JeMalloc::JeMalloc) +- endif() ++ if(WITH_JEMALLOC) ++ find_package(JeMalloc REQUIRED) ++ add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) ++ list(APPEND THIRDPARTY_LIBS JeMalloc::JeMalloc) +@@ -115,0 +114 @@ else() ++endif() +@@ -117,18 +116,14 @@ else() +- if(MINGW) +- option(WITH_GFLAGS "build with GFlags" OFF) +- else() +- option(WITH_GFLAGS "build with GFlags" ON) +- endif() +- set(GFLAGS_LIB) +- if(WITH_GFLAGS) +- # Config with namespace available since gflags 2.2.2 +- option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON) +- find_package(gflags CONFIG) +- if(gflags_FOUND) +- if(TARGET ${GFLAGS_TARGET}) +- # Config with GFLAGS_TARGET available since gflags 2.2.0 +- set(GFLAGS_LIB ${GFLAGS_TARGET}) +- else() +- # Config with GFLAGS_LIBRARIES available since gflags 2.1.0 +- set(GFLAGS_LIB ${gflags_LIBRARIES}) +- endif() ++if(MINGW OR MSVC) ++ option(WITH_GFLAGS "build with GFlags" OFF) ++else() ++ option(WITH_GFLAGS "build with GFlags" ON) ++endif() ++set(GFLAGS_LIB) ++if(WITH_GFLAGS) ++ # Config with namespace available since gflags 2.2.2 ++ option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON) ++ find_package(gflags CONFIG) ++ if(gflags_FOUND) ++ if(TARGET ${GFLAGS_TARGET}) ++ # Config with GFLAGS_TARGET available since gflags 2.2.0 ++ set(GFLAGS_LIB ${GFLAGS_TARGET}) +@@ -136,2 +131,2 @@ else() +- find_package(gflags REQUIRED) +- set(GFLAGS_LIB gflags::gflags) ++ # Config with GFLAGS_LIBRARIES available since gflags 2.1.0 ++ set(GFLAGS_LIB ${gflags_LIBRARIES}) +@@ -139,3 +134,3 @@ else() +- include_directories(${GFLAGS_INCLUDE_DIR}) +- list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB}) +- add_definitions(-DGFLAGS=1) ++ else() ++ find_package(gflags REQUIRED) ++ set(GFLAGS_LIB gflags::gflags) +@@ -142,0 +138,4 @@ else() ++ include_directories(${GFLAGS_INCLUDE_DIR}) ++ list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB}) ++ add_definitions(-DGFLAGS=1) ++endif() +@@ -144,7 +143,4 @@ else() +- if(WITH_SNAPPY) +- find_package(Snappy CONFIG) +- if(NOT Snappy_FOUND) +- find_package(Snappy REQUIRED) +- endif() +- add_definitions(-DSNAPPY) +- list(APPEND THIRDPARTY_LIBS Snappy::snappy) ++if(WITH_SNAPPY) ++ find_package(Snappy CONFIG) ++ if(NOT Snappy_FOUND) ++ find_package(Snappy REQUIRED) +@@ -151,0 +148,3 @@ else() ++ add_definitions(-DSNAPPY) ++ list(APPEND THIRDPARTY_LIBS Snappy::snappy) ++endif() +@@ -153,5 +152,5 @@ else() +- if(WITH_ZLIB) +- find_package(ZLIB REQUIRED) +- add_definitions(-DZLIB) +- list(APPEND THIRDPARTY_LIBS ZLIB::ZLIB) +- endif() ++if(WITH_ZLIB) ++ find_package(ZLIB REQUIRED) ++ add_definitions(-DZLIB) ++ list(APPEND THIRDPARTY_LIBS ZLIB::ZLIB) ++endif() +@@ -159,10 +158,8 @@ else() +- option(WITH_BZ2 "build with bzip2" OFF) +- if(WITH_BZ2) +- find_package(BZip2 REQUIRED) +- add_definitions(-DBZIP2) +- if(BZIP2_INCLUDE_DIRS) +- include_directories(${BZIP2_INCLUDE_DIRS}) +- else() +- include_directories(${BZIP2_INCLUDE_DIR}) +- endif() +- list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) ++option(WITH_BZ2 "build with bzip2" OFF) ++if(WITH_BZ2) ++ find_package(BZip2 REQUIRED) ++ add_definitions(-DBZIP2) ++ if(BZIP2_INCLUDE_DIRS) ++ include_directories(${BZIP2_INCLUDE_DIRS}) ++ else() ++ include_directories(${BZIP2_INCLUDE_DIR}) +@@ -169,0 +167,2 @@ else() ++ list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) ++endif() +@@ -171,5 +170,5 @@ else() +- if(WITH_LZ4) +- find_package(lz4 REQUIRED) +- add_definitions(-DLZ4) +- list(APPEND THIRDPARTY_LIBS lz4::lz4) +- endif() ++if(WITH_LZ4) ++ find_package(lz4 REQUIRED) ++ add_definitions(-DLZ4) ++ list(APPEND THIRDPARTY_LIBS lz4::lz4) ++endif() +@@ -177,6 +176,5 @@ else() +- if(WITH_ZSTD) +- find_package(zstd REQUIRED) +- add_definitions(-DZSTD) +- include_directories(${ZSTD_INCLUDE_DIR}) +- list(APPEND THIRDPARTY_LIBS zstd::zstd) +- endif() ++if(WITH_ZSTD) ++ find_package(zstd REQUIRED) ++ add_definitions(-DZSTD) ++ include_directories(${ZSTD_INCLUDE_DIR}) ++ list(APPEND THIRDPARTY_LIBS zstd::zstd) From dbeb557622957b9c6518c56cc449750b4fdbe351 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 09:21:42 -0500 Subject: [PATCH 05/12] Try again --- .../6.29.5-0002-exclude-thirdparty.patch | 151 +----------------- 1 file changed, 7 insertions(+), 144 deletions(-) diff --git a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch index b6097e32eab..fb0dd0c46b4 100644 --- a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch +++ b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch @@ -1,153 +1,16 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index ec59d4491..f7e7812b5 100644 +index ec59d4491..35577c998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101 +100,0 @@ if(MSVC) - option(WITH_GFLAGS "build with GFlags" OFF) -@@ -103 +102,6 @@ if(MSVC) +@@ -103,2 +102,2 @@ if(MSVC) - include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) +-else() +endif() + -+if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") -+ # FreeBSD has jemalloc as default malloc -+ # but it does not have all the jemalloc files in include/... -+ set(WITH_JEMALLOC ON) -@@ -105,10 +109,4 @@ else() -- if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") -- # FreeBSD has jemalloc as default malloc -- # but it does not have all the jemalloc files in include/... -- set(WITH_JEMALLOC ON) -- else() -- if(WITH_JEMALLOC) -- find_package(JeMalloc REQUIRED) -- add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) -- list(APPEND THIRDPARTY_LIBS JeMalloc::JeMalloc) -- endif() -+ if(WITH_JEMALLOC) -+ find_package(JeMalloc REQUIRED) -+ add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) -+ list(APPEND THIRDPARTY_LIBS JeMalloc::JeMalloc) -@@ -115,0 +114 @@ else() -+endif() -@@ -117,18 +116,14 @@ else() +@@ -117 +116 @@ else() - if(MINGW) -- option(WITH_GFLAGS "build with GFlags" OFF) -- else() -- option(WITH_GFLAGS "build with GFlags" ON) -- endif() -- set(GFLAGS_LIB) -- if(WITH_GFLAGS) -- # Config with namespace available since gflags 2.2.2 -- option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON) -- find_package(gflags CONFIG) -- if(gflags_FOUND) -- if(TARGET ${GFLAGS_TARGET}) -- # Config with GFLAGS_TARGET available since gflags 2.2.0 -- set(GFLAGS_LIB ${GFLAGS_TARGET}) -- else() -- # Config with GFLAGS_LIBRARIES available since gflags 2.1.0 -- set(GFLAGS_LIB ${gflags_LIBRARIES}) -- endif() -+if(MINGW OR MSVC) -+ option(WITH_GFLAGS "build with GFlags" OFF) -+else() -+ option(WITH_GFLAGS "build with GFlags" ON) -+endif() -+set(GFLAGS_LIB) -+if(WITH_GFLAGS) -+ # Config with namespace available since gflags 2.2.2 -+ option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON) -+ find_package(gflags CONFIG) -+ if(gflags_FOUND) -+ if(TARGET ${GFLAGS_TARGET}) -+ # Config with GFLAGS_TARGET available since gflags 2.2.0 -+ set(GFLAGS_LIB ${GFLAGS_TARGET}) -@@ -136,2 +131,2 @@ else() -- find_package(gflags REQUIRED) -- set(GFLAGS_LIB gflags::gflags) -+ # Config with GFLAGS_LIBRARIES available since gflags 2.1.0 -+ set(GFLAGS_LIB ${gflags_LIBRARIES}) -@@ -139,3 +134,3 @@ else() -- include_directories(${GFLAGS_INCLUDE_DIR}) -- list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB}) -- add_definitions(-DGFLAGS=1) -+ else() -+ find_package(gflags REQUIRED) -+ set(GFLAGS_LIB gflags::gflags) -@@ -142,0 +138,4 @@ else() -+ include_directories(${GFLAGS_INCLUDE_DIR}) -+ list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB}) -+ add_definitions(-DGFLAGS=1) -+endif() -@@ -144,7 +143,4 @@ else() -- if(WITH_SNAPPY) -- find_package(Snappy CONFIG) -- if(NOT Snappy_FOUND) -- find_package(Snappy REQUIRED) -- endif() -- add_definitions(-DSNAPPY) -- list(APPEND THIRDPARTY_LIBS Snappy::snappy) -+if(WITH_SNAPPY) -+ find_package(Snappy CONFIG) -+ if(NOT Snappy_FOUND) -+ find_package(Snappy REQUIRED) -@@ -151,0 +148,3 @@ else() -+ add_definitions(-DSNAPPY) -+ list(APPEND THIRDPARTY_LIBS Snappy::snappy) -+endif() -@@ -153,5 +152,5 @@ else() -- if(WITH_ZLIB) -- find_package(ZLIB REQUIRED) -- add_definitions(-DZLIB) -- list(APPEND THIRDPARTY_LIBS ZLIB::ZLIB) -- endif() -+if(WITH_ZLIB) -+ find_package(ZLIB REQUIRED) -+ add_definitions(-DZLIB) -+ list(APPEND THIRDPARTY_LIBS ZLIB::ZLIB) -+endif() -@@ -159,10 +158,8 @@ else() -- option(WITH_BZ2 "build with bzip2" OFF) -- if(WITH_BZ2) -- find_package(BZip2 REQUIRED) -- add_definitions(-DBZIP2) -- if(BZIP2_INCLUDE_DIRS) -- include_directories(${BZIP2_INCLUDE_DIRS}) -- else() -- include_directories(${BZIP2_INCLUDE_DIR}) -- endif() -- list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) -+option(WITH_BZ2 "build with bzip2" OFF) -+if(WITH_BZ2) -+ find_package(BZip2 REQUIRED) -+ add_definitions(-DBZIP2) -+ if(BZIP2_INCLUDE_DIRS) -+ include_directories(${BZIP2_INCLUDE_DIRS}) -+ else() -+ include_directories(${BZIP2_INCLUDE_DIR}) -@@ -169,0 +167,2 @@ else() -+ list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) -+endif() -@@ -171,5 +170,5 @@ else() -- if(WITH_LZ4) -- find_package(lz4 REQUIRED) -- add_definitions(-DLZ4) -- list(APPEND THIRDPARTY_LIBS lz4::lz4) -- endif() -+if(WITH_LZ4) -+ find_package(lz4 REQUIRED) -+ add_definitions(-DLZ4) -+ list(APPEND THIRDPARTY_LIBS lz4::lz4) -+endif() -@@ -177,6 +176,5 @@ else() -- if(WITH_ZSTD) -- find_package(zstd REQUIRED) -- add_definitions(-DZSTD) -- include_directories(${ZSTD_INCLUDE_DIR}) -- list(APPEND THIRDPARTY_LIBS zstd::zstd) -- endif() -+if(WITH_ZSTD) -+ find_package(zstd REQUIRED) -+ add_definitions(-DZSTD) -+ include_directories(${ZSTD_INCLUDE_DIR}) -+ list(APPEND THIRDPARTY_LIBS zstd::zstd) ++ if(MINGW OR MSVC) +@@ -183 +181,0 @@ else() +-endif() From 6de272e6b14ac0ec1898a3a79f0ea201ef96cfda Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 09:54:44 -0500 Subject: [PATCH 06/12] Enable revisions in Conan --- .github/actions/dependencies/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index c61f5b07e56..e65cc80b0ae 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -12,6 +12,7 @@ runs: - name: export custom recipes shell: bash run: | + conan config set general.revisions_enabled=1 conan export external/snappy snappy/1.1.10@ conan export external/rocksdb rocksdb/6.29.5@ conan export external/soci soci/4.0.3@ From 2f1044791a92fb8748a5b556339b07fd2db01e78 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 10:45:25 -0500 Subject: [PATCH 07/12] Rethink dependencies action --- .github/actions/dependencies/action.yml | 21 +++++++++++++++++++++ .github/workflows/nix.yml | 20 -------------------- .github/workflows/windows.yml | 20 -------------------- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index e65cc80b0ae..047afa4417b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -16,6 +16,23 @@ runs: conan export external/snappy snappy/1.1.10@ conan export external/rocksdb rocksdb/6.29.5@ conan export external/soci soci/4.0.3@ + - name: try to authenticate to ripple Conan remote + id: remote + shell: bash + run: | + # Do not quote the URL. An empty string will be accepted (with + # a non-fatal warning), but a missing argument will not. + conan remote add ripple ${{ env.CONAN_URL }} --insert 0 + echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} \ + --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || \ + echo failure) | tee ${GITHUB_OUTPUT} + - name: list missing binaries + id: binaries + shell: bash + # Print the list of dependencies that would need to be built locally. + # A non-empty list means we have "failed" to cache binaries remotely. + run: | + echo missing=$(conan info . --build missing --settings build_type=${{ inputs.configuration }} --json 2>/dev/null | grep '^\[') | tee ${GITHUB_OUTPUT} - name: install dependencies shell: bash run: | @@ -26,3 +43,7 @@ runs: --build missing \ --settings build_type=${{ inputs.configuration }} \ .. + - name: upload dependencies to remote + if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success') + shell: bash + run: conan upload --remote ripple '*' --all --parallel --confirm diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index b494b88718c..7f117498064 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -73,33 +73,13 @@ jobs: conan profile update env.CC=${{ matrix.profile.cc }} default conan profile update env.CXX=${{ matrix.profile.cxx }} default conan profile update conf.tools.build:compiler_executables='{"c": "${{ matrix.profile.cc }}", "cpp": "${{ matrix.profile.cxx }}"}' default - # Do not quote the URL. An empty string will be accepted (with - # a non-fatal warning), but a missing argument will not. - conan remote add ripple ${{ env.CONAN_URL }} --insert 0 - - name: try to authenticate to ripple Conan remote - id: remote - run: | - echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || echo failure) | tee ${GITHUB_OUTPUT} - name: archive profile # Create this archive before dependencies are added to the local cache. run: tar -czf conan.tar -C ~/.conan . - - name: list missing binaries - id: binaries - # Print the list of dependencies that would need to be built locally. - # A non-empty list means we have "failed" to cache binaries remotely. - run: | - echo missing=$(conan info . --build missing --settings build_type=${{ matrix.configuration }} --json 2>/dev/null | grep '^\[') | tee ${GITHUB_OUTPUT} - name: build dependencies - if: (steps.binaries.outputs.missing != '[]') uses: ./.github/actions/dependencies with: configuration: ${{ matrix.configuration }} - - name: upload dependencies to remote - if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success') - run: conan upload --remote ripple '*' --all --parallel --confirm - - name: recreate archive with dependencies - if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'failure') - run: tar -czf conan.tar -C ~/.conan . - name: upload archive uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e0663b3d18d..e1586e1b415 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -59,30 +59,10 @@ jobs: run: | conan profile new default --detect conan profile update settings.compiler.runtime=MT${{ matrix.configuration == 'Debug' && 'd' || '' }} default - # Do not quote the URL. An empty string will be accepted (with - # a non-fatal warning), but a missing argument will not. - conan remote add ripple ${{ env.CONAN_URL }} --insert 0 - - name: try to authenticate to ripple Conan remote - shell: bash - id: remote - run: | - echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} \ - --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || \ - echo failure) | tee ${GITHUB_OUTPUT} - - name: list missing binaries - id: binaries - shell: bash - # Print the list of dependencies that would need to be built locally. - # A non-empty list means we have "failed" to cache binaries remotely. - run: | - echo missing=$(conan info . --build missing --settings build_type=${{ matrix.configuration }} --json 2>/dev/null | grep '^\[') | tee ${GITHUB_OUTPUT} - name: build dependencies uses: ./.github/actions/dependencies with: configuration: ${{ matrix.configuration }} - - name: upload dependencies to remote - if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success') - run: conan upload --remote ripple '*' --all --parallel --confirm - name: build uses: ./.github/actions/build with: From 5630fd5542aa19bd4b9f0504b3906c103cd75661 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 10:52:21 -0500 Subject: [PATCH 08/12] Pass secrets to composite action --- .github/workflows/macos.yml | 3 ++- .github/workflows/nix.yml | 1 + .github/workflows/windows.yml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 120a6ec1782..d18bcea2d74 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -37,10 +37,11 @@ jobs: run : | conan profile get env.CXXFLAGS default || true conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default - - name: dependencies + - name: build dependencies uses: ./.github/actions/dependencies with: configuration: ${{ matrix.configuration }} + secrets: inherit - name: build uses: ./.github/actions/build with: diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 7f117498064..734fc5ef487 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -80,6 +80,7 @@ jobs: uses: ./.github/actions/dependencies with: configuration: ${{ matrix.configuration }} + secrets: inherit - name: upload archive uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e1586e1b415..c5d6b525605 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -63,6 +63,7 @@ jobs: uses: ./.github/actions/dependencies with: configuration: ${{ matrix.configuration }} + secrets: inherit - name: build uses: ./.github/actions/build with: From 9bb9d8f58671962ae93a09ad6ec0ae8059f077db Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 10:59:23 -0500 Subject: [PATCH 09/12] Try again to pass secrets to composite action --- .github/actions/dependencies/action.yml | 4 ++-- .github/workflows/macos.yml | 5 ++++- .github/workflows/nix.yml | 7 ++++--- .github/workflows/windows.yml | 7 ++++--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 047afa4417b..17dbc49c45d 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -23,8 +23,8 @@ runs: # Do not quote the URL. An empty string will be accepted (with # a non-fatal warning), but a missing argument will not. conan remote add ripple ${{ env.CONAN_URL }} --insert 0 - echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} \ - --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || \ + echo outcome=$(conan user --remote ripple ${{ env.CONAN_USERNAME }} \ + --password ${{ env.CONAN_TOKEN }} >&2 && echo success || \ echo failure) | tee ${GITHUB_OUTPUT} - name: list missing binaries id: binaries diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d18bcea2d74..d1e00ed7895 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -39,9 +39,12 @@ jobs: conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default - name: build dependencies uses: ./.github/actions/dependencies + env: + CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod + CONAN_USERNAME: ${{ secrets.CONAN_USERNAME }} + CONAN_TOKEN: ${{ secrets.CONAN_TOKEN }} with: configuration: ${{ matrix.configuration }} - secrets: inherit - name: build uses: ./.github/actions/build with: diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 734fc5ef487..0da8cc25622 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -62,8 +62,6 @@ jobs: cmake --version env | sort - name: configure Conan - env: - CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod run: | conan profile new default --detect conan profile update settings.compiler.cppstd=20 default @@ -78,9 +76,12 @@ jobs: run: tar -czf conan.tar -C ~/.conan . - name: build dependencies uses: ./.github/actions/dependencies + env: + CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod + CONAN_USERNAME: ${{ secrets.CONAN_USERNAME }} + CONAN_TOKEN: ${{ secrets.CONAN_TOKEN }} with: configuration: ${{ matrix.configuration }} - secrets: inherit - name: upload archive uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index c5d6b525605..60419944ed6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,16 +54,17 @@ jobs: dir env: - name: configure Conan shell: bash - env: - CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod run: | conan profile new default --detect conan profile update settings.compiler.runtime=MT${{ matrix.configuration == 'Debug' && 'd' || '' }} default - name: build dependencies uses: ./.github/actions/dependencies + env: + CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod + CONAN_USERNAME: ${{ secrets.CONAN_USERNAME }} + CONAN_TOKEN: ${{ secrets.CONAN_TOKEN }} with: configuration: ${{ matrix.configuration }} - secrets: inherit - name: build uses: ./.github/actions/build with: From e94e41733d57878d5409947d1b26ec8e0c384357 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 11:03:49 -0500 Subject: [PATCH 10/12] Forgot action parameter --- .github/workflows/nix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 0da8cc25622..e8f9f373b2d 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -130,6 +130,8 @@ jobs: uses: actions/checkout@v3 - name: dependencies uses: ./.github/actions/dependencies + env: + CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod with: configuration: ${{ matrix.configuration }} - name: build From 82526485bed361f6ae473f7b033b5d78ff886944 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 11:07:00 -0500 Subject: [PATCH 11/12] Do not add duplicate remote on macOS --- .github/workflows/macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d1e00ed7895..3462939c987 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -37,6 +37,7 @@ jobs: run : | conan profile get env.CXXFLAGS default || true conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default + conan remote remove ripple - name: build dependencies uses: ./.github/actions/dependencies env: From 58eafe1670fa8ac9cc0a9b220df9fd974c644a94 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 11:16:18 -0500 Subject: [PATCH 12/12] Forgot an action parameter --- .github/workflows/nix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index e8f9f373b2d..97edbeb17f3 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -181,6 +181,8 @@ jobs: uses: actions/checkout@v3 - name: dependencies uses: ./.github/actions/dependencies + env: + CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod with: configuration: ${{ matrix.configuration }} - name: build