From 16be108d16519d1e428cf8119fef5df5dbe782f9 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 25 Apr 2023 12:53:47 -0500 Subject: [PATCH] Add patched recipe for SOCI See https://github.com/conan-io/conan-center-index/pull/17026#issuecomment-1522119607 Add patched recipe for SOCI: (#4510) SOCI is the C++ database access library. The SOCI recipe was updated in Conan Center Index (CCI), and it breaks for our choice of options. This breakage occurs when you build with a fresh Conan cache (e.g. when you submit a PR, or delete `~/.conan/data`). * Add a custom Conan recipe for SOCI v4.0.3 * Update dependency building to handle exporting and installing Snappy and SOCI * Fix workflows to use custom SOCI recipe * Update BUILD.md to include instruction for exporting the SOCI Conan recipe: * `conan export external/soci soci/4.0.3@` This solution has been verified on Ubuntu 20.04 and macOS. Context: * There is a compiler error that the `sqlite3.h` header is not available when building soci. * When package B depends on package A, it finds the pieces it needs by importing the Package Configuration File (PCF) that Conan generates for package A. * Read the CMake written by package B to check that it is importing the PCF correctly and linking its exports correctly. * Since this can be difficult, it is often more efficient to check https://github.com/conan-io/conan-center-index/issues for package B to see if anyone else has seen a similar problem. * One of the issues points to a problem area in soci's CMake. To confirm the diagnosis, review soci's CMake (after any patches are applied) in the Conan build directory `build/$buildId/src/`. * Review the Conan-generated PCF in `build/$buildId/build/$buildType/generators/`. * In this case, the problem was likely (re)introduced by https://github.com/conan-io/conan-center-index/pull/17026 * If there is a problem in the source or in the Conan recipe, the fastest fix is to copy the recipe and either: * Add a source patch to fix any problems in the source. * Change the recipe to fix any problems in the recipe. * In this case, this can be done by finding soci's Conan recipe at https://github.com/conan-io/conan-center-index/tree/master/recipes/soci and then copying the `all` directory as `external/$packageName` in our project. Then, make any changes. * Test packages can be removed from the recipe folder as they are not needed. * If adding a patch in the `patches` directory, add a description for it to `conandata.yml`. * Since `conanfile.py` has no `version` property on the recipe class, builders need to pass a version on the command line (like they do for our `snappy` recipe). * Add an example command to `BUILD.md`. Future work: It may make sense to refer to recipes by revision, by checking in a lockfile. --- .github/actions/build/action.yml | 33 +++ .github/actions/dependencies/action.yml | 23 ++ .github/workflows/nix.yml | 59 +++++ .github/workflows/windows.yml | 6 +- BUILD.md | 7 + external/soci/conandata.yml | 12 + external/soci/conanfile.py | 212 ++++++++++++++++++ ...-INSTALL_NAME_DIR-for-relocatable-li.patch | 39 ++++ .../soci/patches/0002-Fix-soci_backend.patch | 24 ++ 9 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 .github/actions/build/action.yml create mode 100644 .github/actions/dependencies/action.yml create mode 100644 external/soci/conandata.yml create mode 100644 external/soci/conanfile.py create mode 100644 external/soci/patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch create mode 100644 external/soci/patches/0002-Fix-soci_backend.patch diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 00000000000..2c4135e0cf1 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,33 @@ +name: build +inputs: + generator: + default: null + configuration: + required: true + cmake-args: + default: null +# An implicit input is the environment variable `build_dir`. +runs: + using: composite + steps: + - name: dependencies + uses: ./.github/actions/dependencies + with: + configuration: ${{ inputs.configuration }} + - name: configure + shell: bash + run: | + cd ${build_dir} + cmake \ + ${{ inputs.generator && format('-G {0}', inputs.generator) || '' }} \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_BUILD_TYPE=${{ inputs.configuration }} \ + ${{ inputs.cmake-args }} \ + .. + - name: build + shell: bash + run: | + cmake \ + --build ${build_dir} \ + --config ${{ inputs.configuration }} \ + --parallel ${NUM_PROCESSORS:-$(nproc)} diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml new file mode 100644 index 00000000000..af570f21b82 --- /dev/null +++ b/.github/actions/dependencies/action.yml @@ -0,0 +1,23 @@ +name: dependencies +inputs: + configuration: + required: true +# An implicit input is the environment variable `build_dir`. +runs: + using: composite + steps: + - name: export custom recipes + shell: bash + run: | + conan export external/snappy snappy/1.1.9@ + conan export external/soci soci/4.0.3@ + - name: install dependencies + shell: bash + run: | + mkdir ${build_dir} + cd ${build_dir} + conan install \ + --output-folder . \ + --build missing \ + --settings build_type=${{ inputs.configuration }} \ + .. diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 8ccb1f52733..d7de297c13c 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -60,6 +60,7 @@ jobs: if: runner.os == 'Linux' run: | conan profile update settings.compiler.libcxx=libstdc++11 default +<<<<<<< HEAD - name: learn Conan cache directory id: conan-cache run: | @@ -88,6 +89,64 @@ jobs: -Dreporting=OFF \ -Dunity=OFF \ .. +======= + conan profile update env.CC=${{ matrix.profile.cc }} default + conan profile update env.CXX=${{ matrix.profile.cxx }} default + conan profile update conf.tools.build:compiler_executables='{"c": "${{ matrix.profile.cc }}", "cpp": "${{ matrix.profile.cxx }}"}' default + - name: checkout + uses: actions/checkout@v3 + - name: dependencies + uses: ./.github/actions/dependencies + with: + configuration: ${{ matrix.configuration }} + - name: archive cache + run: tar -czf conan.tar -C ~/.conan . + - name: upload cache + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }} + path: conan.tar + + + test: + strategy: + fail-fast: false + matrix: + platform: + - linux + compiler: + - gcc + - clang + configuration: + - Debug + - Release + cmake-args: + - + - "-Dunity=ON" + needs: dependencies + runs-on: [self-hosted, heavy] + container: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b + env: + build_dir: .build + steps: + - name: download cache + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }} + - name: extract cache + run: | + mkdir -p ~/.conan + tar -xzf conan.tar -C ~/.conan + - name: check environment + run: | + echo ${PATH} | tr ':' '\n' + conan --version + cmake --version + env + ls ~/.conan + - name: checkout + uses: actions/checkout@v3 +>>>>>>> c7ef4c978 (Add patched recipe for SOCI: (#4510)) - name: build run: | cmake --build ${build_dir} --target rippled --parallel $(nproc) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 51fdc5d27cb..1c24f65e600 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -69,8 +69,10 @@ jobs: with: path: ${{ steps.conan-cache.outputs.dir }} key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/windows.yml') }} - - name: export Snappy - run: conan export external/snappy snappy/1.1.9@ + - name: export custom recipes + run: | + conan export external/snappy snappy/1.1.9@ + conan export external/soci soci/4.0.3@ - name: install dependencies run: | mkdir $env:build_dir diff --git a/BUILD.md b/BUILD.md index 3a7283407d9..be310f32a00 100644 --- a/BUILD.md +++ b/BUILD.md @@ -127,6 +127,13 @@ which allows you to statically link it with GCC, if you want. conan export external/snappy snappy/1.1.9@ ``` +Export our [Conan recipe for SOCI](./external/soci). +It patches their CMake to correctly import its dependencies. + +``` +conan export external/soci soci/4.0.3@ +``` + ## How to build and test Let's start with a couple of examples of common workflows. diff --git a/external/soci/conandata.yml b/external/soci/conandata.yml new file mode 100644 index 00000000000..6eb59aaffa2 --- /dev/null +++ b/external/soci/conandata.yml @@ -0,0 +1,12 @@ +sources: + "4.0.3": + url: "https://github.com/SOCI/soci/archive/v4.0.3.tar.gz" + sha256: "4b1ff9c8545c5d802fbe06ee6cd2886630e5c03bf740e269bb625b45cf934928" +patches: + "4.0.3": + - patch_file: "patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch" + patch_description: "Generate relocatable libraries on MacOS" + patch_type: "portability" + - patch_file: "patches/0002-Fix-soci_backend.patch" + patch_description: "Fix variable names for dependencies" + patch_type: "conan" diff --git a/external/soci/conanfile.py b/external/soci/conanfile.py new file mode 100644 index 00000000000..67c572d5ad8 --- /dev/null +++ b/external/soci/conanfile.py @@ -0,0 +1,212 @@ +from conan import ConanFile +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, rmdir +from conan.tools.microsoft import is_msvc +from conan.tools.scm import Version +from conan.errors import ConanInvalidConfiguration +import os + +required_conan_version = ">=1.55.0" + + +class SociConan(ConanFile): + name = "soci" + homepage = "https://github.com/SOCI/soci" + url = "https://github.com/conan-io/conan-center-index" + description = "The C++ Database Access Library " + topics = ("mysql", "odbc", "postgresql", "sqlite3") + license = "BSL-1.0" + + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "empty": [True, False], + "with_sqlite3": [True, False], + "with_db2": [True, False], + "with_odbc": [True, False], + "with_oracle": [True, False], + "with_firebird": [True, False], + "with_mysql": [True, False], + "with_postgresql": [True, False], + "with_boost": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "empty": False, + "with_sqlite3": False, + "with_db2": False, + "with_odbc": False, + "with_oracle": False, + "with_firebird": False, + "with_mysql": False, + "with_postgresql": False, + "with_boost": False, + } + + def export_sources(self): + export_conandata_patches(self) + + def layout(self): + cmake_layout(self, src_folder="src") + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def requirements(self): + if self.options.with_sqlite3: + self.requires("sqlite3/3.41.1") + if self.options.with_odbc and self.settings.os != "Windows": + self.requires("odbc/2.3.11") + if self.options.with_mysql: + self.requires("libmysqlclient/8.0.31") + if self.options.with_postgresql: + self.requires("libpq/14.7") + if self.options.with_boost: + self.requires("boost/1.81.0") + + @property + def _minimum_compilers_version(self): + return { + "Visual Studio": "14", + "gcc": "4.8", + "clang": "3.8", + "apple-clang": "8.0" + } + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, 11) + + compiler = str(self.settings.compiler) + compiler_version = Version(self.settings.compiler.version.value) + if compiler not in self._minimum_compilers_version: + self.output.warning("{} recipe lacks information about the {} compiler support.".format(self.name, self.settings.compiler)) + elif compiler_version < self._minimum_compilers_version[compiler]: + raise ConanInvalidConfiguration("{} requires a {} version >= {}".format(self.name, compiler, compiler_version)) + + prefix = "Dependencies for" + message = "not configured in this conan package." + if self.options.with_db2: + # self.requires("db2/0.0.0") # TODO add support for db2 + raise ConanInvalidConfiguration("{} DB2 {} ".format(prefix, message)) + if self.options.with_oracle: + # self.requires("oracle_db/0.0.0") # TODO add support for oracle + raise ConanInvalidConfiguration("{} ORACLE {} ".format(prefix, message)) + if self.options.with_firebird: + # self.requires("firebird/0.0.0") # TODO add support for firebird + raise ConanInvalidConfiguration("{} firebird {} ".format(prefix, message)) + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + + tc.variables["SOCI_SHARED"] = self.options.shared + tc.variables["SOCI_STATIC"] = not self.options.shared + tc.variables["SOCI_TESTS"] = False + tc.variables["SOCI_CXX11"] = True + tc.variables["SOCI_EMPTY"] = self.options.empty + tc.variables["WITH_SQLITE3"] = self.options.with_sqlite3 + tc.variables["WITH_DB2"] = self.options.with_db2 + tc.variables["WITH_ODBC"] = self.options.with_odbc + tc.variables["WITH_ORACLE"] = self.options.with_oracle + tc.variables["WITH_FIREBIRD"] = self.options.with_firebird + tc.variables["WITH_MYSQL"] = self.options.with_mysql + tc.variables["WITH_POSTGRESQL"] = self.options.with_postgresql + tc.variables["WITH_BOOST"] = self.options.with_boost + tc.generate() + + deps = CMakeDeps(self) + deps.generate() + + def build(self): + apply_conandata_patches(self) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + copy(self, "LICENSE_1_0.txt", dst=os.path.join(self.package_folder, "licenses"), src=self.source_folder) + + cmake = CMake(self) + cmake.install() + + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + + def package_info(self): + self.cpp_info.set_property("cmake_file_name", "SOCI") + + target_suffix = "" if self.options.shared else "_static" + lib_prefix = "lib" if is_msvc(self) and not self.options.shared else "" + version = Version(self.version) + lib_suffix = "_{}_{}".format(version.major, version.minor) if self.settings.os == "Windows" else "" + + # soci_core + self.cpp_info.components["soci_core"].set_property("cmake_target_name", "SOCI::soci_core{}".format(target_suffix)) + self.cpp_info.components["soci_core"].libs = ["{}soci_core{}".format(lib_prefix, lib_suffix)] + if self.options.with_boost: + self.cpp_info.components["soci_core"].requires.append("boost::boost") + + # soci_empty + if self.options.empty: + self.cpp_info.components["soci_empty"].set_property("cmake_target_name", "SOCI::soci_empty{}".format(target_suffix)) + self.cpp_info.components["soci_empty"].libs = ["{}soci_empty{}".format(lib_prefix, lib_suffix)] + self.cpp_info.components["soci_empty"].requires = ["soci_core"] + + # soci_sqlite3 + if self.options.with_sqlite3: + self.cpp_info.components["soci_sqlite3"].set_property("cmake_target_name", "SOCI::soci_sqlite3{}".format(target_suffix)) + self.cpp_info.components["soci_sqlite3"].libs = ["{}soci_sqlite3{}".format(lib_prefix, lib_suffix)] + self.cpp_info.components["soci_sqlite3"].requires = ["soci_core", "sqlite3::sqlite3"] + + # soci_odbc + if self.options.with_odbc: + self.cpp_info.components["soci_odbc"].set_property("cmake_target_name", "SOCI::soci_odbc{}".format(target_suffix)) + self.cpp_info.components["soci_odbc"].libs = ["{}soci_odbc{}".format(lib_prefix, lib_suffix)] + self.cpp_info.components["soci_odbc"].requires = ["soci_core"] + if self.settings.os == "Windows": + self.cpp_info.components["soci_odbc"].system_libs.append("odbc32") + else: + self.cpp_info.components["soci_odbc"].requires.append("odbc::odbc") + + # soci_mysql + if self.options.with_mysql: + self.cpp_info.components["soci_mysql"].set_property("cmake_target_name", "SOCI::soci_mysql{}".format(target_suffix)) + self.cpp_info.components["soci_mysql"].libs = ["{}soci_mysql{}".format(lib_prefix, lib_suffix)] + self.cpp_info.components["soci_mysql"].requires = ["soci_core", "libmysqlclient::libmysqlclient"] + + # soci_postgresql + if self.options.with_postgresql: + self.cpp_info.components["soci_postgresql"].set_property("cmake_target_name", "SOCI::soci_postgresql{}".format(target_suffix)) + self.cpp_info.components["soci_postgresql"].libs = ["{}soci_postgresql{}".format(lib_prefix, lib_suffix)] + self.cpp_info.components["soci_postgresql"].requires = ["soci_core", "libpq::libpq"] + + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "SOCI" + self.cpp_info.names["cmake_find_package_multi"] = "SOCI" + self.cpp_info.components["soci_core"].names["cmake_find_package"] = "soci_core{}".format(target_suffix) + self.cpp_info.components["soci_core"].names["cmake_find_package_multi"] = "soci_core{}".format(target_suffix) + if self.options.empty: + self.cpp_info.components["soci_empty"].names["cmake_find_package"] = "soci_empty{}".format(target_suffix) + self.cpp_info.components["soci_empty"].names["cmake_find_package_multi"] = "soci_empty{}".format(target_suffix) + if self.options.with_sqlite3: + self.cpp_info.components["soci_sqlite3"].names["cmake_find_package"] = "soci_sqlite3{}".format(target_suffix) + self.cpp_info.components["soci_sqlite3"].names["cmake_find_package_multi"] = "soci_sqlite3{}".format(target_suffix) + if self.options.with_odbc: + self.cpp_info.components["soci_odbc"].names["cmake_find_package"] = "soci_odbc{}".format(target_suffix) + self.cpp_info.components["soci_odbc"].names["cmake_find_package_multi"] = "soci_odbc{}".format(target_suffix) + if self.options.with_mysql: + self.cpp_info.components["soci_mysql"].names["cmake_find_package"] = "soci_mysql{}".format(target_suffix) + self.cpp_info.components["soci_mysql"].names["cmake_find_package_multi"] = "soci_mysql{}".format(target_suffix) + if self.options.with_postgresql: + self.cpp_info.components["soci_postgresql"].names["cmake_find_package"] = "soci_postgresql{}".format(target_suffix) + self.cpp_info.components["soci_postgresql"].names["cmake_find_package_multi"] = "soci_postgresql{}".format(target_suffix) diff --git a/external/soci/patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch b/external/soci/patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch new file mode 100644 index 00000000000..5de0027f750 --- /dev/null +++ b/external/soci/patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch @@ -0,0 +1,39 @@ +From d491bf7b5040d314ffd0c6310ba01f78ff44c85e Mon Sep 17 00:00:00 2001 +From: Rasmus Thomsen +Date: Fri, 14 Apr 2023 09:16:29 +0200 +Subject: [PATCH] Remove hardcoded INSTALL_NAME_DIR for relocatable libraries + on MacOS + +--- + cmake/SociBackend.cmake | 2 +- + src/core/CMakeLists.txt | 1 - + 2 files changed, 1 insertion(+), 2 deletions(-) + +diff --git a/cmake/SociBackend.cmake b/cmake/SociBackend.cmake +index 5d4ef0df..39fe1f77 100644 +--- a/cmake/SociBackend.cmake ++++ b/cmake/SociBackend.cmake +@@ -171,7 +171,7 @@ macro(soci_backend NAME) + set_target_properties(${THIS_BACKEND_TARGET} + PROPERTIES + SOVERSION ${${PROJECT_NAME}_SOVERSION} +- INSTALL_NAME_DIR ${CMAKE_INSTALL_PREFIX}/lib) ++ ) + + if(APPLE) + set_target_properties(${THIS_BACKEND_TARGET} +diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt +index 3e7deeae..f9eae564 100644 +--- a/src/core/CMakeLists.txt ++++ b/src/core/CMakeLists.txt +@@ -59,7 +59,6 @@ if (SOCI_SHARED) + PROPERTIES + VERSION ${SOCI_VERSION} + SOVERSION ${SOCI_SOVERSION} +- INSTALL_NAME_DIR ${CMAKE_INSTALL_PREFIX}/lib + CLEAN_DIRECT_OUTPUT 1) + endif() + +-- +2.25.1 + diff --git a/external/soci/patches/0002-Fix-soci_backend.patch b/external/soci/patches/0002-Fix-soci_backend.patch new file mode 100644 index 00000000000..eab3c3763c0 --- /dev/null +++ b/external/soci/patches/0002-Fix-soci_backend.patch @@ -0,0 +1,24 @@ +diff --git a/cmake/SociBackend.cmake b/cmake/SociBackend.cmake +index 0a664667..3fa2ed95 100644 +--- a/cmake/SociBackend.cmake ++++ b/cmake/SociBackend.cmake +@@ -31,14 +31,13 @@ macro(soci_backend_deps_found NAME DEPS SUCCESS) + if(NOT DEPEND_FOUND) + list(APPEND DEPS_NOT_FOUND ${dep}) + else() +- string(TOUPPER "${dep}" DEPU) +- if( ${DEPU}_INCLUDE_DIR ) +- list(APPEND DEPS_INCLUDE_DIRS ${${DEPU}_INCLUDE_DIR}) ++ if( ${dep}_INCLUDE_DIR ) ++ list(APPEND DEPS_INCLUDE_DIRS ${${dep}_INCLUDE_DIR}) + endif() +- if( ${DEPU}_INCLUDE_DIRS ) +- list(APPEND DEPS_INCLUDE_DIRS ${${DEPU}_INCLUDE_DIRS}) ++ if( ${dep}_INCLUDE_DIRS ) ++ list(APPEND DEPS_INCLUDE_DIRS ${${dep}_INCLUDE_DIRS}) + endif() +- list(APPEND DEPS_LIBRARIES ${${DEPU}_LIBRARIES}) ++ list(APPEND DEPS_LIBRARIES ${${dep}_LIBRARIES}) + endif() + endforeach() +