From 289006c5fbd3cd9198d0949f64a0727f5f3ba80e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 8 Mar 2023 12:12:17 +0800 Subject: [PATCH] ci: Improve rocksdb build speed by link dynamic libs (#1502) * ci: Improve rocksdb build speed by link dynamic libs Signed-off-by: Xuanwo * Fix typo Signed-off-by: Xuanwo * Not for test Signed-off-by: Xuanwo * Add note for rocksdb Signed-off-by: Xuanwo * Allow using system lib Signed-off-by: Xuanwo * cover docs Signed-off-by: Xuanwo * lower version Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 11 + .github/workflows/docs.yml | 6 + .github/workflows/service_test_rocksdb.yml | 3 + Cargo.lock | 229 +++++++++++++-------- Cargo.toml | 9 +- src/services/rocksdb/backend.rs | 18 +- src/services/sled/backend.rs | 6 - 7 files changed, 188 insertions(+), 94 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77a078c5fe9b..9cd46eb5ad12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,9 @@ jobs: with: hdfs-version: "3.3.2" + - name: Install rocksdb + run: sudo apt install librocksdb-dev + - uses: Swatinem/rust-cache@v2 - name: Check license headers @@ -58,8 +61,12 @@ jobs: - name: Cargo format run: cargo fmt --all -- --check - name: Cargo clippy + env: + ROCKSDB_LIB_DIR: /usr/lib run: cargo clippy --all-targets --all-features --workspace -- -D warnings - name: Cargo doc + env: + ROCKSDB_LIB_DIR: /usr/lib run: cargo doc --lib --no-deps --all-features msrv_check: @@ -126,9 +133,13 @@ jobs: uses: beyondstorage/setup-hdfs@master with: hdfs-version: "3.3.2" + - name: Install rocksdb + run: sudo apt install librocksdb-dev - uses: Swatinem/rust-cache@v2 - name: Build + env: + ROCKSDB_LIB_DIR: /usr/lib run: cargo build --all-features unit: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b71b35eb1161..0eca3f57d18d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -33,12 +33,18 @@ jobs: distribution: temurin java-version: '11' + - name: Install rocksdb + run: sudo apt install librocksdb-dev + + - uses: Swatinem/rust-cache@v2 + - uses: actions-rs/cargo@v1 with: command: doc args: --lib --no-deps --all-features -p opendal env: LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }} + ROCKSDB_LIB_DIR: /usr/lib - name: Copy vercel.json run: cp ./vercel.json ./target/doc diff --git a/.github/workflows/service_test_rocksdb.yml b/.github/workflows/service_test_rocksdb.yml index f1c1a0031d12..42f50bbf6f5c 100644 --- a/.github/workflows/service_test_rocksdb.yml +++ b/.github/workflows/service_test_rocksdb.yml @@ -36,11 +36,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Install rocksdb + run: sudo apt install librocksdb-dev - uses: Swatinem/rust-cache@v2 - name: Test shell: bash run: cargo test rocksdb --features services-rocksdb -- --show-output --test-threads=1 env: + ROCKSDB_LIB_DIR: /usr/lib RUST_BACKTRACE: full RUST_LOG: debug OPENDAL_ROCKSDB_TEST: on diff --git a/Cargo.lock b/Cargo.lock index 80af4d57dde7..a263b12c6e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" version = "1.0.69" @@ -280,22 +289,26 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.64.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +checksum = "66c0bb6167449588ff70803f4127f0684f9063097eca5016f37eb52b92c2cf36" dependencies = [ "bitflags", "cexpr", + "cfg-if 0.1.10", "clang-sys", + "clap 2.34.0", + "env_logger 0.7.1", "lazy_static", "lazycell", + "log", "peeking_take_while", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn", + "which", ] [[package]] @@ -363,17 +376,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "camino" version = "1.1.3" @@ -422,13 +424,19 @@ dependencies = [ [[package]] name = "cexpr" -version = "0.6.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" dependencies = [ "nom", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -479,13 +487,28 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.0" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" +checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.5.2", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", ] [[package]] @@ -497,7 +520,7 @@ dependencies = [ "bitflags", "clap_lex 0.2.4", "indexmap", - "textwrap", + "textwrap 0.16.0", ] [[package]] @@ -510,7 +533,7 @@ dependencies = [ "clap_lex 0.3.2", "is-terminal", "once_cell", - "strsim", + "strsim 0.10.0", "termcolor", ] @@ -611,7 +634,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -658,7 +681,7 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -668,7 +691,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] @@ -680,7 +703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", "memoffset", "scopeguard", @@ -692,7 +715,7 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -765,7 +788,7 @@ version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown", "lock_api", "once_cell", @@ -882,7 +905,7 @@ version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -897,13 +920,26 @@ dependencies = [ "syn", ] +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + [[package]] name = "env_logger" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ - "humantime", + "humantime 2.1.0", "is-terminal", "log", "regex", @@ -1145,7 +1181,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1156,7 +1192,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] @@ -1348,6 +1384,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "humantime" version = "2.1.0" @@ -1477,7 +1522,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1626,13 +1671,23 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +dependencies = [ + "cc", + "winapi", +] + [[package]] name = "libloading" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "winapi", ] @@ -1644,27 +1699,14 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "librocksdb-sys" -version = "0.10.0+7.9.2" +version = "6.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe4d5874f5ff2bc616e55e8c6086d478fcda13faf9495768a4aa1c22042d30b" +checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09" dependencies = [ "bindgen", - "bzip2-sys", "cc", "glob", "libc", - "libz-sys", -] - -[[package]] -name = "libz-sys" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", ] [[package]] @@ -1704,7 +1746,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "value-bag", ] @@ -1818,12 +1860,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "mio" version = "0.8.6" @@ -1915,7 +1951,7 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "166b5ef52a3ab5575047a9fe8d4a030cdd0f63c96f071cd6907674453b07bae3" dependencies = [ - "libloading", + "libloading 0.7.4", ] [[package]] @@ -1938,12 +1974,12 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.3" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" dependencies = [ "memchr", - "minimal-lexical", + "version_check", ] [[package]] @@ -2071,7 +2107,7 @@ dependencies = [ "anyhow", "assert_cmd", "clap 4.1.8", - "env_logger", + "env_logger 0.10.0", "futures", "log", "opendal", @@ -2103,17 +2139,18 @@ dependencies = [ "base64 0.21.0", "bb8", "bytes", - "cfg-if", + "cfg-if 1.0.0", "criterion", "dashmap", "dotenvy", - "env_logger", + "env_logger 0.10.0", "flagset", "futures", "hdrs", "http", "hyper", "lazy-regex", + "librocksdb-sys", "log", "md-5", "memcache-async", @@ -2180,7 +2217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -2343,7 +2380,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -2357,7 +2394,7 @@ version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", @@ -2495,7 +2532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.0", "libc", "log", "wepoll-ffi", @@ -2605,7 +2642,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06a3d8e8a46ab2738109347433cb7b96dffda2e4a218b03ef27090238886b147" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "indoc", "libc", "memoffset", @@ -2993,9 +3030,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.20.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "015439787fce1e75d55f279078d33ff14b4af5d93d995e8838ee4631301c8a99" +checksum = "23d83c02c429044d58474eaf5ae31e062d0de894e21125b47437ec0edc1397e6" dependencies = [ "libc", "librocksdb-sys", @@ -3028,7 +3065,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "ordered-multimap", ] @@ -3264,7 +3301,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -3281,7 +3318,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -3297,9 +3334,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "signature" @@ -3423,6 +3460,12 @@ dependencies = [ "der", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.10.0" @@ -3479,7 +3522,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "redox_syscall", "rustix", @@ -3501,6 +3544,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "textwrap" version = "0.16.0" @@ -3533,7 +3585,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -3698,7 +3750,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3781,7 +3833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "enum-as-inner", "futures-channel", @@ -3805,7 +3857,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "futures-util", "ipconfig", "lazy_static", @@ -3946,6 +3998,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.4" @@ -4012,7 +4070,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -4037,7 +4095,7 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -4142,6 +4200,15 @@ dependencies = [ "cc", ] +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + [[package]] name = "widestring" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 3fe224402234..abecf43c3b64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,7 +85,7 @@ services-moka = ["dep:moka"] # Enable services redis support services-redis = ["dep:redis"] # Enable services rocksdb support -services-rocksdb = ["dep:rocksdb"] +services-rocksdb = ["dep:rocksdb", "dep:librocksdb-sys"] # Enable services sled support services-sled = ["dep:sled"] @@ -112,6 +112,9 @@ hdrs = { version = "0.2", optional = true, features = ["async_file"] } http = "0.2.5" hyper = "0.14" lazy-regex = { version = "2.4.1", optional = true } +# Notes: +# pin to 6.11 to allow we can use rocksdb 6.11 which is more widely used. +librocksdb-sys = { version = "=6.11", optional = true } log = "0.4" md-5 = "0.10" memcache-async = { version = "0.6", optional = true } @@ -133,7 +136,9 @@ reqwest = { version = "0.11.13", features = [ "multipart", "stream", ], default-features = false } -rocksdb = { version = "0.20", default-features = false, optional = true } +# Notes: +# pin to 0.15 to allow we can use rocksdb 6.x which is more widely used. +rocksdb = { version = "0.15", default-features = false, optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" sled = { version = "0.34.7", optional = true } diff --git a/src/services/rocksdb/backend.rs b/src/services/rocksdb/backend.rs index 7f41aeee30f2..b3ba9c51976c 100644 --- a/src/services/rocksdb/backend.rs +++ b/src/services/rocksdb/backend.rs @@ -18,7 +18,7 @@ use std::fmt::Formatter; use std::sync::Arc; use async_trait::async_trait; -use rocksdb::TransactionDB; +use rocksdb::DB; use crate::raw::adapters::kv; use crate::raw::*; @@ -40,9 +40,17 @@ use crate::*; /// /// # Note /// -/// The storage format for this service is not **stable** yet. +/// OpenDAL will build rocksdb from source by default. /// -/// PLEASE DON'T USE THIS SERVICE FOR PERSIST DATA. +/// To link with existing rocksdb lib, please set one of the following: +/// +/// - `ROCKSDB_LIB_DIR` to the dir that contains `librocksdb.so` +/// - `ROCKSDB_STATIC` to the dir that contains `librocksdb.a` +/// +/// If the version of RocksDB is below 6.0, you may encounter compatibility +/// issues. It is advisable to follow the steps provided in the [`INSTALL`](https://github.com/facebook/rocksdb/blob/main/INSTALL.md) +/// file to build rocksdb, rather than relying on system libraries that +/// may be outdated and incompatible. /// /// # Configuration /// @@ -116,7 +124,7 @@ impl Builder for RocksdbBuilder { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Rocksdb) })?; - let db = TransactionDB::open_default(&path).map_err(|e| { + let db = DB::open_default(&path).map_err(|e| { Error::new(ErrorKind::ConfigInvalid, "open default transaction db") .with_context("service", Scheme::Rocksdb) .with_context("datadir", path) @@ -132,7 +140,7 @@ pub type RocksdbBackend = kv::Backend; #[derive(Clone)] pub struct Adapter { - db: Arc, + db: Arc, } impl Debug for Adapter { diff --git a/src/services/sled/backend.rs b/src/services/sled/backend.rs index 3bfc3503b67d..93b3e46b8fab 100644 --- a/src/services/sled/backend.rs +++ b/src/services/sled/backend.rs @@ -39,12 +39,6 @@ use crate::*; /// - [ ] ~~presign~~ /// - [x] blocking /// -/// # Note -/// -/// The storage format for this service is not **stable** yet. -/// -/// PLEASE DON'T USE THIS SERVICE FOR PERSIST DATA. -/// /// # Configuration /// /// - `datadir`: Set the path to the sled data directory