diff --git a/.github/workflows/metric-engine-ci.yml b/.github/workflows/metric-engine-ci.yml index be80ebe8fb..cad7d3ab8d 100644 --- a/.github/workflows/metric-engine-ci.yml +++ b/.github/workflows/metric-engine-ci.yml @@ -70,3 +70,30 @@ jobs: working-directory: horaedb run: | git diff --exit-code + + unit-test: + name: unit-test + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: | + rustup set auto-self-update disable + rustup toolchain install ${RUST_VERSION} --profile minimal + - name: Release Disk Quota + run: | + sudo make ensure-disk-quota + - name: Setup Build Environment + run: | + sudo apt update + sudo apt install --yes protobuf-compiler + - name: Run Unit Tests + working-directory: horaedb + run: | + make test + - name: Check lock + working-directory: horaedb + run: | + git diff --exit-code diff --git a/horaedb/Cargo.lock b/horaedb/Cargo.lock index 632b74b743..64912637f3 100644 --- a/horaedb/Cargo.lock +++ b/horaedb/Cargo.lock @@ -96,71 +96,35 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" +checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" dependencies = [ - "arrow-arith 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-csv 52.2.0", - "arrow-data 52.2.0", - "arrow-ipc 52.2.0", - "arrow-json 52.2.0", - "arrow-ord 52.2.0", - "arrow-row 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", - "arrow-string 52.2.0", -] - -[[package]] -name = "arrow" -version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba" -dependencies = [ - "arrow-arith 53.0.0", - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-cast 53.1.0", - "arrow-csv 53.0.0", - "arrow-data 53.1.0", - "arrow-ipc 53.1.0", - "arrow-json 53.0.0", - "arrow-ord 53.0.0", - "arrow-row 53.0.0", - "arrow-schema 53.1.0", - "arrow-select 53.1.0", - "arrow-string 53.0.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] name = "arrow-arith" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" +checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "chrono", - "half", - "num", -] - -[[package]] -name = "arrow-arith" -version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a" -dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "half", "num", @@ -168,14 +132,14 @@ dependencies = [ [[package]] name = "arrow-array" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" +checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" dependencies = [ "ahash", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -183,343 +147,164 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-array" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da" -dependencies = [ - "ahash", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", - "chrono", - "half", - "hashbrown", - "num", -] - [[package]] name = "arrow-buffer" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" +checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" dependencies = [ "bytes", "half", "num", ] -[[package]] -name = "arrow-buffer" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80" -dependencies = [ - "bytes", - "half", - "num", -] - -[[package]] -name = "arrow-cast" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" -dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core 0.8.5", - "num", - "ryu", -] - [[package]] name = "arrow-cast" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4" +checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", - "arrow-select 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64", "chrono", "comfy-table", "half", - "lexical-core 1.0.2", + "lexical-core", "num", "ryu", ] [[package]] name = "arrow-csv" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" -dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "chrono", - "csv", - "csv-core", - "lazy_static", - "lexical-core 0.8.5", - "regex", -] - -[[package]] -name = "arrow-csv" -version = "53.0.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740" +checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-cast 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "csv", "csv-core", "lazy_static", - "lexical-core 0.8.5", + "lexical-core", "regex", ] [[package]] name = "arrow-data" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" +checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" dependencies = [ - "arrow-buffer 52.2.0", - "arrow-schema 52.2.0", - "half", - "num", -] - -[[package]] -name = "arrow-data" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634" -dependencies = [ - "arrow-buffer 53.1.0", - "arrow-schema 53.1.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] [[package]] name = "arrow-ipc" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" +checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "flatbuffers", "lz4_flex", ] -[[package]] -name = "arrow-ipc" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2" -dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-cast 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", - "flatbuffers", -] - [[package]] name = "arrow-json" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" +checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", - "lexical-core 0.8.5", + "lexical-core", "num", "serde", "serde_json", ] -[[package]] -name = "arrow-json" -version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6" -dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-cast 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", - "chrono", - "half", - "indexmap", - "lexical-core 0.8.5", - "num", - "serde", - "serde_json", -] - -[[package]] -name = "arrow-ord" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" -dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", - "half", - "num", -] - [[package]] name = "arrow-ord" -version = "53.0.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b" +checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", - "arrow-select 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "half", "num", ] [[package]] name = "arrow-row" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" +checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" dependencies = [ "ahash", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "half", -] - -[[package]] -name = "arrow-row" -version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb" -dependencies = [ - "ahash", - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] [[package]] name = "arrow-schema" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" - -[[package]] -name = "arrow-schema" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794" - -[[package]] -name = "arrow-select" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" -dependencies = [ - "ahash", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "num", -] +checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" [[package]] name = "arrow-select" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a" +checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" dependencies = [ "ahash", - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] [[package]] name = "arrow-string" -version = "52.2.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" +checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", - "memchr", - "num", - "regex", - "regex-syntax", -] - -[[package]] -name = "arrow-string" -version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230" -dependencies = [ - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-data 53.1.0", - "arrow-schema 53.1.0", - "arrow-select 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex", @@ -552,7 +337,7 @@ checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -725,9 +510,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" dependencies = [ "chrono", "chrono-tz-build", @@ -736,12 +521,11 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" dependencies = [ "parse-zoneinfo", - "phf", "phf_codegen", ] @@ -865,15 +649,15 @@ dependencies = [ [[package]] name = "datafusion" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb" +checksum = "3e8053b4cedc24eb158e4c041b38cfa0677ef5f4a7ccaa31ee5dcad61dd7aa54" dependencies = [ "ahash", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-ipc 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", "async-compression", "async-trait", "bytes", @@ -888,6 +672,7 @@ dependencies = [ "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", + "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -900,12 +685,12 @@ dependencies = [ "half", "hashbrown", "indexmap", - "itertools 0.12.1", + "itertools 0.13.0", "log", "num_cpus", - "object_store 0.10.2", + "object_store", "parking_lot", - "parquet 52.2.0", + "parquet", "paste", "pin-project-lite", "rand", @@ -921,56 +706,60 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529" +checksum = "7d95efedb3a32f6f74df5bb8fda7b69fb9babe80e92137f25de6ddb15e8e8801" dependencies = [ - "arrow-schema 52.2.0", + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "parking_lot", ] [[package]] name = "datafusion-common" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c" +checksum = "b7d766e0d3dec01a0ab70b1b31678c286cddc0bd7afc9bd82504a1d9a70a7d94" dependencies = [ "ahash", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", "chrono", "half", "hashbrown", "instant", "libc", "num_cpus", - "object_store 0.10.2", - "parquet 52.2.0", + "object_store", + "parquet", + "paste", "sqlparser", + "tokio", ] [[package]] name = "datafusion-common-runtime" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1" +checksum = "4e55db6df319f9e7cf366d0d4ffae793c863823421b2f2b7314a0fefd8e8c11a" dependencies = [ + "log", "tokio", ] [[package]] name = "datafusion-execution" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" +checksum = "6d0c6dc013f955c382438a78fa3de8b0a8bf7b1a4cda5bc46335fe445ff3ff1a" dependencies = [ - "arrow 52.2.0", + "arrow", "chrono", "dashmap", "datafusion-common", @@ -978,7 +767,7 @@ dependencies = [ "futures", "hashbrown", "log", - "object_store 0.10.2", + "object_store", "parking_lot", "rand", "tempfile", @@ -987,16 +776,19 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2" +checksum = "7f31405c0bb854451d755b224d41dc466a8f7fd36f8c041c29d2d8432bd0c08c" dependencies = [ "ahash", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "paste", "serde_json", "sqlparser", @@ -1004,14 +796,25 @@ dependencies = [ "strum_macros", ] +[[package]] +name = "datafusion-expr-common" +version = "42.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc8266b6627c8264c87bc7c82564e3d89ed5f0f9943b49a30dac1f1ac12e4c0" +dependencies = [ + "arrow", + "datafusion-common", + "paste", +] + [[package]] name = "datafusion-functions" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb" +checksum = "5712668780bc43666ecd10acd188b7df58e2a5501d4dbbd972bf209f1790138b" dependencies = [ - "arrow 52.2.0", - "arrow-buffer 52.2.0", + "arrow", + "arrow-buffer", "base64", "blake2", "blake3", @@ -1021,7 +824,7 @@ dependencies = [ "datafusion-expr", "hashbrown", "hex", - "itertools 0.12.1", + "itertools 0.13.0", "log", "md-5", "rand", @@ -1033,51 +836,81 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb" +checksum = "8ec138af6b7482fb726f1bfeec010fc063b9614594c36a1051a4d3b365ba6a5f" dependencies = [ "ahash", - "arrow 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", "datafusion-physical-expr-common", + "half", "log", "paste", "sqlparser", ] +[[package]] +name = "datafusion-functions-aggregate-common" +version = "42.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "564499c6bdd3ab9f76c7ad727e858bc6791e4de6c1a484d21d2bf49daaa658d6" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", + "rand", +] + [[package]] name = "datafusion-functions-nested" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d" +checksum = "9b55ea2221ae1c1e37d524f8330f763dcdc205edb74fe5f54cbdea475c17fd18" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", - "itertools 0.12.1", + "datafusion-physical-expr-common", + "itertools 0.13.0", "log", "paste", "rand", ] +[[package]] +name = "datafusion-functions-window" +version = "42.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6932996c4407ee1ebf23ffd706e982729cb9b6f7a31a281abac51fe524c3a049" +dependencies = [ + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", +] + [[package]] name = "datafusion-optimizer" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906" +checksum = "f7d8afa1eb44e2f00cc8d82b88803e456a681474b8877ceecc04e9517d5c843c" dependencies = [ - "arrow 52.2.0", + "arrow", "async-trait", "chrono", "datafusion-common", @@ -1085,7 +918,7 @@ dependencies = [ "datafusion-physical-expr", "hashbrown", "indexmap", - "itertools 0.12.1", + "itertools 0.13.0", "log", "paste", "regex-syntax", @@ -1093,28 +926,30 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4" +checksum = "570666d84df483473626fab4e69997d048b40d0e7c67c540299714f244d99e73" dependencies = [ "ahash", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", - "arrow-string 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", "base64", "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", "hashbrown", "hex", "indexmap", - "itertools 0.12.1", + "itertools 0.13.0", "log", "paste", "petgraph", @@ -1123,42 +958,44 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6" +checksum = "3746cbdfb32d67399dcaad17042e419ac6da454a7e38ff098aa2fbf0a7388982" dependencies = [ "ahash", - "arrow 52.2.0", + "arrow", "datafusion-common", - "datafusion-expr", + "datafusion-expr-common", "hashbrown", "rand", ] [[package]] name = "datafusion-physical-optimizer" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0" +checksum = "696f06e79d44f7c50f57cea23493881d86d9d9647884d38ce467c7f75c13e286" dependencies = [ + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-physical-expr", "datafusion-physical-plan", + "itertools 0.13.0", ] [[package]] name = "datafusion-physical-plan" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe" +checksum = "04e1d084224023e09cdea14d01ded0f2092c319c7b4594ebc821283b9c7c4a35" dependencies = [ "ahash", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -1166,13 +1003,14 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", "half", "hashbrown", "indexmap", - "itertools 0.12.1", + "itertools 0.13.0", "log", "once_cell", "parking_lot", @@ -1183,13 +1021,13 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "41.0.0" +version = "42.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0" +checksum = "c105148357dcbd9e4c97eada2930a59f7923215461d9f47de6e76edd60eab2d5" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-schema", "datafusion-common", "datafusion-expr", "log", @@ -1209,12 +1047,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "either" version = "1.13.0" @@ -1334,7 +1166,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -1421,12 +1253,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -1529,15 +1355,6 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b73f1c685cfd8ff8d75698ed87e6188cd09944b30c0863d45c2c3699d1da0c" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1577,41 +1394,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lexical-core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" -dependencies = [ - "lexical-parse-float 0.8.5", - "lexical-parse-integer 0.8.6", - "lexical-util 0.8.5", - "lexical-write-float 0.8.5", - "lexical-write-integer 0.8.5", -] - [[package]] name = "lexical-core" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" dependencies = [ - "lexical-parse-float 1.0.2", - "lexical-parse-integer 1.0.2", - "lexical-util 1.0.3", - "lexical-write-float 1.0.2", - "lexical-write-integer 1.0.2", -] - -[[package]] -name = "lexical-parse-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" -dependencies = [ - "lexical-parse-integer 0.8.6", - "lexical-util 0.8.5", - "static_assertions", + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", ] [[package]] @@ -1620,18 +1413,8 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" dependencies = [ - "lexical-parse-integer 1.0.2", - "lexical-util 1.0.3", - "static_assertions", -] - -[[package]] -name = "lexical-parse-integer" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" -dependencies = [ - "lexical-util 0.8.5", + "lexical-parse-integer", + "lexical-util", "static_assertions", ] @@ -1641,16 +1424,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" dependencies = [ - "lexical-util 1.0.3", - "static_assertions", -] - -[[package]] -name = "lexical-util" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" -dependencies = [ + "lexical-util", "static_assertions", ] @@ -1663,35 +1437,14 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "lexical-write-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" -dependencies = [ - "lexical-util 0.8.5", - "lexical-write-integer 0.8.5", - "static_assertions", -] - [[package]] name = "lexical-write-float" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" dependencies = [ - "lexical-util 1.0.3", - "lexical-write-integer 1.0.2", - "static_assertions", -] - -[[package]] -name = "lexical-write-integer" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" -dependencies = [ - "lexical-util 0.8.5", + "lexical-util", + "lexical-write-integer", "static_assertions", ] @@ -1701,7 +1454,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" dependencies = [ - "lexical-util 1.0.3", + "lexical-util", "static_assertions", ] @@ -1784,7 +1537,7 @@ name = "metric_engine" version = "2.0.0" dependencies = [ "anyhow", - "arrow 53.0.0", + "arrow", "async-trait", "bytes", "datafusion", @@ -1792,8 +1545,8 @@ dependencies = [ "itertools 0.3.25", "lazy_static", "macros", - "object_store 0.11.0", - "parquet 53.1.0", + "object_store", + "parquet", "pb_types", "prost", "thiserror", @@ -1930,27 +1683,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object_store" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "humantime", - "itertools 0.13.0", - "parking_lot", - "percent-encoding", - "snafu 0.7.5", - "tokio", - "tracing", - "url", - "walkdir", -] - [[package]] name = "object_store" version = "0.11.0" @@ -1965,7 +1697,7 @@ dependencies = [ "itertools 0.13.0", "parking_lot", "percent-encoding", - "snafu 0.8.4", + "snafu", "tokio", "tracing", "url", @@ -2016,42 +1748,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "parquet" -version = "52.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" -dependencies = [ - "ahash", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-ipc 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", - "base64", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown", - "lz4_flex", - "num", - "num-bigint", - "object_store 0.10.2", - "paste", - "seq-macro", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", - "zstd-sys", -] - [[package]] name = "parquet" version = "53.1.0" @@ -2059,13 +1755,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e" dependencies = [ "ahash", - "arrow-array 53.1.0", - "arrow-buffer 53.1.0", - "arrow-cast 53.1.0", - "arrow-data 53.1.0", - "arrow-ipc 53.1.0", - "arrow-schema 53.1.0", - "arrow-select 53.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64", "brotli", "bytes", @@ -2077,7 +1773,7 @@ dependencies = [ "lz4_flex", "num", "num-bigint", - "object_store 0.11.0", + "object_store", "paste", "seq-macro", "snap", @@ -2199,7 +1895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "910d41a655dac3b764f1ade94821093d3610248694320cd072303a8eedcf221d" dependencies = [ "proc-macro2", - "syn 2.0.82", + "syn", ] [[package]] @@ -2228,7 +1924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", - "heck 0.5.0", + "heck", "itertools 0.13.0", "log", "multimap", @@ -2238,7 +1934,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.82", + "syn", "tempfile", ] @@ -2252,7 +1948,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2425,7 +2121,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2496,35 +2192,13 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -[[package]] -name = "snafu" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" -dependencies = [ - "doc-comment", - "snafu-derive 0.7.5", -] - [[package]] name = "snafu" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d" dependencies = [ - "snafu-derive 0.8.4", -] - -[[package]] -name = "snafu-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 1.0.109", + "snafu-derive", ] [[package]] @@ -2533,10 +2207,10 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2557,9 +2231,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.49.0" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" +checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac" dependencies = [ "log", "sqlparser_derive", @@ -2573,7 +2247,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2597,11 +2271,11 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.82", + "syn", ] [[package]] @@ -2610,17 +2284,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.82" @@ -2662,7 +2325,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2736,7 +2399,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2771,7 +2434,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] @@ -2928,7 +2591,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.82", + "syn", "wasm-bindgen-shared", ] @@ -2950,7 +2613,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3120,7 +2783,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn", ] [[package]] diff --git a/horaedb/Cargo.toml b/horaedb/Cargo.toml index ee231cbea3..628e47a1a1 100644 --- a/horaedb/Cargo.toml +++ b/horaedb/Cargo.toml @@ -30,7 +30,7 @@ anyhow = { version = "1.0" } metric_engine = { path = "metric_engine" } thiserror = "1" bytes = "1" -datafusion = "41" +datafusion = "42" parquet = { version = "53" } object_store = { version = "0.11" } macros = { path = "../src/components/macros" } diff --git a/horaedb/Makefile b/horaedb/Makefile index 72736b0af4..189b1c2b22 100644 --- a/horaedb/Makefile +++ b/horaedb/Makefile @@ -30,3 +30,6 @@ fmt: fix: cargo fmt cargo sort --workspace + +test: + cargo test --workspace diff --git a/horaedb/metric_engine/src/storage.rs b/horaedb/metric_engine/src/storage.rs index 4c5b2667e6..3a8d42f0fd 100644 --- a/horaedb/metric_engine/src/storage.rs +++ b/horaedb/metric_engine/src/storage.rs @@ -15,13 +15,25 @@ // specific language governing permissions and limitations // under the License. +use std::{sync::Arc, vec}; + use anyhow::Context; use arrow::{ array::{Int64Array, RecordBatch}, datatypes::SchemaRef, }; use async_trait::async_trait; -use datafusion::logical_expr::Expr; +use datafusion::{ + common::DFSchema, + execution::{ + context::ExecutionProps, SendableRecordBatchStream as DFSendableRecordBatchStream, + }, + logical_expr::Expr, + physical_plan::{execute_stream, memory::MemoryExec, sorts::sort::SortExec}, + physical_planner::create_physical_sort_exprs, + prelude::{ident, SessionContext}, +}; +use futures::StreamExt; use macros::ensure; use object_store::path::Path; use parquet::{ @@ -69,6 +81,7 @@ pub struct CloudObjectStorage { path: String, store: ObjectStoreRef, arrow_schema: SchemaRef, + num_primary_key: usize, timestamp_index: usize, manifest: Manifest, } @@ -88,6 +101,7 @@ impl CloudObjectStorage { root_path: String, store: ObjectStoreRef, arrow_schema: SchemaRef, + num_primary_key: usize, timestamp_index: usize, ) -> Result { let manifest_prefix = crate::manifest::PREFIX_PATH; @@ -95,6 +109,7 @@ impl CloudObjectStorage { Manifest::try_new(format!("{root_path}/{manifest_prefix}"), store.clone()).await?; Ok(Self { path: root_path, + num_primary_key, timestamp_index, store, arrow_schema, @@ -117,15 +132,39 @@ impl CloudObjectStorage { AsyncArrowWriter::try_new(object_store_writer, self.schema().clone(), req.props) .context("create arrow writer")?; - // TODO: sort record batch according to primary key columns. - writer - .write(&req.batch) - .await - .context("write arrow batch")?; + // sort record batch + let mut batches = self.sort_batch(req.batch).await?; + while let Some(batch) = batches.next().await { + let batch = batch.context("get sorted batch")?; + writer.write(&batch).await.context("write arrow batch")?; + } writer.close().await.context("close arrow writer")?; Ok(file_id) } + + async fn sort_batch(&self, batch: RecordBatch) -> Result { + let ctx = SessionContext::default(); + let schema = batch.schema(); + let df_schema = DFSchema::try_from(schema.clone()).context("build DFSchema")?; + + let sort_exprs = (0..self.num_primary_key) + .collect::>() + .iter() + .map(|i| ident(schema.clone().field(*i).name()).sort(true, true)) + .collect::>(); + let physical_sort_exprs = + create_physical_sort_exprs(&sort_exprs, &df_schema, &ExecutionProps::default()) + .context("create physical sort exprs")?; + + let batch_plan = + MemoryExec::try_new(&[vec![batch]], schema, None).context("build batch plan")?; + let physical_plan = Arc::new(SortExec::new(physical_sort_exprs, Arc::new(batch_plan))); + + let res = + execute_stream(physical_plan, ctx.task_ctx()).context("execute sort physical plan")?; + Ok(res) + } } #[async_trait] @@ -174,3 +213,62 @@ impl TimeMergeStorage for CloudObjectStorage { todo!() } } + +#[cfg(test)] +mod tests { + use arrow::{ + array::UInt8Array, + datatypes::{DataType, Field, Schema}, + }; + use object_store::local::LocalFileSystem; + + use super::*; + + #[tokio::test] + async fn test_sort_batch() { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::UInt8, false), + Field::new("b", DataType::UInt8, false), + Field::new("c", DataType::UInt8, false), + Field::new("d", DataType::UInt8, false), + ])); + + let store = Arc::new(LocalFileSystem::new()); + let storage = + CloudObjectStorage::try_new("/tmp/storage".to_string(), store, schema.clone(), 1, 1) + .await + .unwrap(); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt8Array::from(vec![2, 1, 3, 4, 8, 6, 5, 7])), + Arc::new(UInt8Array::from(vec![1, 3, 4, 8, 2, 6, 5, 7])), + Arc::new(UInt8Array::from(vec![8, 6, 2, 4, 3, 1, 5, 7])), + Arc::new(UInt8Array::from(vec![2, 7, 4, 6, 1, 3, 5, 8])), + ], + ) + .unwrap(); + + let mut sorted_batches = storage.sort_batch(batch).await.unwrap(); + let expected_bacth = RecordBatch::try_new( + schema, + vec![ + Arc::new(UInt8Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8])), + Arc::new(UInt8Array::from(vec![3, 1, 4, 8, 5, 6, 7, 2])), + Arc::new(UInt8Array::from(vec![6, 8, 2, 4, 5, 1, 7, 3])), + Arc::new(UInt8Array::from(vec![7, 2, 4, 6, 5, 3, 8, 1])), + ], + ) + .unwrap(); + + let mut offset = 0; + while let Some(sorted_batch) = sorted_batches.next().await { + let sorted_batch = sorted_batch.unwrap(); + let length = sorted_batch.num_rows(); + let batch = expected_bacth.slice(offset, length); + assert!(sorted_batch.eq(&batch)); + offset += length; + } + } +}