From 5dcaabd6cd0f02569cd99bf8c48bbc5f24c98b90 Mon Sep 17 00:00:00 2001 From: Daniel Harvey Date: Fri, 20 Dec 2024 17:32:08 +0000 Subject: [PATCH] Remove unused constructors and packages (#46) * Remove unused constructors and packages, add cargo machete job to CI * Rebuild Cargo.lock --- .github/workflows/cargo-machete.yaml | 28 + Cargo.lock | 1034 +---------------- architecture.md | 173 --- crates/cli/Cargo.toml | 7 +- crates/configuration/Cargo.toml | 10 - crates/connectors/ndc-bigquery/Cargo.toml | 9 - .../ndc-bigquery/src/error/convert.rs | 9 - .../ndc-bigquery/src/error/record.rs | 8 +- crates/connectors/ndc-bigquery/src/state.rs | 2 - crates/query-engine/execution/Cargo.toml | 5 - crates/query-engine/execution/src/error.rs | 31 - crates/query-engine/execution/src/metrics.rs | 112 +- crates/query-engine/metadata/Cargo.toml | 2 - crates/query-engine/sql/Cargo.toml | 2 - crates/query-engine/translation/Cargo.toml | 3 +- crates/tests/tests-common/Cargo.toml | 11 - 16 files changed, 69 insertions(+), 1377 deletions(-) create mode 100644 .github/workflows/cargo-machete.yaml delete mode 100644 architecture.md diff --git a/.github/workflows/cargo-machete.yaml b/.github/workflows/cargo-machete.yaml new file mode 100644 index 00000000..c05dfa5e --- /dev/null +++ b/.github/workflows/cargo-machete.yaml @@ -0,0 +1,28 @@ +name: find unused dependencies + +on: + push: + +jobs: + cargo-machete: + name: find unused dependencies with cargo machete + runs-on: ubuntu-latest + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + RUSTFLAGS: "-D warnings" # fail on warnings + steps: + - uses: actions/checkout@v4 + + - name: install tools + run: | + rustup show + cargo install cargo-machete + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "build" # share the cache across jobs + save-if: false + + - name: find unused dependencies + run: | + cargo machete --with-metadata diff --git a/Cargo.lock b/Cargo.lock index ba26e9fc..9b420921 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,20 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "serde", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.0.4" @@ -40,12 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -135,7 +115,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -146,16 +126,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", + "syn", ] [[package]] @@ -285,27 +256,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -317,29 +267,6 @@ name = "bitflags" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" -dependencies = [ - "serde", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata 0.1.10", -] [[package]] name = "build-data" @@ -357,18 +284,6 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" -[[package]] -name = "bytecount" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - [[package]] name = "bytes" version = "1.9.0" @@ -431,10 +346,10 @@ version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -467,12 +382,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "const-oid" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" - [[package]] name = "core-foundation" version = "0.9.3" @@ -489,30 +398,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" -[[package]] -name = "cpufeatures" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" -dependencies = [ - "libc", -] - -[[package]] -name = "crc" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" - [[package]] name = "crc32fast" version = "1.4.2" @@ -532,16 +417,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -551,16 +426,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "darling" version = "0.20.3" @@ -582,7 +447,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.69", + "syn", ] [[package]] @@ -593,18 +458,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.69", -] - -[[package]] -name = "der" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", + "syn", ] [[package]] @@ -616,24 +470,6 @@ dependencies = [ "serde", ] -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "dyn-clone" version = "1.0.13" @@ -645,9 +481,6 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" -dependencies = [ - "serde", -] [[package]] name = "encode_unicode" @@ -681,7 +514,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -734,39 +567,6 @@ dependencies = [ "libc", ] -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fancy-regex" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" -dependencies = [ - "bit-set", - "regex", -] - [[package]] name = "fastrand" version = "2.0.0" @@ -783,18 +583,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "flume" -version = "0.10.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" -dependencies = [ - "futures-core", - "futures-sink", - "pin-project", - "spin 0.9.8", -] - [[package]] name = "fnv" version = "1.0.7" @@ -825,16 +613,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fraction" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3027ae1df8d41b4bed2241c8fdad4acc1e7af60c8e17743534b545e77182d678" -dependencies = [ - "lazy_static", - "num", -] - [[package]] name = "futures" version = "0.3.28" @@ -877,17 +655,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - [[package]] name = "futures-io" version = "0.3.28" @@ -902,7 +669,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -958,16 +725,6 @@ dependencies = [ "yup-oauth2", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.10" @@ -975,10 +732,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] @@ -1023,28 +778,6 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] - -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown 0.14.5", -] - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -dependencies = [ - "unicode-segmentation", -] [[package]] name = "heck" @@ -1058,33 +791,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hkdf" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" -dependencies = [ - "hmac", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "http" version = "0.2.9" @@ -1283,15 +989,6 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" -[[package]] -name = "iso8601" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" -dependencies = [ - "nom", -] - [[package]] name = "itertools" version = "0.10.5" @@ -1316,44 +1013,11 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonschema" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a071f4f7efc9a9118dfb627a0a94ef247986e1ab8606a4c806ae2b3aa3b6978" -dependencies = [ - "ahash", - "anyhow", - "base64 0.21.3", - "bytecount", - "clap", - "fancy-regex", - "fraction", - "getrandom", - "iso8601", - "itoa", - "memchr", - "num-cmp", - "once_cell", - "parking_lot", - "percent-encoding", - "regex", - "reqwest", - "serde", - "serde_json", - "time", - "url", - "uuid", -] - [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] [[package]] name = "libc" @@ -1361,23 +1025,6 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" -[[package]] -name = "libm" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" - -[[package]] -name = "libsqlite3-sys" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1421,15 +1068,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" -[[package]] -name = "md-5" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" -dependencies = [ - "digest", -] - [[package]] name = "memchr" version = "2.6.0" @@ -1511,14 +1149,11 @@ version = "1.0.0" dependencies = [ "async-trait", "axum", - "axum-test-helper", - "enum-iterator", "env_logger", "gcp-bigquery-client", "hyper", "insta", "ndc-bigquery-configuration", - "ndc-models", "ndc-sdk", "ndc-test", "prometheus", @@ -1527,17 +1162,11 @@ dependencies = [ "query-engine-sql", "query-engine-translation", "reqwest", - "schemars", - "serde", - "serde_derive", "serde_json", - "similar-asserts", - "sqlx", "tests-common", "thiserror", "tokio", "tracing", - "url", "yup-oauth2", ] @@ -1548,12 +1177,9 @@ dependencies = [ "anyhow", "build-data", "clap", - "insta", "ndc-bigquery-configuration", "serde", - "serde_json", "serde_yaml", - "tempfile", "thiserror", "tokio", ] @@ -1563,21 +1189,14 @@ name = "ndc-bigquery-configuration" version = "1.0.0" dependencies = [ "anyhow", - "clap", "gcp-bigquery-client", - "jsonschema", "ndc-models", - "prometheus", "query-engine-metadata", - "query-engine-sql", "schemars", "serde", "serde_json", - "smol_str", - "sqlx", "thiserror", "tokio", - "tracing", "yup-oauth2", ] @@ -1669,93 +1288,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-cmp" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1763,7 +1295,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -1813,7 +1344,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -1982,45 +1513,12 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project" version = "1.1.3" @@ -2038,7 +1536,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -2053,62 +1551,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkg-config" version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "postgres-protocol" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" -dependencies = [ - "base64 0.21.3", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md-5", - "memchr", - "rand", - "sha2", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" -dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", -] - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2159,7 +1607,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -2174,25 +1622,19 @@ version = "1.0.0" dependencies = [ "bytes", "gcp-bigquery-client", - "ndc-models", "prometheus", "query-engine-sql", "serde_json", - "sqlformat", - "sqlx", "thiserror", - "tracing", ] [[package]] name = "query-engine-metadata" version = "1.0.0" dependencies = [ - "enum-iterator", "ndc-models", "schemars", "serde", - "serde_json", "smol_str", ] @@ -2201,8 +1643,6 @@ name = "query-engine-sql" version = "1.0.0" dependencies = [ "ndc-models", - "schemars", - "serde", "serde_json", ] @@ -2225,7 +1665,6 @@ dependencies = [ "sqlformat", "thiserror", "tokio", - "tracing", ] [[package]] @@ -2293,7 +1732,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -2383,7 +1822,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.2", + "webpki-roots", "winreg", ] @@ -2411,32 +1850,10 @@ dependencies = [ "cc", "cfg-if", "getrandom", - "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys 0.52.0", -] - -[[package]] -name = "rsa" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" -dependencies = [ - "byteorder", - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-iter", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core", - "signature", - "spki", - "subtle", - "zeroize", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", ] [[package]] @@ -2647,7 +2064,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.69", + "syn", ] [[package]] @@ -2718,7 +2135,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -2729,7 +2146,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -2793,7 +2210,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -2809,28 +2226,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.4" @@ -2849,41 +2244,11 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" -dependencies = [ - "digest", - "rand_core", -] - [[package]] name = "similar" version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" -dependencies = [ - "bstr", - "unicode-segmentation", -] - -[[package]] -name = "similar-asserts" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f" -dependencies = [ - "console", - "similar", -] - -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" @@ -2930,19 +2295,6 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" -dependencies = [ - "base64ct", - "der", -] [[package]] name = "sqlformat" @@ -2954,218 +2306,6 @@ dependencies = [ "unicode_categories", ] -[[package]] -name = "sqlx" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" -dependencies = [ - "ahash", - "atoi", - "byteorder", - "bytes", - "crc", - "crossbeam-queue", - "dotenvy", - "either", - "event-listener", - "futures-channel", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashlink", - "hex", - "indexmap 2.5.0", - "log", - "memchr", - "once_cell", - "paste", - "percent-encoding", - "rustls 0.21.7", - "rustls-pemfile 1.0.3", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlformat", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "url", - "uuid", - "webpki-roots 0.24.0", -] - -[[package]] -name = "sqlx-macros" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 1.0.109", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" -dependencies = [ - "dotenvy", - "either", - "heck 0.4.1", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn 1.0.109", - "tempfile", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" -dependencies = [ - "atoi", - "base64 0.21.3", - "bitflags 2.4.0", - "byteorder", - "bytes", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" -dependencies = [ - "atoi", - "base64 0.21.3", - "bitflags 2.4.0", - "byteorder", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand", - "serde", - "serde_json", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" -dependencies = [ - "atoi", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "sqlx-core", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "stringprep" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "strsim" version = "0.10.0" @@ -3184,17 +2324,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.69" @@ -3250,23 +2379,13 @@ dependencies = [ name = "tests-common" version = "1.0.0" dependencies = [ - "anyhow", "axum", "axum-test-helper", - "env_logger", - "hyper", - "ndc-bigquery", "ndc-sdk", - "ndc-test", - "reqwest", "serde", "serde_derive", "serde_json", - "sqlx", - "tokio", - "tokio-postgres", "tracing", - "uuid", ] [[package]] @@ -3286,7 +2405,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -3380,7 +2499,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -3393,32 +2512,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-postgres" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures-channel", - "futures-util", - "log", - "parking_lot", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "rand", - "socket2", - "tokio", - "tokio-util", - "whoami", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3569,7 +2662,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] [[package]] @@ -3663,15 +2756,9 @@ checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", ] -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - [[package]] name = "unicase" version = "2.7.0" @@ -3702,12 +2789,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -3755,28 +2836,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "uuid" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" -dependencies = [ - "getrandom", - "rand", - "uuid-macro-internal", -] - -[[package]] -name = "uuid-macro-internal" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e1ba1f333bd65ce3c9f27de592fcbc256dafe3af2717f56d7c87761fbaccf4" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.69", -] - [[package]] name = "valuable" version = "0.1.0" @@ -3831,7 +2890,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn", "wasm-bindgen-shared", ] @@ -3865,7 +2924,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3909,31 +2968,12 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" -dependencies = [ - "rustls-webpki 0.101.4", -] - [[package]] name = "webpki-roots" version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" -[[package]] -name = "whoami" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" -dependencies = [ - "wasm-bindgen", - "web-sys", -] - [[package]] name = "winapi" version = "0.3.9" @@ -4216,26 +3256,6 @@ dependencies = [ "url", ] -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.69", -] - [[package]] name = "zeroize" version = "1.6.0" diff --git a/architecture.md b/architecture.md deleted file mode 100644 index 2aa9d5e9..00000000 --- a/architecture.md +++ /dev/null @@ -1,173 +0,0 @@ -# General architecture - -## Query Engine - -The query engine's job is to take a `QueryRequest`, which contains information about the query a user would like to run, -translate it to PostgreSQL SQL, execute it against the database, and return the results as a `QueryResponse`. - -One place in particular that uses the Query Engine is the `/query` endpoint (defined in the `ndc-hub` repository). - -`/query` endpoints receives a `QueryRequest`, and calls the `translate` function from the Query Engine -with it and with the information about the tables tracked in the metadata to receive and `ExecutionPlan`. -It then calls the `execute` function from the Query Engine with the same `ExecutionPlan` -(which then runs it against postgres) and gets back a `QueryResponse` which it can then return to the caller. - -API: - -```rs -pub fn translate( - metadata: &metadata::Metadata, - query_request: models::QueryRequest, -) -> Result -``` - -```rs -pub async fn execute( - pool: sqlx::PgPool, - plan: translation::ExecutionPlan, -) -> Result -``` - -### Translation - -The translation step is essentially side-effect free - we use information from the request, as well as the information -about the metadata to translate the query request into steps to run against the database. - -This process is currently found in the [translation/query/](/crates/query-engine/translation/src/translation/query/) directory -and are split to several modules roughly mimicing the query parts as they are specified in the spec. The API -is the following function in the [mod.rs](/crates/query-engine/translation/src/translation/query/mod.rs) file: - -```rs -pub fn translate( - metadata: &metadata::Metadata, - query_request: models::QueryRequest, -) -> Result -``` - -The `translate` function returns a `ExecutionPlan`. - -```rs -pub struct ExecutionPlan { - pub root_field: String, - /// Run before the query. Should be a sql::ast in the future. - pub pre: Vec, - /// The query. - pub query: sql::ast::Select, - /// Run after the query. Should be a sql::ast in the future. - pub post: Vec, -} -``` - -Right now we don't expect `pre` and `post` to be populated, but it could be used for things like Stored Procedures. - -### SQL AST - -We maintain a SQL AST represented as Rust data types in [sql/ast.rs](/crates/query-engine/sql/src/sql/ast.rs). -We implement our own representation because we want more control over this core component of our application, -and we want to implement exactly what we want and not more or less. Other external libraries such as `sqlparser` -do not have the same goals as us, and we'll have to make compromises that will affect our codebase's complexity -and development velocity. - -We have a few guidelines for the SQL AST: - -#### The SQL AST should mimic PostgreSQL directly - -The SQL AST should look like a **subset** of PostgreSQL SQL, and not contain any high-level abstractions, or try to abstract -multiple SQL ASTs. We should implement exactly what we need, and be precise about it. - -Should we need a higher-level of abstraction, and additional IR should be constructed that will sit before the SQL AST. - -#### Implement what you need and not more - -The SQL AST should contain structures we actively use, and not contain structures we don't use. - -One such example is window functions - we don't need to include them in the AST currently because we don't have features -that use them from GraphQL. - -#### Use sql::helpers to build the AST - -Sometimes we'd like a shorthand to build specific repeating patterns, -such as `SELECT coalesce(json_agg(row_to_json()), '[]') AS FROM as `. -The [sql/helpers.rs](/crates/query-engine/sql/src/sql/helpers.rs) module can come in handy to help -codify certain SQL AST generation patterns. If you end up meeting a repeating long pattern that is used in multiple places, -it might be a good candidate to codify it as a `helpers` function. - -### SQL string - -The SQL string is a stringify representation of the SQL AST. It can be found in [sql/string.rs](/crates/query-engine/sql/src/sql/string.rs). - -We separate the SQL to AST and string representation so we can write transformations and optimizations on the SQL AST. - -The SQL string representation should be generated from the SQL AST by pretty printing the result. -The result of converting ([sql/convert.rs](/crates/query-engine/sql/src/sql/convert.rs)) a sql ast to string should produce -a query string that can be run against postgres as a parameterized query, as well as the parameters that are supplied by the user. - -Please use the API provided by the `SQL` type. It provides functions for constructing SQL strings in an easy way, such as appending syntax (like keywords and punctuation), -identifiers, and params. Don't use `append_syntax` for things that are not syntax. - -### Query Execution - -The query execution receives a pool and a plan and executes it against postgres. It then returns the results from the query part -back to the caller of the function. -The code can be found in [execution.rs](/crates/query-engine/execution/src/execution.rs) - -```rs -pub async fn execute( - pool: sqlx::PgPool, - plan: translation::ExecutionPlan, -) -> Result -``` - -## Patterns and guiding principles - -Here are a few ideas we want to maintain and why: - -### Optimize for inspectability - -One of the most important tools we have to figure out how a program behaves is run it and examine -the way data changes throughout the program. To do so we want to make sure the flow of the program -is easy to follow, and we can examine the relevant data in any given point. Here are a few suggestions: - -1. Prefer functions over abstract trait methods - makes jump to definition easier. -2. Annotate data types with `Debug` so that they can be traced, and prefer data shapes that can be `Debug`ged over those that can't. -3. Prefer to encode information in a way that is inspectable at runtime. - -### Prefer flexibility over rigidness until knowing better - -Abstractions should come up from usage pain points rather than theoretical ideas. -Often we don't know exactly what a good solution looks like in terms of code, and it is better to do a simple thing first and even -duplicate code when in doubt. -Abstracting or extracting duplicate code to a single place too early may lead to rigidness to a codebase before we understand the trade-offs, -so if there's no real necessity (that arises from usage) it might not be worth to "fix" it. - -A possible rule of thumb is that creating an abstraction to build a new thing might raise an alarm bell, -and building an abstraction in the middle of building yet another thing using the same pattern might be more appropriate. -Though still, You Might Not Need It. - -### Simplify testing and avoid infrastructure code - -At this point in time we've settled on a fairly sweet spot for testing, where infrastructure code is very limited and contained in one place. -Since testing can very easily become complex and "another project to maintain", we want to deliberately put limitations to how wildly it can grow, -and contain all infrastructure code to a single module - `tests/common/mod.rs` (in each relevant crate), without infra code in tests. - -We do this so that we will notice when it grows too large and requires some explicit and calculated attention. - -We would also like, in general, to keep infrastructure code to the required minimum, and not add things we'll need to maintain before using them. - -### Consider the generated SQL - -When working on a feature or fixing a bug, consider the generated SQL first. -What does it currently look like? What would you like it to look like? - -Compose a simplified yet runnable example and run it against the database (`just repl-postgres` can be helpful here), -find a query that returns the results you expect, examine edge cases, and then consider the rust implementation of this idea. - -### General coding style ideas - -- Use meaningful names (not shorthands). No need to skimp on vowels. -- Comment code with intention and reasons when those are not trivial. Comment modules, functions and types with a summary of what they are supposed to do. - Often writing summary comments helps us understand our code better and where it can be improved. -- Avoid using dummy values. Prefer throwing an error when in doubt. -- Prefer to avoid generics, traits and macros if possible. The majority of functions and types have a single usecase, and the ones that have multiple - use-cases probably don't have more than a handful of them. Duplicating code is often cheaper than the wrong abstraction, and we can always refactor. - Simple functions can go a long way. diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d652979b..fe436f88 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -13,7 +13,6 @@ ndc-bigquery-configuration = { path = "../configuration" } anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } serde_yaml = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } @@ -21,9 +20,5 @@ tokio = { workspace = true, features = ["full"] } [build-dependencies] build-data = { workspace = true } -[dev-dependencies] -insta = { workspace = true } -tempfile = { workspace = true } - [package.metadata.cargo-machete] -ignored = ["build_data"] # apparently cargo-machete doesn't find dependencies used by build scripts +ignored = ["build_data", "build-data"] # apparently cargo-machete doesn't find dependencies used by build scripts diff --git a/crates/configuration/Cargo.toml b/crates/configuration/Cargo.toml index 22169b0e..f4f0ecda 100644 --- a/crates/configuration/Cargo.toml +++ b/crates/configuration/Cargo.toml @@ -10,22 +10,12 @@ workspace = true [dependencies] ndc-models = { workspace = true } query-engine-metadata = { path = "../query-engine/metadata" } -query-engine-sql = { path = "../query-engine/sql" } anyhow = { workspace = true } -# We only use clap for the derive. -clap = { workspace = true, features = ["derive", "env"] } -prometheus = {workspace = true } schemars = { workspace = true, features = ["smol_str", "preserve_order"] } serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } -smol_str = { workspace = true } -sqlx = { workspace = true, features = ["json", "postgres", "runtime-tokio-rustls"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } -tracing = { workspace = true } yup-oauth2 = { workspace = true } gcp-bigquery-client = { workspace = true } - -[dev-dependencies] -jsonschema = { workspace = true } diff --git a/crates/connectors/ndc-bigquery/Cargo.toml b/crates/connectors/ndc-bigquery/Cargo.toml index 8f970b30..08dd53c9 100644 --- a/crates/connectors/ndc-bigquery/Cargo.toml +++ b/crates/connectors/ndc-bigquery/Cargo.toml @@ -24,29 +24,20 @@ ndc-bigquery-configuration = { path = "../../configuration" } ndc-sdk = { workspace = true } async-trait = { workspace = true } -enum-iterator = { workspace = true } gcp-bigquery-client = { workspace = true } prometheus = { workspace = true } -schemars = { workspace = true, features = ["smol_str", "preserve_order"] } -serde = { workspace = true } -serde_derive = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } -sqlx = { workspace = true, features = [ "json", "postgres", "runtime-tokio-rustls" ] } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } thiserror = { workspace = true } yup-oauth2 = { workspace = true } [dev-dependencies] -ndc-models = { workspace = true } ndc-test = { workspace = true } tests-common = { path = "../../tests/tests-common" } axum = { workspace = true } -axum-test-helper = { workspace = true } insta = { workspace = true, features = ["json"] } env_logger = { workspace = true } hyper = { workspace = true, features = ["tcp"] } reqwest = { workspace = true, features = ["rustls-tls"] } -similar-asserts = { workspace = true } -url = { workspace = true } diff --git a/crates/connectors/ndc-bigquery/src/error/convert.rs b/crates/connectors/ndc-bigquery/src/error/convert.rs index 223f7fba..44321c01 100644 --- a/crates/connectors/ndc-bigquery/src/error/convert.rs +++ b/crates/connectors/ndc-bigquery/src/error/convert.rs @@ -13,16 +13,7 @@ pub fn execution_error_to_response(error: query_engine_execution::error::Error) QueryError::NotSupported(_) => { connector::QueryError::new_unsupported_operation(&query_error.to_string()).into() } - QueryError::DBError(_) => { - connector::QueryError::new_unprocessable_content(&query_error.to_string()).into() - } - QueryError::DBConstraintError(_) | QueryError::MutationConstraintFailed => { - connector::MutationError::new_constraint_not_met(&query_error.to_string()).into() - } }, - Error::DB(_) => { - ErrorResponse::new_internal_with_details(serde_json::Value::String(error.to_string())) - } } } diff --git a/crates/connectors/ndc-bigquery/src/error/record.rs b/crates/connectors/ndc-bigquery/src/error/record.rs index 82294d6f..1f337e6f 100644 --- a/crates/connectors/ndc-bigquery/src/error/record.rs +++ b/crates/connectors/ndc-bigquery/src/error/record.rs @@ -8,19 +8,13 @@ pub fn execution_error(error: &query_engine_execution::error::Error, metrics: &m tracing::error!("{}", error); match error { Error::Query(err) => match &err { - QueryError::VariableNotFound(_) - | QueryError::DBError(_) - | QueryError::MutationConstraintFailed - | QueryError::DBConstraintError(_) => { + QueryError::VariableNotFound(_) => { metrics.error_metrics.record_invalid_request(); } QueryError::NotSupported(_) => { metrics.error_metrics.record_unsupported_feature(); } }, - Error::DB(_) => { - metrics.error_metrics.record_database_error(); - } } } diff --git a/crates/connectors/ndc-bigquery/src/state.rs b/crates/connectors/ndc-bigquery/src/state.rs index 5830b7b4..5f4446d5 100644 --- a/crates/connectors/ndc-bigquery/src/state.rs +++ b/crates/connectors/ndc-bigquery/src/state.rs @@ -53,8 +53,6 @@ pub async fn create_state( /// State initialization error. #[derive(Debug, Error)] pub enum InitializationError { - #[error("unable to initialize connection pool: {0}")] - UnableToCreatePool(sqlx::Error), #[error("error initializing metrics: {0}")] MetricsError(prometheus::Error), } diff --git a/crates/query-engine/execution/Cargo.toml b/crates/query-engine/execution/Cargo.toml index 2d825a91..31f7c94d 100644 --- a/crates/query-engine/execution/Cargo.toml +++ b/crates/query-engine/execution/Cargo.toml @@ -8,15 +8,10 @@ license.workspace = true workspace = true [dependencies] -ndc-models = { workspace = true } - query-engine-sql = { path = "../sql" } gcp-bigquery-client = { workspace = true } prometheus = { workspace = true } serde_json = { workspace = true } -sqlformat = { workspace = true } -sqlx = { workspace = true, features = [ "json", "postgres", "runtime-tokio-rustls", "uuid" ] } thiserror = { workspace = true } -tracing = { workspace = true } bytes = { workspace = true } diff --git a/crates/query-engine/execution/src/error.rs b/crates/query-engine/execution/src/error.rs index a0c5e519..75f1f39c 100644 --- a/crates/query-engine/execution/src/error.rs +++ b/crates/query-engine/execution/src/error.rs @@ -3,8 +3,6 @@ pub enum Error { #[error("{0}")] Query(QueryError), - #[error("{0}")] - DB(sqlx::Error), } /// Query planning error. @@ -14,33 +12,4 @@ pub enum QueryError { VariableNotFound(String), #[error("{0} are not supported.")] NotSupported(String), - #[error("{0}")] - DBError(sqlx::Error), - #[error("{0}")] - DBConstraintError(sqlx::Error), - #[error("Mutation constraint failed.")] - MutationConstraintFailed, -} - -impl From for Error { - fn from(err: sqlx::Error) -> Error { - match err - .as_database_error() - .and_then(|e| e.try_downcast_ref()) - .map(|e: &sqlx::postgres::PgDatabaseError| e.code()) - { - None => Error::DB(err), - Some(code) => { - // We want to map data and constraint exceptions to query errors - // https://www.postgresql.org/docs/current/errcodes-appendix.html - if code.starts_with("22") { - Error::Query(QueryError::DBError(err)) - } else if code.starts_with("23") { - Error::Query(QueryError::DBConstraintError(err)) - } else { - Error::DB(err) - } - } - } - } } diff --git a/crates/query-engine/execution/src/metrics.rs b/crates/query-engine/execution/src/metrics.rs index 0e388f39..8df01350 100644 --- a/crates/query-engine/execution/src/metrics.rs +++ b/crates/query-engine/execution/src/metrics.rs @@ -1,8 +1,6 @@ //! Metrics setup and update for our connector. -use std::time::Duration; - -use prometheus::{Gauge, Histogram, HistogramTimer, IntCounter, IntGauge, Registry}; +use prometheus::{Histogram, HistogramTimer, IntCounter, Registry}; /// The collection of all metrics exposed through the `/metrics` endpoint. #[derive(Debug, Clone)] @@ -11,11 +9,6 @@ pub struct Metrics { explain_total: IntCounter, query_plan_time: Histogram, query_execution_time: Histogram, - pool_max_connections: IntGauge, - pool_min_connections: IntGauge, - pool_acquire_timeout: Gauge, - pool_max_lifetime: Gauge, - pool_idle_timeout: Gauge, pub error_metrics: ErrorMetrics, } @@ -24,58 +17,28 @@ impl Metrics { pub fn initialize(metrics_registry: &mut Registry) -> Result { let query_total = add_int_counter_metric( metrics_registry, - "postgres_ndc_query_total", + "bigquery_ndc_query_total", "Total successful queries.", )?; let explain_total = add_int_counter_metric( metrics_registry, - "postgres_ndc_explain_total", + "bigquery_ndc_explain_total", "Total successful explains.", )?; let query_plan_time = add_histogram_metric( metrics_registry, - "postgres_ndc_query_plan_time", + "bigquery_ndc_query_plan_time", "Time taken to plan a query for execution, in seconds.", )?; let query_execution_time = add_histogram_metric( metrics_registry, - "postgres_ndc_query_execution_time", + "bigquery_ndc_query_execution_time", "Time taken to execute an already-planned query, in seconds.", )?; - let pool_max_connections = add_int_gauge_metric( - metrics_registry, - "postgres_ndc_pool_max_connections", - "The maximum number of connections that this pool should maintain.", - )?; - - let pool_min_connections = add_int_gauge_metric( - metrics_registry, - "postgres_ndc_pool_min_connections", - "The minimum number of connections that this pool should maintain.", - )?; - - let pool_acquire_timeout = add_gauge_metric( - metrics_registry, - "postgres_ndc_pool_acquire_timeout", - "Get the maximum amount of time to spend waiting for a connection, in seconds.", - )?; - - let pool_idle_timeout = add_gauge_metric( - metrics_registry, - "postgres_ndc_pool_idle_timeout", - "Get the maximum idle duration for individual connections, in seconds.", - )?; - - let pool_max_lifetime = add_gauge_metric( - metrics_registry, - "postgres_ndc_pool_max_lifetime", - "Get the maximum lifetime of individual connections, in seconds.", - )?; - let error_metrics = ErrorMetrics::initialize(metrics_registry)?; Ok(Self { @@ -83,11 +46,6 @@ impl Metrics { explain_total, query_plan_time, query_execution_time, - pool_max_connections, - pool_min_connections, - pool_acquire_timeout, - pool_max_lifetime, - pool_idle_timeout, error_metrics, }) } @@ -107,34 +65,6 @@ impl Metrics { pub fn time_query_execution(&self) -> Timer { Timer(self.query_execution_time.start_timer()) } - - // Set the metrics populated from the pool options. - // - // This only needs to be called once, as the options don't change. - pub fn set_pool_options_metrics(&self, pool_options: &sqlx::pool::PoolOptions) { - let max_connections: i64 = pool_options.get_max_connections().into(); - self.pool_max_connections.set(max_connections); - - let min_connections: i64 = pool_options.get_min_connections().into(); - self.pool_min_connections.set(min_connections); - - let acquire_timeout: f64 = pool_options.get_acquire_timeout().as_secs_f64(); - self.pool_acquire_timeout.set(acquire_timeout); - - // if nothing is set, return 0 - let idle_timeout: f64 = pool_options - .get_idle_timeout() - .unwrap_or(Duration::ZERO) - .as_secs_f64(); - self.pool_idle_timeout.set(idle_timeout); - - // if nothing is set, return 0 - let max_lifetime: f64 = pool_options - .get_max_lifetime() - .unwrap_or(Duration::ZERO) - .as_secs_f64(); - self.pool_max_lifetime.set(max_lifetime); - } } /// Create a new int counter metric and register it with the provided Prometheus Registry @@ -148,26 +78,6 @@ fn add_int_counter_metric( register_collector(metrics_registry, int_counter) } -/// Create a new int gauge metric and register it with the provided Prometheus Registry -fn add_int_gauge_metric( - metrics_registry: &mut Registry, - metric_name: &str, - metric_description: &str, -) -> Result { - let int_gauge = IntGauge::with_opts(prometheus::Opts::new(metric_name, metric_description))?; - register_collector(metrics_registry, int_gauge) -} - -/// Create a new gauge metric and register it with the provided Prometheus Registry -fn add_gauge_metric( - metrics_registry: &mut Registry, - metric_name: &str, - metric_description: &str, -) -> Result { - let gauge = Gauge::with_opts(prometheus::Opts::new(metric_name, metric_description))?; - register_collector(metrics_registry, gauge) -} - /// Create a new histogram metric using the default buckets, and register it with the provided /// Prometheus Registry. fn add_histogram_metric( @@ -249,37 +159,37 @@ impl ErrorMetrics { ) -> Result { let invalid_request_total = add_int_counter_metric( metrics_registry, - "ndc_postgres_error_invalid_request_total_count", + "ndc_bigquery_error_invalid_request_total_count", "Total number of invalid requests encountered.", )?; let unsupported_capability_total = add_int_counter_metric( metrics_registry, - "ndc_postgres_error_unsupported_capability_total_count", + "ndc_bigquery_error_unsupported_capability_total_count", "Total number of invalid requests with unsupported capabilities encountered.", )?; let unsupported_feature_total = add_int_counter_metric( metrics_registry, - "ndc_postgres_error_unsupported_capabilities_total_count", + "ndc_bigquery_error_unsupported_capabilities_total_count", "Total number of invalid requests with unsupported capabilities encountered.", )?; let connector_error_total = add_int_counter_metric( metrics_registry, - "ndc_postgres_error_connector_error_total_count", + "ndc_bigquery_error_connector_error_total_count", "Total number of requests failed due to an internal conenctor error.", )?; let database_error_total = add_int_counter_metric( metrics_registry, - "ndc_postgres_error_database_error_total_count", + "ndc_bigquery_error_database_error_total_count", "Total number of requests failed due to a database error.", )?; let connection_acquisition_error_total = add_int_counter_metric( metrics_registry, - "ndc_postgres_error_connection_acquisition_error_total_count", + "ndc_bigquery_error_connection_acquisition_error_total_count", "Total number of failures to acquire a database connection.", )?; diff --git a/crates/query-engine/metadata/Cargo.toml b/crates/query-engine/metadata/Cargo.toml index da4519b2..6cf6cf29 100644 --- a/crates/query-engine/metadata/Cargo.toml +++ b/crates/query-engine/metadata/Cargo.toml @@ -11,7 +11,5 @@ workspace = true ndc-models = { workspace = true } smol_str = { workspace = true } -enum-iterator = { workspace = true } schemars = { workspace = true, features = ["smol_str"] } serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } diff --git a/crates/query-engine/sql/Cargo.toml b/crates/query-engine/sql/Cargo.toml index 7abc9e84..e539c8c9 100644 --- a/crates/query-engine/sql/Cargo.toml +++ b/crates/query-engine/sql/Cargo.toml @@ -10,6 +10,4 @@ workspace = true [dependencies] ndc-models = { workspace = true } -schemars = { workspace = true, features = ["smol_str", "preserve_order"] } -serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/query-engine/translation/Cargo.toml b/crates/query-engine/translation/Cargo.toml index 8f143a75..5c5e5e32 100644 --- a/crates/query-engine/translation/Cargo.toml +++ b/crates/query-engine/translation/Cargo.toml @@ -17,10 +17,9 @@ multimap = { workspace = true } ref-cast = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -tracing = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } [dev-dependencies] insta = { workspace = true, features = ["json"] } -sqlformat = { workspace = true } \ No newline at end of file +sqlformat = { workspace = true } diff --git a/crates/tests/tests-common/Cargo.toml b/crates/tests/tests-common/Cargo.toml index 7f478fe9..2fb34cb6 100644 --- a/crates/tests/tests-common/Cargo.toml +++ b/crates/tests/tests-common/Cargo.toml @@ -12,22 +12,11 @@ name = "tests_common" path = "src/lib.rs" [dependencies] -ndc-bigquery = { path = "../../connectors/ndc-bigquery" } - ndc-sdk = { workspace = true } -ndc-test = { workspace = true } -anyhow = { workspace = true } axum = { workspace = true } axum-test-helper = { workspace = true } -env_logger = { workspace = true } -hyper = { workspace = true, features = ["tcp"] } -reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } serde_derive = { workspace = true } -sqlx = { workspace = true, features = [ "json", "postgres", "runtime-tokio-rustls" ] } -tokio = { workspace = true, features = ["full"] } -tokio-postgres = { workspace = true } tracing = { workspace = true } -uuid = { workspace = true, features = [ "v4", "fast-rng", "macro-diagnostics" ] }