diff --git a/Cargo.lock b/Cargo.lock index 58fbd4780..58d8fc41f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,12 +73,6 @@ dependencies = [ "nodrop", ] -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.2" @@ -107,9 +101,9 @@ dependencies = [ [[package]] name = "atoi" -version = "0.4.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" dependencies = [ "num-traits", ] @@ -125,12 +119,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - [[package]] name = "autocfg" version = "1.0.1" @@ -152,12 +140,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base-x" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" - [[package]] name = "base64" version = "0.13.0" @@ -179,33 +161,12 @@ dependencies = [ "serde", ] -[[package]] -name = "bigdecimal" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256" -dependencies = [ - "num-bigint 0.3.3", - "num-integer", - "num-traits", - "serde", -] - [[package]] name = "bit-vec" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -228,34 +189,22 @@ dependencies = [ "cty", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.4", + "generic-array", ] [[package]] -name = "block-padding" -version = "0.1.5" +name = "block-buffer" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ - "byte-tools", + "generic-array", ] [[package]] @@ -276,12 +225,6 @@ version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "bytemuck" version = "1.7.2" @@ -309,7 +252,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" dependencies = [ - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -339,8 +282,7 @@ dependencies = [ "libc", "num-integer", "num-traits", - "serde", - "time 0.1.43", + "time", "winapi", ] @@ -350,7 +292,7 @@ version = "0.101.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e825530271e31d1439ae1b4a74d99d0429b4266fcac0ae0bc92a843911e0de72" dependencies = [ - "crossbeam-channel 0.5.1", + "crossbeam-channel", ] [[package]] @@ -477,7 +419,7 @@ version = "0.101.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e48888217f8778108375fb656915d8a66b813691d740368eb5d88b09f40f0b3" dependencies = [ - "bit-vec 0.5.1", + "bit-vec", "bitflags", "bytes", "ckb-channel", @@ -521,7 +463,7 @@ dependencies = [ "creep", "derive_more", "hex", - "minstant 0.0.2", + "minstant", "once_cell", "rand 0.8.4", "rust-crypto", @@ -541,16 +483,8 @@ dependencies = [ "log4rs", "serde", "serde_derive", - "tracing 0.1.0", - "tracing-derive", ] -[[package]] -name = "const_fn" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" - [[package]] name = "convert_case" version = "0.4.0" @@ -564,7 +498,6 @@ dependencies = [ "ckb-jsonrpc-types", "ckb-types", "common", - "common-logger", "core-storage", "core-synchronization", "jsonrpc-core", @@ -621,13 +554,13 @@ dependencies = [ "ckb-types", "clap", "common", - "common-logger", "core-ckb-client", "core-cli", "core-rpc-types", "core-storage", - "crossbeam-channel 0.5.1", + "crossbeam-channel", "dashmap", + "db-sqlx", "env_logger", "hex", "jsonrpsee", @@ -635,17 +568,17 @@ dependencies = [ "jsonrpsee-proc-macros", "lazy_static", "log", - "num-bigint 0.4.3", + "num-bigint", "num-traits", "parking_lot 0.12.0", "pprof", "rand 0.7.3", - "rbatis", "reqwest", "serde", "serde_json", + "sqlx", "tokio", - "xsql", + "xsql-test", ] [[package]] @@ -662,20 +595,7 @@ dependencies = [ "parking_lot 0.12.0", "protocol", "serde", -] - -[[package]] -name = "core-rpc-utility" -version = "0.4.1" -dependencies = [ - "ckb-dao-utils", - "ckb-jsonrpc-types", - "ckb-types", - "common", - "common-logger", - "core-ckb-client", - "core-rpc-types", - "core-storage", + "serde_json", ] [[package]] @@ -707,22 +627,22 @@ dependencies = [ "ckb-types", "clap", "common", - "common-logger", + "core-rpc-types", "criterion", "dashmap", + "db-sqlx", "env_logger", "hex", "lazy_static", "log", "protocol", - "rand 0.7.3", - "rbatis", - "rbson", + "seq-macro", "serde", "serde_json", + "sql-builder", + "sqlx", "tokio", "tokio-stream", - "xsql", "xsql-test", ] @@ -735,18 +655,22 @@ dependencies = [ "common", "core-rpc-types", "core-storage", + "db-sqlx", "env_logger", "futures", + "hex", "itertools", "lazy_static", "log", "parking_lot 0.12.0", - "rbatis", - "rbson", + "rand 0.7.3", + "seq-macro", "serde", "serde_json", + "sql-builder", + "sqlx", "tokio", - "xsql", + "xsql-test", ] [[package]] @@ -769,18 +693,18 @@ dependencies = [ [[package]] name = "crc" -version = "2.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "1.1.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff" [[package]] name = "creep" @@ -829,30 +753,6 @@ dependencies = [ "itertools", ] -[[package]] -name = "crossbeam" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-channel 0.4.4", - "crossbeam-deque 0.7.4", - "crossbeam-epoch 0.8.2", - "crossbeam-queue 0.2.3", - "crossbeam-utils 0.7.2", -] - -[[package]] -name = "crossbeam-channel" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" -dependencies = [ - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -860,18 +760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.5", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -881,23 +770,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.5", - "crossbeam-utils 0.8.5", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg 1.0.1", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -907,23 +781,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.5", + "crossbeam-utils", "lazy_static", - "memoffset 0.6.4", + "memoffset", "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-queue" version = "0.3.2" @@ -931,18 +794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.5", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg 1.0.1", - "cfg-if 0.1.10", - "lazy_static", + "crossbeam-utils", ] [[package]] @@ -956,13 +808,13 @@ dependencies = [ ] [[package]] -name = "crypto-mac" -version = "0.11.1" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.4", - "subtle", + "generic-array", + "typenum", ] [[package]] @@ -973,7 +825,7 @@ checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -1013,6 +865,22 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "db-sqlx" +version = "0.1.0" +dependencies = [ + "ckb-types", + "common", + "futures", + "hex", + "log", + "once_cell", + "protocol", + "sql-builder", + "sqlx", + "tokio", +] + [[package]] name = "debugid" version = "0.7.2" @@ -1042,33 +910,35 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.0", + "rustc_version", "syn", ] [[package]] name = "digest" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.12.4", + "generic-array", ] [[package]] name = "digest" -version = "0.9.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "generic-array 0.14.4", + "block-buffer 0.10.2", + "crypto-common", + "subtle", ] [[package]] name = "dirs" -version = "3.0.2" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ "dirs-sys", ] @@ -1085,10 +955,13 @@ dependencies = [ ] [[package]] -name = "discard" -version = "1.0.4" +name = "dotenvy" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +checksum = "7e851a83c30366fd01d75b913588e95e74a1705c1ecc5d58b1f8e1a6d556525f" +dependencies = [ + "dirs", +] [[package]] name = "dtoa" @@ -1107,9 +980,6 @@ name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" -dependencies = [ - "serde", -] [[package]] name = "encoding_rs" @@ -1139,12 +1009,6 @@ version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "faster-hex" version = "0.6.1" @@ -1163,6 +1027,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin 0.9.4", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1217,9 +1093,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", "futures-sink", @@ -1227,15 +1103,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-executor" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b808bf53348a36cab739d7e04755909b9fcaaa69b7d7e588b37b6ec62704c97" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" dependencies = [ "futures-core", "futures-task", @@ -1255,15 +1131,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-macro" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89f17b21645bc4ed773c69af9c9a0effd4a3f1a3876eadd453469f8854e7fdd" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ "proc-macro2", "quote", @@ -1272,15 +1148,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" [[package]] name = "futures-task" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-timer" @@ -1294,9 +1170,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.18" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d22213122356472061ac0f1ab2cee28d2bac8491410fd68c2af53d1cedb83e" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ "futures-channel", "futures-core", @@ -1316,15 +1192,6 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.4" @@ -1387,7 +1254,7 @@ dependencies = [ "futures-sink", "gloo-utils", "js-sys", - "pin-project 1.0.10", + "pin-project", "serde", "serde_json", "thiserror", @@ -1435,7 +1302,7 @@ dependencies = [ "slab", "tokio", "tokio-util 0.6.9", - "tracing 0.1.29", + "tracing", ] [[package]] @@ -1449,17 +1316,23 @@ name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ "ahash", ] [[package]] name = "hashlink" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +checksum = "d452c155cb93fecdfb02a73dd57b5d8e442c2063bd7aac72f1bc5e4263a43086" dependencies = [ - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -1473,9 +1346,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" dependencies = [ "unicode-segmentation", ] @@ -1496,28 +1369,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hmac" -version = "0.11.0" +name = "hkdf" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" dependencies = [ - "crypto-mac", - "digest 0.9.0", + "hmac", ] [[package]] -name = "html_parser" -version = "0.6.2" +name = "hmac" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58112d7b68ff61447bd22a489d7877a79ee688f6f34d57d3857842776d42dd0c" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "pest", - "pest_derive", - "serde", - "serde_derive", - "serde_json", - "structopt", - "thiserror", + "digest 0.10.3", ] [[package]] @@ -1528,7 +1394,7 @@ checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 0.4.8", ] [[package]] @@ -1575,12 +1441,12 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 0.4.8", "pin-project-lite", "socket2", "tokio", "tower-service", - "tracing 0.1.29", + "tracing", "want", ] @@ -1630,8 +1496,8 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ - "autocfg 1.0.1", - "hashbrown", + "autocfg", + "hashbrown 0.11.2", ] [[package]] @@ -1643,7 +1509,7 @@ dependencies = [ "ahash", "atty", "indexmap", - "itoa", + "itoa 0.4.8", "lazy_static", "log", "num-format", @@ -1667,15 +1533,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" -[[package]] -name = "ipnetwork" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c3eaab3ac0ede60ffa41add21970a7df7d91772c03383aac6c2c3d53cc716b" -dependencies = [ - "serde", -] - [[package]] name = "itertools" version = "0.10.1" @@ -1691,6 +1548,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + [[package]] name = "js-sys" version = "0.3.55" @@ -1736,7 +1599,7 @@ dependencies = [ "jsonrpsee-wasm-client", "jsonrpsee-ws-client", "jsonrpsee-ws-server", - "tracing 0.1.29", + "tracing", ] [[package]] @@ -1753,14 +1616,14 @@ dependencies = [ "http", "jsonrpsee-core", "jsonrpsee-types", - "pin-project 1.0.10", + "pin-project", "rustls-native-certs", "soketto", "thiserror", "tokio", "tokio-rustls", "tokio-util 0.7.2", - "tracing 0.1.29", + "tracing", "webpki-roots", ] @@ -1788,7 +1651,7 @@ dependencies = [ "soketto", "thiserror", "tokio", - "tracing 0.1.29", + "tracing", "wasm-bindgen-futures", ] @@ -1808,7 +1671,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "tracing 0.1.29", + "tracing", ] [[package]] @@ -1826,7 +1689,7 @@ dependencies = [ "lazy_static", "serde_json", "tokio", - "tracing 0.1.29", + "tracing", "unicase", ] @@ -1853,7 +1716,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "tracing 0.1.29", + "tracing", ] [[package]] @@ -1892,7 +1755,7 @@ dependencies = [ "soketto", "tokio", "tokio-util 0.7.2", - "tracing 0.1.29", + "tracing", ] [[package]] @@ -1900,9 +1763,6 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin", -] [[package]] name = "libc" @@ -1910,17 +1770,11 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" -[[package]] -name = "libm" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" - [[package]] name = "libsqlite3-sys" -version = "0.22.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" dependencies = [ "cc", "pkg-config", @@ -1985,33 +1839,19 @@ dependencies = [ "winapi", ] -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - [[package]] name = "matches" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md-5" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest 0.10.3", ] [[package]] @@ -2030,22 +1870,13 @@ dependencies = [ "winapi", ] -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg 1.0.1", -] - [[package]] name = "memoffset" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] @@ -2081,28 +1912,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "minitrace" -version = "0.1.0" -source = "git+https://github.com/tikv/minitrace-rust.git#6badc9866af3a80219613c2d5c19af41c336d18f" +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ - "crossbeam", - "minstant 0.0.1", - "pin-project 0.4.28", + "adler", + "autocfg", ] [[package]] -name = "minitrace-jaeger" -version = "0.1.0" -source = "git+https://github.com/tikv/minitrace-rust.git#6badc9866af3a80219613c2d5c19af41c336d18f" +name = "minstant" +version = "0.0.2" +source = "git+https://github.com/tikv/minstant.git?branch=master#dcd3e1e66aef844ce0bb5433d46f7a779fbff2e8" dependencies = [ - "minitrace", - "thrift_codec", + "ctor", + "libc", + "minstant_macro", + "wasi 0.7.0", ] [[package]] -name = "minitrace-macro" +name = "minstant_macro" version = "0.1.0" -source = "git+https://github.com/tikv/minitrace-rust.git#6badc9866af3a80219613c2d5c19af41c336d18f" +source = "git+https://github.com/tikv/minstant.git?branch=master#dcd3e1e66aef844ce0bb5433d46f7a779fbff2e8" dependencies = [ "proc-macro-error", "quote", @@ -2110,49 +1943,8 @@ dependencies = [ ] [[package]] -name = "miniz_oxide" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg 1.0.1", -] - -[[package]] -name = "minstant" -version = "0.0.1" -source = "git+https://github.com/zhongzc/minstant.git?rev=dc7dd5c17c564601afff7c0b640fd430728bfcd5#dc7dd5c17c564601afff7c0b640fd430728bfcd5" -dependencies = [ - "lazy_static", - "libc", - "wasi 0.7.0", -] - -[[package]] -name = "minstant" -version = "0.0.2" -source = "git+https://github.com/tikv/minstant.git?branch=master#dcd3e1e66aef844ce0bb5433d46f7a779fbff2e8" -dependencies = [ - "ctor", - "libc", - "minstant_macro", - "wasi 0.7.0", -] - -[[package]] -name = "minstant_macro" -version = "0.1.0" -source = "git+https://github.com/tikv/minstant.git?branch=master#dcd3e1e66aef844ce0bb5433d46f7a779fbff2e8" -dependencies = [ - "proc-macro-error", - "quote", - "syn", -] - -[[package]] -name = "mio" -version = "0.8.3" +name = "mio" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ @@ -2201,7 +1993,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "memoffset 0.6.4", + "memoffset", ] [[package]] @@ -2221,44 +2013,15 @@ dependencies = [ "version_check", ] -[[package]] -name = "num-bigint" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" -dependencies = [ - "autocfg 1.0.1", - "num-integer", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ - "autocfg 1.0.1", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-bigint-dig" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" -dependencies = [ - "autocfg 0.1.7", - "byteorder", - "lazy_static", - "libm", + "autocfg", "num-integer", - "num-iter", "num-traits", - "rand 0.8.4", - "smallvec", - "zeroize", ] [[package]] @@ -2268,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" dependencies = [ "arrayvec 0.4.12", - "itoa", + "itoa 0.4.8", ] [[package]] @@ -2277,18 +2040,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg 1.0.1", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" -dependencies = [ - "autocfg 1.0.1", - "num-integer", + "autocfg", "num-traits", ] @@ -2298,8 +2050,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg 1.0.1", - "libm", + "autocfg", ] [[package]] @@ -2379,12 +2130,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.0" @@ -2417,7 +2162,7 @@ version = "0.9.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df13d165e607909b363a4757a6f133f8a818a74e9d3a98d09c6128e15fa4c73" dependencies = [ - "autocfg 1.0.1", + "autocfg", "cc", "libc", "pkg-config", @@ -2482,15 +2227,10 @@ dependencies = [ ] [[package]] -name = "pem" -version = "0.8.3" +name = "paste" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" -dependencies = [ - "base64", - "once_cell", - "regex", -] +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" [[package]] name = "percent-encoding" @@ -2498,76 +2238,13 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "pest" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" -dependencies = [ - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pest_meta" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" -dependencies = [ - "maplit", - "pest", - "sha-1 0.8.2", -] - -[[package]] -name = "pin-project" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918192b5c59119d51e0cd221f4d49dde9112824ba717369e903c97d076083d0f" -dependencies = [ - "pin-project-internal 0.4.28", -] - [[package]] name = "pin-project" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" dependencies = [ - "pin-project-internal 1.0.10", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be26700300be6d9d23264c73211d8190e755b6b5ca7a1b28230025511b52a5e" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "pin-project-internal", ] [[package]] @@ -2687,19 +2364,13 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - [[package]] name = "proc-macro2" -version = "1.0.32" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -2712,18 +2383,6 @@ dependencies = [ "serde", ] -[[package]] -name = "py_sql" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406b706c360fcde6e93391e83ef16aed0798a96f08e8f0e174de0681ec0a6060" -dependencies = [ - "dashmap", - "rexpr", - "serde", - "serde_json", -] - [[package]] name = "quick-xml" version = "0.22.0" @@ -2735,9 +2394,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.10" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] @@ -2867,8 +2526,8 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ - "autocfg 1.0.1", - "crossbeam-deque 0.8.1", + "autocfg", + "crossbeam-deque", "either", "rayon-core", ] @@ -2879,121 +2538,13 @@ version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ - "crossbeam-channel 0.5.1", - "crossbeam-deque 0.8.1", - "crossbeam-utils 0.8.5", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", "lazy_static", "num_cpus", ] -[[package]] -name = "rbatis" -version = "3.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce45febe5f43f2495d3d55c1342ebadc49444b77a830bcad7599a6481c8686a6" -dependencies = [ - "async-trait", - "chrono", - "futures", - "futures-core", - "hex", - "lazy_static", - "log", - "once_cell", - "rand 0.8.4", - "rbatis-core", - "rbatis-macro-driver", - "rbatis_sql", - "rbson", - "serde", - "uuid", -] - -[[package]] -name = "rbatis-core" -version = "3.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9baa1d636ad53d589680dcc9674fb58829abb42e0b9e694b50ff3da6e2d26395" -dependencies = [ - "base64", - "bigdecimal", - "bit-vec 0.6.3", - "chrono", - "hex", - "ipnetwork", - "lazy_static", - "log", - "py_sql", - "rbson", - "rexpr", - "serde", - "serde_json", - "sqlx-core", - "time 0.2.27", - "uuid", -] - -[[package]] -name = "rbatis-macro-driver" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82273c716eca70f4945cd0485dbc3a525726242ea7fee713ca490f081ca47f86" -dependencies = [ - "html_parser", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "rbatis_sql" -version = "3.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852b627d81d1e2d8313b024d899190baaf8e75429f5d49e842a0bdac77474c4" -dependencies = [ - "async-trait", - "base64", - "dashmap", - "rbatis_sql_macro", - "rbson", - "serde", -] - -[[package]] -name = "rbatis_sql_macro" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5050d3450a18cb42f377c8881df476fd33e6a65df28309bf8be08995ac8e4935" -dependencies = [ - "async-trait", - "base64", - "html_parser", - "proc-macro2", - "quote", - "syn", - "url", - "xml-rs", -] - -[[package]] -name = "rbson" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f7c79c3b796113c856d3f1b7ca27c453e1efea1096e9d4fac6dba84d641dbc" -dependencies = [ - "ahash", - "base64", - "chrono", - "hex", - "indexmap", - "lazy_static", - "rand 0.8.4", - "serde", - "serde_bytes", - "serde_json", - "uuid", -] - [[package]] name = "rdrand" version = "0.4.0" @@ -3095,17 +2646,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rexpr" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4753cc5e42400348f62002a279dd25e135fa46b95d0cac6935c6380deb31feda" -dependencies = [ - "dashmap", - "serde", - "serde_json", -] - [[package]] name = "rgb" version = "0.8.30" @@ -3124,32 +2664,12 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", ] -[[package]] -name = "rsa" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0aeddcca1082112a6eeb43bf25fd7820b066aaf6eaef776e19d0a1febe38fe" -dependencies = [ - "byteorder", - "digest 0.9.0", - "lazy_static", - "num-bigint-dig", - "num-integer", - "num-iter", - "num-traits", - "pem", - "rand 0.8.4", - "simple_asn1", - "subtle", - "zeroize", -] - [[package]] name = "rust-crypto" version = "0.2.36" @@ -3160,18 +2680,7 @@ dependencies = [ "libc", "rand 0.3.23", "rustc-serialize", - "time 0.1.43", -] - -[[package]] -name = "rust_decimal" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b5a9625a7e6060b23db692facf49082cc78889a7e6ac94a735356ae49db4b0" -dependencies = [ - "arrayvec 0.5.2", - "num-traits", - "serde", + "time", ] [[package]] @@ -3192,22 +2701,13 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.4", + "semver", ] [[package]] @@ -3325,38 +2825,29 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "send_wrapper" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" +[[package]] +name = "seq-macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d9841243dbc9928f5fed7946d2862292eebd823d96e13e556924d3db0120d2" + [[package]] name = "serde" -version = "1.0.130" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" dependencies = [ "serde_derive", ] @@ -3371,15 +2862,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_bytes" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" -dependencies = [ - "serde", -] - [[package]] name = "serde_cbor" version = "0.11.2" @@ -3392,9 +2874,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" dependencies = [ "proc-macro2", "quote", @@ -3403,12 +2885,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.72" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ - "indexmap", - "itoa", + "itoa 1.0.2", "ryu", "serde", ] @@ -3420,7 +2901,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ "form_urlencoded", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -3437,18 +2918,6 @@ dependencies = [ "yaml-rust", ] -[[package]] -name = "sha-1" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - [[package]] name = "sha-1" version = "0.9.8" @@ -3459,38 +2928,38 @@ dependencies = [ "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] -name = "sha1" -version = "0.6.0" +name = "sha-1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] [[package]] name = "sha2" -version = "0.9.8" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ - "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest 0.10.3", ] [[package]] -name = "simple_asn1" -version = "0.5.4" +name = "signal-hook-registry" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb4ea60fb301dc81dfc113df680571045d375ab7345d171c5dc7d7e13107a80" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ - "chrono", - "num-bigint 0.4.3", - "num-traits", - "thiserror", + "libc", ] [[package]] @@ -3536,6 +3005,25 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +dependencies = [ + "lock_api", +] + +[[package]] +name = "sql-builder" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1008d95d2ec2d062959352527be30e10fec42a1aa5e5a48d990a5ff0fb9bdc0" +dependencies = [ + "anyhow", + "thiserror", +] + [[package]] name = "sqlformat" version = "0.1.8" @@ -3547,143 +3035,105 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "sqlx" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "788841def501aabde58d3666fcea11351ec3962e6ea75dbcd05c84a71d68bcd1" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + [[package]] name = "sqlx-core" -version = "0.5.9" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec89bfaca8f7737439bad16d52b07f1ccd0730520d3bf6ae9d069fe4b641fb1" +checksum = "8c21d3b5e7cadfe9ba7cdc1295f72cc556c750b4419c27c219c0693198901f8e" dependencies = [ "ahash", "atoi", "base64", - "bigdecimal", - "bit-vec 0.6.3", "bitflags", "byteorder", "bytes", - "chrono", "crc", - "crossbeam-channel 0.5.1", - "crossbeam-queue 0.3.2", - "crossbeam-utils 0.8.5", - "digest 0.9.0", + "crossbeam-queue", "dirs", + "dotenvy", "either", - "encoding_rs", + "event-listener", + "flume", "futures-channel", "futures-core", + "futures-executor", "futures-intrusive", "futures-util", - "generic-array 0.14.4", "hashlink", "hex", + "hkdf", "hmac", "indexmap", - "itoa", + "itoa 1.0.2", "libc", "libsqlite3-sys", "log", "md-5", "memchr", - "num-bigint 0.3.3", "once_cell", - "parking_lot 0.11.2", + "paste", "percent-encoding", "rand 0.8.4", - "regex", - "rsa", - "rust_decimal", "serde", "serde_json", - "sha-1 0.9.8", + "sha-1 0.10.0", "sha2", "smallvec", "sqlformat", "sqlx-rt", "stringprep", "thiserror", - "time 0.2.27", "tokio-stream", "url", - "uuid", "whoami", ] [[package]] -name = "sqlx-rt" -version = "0.5.9" +name = "sqlx-macros" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0" +checksum = "4adfd2df3557bddd3b91377fc7893e8fa899e9b4061737cbade4e1bb85f1b45c" dependencies = [ - "native-tls", + "dotenvy", + "either", + "heck", "once_cell", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ "proc-macro2", "quote", - "serde", - "serde_derive", + "sha2", + "sqlx-core", + "sqlx-rt", "syn", + "url", ] [[package]] -name = "stdweb-internal-macros" -version = "0.2.9" +name = "sqlx-rt" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +checksum = "7be52fc7c96c136cedea840ed54f7d446ff31ad670c9dea95ebcb998530971a3" dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn", + "native-tls", + "once_cell", + "tokio", + "tokio-native-tls", ] [[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" +name = "stable_deref_trait" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "str_stack" @@ -3707,30 +3157,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -[[package]] -name = "structopt" -version = "0.3.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c" -dependencies = [ - "clap", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "subtle" version = "2.4.1" @@ -3762,25 +3188,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.82" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", - "unicode-xid", -] - -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -3846,16 +3260,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "thrift_codec" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb61fb3d0a0af14949f3a6949b2639112e13226647112824f4d081533f9b1a8" -dependencies = [ - "byteorder", - "trackable 0.2.24", -] - [[package]] name = "time" version = "0.1.43" @@ -3866,45 +3270,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "serde", - "standback", - "stdweb", - "time-macros", - "version_check", - "winapi", -] - -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -3942,7 +3307,9 @@ dependencies = [ "mio", "num_cpus", "once_cell", + "parking_lot 0.12.0", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "winapi", @@ -4035,18 +3402,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" -[[package]] -name = "tracing" -version = "0.1.0" -dependencies = [ - "arc-swap 1.5.0", - "lazy_static", - "minitrace", - "minitrace-jaeger", - "minitrace-macro", - "tokio", -] - [[package]] name = "tracing" version = "0.1.29" @@ -4079,46 +3434,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "tracing-derive" -version = "0.1.0" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "serde_json", - "syn", -] - -[[package]] -name = "trackable" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98abb9e7300b9ac902cc04920945a874c1973e08c310627cc4458c04b70dd32" -dependencies = [ - "trackable 1.2.0", - "trackable_derive", -] - -[[package]] -name = "trackable" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "017e2a1a93718e4e8386d037cfb8add78f1d690467f4350fb582f55af1203167" -dependencies = [ - "trackable_derive", -] - -[[package]] -name = "trackable_derive" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" -dependencies = [ - "quote", - "syn", -] - [[package]] name = "traitobject" version = "0.1.0" @@ -4146,12 +3461,6 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" -[[package]] -name = "ucd-trie" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" - [[package]] name = "unicase" version = "2.6.0" @@ -4167,6 +3476,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +[[package]] +name = "unicode-ident" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" + [[package]] name = "unicode-normalization" version = "0.1.19" @@ -4188,12 +3503,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -4232,10 +3541,6 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.3", - "serde", -] [[package]] name = "vcpkg" @@ -4533,40 +3838,12 @@ dependencies = [ "winapi", ] -[[package]] -name = "xml-rs" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" - -[[package]] -name = "xsql" -version = "0.1.0" -dependencies = [ - "arc-swap 1.5.0", - "base64", - "common", - "criterion", - "env_logger", - "log", - "protocol", - "rand 0.7.3", - "rbatis", - "rbson", - "serde", - "serde_json", -] - [[package]] name = "xsql-test" version = "0.1.0" dependencies = [ - "ckb-jsonrpc-types", "common", - "protocol", - "rbatis", - "serde_json", - "xsql", + "sqlx", ] [[package]] @@ -4577,24 +3854,3 @@ checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" dependencies = [ "linked-hash-map", ] - -[[package]] -name = "zeroize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] diff --git a/Cargo.toml b/Cargo.toml index 2df6bd77a..e0c1c1d84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] core-cli = { path = "core/cli" } log = "0.4" -tokio = { version = "1.14", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [dev-dependencies] criterion = { version = "0.3", features = ["async_tokio", "cargo_bench_support"] } @@ -16,9 +16,6 @@ rand = "0.8" [workspace] members = [ - "apm/tracing", - "apm/tracing-derive", - "common", "logger", "protocol", @@ -27,13 +24,12 @@ members = [ "core/ckb-client", "core/rpc/core", "core/rpc/types", - "core/rpc/utility", "core/service", "core/storage", "core/synchronization", - "db/xsql", - "db/xsql-test", + "db/db-sqlx", + "db/xsql-test" ] [profile.release] diff --git a/apm/tracing-derive/Cargo.toml b/apm/tracing-derive/Cargo.toml deleted file mode 100644 index 68a3b77a5..000000000 --- a/apm/tracing-derive/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "tracing-derive" -version = "0.1.0" -authors = ["Nervos Network"] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -proc-macro = true - -[dependencies] -proc-macro2 = "1.0" -proc-macro-error = "1.0" -quote = "1.0" -serde_json = "1.0" -syn = { version = "1.0", features = ["full"] } diff --git a/apm/tracing-derive/src/attr_parse.rs b/apm/tracing-derive/src/attr_parse.rs deleted file mode 100644 index 2bbd56cb0..000000000 --- a/apm/tracing-derive/src/attr_parse.rs +++ /dev/null @@ -1,137 +0,0 @@ -use proc_macro2::TokenStream; -use quote::quote; -use syn::{parse_str, Expr, Lit, Meta, NestedMeta}; - -use std::collections::HashMap; - -static KIND: &str = "kind"; -static TRACING_NAME: &str = "name"; -static TRACING_TAGS: &str = "tags"; -static TRACING_LOGS: &str = "logs"; - -#[derive(Default)] -pub struct TracingAttrs { - pub kind: String, - pub tracing_name: Option, - pub tracing_tags: HashMap, - pub tracing_logs: HashMap, -} - -impl TracingAttrs { - pub fn _get_tracing_name(&self) -> Option { - self.tracing_name.clone() - } - - pub fn get_tag_map(&self) -> HashMap { - let mut res = self.tracing_tags.clone(); - res.insert("kind".to_string(), self.kind.clone()); - res - } - - pub fn get_log_map(&self) -> HashMap { - self.tracing_logs.clone() - } - - fn set_kind(&mut self, kind: String) { - self.kind = kind; - } - - fn set_tracing_name(&mut self, name: String) { - self.tracing_name = Some(name); - } - - fn set_tracing_tags(&mut self, tags: HashMap) { - self.tracing_tags = tags; - } - - fn set_tracing_logs(&mut self, logs: HashMap) { - self.tracing_logs = logs; - } -} - -pub fn parse_attrs(input: Vec) -> TracingAttrs { - let mut attrs = TracingAttrs::default(); - for attr in input.iter() { - match_attr(&mut attrs, attr); - } - - attrs -} - -pub fn span_log(key: String, val: String) -> TokenStream { - if let Ok(expr) = parse_str::(&val) { - quote! { span_logs.push(LogField::new(#key, (#expr).to_string())); } - } else { - quote! { span_logs.push(LogField::new(#key, #val)); } - } -} - -pub fn span_tag(key: String, val: String) -> TokenStream { - if key == KIND { - return quote! { span_tags.push((#key.as_str(), #val)); }; - } - - if let Ok(expr) = parse_str::(&val) { - quote! { span_tags.push(Tag::new(#key.as_str(), (#expr).to_string())); } - } else { - quote! { span_tags.push(Tag::new(#key.as_str(), #val)); } - } -} - -fn match_attr(tracing_attrs: &mut TracingAttrs, input: &NestedMeta) { - match input { - NestedMeta::Meta(data) => match data { - Meta::NameValue(name_value) => { - let ident = &name_value - .path - .segments - .first() - .expect("there must be at least 1 segment") - .ident; - - if ident == KIND { - tracing_attrs.set_kind(get_lit_str(&name_value.lit)); - } else if ident == TRACING_NAME { - tracing_attrs.set_tracing_name(get_lit_str(&name_value.lit)); - } else if ident == TRACING_TAGS { - tracing_attrs.set_tracing_tags(parse_json(&get_lit_str(&name_value.lit))); - } else if ident == TRACING_LOGS { - tracing_attrs.set_tracing_logs(parse_json(&get_lit_str(&name_value.lit))); - } else { - panic!(""); - } - } - _ => unreachable!("name_value"), - }, - _ => unreachable!("meta"), - }; -} - -fn get_lit_str(lit: &Lit) -> String { - match lit { - Lit::Str(value) => value.value(), - _ => unreachable!("lit_str"), - } -} - -fn parse_json(input: &str) -> HashMap { - serde_json::from_str::>(&transfer_string(input)) - .expect("deserialize json error") -} - -fn transfer_string(input: &str) -> String { - input.replace('\'', "\"") -} - -#[cfg(test)] -mod test { - use super::transfer_string; - - #[test] - fn test_transfer_string() { - assert_eq!( - transfer_string("{'a': 'b', 'c': 'd'}"), - "{\"a\": \"b\", \"c\": \"d\"}", - ); - } -} diff --git a/apm/tracing-derive/src/lib.rs b/apm/tracing-derive/src/lib.rs deleted file mode 100644 index 45113fd23..000000000 --- a/apm/tracing-derive/src/lib.rs +++ /dev/null @@ -1,121 +0,0 @@ -#![allow(clippy::cmp_owned, dead_code)] - -mod attr_parse; - -#[macro_use] -extern crate proc_macro_error; -extern crate proc_macro; - -use proc_macro::TokenStream; -use syn::{parse_macro_input, spanned::Spanned, token::Async, Ident, ItemFn, Signature}; - -#[proc_macro_attribute] -#[proc_macro_error] -pub fn tracing(_args: TokenStream, item: TokenStream) -> TokenStream { - let input = parse_macro_input!(item as ItemFn); - - let ItemFn { - attrs, - vis, - block, - sig, - } = input; - - let Signature { - output: return_type, - inputs: params, - unsafety, - asyncness, - constness, - abi, - ident, - generics: - syn::Generics { - params: gen_params, - where_clause, - .. - }, - .. - } = sig; - - if asyncness.is_some() { - abort!( - asyncness, - "Unexpected async\nIf want to trace async function, consider `minitrace::trace_async`" - ); - }; - - let trace_name = ident.to_string(); - - quote::quote!( - #(#attrs) * - #vis #constness #unsafety #asyncness #abi fn #ident<#gen_params>(#params) #return_type - #where_clause - { - let _guard = common_logger::LocalSpan::enter(#trace_name); - #block - } - ) - .into() -} - -#[proc_macro_attribute] -#[proc_macro_error] -pub fn tracing_async(_args: TokenStream, item: TokenStream) -> TokenStream { - let input = parse_macro_input!(item as syn::ItemFn); - - let ItemFn { - attrs, - vis, - block, - sig, - } = input; - - let Signature { - output: return_type, - inputs: params, - unsafety, - asyncness, - constness, - abi, - ident, - generics: - syn::Generics { - params: gen_params, - where_clause, - .. - }, - .. - } = sig; - - let trace_name = ident.to_string(); - - let body = if asyncness.is_some() { - let async_kwd = Async { span: block.span() }; - let await_kwd = Ident::new("await", block.span()); - quote::quote_spanned! {block.span() => - #async_kwd move { - #block - } - .in_local_span(#trace_name) - .#await_kwd - } - } else { - quote::quote_spanned! { - block.span() => std::boxed::Box::pin({ - #block.in_local_span(#trace_name) - }) - } - }; - - quote::quote!( - #(#attrs) * - #vis #constness #unsafety #asyncness #abi fn #ident<#gen_params>(#params) #return_type - #where_clause - { - use common_logger::FutureExt; - #body - } - ) - .into() -} diff --git a/apm/tracing/Cargo.toml b/apm/tracing/Cargo.toml deleted file mode 100644 index 00dba6446..000000000 --- a/apm/tracing/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "tracing" -version = "0.1.0" -authors = ["Nervos Network"] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -arc-swap = "1.5" -lazy_static = "1.4" -minitrace = { git = "https://github.com/tikv/minitrace-rust.git" } -minitrace-jaeger = { git = "https://github.com/tikv/minitrace-rust.git" } -minitrace-macro = { git = "https://github.com/tikv/minitrace-rust.git" } -tokio = { version = "1.14", features = ["rt", "sync"] } diff --git a/apm/tracing/src/lib.rs b/apm/tracing/src/lib.rs deleted file mode 100644 index 49d598b40..000000000 --- a/apm/tracing/src/lib.rs +++ /dev/null @@ -1,74 +0,0 @@ -pub use minitrace::{FutureExt, LocalSpan, Span}; -pub use minitrace_macro::{trace, trace_async}; - -use arc_swap::ArcSwap; -use minitrace::{span, Collector}; -use minitrace_jaeger::Reporter; -use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; - -use std::net::SocketAddr; -use std::sync::Arc; - -lazy_static::lazy_static! { - pub static ref TRACING_SPAN_TX: ArcSwap>> = { - let (tx, _) = unbounded_channel(); - ArcSwap::from_pointee(tx) - }; -} - -pub fn init_jaeger(jaeger_uri: String) { - let (tx, mut rx) = unbounded_channel(); - TRACING_SPAN_TX.swap(Arc::new(tx)); - let uri = jaeger_uri.parse::().expect("parse jaeger uri"); - - tokio::spawn(async move { - loop { - if let Some(spans) = rx.recv().await { - if !spans.is_empty() { - let s = spans.get(0).cloned().unwrap(); - let bytes = Reporter::encode( - s.event.to_string(), - s.id.into(), - s.parent_id.into(), - 0, - &spans, - ) - .expect("reporter encode"); - Reporter::report(uri, &bytes).ok(); - } - } - } - }); -} - -pub struct MercuryTrace { - collector: Option, - tx: Arc>>, -} - -impl Default for MercuryTrace { - fn default() -> Self { - MercuryTrace { - collector: None, - tx: Arc::clone(&(*TRACING_SPAN_TX.load())), - } - } -} - -impl MercuryTrace { - pub fn new(collector: Collector) -> Self { - MercuryTrace { - collector: Some(collector), - tx: Arc::clone(&(*TRACING_SPAN_TX.load())), - } - } -} - -impl Drop for MercuryTrace { - fn drop(&mut self) { - if let Some(collector) = self.collector.take() { - let spans = collector.collect(); - let _ = self.tx.send(spans); - } - } -} diff --git a/common/src/address.rs b/common/src/address.rs index 469775771..1ee1e7b94 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -1,6 +1,7 @@ use crate::lazy::{ACP_CODE_HASH, PW_LOCK_CODE_HASH, SECP256K1_CODE_HASH}; use crate::{NetworkType, MULTISIG_TYPE_HASH}; +use anyhow::Result; use bech32::{convert_bits, ToBase32, Variant}; use ckb_hash::blake2b_256; use ckb_types::{bytes::Bytes, core::ScriptHashType, packed, prelude::*, H160}; diff --git a/common/src/lib.rs b/common/src/lib.rs index 83ef22f53..1ae41081c 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -184,7 +184,7 @@ impl Range { pub struct PaginationRequest { pub cursor: Option, pub order: Order, - pub limit: Option, + pub limit: Option, pub skip: Option, pub return_count: bool, } @@ -193,7 +193,7 @@ impl PaginationRequest { pub fn new( cursor: Option, order: Order, - limit: Option, + limit: Option, skip: Option, return_count: bool, ) -> PaginationRequest { @@ -215,12 +215,12 @@ impl PaginationRequest { self.order = order; } - pub fn limit(mut self, limit: Option) -> Self { + pub fn limit(mut self, limit: Option) -> Self { self.set_limit(limit); self } - pub fn set_limit(&mut self, limit: Option) { + pub fn set_limit(&mut self, limit: Option) { self.limit = limit; } diff --git a/core/ckb-client/Cargo.toml b/core/ckb-client/Cargo.toml index 3421b34ee..e1817d629 100644 --- a/core/ckb-client/Cargo.toml +++ b/core/ckb-client/Cargo.toml @@ -13,9 +13,8 @@ log = "0.4" reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.13", features = ["macros", "rt-multi-thread", "sync"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] } common = { path = "../../common" } -common-logger = { path = "../../logger" } core-storage = { path = "../storage" } core-synchronization = { path = "../synchronization"} diff --git a/core/ckb-client/src/lib.rs b/core/ckb-client/src/lib.rs index b7bb20a48..e42469382 100644 --- a/core/ckb-client/src/lib.rs +++ b/core/ckb-client/src/lib.rs @@ -4,7 +4,6 @@ pub mod error; pub use client::CkbRpcClient; use common::{async_trait, Result}; -use core_synchronization::SyncAdapter; use ckb_jsonrpc_types::{ BlockView, EpochView, LocalNode, RawTxPool, TransactionWithStatus, Uint64, @@ -35,27 +34,3 @@ pub trait CkbRpc: Sync + Send + 'static { async fn get_block(&self, block_hash: H256, use_hex_format: bool) -> Result>; } - -#[async_trait] -impl SyncAdapter for dyn CkbRpc { - async fn pull_blocks( - &self, - block_numbers: Vec, - ) -> Result> { - let mut ret = Vec::new(); - for (idx, block) in self - .get_blocks_by_number(block_numbers.clone()) - .await? - .iter() - .enumerate() - { - if let Some(b) = block { - ret.push(core::BlockView::from(b.to_owned())); - } else { - log::error!("[sync] Get none block {:?} from node", block_numbers[idx]); - } - } - - Ok(ret) - } -} diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index aefb3e813..c1d23ce4b 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml @@ -15,7 +15,7 @@ log = "0.4" log4rs = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.14", features = ["time"] } +tokio = { version = "1", features = ["time"] } toml = "0.5" common = { path = "../../common" } diff --git a/core/cli/src/config.rs b/core/cli/src/config.rs index 9f3ef5bdf..12b803df7 100644 --- a/core/cli/src/config.rs +++ b/core/cli/src/config.rs @@ -38,7 +38,6 @@ pub struct DBConfig { pub db_name: String, pub db_user: String, pub password: String, - pub db_log_level: String, } #[derive(Deserialize, Default, Clone, Debug)] @@ -118,7 +117,7 @@ pub struct MercuryConfig { pub extensions_config: Vec, #[serde(default = "default_pool_cache_size")] - pub pool_cache_size: u64, + pub pool_cache_size: u16, #[serde(default = "default_is_pprof_enabled")] pub is_pprof_enabled: bool, @@ -215,8 +214,8 @@ fn default_extensions_config() -> Vec { vec![] } -fn default_pool_cache_size() -> u64 { - 100u64 +fn default_pool_cache_size() -> u16 { + 100u16 } fn default_is_pprof_enabled() -> bool { diff --git a/core/cli/src/lib.rs b/core/cli/src/lib.rs index 8739c8aea..cec4ba81f 100644 --- a/core/cli/src/lib.rs +++ b/core/cli/src/lib.rs @@ -2,12 +2,11 @@ pub mod config; use crate::config::{parse, MercuryConfig}; -use common_logger::init_jaeger; use core_service::Service; use ansi_term::Colour::Green; use clap::{crate_version, App, Arg, ArgMatches, SubCommand}; -use log::{info, LevelFilter}; +use log::info; use std::path::PathBuf; use std::str::FromStr; @@ -104,16 +103,6 @@ impl<'a> Cli<'a> { self.print_logo(); self.log_init(); - if self.config.log_config.use_apm { - init_jaeger( - self.config - .log_config - .jaeger_uri - .clone() - .expect("init jaeger"), - ); - } - let mut service = Service::new( self.config.db_config.center_id, self.config.db_config.machine_id, @@ -130,7 +119,6 @@ impl<'a> Cli<'a> { self.config.cellbase_maturity, self.parse_cmd_args("ckb_uri", self.config.network_config.ckb_uri.clone()), self.config.cheque_since, - LevelFilter::from_str(&self.config.db_config.db_log_level).expect("parse log level"), self.config.pool_cache_size, self.config.is_pprof_enabled, ); diff --git a/core/rpc/README.md b/core/rpc/README.md index 3102ae4d3..dc4711afa 100644 --- a/core/rpc/README.md +++ b/core/rpc/README.md @@ -51,6 +51,7 @@ - [Type `DBInfo`](#type-dbinfo) - [Type `SyncState`](#type-syncstate) - [Type `SyncProgress`](#type-syncprogress) + - [Type `Uint16`](#type-uint16) - [Type `Uint32`](#type-uint32) - [Type `Uint64`](#type-uint64) - [Type `Uint128`](#type-uint128) @@ -2205,7 +2206,7 @@ Fields - Start from the biggest cursor for descending order - Start from the smallest cursor for ascending order - `order` (Type: `"Asc"` | `"Desc"`): Specify the order of the returning data. -- `limit` (Type: `Uint64` | `null` ): Specify the entry limit per page of the query. +- `limit` (Type: `Uint16` | `null` ): Specify the entry limit per page of the query. - `return_count` (Type: `bool`): Specify whether to return the total count. ### Type `BlockInfo` @@ -2379,17 +2380,60 @@ Fields - `target`(Type: `string`): target number at the current stage. - `progress`(Type: `string`): Percentage of progress calculated based on current and target. +### Type `Uint16` + +The 16-bit unsigned integer type encoded as the 0x-prefixed hex string in JSON. + +##### Examples + + +| JSON | Decimal Value | +| --- |--- | +| “0x0” | 0 | +| “0x10” | 16 | +| “10” | Invalid, 0x is required | +| “0x01” | Invalid, redundant leading 0 | + ### Type `Uint32` -The [32-bit unsigned integer type](https://github.com/nervosnetwork/ckb/blob/develop/rpc/README.md#type-uint32) encoded as the 0x-prefixed hex string in JSON. +The 32-bit unsigned integer type encoded as the 0x-prefixed hex string in JSON. + +##### Examples + +| JSON | Decimal Value | +| --- |--- | +| “0x0” | 0 | +| “0x10” | 16 | +| “10” | Invalid, 0x is required | +| “0x01” | Invalid, redundant leading 0 | ### Type `Uint64` -The [64-bit unsigned integer type](https://github.com/nervosnetwork/ckb/blob/develop/rpc/README.md#type-uint64) encoded as the 0x-prefixed hex string in JSON. +The 64-bit unsigned integer type encoded as the 0x-prefixed hex string in JSON. + +##### Examples + + +| JSON | Decimal Value | +| --- |--- | +| “0x0” | 0 | +| “0x10” | 16 | +| “10” | Invalid, 0x is required | +| “0x01” | Invalid, redundant leading 0 | ### Type `Uint128` -The [128-bit unsigned integer type](https://github.com/nervosnetwork/ckb/blob/develop/rpc/README.md#type-uint128) encoded as the 0x-prefixed hex string in JSON. +The 128-bit unsigned integer type encoded as the 0x-prefixed hex string in JSON. + +##### Examples + + +| JSON | Decimal Value | +| --- |--- | +| “0x0” | 0 | +| “0x10” | 16 | +| “10” | Invalid, 0x is required | +| “0x01” | Invalid, redundant leading 0 | ### Type `BlockNumber` diff --git a/core/rpc/core/Cargo.toml b/core/rpc/core/Cargo.toml index 70abb3e8e..eeba16ebe 100644 --- a/core/rpc/core/Cargo.toml +++ b/core/rpc/core/Cargo.toml @@ -8,7 +8,6 @@ edition = "2021" [dependencies] arc-swap = "1.4" clap = "2.34" -# ckb-indexer = { git = "https://github.com/KaoImin/ckb-indexer", branch = "mercury" } ckb-jsonrpc-types = "0.101" ckb-types = "0.101" ckb-dao-utils = "0.101" @@ -27,10 +26,9 @@ pprof = { version = "0.6", features = ["flamegraph", "cpp"]} reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.13", features = ["macros", "rt-multi-thread", "sync"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] } common = { path = "../../../common" } -common-logger = { path = "../../../logger" } core-ckb-client = { path = "../../ckb-client" } core-rpc-types = { path = "../types" } core-storage = { path = "../../storage" } @@ -38,7 +36,9 @@ core-storage = { path = "../../storage" } [dev-dependencies] env_logger = "0.9" rand = "0.7" -rbatis = { version = "3.0", default-features = false, features = ["all-database", "runtime-tokio-native-tls", "upper_case_sql_keyword"] } +sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "sqlite"] } core-cli = { path = "../../cli" } -xsql = { path = "../../../db/xsql" } +db-sqlx = { path = "../../../db/db-sqlx" , package = "db-sqlx"} +xsql-test = { path = "../../../db/xsql-test" } + diff --git a/core/rpc/core/src/impl.rs b/core/rpc/core/src/impl.rs index e4dd87045..e89f221c3 100644 --- a/core/rpc/core/src/impl.rs +++ b/core/rpc/core/src/impl.rs @@ -8,7 +8,6 @@ pub(crate) mod utils_types; use crate::r#impl::build_tx::calculate_tx_size; use crate::{error::CoreError, MercuryRpcServer}; -use ckb_jsonrpc_types::Uint64; use ckb_types::core::RationalU256; use ckb_types::{packed, prelude::*, H160, H256}; use clap::crate_version; @@ -24,12 +23,13 @@ use common::{ use core_ckb_client::CkbRpc; use core_rpc_types::error::MercuryRpcError; use core_rpc_types::{ - indexer, AdjustAccountPayload, BlockInfo, DaoClaimPayload, DaoDepositPayload, - DaoWithdrawPayload, GetAccountInfoPayload, GetAccountInfoResponse, GetBalancePayload, - GetBalanceResponse, GetBlockInfoPayload, GetSpentTransactionPayload, - GetTransactionInfoResponse, MercuryInfo, PaginationResponse, QueryTransactionsPayload, - SimpleTransferPayload, SudtIssuePayload, SyncState, TransactionCompletionResponse, - TransferPayload, TxView, + indexer, + uints::{Uint16, Uint64}, + AdjustAccountPayload, BlockInfo, DaoClaimPayload, DaoDepositPayload, DaoWithdrawPayload, + GetAccountInfoPayload, GetAccountInfoResponse, GetBalancePayload, GetBalanceResponse, + GetBlockInfoPayload, GetSpentTransactionPayload, GetTransactionInfoResponse, MercuryInfo, + PaginationResponse, QueryTransactionsPayload, SimpleTransferPayload, SudtIssuePayload, + SyncState, TransactionCompletionResponse, TransferPayload, TxView, }; use core_storage::{DBInfo, RelationalStorage}; use jsonrpsee_core::{Error, RpcResult}; @@ -46,9 +46,6 @@ lazy_static::lazy_static! { macro_rules! rpc_impl { ($self_: ident, $func: ident, $payload: expr) => {{ - let (_, collector) = common_logger::Span::root("trace_name"); - let _collector = common_logger::MercuryTrace::new(collector); - $self_ .$func(Context::new(), $payload) .await @@ -64,7 +61,7 @@ pub struct MercuryRpcImpl { cheque_timeout: RationalU256, cellbase_maturity: RationalU256, sync_state: Arc>, - pool_cache_size: u64, + pool_cache_size: u16, is_pprof_enabled: bool, } @@ -186,7 +183,7 @@ impl MercuryRpcServer for MercuryRpcImpl { &self, search_key: indexer::SearchKey, order: Order, - limit: Uint64, + limit: Uint16, after_cursor: Option, ) -> RpcResult> { self.inner_get_cells( @@ -213,7 +210,7 @@ impl MercuryRpcServer for MercuryRpcImpl { &self, search_key: indexer::SearchKey, order: Order, - limit: Uint64, + limit: Uint16, after_cursor: Option, ) -> RpcResult> { self.inner_get_transaction( @@ -334,7 +331,7 @@ impl MercuryRpcImpl { cheque_timeout: RationalU256, cellbase_maturity: RationalU256, sync_state: Arc>, - pool_cache_size: u64, + pool_cache_size: u16, is_pprof_enabled: bool, ) -> Self { load_code_hash(&builtin_scripts); diff --git a/core/rpc/core/src/impl/adjust_account.rs b/core/rpc/core/src/impl/adjust_account.rs index 74885c968..24e5c387f 100644 --- a/core/rpc/core/src/impl/adjust_account.rs +++ b/core/rpc/core/src/impl/adjust_account.rs @@ -9,7 +9,6 @@ use common::hash::blake2b_256_to_160; use common::lazy::{ACP_CODE_HASH, PW_LOCK_CODE_HASH, SECP256K1_CODE_HASH}; use common::utils::decode_udt_amount; use common::{Context, DetailedCell, PaginationRequest, ACP, PW_LOCK, SECP256K1, SUDT}; -use common_logger::tracing_async; use core_ckb_client::CkbRpc; use core_rpc_types::consts::{ckb, DEFAULT_FEE_RATE, STANDARD_SUDT_CAPACITY}; use core_rpc_types::{ @@ -21,7 +20,6 @@ use std::collections::{BTreeSet, HashSet}; use std::convert::TryInto; impl MercuryRpcImpl { - #[tracing_async] pub(crate) async fn inner_build_adjust_account_transaction( &self, ctx: Context, @@ -96,7 +94,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] async fn build_create_acp_transaction_fixed_fee( &self, ctx: Context, diff --git a/core/rpc/core/src/impl/build_tx.rs b/core/rpc/core/src/impl/build_tx.rs index 72d9a2fba..945be680a 100644 --- a/core/rpc/core/src/impl/build_tx.rs +++ b/core/rpc/core/src/impl/build_tx.rs @@ -20,7 +20,6 @@ use common::utils::{decode_udt_amount, encode_udt_amount}; use common::{ Address, Context, DetailedCell, PaginationRequest, ACP, CHEQUE, DAO, PW_LOCK, SECP256K1, SUDT, }; -use common_logger::tracing_async; use core_ckb_client::CkbRpc; use core_rpc_types::consts::{ BYTE_SHANNONS, DEFAULT_FEE_RATE, INIT_ESTIMATE_FEE, MAX_ITEM_NUM, MIN_DAO_CAPACITY, @@ -47,7 +46,6 @@ pub struct CellWithData { } impl MercuryRpcImpl { - #[tracing_async] pub(crate) async fn inner_build_dao_deposit_transaction( &self, ctx: Context, @@ -72,7 +70,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_dao_deposit_transaction( &self, ctx: Context, @@ -118,7 +115,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] pub(crate) async fn inner_build_dao_withdraw_transaction( &self, ctx: Context, @@ -140,7 +136,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_dao_withdraw_transaction( &self, ctx: Context, @@ -261,7 +256,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] pub(crate) async fn inner_build_dao_claim_transaction( &self, ctx: Context, @@ -283,7 +277,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_dao_claim_transaction( &self, ctx: Context, @@ -469,7 +462,6 @@ impl MercuryRpcImpl { .map(|(tx_view, script_groups)| (tx_view, script_groups, change_cell_index)) } - #[tracing_async] pub(crate) async fn inner_build_transfer_transaction( &self, ctx: Context, @@ -507,7 +499,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_transfer_transaction( &self, ctx: Context, @@ -541,7 +532,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] async fn prebuild_ckb_transfer_transaction_from_provide_capacity( &self, ctx: Context, @@ -587,7 +577,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_ckb_transfer_transaction_to_provide_capacity( &self, ctx: Context, @@ -674,7 +663,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_udt_transfer_transaction_from_provide_capacity( &self, ctx: Context, @@ -739,7 +727,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_udt_transfer_transaction_to_provide_capacity( &self, ctx: Context, @@ -827,7 +814,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] pub(crate) async fn inner_build_simple_transfer_transaction( &self, ctx: Context, @@ -983,7 +969,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] pub(crate) async fn build_transaction_with_adjusted_fee<'a, F, Fut, T>( &'a self, prebuild: F, @@ -1025,7 +1010,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] async fn get_simple_transfer_output_capacity_provider( &self, ctx: Context, @@ -1062,7 +1046,6 @@ impl MercuryRpcImpl { Ok(OutputCapacityProvider::To) } - #[tracing_async] pub(crate) async fn prebuild_capacity_balance_tx( &self, ctx: Context, @@ -1105,7 +1088,6 @@ impl MercuryRpcImpl { .map(|(tx_view, script_groups)| (tx_view, script_groups, fee_change_cell_index)) } - #[tracing_async] pub(crate) async fn build_sudt_type_script( &self, ctx: Context, @@ -1229,7 +1211,6 @@ impl MercuryRpcImpl { Ok(inputs) } - #[tracing_async] pub(crate) async fn inner_build_sudt_issue_transaction( &self, ctx: Context, @@ -1268,7 +1249,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_sudt_issue_transaction( &self, ctx: Context, @@ -1287,7 +1267,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] async fn prebuild_sudt_issue_transaction_from_provide_capacity( &self, ctx: Context, @@ -1341,7 +1320,6 @@ impl MercuryRpcImpl { .await } - #[tracing_async] async fn prebuild_sudt_issue_transaction_to_provide_capacity( &self, ctx: Context, diff --git a/core/rpc/core/src/impl/query.rs b/core/rpc/core/src/impl/query.rs index 9460e5bab..fd7b7b7e1 100644 --- a/core/rpc/core/src/impl/query.rs +++ b/core/rpc/core/src/impl/query.rs @@ -2,7 +2,6 @@ use crate::r#impl::utils; use crate::{error::CoreError, InnerResult, MercuryRpcImpl}; use common::{Context, DetailedCell, Order, PaginationRequest, Range}; -use common_logger::tracing_async; use core_ckb_client::CkbRpc; use core_rpc_types::lazy::CURRENT_BLOCK_NUMBER; use core_rpc_types::{ @@ -30,7 +29,6 @@ impl MercuryRpcImpl { .map_err(|error| CoreError::DBError(error.to_string()).into()) } - #[tracing_async] pub(crate) async fn inner_get_balance( &self, ctx: Context, @@ -110,7 +108,6 @@ impl MercuryRpcImpl { }) } - #[tracing_async] pub(crate) async fn inner_get_block_info( &self, ctx: Context, @@ -147,7 +144,6 @@ impl MercuryRpcImpl { }) } - #[tracing_async] pub(crate) async fn inner_query_transactions( &self, ctx: Context, @@ -191,7 +187,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] pub(crate) async fn inner_get_tip(&self, ctx: Context) -> InnerResult> { let block = self .storage @@ -208,13 +203,12 @@ impl MercuryRpcImpl { } } - #[tracing_async] pub(crate) async fn inner_get_cells( &self, ctx: Context, search_key: indexer::SearchKey, order: Order, - limit: u64, + limit: u16, after_cursor: Option, ) -> InnerResult> { let pagination = PaginationRequest::new(after_cursor, order, Some(limit), None, false); @@ -233,7 +227,6 @@ impl MercuryRpcImpl { }) } - #[tracing_async] pub(crate) async fn inner_get_spent_transaction( &self, ctx: Context, @@ -267,7 +260,6 @@ impl MercuryRpcImpl { } } - #[tracing_async] pub(crate) async fn inner_get_cells_capacity( &self, ctx: Context, @@ -298,17 +290,15 @@ impl MercuryRpcImpl { }) } - #[tracing_async] pub(crate) async fn inner_get_transaction( &self, ctx: Context, search_key: indexer::SearchKey, order: Order, - limit: u64, + limit: u16, after_cursor: Option, ) -> InnerResult> { let pagination = PaginationRequest::new(after_cursor, order, Some(limit), None, false); - let script = search_key.script; let (the_other_script, block_range) = if let Some(filter) = search_key.filter { (filter.script, filter.block_range) @@ -335,29 +325,12 @@ impl MercuryRpcImpl { .await .map_err(|error| CoreError::DBError(error.to_string()))?; - let mut objects = Vec::new(); - for cell in db_response.response.iter() { - let object = indexer::Transaction { - tx_hash: H256::from_slice(&cell.tx_hash.inner[0..32]).expect("get tx hash h256"), - block_number: cell.block_number.into(), - tx_index: cell.tx_index.into(), - io_index: cell.io_index.into(), - io_type: if cell.io_type == 0 { - IOType::Input - } else { - IOType::Output - }, - }; - objects.push(object); - } - Ok(indexer::PaginationResponse { - objects, + objects: db_response.response, last_cursor: db_response.next_cursor.map(Into::into), }) } - #[tracing_async] pub(crate) async fn inner_get_live_cells_by_lock_hash( &self, ctx: Context, @@ -378,7 +351,7 @@ impl MercuryRpcImpl { .into()); } let skip = page * per_page; - let limit = per_page; + let limit = per_page as u16; PaginationRequest::new(None, order, Some(limit), Some(skip), false) }; let cells = self @@ -416,7 +389,6 @@ impl MercuryRpcImpl { Ok(res) } - #[tracing_async] pub(crate) async fn inner_get_capacity_by_lock_hash( &self, ctx: Context, @@ -454,7 +426,6 @@ impl MercuryRpcImpl { } #[allow(clippy::unnecessary_unwrap)] - #[tracing_async] pub(crate) async fn inner_get_transactions_by_lock_hash( &self, ctx: Context, @@ -475,7 +446,7 @@ impl MercuryRpcImpl { .into()); } let skip = page * per_page; - let limit = per_page; + let limit = per_page as u16; PaginationRequest::new(None, order, Some(limit), Some(skip), false) }; let db_response = self @@ -536,7 +507,6 @@ impl MercuryRpcImpl { Ok(cell_txs) } - #[tracing_async] pub(crate) async fn inner_get_transaction_with_status( &self, ctx: Context, @@ -563,7 +533,6 @@ impl MercuryRpcImpl { Ok(tx_wrapper) } - #[tracing_async] pub(crate) async fn inner_get_transaction_info( &self, ctx: Context, @@ -580,7 +549,6 @@ impl MercuryRpcImpl { }) } - #[tracing_async] async fn query_transaction_info( &self, ctx: Context, @@ -670,7 +638,6 @@ impl MercuryRpcImpl { }) } - #[tracing_async] async fn get_live_cells_by_search_key( &self, ctx: Context, diff --git a/core/rpc/core/src/impl/utils.rs b/core/rpc/core/src/impl/utils.rs index d9b8a2fda..63871b401 100644 --- a/core/rpc/core/src/impl/utils.rs +++ b/core/rpc/core/src/impl/utils.rs @@ -15,7 +15,6 @@ use common::{ Address, AddressPayload, Context, DetailedCell, PaginationRequest, PaginationResponse, Range, ACP, CHEQUE, DAO, PW_LOCK, SECP256K1, SUDT, }; -use common_logger::tracing_async; use core_ckb_client::CkbRpc; use core_rpc_types::consts::{ MIN_CKB_CAPACITY, MIN_DAO_LOCK_PERIOD, STANDARD_SUDT_CAPACITY, @@ -50,7 +49,6 @@ impl MercuryRpcImpl { } #[allow(clippy::unnecessary_unwrap)] - #[tracing_async] pub(crate) async fn get_scripts_by_identity( &self, ctx: Context, @@ -153,7 +151,6 @@ impl MercuryRpcImpl { ret } - #[tracing_async] pub(crate) async fn get_live_cells_by_item( &self, ctx: Context, @@ -219,7 +216,6 @@ impl MercuryRpcImpl { Ok(cells) } - #[tracing_async] async fn get_live_cells( &self, ctx: Context, @@ -261,7 +257,6 @@ impl MercuryRpcImpl { Ok(cells) } - #[tracing_async] pub(crate) async fn get_transactions_by_item( &self, ctx: Context, @@ -515,7 +510,6 @@ impl MercuryRpcImpl { } #[allow(clippy::unnecessary_unwrap)] - #[tracing_async] pub(crate) async fn to_record( &self, ctx: Context, @@ -612,7 +606,6 @@ impl MercuryRpcImpl { Ok(records) } - #[tracing_async] pub(crate) async fn get_cheque_sender_address( &self, ctx: Context, @@ -668,7 +661,6 @@ impl MercuryRpcImpl { decode_udt_amount(&cell.cell_data).unwrap_or(0) } - #[tracing_async] async fn generate_extra( &self, ctx: Context, @@ -769,7 +761,6 @@ impl MercuryRpcImpl { } /// Calculate maximum withdraw capacity of a deposited dao output - #[tracing_async] pub async fn calculate_maximum_withdraw( &self, ctx: Context, @@ -813,7 +804,6 @@ impl MercuryRpcImpl { Ok(withdraw_capacity) } - #[tracing_async] /// We do not use the accurate `occupied` definition in ckb, which indicates the capacity consumed for storage of the live cells. /// Because by this definition, `occupied` and `free` are both not good indicators for spendable balance. /// @@ -899,7 +889,6 @@ impl MercuryRpcImpl { Ok(()) } - #[tracing_async] pub(crate) async fn get_epoch_by_number( &self, ctx: Context, @@ -1042,7 +1031,6 @@ impl MercuryRpcImpl { ) > self.cellbase_maturity } - #[tracing_async] pub(crate) async fn balance_transfer_tx_capacity( &self, ctx: Context, @@ -1169,7 +1157,6 @@ impl MercuryRpcImpl { Ok(()) } - #[tracing_async] pub(crate) async fn balance_transfer_tx_capacity_fee_by_output( &self, to_items: Vec, @@ -1385,7 +1372,6 @@ impl MercuryRpcImpl { Ok(required_capacity) } - #[tracing_async] pub(crate) async fn balance_transfer_tx_udt( &self, ctx: Context, diff --git a/core/rpc/core/src/lib.rs b/core/rpc/core/src/lib.rs index b09e442f6..aedde779b 100644 --- a/core/rpc/core/src/lib.rs +++ b/core/rpc/core/src/lib.rs @@ -7,17 +7,17 @@ mod tests; pub use r#impl::MercuryRpcImpl; -use ckb_jsonrpc_types::Uint64; use ckb_types::{H160, H256}; use common::{Order, Result}; use core_rpc_types::error::MercuryRpcError; use core_rpc_types::{ - indexer, AdjustAccountPayload, BlockInfo, DaoClaimPayload, DaoDepositPayload, - DaoWithdrawPayload, GetAccountInfoPayload, GetAccountInfoResponse, GetBalancePayload, - GetBalanceResponse, GetBlockInfoPayload, GetSpentTransactionPayload, - GetTransactionInfoResponse, MercuryInfo, PaginationResponse, QueryTransactionsPayload, - SimpleTransferPayload, SudtIssuePayload, SyncState, TransactionCompletionResponse, - TransferPayload, TxView, + indexer, + uints::{Uint16, Uint64}, + AdjustAccountPayload, BlockInfo, DaoClaimPayload, DaoDepositPayload, DaoWithdrawPayload, + GetAccountInfoPayload, GetAccountInfoResponse, GetBalancePayload, GetBalanceResponse, + GetBlockInfoPayload, GetSpentTransactionPayload, GetTransactionInfoResponse, MercuryInfo, + PaginationResponse, QueryTransactionsPayload, SimpleTransferPayload, SudtIssuePayload, + SyncState, TransactionCompletionResponse, TransferPayload, TxView, }; use core_storage::DBInfo; use jsonrpsee_core::RpcResult; @@ -111,7 +111,7 @@ pub trait MercuryRpc { &self, search_key: indexer::SearchKey, order: Order, - limit: Uint64, + limit: Uint16, after_cursor: Option, ) -> RpcResult>; @@ -126,7 +126,7 @@ pub trait MercuryRpc { &self, search_key: indexer::SearchKey, order: Order, - limit: Uint64, + limit: Uint16, after_cursor: Option, ) -> RpcResult>; diff --git a/core/rpc/core/src/tests/mod.rs b/core/rpc/core/src/tests/mod.rs index 90451861c..798b7a323 100644 --- a/core/rpc/core/src/tests/mod.rs +++ b/core/rpc/core/src/tests/mod.rs @@ -3,7 +3,6 @@ mod operation_test; mod query_test; mod rpc_test; -mod sqlite; mod utils_test; use crate::{ @@ -91,14 +90,14 @@ pub struct RpcTestEngine { impl RpcTestEngine { pub async fn new() -> Self { - let store = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30, log::LevelFilter::Info); + let mut store = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30); store .connect(DBDriver::SQLite, MEMORY_DB, "", 0, "", "") .await .unwrap(); - let mut tx = store.pool.transaction().await.unwrap(); - sqlite::create_tables(&mut tx).await.unwrap(); + let tx = store.sqlx_pool.transaction().await.unwrap(); + xsql_test::create_tables(tx).await.unwrap(); let config: MercuryConfig = parse(TESTNET_CONFIG).unwrap(); let script_map = config.to_script_map(); @@ -124,7 +123,7 @@ impl RpcTestEngine { } pub async fn new_pg(net_ty: NetworkType, url: &str) -> Self { - let store = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30, log::LevelFilter::Info); + let mut store = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30); store .connect( DBDriver::PostgreSQL, @@ -293,10 +292,7 @@ impl RpcTestEngine { } pub async fn append(&mut self, block: BlockView) { - self.store - .append_block(Context::new(), block) - .await - .unwrap(); + self.store.append_block(block).await.unwrap(); } pub fn rpc(&self, net_ty: NetworkType) -> MercuryRpcImpl { @@ -308,7 +304,7 @@ impl RpcTestEngine { RationalU256::from_u256(6u64.into()), RationalU256::from_u256(6u64.into()), Arc::new(RwLock::new(SyncState::ReadOnly)), - 100u64, + 100u16, true, ) } diff --git a/core/rpc/core/src/tests/rpc_test.rs b/core/rpc/core/src/tests/rpc_test.rs index ef578b818..397e701a5 100644 --- a/core/rpc/core/src/tests/rpc_test.rs +++ b/core/rpc/core/src/tests/rpc_test.rs @@ -13,8 +13,8 @@ use core_rpc_types::{ }; use tokio::test; -async fn new_rpc(network: NetworkType) -> MercuryRpcImpl { - let engine = RpcTestEngine::new_pg(network, "127.0.0.1").await; +async fn new_rpc(network: NetworkType, url: &str) -> MercuryRpcImpl { + let engine = RpcTestEngine::new_pg(network, url).await; let rpc = engine.rpc(network); let tip = rpc.inner_get_tip(Context::new()).await.unwrap().unwrap(); @@ -56,7 +56,7 @@ fn print_cells(rpc: &MercuryRpcImpl, cells: Vec) { #[ignore] #[test] async fn test_get_live_cells_by_item() { - let rpc = new_rpc(NetworkType::Dev).await; + let rpc = new_rpc(NetworkType::Dev, "127.0.0.1").await; let out_point = new_outpoint( "0496b6d22aa0ac90592a79390d3c2d796a014879ae340682ff3774ad541f4228", diff --git a/core/rpc/core/src/tests/sqlite/mod.rs b/core/rpc/core/src/tests/sqlite/mod.rs deleted file mode 100644 index 6d2430d81..000000000 --- a/core/rpc/core/src/tests/sqlite/mod.rs +++ /dev/null @@ -1,49 +0,0 @@ -pub mod sql; - -use common::{Context, Result}; -use sql::*; - -use ckb_jsonrpc_types::BlockView as JsonBlockView; -use core_storage::{RelationalStorage, Storage}; -use xsql::rbatis::executor::RBatisTxExecutor; - -pub async fn delete_all_data(tx: &mut RBatisTxExecutor<'_>) -> Result<()> { - delete_block_table_data(tx).await?; - delete_transaction_table_data(tx).await?; - delete_cell_table_data(tx).await?; - delete_live_cell_table_data(tx).await?; - delete_script_table_data(tx).await?; - delete_uncle_relationship_table_data(tx).await?; - delete_canonical_chain_table_data(tx).await?; - delete_registered_address_table_data(tx).await?; - tx.commit().await?; - Ok(()) -} - -pub async fn create_tables(tx: &mut RBatisTxExecutor<'_>) -> Result<()> { - create_block_table(tx).await?; - create_transaction_table(tx).await?; - create_cell_table(tx).await?; - create_live_cell_table(tx).await?; - create_script_table(tx).await?; - create_uncle_relationship_table(tx).await?; - create_canonical_chain_table(tx).await?; - create_registered_address_table(tx).await?; - tx.commit().await?; - Ok(()) -} - -pub async fn insert_blocks(pool: RelationalStorage, block_dir: &str) { - let data_path = String::from(block_dir); - for i in 0..10 { - pool.append_block(Context::new(), read_block_view(i, data_path.clone()).into()) - .await - .unwrap(); - } -} - -pub fn read_block_view(number: u64, dir_path: String) -> JsonBlockView { - let file_name = number.to_string() + ".json"; - let path = dir_path + file_name.as_str(); - serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() -} diff --git a/core/rpc/core/src/tests/sqlite/sql.rs b/core/rpc/core/src/tests/sqlite/sql.rs deleted file mode 100644 index 389081ac3..000000000 --- a/core/rpc/core/src/tests/sqlite/sql.rs +++ /dev/null @@ -1,168 +0,0 @@ -use xsql::rbatis::executor::RBatisTxExecutor; -use xsql::rbatis::sql; - -#[sql(tx, "DELETE FROM mercury_block")] -pub async fn delete_block_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_transaction")] -pub async fn delete_transaction_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_cell")] -pub async fn delete_cell_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_live_cell")] -pub async fn delete_live_cell_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_script")] -pub async fn delete_script_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_uncle_relationship")] -pub async fn delete_uncle_relationship_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_canonical_chain")] -pub async fn delete_canonical_chain_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DELETE FROM mercury_registered_address")] -pub async fn delete_registered_address_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_block( - block_hash blob PRIMARY KEY, - block_number int NOT NULL, - version smallint NOT NULL, - compact_target int NOT NULL, - block_timestamp bigint NOT NULL, - epoch_number int NOT NULL, - epoch_index smallint NOT NULL, - epoch_length smallint NOT NULL, - parent_hash blob NOT NULL, - transactions_root blob NOT NULL, - proposals_hash blob NOT NULL, - uncles_hash blob, - dao blob NOT NULL, - nonce blob NOT NULL, - proposals blob -)" -)] -pub async fn create_block_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_transaction( - id bigint PRIMARY KEY, - tx_hash blob NOT NULL, - tx_index smallint NOT NULL, - input_count smallint NOT NULL, - output_count smallint NOT NULL, - block_number int NOT NULL, - block_hash blob NOT NULL, - tx_timestamp bigint NOT NULL, - version smallint NOT NULL, - cell_deps blob, - header_deps blob, - witnesses blob -)" -)] -pub async fn create_transaction_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_cell( - id bigint PRIMARY KEY, - tx_hash blob NOT NULL, - output_index smallint NOT NULL, - tx_index smallint NOT NULL, - block_hash blob NOT NULL, - block_number bigint NOT NULL, - epoch_number bigint NOT NULL, - epoch_index bigint NOT NULL, - epoch_length bigint NOT NULL, - capacity bigint NOT NULL, - lock_hash blob, - lock_code_hash blob, - lock_args blob, - lock_script_type smallint, - type_hash blob, - type_code_hash blob, - type_args blob, - type_script_type smallint, - data blob, - consumed_block_number int, - consumed_block_hash blob, - consumed_tx_hash blob, - consumed_tx_index smallint, - input_index smallint, - since blob -)" -)] -pub async fn create_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_live_cell( - id bigint PRIMARY KEY, - output_index smallint NOT NULL, - tx_hash blob NOT NULL, - tx_index smallint NOT NULL, - block_hash blob NOT NULL, - block_number bigint NOT NULL, - epoch_number bigint NOT NULL, - epoch_index bigint NOT NULL, - epoch_length bigint NOT NULL, - capacity bigint NOT NULL, - lock_hash blob, - lock_code_hash blob, - lock_script_hash blob, - lock_args blob, - lock_script_type smallint, - type_hash blob, - type_code_hash blob, - type_args blob, - type_script_type smallint, - data blob -)" -)] -pub async fn create_live_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_script( - id bigint PRIMARY KEY, - script_hash blob NOT NULL, - script_hash_160 blob NOT NULL, - script_code_hash blob NOT NULL, - script_args blob, - script_type smallint NOT NULL, - script_args_len smallint -)" -)] -pub async fn create_script_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_uncle_relationship( - block_hash blob, - uncle_hashes blob, - PRIMARY KEY(block_hash, uncle_hashes) -)" -)] -pub async fn create_uncle_relationship_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_canonical_chain( - block_number bigint PRIMARY KEY, - block_hash blob NOT NULL -)" -)] -pub async fn create_canonical_chain_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql( - tx, - "CREATE TABLE mercury_registered_address( - lock_hash blob NOT NULL PRIMARY KEY, - address varchar NOT NULL -)" -)] -pub async fn create_registered_address_table(tx: &mut RBatisTxExecutor<'_>) -> () {} diff --git a/core/rpc/types/Cargo.toml b/core/rpc/types/Cargo.toml index 51ae6c3b8..231a0ccb7 100644 --- a/core/rpc/types/Cargo.toml +++ b/core/rpc/types/Cargo.toml @@ -17,3 +17,6 @@ serde = { version = "1.0", features = ["derive"] } common = { path = "../../../common" } protocol = { path = "../../../protocol" } + +[dev-dependencies] +serde_json = "1.0" diff --git a/core/rpc/types/src/indexer.rs b/core/rpc/types/src/indexer.rs index c5dccfe1a..1e0539bb1 100644 --- a/core/rpc/types/src/indexer.rs +++ b/core/rpc/types/src/indexer.rs @@ -1,8 +1,9 @@ -use crate::IOType; - -use ckb_jsonrpc_types::{ - BlockNumber, Capacity, CellOutput, JsonBytes, OutPoint, Script, Uint32, Uint64, +use crate::{ + uints::{Uint32, Uint64}, + IOType, }; + +use ckb_jsonrpc_types::{BlockNumber, Capacity, CellOutput, JsonBytes, OutPoint, Script}; use ckb_types::H256; use serde::{Deserialize, Serialize}; diff --git a/core/rpc/types/src/lib.rs b/core/rpc/types/src/lib.rs index 2a46b838e..539d643bd 100644 --- a/core/rpc/types/src/lib.rs +++ b/core/rpc/types/src/lib.rs @@ -2,12 +2,13 @@ pub mod consts; pub mod error; pub mod indexer; pub mod lazy; +pub mod uints; + +use uints::{Uint128, Uint16, Uint32, Uint64}; use crate::error::TypeError; -use ckb_jsonrpc_types::{ - BlockNumber, CellDep, CellOutput, OutPoint, Script, TransactionView, Uint128, Uint32, Uint64, -}; +use ckb_jsonrpc_types::{BlockNumber, CellDep, CellOutput, OutPoint, Script, TransactionView}; use ckb_types::{bytes::Bytes, H160, H256}; use common::{derive_more::Display, utils::to_fixed_array, NetworkType, Order, Result}; use protocol::db::TransactionWrapper; @@ -591,8 +592,7 @@ impl SyncProgress { pub struct PaginationRequest { pub cursor: Option, pub order: Order, - pub limit: Option, - pub skip: Option, + pub limit: Option, pub return_count: bool, } @@ -600,15 +600,13 @@ impl PaginationRequest { pub fn new( cursor: Option, order: Order, - limit: Option, - skip: Option, + limit: Option, return_count: bool, ) -> PaginationRequest { PaginationRequest { cursor: cursor.map(Into::into), order, limit: limit.map(Into::into), - skip: skip.map(Into::into), return_count, } } @@ -620,7 +618,7 @@ impl std::convert::From for common::PaginationRequest { cursor: page.cursor.map(Into::into), order: page.order, limit: page.limit.map(Into::into), - skip: page.skip.map(Into::into), + skip: None, return_count: page.return_count, } } diff --git a/core/rpc/types/src/uints.rs b/core/rpc/types/src/uints.rs new file mode 100644 index 000000000..83f2e7ef3 --- /dev/null +++ b/core/rpc/types/src/uints.rs @@ -0,0 +1,241 @@ +use common::Result; + +use ckb_types::{ + packed, + prelude::{Pack, Unpack}, +}; +use serde::{ + de::{Error, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; + +use std::{fmt, marker, num}; + +pub trait Uint: Copy + fmt::LowerHex { + const NAME: &'static str; + + fn from_str_radix(src: &str, radix: u32) -> Result; +} + +#[derive(Copy, Clone, Default, PartialEq, PartialOrd, Ord, Eq, Hash, Debug)] +pub struct JsonUint(pub(crate) T); + +struct JsonUintVisitor(marker::PhantomData); + +impl JsonUint { + pub fn value(self) -> T { + self.0 + } +} + +impl fmt::Display for JsonUint { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "0x{:x}", self.value()) + } +} + +impl From for JsonUint { + fn from(value: T) -> Self { + Self(value) + } +} + +impl Serialize for JsonUint { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_str(self) + } +} + +impl JsonUintVisitor { + #[inline] + fn expecting(formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a hex-encoded, 0x-prefixed {}", T::NAME) + } + + #[inline] + fn visit_str(value: &str) -> Result, E> + where + E: Error, + { + let value_bytes = value.as_bytes(); + if value_bytes.len() < 3 || &value_bytes[..2] != b"0x" { + return Err(Error::custom(format!( + "Invalid {} {}: without `0x` prefix", + T::NAME, + value + ))); + } + if value_bytes[2] == b'0' && value_bytes.len() > 3 { + return Err(Error::custom(format!( + "Invalid {} {}: with redundant leading zeros", + T::NAME, + value, + ))); + } + + T::from_str_radix(&value[2..], 16) + .map(JsonUint) + .map_err(|e| Error::custom(format!("Invalid {} {}: {}", T::NAME, value, e))) + } +} + +macro_rules! def_json_uint { + ($alias:ident, $inner:ident, $bits:expr) => { + #[doc = "The "] + #[doc = $bits] + #[doc = r#" unsigned integer type encoded as the 0x-prefixed hex string in JSON. + +## Examples + +| JSON | Decimal Value | +| -------| ---------------------------- | +| "0x0" | 0 | +| "0x10" | 16 | +| "10" | Invalid, 0x is required | +| "0x01" | Invalid, redundant leading 0 |"#] + pub type $alias = JsonUint<$inner>; + + impl Uint for $inner { + const NAME: &'static str = stringify!($alias); + + fn from_str_radix(src: &str, radix: u32) -> Result { + $inner::from_str_radix(src, radix) + } + } + + impl From> for $inner { + fn from(value: JsonUint<$inner>) -> Self { + value.value() + } + } + }; +} + +macro_rules! impl_serde_deserialize { + ($visitor:ident, $inner:ident) => { + impl<'a> Deserialize<'a> for JsonUint<$inner> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'a>, + { + deserializer.deserialize_str($visitor) + } + } + + struct $visitor; + + impl<'a> Visitor<'a> for $visitor { + type Value = JsonUint<$inner>; + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + JsonUintVisitor::<$inner>::expecting(formatter) + } + + fn visit_str(self, value: &str) -> Result + where + E: Error, + { + JsonUintVisitor::<$inner>::visit_str(value) + } + } + }; +} + +macro_rules! impl_pack_and_unpack { + ($packed:ident, $inner:ident) => { + impl Pack for JsonUint<$inner> { + fn pack(&self) -> packed::$packed { + self.value().pack() + } + } + + impl Unpack> for packed::$packed { + fn unpack(&self) -> JsonUint<$inner> { + Unpack::<$inner>::unpack(self).into() + } + } + }; +} + +def_json_uint!(Uint16, u16, "16-bit"); +def_json_uint!(Uint32, u32, "32-bit"); +def_json_uint!(Uint64, u64, "64-bit"); +def_json_uint!(Uint128, u128, "128-bit"); +impl_serde_deserialize!(Uint16Visitor, u16); +impl_serde_deserialize!(Uint32Visitor, u32); +impl_serde_deserialize!(Uint64Visitor, u64); +impl_serde_deserialize!(Uint128Visitor, u128); +impl_pack_and_unpack!(Uint32, u32); +impl_pack_and_unpack!(Uint64, u64); +impl_pack_and_unpack!(Uint128, u128); + +#[cfg(test)] +mod tests { + macro_rules! test_json_uint { + ($tests_name:ident, $name:ident, $inner:ident) => { + mod $tests_name { + use crate::$name; + + #[test] + fn serialize() { + assert_eq!(r#""0xd""#, serde_json::to_string(&$name::from(13)).unwrap()); + assert_eq!(r#""0x0""#, serde_json::to_string(&$name::from(0)).unwrap()); + } + + #[test] + fn deserialize_hexadecimal() { + let s = r#""0xa""#; + let deserialized: $name = serde_json::from_str(s).unwrap(); + assert_eq!(deserialized, $name::from(10)); + } + + #[test] + fn deserialize_decimal() { + let s = r#""10""#; + let ret: Result<$name, _> = serde_json::from_str(s); + assert!(ret.is_err()); + } + + #[test] + fn deserialize_with_redundant_leading_zeros() { + let cases = vec![r#""0x01""#, r#""0x00""#]; + for s in cases { + let ret: Result<$name, _> = serde_json::from_str(s); + assert!(ret.is_err()); + + let err = ret.unwrap_err(); + assert!(err.to_string().contains("with redundant leading zeros"),); + } + } + + #[test] + fn deserialize_with_uppercase() { + let cases = vec![r#""0xFF""#, r#""0xfF""#]; + for s in cases { + let ret: Result<$name, _> = serde_json::from_str(s); + assert!(ret.is_ok()); + } + } + + #[test] + fn deserialize_too_large() { + let s = format!(r#""0x{:x}0""#, $inner::max_value()); + let ret: Result<$name, _> = serde_json::from_str(&s); + assert!(ret.is_err()); + + let err = ret.unwrap_err(); + assert!(err + .to_string() + .contains("number too large to fit in target type"),); + } + } + }; + } + + test_json_uint!(uint16, Uint16, u16); + test_json_uint!(uint32, Uint32, u32); + test_json_uint!(uint64, Uint64, u64); + test_json_uint!(uint128, Uint128, u128); +} diff --git a/core/rpc/utility/Cargo.toml b/core/rpc/utility/Cargo.toml deleted file mode 100644 index 73f8c0614..000000000 --- a/core/rpc/utility/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "core-rpc-utility" -version = "0.4.1" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -ckb-jsonrpc-types = "0.101" -ckb-types = "0.101" -ckb-dao-utils = "0.101" - -common = { path = "../../../common" } -common-logger = { path = "../../../logger" } -core-ckb-client = { path = "../../ckb-client" } -core-rpc-types = { path = "../types" } -core-storage = { path = "../../storage" } diff --git a/core/rpc/utility/src/lib.rs b/core/rpc/utility/src/lib.rs deleted file mode 100644 index 72367b935..000000000 --- a/core/rpc/utility/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use common::async_trait; - -#[async_trait] -pub trait RpcUtility {} diff --git a/core/service/Cargo.toml b/core/service/Cargo.toml index 5c6e690a8..4b69e469a 100644 --- a/core/service/Cargo.toml +++ b/core/service/Cargo.toml @@ -7,14 +7,13 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -# ckb-indexer = { git = "https://github.com/KaoImin/ckb-indexer", branch = "mercury" } ckb-jsonrpc-types = "0.101" ckb-types = "0.101" jsonrpsee-http-server = "0.13" lazy_static = "1.4" log = "0.4" parking_lot = "0.12" -tokio = { version = "1.14", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } common = { path = "../../common" } core-ckb-client = { path = "../ckb-client" } diff --git a/core/service/src/lib.rs b/core/service/src/lib.rs index 545c50f57..58e48529d 100644 --- a/core/service/src/lib.rs +++ b/core/service/src/lib.rs @@ -1,8 +1,5 @@ #![allow(clippy::mutable_key_type, dead_code)] -use ckb_jsonrpc_types::{RawTxPool, TransactionWithStatus}; -use ckb_types::core::{BlockNumber, BlockView, EpochNumberWithFraction, RationalU256}; -use ckb_types::{packed, H256}; use common::{anyhow::anyhow, utils::ScriptInfo, Context, NetworkType, Result}; use core_ckb_client::{CkbRpc, CkbRpcClient}; use core_rpc::{MercuryRpcImpl, MercuryRpcServer}; @@ -10,8 +7,12 @@ use core_rpc_types::lazy::{CURRENT_BLOCK_NUMBER, CURRENT_EPOCH_NUMBER, TX_POOL_C use core_rpc_types::{SyncProgress, SyncState}; use core_storage::{DBDriver, RelationalStorage, Storage}; use core_synchronization::{Synchronization, TASK_LEN}; + +use ckb_jsonrpc_types::{RawTxPool, TransactionWithStatus}; +use ckb_types::core::{BlockNumber, BlockView, EpochNumberWithFraction, RationalU256}; +use ckb_types::{packed, H256}; use jsonrpsee_http_server::{HttpServerBuilder, HttpServerHandle}; -use log::{error, info, warn, LevelFilter}; +use log::{error, info, warn}; use parking_lot::RwLock; use tokio::time::{sleep, Duration}; @@ -32,7 +33,7 @@ pub struct Service { cheque_since: RationalU256, use_tx_pool_cache: bool, sync_state: Arc>, - pool_cache_size: u64, + pool_cache_size: u16, is_pprof_enabled: bool, } @@ -54,8 +55,7 @@ impl Service { cellbase_maturity: u64, ckb_uri: String, cheque_since: u64, - log_level: LevelFilter, - pool_cache_size: u64, + pool_cache_size: u16, is_pprof_enabled: bool, ) -> Self { let ckb_client = CkbRpcClient::new(ckb_uri); @@ -67,7 +67,6 @@ impl Service { connect_timeout, max_lifetime, idle_timeout, - log_level, ); let network_type = NetworkType::from_raw_str(network_ty).expect("invalid network type"); let cellbase_maturity = RationalU256::from_u256(cellbase_maturity.into()); @@ -93,7 +92,7 @@ impl Service { } pub async fn init( - &self, + &mut self, listen_address: String, db_driver: String, db_name: String, @@ -159,7 +158,7 @@ impl Service { } let sync_handler = Synchronization::new( - self.store.inner(), + self.store.get_pool(), Arc::new(self.ckb_client.clone()), max_task_number, node_tip, @@ -173,7 +172,7 @@ impl Service { < TASK_LEN { return Synchronization::new( - self.store.inner(), + self.store.get_pool(), Arc::new(self.ckb_client.clone()), max_task_number, db_tip, @@ -227,10 +226,7 @@ impl Service { let block_number = block.number(); log::info!("append {}, {}", block_number, block.hash()); let start = Instant::now(); - self.store - .append_block(Context::new(), block) - .await - .expect("append block"); + self.store.append_block(block).await.expect("append block"); let duration = start.elapsed(); log::info!( "append {} time elapsed is: {:?} ms", @@ -240,7 +236,7 @@ impl Service { } else { info!("rollback {}, {}", tip_number, tip_hash); self.store - .rollback_block(Context::new(), tip_number, tip_hash) + .rollback_block(tip_number, tip_hash) .await .expect("rollback block"); } @@ -261,10 +257,7 @@ impl Service { Ok(Some(block)) => { log::info!("append {} block", 0); self.change_current_epoch(block.epoch().to_rational()); - self.store - .append_block(Context::new(), block) - .await - .expect("append block"); + self.store.append_block(block).await.expect("append block"); } Ok(None) => { diff --git a/core/storage/Cargo.toml b/core/storage/Cargo.toml index 0547867b7..8359bb3ec 100644 --- a/core/storage/Cargo.toml +++ b/core/storage/Cargo.toml @@ -14,23 +14,24 @@ dashmap = "4.0" hex = "0.4" lazy_static = "1.4" log = "0.4" -rbatis = { version = "3.0", default-features = false, features = ["all-database", "runtime-tokio-native-tls", "upper_case_sql_keyword"] } -rbson = "2.0" +seq-macro = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.14", features = ["macros", "rt-multi-thread", "sync"] } +sql-builder = "3.1" +sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "postgres"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] } tokio-stream = { version = "0.1", features = ["sync"] } common = { path = "../../common" } -common-logger = { path = "../../logger" } -db-xsql = { path = "../../db/xsql", package = "xsql" } +core-rpc-types = { path = "../../core/rpc/types" } +db-sqlx = { path = "../../db/db-sqlx", package = "db-sqlx" } protocol = { path = "../../protocol" } [dev-dependencies] arc-swap = "1.5" criterion = { version = "0.3", features = ["async_tokio", "html_reports"] } env_logger = "0.9" -rand = "0.7" +sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "sqlite"] } xsql-test = { path = "../../db/xsql-test" } [[bench]] diff --git a/core/storage/src/lib.rs b/core/storage/src/lib.rs index b1a301c12..5ebac5371 100644 --- a/core/storage/src/lib.rs +++ b/core/storage/src/lib.rs @@ -3,29 +3,23 @@ pub mod error; pub mod relational; -pub use protocol::db::{DBDriver, DBInfo, SimpleBlock, SimpleTransaction, TransactionWrapper}; -use relational::table::IndexerCellTable; pub use relational::RelationalStorage; +use ckb_types::core::{BlockNumber, BlockView, HeaderView}; +use ckb_types::{bytes::Bytes, packed, H160, H256}; use common::{ async_trait, Context, DetailedCell, PaginationRequest, PaginationResponse, Range, Result, }; - -use ckb_types::core::{BlockNumber, BlockView, HeaderView}; -use ckb_types::{bytes::Bytes, packed, H160, H256}; +use core_rpc_types::indexer::Transaction; +pub use protocol::db::{DBDriver, DBInfo, SimpleBlock, SimpleTransaction, TransactionWrapper}; #[async_trait] pub trait Storage { /// Append the given block to the database. - async fn append_block(&self, ctx: Context, block: BlockView) -> Result<()>; + async fn append_block(&self, block: BlockView) -> Result<()>; /// Rollback a block by block hash and block number from the database. - async fn rollback_block( - &self, - ctx: Context, - block_number: BlockNumber, - block_hash: H256, - ) -> Result<()>; + async fn rollback_block(&self, block_number: BlockNumber, block_hash: H256) -> Result<()>; /// Get live cells from the database according to the given arguments. async fn get_live_cells( @@ -193,7 +187,7 @@ pub trait Storage { type_hashes: Vec, block_range: Option, pagination: PaginationRequest, - ) -> Result>; + ) -> Result>; /// Get the block count. async fn indexer_synced_count(&self) -> Result; @@ -201,88 +195,3 @@ pub trait Storage { /// Get the block count. async fn block_count(&self, ctx: Context) -> Result; } - -#[async_trait] -pub trait ExtensionStorage { - async fn get_live_cells( - &self, - ctx: Context, - out_point: Option, - lock_hashes: Vec, - type_hashes: Vec, - pagination: PaginationRequest, - ) -> Result>; - - async fn get_historical_live_cells( - &self, - ctx: Context, - lock_hashes: Vec, - type_hashes: Vec, - tip_block_number: BlockNumber, - out_point: Option, - pagination: PaginationRequest, - ) -> Result>; - - async fn get_cells( - &self, - ctx: Context, - out_point: Option, - lock_hashes: Vec, - type_hashes: Vec, - block_range: Option, - pagination: PaginationRequest, - ) -> Result>; - - async fn get_transactions_by_hashes( - &self, - ctx: Context, - tx_hashes: Vec, - block_range: Option, - pagination: PaginationRequest, - ) -> Result>; - - async fn get_transactions_by_scripts( - &self, - ctx: Context, - lock_hashes: Vec, - type_hashes: Vec, - block_range: Option, - pagination: PaginationRequest, - ) -> Result>; - - async fn get_block_header( - &self, - ctx: Context, - block_hash: Option, - block_number: Option, - ) -> Result; - - async fn get_scripts( - &self, - ctx: Context, - script_hashes: Vec, - code_hash: Vec, - args_len: Option, - args: Vec, - ) -> Result>; - - async fn get_cells_by_partial_args( - &self, - ctx: Context, - p_lock_args: Option, - p_type_args: Option, - pagination: PaginationRequest, - ) -> Result>; - - async fn get_cells_by_partial_data( - &self, - ctx: Context, - p_data: PartialBytes, - pagination: PaginationRequest, - ) -> Result>; -} - -pub struct PartialBytes { - pub content: Vec, - pub range: std::ops::Range, -} diff --git a/core/storage/src/relational/_sql.html b/core/storage/src/relational/_sql.html deleted file mode 100644 index cd82e9cf8..000000000 --- a/core/storage/src/relational/_sql.html +++ /dev/null @@ -1,97 +0,0 @@ - - - - - diff --git a/core/storage/src/relational/fetch.rs b/core/storage/src/relational/fetch.rs index 74d5c47e9..095fd3eac 100644 --- a/core/storage/src/relational/fetch.rs +++ b/core/storage/src/relational/fetch.rs @@ -1,17 +1,12 @@ use crate::error::DBError; -use crate::relational::table::{ - decode_since, BlockTable, CanonicalChainTable, CellTable, IndexerCellTable, LiveCellTable, - RegisteredAddressTable, ScriptTable, TransactionTable, -}; -use crate::relational::{to_rb_bytes, RelationalStorage}; +use crate::relational::RelationalStorage; use common::{ - utils, utils::to_fixed_array, Context, DetailedCell, PaginationRequest, PaginationResponse, - Range, Result, + utils, utils::to_fixed_array, Context, DetailedCell, Order, PaginationRequest, + PaginationResponse, Range, Result, }; -use common_logger::tracing_async; -use db_xsql::page::PageRequest; -use db_xsql::rbatis::{crud::CRUDMut, plugin::page::Page, Bytes as RbBytes}; +use core_rpc_types::{indexer::Transaction, IOType}; +use db_sqlx::{build_query_page_sql, SQLXPool}; use protocol::db::{SimpleBlock, SimpleTransaction, TransactionWrapper}; use ckb_jsonrpc_types::TransactionWithStatus; @@ -20,42 +15,30 @@ use ckb_types::core::{ BlockBuilder, BlockNumber, BlockView, EpochNumberWithFraction, HeaderBuilder, HeaderView, TransactionBuilder, TransactionView, UncleBlockView, }; -use ckb_types::{packed, prelude::*, H256}; +use ckb_types::{packed, prelude::*, H160, H256}; +use sql_builder::SqlBuilder; +use sqlx::{any::AnyRow, Row}; use std::collections::HashMap; use std::convert::From; -use std::slice::Iter; - -macro_rules! build_next_cursor { - ($page: expr, $pagination: expr) => {{ - if $page.records.is_empty() - || $page.search_count - && $page.total <= $pagination.limit.map(Into::into).unwrap_or(u64::MAX) - || ($page.records.len() as u64) < $pagination.limit.map(Into::into).unwrap_or(u64::MAX) - { - None - } else { - Some($page.records.last().cloned().unwrap().id as u64) - } - }}; -} impl RelationalStorage { pub(crate) async fn query_tip(&self) -> Result> { - let mut conn = self.pool.acquire().await?; - let w = self - .pool - .wrapper() - .order_by(false, &["block_number"]) - .limit(1); - let res: Option = conn.fetch_by_wrapper(w).await?; - - Ok(res.map(|t| { - ( - t.block_number, - H256::from_slice(&t.block_hash.inner[0..32]).expect("get block hash h256"), - ) - })) + let query = SQLXPool::new_query( + r#" + SELECT * FROM mercury_canonical_chain + ORDER BY block_number + DESC LIMIT 1 + "#, + ); + self.sqlx_pool.fetch_optional(query).await.map(|res| { + res.map(|row| { + ( + row.get::("block_number") as u64, + bytes_to_h256(row.get("block_hash")), + ) + }) + }) } pub(crate) async fn get_block_by_number( @@ -72,7 +55,6 @@ impl RelationalStorage { ctx: Context, block_hash: H256, ) -> Result { - let block_hash = to_rb_bytes(block_hash.as_bytes()); let block = self.query_block_by_hash(block_hash).await?; self.get_block_view(ctx, &block).await } @@ -91,7 +73,6 @@ impl RelationalStorage { &self, block_hash: H256, ) -> Result { - let block_hash = to_rb_bytes(block_hash.as_bytes()); let block = self.query_block_by_hash(block_hash).await?; Ok(build_header_view(&block)) } @@ -104,23 +85,23 @@ impl RelationalStorage { Ok(build_header_view(&block)) } - async fn get_block_view(&self, ctx: Context, block: &BlockTable) -> Result { + async fn get_block_view(&self, ctx: Context, block: &AnyRow) -> Result { let header = build_header_view(block); - let uncles = packed::UncleBlockVec::from_slice(&block.uncles.inner)? + let uncles = packed::UncleBlockVec::from_slice(block.get("uncles"))? .into_iter() .map(|uncle| uncle.into_view()) .collect::>(); let txs = self - .get_transactions_by_block_hash(ctx, &block.block_hash) + .get_transactions_by_block_hash(ctx, block.get("block_hash")) .await?; - let proposals = build_proposals(block.proposals.inner.clone()); + let proposals = build_proposals(block.get("proposals")); Ok(build_block_view(header, uncles, txs, proposals)) } async fn get_transactions_by_block_hash( &self, ctx: Context, - block_hash: &RbBytes, + block_hash: &[u8], ) -> Result> { let txs = self.query_transactions_by_block_hash(block_hash).await?; self.get_transaction_views(ctx, txs).await @@ -130,24 +111,24 @@ impl RelationalStorage { &self, tx_hash: H256, ) -> Result { - let mut conn = self.pool.acquire().await?; - let w = self - .pool - .wrapper() - .eq("tx_hash", to_rb_bytes(&tx_hash.0)) - .limit(1); - let res = conn.fetch_by_wrapper::(w).await?; - + let query = SQLXPool::new_query( + r#" + SELECT tx_index, block_number, block_hash, epoch_number, epoch_index, epoch_length + FROM mercury_cell + WHERE tx_hash = $1 + "#, + ) + .bind(tx_hash.as_bytes()); + let cell = self.sqlx_pool.fetch_one(query).await?; let epoch_number = EpochNumberWithFraction::new( - res.epoch_number.into(), - res.epoch_index.into(), - res.epoch_length.into(), + cell.get::("epoch_number").try_into()?, + cell.get::("epoch_index").try_into()?, + cell.get::("epoch_length").try_into()?, ) .to_rational(); - let block_hash = H256::from_slice(&res.block_hash.inner[0..32]).expect("get block hash"); - let block_number = res.block_number; - let tx_index = res.tx_index as u32; - + let block_hash = bytes_to_h256(cell.get("block_hash")); + let block_number = cell.get::("block_number").try_into()?; + let tx_index = cell.get::("tx_index").try_into()?; Ok(SimpleTransaction { epoch_number, block_number, @@ -160,26 +141,25 @@ impl RelationalStorage { &self, out_point: packed::OutPoint, ) -> Result> { - let mut conn = self.pool.acquire().await?; let tx_hash: H256 = out_point.tx_hash().unpack(); let output_index: u32 = out_point.index().unpack(); - let w = self - .pool - .wrapper() - .eq("tx_hash", to_rb_bytes(&tx_hash.0)) - .and() - .eq("output_index", output_index) - .limit(1); - let res: Option = conn.fetch_by_wrapper(w).await?; - - if let Some(table) = res { - if table.consumed_tx_hash.inner.is_empty() { - return Ok(None); + let query = SQLXPool::new_query( + r#" + SELECT tx_hash, output_index, consumed_tx_hash + FROM mercury_cell + WHERE tx_hash = $1 AND output_index = $2 + "#, + ) + .bind(tx_hash.as_bytes()) + .bind(i32::try_from(output_index)?); + let cell = self.sqlx_pool.fetch_optional(query).await?; + if let Some(cell) = cell { + let consumed_tx_hash = cell.get::, _>("consumed_tx_hash"); + if consumed_tx_hash.is_empty() { + Ok(None) + } else { + Ok(Some(bytes_to_h256(&consumed_tx_hash))) } - - Ok(Some( - H256::from_slice(&table.consumed_tx_hash.inner[0..32]).expect("get tx hash"), - )) } else { Ok(None) } @@ -188,7 +168,7 @@ impl RelationalStorage { pub(crate) async fn get_transaction_views( &self, ctx: Context, - txs: Vec, + txs: Vec, ) -> Result> { let txs_wrapper = self.get_transactions_with_status(ctx, txs).await?; let tx_views = txs_wrapper @@ -198,65 +178,81 @@ impl RelationalStorage { Ok(tx_views) } - #[tracing_async] pub(crate) async fn get_transactions_with_status( &self, _ctx: Context, - txs: Vec, + txs: Vec, ) -> Result> { if txs.is_empty() { return Ok(Vec::new()); } - let tx_hashes: Vec = txs.iter().map(|tx| tx.tx_hash.clone()).collect(); - let output_cells = self.query_txs_output_cells(&tx_hashes).await?; - let input_cells = self.query_txs_input_cells(&tx_hashes).await?; - - let mut txs_output_cells: HashMap, Vec> = tx_hashes - .iter() - .map(|tx_hash| (tx_hash.inner.clone(), vec![])) - .collect(); - let mut txs_input_cells: HashMap, Vec> = tx_hashes + let tx_hashes: Vec> = txs .iter() - .map(|tx_hash| (tx_hash.inner.clone(), vec![])) + .map(|tx| tx.get::, _>("tx_hash")) .collect(); + let output_cells = self.query_txs_output_cells(&tx_hashes).await?; + let input_cells = self.query_txs_input_cells(&tx_hashes).await?; + let mut output_cells_group_by_tx_hash = HashMap::new(); for cell in output_cells { - if let Some(set) = txs_output_cells.get_mut(&cell.tx_hash.inner) { - (*set).push(cell) - } + output_cells_group_by_tx_hash + .entry(cell.get::, _>("tx_hash")) + .or_insert_with(Vec::new) + .push(build_detailed_cell(cell)?); } + let mut input_cells_group_by_tx_hash = HashMap::new(); for cell in input_cells { - if let Some(set) = txs_input_cells.get_mut(&cell.consumed_tx_hash.inner) { - (*set).push(cell) - } + input_cells_group_by_tx_hash + .entry(cell.get::, _>("consumed_tx_hash")) + .or_insert_with(Vec::new) + .push(build_detailed_cell(cell)?); } let txs_with_status = txs .into_iter() .map(|tx| { - let witnesses = build_witnesses(tx.witnesses.inner.clone()); - let header_deps = build_header_deps(tx.header_deps.inner.clone()); - let cell_deps = build_cell_deps(tx.cell_deps.inner.clone()); + let witnesses = build_witnesses(tx.get("witnesses")); + let header_deps = build_header_deps(tx.get("header_deps")); + let cell_deps = build_cell_deps(tx.get("cell_deps")); - let input_tables = txs_input_cells - .get(&tx.tx_hash.inner) + let input_cells = input_cells_group_by_tx_hash + .get(&tx.get::, _>("tx_hash")) .cloned() .unwrap_or_default(); - let mut inputs = build_cell_inputs(input_tables.iter()); - if inputs.is_empty() && tx.tx_index == 0 { - inputs = vec![build_cell_base_input(tx.block_number)] + let mut inputs: Vec = input_cells + .iter() + .map(|cell| { + packed::CellInputBuilder::default() + .since(cell.since.expect("get since").pack()) + .previous_output(cell.out_point.clone()) + .build() + }) + .collect(); + if inputs.is_empty() && tx.get::("tx_index") == 0i32 { + inputs = vec![build_cell_base_input( + tx.get::("block_number") + .try_into() + .expect("i32 to u64"), + )] }; - let output_tables = txs_output_cells - .get(&tx.tx_hash.inner) + let output_cells = output_cells_group_by_tx_hash + .get(&tx.get::, _>("tx_hash")) .cloned() .unwrap_or_default(); - let (outputs, outputs_data) = build_cell_outputs(&output_tables); + let outputs = output_cells + .iter() + .map(|cell| cell.cell_output.clone()) + .collect(); + let outputs_data = output_cells + .iter() + .map(|cell| cell.cell_data.pack()) + .collect(); let transaction_view = build_transaction_view( - tx.version as u32, + tx.get::("version").try_into().expect("i16 to u32"), witnesses, inputs, outputs, @@ -266,13 +262,14 @@ impl RelationalStorage { ); let transaction_with_status = TransactionWithStatus::with_committed( Some(transaction_view.clone()), - H256::from_slice(tx.block_hash.inner.as_slice()).expect("get block hash"), + bytes_to_h256(tx.get("block_hash")), ); - let is_cellbase = tx.tx_index == 0; - let input_cells = input_tables.into_iter().map(Into::into).collect(); - let output_cells = output_tables.into_iter().map(Into::into).collect(); - let timestamp = tx.tx_timestamp; + let is_cellbase = tx.get::("tx_index") == 0; + let timestamp = tx + .get::("tx_timestamp") + .try_into() + .expect("i64 to u64"); TransactionWrapper { transaction_with_status, @@ -288,148 +285,179 @@ impl RelationalStorage { } pub(crate) async fn get_tip_simple_block(&self) -> Result { - let tip = self.query_tip().await?; - let (_, block_hash) = match tip { - Some((block_number, block_hash)) => (block_number, block_hash), - None => return Err(DBError::NotExist("tip block".to_string()).into()), - }; - let block_table = self - .query_block_by_hash(to_rb_bytes(block_hash.as_bytes())) - .await?; - self.get_simple_block(&block_table).await + let (block_hash, block_number, parent_hash, block_timestamp) = + self.query_tip_simple_block().await?; + self.get_simple_block(block_hash, block_number, parent_hash, block_timestamp) + .await } pub(crate) async fn get_simple_block_by_block_number( &self, block_number: BlockNumber, ) -> Result { - let block_table = self.query_block_by_number(block_number).await?; - self.get_simple_block(&block_table).await + let (block_hash, block_number, parent_hash, block_timestamp) = + self.query_simple_block_by_number(block_number).await?; + self.get_simple_block(block_hash, block_number, parent_hash, block_timestamp) + .await } pub(crate) async fn get_simple_block_by_block_hash( &self, block_hash: H256, ) -> Result { - let block_table = self - .query_block_by_hash(to_rb_bytes(block_hash.as_bytes())) - .await?; - self.get_simple_block(&block_table).await + let (block_hash, block_number, parent_hash, block_timestamp) = + self.query_simple_block_by_hash(block_hash).await?; + self.get_simple_block(block_hash, block_number, parent_hash, block_timestamp) + .await } - async fn get_simple_block(&self, block_table: &BlockTable) -> Result { - let txs = self - .query_transactions_by_block_hash(&block_table.block_hash) + async fn get_simple_block( + &self, + block_hash: H256, + block_number: BlockNumber, + parent_hash: H256, + timestamp: u64, + ) -> Result { + let transactions = self + .query_transaction_hashes_by_block_hash(block_hash.as_bytes()) .await?; Ok(SimpleBlock { - block_number: block_table.block_number, - block_hash: rb_bytes_to_h256(&block_table.block_hash), - parent_hash: rb_bytes_to_h256(&block_table.parent_hash), - timestamp: block_table.block_timestamp, - transactions: txs - .iter() - .map(|tx| rb_bytes_to_h256(&tx.tx_hash)) - .collect::>(), + block_number, + block_hash, + parent_hash, + timestamp, + transactions, }) } pub(crate) async fn query_scripts( &self, - script_hashes: Vec, - code_hash: Vec, + script_hashes: Vec, + code_hashes: Vec, args_len: Option, - args: Vec, + args: Vec, ) -> Result> { - if script_hashes.is_empty() && code_hash.is_empty() && args_len.is_none() && args.is_empty() - { + if script_hashes.is_empty() && args.is_empty() { return Err(DBError::InvalidParameter( "no valid parameter to query scripts".to_owned(), ) .into()); } - let mut wrapper = self.pool.wrapper(); - + // build query str + let mut query_builder = SqlBuilder::select_from("mercury_script"); + query_builder.field("script_code_hash, script_args, script_type"); if !script_hashes.is_empty() { - wrapper = wrapper.in_array("script_hash_160", &script_hashes) + query_builder.and_where_in( + "script_hash_160", + &sqlx_param_placeholders(1..script_hashes.len())?, + ); } - - if !code_hash.is_empty() { - wrapper = wrapper.and().in_array("script_code_hash", &code_hash); + if !code_hashes.is_empty() { + query_builder.and_where_in( + "script_code_hash", + &sqlx_param_placeholders( + script_hashes.len() + 1..script_hashes.len() + code_hashes.len(), + )?, + ); } - if !args.is_empty() { - wrapper = wrapper.and().in_array("script_args", &args); + query_builder.and_where_in( + "script_args", + &sqlx_param_placeholders( + script_hashes.len() + code_hashes.len() + 1 + ..script_hashes.len() + code_hashes.len() + args.len(), + )?, + ); } - if let Some(len) = args_len { - wrapper = wrapper.and().eq("script_args_len", len); + query_builder.and_where_eq("script_args_len", len); } + let query = query_builder.sql()?.trim_end_matches(';').to_string(); - let mut conn = self.pool.acquire().await?; - let scripts: Vec = conn.fetch_list_by_wrapper(wrapper).await?; + // bind + let mut query = SQLXPool::new_query(&query); + for script_hash in &script_hashes { + query = query.bind(script_hash.as_bytes()); + } + for code_hash in &code_hashes { + query = query.bind(code_hash.as_bytes()); + } + for arg in &args { + query = query.bind(arg.to_vec()); + } - Ok(scripts.into_iter().map(Into::into).collect()) + // fetch + let res = self.sqlx_pool.fetch(query).await?; + Ok(res + .into_iter() + .map(|row| { + packed::ScriptBuilder::default() + .code_hash(bytes_to_h256(row.get("script_code_hash")).pack()) + .args(row.get::, _>("script_args").pack()) + .hash_type(packed::Byte::new(row.get::("script_type") as u8)) + .build() + }) + .collect()) } pub(crate) async fn query_canonical_block_hash( &self, block_number: BlockNumber, ) -> Result { - let mut conn = self.pool.acquire().await?; - let ret = conn - .fetch_by_column::("block_number", block_number) - .await?; - Ok(rb_bytes_to_h256(&ret.block_hash)) + let query = SQLXPool::new_query( + r#" + SELECT block_hash + FROM mercury_canonical_chain + WHERE block_number = $1 + "#, + ) + .bind(i32::try_from(block_number)?); + let row = self.sqlx_pool.fetch_one(query).await?; + let block_hash = row.get("block_hash"); + Ok(bytes_to_h256(block_hash)) } async fn query_live_cell_by_out_point( &self, out_point: packed::OutPoint, ) -> Result { - let mut conn = self.pool.acquire().await?; let tx_hash: H256 = out_point.tx_hash().unpack(); let output_index: u32 = out_point.index().unpack(); - let w = self - .pool - .wrapper() - .eq("tx_hash", to_rb_bytes(&tx_hash.0)) - .and() - .eq("output_index", output_index); - - let res: Option = conn.fetch_by_wrapper(w).await?; - let res = res.ok_or_else(|| { - DBError::NotExist(format!( - "live cell with out point {} {}", - tx_hash, output_index - )) - })?; - let cell: CellTable = res.into(); - Ok(cell.into()) + let query = SQLXPool::new_query( + r#" + SELECT * + FROM mercury_live_cell + WHERE tx_hash = $1 AND output_index = $2 + "#, + ) + .bind(tx_hash.as_bytes()) + .bind(i32::try_from(output_index)?); + let row = self.sqlx_pool.fetch_one(query).await?; + build_detailed_cell(row) } async fn query_cell_by_out_point(&self, out_point: packed::OutPoint) -> Result { - let mut conn = self.pool.acquire().await?; let tx_hash: H256 = out_point.tx_hash().unpack(); let output_index: u32 = out_point.index().unpack(); - let w = self - .pool - .wrapper() - .eq("tx_hash", to_rb_bytes(&tx_hash.0)) - .and() - .eq("output_index", output_index); - - let res = conn.fetch_by_wrapper::(w).await?; - Ok(res.into()) + let query = SQLXPool::new_query( + r#" + SELECT * + FROM mercury_cell + WHERE tx_hash = $1 AND output_index = $2 + "#, + ) + .bind(tx_hash.as_bytes()) + .bind(i32::try_from(output_index)?); + let row = self.sqlx_pool.fetch_one(query).await?; + build_detailed_cell(row) } - #[tracing_async] pub(crate) async fn query_live_cells( &self, - _ctx: Context, out_point: Option, - lock_hashes: Vec, - type_hashes: Vec, + lock_hashes: Vec, + type_hashes: Vec, block_range: Option, capacity_range: Option, data_len_range: Option, @@ -451,37 +479,26 @@ impl RelationalStorage { let mut is_ok = true; let lock_hash: H256 = cell.cell_output.lock().calc_script_hash().unpack(); - let lock_hash = to_rb_bytes(&lock_hash.0); if !lock_hashes.is_empty() { is_ok &= lock_hashes.contains(&lock_hash) }; - if let Some(type_script) = cell.cell_output.type_().to_opt() { let type_hash: H256 = type_script.calc_script_hash().unpack(); - let type_hash = to_rb_bytes(&type_hash.0); if !type_hashes.is_empty() { is_ok &= type_hashes.contains(&type_hash) }; } else if !type_hashes.is_empty() { - let default_hashes = vec![H256::default()] - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - is_ok &= type_hashes == default_hashes + is_ok &= type_hashes == vec![H256::default()] } - if let Some(range) = block_range { is_ok &= range.is_in(cell.block_number); } - if let Some(range) = capacity_range { is_ok &= range.is_in(cell.cell_output.capacity().unpack()) } - if let Some(range) = data_len_range { is_ok &= range.is_in(cell.cell_data.len() as u64) } - let mut response: Vec = vec![]; if is_ok { response.push(cell); @@ -498,63 +515,75 @@ impl RelationalStorage { }); } - let mut wrapper = self.pool.wrapper(); - + let mut query_builder = SqlBuilder::select_from("mercury_live_cell"); + query_builder.field("*"); if !lock_hashes.is_empty() { - wrapper = wrapper.in_array("lock_hash", &lock_hashes); + query_builder + .and_where_in("lock_hash", &sqlx_param_placeholders(1..lock_hashes.len())?); } - if !type_hashes.is_empty() { - wrapper = wrapper.and().in_array("type_hash", &type_hashes); + query_builder.and_where_in( + "type_hash", + &sqlx_param_placeholders( + lock_hashes.len() + 1..lock_hashes.len() + type_hashes.len(), + )?, + ); } - - if let Some(range) = block_range { - wrapper = wrapper - .and() - .between("block_number", range.min(), range.max()) + if let Some(ref range) = block_range { + query_builder.and_where_between( + "block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ); } - if let Some(range) = capacity_range { - wrapper = wrapper.and().between("capacity", range.min(), range.max()) + query_builder.and_where_between( + "capacity", + range.from.min(i64::MAX as u64), + range.to.min(i64::MAX as u64), + ); } if let Some(range) = data_len_range { - wrapper = wrapper - .and() - .between("LENGTH(data)", range.min(), range.max()) + query_builder.and_where_between("LENGTH(data)", range.min(), range.max()); } + let (sql, sql_for_total) = build_query_page_sql(query_builder, &pagination)?; - let mut conn = self.pool.acquire().await?; - let cells: Page = conn - .fetch_page_by_wrapper(wrapper, &PageRequest::from(pagination.clone())) + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in &lock_hashes { + query = query.bind(hash.as_bytes()); + } + for hash in &type_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + let query_total = bind(&sql_for_total); + + // fetch + let page = self + .sqlx_pool + .fetch_page(query, query_total, &pagination) .await?; - let next_cursor = build_next_cursor!(cells, pagination); - let res = cells - .records - .into_iter() - .map(|r| { - let cell: CellTable = r.into(); - cell.into() - }) - .collect(); - Ok(to_pagination_response( - res, - next_cursor, - if pagination.return_count { - Some(cells.total) - } else { - None - }, - )) + let mut cells = vec![]; + for row in page.response { + cells.push(build_detailed_cell(row)?); + } + Ok(PaginationResponse { + response: cells, + next_cursor: page.next_cursor, + count: page.count, + }) } - #[tracing_async] pub(crate) async fn query_cells( &self, - _ctx: Context, out_point: Option, - lock_hashes: Vec, - type_hashes: Vec, + lock_hashes: Vec, + type_hashes: Vec, block_range: Option, limit_cellbase: bool, pagination: PaginationRequest, @@ -574,33 +603,23 @@ impl RelationalStorage { let mut is_ok = true; let lock_hash: H256 = cell.cell_output.lock().calc_script_hash().unpack(); - let lock_hash = to_rb_bytes(&lock_hash.0); if !lock_hashes.is_empty() { is_ok &= lock_hashes.contains(&lock_hash) }; - if let Some(type_script) = cell.cell_output.type_().to_opt() { let type_hash: H256 = type_script.calc_script_hash().unpack(); - let type_hash = to_rb_bytes(&type_hash.0); if !type_hashes.is_empty() { is_ok &= type_hashes.contains(&type_hash) }; } else if !type_hashes.is_empty() { - let default_hashes = vec![H256::default()] - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - is_ok &= type_hashes == default_hashes + is_ok &= type_hashes == vec![H256::default()] } - if let Some(range) = block_range { is_ok &= range.is_in(cell.block_number); } - if limit_cellbase { is_ok &= cell.tx_index == 0; } - let mut response: Vec = vec![]; if is_ok { response.push(cell); @@ -617,274 +636,607 @@ impl RelationalStorage { }); } - let mut wrapper = self.pool.wrapper(); - + let mut query_builder = SqlBuilder::select_from("mercury_cell"); + query_builder.field("*"); if !lock_hashes.is_empty() { - wrapper = wrapper.in_array("lock_hash", &lock_hashes); + query_builder + .and_where_in("lock_hash", &sqlx_param_placeholders(1..lock_hashes.len())?); } - if !type_hashes.is_empty() { - wrapper = wrapper.and().in_array("type_hash", &type_hashes); - } - - if let Some(range) = block_range { - wrapper = wrapper - .and() - .push_sql("(") - .between("block_number", range.min(), range.max()) - .or() - .between("consumed_block_number", range.min(), range.max()) - .push_sql(")"); + query_builder.and_where_in( + "type_hash", + &sqlx_param_placeholders( + lock_hashes.len() + 1..lock_hashes.len() + type_hashes.len(), + )?, + ); } - if limit_cellbase { - wrapper = wrapper.and().eq("tx_index", 0) + query_builder.and_where_eq("tx_index", 0i32); } + if let Some(ref range) = block_range { + query_builder + .and_where_between( + "block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ) + .or_where_between( + "consumed_block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ); + } + let (sql, sql_for_total) = build_query_page_sql(query_builder, &pagination)?; + + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in &lock_hashes { + query = query.bind(hash.as_bytes()); + } + for hash in &type_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + let query_total = bind(&sql_for_total); - let mut conn = self.pool.acquire().await?; - let cells: Page = conn - .fetch_page_by_wrapper(wrapper, &PageRequest::from(pagination.clone())) + // fetch + let page = self + .sqlx_pool + .fetch_page(query, query_total, &pagination) .await?; - let next_cursor = build_next_cursor!(cells, pagination); - let res = cells.records.into_iter().map(Into::into).collect(); - Ok(to_pagination_response( - res, - next_cursor, - if pagination.return_count { - Some(cells.total) - } else { - None - }, - )) + let mut cells = vec![]; + for row in page.response { + cells.push(build_detailed_cell(row)?); + } + Ok(PaginationResponse { + response: cells, + next_cursor: page.next_cursor, + count: page.count, + }) } - #[tracing_async] pub(crate) async fn query_historical_live_cells( &self, - _ctx: Context, - lock_hashes: Vec, - type_hashes: Vec, + lock_hashes: Vec, + type_hashes: Vec, tip_block_number: u64, out_point: Option, pagination: PaginationRequest, ) -> Result> { - let mut w = self - .pool - .wrapper() - .le("block_number", tip_block_number) - .and() - .push_sql("(") - .gt("consumed_block_number", tip_block_number) - .or() - .is_null("consumed_block_number") - .push_sql(")") - .and() - .in_array("lock_hash", &lock_hashes); - if !type_hashes.is_empty() { - w = w.and().in_array("type_hash", &type_hashes); + if lock_hashes.is_empty() && type_hashes.is_empty() && out_point.is_none() { + return Err(DBError::InvalidParameter( + "no valid parameter to query historical live cells".to_owned(), + ) + .into()); + } + + if let Some(op) = out_point { + let cell = self.query_cell_by_out_point(op).await?; + + let mut is_ok = true; + let lock_hash: H256 = cell.cell_output.lock().calc_script_hash().unpack(); + if !lock_hashes.is_empty() { + is_ok &= lock_hashes.contains(&lock_hash) + }; + if let Some(type_script) = cell.cell_output.type_().to_opt() { + let type_hash: H256 = type_script.calc_script_hash().unpack(); + if !type_hashes.is_empty() { + is_ok &= type_hashes.contains(&type_hash) + }; + } else if !type_hashes.is_empty() { + is_ok &= type_hashes == vec![H256::default()] + } + is_ok &= cell.block_number <= tip_block_number; + if let Some(consumed_block_number) = cell.consumed_block_number { + is_ok &= consumed_block_number > tip_block_number + } + let mut response: Vec = vec![]; + if is_ok { + response.push(cell); + } + let count = response.len() as u64; + return Ok(PaginationResponse { + response, + next_cursor: None, + count: if pagination.return_count { + Some(count) + } else { + None + }, + }); } - if let Some(out_point) = out_point { - let tx_hash: H256 = out_point.tx_hash().unpack(); - let output_index: u32 = out_point.index().unpack(); - w = w - .and() - .eq("tx_hash", to_rb_bytes(&tx_hash.0)) - .and() - .eq("output_index", output_index); + + let mut query_builder = SqlBuilder::select_from("mercury_cell"); + query_builder + .field("*") + .and_where_le("block_number", tip_block_number) + .and_where_gt("consumed_block_number", tip_block_number) + .or_where_is_null("consumed_block_number"); + if !lock_hashes.is_empty() { + query_builder + .and_where_in("lock_hash", &sqlx_param_placeholders(1..lock_hashes.len())?); } + if !type_hashes.is_empty() { + query_builder.and_where_in( + "type_hash", + &sqlx_param_placeholders( + lock_hashes.len() + 1..lock_hashes.len() + type_hashes.len(), + )?, + ); + } + let (sql, sql_for_total) = build_query_page_sql(query_builder, &pagination)?; - let mut conn = self.pool.acquire().await?; + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in &lock_hashes { + query = query.bind(hash.as_bytes()); + } + for hash in &type_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + let query_total = bind(&sql_for_total); - let cells: Page = conn - .fetch_page_by_wrapper(w, &PageRequest::from(pagination.clone())) + // fetch + let page = self + .sqlx_pool + .fetch_page(query, query_total, &pagination) .await?; - let next_cursor = build_next_cursor!(cells, pagination); - let res = cells.records.into_iter().map(Into::into).collect(); - Ok(to_pagination_response( - res, - next_cursor, - if pagination.return_count { - Some(cells.total) - } else { - None - }, - )) + let mut cells = vec![]; + for row in page.response { + cells.push(build_detailed_cell(row)?); + } + Ok(PaginationResponse { + response: cells, + next_cursor: page.next_cursor, + count: page.count, + }) } - // TODO: query refactoring - async fn query_tip_block(&self) -> Result { - let wrapper = self - .pool - .wrapper() - .order_by(false, &["block_number"]) - .limit(1); - let block: Option = self.pool.fetch_by_wrapper(wrapper).await?; - let block = match block { - Some(block) => block, - None => return Err(DBError::NotExist("tip block".to_string()).into()), - }; - Ok(block) + async fn query_tip_block(&self) -> Result { + let query = SQLXPool::new_query( + r#" + SELECT * FROM mercury_block + ORDER BY block_number + DESC + "#, + ); + self.sqlx_pool.fetch_one(query).await } - async fn query_block_by_hash(&self, block_hash: RbBytes) -> Result { - let block: Option = - self.pool.fetch_by_column("block_hash", &block_hash).await?; - let block = match block { - Some(block) => block, - None => { - return Err(DBError::NotExist(format!( - "block with hash {:?}", - rb_bytes_to_h256(&block_hash).to_string() - )) - .into()) - } - }; - Ok(block) + async fn query_block_by_hash(&self, block_hash: H256) -> Result { + let query = SQLXPool::new_query( + r#" + SELECT * FROM mercury_block + WHERE block_hash = $1 + "#, + ) + .bind(block_hash.as_bytes()); + self.sqlx_pool.fetch_one(query).await + } + + pub(crate) async fn query_block_by_number(&self, block_number: BlockNumber) -> Result { + let query = SQLXPool::new_query( + r#" + SELECT * FROM mercury_block + WHERE block_number = $1 + "#, + ) + .bind(i64::try_from(block_number)?); + self.sqlx_pool.fetch_one(query).await } - pub(crate) async fn query_indexer_cells( + async fn query_tip_simple_block(&self) -> Result<(H256, BlockNumber, H256, u64)> { + let query = SQLXPool::new_query( + r#" + SELECT block_hash, block_number, parent_hash, block_timestamp + FROM mercury_block + ORDER BY block_number + DESC + "#, + ); + self.sqlx_pool.fetch_one(query).await.map(to_simple_block) + } + + async fn query_simple_block_by_hash( + &self, + block_hash: H256, + ) -> Result<(H256, BlockNumber, H256, u64)> { + let query = SQLXPool::new_query( + r#" + SELECT block_hash, block_number, parent_hash, block_timestamp + FROM mercury_block + WHERE block_hash = $1 + "#, + ) + .bind(block_hash.as_bytes()); + self.sqlx_pool.fetch_one(query).await.map(to_simple_block) + } + + async fn query_simple_block_by_number( + &self, + block_number: BlockNumber, + ) -> Result<(H256, BlockNumber, H256, u64)> { + let query = SQLXPool::new_query( + r#" + SELECT block_hash, block_number, parent_hash, block_timestamp + FROM mercury_block + WHERE block_number = $1 + "#, + ) + .bind(i64::try_from(block_number)?); + self.sqlx_pool.fetch_one(query).await.map(to_simple_block) + } + + pub(crate) async fn query_indexer_transactions( &self, lock_hashes: Vec, type_hashes: Vec, block_range: Option, pagination: PaginationRequest, - ) -> Result> { - let mut w = self.pool.wrapper(); - - if let Some(range) = block_range { - w = w.between("block_number", range.min(), range.max()); + ) -> Result> { + if lock_hashes.is_empty() && type_hashes.is_empty() && block_range.is_none() { + return Err(DBError::InvalidParameter( + "no valid parameter to query indexer transactions".to_owned(), + ) + .into()); } + let mut query_builder = SqlBuilder::select_from("mercury_indexer_cell"); + query_builder.field("id, block_number, io_type, io_index, tx_hash, tx_index"); if !lock_hashes.is_empty() { - let lock_hashes = lock_hashes - .iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - w = w.and().r#in("lock_hash", &lock_hashes); + query_builder + .and_where_in("lock_hash", &sqlx_param_placeholders(1..lock_hashes.len())?); } - if !type_hashes.is_empty() { - let type_hashes = type_hashes - .iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - w = w.and().r#in("type_hash", &type_hashes); + query_builder.and_where_in( + "type_hash", + &sqlx_param_placeholders( + lock_hashes.len() + 1..lock_hashes.len() + type_hashes.len(), + )?, + ); } + if let Some(ref range) = block_range { + query_builder.and_where_between( + "block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ); + } + let (sql, sql_for_total) = build_query_page_sql(query_builder, &pagination)?; - let mut conn = self.pool.acquire().await?; - let res: Page = conn - .fetch_page_by_wrapper(w, &PageRequest::from(pagination.clone())) - .await?; - let next_cursor = build_next_cursor!(res, pagination); + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in &lock_hashes { + query = query.bind(hash.as_bytes()); + } + for hash in &type_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + let query_total = bind(&sql_for_total); - Ok(to_pagination_response( - res.records, - next_cursor, - if pagination.return_count { - Some(res.total) - } else { - None - }, - )) + // fetch + let page = self + .sqlx_pool + .fetch_page(query, query_total, &pagination) + .await?; + let mut cells = vec![]; + for row in page.response { + cells.push(build_indexer_transaction(row)?); + } + Ok(PaginationResponse { + response: cells, + next_cursor: page.next_cursor, + count: page.count, + }) } - pub(crate) async fn query_block_by_number( + pub(crate) async fn query_transactions_by_block_hash( &self, - block_number: BlockNumber, - ) -> Result { - let block: Option = self - .pool - .fetch_by_column("block_number", &block_number) - .await?; - let block = match block { - Some(block) => block, - None => return Err(DBError::WrongHeight.into()), - }; - Ok(block) + block_hash: &[u8], + ) -> Result> { + let query = SQLXPool::new_query( + r#" + SELECT * FROM mercury_transaction + WHERE block_hash = $1 + ORDER BY tx_index + ASC + "#, + ) + .bind(block_hash); + self.sqlx_pool.fetch_all(query).await } - pub(crate) async fn query_transactions_by_block_hash( + pub(crate) async fn query_transaction_hashes_by_block_hash( &self, - block_hash: &RbBytes, - ) -> Result> { - let w = self - .pool - .wrapper() - .eq("block_hash", block_hash) - .order_by(true, &["tx_index"]); - let txs: Vec = self.pool.fetch_list_by_wrapper(w).await?; - Ok(txs) + block_hash: &[u8], + ) -> Result> { + let query = SQLXPool::new_query( + r#" + SELECT tx_hash FROM mercury_transaction + WHERE block_hash = $1 + ORDER BY tx_index + ASC + "#, + ) + .bind(block_hash); + self.sqlx_pool.fetch_all(query).await.map(|tx| { + tx.into_iter() + .map(|tx| bytes_to_h256(tx.get("tx_hash"))) + .collect() + }) } - #[tracing_async] pub(crate) async fn query_transactions( &self, _ctx: Context, - tx_hashes: Vec, + tx_hashes: Vec, block_range: Option, pagination: PaginationRequest, - ) -> Result> { - let mut wrapper = self.pool.wrapper(); + ) -> Result> { + if tx_hashes.is_empty() && block_range.is_none() && pagination.limit == Some(0) { + return Err(DBError::InvalidParameter( + "no valid parameter to query transactions".to_owned(), + ) + .into()); + } + // build query str + let mut query_builder = SqlBuilder::select_from("mercury_transaction"); + query_builder.field("*"); if !tx_hashes.is_empty() { - wrapper = wrapper.in_array("tx_hash", &tx_hashes) + query_builder.and_where_in("tx_hash", &sqlx_param_placeholders(1..tx_hashes.len())?); } - - if let Some(range) = block_range { - wrapper = wrapper.between("block_number", range.min(), range.max()); + if let Some(ref range) = block_range { + query_builder.and_where_between( + "block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ); } + let (sql, sql_for_total) = build_query_page_sql(query_builder, &pagination)?; - let mut conn = self.pool.acquire().await?; - let txs: Page = conn - .fetch_page_by_wrapper(wrapper, &PageRequest::from(pagination.clone())) - .await?; - let next_cursor = build_next_cursor!(txs, pagination); + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in &tx_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + let query_total = bind(&sql_for_total); - Ok(to_pagination_response( - txs.records, - next_cursor, - if pagination.return_count { - Some(txs.total) - } else { - None - }, - )) + // fetch + self.sqlx_pool + .fetch_page(query, query_total, &pagination) + .await } - async fn query_txs_output_cells(&self, tx_hashes: &[RbBytes]) -> Result> { + async fn query_txs_output_cells(&self, tx_hashes: &[Vec]) -> Result> { if tx_hashes.is_empty() { return Ok(Vec::new()); } - let w = self - .pool - .wrapper() - .r#in("tx_hash", tx_hashes) - .order_by(true, &["output_index"]); - let cells: Vec = self.pool.fetch_list_by_wrapper(w).await?; - Ok(cells) + // build query str + let mut query_builder = SqlBuilder::select_from("mercury_cell"); + let sql = query_builder + .field("*") + .and_where_in("tx_hash", &sqlx_param_placeholders(1..tx_hashes.len())?) + .order_by("output_index", false) + .sql()?; + + // bind + let mut query = SQLXPool::new_query(&sql); + for hash in tx_hashes { + query = query.bind(hash); + } + + // fetch + self.sqlx_pool.fetch(query).await + } + + pub(crate) async fn query_transaction_hashes_by_scripts( + &self, + lock_hashes: &[H256], + type_hashes: &[H256], + block_range: &Option, + limit_cellbase: bool, + pagination: &PaginationRequest, + ) -> Result> { + if lock_hashes.is_empty() && type_hashes.is_empty() && block_range.is_none() { + return Err(DBError::InvalidParameter( + "no valid parameter to query transaction hashes".to_owned(), + ) + .into()); + } + + // query str + let mut query_builder = SqlBuilder::select_from("mercury_indexer_cell"); + query_builder.field("DISTINCT ON (tx_hash) tx_hash, id"); + if !lock_hashes.is_empty() { + query_builder + .and_where_in("lock_hash", &sqlx_param_placeholders(1..lock_hashes.len())?); + } + if !type_hashes.is_empty() { + query_builder.and_where_in( + "type_hash", + &sqlx_param_placeholders( + lock_hashes.len() + 1..lock_hashes.len() + type_hashes.len(), + )?, + ); + } + if limit_cellbase { + query_builder.and_where_eq("tx_index", 0); + } + if let Some(ref range) = block_range { + query_builder.and_where_between( + "block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ); + } + if let Some(id) = pagination.cursor { + let id = i64::try_from(id).unwrap_or(i64::MAX); + match pagination.order { + Order::Asc => query_builder.and_where_gt("id", id), + Order::Desc => query_builder.and_where_lt("id", id), + }; + } + match pagination.order { + Order::Asc => query_builder.order_by("tx_hash, id", true), + Order::Desc => query_builder.order_by("tx_hash, id", false), + }; + let sql_sub_query = query_builder.subquery()?; + + let mut query_builder = SqlBuilder::select_from(&format!("{} res", sql_sub_query)); + query_builder.field("*"); + match pagination.order { + Order::Asc => query_builder.order_by("id", false), + Order::Desc => query_builder.order_by("id", true), + }; + query_builder.limit(pagination.limit.unwrap_or(u16::MAX)); + let sql = query_builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in lock_hashes { + query = query.bind(hash.as_bytes()); + } + for hash in type_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + + // fetch + let response = self.sqlx_pool.fetch(query).await?; + Ok(response + .into_iter() + .map(|row| { + ( + bytes_to_h256(row.get("tx_hash")), + row.get::("id") as u64, + ) + }) + .collect()) + } + + pub(crate) async fn query_distinct_tx_hashes_count( + &self, + lock_hashes: &[H256], + type_hashes: &[H256], + block_range: &Option, + limit_cellbase: bool, + ) -> Result { + if lock_hashes.is_empty() && type_hashes.is_empty() && block_range.is_none() { + return Err(DBError::InvalidParameter( + "no valid parameter to query distinct tx_hashes count".to_owned(), + ) + .into()); + } + + // query str + let mut query_builder = SqlBuilder::select_from("mercury_indexer_cell"); + query_builder.field("COUNT(DISTINCT tx_hash) AS count"); + if !lock_hashes.is_empty() { + query_builder + .and_where_in("lock_hash", &sqlx_param_placeholders(1..lock_hashes.len())?); + } + if !type_hashes.is_empty() { + query_builder.and_where_in( + "type_hash", + &sqlx_param_placeholders( + lock_hashes.len() + 1..lock_hashes.len() + type_hashes.len(), + )?, + ); + } + if limit_cellbase { + query_builder.and_where_eq("tx_index", 0); + } + if let Some(ref range) = block_range { + query_builder.and_where_between( + "block_number", + range.from.min(i32::MAX as u64), + range.to.min(i32::MAX as u64), + ); + } + let sql = query_builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let bind = |sql| { + let mut query = SQLXPool::new_query(sql); + for hash in lock_hashes { + query = query.bind(hash.as_bytes()); + } + for hash in type_hashes { + query = query.bind(hash.as_bytes()); + } + query + }; + let query = bind(&sql); + + // fetch + self.sqlx_pool + .fetch_one(query) + .await + .map(|row| row.get::("count") as u64) } - async fn query_txs_input_cells(&self, tx_hashes: &[RbBytes]) -> Result> { + async fn query_txs_input_cells(&self, tx_hashes: &[Vec]) -> Result> { if tx_hashes.is_empty() { return Ok(Vec::new()); } - let w = self - .pool - .wrapper() - .r#in("consumed_tx_hash", tx_hashes) - .order_by(true, &["input_index"]); - let cells: Vec = self.pool.fetch_list_by_wrapper(w).await?; - Ok(cells) + // build query str + let mut query_builder = SqlBuilder::select_from("mercury_cell"); + let sql = query_builder + .field("*") + .and_where_in( + "consumed_tx_hash", + &sqlx_param_placeholders(1..tx_hashes.len())?, + ) + .order_by("input_index", false) + .sql()?; + + // bind + let mut query = SQLXPool::new_query(&sql); + for hash in tx_hashes { + query = query.bind(hash); + } + + // fetch + self.sqlx_pool.fetch(query).await } pub(crate) async fn query_registered_address( &self, - lock_hash: RbBytes, - ) -> Result> { - let address = self.pool.fetch_by_column("lock_hash", &lock_hash).await?; - Ok(address) + lock_hash: &[u8], + ) -> Result> { + let query = SQLXPool::new_query( + r#" + SELECT address + FROM mercury_registered_address + WHERE lock_hash = $1 + "#, + ) + .bind(lock_hash); + self.sqlx_pool + .fetch_optional(query) + .await + .map(|row| row.map(|row| row.get::("address"))) } } @@ -902,37 +1254,39 @@ fn build_block_view( .build() } -fn build_header_view(block: &BlockTable) -> HeaderView { - let epoch = if block.block_number == 0 { +fn build_header_view(block: &AnyRow) -> HeaderView { + let epoch = if block.get::("block_number") == 0 { 0u64.pack() } else { EpochNumberWithFraction::new( - block.epoch_number.into(), - block.epoch_index as u64, - block.epoch_length as u64, + block.get::("epoch_number") as u64, + block.get::("epoch_index") as u64, + block.get::("epoch_length") as u64, ) .full_value() .pack() }; HeaderBuilder::default() - .number(block.block_number.pack()) + .number((block.get::("block_number") as u64).pack()) .parent_hash(packed::Byte32::new(to_fixed_array( - &block.parent_hash.inner, + &block.get::, _>("parent_hash"), ))) - .compact_target(block.compact_target.pack()) - .nonce(utils::decode_nonce(&block.nonce.inner).pack()) - .timestamp(block.block_timestamp.pack()) - .version((block.version as u32).pack()) + .compact_target((block.get::("compact_target") as u32).pack()) + .nonce(utils::decode_nonce(block.get("nonce")).pack()) + .timestamp((block.get::("block_timestamp") as u64).pack()) + .version((block.get::("version") as u32).pack()) .epoch(epoch) - .dao(packed::Byte32::new(to_fixed_array(&block.dao.inner[0..32]))) + .dao(packed::Byte32::new(to_fixed_array( + &block.get::, _>("dao")[0..32], + ))) .transactions_root(packed::Byte32::new(to_fixed_array( - &block.transactions_root.inner[0..32], + &block.get::, _>("transactions_root")[0..32], ))) .proposals_hash(packed::Byte32::new(to_fixed_array( - &block.proposals_hash.inner[0..32], + &block.get::, _>("proposals_hash")[0..32], ))) .extra_hash(packed::Byte32::new(to_fixed_array( - &block.uncles_hash.inner[0..32], + &block.get::, _>("uncles_hash")[0..32], ))) .build() } @@ -964,52 +1318,6 @@ fn build_cell_base_input(block_number: u64) -> packed::CellInput { .build() } -fn build_cell_inputs(input_cells: Iter) -> Vec { - input_cells - .map(|cell| { - let out_point = packed::OutPointBuilder::default() - .tx_hash( - packed::Byte32::from_slice(&cell.tx_hash.inner) - .expect("impossible: fail to pack since"), - ) - .index((cell.output_index as u32).pack()) - .build(); - - packed::CellInputBuilder::default() - .since(decode_since(&cell.since.inner).pack()) - .previous_output(out_point) - .build() - }) - .collect() -} - -fn build_cell_outputs(cells: &[CellTable]) -> (Vec, Vec) { - ( - cells - .iter() - .map(|cell| { - let lock_script: packed::Script = cell.to_lock_script_table().into(); - let type_script_opt = build_script_opt(if cell.has_type_script() { - Some(cell.to_type_script_table()) - } else { - None - }); - packed::CellOutputBuilder::default() - .capacity(cell.capacity.pack()) - .lock(lock_script) - .type_(type_script_opt) - .build() - }) - .collect(), - cells.iter().map(|cell| cell.data.inner.pack()).collect(), - ) -} - -fn build_script_opt(script_opt: Option) -> packed::ScriptOpt { - let script_opt = script_opt.map(|script| script.into()); - packed::ScriptOptBuilder::default().set(script_opt).build() -} - fn build_transaction_view( version: u32, witnesses: packed::BytesVec, @@ -1042,6 +1350,131 @@ pub fn to_pagination_response( } } -pub fn rb_bytes_to_h256(input: &RbBytes) -> H256 { - H256::from_slice(&input.inner[0..32]).expect("rb bytes to h256") +pub(crate) fn bytes_to_h256(input: &[u8]) -> H256 { + H256::from_slice(&input[0..32]).expect("bytes to h256") +} + +fn to_simple_block(block: AnyRow) -> (H256, BlockNumber, H256, u64) { + ( + bytes_to_h256(block.get("block_hash")), + block + .get::("block_number") + .try_into() + .expect("i32 to u64"), + bytes_to_h256(block.get("parent_hash")), + block + .get::("block_timestamp") + .try_into() + .expect("i64 to u64"), + ) +} + +pub(crate) fn sqlx_param_placeholders(range: std::ops::Range) -> Result> { + if range.start == 0 { + return Err(DBError::InvalidParameter("no valid parameter".to_owned()).into()); + } + Ok((1..=range.end) + .map(|i| format!("${}", i)) + .collect::>()) +} + +fn build_detailed_cell(row: AnyRow) -> Result { + let lock_script = packed::ScriptBuilder::default() + .code_hash(to_fixed_array::<32>(&row.get::, _>("lock_code_hash")[0..32]).pack()) + .args(row.get::, _>("lock_args").pack()) + .hash_type(packed::Byte::new( + row.get::("lock_script_type").try_into()?, + )) + .build(); + let type_script = if row.get::, _>("type_hash") == H256::default().as_bytes() { + None + } else { + Some( + packed::ScriptBuilder::default() + .code_hash(H256::from_slice(row.get("type_code_hash"))?.pack()) + .args(row.get::, _>("type_args").pack()) + .hash_type(packed::Byte::new( + row.get::("type_script_type").try_into()?, + )) + .build(), + ) + }; + + let convert_hash = |hash: Option>| -> Option { + if let Some(hash) = hash { + if hash.is_empty() { + None + } else { + Some(H256::from_slice(&hash).expect("convert hash")) + } + } else { + None + } + }; + + let convert_since = |b: Option>| -> Option { + if let Some(b) = b { + if b.is_empty() { + None + } else { + Some(u64::from_be_bytes(to_fixed_array::<8>(&b))) + } + } else { + None + } + }; + + let cell = DetailedCell { + epoch_number: EpochNumberWithFraction::new_unchecked( + row.get::("epoch_number").try_into()?, + row.get::("epoch_index").try_into()?, + row.get::("epoch_length").try_into()?, + ) + .full_value(), + block_number: row.get::("block_number").try_into()?, + block_hash: H256::from_slice(&row.get::, _>("block_hash")[0..32])?, + tx_index: row.get::("tx_index").try_into()?, + out_point: packed::OutPointBuilder::default() + .tx_hash(to_fixed_array::<32>(&row.get::, _>("tx_hash")).pack()) + .index(u32::try_from(row.get::("output_index"))?.pack()) + .build(), + cell_output: packed::CellOutputBuilder::default() + .lock(lock_script) + .type_(type_script.pack()) + .capacity(u64::try_from(row.get::("capacity"))?.pack()) + .build(), + cell_data: row.get::, _>("data").into(), + + // The following fields are in the mercury_cell table, but not in the mercury_live_cell table + consumed_block_hash: convert_hash(row.try_get("consumed_block_hash").ok()), + consumed_block_number: row + .try_get::, _>("consumed_block_number") + .unwrap_or(None) + .map(|block_number| block_number as u64), + consumed_tx_hash: convert_hash(row.try_get("consumed_tx_hash").ok()), + consumed_tx_index: row + .try_get::, _>("consumed_tx_index") + .unwrap_or(None) + .map(|block_number| block_number as u32), + consumed_input_index: row + .try_get::, _>("input_index") + .unwrap_or(None) + .map(|block_number| block_number as u32), + since: convert_since(row.try_get("since").ok()), + }; + Ok(cell) +} + +fn build_indexer_transaction(row: AnyRow) -> Result { + Ok(Transaction { + block_number: u64::try_from(row.get::("block_number"))?.into(), + tx_index: u32::try_from(row.get::("tx_index"))?.into(), + io_index: u32::try_from(row.get::("io_index"))?.into(), + tx_hash: bytes_to_h256(row.get("tx_hash")), + io_type: if u8::try_from(row.get::("io_type"))? == 0 { + IOType::Input + } else { + IOType::Output + }, + }) } diff --git a/core/storage/src/relational/insert.rs b/core/storage/src/relational/insert.rs index cf3f79af2..aa284c2b0 100644 --- a/core/storage/src/relational/insert.rs +++ b/core/storage/src/relational/insert.rs @@ -1,20 +1,26 @@ -use crate::relational::table::{ - BlockTable, CanonicalChainTable, CellTable, ConsumedInfo, IndexerCellTable, LiveCellTable, - RegisteredAddressTable, ScriptTable, SyncStatus, TransactionTable, IO_TYPE_INPUT, - IO_TYPE_OUTPUT, -}; -use crate::relational::{generate_id, sql, to_rb_bytes, RelationalStorage}; +use crate::relational::{generate_id, RelationalStorage}; -use common::{Context, Result}; -use common_logger::tracing_async; -use db_xsql::rbatis::{crud::CRUDMut, executor::RBatisTxExecutor, Bytes as RbBytes}; +use common::Result; +use db_sqlx::SQLXPool; use ckb_types::core::{BlockView, EpochNumberWithFraction, TransactionView}; -use ckb_types::{prelude::*, H256}; - -use std::collections::{HashMap, HashSet}; - -pub const BATCH_SIZE_THRESHOLD: usize = 1_500; +use ckb_types::{prelude::*, H160, H256}; +use seq_macro::seq; +use sql_builder::SqlBuilder; +use sqlx::{Any, Row, Transaction}; +use std::collections::HashSet; + +// Note that every database has a practical limit on the number of bind parameters you can add to a single query. +// This varies by database. +// https://docs.rs/sqlx/0.6.1/sqlx/struct.QueryBuilder.html#note-database-specific-limits +// BATCH_SIZE_THRESHOLD represents the number of rows that can be bound in an insert sql execution. +// The number of columns in each row multiplied by this BATCH_SIZE_THRESHOLD yields the total number of bound parameters, +// which should be within the above limits. +pub const BATCH_SIZE_THRESHOLD: usize = 1_000; + +pub const BLAKE_160_HSAH_LEN: usize = 20; +pub const IO_TYPE_INPUT: u8 = 0; +pub const IO_TYPE_OUTPUT: u8 = 1; #[macro_export] macro_rules! save_batch_slice { @@ -27,335 +33,683 @@ macro_rules! save_batch_slice { } impl RelationalStorage { - #[tracing_async] pub(crate) async fn insert_block_table( &self, - _ctx: Context, block_view: &BlockView, - tx: &mut RBatisTxExecutor<'_>, + tx: &mut Transaction<'_, Any>, ) -> Result<()> { - let block_hash = to_rb_bytes(&block_view.hash().raw_data()); + // insert mercury_block and mercury_canonical_chain + bulk_insert_blocks(&[block_view.to_owned()], tx).await?; - tx.save(&BlockTable::from(block_view), &[]).await?; - tx.save(&SyncStatus::new(block_view.number()), &[]).await?; - tx.save( - &CanonicalChainTable::new(block_view.number(), block_hash), - &[], + // insert mercury_sync_status + SQLXPool::new_query( + r#"INSERT INTO mercury_sync_status(block_number) + VALUES ($1)"#, ) + .bind(i32::try_from(block_view.number())?) + .execute(&mut *tx) .await?; Ok(()) } - #[tracing_async] pub(crate) async fn insert_transaction_table( &self, - _ctx: Context, block_view: &BlockView, - tx: &mut RBatisTxExecutor<'_>, + tx: &mut Transaction<'_, Any>, ) -> Result<()> { - let txs = block_view.transactions(); let block_number = block_view.number(); - let block_hash = to_rb_bytes(&block_view.hash().raw_data()); + let block_hash = block_view.hash().raw_data().to_vec(); let block_timestamp = block_view.timestamp(); - let mut output_cell_set = Vec::new(); - let mut live_cell_set = Vec::new(); - let mut tx_set = Vec::new(); - let mut script_set = HashSet::new(); - let mut consumed_infos = Vec::new(); - let mut indexer_cells = Vec::new(); - - for (idx, transaction) in txs.iter().enumerate() { - let index = idx as u32; - - tx_set.push(TransactionTable::from_view( - transaction, - generate_id(block_number), - index, - block_hash.clone(), - block_number, - block_timestamp, - )); - - self.insert_cell_table( - transaction, - index, - block_view, - tx, - &mut output_cell_set, - &mut live_cell_set, - &mut script_set, - &mut consumed_infos, - &mut indexer_cells, - ) - .await?; - } - - let script_batch = script_set.iter().cloned().collect::>(); - save_batch_slice!(tx, tx_set, output_cell_set, live_cell_set, script_batch); - - self.update_consumed_cells(&consumed_infos, tx).await?; - self.fill_and_save_indexer_cells(block_number, indexer_cells, &consumed_infos, tx) - .await?; + let epoch = block_view.epoch(); + let tx_views = block_view.transactions(); - Ok(()) + bulk_insert_transactions(block_number, &block_hash, block_timestamp, &tx_views, tx).await?; + bulk_insert_output_cells(block_number, &block_hash, epoch, &tx_views, true, tx).await?; + bulk_insert_scripts(&tx_views, tx).await?; + update_consumed_cells(block_number, &block_hash, &tx_views, tx).await?; + bulk_insert_indexer_cells(block_number, &tx_views, tx).await } - async fn fill_and_save_indexer_cells( + pub(crate) async fn bulk_insert_registered_address_table( &self, - block_number: u64, - mut indexer_cells: Vec, - infos: &[ConsumedInfo], - tx: &mut RBatisTxExecutor<'_>, - ) -> Result<()> { - for info in infos.iter() { - let tx_hash: H256 = info.out_point.tx_hash().unpack(); - let output_index: u32 = info.out_point.index().unpack(); - let w = self - .pool - .wrapper() - .eq("tx_hash", to_rb_bytes(&tx_hash.0)) - .and() - .eq("output_index", output_index); - let cell_table = tx.fetch_by_wrapper::(w).await?; - let indexer_cell = IndexerCellTable::new_with_empty_scripts( - block_number, - IO_TYPE_INPUT, - info.input_index, - info.consumed_tx_hash.clone(), - info.consumed_tx_index, + addresses: Vec<(H160, String)>, + ) -> Result> { + let mut to_be_inserted = vec![]; + for (lock_hash_160, address) in addresses.iter() { + if self + .query_registered_address(lock_hash_160.as_bytes()) + .await? + .is_none() + { + to_be_inserted.push((lock_hash_160.as_bytes().to_vec(), address)); + } + } + let mut tx = self.sqlx_pool.transaction().await?; + for start in (0..to_be_inserted.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(to_be_inserted.len()); + + // build query str + let mut builder = SqlBuilder::insert_into("mercury_registered_address"); + builder.field( + r#"lock_hash, + address"#, ); - indexer_cells.push(indexer_cell.update_by_cell_table(&cell_table)); + push_values_placeholders(&mut builder, 2, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in to_be_inserted[start..end].iter() { + seq!(i in 0..2 { + query = query.bind(&row.i); + }); + } + + // execute + query.execute(&mut *tx).await?; } + tx.commit().await?; - indexer_cells.sort(); - indexer_cells - .iter_mut() - .for_each(|c| c.id = generate_id(block_number)); + Ok(addresses.into_iter().map(|(hash, _)| hash).collect()) + } +} - save_batch_slice!(tx, indexer_cells); +pub fn push_values_placeholders( + builder: &mut SqlBuilder, + column_number: usize, + rows_number: usize, +) { + let mut placeholder_idx = 1usize; + for _ in 0..rows_number { + let values = (placeholder_idx..placeholder_idx + column_number) + .map(|i| format!("${}", i)) + .collect::>(); + builder.values(&values); + placeholder_idx += column_number; + } +} - Ok(()) +pub async fn bulk_insert_blocks( + block_views: &[BlockView], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + let block_rows: Vec<_> = block_views + .iter() + .map(|block_view| { + ( + block_view.hash().raw_data().to_vec(), + block_view.number() as i32, + block_view.version() as i16, + block_view.compact_target() as i32, + block_view.timestamp() as i64, + block_view.epoch().number() as i32, + block_view.epoch().index() as i32, + block_view.epoch().length() as i32, + block_view.parent_hash().raw_data().to_vec(), + block_view.transactions_root().raw_data().to_vec(), + block_view.proposals_hash().raw_data().to_vec(), + block_view.extra_hash().raw_data().to_vec(), + block_view.uncles().data().as_slice().to_vec(), + block_view.uncle_hashes().len() as i32, + block_view.dao().raw_data().to_vec(), + block_view.nonce().to_be_bytes().to_vec(), + block_view.data().proposals().as_bytes().to_vec(), + ) + }) + .collect(); + + for start in (0..block_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(block_rows.len()); + + // insert mercury_block + // build query str + let mut builder = SqlBuilder::insert_into("mercury_block"); + builder.field( + r#" + block_hash, + block_number, + version, + compact_target, + block_timestamp, + epoch_number, + epoch_index, + epoch_length, + parent_hash, + transactions_root, + proposals_hash, + uncles_hash, + uncles, + uncles_count, + dao, + nonce, + proposals"#, + ); + push_values_placeholders(&mut builder, 17, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in block_rows[start..end].iter() { + seq!(i in 0..17 { + query = query.bind(&row.i); + }); + } + + // execute + query.execute(&mut *tx).await?; + + // insert mercury_canonical_chain + // build query str + let mut builder = SqlBuilder::insert_into("mercury_canonical_chain"); + builder.field("block_hash, block_number"); + push_values_placeholders(&mut builder, 2, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in block_rows[start..end].iter() { + seq!(i in 0..2 { + query = query.bind(&row.i); + }); + } + + // execute + query.execute(&mut *tx).await?; } - async fn update_consumed_cells( - &self, - infos: &[ConsumedInfo], - tx: &mut RBatisTxExecutor<'_>, - ) -> Result<()> { - for info in infos.iter() { - let tx_hash = to_rb_bytes(&info.out_point.tx_hash().raw_data()); - let output_index: u32 = info.out_point.index().unpack(); + Ok(()) +} - self.remove_live_cell_by_out_point(&info.out_point, tx) - .await?; - sql::update_consume_cell( - tx, - &info.consumed_block_number, - &info.consumed_block_hash.clone(), - &info.consumed_tx_hash.clone(), - &info.consumed_tx_index, - &info.input_index, - &info.since.clone(), - &tx_hash, - &output_index, +pub async fn bulk_insert_transactions( + block_number: u64, + block_hash: &[u8], + block_timestamp: u64, + tx_views: &[TransactionView], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + let tx_rows: Vec<_> = tx_views + .iter() + .enumerate() + .map(|(idx, transaction)| { + ( + generate_id(block_number), + transaction.hash().raw_data().to_vec(), + idx as i32, + transaction.inputs().len() as i32, + transaction.outputs().len() as i32, + block_number as i32, + block_hash.to_vec(), + block_timestamp as i64, + transaction.version() as i16, + transaction.cell_deps().as_bytes().to_vec(), + transaction.header_deps().as_bytes().to_vec(), + transaction.witnesses().as_bytes().to_vec(), ) - .await?; + }) + .collect(); + + for start in (0..tx_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(tx_rows.len()); + + // build query str + let mut builder = SqlBuilder::insert_into("mercury_transaction"); + builder.field( + r#"id, + tx_hash, + tx_index, + input_count, + output_count, + block_number, + block_hash, + tx_timestamp, + version, + cell_deps, + header_deps, + witnesses"#, + ); + push_values_placeholders(&mut builder, 12, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in tx_rows[start..end].iter() { + seq!(i in 0..12 { + query = query.bind(&row.i); + }); } - Ok(()) + // execute + query.execute(&mut *tx).await?; } - async fn insert_cell_table( - &self, - tx_view: &TransactionView, - tx_index: u32, - block_view: &BlockView, - tx: &mut RBatisTxExecutor<'_>, - output_cell_set: &mut Vec, - live_cell_set: &mut Vec, - script_set: &mut HashSet, - consumed_infos: &mut Vec, - indexer_cells: &mut Vec, - ) -> Result<()> { - let block_hash = to_rb_bytes(&block_view.hash().raw_data()); - let block_number = block_view.number(); - let epoch = block_view.epoch(); + Ok(()) +} - if tx_index > 0 { - self.collect_consume_input_cells( - tx_view, - block_number, - block_hash.clone(), - tx_index, - consumed_infos, - ) - .await?; +pub async fn bulk_insert_output_cells( + block_number: u64, + block_hash: &[u8], + epoch: EpochNumberWithFraction, + tx_views: &[TransactionView], + insert_live_cells: bool, + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + let mut output_cell_rows = Vec::new(); + + for (tx_index, tx_view) in tx_views.iter().enumerate() { + let tx_hash = tx_view.hash().raw_data(); + + for (output_index, (cell, data)) in tx_view.outputs_with_data_iter().enumerate() { + let cell_capacity: u64 = cell.capacity().unpack(); + let cell_row = ( + generate_id(block_number), + tx_hash.to_vec(), + i32::try_from(output_index)?, + i32::try_from(tx_index)?, + block_hash.to_vec(), + i32::try_from(block_number)?, + i32::try_from(epoch.number())?, + i32::try_from(epoch.index())?, + i32::try_from(epoch.length())?, + i64::try_from(cell_capacity)?, + cell.lock().calc_script_hash().raw_data().to_vec(), + cell.lock().code_hash().raw_data().to_vec(), + cell.lock().args().raw_data().to_vec(), + i16::try_from(u8::try_from(cell.lock().hash_type())?)?, + if let Some(script) = cell.type_().to_opt() { + script.calc_script_hash().raw_data().to_vec() + } else { + H256::default().0.to_vec() + }, + if let Some(script) = cell.type_().to_opt() { + script.code_hash().raw_data().to_vec() + } else { + Vec::::new() + }, + if let Some(script) = cell.type_().to_opt() { + script.args().raw_data().to_vec() + } else { + Vec::::new() + }, + if let Some(script) = cell.type_().to_opt() { + i16::try_from(u8::try_from(script.hash_type())?)? + } else { + 0i16 + }, + data.to_vec(), + Vec::::new(), + Vec::::new(), + Vec::::new(), + ); + output_cell_rows.push(cell_row); } + } - self.insert_output_cells( - tx_view, + // bulk insert + for start in (0..output_cell_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(output_cell_rows.len()); + + // mercury_cell + // build query str + let mut builder = SqlBuilder::insert_into("mercury_cell"); + builder.field( + r#"id, + tx_hash, + output_index, tx_index, + block_hash, block_number, - block_hash.clone(), - epoch, - tx, - output_cell_set, - live_cell_set, - script_set, - indexer_cells, - ) - .await?; + epoch_number, + epoch_index, + epoch_length, + capacity, + lock_hash, + lock_code_hash, + lock_args, + lock_script_type, + type_hash, + type_code_hash, + type_args, + type_script_type, + data, + consumed_block_hash, + consumed_tx_hash, + since"#, + ); + push_values_placeholders(&mut builder, 22, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in output_cell_rows[start..end].iter() { + seq!(i in 0..22 { + query = query.bind(&row.i); + }); + } - Ok(()) - } + // execute + query.execute(&mut *tx).await?; - async fn insert_output_cells( - &self, - tx_view: &TransactionView, - tx_index: u32, - block_number: u64, - block_hash: RbBytes, - epoch: EpochNumberWithFraction, - tx: &mut RBatisTxExecutor<'_>, - output_cell_set: &mut Vec, - live_cell_set: &mut Vec, - script_set: &mut HashSet, - indexer_cell_set: &mut Vec, - ) -> Result<()> { - let tx_hash = to_rb_bytes(&tx_view.hash().raw_data()); - let mut has_script_cache = HashMap::new(); - - for (idx, (cell, data)) in tx_view.outputs_with_data_iter().enumerate() { - let mut table = CellTable::from_cell( - &cell, - generate_id(block_number), - tx_hash.clone(), - idx as u32, + if insert_live_cells { + // build query str + let mut builder = SqlBuilder::insert_into("mercury_live_cell"); + builder.field( + r#"id, + tx_hash, + output_index, tx_index, + block_hash, block_number, - block_hash.clone(), - epoch, - &data, + epoch_number, + epoch_index, + epoch_length, + capacity, + lock_hash, + lock_code_hash, + lock_args, + lock_script_type, + type_hash, + type_code_hash, + type_args, + type_script_type, + data"#, ); + push_values_placeholders(&mut builder, 19, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in output_cell_rows[start..end].iter() { + seq!(i in 0..19 { + query = query.bind(&row.i); + }); + } - if let Some(type_script) = cell.type_().to_opt() { - table.set_type_script_info(&type_script); - let type_script_table = table.to_type_script_table(); + // execute + query.execute(&mut *tx).await?; + } + } + + Ok(()) +} + +async fn bulk_insert_scripts( + tx_views: &[TransactionView], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + let mut script_set = HashSet::new(); + let mut exist_script_cache = HashSet::new(); - if !script_set.contains(&type_script_table) - && !self - .has_script(&type_script_table, &mut has_script_cache, tx) - .await? + for tx_view in tx_views.iter() { + for (cell, _) in tx_view.outputs_with_data_iter() { + if let Some(type_script) = cell.type_().to_opt() { + let type_hash = type_script.calc_script_hash().raw_data(); + let type_script_args = type_script.args().raw_data(); + + let type_script_row = ( + type_hash.to_vec(), + type_hash.to_vec().split_at(BLAKE_160_HSAH_LEN).0.to_vec(), + type_script.code_hash().raw_data().to_vec(), + type_script_args.to_vec(), + i16::try_from(u8::try_from(type_script.hash_type())?)?, + i32::try_from(type_script_args.to_vec().len())?, + ); + if !script_set.contains(&type_script_row) + && !script_exists(&type_script_row.0, &mut exist_script_cache, tx).await? { - script_set.insert(type_script_table); + exist_script_cache.insert(type_script_row.0.clone()); + script_set.insert(type_script_row); } } - let lock_table = table.to_lock_script_table(); - if !script_set.contains(&lock_table) - && !self - .has_script(&lock_table, &mut has_script_cache, tx) - .await? + let lock_script = cell.lock(); + let lock_hash = lock_script.calc_script_hash().raw_data(); + let lock_script_args = lock_script.args().raw_data(); + let lock_script_row = ( + lock_hash.to_vec(), + lock_hash.to_vec().split_at(BLAKE_160_HSAH_LEN).0.to_vec(), + lock_script.code_hash().raw_data().to_vec(), + lock_script_args.to_vec(), + i16::try_from(u8::try_from(lock_script.hash_type())?)?, + i32::try_from(lock_script_args.to_vec().len())?, + ); + if !script_set.contains(&lock_script_row) + && !script_exists(&lock_script_row.0, &mut exist_script_cache, tx).await? { - script_set.insert(lock_table); + exist_script_cache.insert(lock_script_row.0.clone()); + script_set.insert(lock_script_row); } - - let indexer_cell = IndexerCellTable::new_with_empty_scripts( - block_number, - IO_TYPE_OUTPUT, - idx as u32, - tx_hash.clone(), - tx_index, - ); - - indexer_cell_set.push(indexer_cell.update_by_cell_table(&table)); - output_cell_set.push(table.clone()); - live_cell_set.push(table.into()); } + } - Ok(()) + let script_rows = script_set.iter().cloned().collect::>(); + + // bulk insert + for start in (0..script_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(script_rows.len()); + + // build query str + let mut builder = SqlBuilder::insert_into("mercury_script"); + builder.field( + r#"script_hash, + script_hash_160, + script_code_hash, + script_args, + script_type, + script_args_len"#, + ); + push_values_placeholders(&mut builder, 6, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in script_rows[start..end].iter() { + seq!(i in 0..6 { + query = query.bind(&row.i); + }); + } + // execute + query.execute(&mut *tx).await?; } - async fn collect_consume_input_cells( - &self, - tx_view: &TransactionView, - consumed_block_number: u64, - consumed_block_hash: RbBytes, - tx_index: u32, - consumed_infos: &mut Vec, - ) -> Result<()> { - let consumed_tx_hash = to_rb_bytes(&tx_view.hash().raw_data()); + Ok(()) +} + +async fn update_consumed_cells( + consumed_block_number: u64, + consumed_block_hash: &[u8], + tx_views: &[TransactionView], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + for (tx_index, transaction) in tx_views.iter().enumerate() { + if tx_index == 0 { + continue; + } - for (idx, input) in tx_view.inputs().into_iter().enumerate() { + let consumed_tx_hash = transaction.hash().raw_data().to_vec(); + + for (input_index, input) in transaction.inputs().into_iter().enumerate() { let since: u64 = input.since().unpack(); + let previous_output_tx_hash = input.previous_output().tx_hash().raw_data().to_vec(); + let previous_output_index: u32 = input.previous_output().index().unpack(); - consumed_infos.push(ConsumedInfo { - out_point: input.previous_output(), - input_index: idx as u32, - consumed_block_hash: consumed_block_hash.clone(), - consumed_block_number, - consumed_tx_hash: consumed_tx_hash.clone(), - consumed_tx_index: tx_index, - since: to_rb_bytes(&since.to_be_bytes()), - }); - } + sqlx::query( + r#"DELETE FROM mercury_live_cell + WHERE tx_hash = $1 AND output_index = $2 + "#, + ) + .bind(&previous_output_tx_hash) + .bind(previous_output_index as i32) + .execute(&mut *tx) + .await?; - Ok(()) + sqlx::query( + r#"UPDATE mercury_cell SET + consumed_block_number = $1, + consumed_block_hash = $2, + consumed_tx_hash = $3, + consumed_tx_index = $4, + input_index = $5, + since = $6 + WHERE tx_hash = $7 AND output_index = $8 + "#, + ) + .bind(consumed_block_number as i64) + .bind(consumed_block_hash) + .bind(&consumed_tx_hash) + .bind(tx_index as i32) + .bind(input_index as i32) + .bind(since.to_be_bytes().as_slice()) + .bind(&previous_output_tx_hash) + .bind(previous_output_index as i32) + .execute(&mut *tx) + .await?; + } } - async fn has_script( - &self, - table: &ScriptTable, - has_script_cache: &mut HashMap, bool>, - tx: &mut RBatisTxExecutor<'_>, - ) -> Result { - if let Some(res) = has_script_cache.get(&table.script_hash.inner) { - if *res { - return Ok(true); + Ok(()) +} + +async fn bulk_insert_indexer_cells( + block_number: u64, + tx_views: &[TransactionView], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + let mut indexer_cell_rows = vec![]; + + for (tx_index, tx_view) in tx_views.iter().enumerate() { + let tx_hash = &tx_view.hash().raw_data(); + + if tx_index > 0 { + for (input_index, input) in tx_view.inputs().into_iter().enumerate() { + let previous_output_tx_hash = input.previous_output().tx_hash().raw_data().to_vec(); + let previous_output_index: u32 = input.previous_output().index().unpack(); + + let cell = sqlx::query( + r#"SELECT lock_hash, lock_code_hash, lock_args, lock_script_type, + type_hash, type_code_hash, type_args, type_script_type + FROM mercury_cell + WHERE tx_hash = $1 AND output_index = $2 + "#, + ) + .bind(&previous_output_tx_hash) + .bind(previous_output_index as i32) + .fetch_one(&mut *tx) + .await?; + + let indexer_cell = ( + generate_id(block_number), + i32::try_from(block_number)?, + i16::try_from(IO_TYPE_INPUT)?, + i32::try_from(input_index)?, + tx_hash.to_vec(), + i32::try_from(tx_index)?, + cell.get("lock_hash"), + cell.get("lock_code_hash"), + cell.get("lock_args"), + cell.get("lock_script_type"), + cell.get("type_hash"), + cell.get("type_code_hash"), + cell.get("type_args"), + cell.get("type_script_type"), + ); + indexer_cell_rows.push(indexer_cell); } } - let w = self - .pool - .wrapper() - .eq("script_hash", table.script_hash.clone()); - let res = tx.fetch_count_by_wrapper::(w).await?; - let ret = res != 0; + for (idx, (cell, _)) in tx_view.outputs_with_data_iter().enumerate() { + let indexer_cell = ( + generate_id(block_number), + i32::try_from(block_number)?, + i16::try_from(IO_TYPE_OUTPUT)?, + i32::try_from(idx)?, + tx_hash.to_vec(), + i32::try_from(tx_index)?, + cell.lock().calc_script_hash().raw_data().to_vec(), + cell.lock().code_hash().raw_data().to_vec(), + cell.lock().args().raw_data().to_vec(), + i16::try_from(u8::try_from(cell.lock().hash_type())?)?, + if let Some(script) = cell.type_().to_opt() { + script.calc_script_hash().raw_data().to_vec() + } else { + H256::default().0.to_vec() + }, + if let Some(script) = cell.type_().to_opt() { + script.code_hash().raw_data().to_vec() + } else { + Vec::new() + }, + if let Some(script) = cell.type_().to_opt() { + script.args().raw_data().to_vec() + } else { + Vec::new() + }, + if let Some(script) = cell.type_().to_opt() { + i16::try_from(u8::try_from(script.hash_type())?)? + } else { + 0i16 + }, + ); + indexer_cell_rows.push(indexer_cell); + } + } - has_script_cache - .entry(table.script_hash.inner.clone()) - .or_insert(ret); + // bulk insert + for start in (0..indexer_cell_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(indexer_cell_rows.len()); - Ok(ret) + // build query str + let mut builder = SqlBuilder::insert_into("mercury_indexer_cell"); + builder.field( + r#"id, + block_number, + io_type, + io_index, + tx_hash, + tx_index, + lock_hash, + lock_code_hash, + lock_args, + lock_script_type, + type_hash, + type_code_hash, + type_args, + type_script_type"#, + ); + push_values_placeholders(&mut builder, 14, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in indexer_cell_rows[start..end].iter() { + seq!(i in 0..14 { + query = query.bind(&row.i); + }); + } + // execute + query.execute(&mut *tx).await?; } - pub(crate) async fn insert_registered_address_table( - &self, - addresses: Vec<(RbBytes, String)>, - tx: &mut RBatisTxExecutor<'_>, - ) -> Result> { - let mut res = vec![]; - for item in addresses { - let (lock_hash, address) = item; - if let Err(e) = tx - .save( - &RegisteredAddressTable::new(lock_hash.clone(), address.clone()), - &[], - ) - .await - { - if let Ok(Some(row)) = self.query_registered_address(lock_hash.clone()).await { - if row.address != address { - return Err(e.into()); - } - } else { - return Err(e.into()); - } - } - res.push(lock_hash); - } + Ok(()) +} - Ok(res) +async fn script_exists( + script_hash: &[u8], + exist_script_cache: &mut HashSet>, + tx: &mut Transaction<'_, Any>, +) -> Result { + if exist_script_cache.contains(script_hash) { + return Ok(true); } + + let row = sqlx::query( + "SELECT COUNT(*) as count + FROM mercury_script WHERE + script_hash = $1", + ) + .bind(script_hash) + .fetch_one(tx) + .await?; + + Ok(row.get::("count") != 0) } diff --git a/core/storage/src/relational/mod.rs b/core/storage/src/relational/mod.rs index 575dff032..71420544a 100644 --- a/core/storage/src/relational/mod.rs +++ b/core/storage/src/relational/mod.rs @@ -2,99 +2,70 @@ mod fetch; mod insert; mod remove; mod snowflake; -mod sql; -pub mod table; #[cfg(test)] mod tests; -pub use insert::BATCH_SIZE_THRESHOLD; - use crate::relational::{ - fetch::to_pagination_response, snowflake::Snowflake, table::IndexerCellTable, + fetch::bytes_to_h256, fetch::to_pagination_response, snowflake::Snowflake, }; use crate::{error::DBError, Storage}; +pub use insert::{ + bulk_insert_blocks, bulk_insert_output_cells, bulk_insert_transactions, + push_values_placeholders, BATCH_SIZE_THRESHOLD, BLAKE_160_HSAH_LEN, IO_TYPE_INPUT, + IO_TYPE_OUTPUT, +}; +use remove::remove_block_table; use common::{ - async_trait, utils::to_fixed_array, Context, DetailedCell, Order, PaginationRequest, - PaginationResponse, Range, Result, + async_trait, Context, DetailedCell, Order, PaginationRequest, PaginationResponse, Range, Result, }; -use common_logger::{tracing, tracing_async}; -use db_xsql::{commit_transaction, rbatis::Bytes as RbBytes, XSQLPool}; +use core_rpc_types::indexer::Transaction; +use db_sqlx::{build_next_cursor, SQLXPool}; use protocol::db::{DBDriver, DBInfo, SimpleBlock, SimpleTransaction, TransactionWrapper}; use ckb_types::core::{BlockNumber, BlockView, HeaderView}; -use ckb_types::{bytes::Bytes, packed, H160, H256}; -use log::LevelFilter; +use ckb_types::{bytes::Bytes, packed, prelude::*, H160, H256}; +use sqlx::Row; use std::collections::HashSet; -const HASH160_LEN: usize = 20; - lazy_static::lazy_static! { pub static ref SNOWFLAKE: Snowflake = Snowflake::default(); } #[derive(Clone, Debug)] pub struct RelationalStorage { - pub pool: XSQLPool, + pub sqlx_pool: SQLXPool, } #[async_trait] impl Storage for RelationalStorage { - #[tracing_async] - async fn append_block(&self, ctx: Context, block: BlockView) -> Result<()> { - let mut tx = self.pool.transaction().await?; - self.insert_block_table(ctx.clone(), &block, &mut tx) - .await?; - self.insert_transaction_table(ctx.clone(), &block, &mut tx) - .await?; - - commit_transaction(tx).await?; - Ok(()) + async fn append_block(&self, block: BlockView) -> Result<()> { + let mut tx = self.sqlx_pool.transaction().await?; + self.insert_block_table(&block, &mut tx).await?; + self.insert_transaction_table(&block, &mut tx).await?; + tx.commit().await.map_err(Into::into) } - #[tracing_async] - async fn rollback_block( - &self, - ctx: Context, - block_number: BlockNumber, - block_hash: H256, - ) -> Result<()> { - let mut tx = self.pool.transaction().await?; - let block_hash = to_rb_bytes(&block_hash.0); - - self.remove_tx_and_cell(ctx.clone(), block_number, block_hash.clone(), &mut tx) - .await?; - self.remove_block_table(ctx.clone(), block_number, block_hash, &mut tx) + async fn rollback_block(&self, block_number: BlockNumber, block_hash: H256) -> Result<()> { + let mut tx = self.sqlx_pool.transaction().await?; + self.remove_tx_and_cell(block_number, block_hash.clone(), &mut tx) .await?; - commit_transaction(tx).await?; - - Ok(()) + remove_block_table(block_number, block_hash, &mut tx).await?; + tx.commit().await.map_err(Into::into) } - #[tracing_async] async fn get_cells( &self, - ctx: Context, + _ctx: Context, out_point: Option, lock_hashes: Vec, type_hashes: Vec, block_range: Option, pagination: PaginationRequest, ) -> Result> { - let lock_hashes = lock_hashes - .into_iter() - .map(|hash| to_rb_bytes(hash.as_bytes())) - .collect::>(); - - let type_hashes = type_hashes - .into_iter() - .map(|hash| to_rb_bytes(hash.as_bytes())) - .collect::>(); - self.query_cells( - ctx, out_point, lock_hashes, type_hashes, @@ -105,10 +76,9 @@ impl Storage for RelationalStorage { .await } - #[tracing_async] async fn get_live_cells( &self, - ctx: Context, + _ctx: Context, out_point: Option, lock_hashes: Vec, type_hashes: Vec, @@ -117,18 +87,7 @@ impl Storage for RelationalStorage { data_len_range: Option, pagination: PaginationRequest, ) -> Result> { - let lock_hashes = lock_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - - let type_hashes = type_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - self.query_live_cells( - ctx, out_point, lock_hashes, type_hashes, @@ -140,10 +99,9 @@ impl Storage for RelationalStorage { .await } - #[tracing_async] async fn get_historical_live_cells( &self, - ctx: Context, + _ctx: Context, lock_hashes: Vec, type_hashes: Vec, tip_block_number: BlockNumber, @@ -156,19 +114,7 @@ impl Storage for RelationalStorage { ) .into()); } - - let lock_hashes = lock_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - - let type_hashes = type_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - self.query_historical_live_cells( - ctx, lock_hashes, type_hashes, tip_block_number, @@ -178,7 +124,6 @@ impl Storage for RelationalStorage { .await } - #[tracing_async] async fn get_transactions( &self, ctx: Context, @@ -199,21 +144,10 @@ impl Storage for RelationalStorage { ) .into()); } - - let lock_hashes = lock_hashes - .into_iter() - .map(|hash| to_rb_bytes(hash.as_bytes())) - .collect::>(); - let type_hashes = type_hashes - .into_iter() - .map(|hash| to_rb_bytes(hash.as_bytes())) - .collect::>(); - let mut set = HashSet::new(); if !lock_hashes.is_empty() || !type_hashes.is_empty() || out_point.is_some() { for cell in self .query_cells( - ctx.clone(), out_point, lock_hashes, type_hashes, @@ -223,11 +157,10 @@ impl Storage for RelationalStorage { ) .await? .response - .iter() { - set.insert(cell.out_point.tx_hash().raw_data().to_vec()); + set.insert(cell.out_point.tx_hash().unpack()); if let Some(hash) = &cell.consumed_tx_hash { - set.insert(hash.0.to_vec()); + set.insert(hash.to_owned()); } } if set.is_empty() { @@ -243,7 +176,7 @@ impl Storage for RelationalStorage { } } - let tx_hashes = set.iter().map(|bytes| to_rb_bytes(bytes)).collect(); + let tx_hashes = set.into_iter().collect(); let tx_tables = self .query_transactions(ctx.clone(), tx_hashes, block_range, pagination) .await?; @@ -259,7 +192,6 @@ impl Storage for RelationalStorage { )) } - #[tracing_async] async fn get_transactions_by_hashes( &self, ctx: Context, @@ -273,19 +205,13 @@ impl Storage for RelationalStorage { ) .into()); } - - let tx_hashes = tx_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); let tx_tables = self .query_transactions(ctx.clone(), tx_hashes, block_range, pagination) .await?; let txs_wrapper = self - .get_transactions_with_status(ctx.clone(), tx_tables.response) + .get_transactions_with_status(ctx, tx_tables.response) .await?; let next_cursor: Option = tx_tables.next_cursor.map(Into::into); - Ok(to_pagination_response( txs_wrapper, next_cursor, @@ -293,7 +219,6 @@ impl Storage for RelationalStorage { )) } - #[tracing_async] async fn get_transactions_by_scripts( &self, ctx: Context, @@ -309,60 +234,26 @@ impl Storage for RelationalStorage { ) .into()); } - let is_asc = pagination.order.is_asc(); - let mut conn = self.pool.acquire().await?; - - // The id type in the table definition is i64, - // so it is necessary to limit the numerical range of cursor and limit of type u64, - // therwise a database error will be returned when querying - let cursor = pagination - .cursor - .map(|cur| cur.min(i64::MAX as u64) as i64) - .unwrap_or(if is_asc { 0 } else { i64::MAX }); - let limit = pagination - .limit - .map(|limit| limit.min(i64::MAX as u64) as i64) - .unwrap_or(i64::MAX); - - let lock_hashes = lock_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - let type_hashes = type_hashes - .into_iter() - .map(|hash| to_rb_bytes(&hash.0)) - .collect::>(); - let (from, to) = if let Some(range) = block_range.clone() { - (range.min(), range.max()) - } else { - (0, 1) - }; - - let tx_hashes = sql::fetch_distinct_tx_hashes( - &mut conn, - &cursor, - &from, - &to, - &lock_hashes, - &type_hashes, - &limit, - &is_asc, - &block_range.is_some(), - &limit_cellbase, - ) - .await?; - let count = if pagination.return_count { - let count = sql::fetch_distinct_tx_hashes_count( - &mut conn, - &from, - &to, + let tx_hashes = self + .query_transaction_hashes_by_scripts( &lock_hashes, &type_hashes, - &block_range.is_some(), - &limit_cellbase, + &block_range, + limit_cellbase, + &pagination, ) .await?; + + let count = if pagination.return_count { + let count = self + .query_distinct_tx_hashes_count( + &lock_hashes, + &type_hashes, + &block_range, + limit_cellbase, + ) + .await?; Some(count) } else { None @@ -376,33 +267,30 @@ impl Storage for RelationalStorage { }); } - let next_cursor = if tx_hashes.is_empty() - || count.is_some() && count.unwrap() <= limit as u64 - || limit > tx_hashes.len() as i64 - { + let next_cursor = if tx_hashes.is_empty() { None } else { - tx_hashes.last().map(|tx_hash| tx_hash.id as u64) + build_next_cursor( + pagination.limit.unwrap_or(u16::MAX), + tx_hashes.last().unwrap().1, + tx_hashes.len(), + count, + ) }; - let pag = if is_asc { + let pag = if pagination.order.is_asc() { PaginationRequest::default() } else { PaginationRequest::default().order(Order::Desc) }; + + let tx_hashes = tx_hashes.into_iter().map(|(tx_hash, _)| tx_hash).collect(); let tx_tables = self - .query_transactions( - ctx.clone(), - tx_hashes.into_iter().map(|i| i.tx_hash).collect(), - block_range, - pag, - ) + .query_transactions(ctx.clone(), tx_hashes, block_range, pag) .await?; - let txs_wrapper = self .get_transactions_with_status(ctx, tx_tables.response) .await?; - Ok(fetch::to_pagination_response( txs_wrapper, next_cursor, @@ -410,7 +298,6 @@ impl Storage for RelationalStorage { )) } - #[tracing_async] async fn get_block( &self, ctx: Context, @@ -433,7 +320,6 @@ impl Storage for RelationalStorage { } } - #[tracing_async] async fn get_block_header( &self, _ctx: Context, @@ -456,7 +342,6 @@ impl Storage for RelationalStorage { } } - #[tracing_async] async fn get_scripts( &self, _ctx: Context, @@ -465,29 +350,14 @@ impl Storage for RelationalStorage { args_len: Option, args: Vec, ) -> Result> { - let script_hashes = script_hashes - .into_iter() - .map(|hash| to_rb_bytes(hash.as_bytes())) - .collect::>(); - let code_hashes = code_hashes - .into_iter() - .map(|hash| to_rb_bytes(hash.as_bytes())) - .collect::>(); - let args = args - .into_iter() - .map(|arg| to_rb_bytes(&arg)) - .collect::>(); - self.query_scripts(script_hashes, code_hashes, args_len, args) .await } - #[tracing_async] async fn get_tip(&self, _ctx: Context) -> Result> { self.query_tip().await } - #[tracing_async] async fn get_spent_transaction_hash( &self, _ctx: Context, @@ -496,7 +366,6 @@ impl Storage for RelationalStorage { self.query_spent_tx_hash(out_point).await } - #[tracing_async] async fn get_canonical_block_hash( &self, _ctx: Context, @@ -505,7 +374,6 @@ impl Storage for RelationalStorage { self.query_canonical_block_hash(block_number).await } - #[tracing_async] async fn get_simple_transaction_by_hash( &self, _ctx: Context, @@ -514,7 +382,6 @@ impl Storage for RelationalStorage { self.query_simple_transaction(tx_hash).await } - #[tracing_async] async fn get_scripts_by_partial_arg( &self, _ctx: Context, @@ -522,69 +389,61 @@ impl Storage for RelationalStorage { arg: Bytes, offset_location: (u32, u32), ) -> Result> { - let mut conn = self.pool.acquire().await?; - let offset = offset_location.0 + 1; - let len = offset_location.1 - offset_location.0; - - let ret = sql::query_scripts_by_partial_arg( - &mut conn, - &to_rb_bytes(&code_hash.0), - &to_rb_bytes(&arg), - &offset, - &len, + let offset = i32::try_from(offset_location.0 + 1)?; + let len = i32::try_from(offset_location.1 - offset_location.0)?; + let query = SQLXPool::new_query( + r#" + SELECT script_code_hash, script_args, script_type + FROM mercury_script + WHERE script_code_hash = $1 + AND substring(script_args, $3, $4) = $2 + "#, ) - .await?; - - Ok(ret.into_iter().map(Into::into).collect()) + .bind(code_hash.as_bytes()) + .bind(arg.to_vec()) + .bind(offset) + .bind(len); + let res = self.sqlx_pool.fetch(query).await?; + Ok(res + .into_iter() + .map(|row| { + packed::ScriptBuilder::default() + .code_hash(bytes_to_h256(row.get("script_code_hash")).pack()) + .args(row.get::, _>("script_args").pack()) + .hash_type(packed::Byte::new(row.get::("script_type") as u8)) + .build() + }) + .collect()) } - #[tracing_async] async fn get_registered_address( &self, _ctx: Context, lock_hash: H160, ) -> Result> { - let lock_hash = to_rb_bytes(lock_hash.as_bytes()); - let res = self.query_registered_address(lock_hash).await?; - Ok(res.map(|t| t.address)) + self.query_registered_address(lock_hash.as_bytes()).await } - #[tracing_async] async fn register_addresses( &self, _ctx: Context, addresses: Vec<(H160, String)>, ) -> Result> { - let mut tx = self.pool.transaction().await?; - let addresses = addresses - .into_iter() - .map(|(lock_hash, address)| (to_rb_bytes(lock_hash.as_bytes()), address)) - .collect::>(); - let res = self - .insert_registered_address_table(addresses, &mut tx) - .await?; - commit_transaction(tx).await?; - - Ok(res - .iter() - .map(|hash| H160(to_fixed_array::(&hash.inner))) - .collect()) + self.bulk_insert_registered_address_table(addresses).await } - #[tracing] fn get_db_info(&self, _ctx: Context) -> Result { let info = SNOWFLAKE.get_info(); Ok(DBInfo { version: clap::crate_version!().to_string(), db: DBDriver::PostgreSQL, - conn_size: self.pool.get_max_connections(), + conn_size: self.sqlx_pool.get_max_connections(), center_id: info.0, machine_id: info.1, }) } - #[tracing_async] async fn get_simple_block( &self, _ctx: Context, @@ -607,7 +466,6 @@ impl Storage for RelationalStorage { } } - #[tracing_async] async fn get_indexer_transactions( &self, _ctx: Context, @@ -615,36 +473,23 @@ impl Storage for RelationalStorage { type_hashes: Vec, block_range: Option, pagination: PaginationRequest, - ) -> Result> { + ) -> Result> { if lock_hashes.is_empty() && type_hashes.is_empty() && block_range.is_none() { return Err(DBError::InvalidParameter( "No valid parameter to query indexer cell".to_string(), ) .into()); } - - self.query_indexer_cells(lock_hashes, type_hashes, block_range, pagination) + self.query_indexer_transactions(lock_hashes, type_hashes, block_range, pagination) .await } - #[tracing_async] async fn indexer_synced_count(&self) -> Result { - let w = self.pool.wrapper(); - let ret = self - .pool - .fetch_count_by_wrapper::(w) - .await?; - Ok(ret) + self.sqlx_pool.fetch_count("mercury_sync_status").await } - #[tracing_async] async fn block_count(&self, _ctx: Context) -> Result { - let w = self.pool.wrapper(); - let ret = self - .pool - .fetch_count_by_wrapper::(w) - .await?; - Ok(ret) + self.sqlx_pool.fetch_count("mercury_block").await } } @@ -657,9 +502,8 @@ impl RelationalStorage { connect_timeout: u64, max_lifetime: u64, idle_timeout: u64, - log_level: LevelFilter, ) -> Self { - let pool = XSQLPool::new( + let sqlx_pool = SQLXPool::new( center_id, machine_id, max_connections, @@ -667,13 +511,12 @@ impl RelationalStorage { connect_timeout, max_lifetime, idle_timeout, - log_level, ); - RelationalStorage { pool } + RelationalStorage { sqlx_pool } } pub async fn connect( - &self, + &mut self, db_driver: DBDriver, db_name: &str, host: &str, @@ -681,21 +524,22 @@ impl RelationalStorage { user: &str, password: &str, ) -> Result<()> { - self.pool - .connect(db_driver, db_name, host, port, user, password) + self.sqlx_pool + .connect(&db_driver, db_name, host, port, user, password) .await?; Ok(()) } - /// This function is provided for test. - pub fn inner(&self) -> XSQLPool { - self.pool.clone() + pub fn get_pool(&self) -> SQLXPool { + self.sqlx_pool.clone() } pub async fn get_tip_number(&self) -> Result { - let mut conn = self.pool.acquire().await?; - let res = sql::get_tip_number(&mut conn).await?; - res.ok_or_else(|| DBError::NotExist("genesis block".to_string()).into()) + let query = + SQLXPool::new_query("SELECT MAX(block_number) AS tip FROM mercury_canonical_chain"); + let res = self.sqlx_pool.fetch_optional(query).await?; + res.map(|row| row.get::("tip") as u64) + .ok_or_else(|| DBError::NotExist("genesis block".to_string()).into()) } } @@ -703,11 +547,3 @@ pub fn generate_id(block_number: BlockNumber) -> i64 { let number = block_number as i64; SNOWFLAKE.generate(number) } - -pub fn to_rb_bytes(input: &[u8]) -> RbBytes { - RbBytes::new(input.to_vec()) -} - -pub fn empty_rb_bytes() -> RbBytes { - RbBytes::new(vec![]) -} diff --git a/core/storage/src/relational/remove.rs b/core/storage/src/relational/remove.rs index a515fec12..baa65dad1 100644 --- a/core/storage/src/relational/remove.rs +++ b/core/storage/src/relational/remove.rs @@ -1,80 +1,120 @@ -use crate::relational::table::{ - BlockTable, CanonicalChainTable, CellTable, IndexerCellTable, LiveCellTable, SyncStatus, - TransactionTable, -}; -use crate::relational::{empty_rb_bytes, sql, to_rb_bytes, RelationalStorage}; +use crate::relational::fetch::sqlx_param_placeholders; +use crate::relational::RelationalStorage; -use common::{Context, Result}; -use common_logger::tracing_async; - -use ckb_types::prelude::Unpack; -use db_xsql::rbatis::{crud::CRUDMut, executor::RBatisTxExecutor, Bytes as RbBytes}; - -use ckb_types::{core::BlockNumber, packed}; +use ckb_types::core::BlockNumber; +use ckb_types::H256; +use common::Result; +use sql_builder::SqlBuilder; +use sqlx::{Any, Transaction}; impl RelationalStorage { pub(crate) async fn remove_tx_and_cell( &self, - _ctx: Context, _block_number: BlockNumber, - block_hash: RbBytes, - tx: &mut RBatisTxExecutor<'_>, + block_hash: H256, + tx: &mut Transaction<'_, Any>, ) -> Result<()> { - let tx_hashes = sql::get_tx_hashes_by_block_hash(tx, &block_hash) - .await? - .into_iter() - .map(|hash| hash.inner()) - .collect::>(); - - tx.remove_by_column::("block_hash", block_hash) - .await?; - tx.remove_batch_by_column::("tx_hash", &tx_hashes) - .await?; - tx.remove_batch_by_column::("tx_hash", &tx_hashes) - .await?; - tx.remove_batch_by_column::("tx_hash", &tx_hashes) + let tx_hashes = self + .query_transaction_hashes_by_block_hash(block_hash.as_bytes()) .await?; + remove_txs_by_block_hash(block_hash, tx).await?; + remove_batch_by_tx_hashes("mercury_cell", &tx_hashes, tx).await?; + remove_batch_by_tx_hashes("mercury_live_cell", &tx_hashes, tx).await?; + remove_batch_by_tx_hashes("mercury_indexer_cell", &tx_hashes, tx).await?; + for tx_hash in tx_hashes.iter() { - sql::rollback_consume_cell(tx, &empty_rb_bytes(), tx_hash).await?; + rollback_consume_cell(tx_hash, tx).await?; } Ok(()) } +} - #[tracing_async] - pub(crate) async fn remove_block_table( - &self, - _ctx: Context, - block_number: BlockNumber, - block_hash: RbBytes, - tx: &mut RBatisTxExecutor<'_>, - ) -> Result<()> { - tx.remove_by_column::("block_hash", block_hash.clone()) - .await?; - tx.remove_by_column::("block_hash", block_hash) - .await?; - tx.remove_by_column::("block_number", block_number) - .await?; - Ok(()) - } +pub(crate) async fn remove_block_table( + block_number: BlockNumber, + block_hash: H256, + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + sqlx::query( + r#"DELETE FROM mercury_block + WHERE block_hash = $1"#, + ) + .bind(block_hash.as_bytes()) + .execute(&mut *tx) + .await?; - pub(crate) async fn remove_live_cell_by_out_point( - &self, - out_point: &packed::OutPoint, - tx: &mut RBatisTxExecutor<'_>, - ) -> Result<()> { - let tx_hash = to_rb_bytes(&out_point.tx_hash().raw_data()); - let output_index: u32 = out_point.index().unpack(); + sqlx::query( + r#"DELETE FROM mercury_canonical_chain + WHERE block_hash = $1"#, + ) + .bind(block_hash.as_bytes()) + .execute(&mut *tx) + .await?; - let w = self - .pool - .wrapper() - .eq("tx_hash", tx_hash) - .and() - .eq("output_index", output_index); - tx.remove_by_wrapper::(w).await?; + sqlx::query( + r#"DELETE FROM mercury_sync_status + WHERE block_number = $1"#, + ) + .bind(i32::try_from(block_number)?) + .execute(&mut *tx) + .await?; - Ok(()) + Ok(()) +} + +async fn remove_batch_by_tx_hashes( + table_name: &str, + tx_hashes: &[H256], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + if tx_hashes.is_empty() { + return Ok(()); + } + + // build query str + let mut query_builder = SqlBuilder::delete_from(table_name); + let sql = query_builder + .and_where_in("tx_hash", &sqlx_param_placeholders(1..tx_hashes.len())?) + .sql()?; + + // bind + let mut query = sqlx::query(&sql); + for hash in tx_hashes { + query = query.bind(hash.as_bytes()); } + + // execute + query.execute(tx).await?; + + Ok(()) +} + +async fn remove_txs_by_block_hash(block_hash: H256, tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + r#"DELETE FROM mercury_transaction + WHERE block_hash = $1"#, + ) + .bind(block_hash.as_bytes()) + .execute(&mut *tx) + .await?; + Ok(()) +} + +async fn rollback_consume_cell(tx_hash: &H256, tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "UPDATE mercury_cell SET + consumed_block_hash = $1, + consumed_block_number = NULL, + consumed_tx_hash = $1, + consumed_tx_index = NULL, + input_index = NULL, + since = $1 + WHERE consumed_tx_hash = $2", + ) + .bind(Vec::::new()) + .bind(tx_hash.as_bytes()) + .execute(&mut *tx) + .await?; + Ok(()) } diff --git a/core/storage/src/relational/sql.rs b/core/storage/src/relational/sql.rs deleted file mode 100644 index 10c90608f..000000000 --- a/core/storage/src/relational/sql.rs +++ /dev/null @@ -1,139 +0,0 @@ -#![allow( - clippy::assign_op_pattern, - clippy::manual_range_contains, - clippy::modulo_one -)] - -use crate::relational::table::{IndexerTxHash, ScriptTable, TxHash}; - -use db_xsql::rbatis::executor::{RBatisConnExecutor, RBatisTxExecutor}; -use db_xsql::rbatis::{html_sql, push_index, rb_html, sql, Bytes as RbBytes}; - -#[sql( - tx, - "UPDATE mercury_cell SET - consumed_block_number = $1, - consumed_block_hash = $2::bytea, - consumed_tx_hash = $3::bytea, - consumed_tx_index = $4, - input_index = $5, - since = $6::bytea - WHERE tx_hash = $7::bytea AND output_index = $8" -)] -pub async fn update_consume_cell( - tx: &mut RBatisTxExecutor<'_>, - consumed_block_number: &u64, - consumed_block_hash: &RbBytes, - consumed_tx_hash: &RbBytes, - consumed_tx_index: &u32, - input_index: &u32, - since: &RbBytes, - tx_hash: &RbBytes, - output_index: &u32, -) -> () { -} - -#[sql( - tx, - "UPDATE mercury_cell SET - consumed_block_hash = $1::bytea, - consumed_block_number = NULL, - consumed_tx_hash = $1::bytea, - consumed_tx_index = NULL, - input_index = NULL, - since = $1::bytea WHERE consumed_tx_hash = $2::bytea" -)] -pub async fn rollback_consume_cell( - tx: &mut RBatisTxExecutor<'_>, - empty_bytes: &RbBytes, - consumed_tx_hash: &RbBytes, -) -> () { -} - -#[sql(conn, "SELECT MAX(block_number) FROM mercury_canonical_chain")] -pub async fn get_tip_number(conn: &mut RBatisConnExecutor<'_>) -> Option {} - -#[sql( - conn, - "SELECT id FROM mercury_live_cell WHERE tx_hash = $1::bytea AND output_index = $2" -)] -pub async fn is_live_cell( - conn: &mut RBatisConnExecutor<'_>, - tx_hash: &RbBytes, - index: &u16, -) -> Option { -} - -#[sql( - conn, - "DELETE FROM mercury_live_cell WHERE tx_hash = $1::bytea AND output_index = $2" -)] -pub async fn remove_live_cell( - conn: &mut RBatisConnExecutor<'_>, - tx_hash: &RbBytes, - index: &u16, -) -> () { -} - -#[sql(tx, "SELECT tx_hash FROM mercury_transaction WHERE block_hash = $1")] -pub async fn get_tx_hashes_by_block_hash( - tx: &mut RBatisTxExecutor<'_>, - block_hash: &RbBytes, -) -> Vec { -} - -#[sql( - conn, - "SELECT * FROM mercury_script - WHERE script_code_hash = $1::bytea AND substring(script_args::bytea ,$3::int ,$4::int) = $2::bytea" -)] -pub async fn query_scripts_by_partial_arg( - conn: &mut RBatisConnExecutor<'_>, - code_hash: &RbBytes, - arg: &RbBytes, - from: &u32, - len: &u32, -) -> Vec { -} - -#[html_sql(conn, "core/storage/src/relational/_sql.html")] -pub async fn fetch_distinct_tx_hashes_count( - conn: &mut RBatisConnExecutor<'_>, - from: &u64, - to: &u64, - lock_hashes: &[RbBytes], - type_hashes: &[RbBytes], - limit_range: &bool, - limit_cellbase: &bool, -) -> u64 { -} - -#[html_sql(conn, "core/storage/src/relational/_sql.html")] -pub async fn fetch_distinct_tx_hashes( - conn: &mut RBatisConnExecutor<'_>, - cursor: &i64, - from: &u64, - to: &u64, - lock_hashes: &[RbBytes], - type_hashes: &[RbBytes], - limit: &i64, - is_asc: &bool, - limit_range: &bool, - limit_cellbase: &bool, -) -> Vec { -} - -#[sql( - tx, - "UPDATE mercury_sync_dead_cell SET is_delete = true WHERE tx_hash = $1::bytea and output_index = $2" -)] -pub async fn update_sync_dead_cell( - tx: &mut RBatisTxExecutor<'_>, - tx_hash: &RbBytes, - index: &u32, -) -> () { -} - -#[cfg(test)] -#[sql(conn, "SELECT COUNT(1) FROM mercury_consume_info")] -pub async fn fetch_cunsumed_cell_count(conn: &mut RBatisConnExecutor<'_>) -> u64 {} diff --git a/core/storage/src/relational/table.rs b/core/storage/src/relational/table.rs deleted file mode 100644 index bf6bf97ef..000000000 --- a/core/storage/src/relational/table.rs +++ /dev/null @@ -1,684 +0,0 @@ -use crate::relational::{empty_rb_bytes, to_rb_bytes}; - -use common::utils::to_fixed_array; -use common::DetailedCell; -use db_xsql::rbatis::{crud_table, Bytes as RbBytes}; - -use ckb_types::core::{BlockView, EpochNumberWithFraction, TransactionView}; -use ckb_types::{packed, prelude::*, H256}; - -use serde::{Deserialize, Serialize}; - -use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; -use std::hash::{Hash, Hasher}; - -const BLAKE_160_HSAH_LEN: usize = 20; -const HASH256_LEN: usize = 32; -pub const IO_TYPE_INPUT: u8 = 0; -pub const IO_TYPE_OUTPUT: u8 = 1; - -#[macro_export] -macro_rules! single_sql_return { - ($name: ident, $field: ident, $ty: ident) => { - #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] - pub struct $name { - pub $field: $ty, - } - - impl $name { - pub fn inner(self) -> $ty { - self.$field - } - } - }; -} - -single_sql_return!(TxHash, tx_hash, RbBytes); - -#[crud_table(table_name: "mercury_block")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct BlockTable { - pub block_hash: RbBytes, - pub block_number: u64, - pub version: u16, - pub compact_target: u32, - pub block_timestamp: u64, - pub epoch_number: u32, - pub epoch_index: u32, - pub epoch_length: u32, - pub parent_hash: RbBytes, - pub transactions_root: RbBytes, - pub proposals_hash: RbBytes, - pub uncles_hash: RbBytes, - pub uncles: RbBytes, - pub uncles_count: u32, - pub dao: RbBytes, - pub nonce: RbBytes, - pub proposals: RbBytes, -} - -impl From<&BlockView> for BlockTable { - fn from(block: &BlockView) -> Self { - let epoch = block.epoch(); - - BlockTable { - block_hash: to_rb_bytes(&block.hash().raw_data()), - block_number: block.number(), - version: block.version() as u16, - compact_target: block.compact_target(), - block_timestamp: block.timestamp(), - epoch_number: epoch.number() as u32, - epoch_index: epoch.index() as u32, - epoch_length: epoch.length() as u32, - parent_hash: to_rb_bytes(&block.parent_hash().raw_data()), - transactions_root: to_rb_bytes(&block.transactions_root().raw_data()), - proposals_hash: to_rb_bytes(&block.proposals_hash().raw_data()), - uncles_hash: to_rb_bytes(&block.extra_hash().raw_data()), - uncles: to_rb_bytes(block.uncles().data().as_slice()), - uncles_count: block.uncle_hashes().len() as u32, - dao: to_rb_bytes(&block.dao().raw_data()), - nonce: to_rb_bytes(&block.nonce().to_be_bytes()), - proposals: to_rb_bytes(&block.data().proposals().as_bytes()), - } - } -} - -#[crud_table(table_name: "mercury_transaction")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct TransactionTable { - pub id: i64, - pub tx_hash: RbBytes, - pub tx_index: u32, - pub input_count: u32, - pub output_count: u32, - pub block_number: u64, - pub block_hash: RbBytes, - pub tx_timestamp: u64, - pub version: u16, - pub cell_deps: RbBytes, - pub header_deps: RbBytes, - pub witnesses: RbBytes, -} - -impl TransactionTable { - pub fn from_view( - view: &TransactionView, - id: i64, - tx_index: u32, - block_hash: RbBytes, - block_number: u64, - tx_timestamp: u64, - ) -> Self { - TransactionTable { - id, - block_hash, - block_number, - tx_index, - tx_timestamp, - tx_hash: to_rb_bytes(&view.hash().raw_data()), - input_count: view.inputs().len() as u32, - output_count: view.outputs().len() as u32, - cell_deps: to_rb_bytes(&view.cell_deps().as_bytes()), - header_deps: to_rb_bytes(&view.header_deps().as_bytes()), - witnesses: to_rb_bytes(&view.witnesses().as_bytes()), - version: view.version() as u16, - } - } -} - -#[crud_table( - table_name: "mercury_cell" | formats_pg: " - consumed_block_number:{}::bigint, - consumed_tx_index:{}::bigint, - input_index:{}::bigint" -)] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct CellTable { - pub id: i64, - pub tx_hash: RbBytes, - pub output_index: u32, - pub tx_index: u32, - pub block_number: u64, - pub block_hash: RbBytes, - pub epoch_number: u32, - pub epoch_index: u32, - pub epoch_length: u32, - pub capacity: u64, - pub lock_hash: RbBytes, - pub lock_code_hash: RbBytes, - pub lock_args: RbBytes, - pub lock_script_type: u8, - pub type_hash: RbBytes, - pub type_code_hash: RbBytes, - pub type_args: RbBytes, - pub type_script_type: u8, - pub data: RbBytes, - pub consumed_block_number: Option, - pub consumed_block_hash: RbBytes, - pub consumed_tx_hash: RbBytes, - pub consumed_tx_index: Option, - pub input_index: Option, - pub since: RbBytes, -} - -impl From for CellTable { - fn from(s: LiveCellTable) -> Self { - CellTable { - id: s.id, - tx_hash: s.tx_hash, - output_index: s.output_index, - block_number: s.block_number, - block_hash: s.block_hash, - tx_index: s.tx_index, - epoch_number: s.epoch_number, - epoch_index: s.epoch_index, - epoch_length: s.epoch_length, - capacity: s.capacity, - lock_hash: s.lock_hash, - lock_code_hash: s.lock_code_hash, - lock_args: s.lock_args, - lock_script_type: s.lock_script_type, - type_hash: s.type_hash, - type_code_hash: s.type_code_hash, - type_args: s.type_args, - type_script_type: s.type_script_type, - data: s.data, - consumed_block_hash: empty_rb_bytes(), - consumed_block_number: None, - consumed_tx_hash: empty_rb_bytes(), - consumed_tx_index: None, - input_index: None, - since: empty_rb_bytes(), - } - } -} - -impl CellTable { - pub fn from_cell( - cell: &packed::CellOutput, - id: i64, - tx_hash: RbBytes, - output_index: u32, - tx_index: u32, - block_number: u64, - block_hash: RbBytes, - epoch: EpochNumberWithFraction, - cell_data: &[u8], - ) -> Self { - let mut ret = CellTable { - id, - tx_hash, - output_index, - tx_index, - block_number, - block_hash, - epoch_number: epoch.number() as u32, - epoch_index: epoch.index() as u32, - epoch_length: epoch.length() as u32, - capacity: cell.capacity().unpack(), - lock_hash: to_rb_bytes(&cell.lock().calc_script_hash().raw_data()), - lock_code_hash: to_rb_bytes(&cell.lock().code_hash().raw_data()), - lock_args: to_rb_bytes(&cell.lock().args().raw_data()), - lock_script_type: cell.lock().hash_type().into(), - type_hash: to_rb_bytes(&H256::default().0), - type_code_hash: empty_rb_bytes(), - type_args: empty_rb_bytes(), - type_script_type: 0u8, - data: to_rb_bytes(cell_data), - consumed_block_number: None, - consumed_block_hash: empty_rb_bytes(), - consumed_tx_index: None, - consumed_tx_hash: empty_rb_bytes(), - input_index: None, - since: empty_rb_bytes(), - }; - - if let Some(script) = cell.type_().to_opt() { - ret.set_type_script_info(&script); - } - - ret - } - - pub fn has_type_script(&self) -> bool { - self.type_hash.inner != H256::default().0.to_vec() - } - - pub fn set_type_script_info(&mut self, script: &packed::Script) { - self.type_hash = to_rb_bytes(&script.calc_script_hash().raw_data()); - self.type_code_hash = to_rb_bytes(&script.code_hash().raw_data()); - self.type_args = to_rb_bytes(&script.args().raw_data()); - self.type_script_type = script.hash_type().into(); - } - - pub fn to_lock_script_table(&self) -> ScriptTable { - ScriptTable { - script_hash: self.lock_hash.clone(), - script_args: self.lock_args.clone(), - script_args_len: self.lock_args.inner.len() as u32, - script_code_hash: self.lock_code_hash.clone(), - script_type: self.lock_script_type, - script_hash_160: to_rb_bytes(self.lock_hash.inner.split_at(BLAKE_160_HSAH_LEN).0), - } - } - - pub fn to_type_script_table(&self) -> ScriptTable { - let type_hash = self.type_hash.clone(); - let type_script_args = self.type_args.clone(); - - ScriptTable { - script_hash: type_hash.clone(), - script_hash_160: to_rb_bytes(type_hash.inner.split_at(BLAKE_160_HSAH_LEN).0), - script_args_len: type_script_args.inner.len() as u32, - script_args: type_script_args, - script_code_hash: self.type_code_hash.clone(), - script_type: self.type_script_type, - } - } - - fn build_detailed_cell(&self, data: Vec) -> DetailedCell { - let lock_script = packed::ScriptBuilder::default() - .code_hash(to_fixed_array::(&self.lock_code_hash.inner[0..32]).pack()) - .args(self.lock_args.inner.pack()) - .hash_type(packed::Byte::new(self.lock_script_type)) - .build(); - let type_script = if self.type_hash.inner == H256::default().0 { - None - } else { - Some( - packed::ScriptBuilder::default() - .code_hash( - H256::from_slice(&self.type_code_hash.inner) - .expect("get type code hash") - .pack(), - ) - .args(self.type_args.inner.pack()) - .hash_type(packed::Byte::new(self.type_script_type)) - .build(), - ) - }; - - let convert_hash = |b: &RbBytes| -> Option { - if b.inner.is_empty() { - None - } else { - Some(H256::from_slice(&b.inner).expect("convert hash")) - } - }; - - let convert_since = |b: &RbBytes| -> Option { - if b.inner.is_empty() { - None - } else { - Some(u64::from_be_bytes(to_fixed_array::<8>(&b.inner))) - } - }; - - DetailedCell { - epoch_number: EpochNumberWithFraction::new_unchecked( - self.epoch_number.into(), - self.epoch_index.into(), - self.epoch_length.into(), - ) - .full_value(), - block_number: self.block_number as u64, - block_hash: H256::from_slice(&self.block_hash.inner[0..32]).expect("get block hash"), - tx_index: self.tx_index, - out_point: packed::OutPointBuilder::default() - .tx_hash(to_fixed_array::<32>(&self.tx_hash.inner).pack()) - .index((self.output_index as u32).pack()) - .build(), - cell_output: packed::CellOutputBuilder::default() - .lock(lock_script) - .type_(type_script.pack()) - .capacity(self.capacity.pack()) - .build(), - cell_data: data.into(), - consumed_block_hash: convert_hash(&self.consumed_block_hash), - consumed_block_number: self.consumed_block_number, - consumed_tx_hash: convert_hash(&self.consumed_tx_hash), - consumed_tx_index: self.consumed_tx_index, - consumed_input_index: self.input_index, - since: convert_since(&self.since), - } - } -} - -impl From for DetailedCell { - fn from(cell_table: CellTable) -> DetailedCell { - let cell_data = cell_table.data.inner.clone(); - cell_table.build_detailed_cell(cell_data) - } -} - -#[crud_table(table_name: "mercury_live_cell")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct LiveCellTable { - pub id: i64, - pub tx_hash: RbBytes, - pub output_index: u32, - pub tx_index: u32, - pub block_number: u64, - pub block_hash: RbBytes, - pub epoch_number: u32, - pub epoch_index: u32, - pub epoch_length: u32, - pub capacity: u64, - pub lock_hash: RbBytes, - pub lock_code_hash: RbBytes, - pub lock_args: RbBytes, - pub lock_script_type: u8, - pub type_hash: RbBytes, - pub type_code_hash: RbBytes, - pub type_args: RbBytes, - pub type_script_type: u8, - pub data: RbBytes, -} - -impl From for LiveCellTable { - fn from(s: CellTable) -> Self { - LiveCellTable { - id: s.id, - tx_hash: s.tx_hash, - output_index: s.output_index, - block_number: s.block_number, - block_hash: s.block_hash, - tx_index: s.tx_index, - epoch_number: s.epoch_number, - epoch_index: s.epoch_index, - epoch_length: s.epoch_length, - capacity: s.capacity, - lock_hash: s.lock_hash, - lock_code_hash: s.lock_code_hash, - lock_args: s.lock_args, - lock_script_type: s.lock_script_type, - type_hash: s.type_hash, - type_code_hash: s.type_code_hash, - type_args: s.type_args, - type_script_type: s.type_script_type, - data: s.data, - } - } -} - -#[crud_table(table_name: "mercury_indexer_cell")] -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -pub struct IndexerCellTable { - pub id: i64, - pub block_number: u64, - pub io_type: u8, - pub io_index: u32, - pub tx_hash: RbBytes, - pub tx_index: u32, - pub lock_hash: RbBytes, - pub lock_code_hash: RbBytes, - pub lock_args: RbBytes, - pub lock_script_type: u8, - pub type_hash: RbBytes, - pub type_code_hash: RbBytes, - pub type_args: RbBytes, - pub type_script_type: u8, -} - -impl Ord for IndexerCellTable { - fn cmp(&self, other: &Self) -> Ordering { - if self.block_number != other.block_number { - self.block_number.cmp(&other.block_number) - } else if self.tx_index != other.tx_index { - self.tx_index.cmp(&other.tx_index) - } else if self.io_type != other.io_type { - self.io_type.cmp(&other.io_type) - } else { - self.io_index.cmp(&other.io_index) - } - } -} - -impl PartialOrd for IndexerCellTable { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl IndexerCellTable { - pub fn new_with_empty_scripts( - block_number: u64, - io_type: u8, - io_index: u32, - tx_hash: RbBytes, - tx_index: u32, - ) -> Self { - IndexerCellTable { - id: 0, - block_number, - io_type, - io_index, - tx_hash, - tx_index, - lock_hash: empty_rb_bytes(), - lock_code_hash: empty_rb_bytes(), - lock_args: empty_rb_bytes(), - lock_script_type: 0, - type_hash: empty_rb_bytes(), - type_code_hash: empty_rb_bytes(), - type_args: empty_rb_bytes(), - type_script_type: 0, - } - } - - pub fn update_by_cell_table(mut self, cell_table: &CellTable) -> Self { - self.lock_hash = cell_table.lock_hash.clone(); - self.lock_code_hash = cell_table.lock_code_hash.clone(); - self.lock_args = cell_table.lock_args.clone(); - self.lock_script_type = cell_table.lock_script_type; - self.type_hash = cell_table.type_hash.clone(); - self.type_code_hash = cell_table.type_code_hash.clone(); - self.type_args = cell_table.type_args.clone(); - self.type_script_type = cell_table.type_script_type; - self - } -} - -#[crud_table(table_name: "mercury_script")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ScriptTable { - pub script_hash: RbBytes, - pub script_hash_160: RbBytes, - pub script_code_hash: RbBytes, - pub script_args: RbBytes, - pub script_type: u8, - pub script_args_len: u32, -} - -#[allow(clippy::from_over_into)] -impl Into for ScriptTable { - fn into(self) -> packed::Script { - packed::ScriptBuilder::default() - .code_hash( - H256::from_slice(&self.script_code_hash.inner[0..32]) - .expect("get code hash h256") - .pack(), - ) - .args(self.script_args.inner.pack()) - .hash_type(packed::Byte::new(self.script_type)) - .build() - } -} - -impl Hash for ScriptTable { - fn hash(&self, state: &mut H) { - self.script_hash.inner.hash(state); - } -} - -impl PartialEq for ScriptTable { - fn eq(&self, other: &Self) -> bool { - self.script_hash == other.script_hash - && self.script_code_hash == other.script_code_hash - && self.script_type == other.script_type - && self.script_args == other.script_args - } -} - -impl Eq for ScriptTable {} - -impl ScriptTable { - pub fn as_bytes(&self) -> Vec { - let mut encode = self.script_hash.inner.clone(); - encode.extend_from_slice(&self.script_hash_160.inner); - encode.extend_from_slice(&self.script_code_hash.inner); - encode.extend_from_slice(&self.script_args_len.to_be_bytes()); - encode.push(self.script_type); - encode.extend_from_slice(&self.script_args.inner); - encode - } - - pub fn from_bytes(bytes: &[u8]) -> Self { - ScriptTable { - script_hash: to_rb_bytes(&bytes[0..32]), - script_hash_160: to_rb_bytes(&bytes[32..52]), - script_code_hash: to_rb_bytes(&bytes[52..84]), - script_args: to_rb_bytes(&bytes[89..]), - script_args_len: u32::from_be_bytes(to_fixed_array::<4>(&bytes[84..88])), - script_type: bytes[88], - } - } -} - -#[crud_table(table_name: "mercury_sync_status")] -#[derive(Serialize, Deserialize, Clone, Debug, Hash, PartialEq, Eq)] -pub struct SyncStatus { - pub block_number: u64, -} - -impl SyncStatus { - pub fn new(block_number: u64) -> Self { - SyncStatus { block_number } - } -} - -#[crud_table(table_name: "mercury_canonical_chain")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct CanonicalChainTable { - pub block_number: u64, - pub block_hash: RbBytes, -} - -impl Default for CanonicalChainTable { - fn default() -> Self { - CanonicalChainTable { - block_number: 0, - block_hash: empty_rb_bytes(), - } - } -} - -impl PartialEq for CanonicalChainTable { - fn eq(&self, other: &Self) -> bool { - self.block_number == other.block_number && self.block_hash == other.block_hash - } -} - -impl Eq for CanonicalChainTable {} - -impl PartialOrd for CanonicalChainTable { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for CanonicalChainTable { - fn cmp(&self, other: &Self) -> Ordering { - self.block_number.cmp(&other.block_number) - } -} - -impl CanonicalChainTable { - pub fn new(block_number: u64, block_hash: RbBytes) -> Self { - CanonicalChainTable { - block_number, - block_hash, - } - } -} - -#[crud_table(table_name: "mercury_registered_address")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct RegisteredAddressTable { - pub lock_hash: RbBytes, - pub address: String, -} - -impl RegisteredAddressTable { - pub fn new(lock_hash: RbBytes, address: String) -> Self { - RegisteredAddressTable { lock_hash, address } - } -} - -pub fn decode_since(input: &[u8]) -> u64 { - u64::from_be_bytes(to_fixed_array::<8>(input)) -} - -pub(crate) struct ConsumedInfo { - pub(crate) out_point: packed::OutPoint, - pub(crate) consumed_block_number: u64, - pub(crate) consumed_block_hash: RbBytes, - pub(crate) consumed_tx_hash: RbBytes, - pub(crate) consumed_tx_index: u32, - pub(crate) input_index: u32, - pub(crate) since: RbBytes, -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] -pub struct IndexerTxHash { - pub id: i64, - pub tx_hash: RbBytes, -} - -impl PartialOrd for IndexerTxHash { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for IndexerTxHash { - fn cmp(&self, other: &Self) -> Ordering { - self.id.cmp(&other.id) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rand::random; - - fn rand_bytes(len: usize) -> Vec { - (0..len).map(|_| random::()).collect() - } - - fn generate_script_table(args: Vec) -> ScriptTable { - ScriptTable { - script_hash: to_rb_bytes(&rand_bytes(32)), - script_hash_160: to_rb_bytes(&rand_bytes(20)), - script_code_hash: to_rb_bytes(&rand_bytes(32)), - script_args: to_rb_bytes(&args), - script_args_len: args.len() as u32, - script_type: 1, - } - } - - #[test] - fn test_script_table_codec() { - let script = generate_script_table(rand_bytes(20)); - let bytes = script.as_bytes(); - let decode = ScriptTable::from_bytes(&bytes); - - assert_eq!(script, decode); - - let script = generate_script_table(vec![]); - let bytes = script.as_bytes(); - let decode = ScriptTable::from_bytes(&bytes); - - assert_eq!(script, decode); - } -} diff --git a/core/storage/src/relational/tests/fetch_mod_test.rs b/core/storage/src/relational/tests/fetch_mod_test.rs index 147573b67..e6711f04e 100644 --- a/core/storage/src/relational/tests/fetch_mod_test.rs +++ b/core/storage/src/relational/tests/fetch_mod_test.rs @@ -6,7 +6,6 @@ async fn test_query_live_cells() { let ret = pool .query_live_cells( - Context::new(), None, vec![], vec![], @@ -28,7 +27,6 @@ async fn test_query_live_cells() { let ret = pool .query_live_cells( - Context::new(), None, vec![], vec![], @@ -54,25 +52,36 @@ async fn test_query_indexer_cells() { let pool = connect_and_insert_blocks().await; let ret = pool - .query_indexer_cells( + .query_indexer_transactions( vec![], vec![], Some(Range::new(0, 1)), PaginationRequest { cursor: None, order: Order::Desc, - limit: Some(2), + limit: None, skip: None, return_count: true, }, ) .await .unwrap(); - assert_eq!(2, ret.response.len()); + let txs_input_count = ret + .response + .iter() + .filter(|tx| tx.io_type == IOType::Input) + .count(); + let txs_output_count = ret + .response + .iter() + .filter(|tx| tx.io_type == IOType::Output) + .count(); assert_eq!(Some(13), ret.count); + assert_eq!(1, txs_input_count); + assert_eq!(12, txs_output_count); let ret = pool - .query_indexer_cells( + .query_indexer_transactions( vec![], vec![], Some(Range::new(0, 10)), @@ -130,3 +139,69 @@ async fn test_query_transactions() { assert_eq!(3, ret.response.len()); assert_eq!(None, ret.count); } + +#[tokio::test] +async fn test_get_scripts() { + use ckb_types::bytes::Bytes; + use common::hash::blake2b_256_to_160; + + let code_hash = "9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8"; + let args = "3f1573b44218d4c12a91919a58a863be415a2bc3"; + let script_hash = "8abf38905f28fd36088ebbbfdb021c2f4a853d2c9e8809338a381561a77bb241"; + + let pool = connect_and_insert_blocks().await; + let args = Bytes::from(hex::decode(args).unwrap()); + let script_hash = blake2b_256_to_160(&H256::from_str(script_hash).unwrap()); + let code_hash = H256::from_str(code_hash).unwrap(); + + let ret = pool + .get_scripts( + Context::new(), + vec![script_hash.clone()], + vec![], + None, + vec![], + ) + .await + .unwrap(); + assert_eq!(1, ret.len()); + + let ret = pool + .get_scripts(Context::new(), vec![], vec![], None, vec![args.clone()]) + .await + .unwrap(); + assert_eq!(1, ret.len()); + + let ret = pool + .get_scripts( + Context::new(), + vec![script_hash], + vec![code_hash], + Some(args.len()), + vec![args], + ) + .await + .unwrap(); + assert_eq!(1, ret.len()); +} + +#[tokio::test] +async fn test_get_scripts_by_partial_arg() { + use ckb_types::bytes::Bytes; + + let code_hash = "9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8"; + let args = "3f1573b44218d4c12a91919a58a863be415a2bc3"; + let script_hash = "8abf38905f28fd36088ebbbfdb021c2f4a853d2c9e8809338a381561a77bb241"; + + let pool = connect_and_insert_blocks().await; + let args = Bytes::from(hex::decode(args).unwrap()); + let script_hash = H256::from_str(script_hash).unwrap(); + let code_hash = H256::from_str(code_hash).unwrap(); + + let ret = pool + .get_scripts_by_partial_arg(Context::new(), &code_hash, args, (0, 20)) + .await + .unwrap(); + assert_eq!(1, ret.len()); + assert_eq!(script_hash, ret[0].calc_script_hash().unpack()) +} diff --git a/core/storage/src/relational/tests/get_block_test.rs b/core/storage/src/relational/tests/get_block_test.rs index 55e05754c..feff5f264 100644 --- a/core/storage/src/relational/tests/get_block_test.rs +++ b/core/storage/src/relational/tests/get_block_test.rs @@ -1,7 +1,9 @@ use super::*; +use sqlx::Row; + #[tokio::test] -async fn test_get_genesis_block_header() { +async fn test_get_block_header_of_genesis() { let pool = connect_and_insert_blocks().await; let res = pool .get_block_header(Context::new(), None, Some(0)) @@ -23,47 +25,27 @@ async fn test_get_block_header_by_number() { } #[tokio::test] -async fn test_get_genesis_block() { - let pool = connect_and_insert_blocks().await; - let res = pool.get_block(Context::new(), None, Some(0)).await.unwrap(); - let block: BlockView = read_block_view(0, BLOCK_DIR.to_string()).into(); - assert_eq!(block.data(), res.data()); -} - -#[tokio::test] -async fn test_get_block_by_number() { - let pool = connect_and_insert_blocks().await; - let res = pool.get_block(Context::new(), None, Some(1)).await.unwrap(); - let block: BlockView = read_block_view(1, BLOCK_DIR.to_string()).into(); - assert_eq!(block.data(), res.data()); -} - -#[tokio::test] -async fn test_get_block_info() { +async fn test_get_simple_block() { let pool = connect_and_insert_blocks().await; let block_table = pool.query_block_by_number(0).await.unwrap(); - let tx_tables = pool - .query_transactions_by_block_hash(&block_table.block_hash) + let block_hash = H256::from_slice(block_table.get("block_hash")).unwrap(); + let tx_hashes = pool + .query_transaction_hashes_by_block_hash(block_hash.as_bytes()) .await .unwrap(); - let tx_hashes: Vec = tx_tables - .iter() - .map(|tx| rb_bytes_to_h256(&tx.tx_hash)) - .collect(); - let block_info = pool .get_simple_block(Context::new(), None, Some(0)) .await .unwrap(); assert_eq!( - block_table.block_hash, - to_rb_bytes(block_info.block_hash.as_bytes()) + &block_table.get::, _>("block_hash"), + block_info.block_hash.as_bytes() ); assert_eq!(tx_hashes, block_info.transactions); } #[tokio::test] -async fn test_get_genesis_block_hash() { +async fn test_get_block_of_genesis() { let pool = connect_and_insert_blocks().await; // from json deserialization @@ -81,7 +63,7 @@ async fn test_get_genesis_block_hash() { // from block table let block_table = pool.query_block_by_number(0).await.unwrap(); - let block_hash_from_table = rb_bytes_to_h256(&block_table.block_hash); + let block_hash_from_table = bytes_to_h256(block_table.get("block_hash")); println!( "hash in block table: {:?}", block_hash_from_table.to_string() @@ -95,10 +77,11 @@ async fn test_get_genesis_block_hash() { println!("block hash is {:?}", block_built_hash.to_string()); assert_eq!(block_hash_from_json, block_built_hash); + assert_eq!(block_core_view.data(), res.data()); } #[tokio::test] -async fn test_get_block_hash() { +async fn test_get_block_by_number() { let pool = connect_and_insert_blocks().await; // from json deserialization @@ -116,7 +99,7 @@ async fn test_get_block_hash() { // from block table let block_table = pool.query_block_by_number(1).await.unwrap(); - let block_hash_from_table = rb_bytes_to_h256(&block_table.block_hash); + let block_hash_from_table = bytes_to_h256(block_table.get("block_hash")); println!( "hash in block table: {:?}", block_hash_from_table.to_string() @@ -125,9 +108,75 @@ async fn test_get_block_hash() { assert_eq!(block_hash_from_json, block_hash_from_table); // from built block view + let count = pool.block_count(Context::new()).await.unwrap(); let res = pool.get_block(Context::new(), None, Some(1)).await.unwrap(); let block_built_hash: H256 = res.hash().unpack(); println!("block hash is {:?}", block_built_hash.to_string()); + assert_eq!(10, count); assert_eq!(block_hash_from_json, block_built_hash); + assert_eq!(block_core_view.data(), res.data()); +} + +#[tokio::test] +async fn test_get_block_by_hash() { + let pool = connect_and_insert_blocks().await; + + // from built block view + let block_hash = + H256::from_str("d5ac7cf8c34a975bf258a34f1c2507638487ab71aa4d10a9ec73704aa3abf9cd").unwrap(); + let res = pool + .get_block(Context::new(), Some(block_hash.clone()), None) + .await + .unwrap(); + let block_built_hash: H256 = res.hash().unpack(); + assert_eq!(block_hash, block_built_hash); + + // get tip + let res = pool.get_block(Context::new(), None, None).await.unwrap(); + let block_built_hash: H256 = res.hash().unpack(); + assert_eq!( + "953761d56c03bfedf5e70dde0583470383184c41331f709df55d4acab5358640".to_string(), + block_built_hash.to_string() + ); + + // query block hash and block number at the same time + let block_number_9_hash = + H256::from_str("953761d56c03bfedf5e70dde0583470383184c41331f709df55d4acab5358640").unwrap(); + let res_1 = pool + .get_block(Context::new(), Some(block_number_9_hash.clone()), Some(9)) + .await; + let res_2 = pool + .get_block(Context::new(), Some(block_number_9_hash), Some(1)) + .await; + assert!(res_1.is_ok()); + assert!(res_2.is_err()); +} + +#[tokio::test] +async fn test_query_tip() { + let pool = connect_and_create_tables().await; + let res = pool.query_tip().await.unwrap(); + assert!(res.is_none()); + + let pool = connect_and_insert_blocks().await; + let (block_number, block_hash) = pool.query_tip().await.unwrap().unwrap(); + assert_eq!(9, block_number); + assert_eq!( + "953761d56c03bfedf5e70dde0583470383184c41331f709df55d4acab5358640".to_string(), + block_hash.to_string() + ); +} + +#[tokio::test] +async fn test_get_canonical_block_hash() { + let pool = connect_and_insert_blocks().await; + let res = pool + .get_canonical_block_hash(Context::new(), 0) + .await + .unwrap(); + assert_eq!( + h256!("0x10639e0895502b5688a6be8cf69460d76541bfa4821629d86d62ba0aae3f9606"), + res + ); } diff --git a/core/storage/src/relational/tests/get_cell_test.rs b/core/storage/src/relational/tests/get_cell_test.rs index 5198c4481..4d1deb450 100644 --- a/core/storage/src/relational/tests/get_cell_test.rs +++ b/core/storage/src/relational/tests/get_cell_test.rs @@ -51,32 +51,195 @@ async fn test_get_cells_pagination_return_count() { #[tokio::test] async fn test_is_not_live_cell() { - let pool = connect_and_insert_blocks().await; - let mut conn = pool.pool.acquire().await.unwrap(); + let storage = connect_and_insert_blocks().await; let tx_hash = - hex::decode("e2fb199810d49a4d8beec56718ba2593b665db9d52299a0f9e6e75416d73ff5c").unwrap(); - let res = sql::is_live_cell(&mut conn, &to_rb_bytes(&tx_hash), &5) - .await - .unwrap(); + hex::decode("8f8c79eb6671709633fe6a46de93c0fedc9c1b8a6527a18d3983879542635c9f").unwrap(); + let query = sqlx::query( + "SELECT id FROM mercury_live_cell + WHERE tx_hash = $1 AND output_index = $2", + ) + .bind(tx_hash) + .bind(5); + let pool = storage.sqlx_pool.get_pool().unwrap(); + let res = query.fetch_optional(pool).await.unwrap(); assert!(res.is_none()); } #[tokio::test] async fn test_is_live_cell() { - let pool = connect_and_insert_blocks().await; - let mut conn = pool.pool.acquire().await.unwrap(); + let storage = connect_and_insert_blocks().await; let tx_hash = hex::decode("8f8c79eb6671709633fe6a46de93c0fedc9c1b8a6527a18d3983879542635c9f").unwrap(); - let res = sql::is_live_cell(&mut conn, &to_rb_bytes(&tx_hash), &0) + let query = sqlx::query( + "SELECT id FROM mercury_live_cell + WHERE tx_hash = $1 AND output_index = $2", + ) + .bind(tx_hash) + .bind(0); + let pool = storage.sqlx_pool.get_pool().unwrap(); + let res = query.fetch_optional(pool).await.unwrap(); + assert!(res.is_some()); +} + +#[tokio::test] +async fn test_get_cells_out_point() { + let pool = connect_and_insert_blocks().await; + + let tx_hash = + h256!("0x8f8c79eb6671709633fe6a46de93c0fedc9c1b8a6527a18d3983879542635c9f").pack(); + let out_point = packed::OutPoint::new(tx_hash, 5); + let cells = pool + .get_cells( + Context::new(), + Some(out_point), + vec![], + vec![], + None, + PaginationRequest { + cursor: Some(u64::MAX), + order: Order::Desc, + limit: None, + skip: None, + return_count: true, + }, + ) .await .unwrap(); - assert!(res.is_some()); + assert_eq!(Some(1), cells.count); + assert_eq!(Some(0), cells.response[0].consumed_block_number); + assert_eq!(Some(1), cells.response[0].consumed_tx_index); + + let tx_hash = + h256!("0xf8de3bb47d055cdf460d93a2a6e1b05f7432f9777c8c474abf4eec1d4aee5d37").pack(); + let out_point = packed::OutPoint::new(tx_hash, 1); + let cells = pool + .get_cells( + Context::new(), + Some(out_point), + vec![], + vec![], + None, + PaginationRequest { + cursor: Some(u64::MAX), + order: Order::Desc, + limit: None, + skip: None, + return_count: true, + }, + ) + .await + .unwrap(); + assert_eq!(Some(1), cells.count); + assert_eq!(None, cells.response[0].consumed_tx_index); + assert_eq!(None, cells.response[0].consumed_block_number); +} + +#[tokio::test] +async fn test_get_cells_pagination_cursor() { + let pool = connect_and_insert_blocks().await; + + let cells = pool + .get_cells( + Context::new(), + None, + vec![], + vec![], + Some(Range::new(0, 9)), + PaginationRequest { + cursor: None, + order: Order::Asc, + limit: Some(2), + skip: None, + return_count: true, + }, + ) + .await + .unwrap(); + let index_0: u32 = cells.response[0].out_point.index().unpack(); + let index_1: u32 = cells.response[1].out_point.index().unpack(); + + assert_eq!(Some(12), cells.count); + assert_eq!(2, cells.response.len()); + assert_eq!(0, index_0); + assert_eq!(1, index_1); + + let cells = pool + .get_cells( + Context::new(), + None, + vec![], + vec![], + Some(Range::new(0, 9)), + PaginationRequest { + cursor: cells.next_cursor, + order: Order::Asc, + limit: Some(2), + skip: None, + return_count: true, + }, + ) + .await + .unwrap(); + let index_2: u32 = cells.response[0].out_point.index().unpack(); + let index_3: u32 = cells.response[1].out_point.index().unpack(); + + assert_eq!(Some(12), cells.count); + assert_eq!(2, cells.response.len()); + assert_eq!(2, index_2); + assert_eq!(3, index_3); } #[tokio::test] -async fn test_to_rb_bytes() { - let tx_hash = hex::decode("63000000000000000000000000000000").unwrap(); - let ret_rbatis_bytes = to_rb_bytes(&tx_hash); - let ret_bytes = Bytes::from(tx_hash); - assert_eq!(ret_rbatis_bytes.len(), ret_bytes.len()); +async fn test_get_cells_pagination_skip() { + let pool = connect_and_insert_blocks().await; + + let cells = pool + .get_cells( + Context::new(), + None, + vec![], + vec![], + Some(Range::new(0, 9)), + PaginationRequest { + cursor: None, + order: Order::Asc, + limit: Some(2), + skip: Some(4), + return_count: false, + }, + ) + .await + .unwrap(); + let index_4: u32 = cells.response[0].out_point.index().unpack(); + let index_5: u32 = cells.response[1].out_point.index().unpack(); + + assert_eq!(None, cells.count); + assert_eq!(2, cells.response.len()); + assert_eq!(4, index_4); + assert_eq!(5, index_5); + + let cells = pool + .get_cells( + Context::new(), + None, + vec![], + vec![], + Some(Range::new(0, 9)), + PaginationRequest { + cursor: None, + order: Order::Desc, + limit: Some(2), + skip: Some(4), + return_count: false, + }, + ) + .await + .unwrap(); + let index_7: u32 = cells.response[0].out_point.index().unpack(); + let index_6: u32 = cells.response[1].out_point.index().unpack(); + + assert_eq!(None, cells.count); + assert_eq!(2, cells.response.len()); + assert_eq!(7, index_7); + assert_eq!(6, index_6); } diff --git a/core/storage/src/relational/tests/get_historical_live_cells_test.rs b/core/storage/src/relational/tests/get_historical_live_cells_test.rs index 1b9a75184..3d410d4d6 100644 --- a/core/storage/src/relational/tests/get_historical_live_cells_test.rs +++ b/core/storage/src/relational/tests/get_historical_live_cells_test.rs @@ -57,7 +57,7 @@ async fn test_get_historical_live_cells_desc() { PaginationRequest { cursor: Some(u64::MAX), order: Order::Desc, - limit: Some(u64::MAX), + limit: Some(u16::MAX), skip: None, return_count: false, }, @@ -205,6 +205,8 @@ async fn test_get_historical_live_cells_asc() { .await .unwrap(); assert_eq!(2, ret.response.len()); + let index: u32 = ret.response[0].out_point.index().unpack(); + assert_eq!(8u32, index); let index: u32 = ret.response[1].out_point.index().unpack(); assert_eq!(9u32, index); diff --git a/core/storage/src/relational/tests/get_tx_test.rs b/core/storage/src/relational/tests/get_tx_test.rs index cc19c1e08..b84899f53 100644 --- a/core/storage/src/relational/tests/get_tx_test.rs +++ b/core/storage/src/relational/tests/get_tx_test.rs @@ -3,6 +3,45 @@ use super::*; #[tokio::test] async fn test_get_txs() { let pool = connect_and_insert_blocks().await; + + let lock_hash: H256 = caculate_lock_hash( + "709f3fda12f561cfacf92273c57a98fede188a3f1a59b1f888d113f9cce08649", + "b73961e46d9eb118d3de1d1e8f30b3af7bbf3160", + ScriptHashType::Data, + ); + + let txs_from_db = pool + .get_transactions( + Context::new(), + None, + vec![lock_hash], + vec![], + Some(Range::new(0, 9)), + false, + PaginationRequest::new(Some(0), Order::Asc, Some(20), None, true), + ) + .await + .unwrap() + .response; + let tx_hashes_from_db: Vec = txs_from_db + .iter() + .map(|tx| tx.transaction_with_status.transaction.clone().unwrap().hash) + .collect(); + + assert_eq!(2, txs_from_db.len()); + assert_eq!( + "8f8c79eb6671709633fe6a46de93c0fedc9c1b8a6527a18d3983879542635c9f", + &tx_hashes_from_db[0].to_string() + ); + assert_eq!( + "f8de3bb47d055cdf460d93a2a6e1b05f7432f9777c8c474abf4eec1d4aee5d37", + &tx_hashes_from_db[1].to_string() + ); +} + +#[tokio::test] +async fn test_get_txs_by_block_range() { + let pool = connect_and_insert_blocks().await; let txs_from_db = pool .get_transactions( Context::new(), @@ -38,7 +77,7 @@ async fn test_get_spent_transaction_hash() { let block: BlockView = read_block_view(0, BLOCK_DIR.to_string()).into(); let tx = &block.transaction(0).unwrap(); let outpoint = ckb_jsonrpc_types::OutPoint { - tx_hash: tx.hash().unpack(), // 0xb50ef2272f9f72b11e21ec12bd1b8fc9136cafc25c197b6fd4c2eb4b19fa905c + tx_hash: tx.hash().unpack(), index: 0u32.into(), }; let res = pool @@ -78,3 +117,44 @@ async fn test_get_tx_timestamp() { assert_eq!(timestamps_from_json, timestamps); } + +#[tokio::test] +async fn test_get_simple_transaction_by_hash() { + let pool = connect_and_insert_blocks_16().await; + + let simple_tx = pool + .get_simple_transaction_by_hash( + Context::new(), + h256!("0xa6789f42b0568b1872e5a5858f0c42148dd8d313f844252f5fe3dfe556958ba9"), + ) + .await + .unwrap(); + + assert_eq!( + "fb27201670e48f65b93b58c4cac7348c54554ad831ed5c1b386c9bd3c24fa911".to_string(), + simple_tx.block_hash.to_string() + ); + assert_eq!(0, simple_tx.tx_index); + assert_eq!(12, simple_tx.block_number); + println!("{:?}", simple_tx.epoch_number.to_string()); +} + +#[tokio::test] +async fn test_query_spent_tx_hash() { + let pool = connect_and_insert_blocks_16().await; + + let tx_hash = + h256!("0xf8de3bb47d055cdf460d93a2a6e1b05f7432f9777c8c474abf4eec1d4aee5d37").pack(); + let out_point = packed::OutPoint::new(tx_hash, 1); + let spent_tx = pool.query_spent_tx_hash(out_point).await.unwrap(); + assert!(spent_tx.is_none()); + + let tx_hash = + h256!("0x8f8c79eb6671709633fe6a46de93c0fedc9c1b8a6527a18d3983879542635c9f").pack(); + let out_point = packed::OutPoint::new(tx_hash, 5); + let spent_tx = pool.query_spent_tx_hash(out_point).await.unwrap().unwrap(); + assert_eq!( + spent_tx, + h256!("0xf8de3bb47d055cdf460d93a2a6e1b05f7432f9777c8c474abf4eec1d4aee5d37") + ); +} diff --git a/core/storage/src/relational/tests/mod.rs b/core/storage/src/relational/tests/mod.rs index e261d01f7..43eb17db2 100644 --- a/core/storage/src/relational/tests/mod.rs +++ b/core/storage/src/relational/tests/mod.rs @@ -6,38 +6,21 @@ mod get_tx_test; mod other_test; mod single_sql_test; -use crate::relational::fetch::rb_bytes_to_h256; -use crate::relational::{sql, to_rb_bytes, DBDriver, PaginationRequest}; +use crate::relational::fetch::bytes_to_h256; +use crate::relational::{DBDriver, PaginationRequest}; use crate::{relational::RelationalStorage, Storage}; use ckb_jsonrpc_types::BlockView as JsonBlockView; use ckb_types::core::ScriptHashType; -use ckb_types::{bytes::Bytes, core::BlockView, h160, packed, prelude::*, H160, H256}; +use ckb_types::{core::BlockView, h160, h256, packed, prelude::*, H160, H256}; use common::{Context, Order, Range}; +use core_rpc_types::IOType; use std::str::FromStr; const MEMORY_DB: &str = ":memory:"; -const POSTGRES_DB: &str = "127.0.0.1"; const BLOCK_DIR: &str = "../../devtools/test_data/blocks/"; -pub async fn connect_pg_pool() -> RelationalStorage { - init_debugger(true); - let pool = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30, log::LevelFilter::Debug); - pool.connect( - DBDriver::PostgreSQL, - "mercury", - POSTGRES_DB, - 8432, - "postgres", - "123456", - ) - .await - .unwrap(); - - pool -} - fn init_debugger(option: bool) { if option { env_logger::builder() @@ -48,33 +31,56 @@ fn init_debugger(option: bool) { async fn connect_sqlite() -> RelationalStorage { init_debugger(false); - let pool = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30, log::LevelFilter::Info); + let mut pool = RelationalStorage::new(0, 0, 100, 0, 60, 1800, 30); pool.connect(DBDriver::SQLite, MEMORY_DB, "", 0, "", "") .await .unwrap(); pool } -pub fn read_block_view(number: u64, dir_path: String) -> JsonBlockView { - let file_name = number.to_string() + ".json"; - let path = dir_path + file_name.as_str(); - serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +async fn connect_and_create_tables() -> RelationalStorage { + let pool = connect_sqlite().await; + let tx = pool.sqlx_pool.transaction().await.unwrap(); + xsql_test::create_tables(tx).await.unwrap(); + pool } async fn connect_and_insert_blocks() -> RelationalStorage { - let pool = connect_sqlite().await; - let mut tx = pool.pool.transaction().await.unwrap(); - xsql_test::create_tables(&mut tx).await.unwrap(); + let storage = connect_sqlite().await; + + let tx = storage.sqlx_pool.transaction().await.unwrap(); + xsql_test::create_tables(tx).await.unwrap(); let data_path = String::from(BLOCK_DIR); for i in 0..10 { - pool.append_block(Context::new(), read_block_view(i, data_path.clone()).into()) + storage + .append_block(read_block_view(i, data_path.clone()).into()) + .await + .unwrap(); + } + storage +} + +async fn connect_and_insert_blocks_16() -> RelationalStorage { + let pool = connect_sqlite().await; + let tx = pool.sqlx_pool.transaction().await.unwrap(); + xsql_test::create_tables(tx).await.unwrap(); + + let data_path = String::from(BLOCK_DIR); + for i in 0..16 { + pool.append_block(read_block_view(i, data_path.clone()).into()) .await .unwrap(); } pool } +pub fn read_block_view(number: u64, dir_path: String) -> JsonBlockView { + let file_name = number.to_string() + ".json"; + let path = dir_path + file_name.as_str(); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +} + fn caculate_lock_hash(code_hash: &str, args: &str, script_hash_type: ScriptHashType) -> H256 { let code_hash = H256::from_str(code_hash).unwrap(); let args = H160::from_str(args).unwrap(); diff --git a/core/storage/src/relational/tests/other_test.rs b/core/storage/src/relational/tests/other_test.rs index 65b1018d8..19d840ec3 100644 --- a/core/storage/src/relational/tests/other_test.rs +++ b/core/storage/src/relational/tests/other_test.rs @@ -2,21 +2,95 @@ use super::*; #[tokio::test] async fn test_insert() { - let _pool = connect_and_insert_blocks().await; + let storage = connect_and_insert_blocks().await; + + let pool = storage.get_pool(); + assert_eq!(10, pool.fetch_count("mercury_block").await.unwrap()); + assert_eq!(11, pool.fetch_count("mercury_transaction").await.unwrap()); + assert_eq!(12, pool.fetch_count("mercury_cell").await.unwrap()); + assert_eq!(11, pool.fetch_count("mercury_live_cell").await.unwrap()); + assert_eq!(13, pool.fetch_count("mercury_indexer_cell").await.unwrap()); + assert_eq!(9, pool.fetch_count("mercury_script").await.unwrap()); + assert_eq!( + 10, + pool.fetch_count("mercury_canonical_chain").await.unwrap() + ); + assert_eq!( + 0, + pool.fetch_count("mercury_registered_address") + .await + .unwrap() + ); + assert_eq!(10, pool.fetch_count("mercury_sync_status").await.unwrap()); + assert_eq!(0, pool.fetch_count("mercury_in_update").await.unwrap()); } #[tokio::test] async fn test_remove_all() { - let pool = connect_and_insert_blocks().await; - let mut tx = pool.pool.transaction().await.unwrap(); - xsql_test::delete_all_data(&mut tx).await.unwrap(); + let storage = connect_and_insert_blocks().await; + + let tx = storage.sqlx_pool.transaction().await.unwrap(); + xsql_test::delete_all_data(tx).await.unwrap(); + + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_block") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_transaction") + .await + .unwrap() + ); + assert_eq!( + 0, + storage.sqlx_pool.fetch_count("mercury_cell").await.unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_live_cell") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_indexer_cell") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_script") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_canonical_chain") + .await + .unwrap() + ); } #[tokio::test] async fn test_register_addresses() { let pool = connect_sqlite().await; - let mut tx = pool.pool.transaction().await.unwrap(); - xsql_test::create_tables(&mut tx).await.unwrap(); + let tx = pool.sqlx_pool.transaction().await.unwrap(); + xsql_test::create_tables(tx).await.unwrap(); let lock_hash = h160!("0xb39bbc0b3673c7d36450bc14cfcdad2d559c6c64"); let address = String::from("ckb1qyqt8xaupvm8837nv3gtc9x0ekkj64vud3jqfwyw5v"); @@ -44,15 +118,161 @@ async fn test_get_db_info() { assert_eq!(res.conn_size, 100); } -#[ignore] #[tokio::test] async fn test_get_tx_hash() { - let rdb = connect_pg_pool().await; - let mut tx = rdb.pool.transaction().await.unwrap(); + let pool = connect_and_insert_blocks().await; let block_hash = hex::decode("bc5969d7829ea32aca5784a9eb94cf219f084d2451706bec378f08e23c417ce3").unwrap(); - let res = sql::get_tx_hashes_by_block_hash(&mut tx, &to_rb_bytes(&block_hash)) + let res = pool + .query_transaction_hashes_by_block_hash(&block_hash) .await .unwrap(); - println!("{:?}", res); + assert!(res.is_empty()); + let block_hash = + hex::decode("10639e0895502b5688a6be8cf69460d76541bfa4821629d86d62ba0aae3f9606").unwrap(); + let res = pool + .query_transaction_hashes_by_block_hash(&block_hash) + .await + .unwrap(); + assert_eq!(2, res.len()); +} + +#[tokio::test] +async fn test_rollback_block() { + let storage = connect_sqlite().await; + + let tx = storage.sqlx_pool.transaction().await.unwrap(); + xsql_test::create_tables(tx).await.unwrap(); + + let data_path = String::from(BLOCK_DIR); + storage + .append_block(read_block_view(0, data_path.clone()).into()) + .await + .unwrap(); + + assert_eq!( + 1, + storage + .sqlx_pool + .fetch_count("mercury_block") + .await + .unwrap() + ); + assert_eq!( + 1, + storage + .sqlx_pool + .fetch_count("mercury_sync_status") + .await + .unwrap() + ); + assert_eq!( + 1, + storage + .sqlx_pool + .fetch_count("mercury_canonical_chain") + .await + .unwrap() + ); + assert_eq!( + 2, + storage + .sqlx_pool + .fetch_count("mercury_transaction") + .await + .unwrap() + ); + assert_eq!( + 12, + storage.sqlx_pool.fetch_count("mercury_cell").await.unwrap() + ); + assert_eq!( + 11, + storage + .sqlx_pool + .fetch_count("mercury_live_cell") + .await + .unwrap() + ); + assert_eq!( + 9, + storage + .sqlx_pool + .fetch_count("mercury_script") + .await + .unwrap() + ); + assert_eq!( + 13, + storage + .sqlx_pool + .fetch_count("mercury_indexer_cell") + .await + .unwrap() + ); + + let block_hash = + H256::from_str("10639e0895502b5688a6be8cf69460d76541bfa4821629d86d62ba0aae3f9606").unwrap(); + storage.rollback_block(0, block_hash).await.unwrap(); + + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_block") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_sync_status") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_canonical_chain") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_transaction") + .await + .unwrap() + ); + assert_eq!( + 0, + storage.sqlx_pool.fetch_count("mercury_cell").await.unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_live_cell") + .await + .unwrap() + ); + assert_eq!( + 9, + storage + .sqlx_pool + .fetch_count("mercury_script") + .await + .unwrap() + ); + assert_eq!( + 0, + storage + .sqlx_pool + .fetch_count("mercury_indexer_cell") + .await + .unwrap() + ); } diff --git a/core/storage/src/relational/tests/single_sql_test.rs b/core/storage/src/relational/tests/single_sql_test.rs index f46070405..2d94ebe2d 100644 --- a/core/storage/src/relational/tests/single_sql_test.rs +++ b/core/storage/src/relational/tests/single_sql_test.rs @@ -4,64 +4,49 @@ use ckb_types::H256; use std::str::FromStr; #[tokio::test] -async fn test_fetch_distinct_tx_hashes_count_by_range() { - let pool = connect_and_insert_blocks().await.pool; - let mut conn = pool.acquire().await.unwrap(); - - let res = - sql::fetch_distinct_tx_hashes_count(&mut conn, &0, &10, &[], &[], &true, &false).await; - assert_eq!(2, res.unwrap()); - - let res = - sql::fetch_distinct_tx_hashes_count(&mut conn, &1, &10, &[], &[], &true, &false).await; - assert_eq!(0, res.unwrap()); - - let res = - sql::fetch_distinct_tx_hashes_count(&mut conn, &1, &10, &[], &[], &false, &false).await; - assert_eq!(2, res.unwrap()); +async fn test_fetch_distinct_tx_hashes_count_by_range_() { + let pool = connect_and_insert_blocks().await; + + let res = pool + .query_distinct_tx_hashes_count(&[], &[], &Some(Range::new(0, 10)), false) + .await + .unwrap(); + assert_eq!(2, res); + + let res = pool + .query_distinct_tx_hashes_count(&[], &[], &Some(Range::new(1, 10)), false) + .await + .unwrap(); + assert_eq!(0, res); + + let res = pool + .query_distinct_tx_hashes_count(&[], &[], &None, false) + .await; + assert!(res.is_err()); } #[tokio::test] async fn test_fetch_distinct_tx_hashes_count_by_lock_hash() { - let pool = connect_and_insert_blocks().await.pool; - let mut conn = pool.acquire().await.unwrap(); + let pool = connect_and_insert_blocks().await; let lock_hash = H256::from_str("ba93972fbe398074f4e0bc538d7e36e61a8b140585b52deb4d2890e8d9d320f0").unwrap(); - let res = sql::fetch_distinct_tx_hashes_count( - &mut conn, - &0, - &10, - &[to_rb_bytes(&lock_hash.0)], - &[], - &true, - &false, - ) - .await; - assert_eq!(1, res.unwrap()); - - let res = sql::fetch_distinct_tx_hashes_count( - &mut conn, - &1, - &10, - &[to_rb_bytes(&lock_hash.0)], - &[], - &true, - &false, - ) - .await; - assert_eq!(0, res.unwrap()); - - let res = sql::fetch_distinct_tx_hashes_count( - &mut conn, - &1, - &10, - &[to_rb_bytes(&lock_hash.0)], - &[], - &false, - &false, - ) - .await; - assert_eq!(1, res.unwrap()); + let res = pool + .query_distinct_tx_hashes_count(&[lock_hash.clone()], &[], &Some(Range::new(0, 10)), false) + .await + .unwrap(); + assert_eq!(1, res); + + let res = pool + .query_distinct_tx_hashes_count(&[lock_hash.clone()], &[], &Some(Range::new(1, 10)), false) + .await + .unwrap(); + assert_eq!(0, res); + + let res = pool + .query_distinct_tx_hashes_count(&[lock_hash], &[], &None, false) + .await + .unwrap(); + assert_eq!(1, res); } diff --git a/core/synchronization/Cargo.toml b/core/synchronization/Cargo.toml index 474e77ad1..9b9def518 100644 --- a/core/synchronization/Cargo.toml +++ b/core/synchronization/Cargo.toml @@ -7,23 +7,27 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -ckb-jsonrpc-types = "0.101" ckb-types = "0.101" futures = "0.3" +hex = "0.4" itertools = "0.10" lazy_static = "1.4" log = "0.4" -rbatis = { version = "3.0", default-features = false, features = ["all-database", "runtime-tokio-native-tls", "upper_case_sql_keyword"] } -rbson = "2.0" parking_lot = "0.12" +seq-macro = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.14", features = ["macros", "rt-multi-thread", "sync", "time"] } +sql-builder = "3.1" +sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "sqlite"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } common = { path = "../../common" } core-rpc-types = { path = "../rpc/types" } core-storage = { path = "../storage" } -db_xsql = { path = "../../db/xsql", package = "xsql" } +db-sqlx = { path = "../../db/db-sqlx", package = "db-sqlx" } [dev-dependencies] +ckb-jsonrpc-types = "0.101" env_logger = "0.9" +rand = "0.7" +xsql-test = { path = "../../db/xsql-test" } diff --git a/core/synchronization/src/lib.rs b/core/synchronization/src/lib.rs index 8e27a2ec6..f4c1712b7 100644 --- a/core/synchronization/src/lib.rs +++ b/core/synchronization/src/lib.rs @@ -1,17 +1,18 @@ mod sql; -mod table; mod task; -use crate::table::InUpdate; +#[cfg(test)] +mod tests; + use crate::task::{Task, TaskType}; use common::{async_trait, Result}; use core_rpc_types::{SyncProgress, SyncState}; -use db_xsql::{rbatis::crud::CRUDMut, XSQLPool}; +use db_sqlx::SQLXPool; use ckb_types::core::{BlockNumber, BlockView}; use parking_lot::RwLock; -use rbatis::executor::RBatisTxExecutor; +use sqlx::Row; use tokio::time::sleep; use std::{ops::Range, sync::Arc, time::Duration}; @@ -29,7 +30,7 @@ pub trait SyncAdapter: Sync + Send + 'static { } pub struct Synchronization { - pool: XSQLPool, + sqlx_pool: SQLXPool, max_task_number: usize, chain_tip: u64, sync_state: Arc>, @@ -39,14 +40,14 @@ pub struct Synchronization { impl Synchronization { pub fn new( - pool: XSQLPool, + sqlx_pool: SQLXPool, adapter: Arc, max_task_number: usize, chain_tip: u64, sync_state: Arc>, ) -> Self { Synchronization { - pool, + sqlx_pool, max_task_number, chain_tip, sync_state, @@ -70,7 +71,7 @@ impl Synchronization { self.wait_insertion_complete().await; log::info!("[sync] insert into live cell table"); - let mut tx = self.pool.transaction().await?; + let mut tx = self.sqlx_pool.transaction().await?; sql::drop_live_cell_table(&mut tx).await?; sql::drop_script_table(&mut tx).await?; sql::create_live_cell_table(&mut tx).await?; @@ -80,26 +81,24 @@ impl Synchronization { { let end = i + INSERT_INTO_BATCH_SIZE as u32; log::info!("[sync] update cell table from {} to {}", i, end); - sql::update_cell_table(&mut tx, &i, &end).await? + sql::update_cell_table(&mut tx, i, end).await? } for i in page_range(self.chain_tip, INSERT_INTO_BATCH_SIZE).step_by(INSERT_INTO_BATCH_SIZE) { let end = i + INSERT_INTO_BATCH_SIZE as u32; log::info!("[sync] insert into live cell table {} to {}", i, end); - sql::insert_into_live_cell(&mut tx, &i, &end).await? + sql::insert_into_live_cell(&mut tx, i, end).await? } log::info!("[sync] insert into script table"); - sql::insert_into_script(&mut tx).await?; sql::drop_consume_info_table(&mut tx).await?; log::info!("[sync] remove in update"); + sql::remove_in_update(&mut tx).await?; - self.remove_in_update(&mut tx).await?; tx.commit().await.expect("insert into"); - let _ = tx.take_conn().expect("take connection").close().await; sleep(Duration::from_secs(10)).await; Ok(()) } @@ -114,7 +113,7 @@ impl Synchronization { let mut task = Task::new( id, self.chain_tip, - self.pool.clone(), + self.sqlx_pool.clone(), Arc::clone(&self.adapter), TaskType::SyncIndexerCell, ); @@ -127,7 +126,10 @@ impl Synchronization { let task_number = current_task_count(); if task_number < self.max_task_number { tokio::spawn(async move { - let _ = task.sync_indexer_cell_process().await; + let ret = task.sync_indexer_cell_process().await; + if ret.is_err() { + log::error!("[sync] sync indexer cell process: {:?}", ret); + } }); break; } else { @@ -140,10 +142,10 @@ impl Synchronization { log::info!("[sync]finish"); Ok(()) } + async fn try_create_consume_info_table(&self) -> Result<()> { - let mut conn = self.pool.acquire().await?; - let _ = sql::create_consume_info_table(&mut conn).await; - Ok(()) + let pool = self.sqlx_pool.get_pool()?; + sql::create_consume_info_table(pool).await } async fn sync_metadata(&self) { @@ -153,7 +155,7 @@ impl Synchronization { let mut task = Task::new( id, self.chain_tip, - self.pool.clone(), + self.sqlx_pool.clone(), Arc::clone(&self.adapter), TaskType::SyncMetadata, ); @@ -166,7 +168,10 @@ impl Synchronization { let task_number = current_task_count(); if task_number < self.max_task_number { tokio::spawn(async move { - let _ = task.sync_metadata_process().await; + let ret = task.sync_metadata_process().await; + if ret.is_err() { + log::error!("[sync] sync metadata process: {:?}", ret); + } }); break; } else { @@ -186,26 +191,33 @@ impl Synchronization { log::info!("current thread number {}", current_task_count()); } } + pub async fn is_previous_in_update(&self) -> Result { - let w = self.pool.wrapper().eq("is_in", true); - Ok(self.pool.fetch_count_by_wrapper::(w).await? == 1) + let pool = self.sqlx_pool.get_pool()?; + let row = sqlx::query( + "SELECT COUNT(*) as count + FROM mercury_in_update + WHERE is_in = $1", + ) + .bind(true) + .fetch_one(pool) + .await?; + Ok(row.get::("count") == 1) } + async fn set_in_update(&self) -> Result<()> { if self.is_previous_in_update().await? { return Ok(()); } - let mut acquire = self.pool.acquire().await?; - acquire.save(&InUpdate { is_in: true }, &[]).await?; - Ok(()) - } - async fn remove_in_update(&self, tx: &mut RBatisTxExecutor<'_>) -> Result<()> { - let w = self - .pool - .wrapper() - .eq("is_in", true) - .or() - .eq("is_in", false); - tx.remove_by_wrapper::(w).await?; + let pool = self.sqlx_pool.get_pool()?; + SQLXPool::new_query( + r#" + INSERT INTO mercury_in_update(is_in) + VALUES (true) + "#, + ) + .execute(pool) + .await?; Ok(()) } } diff --git a/core/synchronization/src/sql.rs b/core/synchronization/src/sql.rs index 73751dd47..d9bd03ab0 100644 --- a/core/synchronization/src/sql.rs +++ b/core/synchronization/src/sql.rs @@ -1,155 +1,182 @@ -use crate::table::{ScriptHash, SyncNumber}; - -use db_xsql::rbatis::executor::{RBatisConnExecutor, RBatisTxExecutor}; -use db_xsql::rbatis::sql; - -#[sql( - tx, - "UPDATE mercury_cell AS cell - SET consumed_block_number = consume.consumed_block_number, - consumed_block_hash = consume.consumed_block_hash, - consumed_tx_index = consume.consumed_tx_index, - consumed_tx_hash = consume.consumed_tx_hash, - input_index = consume.input_index, - since = consume.since - FROM mercury_consume_info AS consume - WHERE consume.consumed_block_number >= $1 AND consume.consumed_block_number < $2 AND consume.tx_hash = cell.tx_hash AND consume.output_index = cell.output_index" -)] -pub async fn update_cell_table(tx: &mut RBatisTxExecutor<'_>, from: &u32, to: &u32) -> () {} - -#[sql( - tx, - "INSERT INTO mercury_live_cell (id, tx_hash, output_index, tx_index, block_hash, block_number, epoch_number, epoch_index, epoch_length, capacity, lock_hash, lock_code_hash, lock_args, lock_script_type, type_hash, type_code_hash, type_args, type_script_type, data) - SELECT cell.id, cell.tx_hash, cell.output_index, cell.tx_index, cell.block_hash, cell.block_number, cell.epoch_number, cell.epoch_index, cell.epoch_length, cell.capacity, cell.lock_hash, cell.lock_code_hash, cell.lock_args, cell.lock_script_type, cell.type_hash, cell.type_code_hash, cell.type_args, cell.type_script_type, cell.data - FROM mercury_cell AS cell - WHERE cell.block_number >= $1::INT AND cell.block_number < $2::INT AND cell.consumed_block_number IS NULL" -)] -pub async fn insert_into_live_cell(tx: &mut RBatisTxExecutor<'_>, from: &u32, to: &u32) -> () {} - -#[sql( - tx, - "INSERT INTO mercury_script(script_hash, script_hash_160, script_code_hash, script_args, script_type, script_args_len) - SELECT DISTINCT cell.script_hash, cell.script_hash_160, cell.script_code_hash, cell.script_args, cell.script_type, cell.script_args_len - FROM(SELECT DISTINCT cell_lock.lock_hash AS script_hash, SUBSTRING(cell_lock.lock_hash::bytea, 1::INT, 20::INT) AS script_hash_160, cell_lock.lock_code_hash AS script_code_hash, cell_lock.lock_args AS script_args, cell_lock.lock_script_type AS script_type, LENGTH(cell_lock.lock_args) AS script_args_len - FROM mercury_cell AS cell_lock UNION ALL - SELECT DISTINCT cell_type.type_hash AS script_hash, SUBSTRING(cell_type.type_hash::bytea, 1::INT, 20::INT) AS script_hash_160, cell_type.type_code_hash AS script_code_hash, cell_type.type_args AS script_args, cell_type.type_script_type AS script_type, LENGTH(cell_type.type_args) AS script_args_len - FROM mercury_cell AS cell_type) AS cell" -)] -pub async fn insert_into_script(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(conn, "SELECT script_hash::bytea from mercury_script")] -pub async fn fetch_exist_script_hash(conn: &mut RBatisConnExecutor<'_>) -> Vec {} - -#[sql(tx, "DROP TABLE mercury_live_cell")] -pub async fn drop_live_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DROP TABLE mercury_script")] -pub async fn drop_script_table(tx: &mut RBatisTxExecutor<'_>) -> () {} - -#[sql(tx, "DROP TABLE mercury_consume_info")] -pub async fn drop_consume_info_table(tx: &mut RBatisTxExecutor<'_>) -> () {} +use common::anyhow::Result; +use sqlx::{Any, AnyPool, Executor, Transaction}; + +pub async fn update_cell_table(tx: &mut Transaction<'_, Any>, from: u32, to: u32) -> Result<()> { + sqlx::query( + " + UPDATE mercury_cell AS cell + SET consumed_block_number = consume.consumed_block_number, + consumed_block_hash = consume.consumed_block_hash, + consumed_tx_index = consume.consumed_tx_index, + consumed_tx_hash = consume.consumed_tx_hash, + input_index = consume.input_index, + since = consume.since + FROM mercury_consume_info AS consume + WHERE $1 <= consume.consumed_block_number + AND $2 > consume.consumed_block_number + AND cell.tx_hash = consume.tx_hash + AND cell.output_index = consume.output_index", + ) + .bind(i64::try_from(from)?) + .bind(i64::try_from(to)?) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - " - CREATE TABLE mercury_live_cell( - id bigint PRIMARY KEY, - tx_hash bytea NOT NULL, - output_index int NOT NULL, - tx_index int NOT NULL, - block_hash bytea NOT NULL, - block_number int NOT NULL, - epoch_number int NOT NULL, - epoch_index int NOT NULL, - epoch_length int NOT NULL, - capacity bigint NOT NULL, - lock_hash bytea, - lock_code_hash bytea, - lock_args bytea, - lock_script_type smallint, - type_hash bytea, - type_code_hash bytea, - type_args bytea, - type_script_type smallint, - data bytea - ); - CREATE INDEX \"index_live_cell_table_block_hash\" ON \"mercury_live_cell\" (\"block_hash\"); - CREATE INDEX \"index_live_cell_table_block_number\" ON \"mercury_live_cell\" (\"block_number\"); - CREATE INDEX \"index_live_cell_table_tx_hash_and_output_index\" ON \"mercury_live_cell\" (\"tx_hash\", \"output_index\"); - CREATE INDEX \"index_live_cell_table_lock_hash\" ON \"mercury_live_cell\" (\"lock_hash\"); - CREATE INDEX \"index_live_cell_table_lock_code_hash_and_lock_script_type\" ON \"mercury_live_cell\" (\"lock_code_hash\", \"lock_script_type\"); - CREATE INDEX \"index_live_cell_table_type_code_hash_and_type_script_type\" ON \"mercury_live_cell\" (\"type_code_hash\", \"type_script_type\"); - " -)] -pub async fn create_live_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn insert_into_live_cell( + tx: &mut Transaction<'_, Any>, + from: u32, + to: u32, +) -> Result<()> { + sqlx::query(" + INSERT INTO mercury_live_cell ( + id, tx_hash, output_index, tx_index, block_hash, + block_number, epoch_number, epoch_index, epoch_length, capacity, + lock_hash, lock_code_hash, lock_args, lock_script_type, type_hash, + type_code_hash, type_args, type_script_type, data) + SELECT cell.id, cell.tx_hash, cell.output_index, cell.tx_index, cell.block_hash, + cell.block_number, cell.epoch_number, cell.epoch_index, cell.epoch_length, cell.capacity, + cell.lock_hash, cell.lock_code_hash, cell.lock_args, cell.lock_script_type, cell.type_hash, + cell.type_code_hash, cell.type_args, cell.type_script_type, cell.data + FROM mercury_cell AS cell + WHERE cell.block_number >= $1 + AND cell.block_number < $2 + AND cell.consumed_block_number IS NULL", + ) + .bind(i64::try_from(from)?) + .bind(i64::try_from(to)?) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - " - CREATE TABLE mercury_script( - script_hash bytea NOT NULL PRIMARY KEY, - script_hash_160 bytea NOT NULL, - script_code_hash bytea NOT NULL, - script_args bytea, - script_type smallint NOT NULL, - script_args_len int - ); - CREATE INDEX \"index_script_table_script_hash\" ON \"mercury_script\" (\"script_hash\"); - CREATE INDEX \"index_script_table_code_hash\" ON \"mercury_script\" (\"script_code_hash\"); - CREATE INDEX \"index_script_table_args\" ON \"mercury_script\" (\"script_args\"); - CREATE INDEX \"index_script_table_script_hash_160\" ON \"mercury_script\" USING btree (\"script_hash_160\"); - ", -)] -pub async fn create_script_table(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn insert_into_script(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query(" + INSERT INTO mercury_script( + script_hash, script_hash_160, script_code_hash, script_args, script_type, script_args_len) + SELECT DISTINCT cell.script_hash, cell.script_hash_160, cell.script_code_hash, + cell.script_args, cell.script_type, cell.script_args_len + FROM( + SELECT DISTINCT cell_lock.lock_hash AS script_hash, + SUBSTRING(cell_lock.lock_hash, 1, 20) AS script_hash_160, + cell_lock.lock_code_hash AS script_code_hash, + cell_lock.lock_args AS script_args, + cell_lock.lock_script_type AS script_type, + LENGTH(cell_lock.lock_args) AS script_args_len + FROM mercury_cell AS cell_lock + UNION ALL + SELECT DISTINCT cell_type.type_hash AS script_hash, + SUBSTRING(cell_type.type_hash, 1, 20) AS script_hash_160, + cell_type.type_code_hash AS script_code_hash, + cell_type.type_args AS script_args, + cell_type.type_script_type AS script_type, + LENGTH(cell_type.type_args) AS script_args_len + FROM mercury_cell AS cell_type + ) AS cell", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE IF NOT EXISTS mercury_consume_info( - tx_hash bytea NOT NULL, - output_index int NOT NULL, - consumed_block_number bigint NOT NULL, - consumed_block_hash bytea NOT NULL, - consumed_tx_hash bytea NOT NULL, - consumed_tx_index int NOT NULL, - input_index int NOT NULL, - since bytea NOT NULL, - PRIMARY KEY(tx_hash, output_index) - )" -)] -pub async fn create_consume_info_table(tx: &mut RBatisConnExecutor<'_>) -> () {} +pub async fn drop_live_cell_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DROP TABLE mercury_live_cell") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "SELECT block_number FROM mercury_canonical_chain")] -pub async fn get_sync_completed_numbers(tx: &mut RBatisConnExecutor<'_>) -> Vec {} +pub async fn drop_script_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DROP TABLE mercury_script") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[cfg(test)] -mod tests { - use super::*; - use db_xsql::XSQLPool; +pub async fn drop_consume_info_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DROP TABLE mercury_consume_info") + .execute(&mut *tx) + .await?; + Ok(()) +} - async fn connect_pool() -> XSQLPool { - env_logger::init(); - let pool = XSQLPool::new(0, 0, 100, 0, 60, 1800, 30, log::LevelFilter::Debug); - pool.connect( - core_storage::DBDriver::PostgreSQL, - "mercury", - "127.0.0.1", - 8432, - "postgres", - "123456", - ) - .await - .unwrap(); +pub async fn create_live_cell_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + let query = r#" + CREATE TABLE mercury_live_cell( + id bigint PRIMARY KEY, + tx_hash bytea NOT NULL, + output_index int NOT NULL, + tx_index int NOT NULL, + block_hash bytea NOT NULL, + block_number int NOT NULL, + epoch_number int NOT NULL, + epoch_index int NOT NULL, + epoch_length int NOT NULL, + capacity bigint NOT NULL, + lock_hash bytea, + lock_code_hash bytea, + lock_args bytea, + lock_script_type smallint, + type_hash bytea, + type_code_hash bytea, + type_args bytea, + type_script_type smallint, + data bytea + ); + CREATE INDEX "index_live_cell_table_block_hash" ON "mercury_live_cell" ("block_hash"); + CREATE INDEX "index_live_cell_table_block_number" ON "mercury_live_cell" ("block_number"); + CREATE INDEX "index_live_cell_table_tx_hash_and_output_index" ON "mercury_live_cell" ("tx_hash", "output_index"); + CREATE INDEX "index_live_cell_table_lock_hash" ON "mercury_live_cell" ("lock_hash"); + CREATE INDEX "index_live_cell_table_lock_code_hash_and_lock_script_type" ON "mercury_live_cell" ("lock_code_hash", "lock_script_type"); + CREATE INDEX "index_live_cell_table_type_code_hash_and_type_script_type" ON "mercury_live_cell" ("type_code_hash", "type_script_type"); + "#; + Executor::execute(&mut *tx, query).await?; + Ok(()) +} - pool - } +pub async fn create_script_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + let query = r#" + CREATE TABLE mercury_script( + script_hash bytea NOT NULL PRIMARY KEY, + script_hash_160 bytea NOT NULL, + script_code_hash bytea NOT NULL, + script_args bytea, + script_type smallint NOT NULL, + script_args_len int + ); + CREATE INDEX "index_script_table_script_hash" ON "mercury_script" ("script_hash"); + CREATE INDEX "index_script_table_code_hash" ON "mercury_script" ("script_code_hash"); + CREATE INDEX "index_script_table_args" ON "mercury_script" ("script_args"); + CREATE INDEX "index_script_table_script_hash_160" ON "mercury_script" ("script_hash_160"); + "#; + Executor::execute(&mut *tx, query).await?; + Ok(()) +} - #[ignore] - #[tokio::test] - async fn test_get_script() { - let pool = connect_pool().await; - let mut conn = pool.acquire().await.unwrap(); +pub async fn create_consume_info_table(pool: &AnyPool) -> Result<()> { + sqlx::query( + " + CREATE TABLE IF NOT EXISTS mercury_consume_info( + tx_hash bytea NOT NULL, + output_index int NOT NULL, + consumed_block_number bigint NOT NULL, + consumed_block_hash bytea NOT NULL, + consumed_tx_hash bytea NOT NULL, + consumed_tx_index int NOT NULL, + input_index int NOT NULL, + since bytea NOT NULL, + PRIMARY KEY(tx_hash, output_index) + )", + ) + .execute(pool) + .await?; + Ok(()) +} - let res = fetch_exist_script_hash(&mut conn).await.unwrap(); - println!("{:?}", res); - } +pub async fn remove_in_update(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_in_update") + .execute(&mut *tx) + .await?; + Ok(()) } diff --git a/core/synchronization/src/table.rs b/core/synchronization/src/table.rs deleted file mode 100644 index d66ae1ff3..000000000 --- a/core/synchronization/src/table.rs +++ /dev/null @@ -1,54 +0,0 @@ -use core_storage::relational::to_rb_bytes; -use core_storage::single_sql_return; -use db_xsql::rbatis::{crud_table, Bytes as RbBytes}; - -use ckb_types::{packed, prelude::*}; -use serde::{Deserialize, Serialize}; - -single_sql_return!(ScriptHash, script_hash, RbBytes); -single_sql_return!(SyncNumber, block_number, u64); - -#[crud_table(table_name: "mercury_consume_info")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ConsumeInfoTable { - pub tx_hash: RbBytes, - pub output_index: u32, - pub consumed_block_number: u64, - pub consumed_block_hash: RbBytes, - pub consumed_tx_hash: RbBytes, - pub consumed_tx_index: u32, - pub input_index: u32, - pub since: RbBytes, -} - -impl ConsumeInfoTable { - pub fn new( - out_point: packed::OutPoint, - consumed_block_number: u64, - consumed_block_hash: RbBytes, - consumed_tx_hash: RbBytes, - consumed_tx_index: u32, - input_index: u32, - since: u64, - ) -> Self { - let tx_hash = to_rb_bytes(&out_point.tx_hash().raw_data()); - let output_index: u32 = out_point.index().unpack(); - - ConsumeInfoTable { - tx_hash, - output_index, - consumed_block_number, - consumed_block_hash, - consumed_tx_hash, - consumed_tx_index, - input_index, - since: to_rb_bytes(&since.to_be_bytes()), - } - } -} - -#[crud_table(table_name: "mercury_in_update")] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct InUpdate { - pub is_in: bool, -} diff --git a/core/synchronization/src/task.rs b/core/synchronization/src/task.rs index 416d24168..8f94b5b4f 100644 --- a/core/synchronization/src/task.rs +++ b/core/synchronization/src/task.rs @@ -1,15 +1,19 @@ -use crate::table::ConsumeInfoTable; use crate::{add_one_task, free_one_task, SyncAdapter, TASK_LEN}; use common::{anyhow::anyhow, Result}; -use core_storage::relational::table::{ - BlockTable, CanonicalChainTable, CellTable, IndexerCellTable, SyncStatus, TransactionTable, - IO_TYPE_INPUT, IO_TYPE_OUTPUT, +use core_storage::relational::{ + bulk_insert_blocks, bulk_insert_output_cells, bulk_insert_transactions, generate_id, + push_values_placeholders, BATCH_SIZE_THRESHOLD, IO_TYPE_INPUT, IO_TYPE_OUTPUT, }; -use core_storage::relational::{generate_id, to_rb_bytes, BATCH_SIZE_THRESHOLD}; -use db_xsql::{commit_transaction, rbatis::crud::CRUDMut, XSQLPool}; +use db_sqlx::SQLXPool; -use ckb_types::{core::BlockView, prelude::*}; +use ckb_types::{ + core::{BlockView, TransactionView}, + prelude::*, +}; +use seq_macro::seq; +use sql_builder::SqlBuilder; +use sqlx::{Any, Row, Transaction}; use tokio::time::sleep; use std::future::Future; @@ -35,7 +39,7 @@ impl TaskType { pub struct Task { id: u64, tip: u64, - store: XSQLPool, + pool: SQLXPool, type_: TaskType, state_cursor: Option, @@ -43,11 +47,11 @@ pub struct Task { } impl Task { - pub fn new(id: u64, tip: u64, store: XSQLPool, adapter: Arc, type_: TaskType) -> Task { + pub fn new(id: u64, tip: u64, pool: SQLXPool, adapter: Arc, type_: TaskType) -> Task { Task { id, tip, - store, + pool, type_, state_cursor: None, adapter, @@ -57,20 +61,23 @@ impl Task { async fn set_state_cursor(&mut self) -> Result<()> { let last = self.last_number(); - let w = self - .store - .wrapper() - .between("block_number", self.id, last) - .order_by(false, &["block_number"]) - .limit(1); - - let cursor = if self.type_.is_metadata_task() { - let block: Option = self.store.fetch_by_wrapper(w).await?; - block.map_or_else(|| self.id, |b| (b.block_number + 1).min(last)) + let mut query = SqlBuilder::select_from(if self.type_.is_metadata_task() { + "mercury_block" } else { - let cell: Option = self.store.fetch_by_wrapper(w).await?; - cell.map_or_else(|| self.id, |c| (c.block_number + 1).min(last)) - }; + "mercury_sync_status" + }); + query + .field("block_number") + .and_where_between("block_number", self.id, last) + .order_by("block_number", true) + .limit(1); + let sql = query.sql()?; + let query = SQLXPool::new_query(&sql); + let row = self.pool.fetch_optional(query).await?; + let cursor = row.map_or_else( + || self.id, + |row| (row.get::("block_number") as u64 + 1).min(last), + ); self.state_cursor = Some(cursor); Ok(()) @@ -108,7 +115,7 @@ impl Task { let end = (start + PULL_BLOCK_BATCH_SIZE).min(last + 1); let sub_task = (start..end).collect(); let blocks = self.poll_call(Self::pull_blocks, sub_task).await; - sync_blocks(blocks, self.store.clone()).await?; + sync_blocks(blocks, self.pool.clone()).await?; } free_one_task(); @@ -144,7 +151,7 @@ impl Task { for start in (cursor..=last).step_by(PULL_BLOCK_BATCH_SIZE as usize) { let end = (start + PULL_BLOCK_BATCH_SIZE).min(last + 1); let sub_task = (start..end).collect::>(); - sync_indexer_cells(&sub_task, self.store.clone()).await?; + sync_indexer_cells(&sub_task, self.pool.clone()).await?; } free_one_task(); @@ -174,129 +181,252 @@ impl Task { } } -async fn sync_blocks(blocks: Vec, rdb: XSQLPool) -> Result<()> { - let mut block_table_batch: Vec = Vec::new(); - let mut tx_table_batch = Vec::new(); - let mut cell_table_batch = Vec::new(); - let mut consume_info_batch = Vec::new(); - let mut canonical_data_table_batch = Vec::new(); - let mut tx = rdb.transaction().await?; +async fn sync_blocks(blocks: Vec, pool: SQLXPool) -> Result<()> { + let mut tx = pool.transaction().await?; + + bulk_insert_blocks(&blocks, &mut tx).await?; for block in blocks.iter() { let block_number = block.number(); let block_hash = block.hash().raw_data().to_vec(); let block_timestamp = block.timestamp(); - let block_epoch = block.epoch(); + let epoch = block.epoch(); + let tx_views = block.transactions(); - block_table_batch.push(block.into()); - canonical_data_table_batch.push(CanonicalChainTable::new( + bulk_insert_transactions( block_number, - to_rb_bytes(&block_hash), - )); - - for (tx_idx, transaction) in block.transactions().iter().enumerate() { - let tx_hash = to_rb_bytes(&transaction.hash().raw_data()); - tx_table_batch.push(TransactionTable::from_view( - transaction, - generate_id(block_number), - tx_idx as u32, - to_rb_bytes(&block_hash), - block_number, - block_timestamp, - )); - - // skip cellbase - if tx_idx != 0 { - for (input_idx, input) in transaction.inputs().into_iter().enumerate() { - consume_info_batch.push(ConsumeInfoTable::new( - input.previous_output(), - block_number, - to_rb_bytes(&block_hash), - tx_hash.clone(), - tx_idx as u32, - input_idx as u32, - input.since().unpack(), - )); - } - } + &block_hash, + block_timestamp, + &tx_views, + &mut tx, + ) + .await?; + bulk_insert_output_cells(block_number, &block_hash, epoch, &tx_views, false, &mut tx) + .await?; + bulk_insert_consume_info(block_number, &block_hash, &tx_views, &mut tx).await?; + } - for (output_idx, (cell, data)) in transaction.outputs_with_data_iter().enumerate() { - cell_table_batch.push(CellTable::from_cell( - &cell, - generate_id(block_number), - tx_hash.clone(), - output_idx as u32, - tx_idx as u32, - block_number, - to_rb_bytes(&block_hash), - block_epoch, - &data, - )); - } + tx.commit().await.map_err(Into::into) +} + +async fn sync_indexer_cells(sub_task: &[u64], pool: SQLXPool) -> Result<()> { + let mut tx = pool.transaction().await?; + bulk_insert_indexer_cells(sub_task, &mut tx).await?; + bulk_insert_sync_status(sub_task, &mut tx).await?; + tx.commit().await.map_err(Into::into) +} + +async fn bulk_insert_consume_info( + block_number: u64, + block_hash: &[u8], + tx_views: &[TransactionView], + tx: &mut Transaction<'_, Any>, +) -> Result<()> { + let mut consume_info_rows = Vec::new(); + + for (tx_index, transaction) in tx_views.iter().enumerate() { + if tx_index == 0 { + continue; + } + + let tx_hash = transaction.hash().raw_data(); + + for (input_index, input) in transaction.inputs().into_iter().enumerate() { + let previous_output = input.previous_output(); + let previous_output_tx_hash = previous_output.tx_hash().raw_data(); + let previous_output_index: u32 = previous_output.index().unpack(); + let since: u64 = input.since().unpack(); + + let consume_info = ( + previous_output_tx_hash.to_vec(), + previous_output_index as i32, + block_number as i64, + block_hash.to_vec(), + tx_hash.to_vec(), + tx_index as i32, + input_index as i32, + since.to_be_bytes().to_vec(), + ); + + consume_info_rows.push(consume_info); } } - core_storage::save_batch_slice!( - tx, - block_table_batch, - tx_table_batch, - cell_table_batch, - consume_info_batch, - canonical_data_table_batch - ); + // bulk insert + for start in (0..consume_info_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(consume_info_rows.len()); + + // build query str + let mut builder = SqlBuilder::insert_into("mercury_consume_info"); + builder.field( + r#" + tx_hash, + output_index, + consumed_block_number, + consumed_block_hash, + consumed_tx_hash, + consumed_tx_index, + input_index, + since"#, + ); + push_values_placeholders(&mut builder, 8, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in consume_info_rows[start..end].iter() { + seq!(i in 0..8 { + query = query.bind(&row.i); + }); + } - commit_transaction(tx).await?; + // execute + query.execute(&mut *tx).await?; + } Ok(()) } -pub async fn sync_indexer_cells(sub_task: &[u64], rdb: XSQLPool) -> Result<()> { - let mut indexer_cells = Vec::new(); - let mut tx = rdb.transaction().await?; - let mut status_list = Vec::new(); - - let w = rdb - .wrapper() - .r#in("block_number", sub_task) - .or() - .r#in("consumed_block_number", sub_task); - let cells = tx.fetch_list_by_wrapper::(w).await?; - +async fn bulk_insert_indexer_cells(sub_task: &[u64], tx: &mut Transaction<'_, Any>) -> Result<()> { + let mut query = SqlBuilder::select_from("mercury_cell"); + query + .field( + "id, tx_hash, output_index, tx_index, block_number, + lock_hash, lock_code_hash, lock_args, lock_script_type, + type_hash, type_code_hash, type_args, type_script_type, + consumed_block_number, consumed_tx_hash, consumed_tx_index, input_index", + ) + .and_where_in("block_number", sub_task) + .or_where_in("consumed_block_number", sub_task); + let sql = query.sql()?; + let query = SQLXPool::new_query(&sql); + let cells = query.fetch_all(&mut *tx).await?; + + let mut indexer_cell_rows = Vec::new(); for cell in cells.iter() { - if sub_task.contains(&cell.block_number) { - let i_cell = IndexerCellTable::new_with_empty_scripts( - cell.block_number, - IO_TYPE_OUTPUT, - cell.output_index, - cell.tx_hash.clone(), - cell.tx_index, + if sub_task.contains(&(cell.get::("block_number") as u64)) { + let indexer_cell = ( + 0i64, + cell.get::("block_number"), + i16::try_from(IO_TYPE_OUTPUT)?, + cell.get::("output_index"), + cell.get::, _>("tx_hash"), + cell.get::("tx_index"), + cell.get::, _>("lock_hash"), + cell.get::, _>("lock_code_hash"), + cell.get::, _>("lock_args"), + cell.get::("lock_script_type"), + cell.get::, _>("type_hash"), + cell.get::, _>("type_code_hash"), + cell.get::, _>("type_args"), + cell.get::("type_script_type"), ); - indexer_cells.push(i_cell.update_by_cell_table(cell)); + indexer_cell_rows.push(indexer_cell); } - if let Some(consume_number) = cell.consumed_block_number { - if sub_task.contains(&consume_number) { - let i_cell = IndexerCellTable::new_with_empty_scripts( - consume_number, - IO_TYPE_INPUT, - cell.input_index.expect("cell input index"), - cell.consumed_tx_hash.clone(), - cell.consumed_tx_index.expect("cell consumed tx index"), + if let Some(consume_number) = cell.get::, _>("consumed_block_number") { + if sub_task.contains(&(consume_number as u64)) { + let indexer_cell = ( + 0i64, + i32::try_from(consume_number)?, + i16::try_from(IO_TYPE_INPUT)?, + cell.get("input_index"), + cell.get("consumed_tx_hash"), + cell.get("consumed_tx_index"), + cell.get("lock_hash"), + cell.get("lock_code_hash"), + cell.get("lock_args"), + cell.get("lock_script_type"), + cell.get("type_hash"), + cell.get("type_code_hash"), + cell.get("type_args"), + cell.get("type_script_type"), ); - indexer_cells.push(i_cell.update_by_cell_table(cell)); + indexer_cell_rows.push(indexer_cell); } } } - status_list.extend(sub_task.iter().map(|num| SyncStatus::new(*num))); - - indexer_cells.sort(); - indexer_cells + indexer_cell_rows.sort_unstable_by(|a, b| { + if a.1 != b.1 { + // 1 block_number + a.1.cmp(&b.1) + } else if a.5 != b.5 { + // 5 tx_index + a.5.cmp(&b.5) + } else if a.2 != b.2 { + // 2 io_type + a.2.cmp(&b.2) + } else { + // 3 io_index + a.3.cmp(&b.3) + } + }); + indexer_cell_rows .iter_mut() - .for_each(|c| c.id = generate_id(c.block_number)); - core_storage::save_batch_slice!(tx, indexer_cells, status_list); + .for_each(|row| row.0 = generate_id(row.1 as u64)); + + // bulk insert indexer cells + for start in (0..indexer_cell_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(indexer_cell_rows.len()); + + // build query str + let mut builder = SqlBuilder::insert_into("mercury_indexer_cell"); + builder.field( + r#"id, + block_number, + io_type, + io_index, + tx_hash, + tx_index, + lock_hash, + lock_code_hash, + lock_args, + lock_script_type, + type_hash, + type_code_hash, + type_args, + type_script_type"#, + ); + push_values_placeholders(&mut builder, 14, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in indexer_cell_rows[start..end].iter() { + seq!(i in 0..14 { + query = query.bind(&row.i); + }); + } + // execute + query.execute(&mut *tx).await?; + } + + Ok(()) +} + +async fn bulk_insert_sync_status(sub_task: &[u64], tx: &mut Transaction<'_, Any>) -> Result<()> { + let sync_status_rows: Vec = sub_task.iter().map(|num| *num as i32).collect(); - commit_transaction(tx).await?; + // bulk insert sync status + for start in (0..sync_status_rows.len()).step_by(BATCH_SIZE_THRESHOLD) { + let end = (start + BATCH_SIZE_THRESHOLD).min(sync_status_rows.len()); + + // build query str + let mut builder = SqlBuilder::insert_into("mercury_sync_status"); + builder.field("block_number"); + push_values_placeholders(&mut builder, 1, end - start); + let sql = builder.sql()?.trim_end_matches(';').to_string(); + + // bind + let mut query = SQLXPool::new_query(&sql); + for row in sync_status_rows[start..end].iter() { + query = query.bind(row); + } + + // execute + query.execute(&mut *tx).await?; + } Ok(()) } diff --git a/core/synchronization/src/tests/mod.rs b/core/synchronization/src/tests/mod.rs new file mode 100644 index 000000000..4cbc4ce8d --- /dev/null +++ b/core/synchronization/src/tests/mod.rs @@ -0,0 +1,50 @@ +mod sync_test; + +use crate::SyncAdapter; + +use common::{async_trait, Order, PaginationRequest, Range, Result}; +use core_rpc_types::IOType; +use core_storage::{relational::RelationalStorage, DBDriver}; + +use ckb_jsonrpc_types::BlockView as JsonBlockView; +use ckb_types::core::{BlockNumber, BlockView}; + +const MEMORY_DB: &str = ":memory:"; +const BLOCK_DIR: &str = "../../devtools/test_data/blocks/"; + +async fn connect_sqlite() -> Result { + let mut pool = RelationalStorage::new(0, 0, 1, 0, 60, 1800, 30); + pool.connect(DBDriver::SQLite, MEMORY_DB, "", 0, "", "") + .await?; + Ok(pool) +} + +async fn connect_and_create_tables() -> Result { + let pool = connect_sqlite().await?; + let tx = pool.sqlx_pool.transaction().await?; + xsql_test::create_tables(tx).await?; + Ok(pool) +} + +pub fn read_block_view(number: u64, dir_path: String) -> JsonBlockView { + let file_name = number.to_string() + ".json"; + let path = dir_path + file_name.as_str(); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +} + +#[derive(Clone, Debug)] +pub struct CkbRpcTestClient; + +#[async_trait] +impl SyncAdapter for CkbRpcTestClient { + async fn pull_blocks(&self, _block_numbers: Vec) -> Result> { + let ret = (0..10) + .map(|i| { + let block_view: BlockView = read_block_view(i, String::from(BLOCK_DIR)).into(); + block_view + }) + .into_iter() + .collect(); + Ok(ret) + } +} diff --git a/core/synchronization/src/tests/sync_test.rs b/core/synchronization/src/tests/sync_test.rs new file mode 100644 index 000000000..05a5582ae --- /dev/null +++ b/core/synchronization/src/tests/sync_test.rs @@ -0,0 +1,101 @@ +use super::*; + +use crate::Synchronization; + +use common::Context; +use core_rpc_types::SyncState; +use core_storage::Storage; + +use ckb_types::prelude::Unpack; +use ckb_types::H256; +use parking_lot::RwLock; + +use std::str::FromStr; +use std::sync::Arc; + +#[tokio::test] +async fn test_sync() { + let res = connect_and_create_tables().await; + assert!(res.is_ok()); + + let storage = res.unwrap(); + let sync_handler = Synchronization::new( + storage.get_pool(), + Arc::new(CkbRpcTestClient), + 4, + 9, + Arc::new(RwLock::new(SyncState::ReadOnly)), + ); + sync_handler.do_sync().await.unwrap(); + sync_handler.build_indexer_cell_table().await.unwrap(); + + let pool = storage.get_pool(); + assert_eq!(10, pool.fetch_count("mercury_block").await.unwrap()); + assert_eq!(11, pool.fetch_count("mercury_transaction").await.unwrap()); + assert_eq!(12, pool.fetch_count("mercury_cell").await.unwrap()); + assert_eq!(11, pool.fetch_count("mercury_live_cell").await.unwrap()); + assert_eq!(13, pool.fetch_count("mercury_indexer_cell").await.unwrap()); + + // During parallel synchronization, H256::default() will be added to the script table as the script hash of typescript, + // so there will be one more than normal serial synchronization (append_block from genesis block). + assert_eq!(10, pool.fetch_count("mercury_script").await.unwrap()); + + assert_eq!( + 10, + pool.fetch_count("mercury_canonical_chain").await.unwrap() + ); + assert_eq!( + 0, + pool.fetch_count("mercury_registered_address") + .await + .unwrap() + ); + assert_eq!(10, pool.fetch_count("mercury_sync_status").await.unwrap()); + assert_eq!(0, pool.fetch_count("mercury_in_update").await.unwrap()); + + // check build block view + let block_hash = + H256::from_str("10639e0895502b5688a6be8cf69460d76541bfa4821629d86d62ba0aae3f9606").unwrap(); + let res_block = storage + .get_block(Context::new(), Some(block_hash.clone()), None) + .await + .unwrap(); + let res_block_hash: H256 = res_block.hash().unpack(); + assert_eq!(block_hash, res_block_hash); + + // check indexer cells + let ret = storage + .get_indexer_transactions( + Context::new(), + vec![], + vec![], + Some(Range::new(0, 1)), + PaginationRequest { + cursor: None, + order: Order::Desc, + limit: None, + skip: None, + return_count: true, + }, + ) + .await + .unwrap(); + + let txs_input_count = ret + .response + .iter() + .filter(|tx| tx.io_type == IOType::Input) + .count(); + let txs_output_count = ret + .response + .iter() + .filter(|tx| tx.io_type == IOType::Output) + .count(); + assert_eq!(Some(13), ret.count); + assert_eq!(1, txs_input_count); + assert_eq!(12, txs_output_count); + assert_eq!(IOType::Output, ret.response[0].io_type); + assert_eq!(IOType::Output, ret.response[1].io_type); + assert_eq!(IOType::Input, ret.response[2].io_type); + assert_eq!(IOType::Output, ret.response[3].io_type); +} diff --git a/db/db-sqlx/Cargo.toml b/db/db-sqlx/Cargo.toml new file mode 100644 index 000000000..34c4e573c --- /dev/null +++ b/db/db-sqlx/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "db-sqlx" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ckb-types = "0.101" +log = "0.4" +futures = "0.3" +hex = "0.4" +once_cell = "1.10" +sql-builder = "3.1" +sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "postgres"] } +tokio = { version = "1", features = ["full"] } + +common = { path = "../../common" } +protocol = { path = "../../protocol" } diff --git a/db/db-sqlx/src/lib.rs b/db/db-sqlx/src/lib.rs new file mode 100644 index 000000000..465e327ee --- /dev/null +++ b/db/db-sqlx/src/lib.rs @@ -0,0 +1,245 @@ +pub mod page; + +pub use crate::page::{build_next_cursor, build_query_page_sql}; + +use crate::page::COUNT_COLUMN; + +use common::{anyhow::anyhow, Order, PaginationRequest, PaginationResponse, Result}; +use protocol::db::DBDriver; + +use futures::TryStreamExt; +use log::LevelFilter; +use once_cell::sync::OnceCell; +use sql_builder::SqlBuilder; +use sqlx::any::{Any, AnyArguments, AnyConnectOptions, AnyPool, AnyPoolOptions, AnyRow}; +use sqlx::query::{Query, QueryAs}; +use sqlx::{ConnectOptions, IntoArguments, Row, Transaction}; + +use std::marker::{Send, Unpin}; +use std::str::FromStr; +use std::{fmt::Debug, sync::Arc, time::Duration}; + +#[derive(Clone)] +pub struct SQLXPool { + pool: Arc>, + center_id: u16, + node_id: u16, + max_conn: u32, + min_conn: u32, + conn_timeout: Duration, + max_lifetime: Duration, + idle_timeout: Duration, +} + +impl Debug for SQLXPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SQLXPool") + .field("center_id", &self.center_id) + .field("node_id", &self.node_id) + .field("max_conn", &self.max_conn) + .field("min_conn", &self.min_conn) + .field("conn_timeout", &self.conn_timeout) + .field("max_lifetime", &self.max_lifetime) + .field("idle_timeout", &self.idle_timeout) + .finish() + } +} + +impl SQLXPool { + pub fn new( + center_id: u16, + node_id: u16, + max_connections: u32, + min_connections: u32, + connection_timeout: u64, + max_lifetime: u64, + idle_timeout: u64, + ) -> Self { + SQLXPool { + pool: Arc::new(OnceCell::new()), + center_id, + node_id, + max_conn: max_connections, + min_conn: min_connections, + conn_timeout: Duration::from_secs(connection_timeout), + max_lifetime: Duration::from_secs(max_lifetime), + idle_timeout: Duration::from_secs(idle_timeout), + } + } + + pub async fn connect( + &mut self, + db_driver: &DBDriver, + db_name: &str, + host: &str, + port: u16, + user: &str, + password: &str, + ) -> Result<()> { + let pool_options = AnyPoolOptions::new() + .max_connections(self.max_conn) + .min_connections(self.min_conn) + .acquire_timeout(self.conn_timeout) + .max_lifetime(self.max_lifetime) + .idle_timeout(self.idle_timeout); + let uri = build_url(db_driver.into(), db_name, host, port, user, password); + let mut connection_options = AnyConnectOptions::from_str(&uri)?; + connection_options.log_statements(LevelFilter::Trace); + let pool = pool_options.connect_with(connection_options).await?; + self.pool + .set(pool) + .map_err(|_| anyhow!("set pg pool failed!")) + } + + pub async fn fetch_count(&self, table_name: &str) -> Result { + let pool = self.get_pool()?; + let row = sqlx::query(&fetch_count_sql(table_name)) + .fetch_one(pool) + .await?; + let count: i64 = row.get::(COUNT_COLUMN); + Ok(count.try_into().expect("i64 to u64")) + } + + pub fn new_query(sql: &str) -> Query { + sqlx::query(sql) + } + + pub fn new_query_as(sql: &str) -> QueryAs + where + T: for<'r> sqlx::FromRow<'r, AnyRow>, + { + sqlx::query_as(sql) + } + + pub async fn fetch_optional<'a, T>(&self, query: Query<'a, Any, T>) -> Result> + where + T: Send + IntoArguments<'a, Any> + 'a, + { + let pool = self.get_pool()?; + query.fetch_optional(pool).await.map_err(Into::into) + } + + pub async fn fetch_one<'a, T>(&self, query: Query<'a, Any, T>) -> Result + where + T: Send + IntoArguments<'a, Any> + 'a, + { + let pool = self.get_pool()?; + query.fetch_one(pool).await.map_err(Into::into) + } + + pub async fn fetch_all<'a, T>(&self, query: Query<'a, Any, T>) -> Result> + where + T: Send + IntoArguments<'a, Any> + 'a, + { + let pool = self.get_pool()?; + query.fetch_all(pool).await.map_err(Into::into) + } + + pub async fn fetch<'a, T>(&self, query: Query<'a, Any, T>) -> Result> + where + T: Send + IntoArguments<'a, Any> + 'a, + { + let pool = self.get_pool()?; + let mut res = vec![]; + let mut rows = query.fetch(pool); + while let Some(row) = rows.try_next().await? { + res.push(row) + } + Ok(res) + } + + pub async fn fetch_one_by_query_as( + &self, + query: QueryAs<'static, Any, T, AnyArguments<'static>>, + ) -> Result + where + T: for<'r> sqlx::FromRow<'r, AnyRow> + Unpin + Send, + { + let pool = self.get_pool()?; + query.fetch_one(pool).await.map_err(Into::into) + } + + pub async fn fetch_page<'a>( + &self, + query: Query<'a, Any, AnyArguments<'a>>, + query_total: Query<'a, Any, AnyArguments<'a>>, + pagination: &PaginationRequest, + ) -> Result> { + let response = self.fetch(query).await?; + let count = if pagination.return_count { + let count = self + .fetch_one(query_total) + .await? + .get::(COUNT_COLUMN) as u64; + Some(count) + } else { + None + }; + + let next_cursor = if response.is_empty() { + None + } else { + build_next_cursor( + pagination.limit.unwrap_or(u16::MAX), + response.last().unwrap().get::("id") as u64, + response.len(), + count, + ) + }; + + Ok(PaginationResponse { + response, + next_cursor, + count, + }) + } + + pub async fn transaction(&self) -> Result> { + let pool = self.get_pool()?; + pool.begin().await.map_err(Into::into) + } + + pub fn get_pool(&self) -> Result<&AnyPool> { + self.pool.get().ok_or(anyhow!("pg pool not inited!")) + } + + pub fn center_id(&self) -> u16 { + self.center_id + } + + pub fn node_id(&self) -> u16 { + self.node_id + } + + pub fn get_max_connections(&self) -> u32 { + self.max_conn + } +} + +fn build_url( + db_type: &str, + db_name: &str, + host: &str, + port: u16, + user: &str, + password: &str, +) -> String { + if db_type == protocol::db::SQLITE { + return db_type.to_string() + db_name; + } + + db_type.to_string() + + user + + ":" + + password + + "@" + + host + + ":" + + port.to_string().as_str() + + "/" + + db_name +} + +pub(crate) fn fetch_count_sql(table_name: &str) -> String { + format!("SELECT COUNT(*) as {} FROM {}", COUNT_COLUMN, table_name) +} diff --git a/db/db-sqlx/src/page.rs b/db/db-sqlx/src/page.rs new file mode 100644 index 000000000..a6a9fbc64 --- /dev/null +++ b/db/db-sqlx/src/page.rs @@ -0,0 +1,51 @@ +use super::*; + +pub(crate) const COUNT_COLUMN: &str = "count"; + +pub fn build_query_page_sql( + mut query_builder: SqlBuilder, + pagination: &PaginationRequest, +) -> Result<(String, String)> { + let sql_sub_query = query_builder.subquery()?; + + if let Some(id) = pagination.cursor { + let id = i64::try_from(id).unwrap_or(i64::MAX); + match pagination.order { + Order::Asc => query_builder.and_where_gt("id", id), + Order::Desc => query_builder.and_where_lt("id", id), + }; + } + match pagination.order { + Order::Asc => query_builder.order_by("id", false), + Order::Desc => query_builder.order_by("id", true), + }; + query_builder.limit(pagination.limit.unwrap_or(u16::MAX)); + if let Some(skip) = pagination.skip { + let offset = i64::try_from(skip).unwrap_or(i64::MAX); + query_builder.offset(offset); + } + + let query = query_builder.sql()?.trim_end_matches(';').to_string(); + let sub_query_for_count = fetch_count_sql(&format!("{} res", sql_sub_query)); + + Ok((query, sub_query_for_count)) +} + +pub fn build_next_cursor( + limit: u16, + last_id: u64, + records_size: usize, + total: Option, +) -> Option { + let mut next_cursor = None; + if records_size == limit as usize { + if let Some(total) = total { + if total > limit as u64 { + next_cursor = Some(last_id) + } + } else { + next_cursor = Some(last_id); + } + } + next_cursor +} diff --git a/db/xsql-test/Cargo.toml b/db/xsql-test/Cargo.toml index a53f09b9d..35ead221e 100644 --- a/db/xsql-test/Cargo.toml +++ b/db/xsql-test/Cargo.toml @@ -6,10 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -ckb-jsonrpc-types = "0.101" -rbatis = { version = "3.0", default-features = false, features = ["all-database", "runtime-tokio-native-tls"] } -serde_json = "1.0" +sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "sqlite"] } common = { path = "../../common" } -protocol = { path = "../../protocol" } -xsql = { path = "../xsql" } diff --git a/db/xsql-test/src/lib.rs b/db/xsql-test/src/lib.rs index dc6c6e235..426548b35 100644 --- a/db/xsql-test/src/lib.rs +++ b/db/xsql-test/src/lib.rs @@ -2,32 +2,34 @@ pub mod sql; use crate::sql::*; use common::anyhow::Result; -use rbatis::executor::RBatisTxExecutor; +use sqlx::{Any, Transaction}; -pub async fn delete_all_data(tx: &mut RBatisTxExecutor<'_>) -> Result<()> { - delete_block_table_data(tx).await?; - delete_transaction_table_data(tx).await?; - delete_cell_table_data(tx).await?; - delete_consume_info_table_data(tx).await?; - delete_live_cell_table_data(tx).await?; - delete_script_table_data(tx).await?; - delete_canonical_chain_table_data(tx).await?; - delete_registered_address_table_data(tx).await?; +pub async fn delete_all_data(mut tx: Transaction<'_, Any>) -> Result<()> { + delete_block_table_data(&mut tx).await?; + delete_transaction_table_data(&mut tx).await?; + delete_cell_table_data(&mut tx).await?; + delete_live_cell_table_data(&mut tx).await?; + delete_indexer_cell_table_data(&mut tx).await?; + delete_script_table_data(&mut tx).await?; + delete_canonical_chain_table_data(&mut tx).await?; + delete_registered_address_table_data(&mut tx).await?; + delete_sync_status_table_data(&mut tx).await?; + delete_in_update_table_data(&mut tx).await?; tx.commit().await?; Ok(()) } -pub async fn create_tables(tx: &mut RBatisTxExecutor<'_>) -> Result<()> { - create_block_table(tx).await?; - create_transaction_table(tx).await?; - create_cell_table(tx).await?; - create_consume_info_table(tx).await?; - create_live_cell_table(tx).await?; - create_indexer_cell_table(tx).await?; - create_script_table(tx).await?; - create_canonical_chain_table(tx).await?; - create_registered_address_table(tx).await?; - create_sync_status_table(tx).await?; +pub async fn create_tables(mut tx: Transaction<'_, Any>) -> Result<()> { + create_block_table(&mut tx).await?; + create_transaction_table(&mut tx).await?; + create_cell_table(&mut tx).await?; + create_live_cell_table(&mut tx).await?; + create_indexer_cell_table(&mut tx).await?; + create_script_table(&mut tx).await?; + create_canonical_chain_table(&mut tx).await?; + create_registered_address_table(&mut tx).await?; + create_sync_status_table(&mut tx).await?; + create_in_update_table(&mut tx).await?; tx.commit().await?; Ok(()) } diff --git a/db/xsql-test/src/sql.rs b/db/xsql-test/src/sql.rs index 520682762..0538ea320 100644 --- a/db/xsql-test/src/sql.rs +++ b/db/xsql-test/src/sql.rs @@ -1,36 +1,86 @@ -use rbatis::executor::RBatisTxExecutor; -use rbatis::sql; +use common::anyhow::Result; +use sqlx::{Any, Transaction}; -#[sql(tx, "DELETE FROM mercury_block")] -pub async fn delete_block_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_block_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_block") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_transaction")] -pub async fn delete_transaction_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_transaction_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_transaction") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_cell")] -pub async fn delete_cell_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_cell_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_cell") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_consume_info")] -pub async fn delete_consume_info_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_consume_info_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_consume_info") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_live_cell")] -pub async fn delete_live_cell_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_live_cell_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_live_cell") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_indexer_cell")] -pub async fn delete_indexer_cell_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_indexer_cell_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_indexer_cell") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_script")] -pub async fn delete_script_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_script_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_script") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_canonical_chain")] -pub async fn delete_canonical_chain_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_canonical_chain_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_canonical_chain") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql(tx, "DELETE FROM mercury_registered_address")] -pub async fn delete_registered_address_table_data(tx: &mut RBatisTxExecutor<'_>) -> () {} +pub async fn delete_registered_address_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_registered_address") + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_block( +pub async fn delete_sync_status_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_sync_status") + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn delete_in_update_table_data(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query("DELETE FROM mercury_in_update") + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn create_block_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_block( block_hash blob PRIMARY KEY, block_number int NOT NULL, version smallint NOT NULL, @@ -48,13 +98,16 @@ pub async fn delete_registered_address_table_data(tx: &mut RBatisTxExecutor<'_>) dao blob NOT NULL, nonce blob NOT NULL, proposals blob - )" -)] -pub async fn create_block_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_transaction( +pub async fn create_transaction_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_transaction( id bigint PRIMARY KEY, tx_hash blob NOT NULL, tx_index smallint NOT NULL, @@ -67,13 +120,16 @@ pub async fn create_block_table(tx: &mut RBatisTxExecutor<'_>) -> () {} cell_deps blob, header_deps blob, witnesses blob - )" -)] -pub async fn create_transaction_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_cell( +pub async fn create_cell_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_cell( id bigint PRIMARY KEY, tx_hash blob NOT NULL, output_index smallint NOT NULL, @@ -99,13 +155,16 @@ pub async fn create_transaction_table(tx: &mut RBatisTxExecutor<'_>) -> () {} consumed_tx_index int, input_index int, since blob - )" -)] -pub async fn create_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_consume_info( +pub async fn create_consume_info_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_consume_info( tx_hash blob NOT NULL, output_index int NOT NULL, consumed_block_number bigint NOT NULL, @@ -115,13 +174,16 @@ pub async fn create_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} input_index int NOT NULL, since blob NOT NULL, PRIMARY KEY(tx_hash, output_index) - )" -)] -pub async fn create_consume_info_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_live_cell( +pub async fn create_live_cell_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_live_cell( id bigint PRIMARY KEY, output_index smallint NOT NULL, tx_hash blob NOT NULL, @@ -142,13 +204,16 @@ pub async fn create_consume_info_table(tx: &mut RBatisTxExecutor<'_>) -> () {} type_args blob, type_script_type smallint, data blob - )" -)] -pub async fn create_live_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_indexer_cell( +pub async fn create_indexer_cell_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_indexer_cell( id bigint PRIMARY KEY, block_number int NOT NULL, io_type smallint NOT NULL, @@ -163,46 +228,71 @@ pub async fn create_live_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} type_code_hash blob, type_args blob, type_script_type smallint - )" -)] -pub async fn create_indexer_cell_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_script( - id bigint PRIMARY KEY, - script_hash blob NOT NULL, +pub async fn create_script_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_script( + script_hash blob NOT NULL PRIMARY KEY, script_hash_160 blob NOT NULL, script_code_hash blob NOT NULL, script_args blob, script_type smallint NOT NULL, script_args_len smallint - )" -)] -pub async fn create_script_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_canonical_chain( +pub async fn create_canonical_chain_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_canonical_chain( block_number bigint PRIMARY KEY, block_hash blob NOT NULL - )" -)] -pub async fn create_canonical_chain_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_registered_address( +pub async fn create_registered_address_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_registered_address( lock_hash blob NOT NULL PRIMARY KEY, address varchar NOT NULL - )" -)] -pub async fn create_registered_address_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} -#[sql( - tx, - "CREATE TABLE mercury_sync_status( +pub async fn create_sync_status_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_sync_status( block_number int NOT NULL PRIMARY KEY - )" -)] -pub async fn create_sync_status_table(tx: &mut RBatisTxExecutor<'_>) -> () {} + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn create_in_update_table(tx: &mut Transaction<'_, Any>) -> Result<()> { + sqlx::query( + "CREATE TABLE mercury_in_update( + is_in bool NOT NULL PRIMARY KEY + )", + ) + .execute(&mut *tx) + .await?; + Ok(()) +} diff --git a/db/xsql/Cargo.toml b/db/xsql/Cargo.toml deleted file mode 100644 index c066d6ad0..000000000 --- a/db/xsql/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "xsql" -version = "0.1.0" -authors = ["Nervos Network"] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -base64 = "0.13" -log = "0.4" -rbatis = { version = "3.0", default-features = false, features = ["all-database", "runtime-tokio-native-tls", "upper_case_sql_keyword"] } -rbson = "2.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" - -common = { path = "../../common" } -protocol = { path = "../../protocol" } - -[dev-dependencies] -arc-swap = "1.5" -criterion = { version = "0.3", features = ["async_tokio"] } -env_logger = "0.9" -rand = "0.7" diff --git a/db/xsql/src/lib.rs b/db/xsql/src/lib.rs deleted file mode 100644 index f059d1c8a..000000000 --- a/db/xsql/src/lib.rs +++ /dev/null @@ -1,195 +0,0 @@ -pub mod page; - -pub use rbatis; - -use common::{anyhow::anyhow, Result}; -use protocol::db::DBDriver; - -use log::LevelFilter; -use rbatis::crud::{CRUDTable, CRUD}; -use rbatis::executor::{RBatisConnExecutor, RBatisTxExecutor}; -use rbatis::{ - core::db::DBPoolOptions, plugin::log::RbatisLogPlugin, rbatis::Rbatis, wrapper::Wrapper, -}; -use serde::{de::DeserializeOwned, ser::Serialize}; - -use std::{fmt::Debug, sync::Arc, time::Duration}; - -#[derive(Clone)] -pub struct XSQLPool { - pool: Arc, - center_id: u16, - node_id: u16, - max_conn: u32, - _min_conn: u32, - _conn_timeout: Duration, - _max_lifetime: Duration, - _idle_timeout: Duration, -} - -impl Debug for XSQLPool { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("XSQLPool") - .field("center_id", &self.center_id) - .field("node_id", &self.node_id) - .field("max_conn", &self.max_conn) - .finish() - } -} - -impl XSQLPool { - pub fn new( - center_id: u16, - node_id: u16, - max_connections: u32, - min_connections: u32, - connection_timeout: u64, - max_lifetime: u64, - idle_timeout: u64, - log_level: LevelFilter, - ) -> Self { - let mut rbatis = Rbatis::new(); - rbatis.set_log_plugin(RbatisLogPlugin { - level_filter: log_level, - }); - rbatis.set_page_plugin(page::CursorPagePlugin); - - XSQLPool { - pool: Arc::new(rbatis), - center_id, - node_id, - max_conn: max_connections, - _min_conn: min_connections, - _conn_timeout: Duration::from_secs(connection_timeout), - _max_lifetime: Duration::from_secs(max_lifetime), - _idle_timeout: Duration::from_secs(idle_timeout), - } - } - - pub async fn connect( - &self, - db_driver: DBDriver, - db_name: &str, - host: &str, - port: u16, - user: &str, - password: &str, - ) -> Result<()> { - self.pool - .link_opt( - &build_url(db_driver.into(), db_name, host, port, user, password), - DBPoolOptions { - max_connections: self.max_conn, - min_connections: 2, - idle_timeout: Some(Duration::from_secs(3)), - test_before_acquire: false, - ..Default::default() - }, - ) - .await - .map_err(Into::into) - } - - pub async fn transaction(&self) -> Result> { - let tx = self.pool.acquire_begin().await?; - Ok(tx) - } - - pub async fn acquire(&self) -> Result> { - let conn = self.pool.acquire().await?; - Ok(conn) - } - - pub async fn fetch_count_by_wrapper(&self, w: Wrapper) -> Result { - let ret = self.pool.fetch_count_by_wrapper::(w).await?; - Ok(ret) - } - - pub async fn fetch_by_wrapper(&self, w: Wrapper) -> Result { - let ret = self.pool.fetch_by_wrapper(w).await?; - Ok(ret) - } - - pub async fn fetch_list_by_wrapper( - &self, - w: Wrapper, - ) -> Result> { - let ret = self.pool.fetch_list_by_wrapper(w).await?; - Ok(ret) - } - - pub async fn fetch_by_column( - &self, - column: &str, - value: &C, - ) -> Result { - let ret = self.pool.fetch_by_column(column, value).await?; - Ok(ret) - } - - pub async fn fetch_list_by_column< - T: CRUDTable + DeserializeOwned, - C: Serialize + Sync + Send, - >( - &self, - column: &str, - values: &[C], - ) -> Result> { - let ret = self.pool.fetch_list_by_column(column, values).await?; - Ok(ret) - } - - pub async fn fetch_list(&self) -> Result> { - let ret = self.pool.fetch_list().await?; - Ok(ret) - } - - pub fn wrapper(&self) -> Wrapper { - self.pool.new_wrapper() - } - - pub fn center_id(&self) -> u16 { - self.center_id - } - - pub fn node_id(&self) -> u16 { - self.node_id - } - - pub fn get_max_connections(&self) -> u32 { - self.max_conn - } -} - -pub async fn commit_transaction(mut tx: RBatisTxExecutor<'_>) -> Result<()> { - if tx.commit().await.is_err() { - tx.rollback().await?; - return Err(anyhow!("Commit transaction failed, transaction rollback!")); - } - - Ok(()) -} - -fn build_url( - db_type: &str, - db_name: &str, - host: &str, - port: u16, - user: &str, - password: &str, -) -> String { - if db_type == protocol::db::SQLITE { - return db_type.to_string() + db_name; - } - - db_type.to_string() - + user - + ":" - + password - + "@" - + host - + ":" - + port.to_string().as_str() - + "/" - + db_name -} diff --git a/db/xsql/src/page.rs b/db/xsql/src/page.rs deleted file mode 100644 index d2be87e6a..000000000 --- a/db/xsql/src/page.rs +++ /dev/null @@ -1,191 +0,0 @@ -use common::PaginationRequest; - -use rbatis::plugin::page::{IPageRequest, PagePlugin}; -use rbatis::{core::Error as RbError, sql::TEMPLATE, DriverType}; -use rbson::Bson; -use serde::{Deserialize, Serialize}; - -#[derive(Default, Clone, Debug)] -pub struct CursorPagePlugin; - -impl PagePlugin for CursorPagePlugin { - fn make_page_sql( - &self, - _dtype: &DriverType, - sql: &str, - _args: &Vec, - page: &dyn IPageRequest, - ) -> Result<(String, String), RbError> { - let sql = sql.trim().to_owned(); - if !sql.starts_with(TEMPLATE.select.right_space) - && !sql.contains(TEMPLATE.from.left_right_space) - { - return Err(RbError::from("sql must contains 'select ' And ' from '")); - } - - let mut count_sql = sql.clone(); - if page.is_search_count() { - // make count sql - count_sql = self.make_count_sql(&count_sql); - } - - let (first_part, second_part, has_where) = self.split_sql(&sql); - - let compare = if page.get_total() == 1 { - String::from(">") - } else { - String::from("<") - }; - - let page_part = if has_where { - format!( - "id {} {} {}", - compare, - page.get_page_no(), - TEMPLATE.and.value - ) - } else { - format!( - "{} id {} {}", - TEMPLATE.r#where.value, - compare, - page.get_page_no() - ) - }; - - let mut order_by_part = format!("{} id ", TEMPLATE.order_by.value); - if page.get_total() == 1 { - order_by_part += TEMPLATE.asc.value; - } else { - order_by_part += TEMPLATE.desc.value; - }; - - let limit_part = format!( - "{} {} {} {}", - TEMPLATE.limit.value, - page.get_page_size(), - TEMPLATE.offset.value, - page.offset(), - ); - - let limit_sql = format!( - "{} {} {} {} {}", - first_part, page_part, second_part, order_by_part, limit_part - ); - - Ok((count_sql, limit_sql)) - } -} - -impl CursorPagePlugin { - fn split_sql(&self, sql: &str) -> (String, String, bool) { - let (mid, has_where) = if sql.contains(TEMPLATE.r#where.left_right_space) { - ( - sql.find(TEMPLATE.r#where.left_right_space).unwrap() + 6, - true, - ) - } else { - (sql.len(), false) - }; - let (a, b) = sql.split_at(mid); - - (a.to_string(), b.to_string(), has_where) - } - - fn make_count_sql(&self, sql: &str) -> String { - let mut from_index = sql.find(TEMPLATE.from.left_right_space); - if from_index.is_some() { - from_index = Option::Some(from_index.unwrap() + TEMPLATE.from.left_right_space.len()); - } - let mut where_sql = sql[from_index.unwrap_or(0)..sql.len()].to_string(); - - // Remove ORDER_BY. - if where_sql.contains(TEMPLATE.order_by.left_right_space) { - where_sql = where_sql[0..where_sql - .rfind(TEMPLATE.order_by.left_right_space) - .unwrap_or(where_sql.len())] - .to_string(); - } - - // // Remove LIMIT. - // if where_sql.contains(TEMPLATE.limit.left_right_space) { - // where_sql = where_sql[0..where_sql - // .rfind(TEMPLATE.limit.left_right_space) - // .unwrap_or_else(|| where_sql.len())] - // .to_string(); - // } - - format!("{} count(1) FROM {} ", TEMPLATE.select.value, where_sql) - } -} - -#[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq, Eq)] -pub struct PageRequest { - pub cursor: u64, - pub skip: u64, - pub count: u64, - pub is_asc: bool, - pub search_count: bool, -} - -impl From for PageRequest { - // The id type in the table definition is i64, - // so it is necessary to limit the numerical range of cursor, limit and skip of type u64, - // therwise a database error will be returned when querying - fn from(page: PaginationRequest) -> Self { - PageRequest { - cursor: page - .cursor - .map(|cursor| cursor.min(i64::MAX as u64)) - .unwrap_or(if page.order.is_asc() { - 0u64 - } else { - i64::MAX as u64 - }), - count: page - .limit - .map(|limit| limit.min(i64::MAX as u64)) - .unwrap_or(i64::MAX as u64), - skip: page - .skip - .map(|skip| skip.min(i64::MAX as u64)) - .unwrap_or(0u64), - is_asc: page.order.is_asc(), - search_count: page.return_count, - } - } -} - -impl IPageRequest for PageRequest { - fn get_page_size(&self) -> u64 { - self.count - } - - fn get_page_no(&self) -> u64 { - self.cursor as u64 - } - - fn offset(&self) -> u64 { - self.skip - } - - fn is_search_count(&self) -> bool { - self.search_count - } - - fn get_total(&self) -> u64 { - self.is_asc.into() - } - - fn set_page_size(&mut self, arg: u64) { - self.count = arg; - } - - fn set_search_count(&mut self, arg: bool) { - self.search_count = arg; - } - - fn set_total(&mut self, _: u64) {} - - fn set_page_no(&mut self, _: u64) {} -} diff --git a/devtools/config/docker_compose_config.toml b/devtools/config/docker_compose_config.toml index c3b9ae92a..add355812 100644 --- a/devtools/config/docker_compose_config.toml +++ b/devtools/config/docker_compose_config.toml @@ -30,7 +30,6 @@ db_port = 5432 db_name = "mercury" db_user = "postgres" password = "123456" -db_log_level = "WARN" [log_config] diff --git a/devtools/config/mainnet_config.toml b/devtools/config/mainnet_config.toml index 23fef42ad..b282bfe95 100644 --- a/devtools/config/mainnet_config.toml +++ b/devtools/config/mainnet_config.toml @@ -29,7 +29,6 @@ db_port = 8432 db_name = "mercury" db_user = "postgres" password = "123456" -db_log_level = "WARN" [log_config] diff --git a/devtools/config/testnet_config.toml b/devtools/config/testnet_config.toml index 72bd57d62..bf4611ac5 100644 --- a/devtools/config/testnet_config.toml +++ b/devtools/config/testnet_config.toml @@ -5,16 +5,11 @@ allow_parallel_sync = true rpc_thread_num = 3 flush_tx_pool_cache_interval = 300 - # Fill cellbase maturity in the same as the ckb genesis block. cellbase_maturity = 4 - cheque_since = 6 - pool_cache_size = 100 - is_pprof_enabled = true - [db_config] center_id = 0 machine_id = 0 @@ -29,7 +24,6 @@ db_port = 8432 db_name = "mercury" db_user = "postgres" password = "123456" -db_log_level = "WARN" [log_config] diff --git a/devtools/create_table/create_sqlite_table.sql b/devtools/create_table/create_sqlite_table.sql index 4bc769c12..332d2fe32 100644 --- a/devtools/create_table/create_sqlite_table.sql +++ b/devtools/create_table/create_sqlite_table.sql @@ -102,8 +102,7 @@ CREATE TABLE mercury_indexer_cell( ); CREATE TABLE mercury_script( - id bigint PRIMARY KEY, - script_hash blob NOT NULL, + script_hash blob NOT NULL PRIMARY KEY, script_hash_160 blob NOT NULL, script_code_hash blob NOT NULL, script_args blob, diff --git a/devtools/test_data/blocks/10.json b/devtools/test_data/blocks/10.json new file mode 100644 index 000000000..72d13cb59 --- /dev/null +++ b/devtools/test_data/blocks/10.json @@ -0,0 +1,40 @@ +{ + "header": { + "compact_target": "0x1e015555", + "dao": "0x2c6f4996b920a12ed2ec9edcf286230092c90fbd8700000000b2b49f02fbfe06", + "epoch": "0x3e8000a000000", + "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x4bdff7e0bd58c9cca74af5a51792e8119958ab6b5e5cacbb8f8c615e5c4b61dc", + "nonce": "0xc58787635b7666eac7aad17bcfe01518", + "number": "0xa", + "parent_hash": "0x953761d56c03bfedf5e70dde0583470383184c41331f709df55d4acab5358640", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b99fa7a", + "transactions_root": "0xba4fff1f10b3505e6f63ad1ed761d40effe993963c7c68174ae7abbac94613cf", + "version": "0x0" + }, + "proposals": [ ], + "transactions": [ + { + "cell_deps": [ ], + "hash": "0x672afe6ede97904ccddcbb38910c7e1c5aa9066b7e6715ee9e72d0153a471341", + "header_deps": [ ], + "inputs": [ + { + "previous_output": { + "index": "0xffffffff", + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "since": "0xa" + } + ], + "outputs": [ ], + "outputs_data": [ ], + "version": "0x0", + "witnesses": [ + "0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef" + ] + } + ], + "uncles": [ ] + } diff --git a/devtools/test_data/blocks/100.json b/devtools/test_data/blocks/100.json deleted file mode 100644 index 5c80de640..000000000 --- a/devtools/test_data/blocks/100.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "header": { - "version": "0x0", - "compact_target": "0x1a08a97e", - "timestamp": "0x16e71735ae1", - "number": "0x64", - "epoch": "0x6cf0064000000", - "parent_hash": "0x4d0913d3d9330b1f2acf70d1b38baffa1d0588a92b006be3c5a0ca031e9841c7", - "transactions_root": "0x70575fdaab2e04e75be67e92e36f1f1bc24f2fc73db54c47fd111061fca60c0d", - "proposals_hash": "0x4dfc5e9a95d5c148d32719114985079b64aa4183582fdf8da0db9229e6fac936", - "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "dao": "0xa8f90c178a2ba12e1a1659e0f4862300f9608bfbc002000000ac29627cfffe06", - "nonce": "0x451b24b070000000000000094e8e0c5", - "hash": "0xdf49b9fe0e635144f181c74a4d36da231acc4810c813c7f29dd0702bf5cdfb1d" - }, - "uncles": [], - "transactions": [ - { - "version": "0x0", - "cell_deps": [], - "header_deps": [], - "inputs": [ - { - "since": "0x64", - "previous_output": { - "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "index": "0xffffffff" - } - } - ], - "outputs": [ - { - "capacity": "0x1ad91e82d7", - "lock": { - "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", - "hash_type": "type", - "args": "0xdde7801c073dfb3464c7b1f05b806bb2bbb84e99" - }, - "type": null - } - ], - "outputs_data": [ - "0x" - ], - "witnesses": [ - "0x5f0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000ddc1ddf9c135061b7635ca51e735fc2b03cee339060000007575706f6f6c" - ], - "hash": "0x41169e91e5d4ec4408d0c6336191476bc88b385422d5b078e351c43dd3055797" - }, - { - "version": "0x0", - "cell_deps": [ - { - "out_point": { - "tx_hash": "0x71a7ba8fc96349fea0ed3a5c47992e3b4084b031a42264a018e0072e8172e46c", - "index": "0x0" - }, - "dep_type": "dep_group" - } - ], - "header_deps": [], - "inputs": [ - { - "since": "0x0", - "previous_output": { - "tx_hash": "0xe2fb199810d49a4d8beec56718ba2593b665db9d52299a0f9e6e75416d73ff5c", - "index": "0x69" - } - } - ], - "outputs": [ - { - "capacity": "0x16b969d00", - "lock": { - "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", - "hash_type": "type", - "args": "0xb2df025a0b1bdee6a7117cbbb0543bc12e82086c" - }, - "type": null - }, - { - "capacity": "0x6a94d5e3ac6130", - "lock": { - "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", - "hash_type": "type", - "args": "0x02146339de92d43813333794acfed8a8b3266a79" - }, - "type": null - } - ], - "outputs_data": [ - "0x", - "0x" - ], - "witnesses": [ - "0x55000000100000005500000055000000410000004f881edad42c5a0b75f450decfcf6c9c25012803c7ca2065a55c1bad86ce24f34de5c03636e698901fcb48856e5502d62715a82860bc7e862fce227c15c2cf1e00" - ], - "hash": "0x3e0faf611735129e329088a6aeb0c1adcf86123d6e6c0063700d89e6411bb7fb" - } - ], - "proposals": [ - "0x785319a921bf0f44d39e", - "0x7ebd82392b2f3172965e", - "0x81f6467121a45c45d2c3", - "0x5010efc26b9c03a98ca2", - "0x377adba95da7c942044e", - "0xd5bbd540d5f5512cd480", - "0x330c8b030e81bc1805ef", - "0x2749054fe16f1770e656" - ] - } diff --git a/devtools/test_data/blocks/11.json b/devtools/test_data/blocks/11.json new file mode 100644 index 000000000..288f09835 --- /dev/null +++ b/devtools/test_data/blocks/11.json @@ -0,0 +1,40 @@ +{ + "header": { + "compact_target": "0x1e015555", + "dao": "0x30563e87f420a12eaeea81e7f28623004bd82ce29300000000b2b49f02fbfe06", + "epoch": "0x3e8000b000000", + "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x18b8cda2aecd83b25df917e28fde1bff31f032085463dd805074df0e68edeec6", + "nonce": "0x2afa31617052a71d4459a64d57e79254", + "number": "0xb", + "parent_hash": "0x4bdff7e0bd58c9cca74af5a51792e8119958ab6b5e5cacbb8f8c615e5c4b61dc", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b9a054e", + "transactions_root": "0xa36b11a686863d353d9f4f638b74ebcbeba8e765ceec7186e2f3dcb0c2ec64e2", + "version": "0x0" + }, + "proposals": [ ], + "transactions": [ + { + "cell_deps": [ ], + "hash": "0x4298daf91148df9093c844d2ae7d16bee6b74e7ab1ccccd108ce834d1ca1a56c", + "header_deps": [ ], + "inputs": [ + { + "previous_output": { + "index": "0xffffffff", + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "since": "0xb" + } + ], + "outputs": [ ], + "outputs_data": [ ], + "version": "0x0", + "witnesses": [ + "0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef" + ] + } + ], + "uncles": [ ] + } diff --git a/devtools/test_data/blocks/12.json b/devtools/test_data/blocks/12.json new file mode 100644 index 000000000..7b7793e39 --- /dev/null +++ b/devtools/test_data/blocks/12.json @@ -0,0 +1,70 @@ +{ + "header": { + "compact_target": "0x1e015555", + "dao": "0x343d33782f21a12e7fe864f2f2862300b9e94907a0000000004f4b0b04fbfe06", + "epoch": "0x3e8000c000000", + "extra_hash": "0xad22239f5cf73ca4a75261ac3ea96169e8543226b143205d0f6987a26c8aa39b", + "hash": "0xfb27201670e48f65b93b58c4cac7348c54554ad831ed5c1b386c9bd3c24fa911", + "nonce": "0x50e30c2a3df9de17fd0d1abb3b5e9206", + "number": "0xc", + "parent_hash": "0x18b8cda2aecd83b25df917e28fde1bff31f032085463dd805074df0e68edeec6", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b9a0f32", + "transactions_root": "0x556ef527219d99eaf3909a715facd3579e3afb8a50f79e00bc58ecff0da1b847", + "version": "0x0" + }, + "proposals": [ ], + "transactions": [ + { + "cell_deps": [ ], + "hash": "0xa6789f42b0568b1872e5a5858f0c42148dd8d313f844252f5fe3dfe556958ba9", + "header_deps": [ ], + "inputs": [ + { + "previous_output": { + "index": "0xffffffff", + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "since": "0xc" + } + ], + "outputs": [ + { + "capacity": "0x2ecbd7f365", + "lock": { + "args": "0xda648442dbb7347e467d1d09da13e5cd3a0ef0e1", + "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", + "hash_type": "type" + }, + "type": null + } + ], + "outputs_data": [ + "0x" + ], + "version": "0x0", + "witnesses": [ + "0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef" + ] + } + ], + "uncles": [ + { + "header": { + "compact_target": "0x1e015555", + "dao": "0x101e97ff1c1fa12eacfa6990f286230065ae44b93200000000b2b49f02fbfe06", + "epoch": "0x3e80003000000", + "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x43f942855ad94439199b2ccaebe8fa07d878cd6e1765c45a3801a52aad0a790a", + "nonce": "0x9ade5d62d366a6cb921ef94cc6d1f89b", + "number": "0x3", + "parent_hash": "0x2f75669bfa970594b74126ea528ee7fe59f11a607f1c247b42171ddd611755e1", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b987a5e", + "transactions_root": "0x1f8d4796f6bd8031c9b9574dd058588acb2ba642ee3e974dfca1df82eeaf1d10", + "version": "0x0" + }, + "proposals": [ ] + } + ] + } diff --git a/devtools/test_data/blocks/13.json b/devtools/test_data/blocks/13.json new file mode 100644 index 000000000..7c9d2c04b --- /dev/null +++ b/devtools/test_data/blocks/13.json @@ -0,0 +1,52 @@ +{ + "header": { + "compact_target": "0x1e015555", + "dao": "0x382428696a21a12e46e647fdf28623006efd662cac00000000ece17605fbfe06", + "epoch": "0x3e8000d000000", + "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x9acca826acbf5278c166479847966c2da3885c6458d8de002c36f7682fd88dc5", + "nonce": "0x8dfaaa7ce8bca95198bba486dd50c17c", + "number": "0xd", + "parent_hash": "0xfb27201670e48f65b93b58c4cac7348c54554ad831ed5c1b386c9bd3c24fa911", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b9a3bfe", + "transactions_root": "0x23d68324870b6e419e9f1faf7139deee6566c10237b1f928419e8446cba48943", + "version": "0x0" + }, + "proposals": [ ], + "transactions": [ + { + "cell_deps": [ ], + "hash": "0x3de316b6fd11ed8be131559a3e09f6c4a34f86bbdfbd83404a41cd3bc6551ede", + "header_deps": [ ], + "inputs": [ + { + "previous_output": { + "index": "0xffffffff", + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "since": "0xd" + } + ], + "outputs": [ + { + "capacity": "0x2ecbd7f0af", + "lock": { + "args": "0xda648442dbb7347e467d1d09da13e5cd3a0ef0e1", + "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", + "hash_type": "type" + }, + "type": null + } + ], + "outputs_data": [ + "0x" + ], + "version": "0x0", + "witnesses": [ + "0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef" + ] + } + ], + "uncles": [ ] + } diff --git a/devtools/test_data/blocks/14.json b/devtools/test_data/blocks/14.json new file mode 100644 index 000000000..2543501af --- /dev/null +++ b/devtools/test_data/blocks/14.json @@ -0,0 +1,52 @@ +{ + "header": { + "compact_target": "0x1e015555", + "dao": "0x3c0b1d5aa521a12e02e42a08f386230069138451b8000000008978e206fbfe06", + "epoch": "0x3e8000e000000", + "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x8aaeb0a28d3871472b361d5e5daa636e669459871a53509d5a707c95bb97a423", + "nonce": "0xb3a0887fc164e00f685e343e3b771eb8", + "number": "0xe", + "parent_hash": "0x9acca826acbf5278c166479847966c2da3885c6458d8de002c36f7682fd88dc5", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b9a6165", + "transactions_root": "0x3379f4fa262ea20cf2ba80e9a995f5f585b3563eee721a6e8db543f5ace4f4b6", + "version": "0x0" + }, + "proposals": [ ], + "transactions": [ + { + "cell_deps": [ ], + "hash": "0x9527ee3809675e766ed13ae6e62ff2a6bc8cd5df96313b8045d6e471006380d3", + "header_deps": [ ], + "inputs": [ + { + "previous_output": { + "index": "0xffffffff", + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "since": "0xe" + } + ], + "outputs": [ + { + "capacity": "0x2ecbd7edf9", + "lock": { + "args": "0xda648442dbb7347e467d1d09da13e5cd3a0ef0e1", + "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", + "hash_type": "type" + }, + "type": null + } + ], + "outputs_data": [ + "0x" + ], + "version": "0x0", + "witnesses": [ + "0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef" + ] + } + ], + "uncles": [ ] + } diff --git a/devtools/test_data/blocks/15.json b/devtools/test_data/blocks/15.json new file mode 100644 index 000000000..00ec09347 --- /dev/null +++ b/devtools/test_data/blocks/15.json @@ -0,0 +1,52 @@ +{ + "header": { + "compact_target": "0x1e015555", + "dao": "0x40f2114be021a12eb4e10d13f3862300aa2ba176c400000000260f4e08fbfe06", + "epoch": "0x3e8000f000000", + "extra_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x5d175450c75eb22e4dcddecc1de9db47ecea7e7d5a0c0474fbfa2d8fd931b5ef", + "nonce": "0xa0dd32f178c695253c52d8b27816403e", + "number": "0xf", + "parent_hash": "0x8aaeb0a28d3871472b361d5e5daa636e669459871a53509d5a707c95bb97a423", + "proposals_hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x1723b9ac6e5", + "transactions_root": "0xd71cfebdebfe7ccd25d6100b276f408938331e7e4901c522449d11d818ad0693", + "version": "0x0" + }, + "proposals": [ ], + "transactions": [ + { + "cell_deps": [ ], + "hash": "0xcfade214c4a4b06952734234767681d802888316f53f3a13140fb5f6f5a48edd", + "header_deps": [ ], + "inputs": [ + { + "previous_output": { + "index": "0xffffffff", + "tx_hash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "since": "0xf" + } + ], + "outputs": [ + { + "capacity": "0x2ecbd7eb43", + "lock": { + "args": "0xda648442dbb7347e467d1d09da13e5cd3a0ef0e1", + "code_hash": "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", + "hash_type": "type" + }, + "type": null + } + ], + "outputs_data": [ + "0x" + ], + "version": "0x0", + "witnesses": [ + "0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef" + ] + } + ], + "uncles": [ ] + } diff --git a/docs/config.md b/docs/config.md index 39e4b8633..9bc3e0bee 100644 --- a/docs/config.md +++ b/docs/config.md @@ -120,12 +120,6 @@ The password of the database. type: `String` -### `db_log_level` - -The log level of the database, uppercase. - -type: `String` - ## Log configuration ### `log_level` diff --git a/docs/layout.md b/docs/layout.md index 2e26bcac9..f47e310e0 100644 --- a/docs/layout.md +++ b/docs/layout.md @@ -2,9 +2,6 @@ ```sh . -├── apm -│   ├── tracing -│   └── tracing-derive ├── common │   ├── address.rs │   ├── hash.rs @@ -22,7 +19,7 @@ │   ├── storage │   └── synchronization ├── db -│   ├── xsql +│   ├── db-sqlx │   └── xsql-test ├── devtools │   ├── config @@ -43,7 +40,6 @@ A brief description: -- `apm` Contains the application performance monitor. - `common` Contains utilities for mercury. - `core` Contains implementations of module traits. - `db` Contains the database implementation. diff --git a/integration/Cargo.lock b/integration/Cargo.lock index 1107e32bd..aa0380c83 100644 --- a/integration/Cargo.lock +++ b/integration/Cargo.lock @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "common" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "async-trait", @@ -343,7 +343,7 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "core-rpc-types" -version = "0.4.0" +version = "0.4.1" dependencies = [ "arc-swap", "ckb-jsonrpc-types", @@ -1187,7 +1187,7 @@ dependencies = [ [[package]] name = "protocol" -version = "0.4.0" +version = "0.4.1" dependencies = [ "ckb-jsonrpc-types", "ckb-types", diff --git a/integration/dev_chain/devnet_config.toml b/integration/dev_chain/devnet_config.toml index 3776e891c..ff8858c51 100644 --- a/integration/dev_chain/devnet_config.toml +++ b/integration/dev_chain/devnet_config.toml @@ -29,7 +29,6 @@ db_port = 5432 db_name = "mercury-dev" db_user = "postgres" password = "123456789" -db_log_level = "WARN" [log_config] diff --git a/integration/src/tests/build_simple_transfer_transaction.rs b/integration/src/tests/build_simple_transfer_transaction.rs index fa877000d..6dcd5f099 100644 --- a/integration/src/tests/build_simple_transfer_transaction.rs +++ b/integration/src/tests/build_simple_transfer_transaction.rs @@ -177,7 +177,7 @@ fn test_simple_transfer_udt_from_provide_capacity() { // build tx let payload = SimpleTransferPayload { - asset_info: AssetInfo::new_udt(udt_hash.to_owned()), + asset_info: AssetInfo::new_udt(udt_hash), from: vec![from_address.to_string()], to: vec![ToInfo { address: to_address_secp.to_string(), @@ -190,7 +190,7 @@ fn test_simple_transfer_udt_from_provide_capacity() { let tx = mercury_client .build_simple_transfer_transaction(payload) .unwrap(); - let tx = sign_transaction(tx, &[from_address_pk.to_owned()]).unwrap(); + let tx = sign_transaction(tx, &[from_address_pk]).unwrap(); // send tx to ckb node let _tx_hash = send_transaction_to_ckb(tx).unwrap(); diff --git a/integration/src/utils/address.rs b/integration/src/utils/address.rs index 2870ab751..5e0f358ba 100644 --- a/integration/src/utils/address.rs +++ b/integration/src/utils/address.rs @@ -8,7 +8,9 @@ use anyhow::{anyhow, Result}; use ckb_hash::blake2b_256; use ckb_types::{bytes::Bytes, core::ScriptHashType, packed, prelude::*, H160, H256}; use common::{ - address::is_acp, address::is_secp256k1, hash::blake2b_160, Address, AddressPayload, NetworkType, + address::{is_acp, is_secp256k1}, + hash::blake2b_160, + Address, AddressPayload, NetworkType, }; use core_rpc_types::{Identity, IdentityFlag}; use crypto::digest::Digest; @@ -66,17 +68,6 @@ fn generate_rand_private_key() -> H256 { H256(rand::thread_rng().gen::<[u8; 32]>()) } -fn _caculate_scirpt_hash(code_hash: &str, args: &str, script_hash_type: ScriptHashType) -> H256 { - let code_hash = H256::from_str(code_hash).unwrap(); - let args = H256::from_str(args).unwrap(); - let script = packed::Script::new_builder() - .hash_type(script_hash_type.into()) - .code_hash(code_hash.pack()) - .args(ckb_types::bytes::Bytes::from(args.as_bytes().to_owned()).pack()) - .build(); - script.calc_script_hash().unpack() -} - pub fn build_cheque_address( receiver_address: &Address, sender_address: &Address, @@ -150,42 +141,25 @@ pub(crate) fn generate_rand_pw_address_pk_pair() -> (Address, H256) { (address, pk) } -#[test] -fn test_caculate_lock_hash() { - let code_hash = "00000000000000000000000000000000000000000000000000545950455f4944"; - - // sudt - let args = "314f67c0ffd0c6fbffe886f03c6b00b42e4e66e3e71d32a66b8a38d69e6a4250"; - let script_hash_type = ScriptHashType::Type; - let script_hash = _caculate_scirpt_hash(code_hash, args, script_hash_type); - assert_eq!( - "9c6933d977360f115a3e9cd5a2e0e475853681b80d775d93ad0f8969da343e56", - &script_hash.to_string() - ); - - // anyone_can_pay - let args = "57fdfd0617dcb74d1287bb78a7368a3a4bf9a790cfdcf5c1a105fd7cb406de0d"; - let script_hash_type = ScriptHashType::Type; - let script_hash = _caculate_scirpt_hash(code_hash, args, script_hash_type); - assert_eq!( - "6283a479a3cf5d4276cd93594de9f1827ab9b55c7b05b3d28e4c2e0a696cfefd", - &script_hash.to_string() - ); -} +#[cfg(test)] +mod test { + use super::*; -#[test] -fn test_build_addresses() { - let _ = common::lazy::SECP256K1_CODE_HASH.set(SIGHASH_TYPE_HASH); + #[test] + fn test_build_addresses() { + let _ = common::lazy::SECP256K1_CODE_HASH.set(SIGHASH_TYPE_HASH); - let (address, _) = generate_rand_secp_address_pk_pair(); - assert!(is_secp256k1(&address)); + let (address, _) = generate_rand_secp_address_pk_pair(); + assert!(is_secp256k1(&address)); - let sender = Address::from_str("ckt1qzda0cr08m85hc8jlnfp3zer7xulejywt49kt2rr0vthywaa50xwsq06y24q4tc4tfkgze35cc23yprtpzfrzygljdjh9").unwrap(); - let receiver = Address::from_str("ckt1qyqf4n4g6qfrvnp78ry4sm0tn8wgpjqf6ufq74srld").unwrap(); - let cheque = build_cheque_address(&receiver, &sender).unwrap(); - assert_eq!("ckt1qqdpunl0xn6es2gx7azmqj870vggjer7sg6xqa8q7vkzan3xea43uqt6g2dxvxxjtdhfvfs0f67gwzgrcrfg3gj9yywse6zu05ez3s64xmtdkl6074rac6q3f7cvk".to_string(), cheque.to_string()); + let sender = Address::from_str("ckt1qzda0cr08m85hc8jlnfp3zer7xulejywt49kt2rr0vthywaa50xwsq06y24q4tc4tfkgze35cc23yprtpzfrzygljdjh9").unwrap(); + let receiver = Address::from_str("ckt1qyqf4n4g6qfrvnp78ry4sm0tn8wgpjqf6ufq74srld").unwrap(); + let cheque = build_cheque_address(&receiver, &sender).unwrap(); + assert_eq!("ckt1qqdpunl0xn6es2gx7azmqj870vggjer7sg6xqa8q7vkzan3xea43uqt6g2dxvxxjtdhfvfs0f67gwzgrcrfg3gj9yywse6zu05ez3s64xmtdkl6074rac6q3f7cvk".to_string(), cheque.to_string()); - let address_secp = Address::from_str("ckt1qyqf4n4g6qfrvnp78ry4sm0tn8wgpjqf6ufq74srld").unwrap(); - let acp_address = build_acp_address(&address_secp).unwrap(); - assert_eq!("ckt1qp3g8fre50846snkekf4jn0f7xp84wd4t3astv7j3exzuznfdnl06qv6e65dqy3kfslr3j2cdh4enhyqeqyawysf7sf4c".to_string(), acp_address.to_string()); + let address_secp = + Address::from_str("ckt1qyqf4n4g6qfrvnp78ry4sm0tn8wgpjqf6ufq74srld").unwrap(); + let acp_address = build_acp_address(&address_secp).unwrap(); + assert_eq!("ckt1qp3g8fre50846snkekf4jn0f7xp84wd4t3astv7j3exzuznfdnl06qv6e65dqy3kfslr3j2cdh4enhyqeqyawysf7sf4c".to_string(), acp_address.to_string()); + } } diff --git a/logger/Cargo.toml b/logger/Cargo.toml index 674ec3ea9..7b946d2a9 100644 --- a/logger/Cargo.toml +++ b/logger/Cargo.toml @@ -16,8 +16,6 @@ serde_derive = "1.0" chrono = "0.4" common = { path = "../common" } -tracing = { path = "../apm/tracing" } -tracing-derive = { path = "../apm/tracing-derive" } [dev-dependencies] env_logger = "0.9" diff --git a/logger/src/lib.rs b/logger/src/lib.rs index 2e941dd32..42270cea6 100644 --- a/logger/src/lib.rs +++ b/logger/src/lib.rs @@ -1,8 +1,6 @@ mod date_fixed_roller; pub use json::{array, object}; -pub use tracing::{init_jaeger, FutureExt, LocalSpan, MercuryTrace, Span, TRACING_SPAN_TX}; -pub use tracing_derive::{tracing, tracing_async}; use date_fixed_roller::DateFixedWindowRoller; diff --git a/protocol/src/db.rs b/protocol/src/db.rs index 7ef264655..3c48f4cdb 100644 --- a/protocol/src/db.rs +++ b/protocol/src/db.rs @@ -23,6 +23,17 @@ impl Default for DBDriver { } } +#[allow(clippy::from_over_into)] +impl Into<&str> for &DBDriver { + fn into(self) -> &'static str { + match self { + DBDriver::PostgreSQL => PGSQL, + DBDriver::MySQL => MYSQL, + DBDriver::SQLite => SQLITE, + } + } +} + impl DBDriver { #[allow(clippy::should_implement_trait)] pub fn from_str(s: &str) -> Self { @@ -44,17 +55,6 @@ pub struct DBInfo { pub machine_id: i64, } -#[allow(clippy::from_over_into)] -impl Into<&str> for DBDriver { - fn into(self) -> &'static str { - match self { - DBDriver::PostgreSQL => PGSQL, - DBDriver::MySQL => MYSQL, - DBDriver::SQLite => SQLITE, - } - } -} - #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] pub struct SimpleTransaction { pub epoch_number: RationalU256, diff --git a/tests/tests/rpc/query_transactions.rs b/tests/tests/rpc/query_transactions.rs index cf86753da..89e4d0b68 100644 --- a/tests/tests/rpc/query_transactions.rs +++ b/tests/tests/rpc/query_transactions.rs @@ -602,6 +602,7 @@ fn test_query_by_pagination_limit() { let txs = &r["response"].as_array().unwrap(); assert_eq!(txs.len(), 7); + assert_eq!(r["count"], "0x9"); assert_eq!( txs[0]["value"]["tx_hash"],