diff --git a/Cargo.lock b/Cargo.lock index eff6eeb52d..02fba36015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,7 +39,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -48,7 +48,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -91,7 +91,7 @@ checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" dependencies = [ "libc", "termion", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -100,6 +100,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + [[package]] name = "backtrace" version = "0.3.40" @@ -325,6 +331,7 @@ dependencies = [ "ckb-indexer", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-miner", "ckb-network", "ckb-network-alert", @@ -379,6 +386,7 @@ dependencies = [ "ckb-instrument", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-miner", "ckb-network", "ckb-network-alert", @@ -500,7 +508,7 @@ version = "0.30.1-pre" dependencies = [ "ckb-logger", "ckb-types", - "futures", + "futures 0.1.29", "rand 0.6.5", "tentacle", "tokio", @@ -559,7 +567,7 @@ name = "ckb-future-executor" version = "0.30.1-pre" dependencies = [ "crossbeam-channel", - "futures", + "futures 0.1.29", "num_cpus", "tokio", "tokio-executor", @@ -655,6 +663,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ckb-memory-tracker" +version = "0.30.1-pre" +dependencies = [ + "ckb-logger", + "futures 0.3.4", + "heim", + "jemalloc-ctl", + "jemalloc-sys", + "serde", +] + [[package]] name = "ckb-miner" version = "0.30.1-pre" @@ -668,7 +688,7 @@ dependencies = [ "crossbeam-channel", "eaglesong", "failure", - "futures", + "futures 0.1.29", "hyper", "indicatif", "lru-cache", @@ -705,7 +725,7 @@ dependencies = [ "crossbeam-channel", "faketime", "faster-hex 0.4.1", - "futures", + "futures 0.1.29", "ipnetwork", "lazy_static", "num_cpus", @@ -786,7 +806,7 @@ version = "0.30.1-pre" dependencies = [ "ckb-logger", "ckb-types", - "futures", + "futures 0.1.29", "tentacle", ] @@ -869,6 +889,7 @@ dependencies = [ "ckb-indexer", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-network", "ckb-network-alert", "ckb-notify", @@ -885,7 +906,7 @@ dependencies = [ "ckb-verification", "crossbeam-channel", "faketime", - "futures", + "futures 0.1.29", "jsonrpc-core", "jsonrpc-derive", "jsonrpc-http-server", @@ -976,7 +997,7 @@ version = "0.30.1-pre" dependencies = [ "ckb-logger", "crossbeam-channel", - "futures", + "futures 0.1.29", "parking_lot 0.7.1", ] @@ -1016,7 +1037,7 @@ dependencies = [ "ckb-verification", "failure", "faketime", - "futures", + "futures 0.1.29", "lru-cache", "rand 0.6.5", "sentry", @@ -1079,7 +1100,7 @@ dependencies = [ "crossbeam-channel", "failure", "faketime", - "futures", + "futures 0.1.29", "lru-cache", "serde", "tokio", @@ -1137,7 +1158,7 @@ dependencies = [ "enum-display-derive", "failure", "faketime", - "futures", + "futures 0.1.29", "lru-cache", "rand 0.6.5", "rayon", @@ -1190,7 +1211,7 @@ dependencies = [ "atty", "lazy_static", "libc", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -1215,7 +1236,7 @@ dependencies = [ "regex", "termios", "unicode-width", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -1252,7 +1273,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "286e0b41c3a20da26536c6000a280585d519fd07b3956b43aed8a79e9edce980" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.5.1", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", "libc", ] @@ -1265,6 +1296,12 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + [[package]] name = "crc" version = "1.8.1" @@ -1424,8 +1461,28 @@ version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7dfd2d8b4c82121dfdff120f818e09fc4380b0b7e17a742081a89b94853e87f" dependencies = [ - "nix", - "winapi 0.3.7", + "nix 0.14.1", + "winapi 0.3.8", +] + +[[package]] +name = "darwin-libproc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade5a88af8d9646bf770687321a9488a0f2b4610aa08b0373016cd1af37f0a31" +dependencies = [ + "darwin-libproc-sys", + "libc", + "memchr", +] + +[[package]] +name = "darwin-libproc-sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30d1a078d74da1183b02fed8a8b07afc412d3998334b53b750d0ed03b031541" +dependencies = [ + "libc", ] [[package]] @@ -1659,16 +1716,107 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" +[[package]] +name = "futures" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" + [[package]] name = "futures-cpupool" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.29", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" + +[[package]] +name = "futures-macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +dependencies = [ + "proc-macro-hack", + "proc-macro2 1.0.8", + "quote 1.0.3", + "syn 1.0.14", +] + +[[package]] +name = "futures-sink" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" + +[[package]] +name = "futures-task" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" + +[[package]] +name = "futures-util" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + [[package]] name = "generic-array" version = "0.12.3" @@ -1738,7 +1886,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "fnv", - "futures", + "futures 0.1.29", "http", "indexmap", "log 0.4.8", @@ -1753,7 +1901,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -1765,6 +1913,180 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heim" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b8e403c0598237327b4143bdc1c7e01db0b358b5420579637e27cc117d573c" +dependencies = [ + "heim-common", + "heim-cpu", + "heim-disk", + "heim-host", + "heim-memory", + "heim-net", + "heim-process", + "heim-runtime", + "heim-sensors", + "heim-virt", +] + +[[package]] +name = "heim-common" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144af2a423102dd4dd3e9fdd2c77c4756fb3c6c009d691628997fd5a2247719a" +dependencies = [ + "cfg-if", + "core-foundation 0.7.0", + "futures-core", + "futures-util", + "lazy_static", + "libc", + "mach", + "nix 0.17.0", + "pin-utils", + "uom", + "winapi 0.3.8", +] + +[[package]] +name = "heim-cpu" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8abb50616d331bd75cd7f52d56b66114a0e8813fda2201c3618a105baa2b470d" +dependencies = [ + "cfg-if", + "heim-common", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "winapi 0.3.8", +] + +[[package]] +name = "heim-disk" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e5740e687a7e1f1db597e14aff112b076f48997fe617b9b165e7c2f139d248" +dependencies = [ + "bitflags", + "cfg-if", + "core-foundation 0.7.0", + "heim-common", + "heim-runtime", + "libc", + "mach", + "widestring", + "winapi 0.3.8", +] + +[[package]] +name = "heim-host" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d46019db89b0021344fd5bb8626a12cb5d9556aa8b79685ecc907b0c43e3c5f0" +dependencies = [ + "cfg-if", + "heim-common", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "platforms", + "winapi 0.3.8", +] + +[[package]] +name = "heim-memory" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216b66fececba2f68a08d15b893f0c5826346b5fb4c8d5201494f8ac2347d3bf" +dependencies = [ + "cfg-if", + "heim-common", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "winapi 0.3.8", +] + +[[package]] +name = "heim-net" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b790922244bd5b139254a0411ea2b4e4175b3e4c261d52011a9c42535c7aa7" +dependencies = [ + "bitflags", + "cfg-if", + "heim-common", + "heim-runtime", + "hex", + "libc", + "macaddr", + "nix 0.17.0", +] + +[[package]] +name = "heim-process" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8e29fca8becea6bc261c38230f3f45a358b7922451353dd068247a3ef059de" +dependencies = [ + "cfg-if", + "darwin-libproc", + "heim-common", + "heim-cpu", + "heim-host", + "heim-net", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "memchr", + "ntapi", + "ordered-float", + "winapi 0.3.8", +] + +[[package]] +name = "heim-runtime" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bd0a5a5e4af50d5d7d9537d3ecf02dca42c26bbbd8dd76621c5116dab14f69" +dependencies = [ + "cfg-if", + "futures-channel", + "heim-common", + "lazy_static", + "threadpool", +] + +[[package]] +name = "heim-sensors" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1250df4d79d4238261588e5aaf1de0e7853dbe632f5bd6868e394d1e730024" +dependencies = [ + "cfg-if", + "heim-common", + "heim-runtime", +] + +[[package]] +name = "heim-virt" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4b6ac6721ba5856659a197934ce522250a3ddd2e8646daa5660b41f1ba96457" +dependencies = [ + "cfg-if", + "heim-common", + "heim-runtime", + "raw-cpuid", +] + [[package]] name = "hermit-abi" version = "0.1.3" @@ -1774,6 +2096,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" + [[package]] name = "hostname" version = "0.1.5" @@ -1802,7 +2130,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "http", "tokio-buf", ] @@ -1835,7 +2163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "futures-cpupool", "h2", "http", @@ -1865,7 +2193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "hyper", "native-tls", "tokio-io", @@ -1993,11 +2321,22 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +[[package]] +name = "jemalloc-ctl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c502a5ff9dd2924f1ed32ba96e3b65735d837b4bfd978d3161b1702e66aca4b7" +dependencies = [ + "jemalloc-sys", + "libc", + "paste", +] + [[package]] name = "jemalloc-sys" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bef0d4ce37578dfd80b466e3d8324bd9de788e249f1accebb0c472ea4b52bdc" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" dependencies = [ "cc", "fs_extra", @@ -2029,7 +2368,7 @@ version = "14.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe3b688648f1ef5d5072229e2d672ecb92cbff7d1c79bcf3fd5898f3f3df0970" dependencies = [ - "futures", + "futures 0.1.29", "log 0.4.8", "serde", "serde_derive", @@ -2212,6 +2551,21 @@ dependencies = [ "linked-hash-map 0.5.2", ] +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matches" version = "0.1.8" @@ -2234,7 +2588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" dependencies = [ "libc", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2404,7 +2758,7 @@ checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" dependencies = [ "cfg-if", "libc", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2420,6 +2774,19 @@ dependencies = [ "void", ] +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", + "void", +] + [[package]] name = "nodrop" version = "0.1.13" @@ -2436,20 +2803,44 @@ dependencies = [ "version_check", ] +[[package]] +name = "ntapi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26e041cd983acbc087e30fcba770380cfa352d0e392e175b2344ebaf7ea0602" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "num-integer" -version = "0.1.39" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +dependencies = [ + "autocfg 1.0.0", + "num-integer", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.6" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +dependencies = [ + "autocfg 1.0.0", +] [[package]] name = "num_cpus" @@ -2550,13 +2941,22 @@ version = "0.9.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c977d08e1312e2f7e4b86f9ebaa0ed3b19d1daff75fae88bbb88108afbd801fc" dependencies = [ - "autocfg", + "autocfg 0.1.2", "cc", "libc", "pkg-config", "vcpkg", ] +[[package]] +name = "ordered-float" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18869315e81473c951eb56ad5558bbc56978562d3ecfb87abb7a1e944cea4518" +dependencies = [ + "num-traits", +] + [[package]] name = "ordermap" version = "0.3.5" @@ -2569,7 +2969,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2647,7 +3047,7 @@ dependencies = [ "rustc_version", "smallvec", "thread-id", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2662,7 +3062,29 @@ dependencies = [ "redox_syscall", "rustc_version", "smallvec", - "winapi 0.3.7", + "winapi 0.3.8", +] + +[[package]] +name = "paste" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "423a519e1c6e828f1e73b720f9d9ed2fa643dce8a7737fb43235ce0b41eeaa49" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4214c9e912ef61bf42b81ba9a47e8aad1b2ffaf739ab162bf96d1e011f54e6c5" +dependencies = [ + "proc-macro-hack", + "proc-macro2 1.0.8", + "quote 1.0.3", + "syn 1.0.14", ] [[package]] @@ -2732,6 +3154,12 @@ dependencies = [ "unicase 1.4.2", ] +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" + [[package]] name = "pkg-config" version = "0.3.14" @@ -2744,6 +3172,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "platforms" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb3b2b1033b8a60b4da6ee470325f887758c95d5320f52f9ce0df055a55940e" + [[package]] name = "ppv-lite86" version = "0.2.5" @@ -2782,6 +3216,12 @@ dependencies = [ "syn 1.0.14", ] +[[package]] +name = "proc-macro-nested" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" + [[package]] name = "proc-macro2" version = "0.4.27" @@ -2883,7 +3323,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2896,7 +3336,7 @@ dependencies = [ "fuchsia-cprng", "libc", "rand_core 0.3.1", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2905,7 +3345,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ - "autocfg", + "autocfg 0.1.2", "libc", "rand_chacha 0.1.1", "rand_core 0.4.0", @@ -2915,7 +3355,7 @@ dependencies = [ "rand_os 0.1.3", "rand_pcg", "rand_xorshift", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2937,7 +3377,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" dependencies = [ - "autocfg", + "autocfg 0.1.2", "rand_core 0.3.1", ] @@ -2947,7 +3387,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e193067942ef6f485a349a113329140d0ab9e2168ce92274499bb0e9a4190d9d" dependencies = [ - "autocfg", + "autocfg 0.1.2", "c2-chacha", "rand_core 0.5.0", ] @@ -3011,7 +3451,7 @@ checksum = "7b9ea758282efe12823e0d952ddb269d2e1897227e464919a554f2a03ef1b832" dependencies = [ "libc", "rand_core 0.4.0", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3025,7 +3465,7 @@ dependencies = [ "libc", "rand_core 0.4.0", "rdrand", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3044,7 +3484,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "autocfg", + "autocfg 0.1.2", "rand_core 0.4.0", ] @@ -3066,6 +3506,17 @@ dependencies = [ "rand_core 0.5.0", ] +[[package]] +name = "raw-cpuid" +version = "7.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf" +dependencies = [ + "bitflags", + "cc", + "rustc_version", +] + [[package]] name = "rayon" version = "1.2.0" @@ -3147,7 +3598,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3162,7 +3613,7 @@ dependencies = [ "cookie_store", "encoding_rs", "flate2", - "futures", + "futures 0.1.29", "http", "hyper", "hyper-tls", @@ -3208,7 +3659,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3266,7 +3717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f6abf258d99c3c1c5c2131d99d064e94b7b3dd5f416483057f308fea253339" dependencies = [ "lazy_static", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3317,8 +3768,8 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfab8dda0e7a327c696d893df9ffa19cadc4bd195797997f5223cf5831beaf05" dependencies = [ - "core-foundation", - "core-foundation-sys", + "core-foundation 0.5.1", + "core-foundation-sys 0.5.1", "libc", "security-framework-sys", ] @@ -3330,7 +3781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6696852716b589dff9e886ff83778bb635150168e83afa8ac6b8a78cb82abc" dependencies = [ "MacTypes-sys", - "core-foundation-sys", + "core-foundation-sys 0.5.1", "libc", ] @@ -3616,7 +4067,7 @@ dependencies = [ "rand 0.7.0", "redox_syscall", "remove_dir_all", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3626,7 +4077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926e291c893eed8a1ca9c18e7159cc9008bde470bd9037ca37370e7616fa587b" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "igd", "libc", "log 0.4.8", @@ -3636,7 +4087,7 @@ dependencies = [ "tokio", "tokio-threadpool", "tokio-yamux", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3647,7 +4098,7 @@ checksum = "f639e9adcc3f3d89ec67b65799651bb0e34db476ebd0562caabd960cfcd920b7" dependencies = [ "bs58 0.3.0", "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", "molecule 0.4.0", "openssl", @@ -3705,7 +4156,7 @@ checksum = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" dependencies = [ "libc", "redox_syscall", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3717,6 +4168,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "threadpool" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2f0c90a5f3459330ac8bc0d2f879c693bb7a2f59689c1083fc4ef83834da865" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.1.42" @@ -3725,7 +4185,7 @@ checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" dependencies = [ "libc", "redox_syscall", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3754,7 +4214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "mio", "num_cpus", "tokio-codec", @@ -3779,7 +4239,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" dependencies = [ "bytes 0.4.12", "either", - "futures", + "futures 0.1.29", ] [[package]] @@ -3789,7 +4249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "tokio-io", ] @@ -3799,7 +4259,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443" dependencies = [ - "futures", + "futures 0.1.29", "tokio-executor", ] @@ -3810,7 +4270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27ee0e6db01c5f0b2973824547ce7e637b2ed79b891a9677b0de9bd532b6ac" dependencies = [ "crossbeam-utils", - "futures", + "futures 0.1.29", ] [[package]] @@ -3819,7 +4279,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fe6dc22b08d6993916647d108a1a7d15b9cd29c4f4496c62b92c45b5041b7af" dependencies = [ - "futures", + "futures 0.1.29", "tokio-io", "tokio-threadpool", ] @@ -3831,7 +4291,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", ] @@ -3842,7 +4302,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af16bfac7e112bea8b0442542161bfc41cbfa4466b580bdda7d18cb88b911ce" dependencies = [ "crossbeam-utils", - "futures", + "futures 0.1.29", "lazy_static", "log 0.4.8", "mio", @@ -3860,7 +4320,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" dependencies = [ - "futures", + "futures 0.1.29", ] [[package]] @@ -3870,7 +4330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2162248ff317e2bc713b261f242b69dbb838b85248ed20bb21df56d60ea4cae7" dependencies = [ "fnv", - "futures", + "futures 0.1.29", ] [[package]] @@ -3880,7 +4340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "iovec", "mio", "tokio-io", @@ -3896,7 +4356,7 @@ dependencies = [ "crossbeam-deque", "crossbeam-queue", "crossbeam-utils", - "futures", + "futures 0.1.29", "log 0.4.8", "num_cpus", "rand 0.6.5", @@ -3911,7 +4371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2910970404ba6fa78c5539126a9ae2045d62e3713041e447f695f41405a120c6" dependencies = [ "crossbeam-utils", - "futures", + "futures 0.1.29", "slab", "tokio-executor", ] @@ -3923,7 +4383,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66268575b80f4a4a710ef83d087fdfeeabdce9b74c797535fbac18a2cb906e92" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", "mio", "tokio-codec", @@ -3938,7 +4398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "iovec", "libc", "log 0.4.8", @@ -3957,7 +4417,7 @@ checksum = "723c77f5e71061a1363d8402418ac87be5570fe548a638a4eb231fa01d6124b6" dependencies = [ "byteorder", "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", "tokio", ] @@ -4079,6 +4539,17 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece" +[[package]] +name = "uom" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51fc04fb44bcb7806da71885872cb15d123b681e459a476ef8a0bab287bee0cd" +dependencies = [ + "num-rational", + "num-traits", + "typenum", +] + [[package]] name = "url" version = "1.7.2" @@ -4161,7 +4632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9658c94fa8b940eab2250bd5a457f9c48b748420d71293b165c8cdbe2f55f71e" dependencies = [ "same-file", - "winapi 0.3.7", + "winapi 0.3.8", "winapi-util", ] @@ -4171,7 +4642,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ - "futures", + "futures 0.1.29", "log 0.4.8", "try-lock", ] @@ -4267,6 +4738,12 @@ dependencies = [ "nom", ] +[[package]] +name = "widestring" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effc0e4ff8085673ea7b9b2e3c73f6bd4d118810c9009ed8f1e16bd96c331db6" + [[package]] name = "winapi" version = "0.2.8" @@ -4275,9 +4752,9 @@ checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -4301,7 +4778,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -4316,7 +4793,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", "winapi-util", ] @@ -4326,7 +4803,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f1f3c6c4d3cab118551b96c476a2caab920701e28875b64a458f2ecb96ec9d" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -4335,7 +4812,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 836ce28de1..57d1d6d08c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ # Members are ordered by dependencies. Crates at top has fewer dependencies. "util/build-info", "util/logger", + "util/memory-tracker", "util", "util/hash", "util/fixed-hash", @@ -69,9 +70,12 @@ members = [ [profile.release] overflow-checks = true -[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies] +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), not(feature = "profiling")))'.dependencies] jemallocator = { version = "0.3.0", features = ["unprefixed_malloc_on_supported_platforms"] } +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), feature = "profiling"))'.dependencies] +jemallocator = { version = "0.3.0", features = ["unprefixed_malloc_on_supported_platforms", "profiling"] } [features] default = [] deadlock_detection = ["ckb-bin/deadlock_detection"] +profiling = [] diff --git a/Makefile b/Makefile index da40919bef..47d15f7b45 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,10 @@ check: setup-ckb-test ## Runs all of the compiler's checks. build: ## Build binary with release profile. cargo build ${VERBOSE} --release +.PHONY: build-for-profiling +build-for-profiling: ## Build binary with for profiling. + JEMALLOC_SYS_WITH_MALLOC_CONF="prof:true" cargo build ${VERBOSE} --features "profiling" + .PHONY: prod prod: ## Build binary for production release. RUSTFLAGS="--cfg disable_faketime" cargo build ${VERBOSE} --release diff --git a/ckb-bin/Cargo.toml b/ckb-bin/Cargo.toml index 811b8dd234..480c300d40 100644 --- a/ckb-bin/Cargo.toml +++ b/ckb-bin/Cargo.toml @@ -29,9 +29,9 @@ ctrlc = { version = "3.1", features = ["termination"] } ckb-sync = { path = "../sync"} ckb-instrument = { path = "../util/instrument", features = ["progress_bar"] } ckb-build-info = { path = "../util/build-info" } +ckb-memory-tracker = { path = "../util/memory-tracker" } ckb-verification = { path = "../verification" } base64 = "0.10.1" - [features] deadlock_detection = ["ckb-util/deadlock_detection"] diff --git a/ckb-bin/src/subcommand/miner.rs b/ckb-bin/src/subcommand/miner.rs index e087322640..f2cc7d46fc 100644 --- a/ckb-bin/src/subcommand/miner.rs +++ b/ckb-bin/src/subcommand/miner.rs @@ -10,6 +10,8 @@ pub fn miner(args: MinerArgs) -> Result<(), ExitCode> { let mut client = Client::new(new_work_tx, client); let mut miner = Miner::new(args.pow_engine, client.clone(), new_work_rx, &workers); + ckb_memory_tracker::track_current_process(args.memory_tracker.interval); + thread::Builder::new() .name("client".to_string()) .spawn(move || client.poll_block_template()) diff --git a/ckb-bin/src/subcommand/run.rs b/ckb-bin/src/subcommand/run.rs index 80718f3150..f964c9c594 100644 --- a/ckb-bin/src/subcommand/run.rs +++ b/ckb-bin/src/subcommand/run.rs @@ -42,6 +42,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> { // Verify genesis every time starting node verify_genesis(&shared)?; + ckb_memory_tracker::track_current_process(args.config.memory_tracker.interval); + let chain_service = ChainService::new(shared.clone(), table); let chain_controller = chain_service.start(Some("ChainService")); info_target!(crate::LOG_TARGET_MAIN, "ckb version: {}", version); @@ -142,7 +144,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> { .enable_experiment(shared.clone()) .enable_integration_test(shared.clone(), network_controller.clone(), chain_controller) .enable_alert(alert_verifier, alert_notifier, network_controller) - .enable_indexer(&args.config.indexer, shared.clone()); + .enable_indexer(&args.config.indexer, shared.clone()) + .enable_debug(); let io_handler = builder.build(); let rpc_server = RpcServer::new(args.config.rpc, io_handler, shared.notify_controller()); diff --git a/devtools/ci/check-cargotoml.sh b/devtools/ci/check-cargotoml.sh index 3f4115449d..afd2976950 100755 --- a/devtools/ci/check-cargotoml.sh +++ b/devtools/ci/check-cargotoml.sh @@ -83,7 +83,7 @@ function search_crate() { | wc -l) depcnt=$((depcnt + tmpcnt)) tmpcnt=$({\ - ${GREP} ${grepopts} "[ (<]\(::\|\)${crate}::" "${source}" \ + ${GREP} ${grepopts} "\(^\|[ (<]\)\(::\|\)${crate}::" "${source}" \ || true; }\ | wc -l) depcnt=$((depcnt + tmpcnt)) diff --git a/docs/ckb-debugging.md b/docs/ckb-debugging.md new file mode 100644 index 0000000000..12532364eb --- /dev/null +++ b/docs/ckb-debugging.md @@ -0,0 +1,62 @@ +# CKB Debugging + +## Memory + +**Only linux versions supported.** + +### Tracking Memory Usage in Logs + +Add the follow configuration into `ckb.toml`: + +```toml +[logger] +filter = "error,ckb-memory-tracker=trace" + +[memory_tracker] +# Seconds between checking the process, 0 is disable, default is 0. +interval = 600 +``` + +### Memory Profiling + +- Compile `ckb` with feature `profiling`. + + ```sh + make build-for-profiling` + ``` + + After compiling, a script named `jeprof` will be generated in `target` direcotry. + + ```sh + find target/ -name "jeprof" + ``` + +- Enable RPC module `Debug` in `ckb.toml`. + + ```toml + [rpc] + modules = ["Debug"] + ``` + +- Run `ckb`. + +- Dump memory usage to a file via call RPC `jemalloc_profiling_dump`. + + ```sh + curl -H 'content-type: application/json' -d '{ "id": 2, "jsonrpc": "2.0", "method": "jemalloc_profiling_dump", "params": [] }' http://localhost:8114 + ``` + + Then, a file named `ckb-jeprof.$TIMESTAMP.heap` will be generated in the working directory of the running `ckb`. + +- Generate a PDF of the call graph. + + **Required**: graphviz and ghostscript + + ```sh + jeprof --show_bytes --pdf target/debug/ckb ckb-jeprof.$TIMESTAMP.heap > call-graph.pdf + ``` + +## References: + +- [JEMALLOC: Use Case: Leak Checking](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking) +- [JEMALLOC: Use Case: Heap Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling) diff --git a/resource/ckb-miner.toml b/resource/ckb-miner.toml index 8385346e37..53db5b9e9c 100644 --- a/resource/ckb-miner.toml +++ b/resource/ckb-miner.toml @@ -39,6 +39,10 @@ dsn = "" # {{ # please leave a way to contact you when we have troubles to reproduce the errors. # org_contact = "" +# [memory_tracker] +# # Seconds between checking the process, 0 is disable, default is 0. +# interval = 600 + [miner.client] rpc_url = "http://127.0.0.1:8114/" # {{ # _ => rpc_url = "http://127.0.0.1:{rpc_port}/" diff --git a/resource/ckb.toml b/resource/ckb.toml index 5d1f2a1f3c..af046a8f05 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -42,6 +42,10 @@ dsn = "" # {{ # please leave a way to contact you when we have troubles to reproduce the errors. # org_contact = "" +# [memory_tracker] +# # Seconds between checking the process, 0 is disable, default is 0. +# interval = 600 + [network] listen_addresses = ["/ip4/0.0.0.0/tcp/8115"] # {{ # _ => listen_addresses = ["/ip4/0.0.0.0/tcp/{p2p_port}"] @@ -91,7 +95,7 @@ listen_address = "127.0.0.1:8114" # {{ # Default is 10MiB = 10 * 1024 * 1024 max_request_body_size = 10485760 -# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment"] +# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment", "Debug"] modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment"] # {{ # integration => modules = ["Net", "Pool", "Miner", "Chain", "Experiment", "Stats", "Indexer", "IntegrationTest"] # }} diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 9b05737789..f4ef59e58e 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -41,6 +41,7 @@ ckb-error = { path = "../error" } ckb-reward-calculator = { path = "../util/reward-calculator" } ckb-tx-pool = { path = "../tx-pool" } ckb-script = { path = "../script" } +ckb-memory-tracker = { path = "../util/memory-tracker" } [dev-dependencies] reqwest = "0.9.16" diff --git a/rpc/src/config.rs b/rpc/src/config.rs index 5b4342a1c8..7844b9d08b 100644 --- a/rpc/src/config.rs +++ b/rpc/src/config.rs @@ -12,6 +12,7 @@ pub enum Module { IntegrationTest, Alert, Subscription, + Debug, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -69,4 +70,8 @@ impl Config { pub(crate) fn alert_enable(&self) -> bool { self.modules.contains(&Module::Alert) } + + pub(crate) fn debug_enable(&self) -> bool { + self.modules.contains(&Module::Debug) + } } diff --git a/rpc/src/module/debug.rs b/rpc/src/module/debug.rs new file mode 100644 index 0000000000..6f02f26ff4 --- /dev/null +++ b/rpc/src/module/debug.rs @@ -0,0 +1,23 @@ +use jsonrpc_core::Result; +use jsonrpc_derive::rpc; +use std::time; + +#[rpc(server)] +pub trait DebugRpc { + #[rpc(name = "jemalloc_profiling_dump")] + fn jemalloc_profiling_dump(&self) -> Result<()>; +} + +pub(crate) struct DebugRpcImpl {} + +impl DebugRpc for DebugRpcImpl { + fn jemalloc_profiling_dump(&self) -> Result<()> { + let timestamp = time::SystemTime::now() + .duration_since(time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let filename = format!("ckb-jeprof.{}.heap\0", timestamp); + ckb_memory_tracker::jemalloc_profiling_dump(filename); + Ok(()) + } +} diff --git a/rpc/src/module/mod.rs b/rpc/src/module/mod.rs index a7613a86de..e025e5570f 100644 --- a/rpc/src/module/mod.rs +++ b/rpc/src/module/mod.rs @@ -1,5 +1,6 @@ mod alert; mod chain; +mod debug; mod experiment; mod indexer; mod miner; @@ -11,6 +12,7 @@ mod test; pub(crate) use self::alert::{AlertRpc, AlertRpcImpl}; pub(crate) use self::chain::{ChainRpc, ChainRpcImpl}; +pub(crate) use self::debug::{DebugRpc, DebugRpcImpl}; pub(crate) use self::experiment::{ExperimentRpc, ExperimentRpcImpl}; pub(crate) use self::indexer::{IndexerRpc, IndexerRpcImpl}; pub(crate) use self::miner::{MinerRpc, MinerRpcImpl}; diff --git a/rpc/src/service_builder.rs b/rpc/src/service_builder.rs index b17f1446e2..2967bda08d 100644 --- a/rpc/src/service_builder.rs +++ b/rpc/src/service_builder.rs @@ -1,8 +1,9 @@ use crate::config::Config; use crate::module::{ - AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, ExperimentRpc, ExperimentRpcImpl, IndexerRpc, - IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, MinerRpc, MinerRpcImpl, NetworkRpc, - NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, StatsRpcImpl, + AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, DebugRpc, DebugRpcImpl, ExperimentRpc, + ExperimentRpcImpl, IndexerRpc, IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, + MinerRpc, MinerRpcImpl, NetworkRpc, NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, + StatsRpcImpl, }; use crate::IoHandler; use ckb_chain::chain::ChainController; @@ -172,6 +173,13 @@ impl<'a> ServiceBuilder<'a> { self } + pub fn enable_debug(mut self) -> Self { + if self.config.debug_enable() { + self.io_handler.extend_with(DebugRpcImpl {}.to_delegate()); + } + self + } + fn update_disabled_methods(&mut self, module: &str, rpc_method: I) where I: IntoIterator, diff --git a/util/app-config/Cargo.toml b/util/app-config/Cargo.toml index a1cb27764d..4ea8c4fc60 100644 --- a/util/app-config/Cargo.toml +++ b/util/app-config/Cargo.toml @@ -24,6 +24,7 @@ ckb-resource = { path = "../../resource"} ckb-store = { path = "../../store" } ckb-network-alert = { path = "../network-alert" } ckb-build-info = { path = "../build-info" } +ckb-memory-tracker = { path = "../memory-tracker" } ckb-indexer = { path = "../../indexer" } ckb-tx-pool = { path = "../../tx-pool" } ckb-notify = { path = "../../notify" } diff --git a/util/app-config/src/app_config.rs b/util/app-config/src/app_config.rs index b04ae7bdff..3841bcc995 100644 --- a/util/app-config/src/app_config.rs +++ b/util/app-config/src/app_config.rs @@ -14,6 +14,7 @@ use ckb_chain_spec::ChainSpec; use ckb_db::DBConfig; use ckb_indexer::IndexerConfig; use ckb_logger::Config as LogConfig; +use ckb_memory_tracker::Config as MemoryTrackerConfig; use ckb_miner::MinerConfig; use ckb_network::NetworkConfig; use ckb_network_alert::config::SignatureConfig as AlertSignatureConfig; @@ -37,6 +38,8 @@ pub struct CKBAppConfig { pub data_dir: PathBuf, pub logger: LogConfig, pub sentry: SentryConfig, + #[serde(default)] + pub memory_tracker: MemoryTrackerConfig, pub chain: ChainConfig, pub block_assembler: Option, @@ -61,6 +64,8 @@ pub struct MinerAppConfig { pub chain: ChainConfig, pub logger: LogConfig, pub sentry: SentryConfig, + #[serde(default)] + pub memory_tracker: MemoryTrackerConfig, pub miner: MinerConfig, } @@ -108,6 +113,13 @@ impl AppConfig { } } + pub fn memory_tracker(&self) -> &MemoryTrackerConfig { + match self { + AppConfig::CKB(config) => &config.memory_tracker, + AppConfig::Miner(config) => &config.memory_tracker, + } + } + pub fn chain_spec(&self) -> Result { let spec_resource = match self { AppConfig::CKB(config) => &config.chain.spec, diff --git a/util/app-config/src/args.rs b/util/app-config/src/args.rs index b00feb2e4b..7bb8870f6f 100644 --- a/util/app-config/src/args.rs +++ b/util/app-config/src/args.rs @@ -1,6 +1,7 @@ use super::app_config::CKBAppConfig; use ckb_chain_spec::consensus::Consensus; use ckb_jsonrpc_types::ScriptHashType; +use ckb_memory_tracker::Config as MemoryTrackerConfig; use ckb_miner::MinerConfig; use ckb_pow::PowEngine; use std::path::PathBuf; @@ -34,6 +35,7 @@ pub struct ProfArgs { pub struct MinerArgs { pub config: MinerConfig, pub pow_engine: Arc, + pub memory_tracker: MemoryTrackerConfig, } pub struct StatsArgs { diff --git a/util/app-config/src/lib.rs b/util/app-config/src/lib.rs index 4b3c574a63..e0e9fea9ea 100644 --- a/util/app-config/src/lib.rs +++ b/util/app-config/src/lib.rs @@ -107,12 +107,14 @@ impl Setup { pub fn miner(self) -> Result { let spec = self.chain_spec()?; + let memory_tracker = self.config.memory_tracker().to_owned(); let config = self.config.into_miner()?; let pow_engine = spec.pow_engine(); Ok(MinerArgs { pow_engine, config: config.miner, + memory_tracker, }) } diff --git a/util/memory-tracker/Cargo.toml b/util/memory-tracker/Cargo.toml new file mode 100644 index 0000000000..f5e384b60f --- /dev/null +++ b/util/memory-tracker/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "ckb-memory-tracker" +version = "0.30.1-pre" +authors = ["Nervos Core Dev "] +edition = "2018" +license = "MIT" + +[dependencies] +ckb-logger = { path = "../logger" } +serde = { version = "1.0", features = ["derive"] } + +# TODO Why don't disable this crate by "target.*" in the crates which are dependent on this crate? +# +# I really don't like to write such dirty code. I try a lot. But due to the limit of the "cargo" +# (and a lot of stupid bugs), at last, I have to write these stupid code. +# +# References: +# - [Cargo Issue-1197: Target-specific features](https://github.com/rust-lang/cargo/issues/1197) +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies] +heim = "0.0.10" +futures = "0.3.1" +jemalloc-ctl = "0.3.3" +jemalloc-sys = "0.3.2" diff --git a/util/memory-tracker/src/config.rs b/util/memory-tracker/src/config.rs new file mode 100644 index 0000000000..c3273373e4 --- /dev/null +++ b/util/memory-tracker/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Config { + pub interval: u64, +} + +impl Default for Config { + fn default() -> Self { + Self { interval: 0 } + } +} diff --git a/util/memory-tracker/src/jemalloc-mock.rs b/util/memory-tracker/src/jemalloc-mock.rs new file mode 100644 index 0000000000..60a88ae407 --- /dev/null +++ b/util/memory-tracker/src/jemalloc-mock.rs @@ -0,0 +1,5 @@ +use ckb_logger::warn; + +pub fn jemalloc_profiling_dump(_: String) { + warn!("jemalloc profiling dump: unsupported"); +} diff --git a/util/memory-tracker/src/jemalloc.rs b/util/memory-tracker/src/jemalloc.rs new file mode 100644 index 0000000000..250ce3e908 --- /dev/null +++ b/util/memory-tracker/src/jemalloc.rs @@ -0,0 +1,15 @@ +use std::{ffi, mem, ptr}; + +pub fn jemalloc_profiling_dump(mut filename: String) { + let opt_name = "prof.dump"; + let opt_c_name = ffi::CString::new(opt_name).unwrap(); + unsafe { + jemalloc_sys::mallctl( + opt_c_name.as_ptr(), + ptr::null_mut(), + ptr::null_mut(), + &mut filename as *mut _ as *mut _, + mem::size_of::<*mut ffi::c_void>(), + ); + } +} diff --git a/util/memory-tracker/src/lib.rs b/util/memory-tracker/src/lib.rs new file mode 100644 index 0000000000..313d4dbd53 --- /dev/null +++ b/util/memory-tracker/src/lib.rs @@ -0,0 +1,23 @@ +mod config; +#[cfg_attr( + all(not(target_env = "msvc"), not(target_os = "macos")), + path = "jemalloc.rs" +)] +#[cfg_attr( + not(all(not(target_env = "msvc"), not(target_os = "macos"))), + path = "jemalloc-mock.rs" +)] +mod jemalloc; +#[cfg_attr( + all(not(target_env = "msvc"), not(target_os = "macos")), + path = "process.rs" +)] +#[cfg_attr( + not(all(not(target_env = "msvc"), not(target_os = "macos"))), + path = "process-mock.rs" +)] +mod process; + +pub use config::Config; +pub use jemalloc::jemalloc_profiling_dump; +pub use process::track_current_process; diff --git a/util/memory-tracker/src/process-mock.rs b/util/memory-tracker/src/process-mock.rs new file mode 100644 index 0000000000..9f84c3158b --- /dev/null +++ b/util/memory-tracker/src/process-mock.rs @@ -0,0 +1,5 @@ +use ckb_logger::info; + +pub fn track_current_process(_: u64) { + info!("track current process: unsupported"); +} diff --git a/util/memory-tracker/src/process.rs b/util/memory-tracker/src/process.rs new file mode 100644 index 0000000000..8d73e09728 --- /dev/null +++ b/util/memory-tracker/src/process.rs @@ -0,0 +1,105 @@ +use ckb_logger::{error, info, trace}; +use futures::executor::block_on; +use heim::units::information::kibibyte; +use jemalloc_ctl::{epoch, stats}; +use std::{thread, time}; + +macro_rules! je_mib { + ($key:ty) => { + if let Ok(value) = <$key>::mib() { + value + } else { + error!("failed to lookup jemalloc mib for {}", stringify!($key)); + return; + } + }; +} + +macro_rules! mib_read { + ($mib:ident) => { + if let Ok(value) = $mib.read() { + value / 1024 + } else { + error!("failed to read jemalloc stats for {}", stringify!($mib)); + return; + } + }; +} + +pub fn track_current_process(interval: u64) { + if interval == 0 { + info!("track current process: disable"); + } else { + info!("track current process: enable"); + let wait_secs = time::Duration::from_secs(interval); + + let je_epoch = je_mib!(epoch); + // Bytes allocated by the application. + let allocated = je_mib!(stats::allocated); + // Bytes in physically resident data pages mapped by the allocator. + let resident = je_mib!(stats::resident); + // Bytes in active pages allocated by the application. + let active = je_mib!(stats::active); + // Bytes in active extents mapped by the allocator. + let mapped = je_mib!(stats::mapped); + // Bytes in virtual memory mappings that were retained + // rather than being returned to the operating system + let retained = je_mib!(stats::retained); + // Bytes dedicated to jemalloc metadata. + let metadata = je_mib!(stats::metadata); + + if let Err(err) = thread::Builder::new() + .name("MemoryTracker".to_string()) + .spawn(move || { + if let Ok(process) = block_on(heim::process::current()) { + let pid = process.pid(); + loop { + if je_epoch.advance().is_err() { + error!("failed to refresh the jemalloc stats"); + return; + } + if let Ok(memory) = block_on(process.memory()) { + // Resident set size, amount of non-swapped physical memory. + let rss = memory.rss().get::(); + // Virtual memory size, total amount of memory. + let virt = memory.vms().get::(); + + let allocated = mib_read!(allocated); + let resident = mib_read!(resident); + let active = mib_read!(active); + let mapped = mib_read!(mapped); + let retained = mib_read!(retained); + let metadata = mib_read!(metadata); + + trace!( + "CurrentProcess {{ pid: {}, rss: {} KiB, virt: {} KiB, \ + Jemalloc: {{ allocated: {} KiB, resident: {} KiB, \ + active: {} KiB, mapped: {} KiB, retained: {} KiB, \ + metadata: {} KiB }} }}", + pid, + rss, + virt, + allocated, + resident, + active, + mapped, + retained, + metadata + ); + } else { + error!("failed to fetch the memory information about current process"); + } + thread::sleep(wait_secs); + } + } else { + error!("failed to track the currently running program"); + } + }) + { + error!( + "failed to spawn the thread to track current process: {}", + err + ); + } + } +}