From 5c34321535f8a634a750345ef9ddb2703d741912 Mon Sep 17 00:00:00 2001 From: Boyu Yang Date: Fri, 7 Feb 2020 09:25:59 +0800 Subject: [PATCH 1/3] fix: fix a ci script which check redundant dependencies --- devtools/ci/check-cargotoml.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/ci/check-cargotoml.sh b/devtools/ci/check-cargotoml.sh index 8e5417123c..31e4a9b06b 100755 --- a/devtools/ci/check-cargotoml.sh +++ b/devtools/ci/check-cargotoml.sh @@ -83,7 +83,7 @@ function search_crate() { | wc -l) depcnt=$((depcnt + tmpcnt)) tmpcnt=$({\ - ${GREP} ${grepopts} "[ (<]\(::\|\)${crate}::" "${source}" \ + ${GREP} ${grepopts} "\(^\|[ (<]\)\(::\|\)${crate}::" "${source}" \ || true; }\ | wc -l) depcnt=$((depcnt + tmpcnt)) From 6019bc021986e47e6ba7228fb7e31aff1391f0a2 Mon Sep 17 00:00:00 2001 From: Boyu Yang Date: Thu, 13 Feb 2020 01:28:37 +0800 Subject: [PATCH 2/3] feat: run a memory tracker thread to record the memory statistics --- Cargo.lock | 660 +++++++++++++++++++++--- ckb-bin/Cargo.toml | 2 +- ckb-bin/src/subcommand/miner.rs | 2 + ckb-bin/src/subcommand/run.rs | 2 + resource/ckb-miner.toml | 4 + resource/ckb.toml | 4 + util/app-config/Cargo.toml | 1 + util/app-config/src/app_config.rs | 12 + util/app-config/src/args.rs | 2 + util/app-config/src/lib.rs | 2 + util/memory-tracker/Cargo.toml | 22 + util/memory-tracker/src/config.rs | 12 + util/memory-tracker/src/lib.rs | 13 + util/memory-tracker/src/process-mock.rs | 5 + util/memory-tracker/src/process.rs | 105 ++++ 15 files changed, 764 insertions(+), 84 deletions(-) create mode 100644 util/memory-tracker/Cargo.toml create mode 100644 util/memory-tracker/src/config.rs create mode 100644 util/memory-tracker/src/lib.rs create mode 100644 util/memory-tracker/src/process-mock.rs create mode 100644 util/memory-tracker/src/process.rs diff --git a/Cargo.lock b/Cargo.lock index 08e33b47ca..eef00c33e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,7 +39,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -48,7 +48,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -91,7 +91,7 @@ checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" dependencies = [ "libc", "termion", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -100,6 +100,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + [[package]] name = "backtrace" version = "0.3.40" @@ -325,6 +331,7 @@ dependencies = [ "ckb-indexer", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-miner", "ckb-network", "ckb-network-alert", @@ -379,6 +386,7 @@ dependencies = [ "ckb-instrument", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-miner", "ckb-network", "ckb-network-alert", @@ -500,7 +508,7 @@ version = "0.28.1-pre" dependencies = [ "ckb-logger", "ckb-types", - "futures", + "futures 0.1.29", "rand 0.6.5", "tentacle", "tokio", @@ -558,7 +566,7 @@ name = "ckb-future-executor" version = "0.28.1-pre" dependencies = [ "crossbeam-channel", - "futures", + "futures 0.1.29", "num_cpus", "tokio", "tokio-executor", @@ -642,6 +650,17 @@ dependencies = [ "serde", ] +[[package]] +name = "ckb-memory-tracker" +version = "0.28.1-pre" +dependencies = [ + "ckb-logger", + "futures 0.3.4", + "heim", + "jemalloc-ctl", + "serde", +] + [[package]] name = "ckb-miner" version = "0.28.1-pre" @@ -655,7 +674,7 @@ dependencies = [ "crossbeam-channel", "eaglesong", "failure", - "futures", + "futures 0.1.29", "hyper", "indicatif", "lru-cache", @@ -692,7 +711,7 @@ dependencies = [ "crossbeam-channel", "faketime", "faster-hex 0.4.1", - "futures", + "futures 0.1.29", "ipnetwork", "lazy_static", "num_cpus", @@ -773,7 +792,7 @@ version = "0.28.1-pre" dependencies = [ "ckb-logger", "ckb-types", - "futures", + "futures 0.1.29", "tentacle", ] @@ -862,7 +881,7 @@ dependencies = [ "ckb-verification", "crossbeam-channel", "faketime", - "futures", + "futures 0.1.29", "jsonrpc-core", "jsonrpc-derive", "jsonrpc-http-server", @@ -953,7 +972,7 @@ version = "0.28.1-pre" dependencies = [ "ckb-logger", "crossbeam-channel", - "futures", + "futures 0.1.29", "parking_lot 0.7.1", ] @@ -993,7 +1012,7 @@ dependencies = [ "ckb-verification", "failure", "faketime", - "futures", + "futures 0.1.29", "lru-cache", "rand 0.6.5", "sentry", @@ -1056,7 +1075,7 @@ dependencies = [ "crossbeam-channel", "failure", "faketime", - "futures", + "futures 0.1.29", "lru-cache", "serde", "tokio", @@ -1114,7 +1133,7 @@ dependencies = [ "enum-display-derive", "failure", "faketime", - "futures", + "futures 0.1.29", "lru-cache", "rand 0.6.5", "rayon", @@ -1167,7 +1186,7 @@ dependencies = [ "atty", "lazy_static", "libc", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -1192,7 +1211,7 @@ dependencies = [ "regex", "termios", "unicode-width", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -1229,7 +1248,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "286e0b41c3a20da26536c6000a280585d519fd07b3956b43aed8a79e9edce980" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.5.1", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", "libc", ] @@ -1242,6 +1271,12 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + [[package]] name = "crc" version = "1.8.1" @@ -1401,8 +1436,28 @@ version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7dfd2d8b4c82121dfdff120f818e09fc4380b0b7e17a742081a89b94853e87f" dependencies = [ - "nix", - "winapi 0.3.7", + "nix 0.14.1", + "winapi 0.3.8", +] + +[[package]] +name = "darwin-libproc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade5a88af8d9646bf770687321a9488a0f2b4610aa08b0373016cd1af37f0a31" +dependencies = [ + "darwin-libproc-sys", + "libc", + "memchr", +] + +[[package]] +name = "darwin-libproc-sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30d1a078d74da1183b02fed8a8b07afc412d3998334b53b750d0ed03b031541" +dependencies = [ + "libc", ] [[package]] @@ -1636,16 +1691,107 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" +[[package]] +name = "futures" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" + [[package]] name = "futures-cpupool" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.29", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" + +[[package]] +name = "futures-macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +dependencies = [ + "proc-macro-hack", + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", +] + +[[package]] +name = "futures-sink" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" + +[[package]] +name = "futures-task" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" + +[[package]] +name = "futures-util" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + [[package]] name = "generic-array" version = "0.12.3" @@ -1715,7 +1861,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "fnv", - "futures", + "futures 0.1.29", "http", "indexmap", "log 0.4.8", @@ -1730,7 +1876,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -1742,6 +1888,199 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heim" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f11cfed41a4703f8f56ccbe411073c52bd3996d92e3ccac90d36bd0e86e0eb" +dependencies = [ + "heim-common", + "heim-cpu", + "heim-derive", + "heim-disk", + "heim-host", + "heim-memory", + "heim-net", + "heim-process", + "heim-runtime", + "heim-sensors", + "heim-virt", +] + +[[package]] +name = "heim-common" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97d185137ab6437750cb22a115d4c64027620d48c57893d31720abcba42c388" +dependencies = [ + "cfg-if", + "core-foundation 0.7.0", + "futures-core", + "futures-util", + "lazy_static", + "libc", + "mach", + "nix 0.16.1", + "pin-utils", + "uom", + "winapi 0.3.8", +] + +[[package]] +name = "heim-cpu" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328091d34621a28af550523e1c139749fa4bc4590a8ea91b160a0f4b82dd8b06" +dependencies = [ + "cfg-if", + "heim-common", + "heim-derive", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "winapi 0.3.8", +] + +[[package]] +name = "heim-derive" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc97d9bf8ca9af8a02a533acea548756f23d4759474a1df8e79eb2b57f3cf7ac" +dependencies = [ + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", +] + +[[package]] +name = "heim-disk" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82179359c7b215ae46fe8ac0f9015b8e5dfd6368cd0d113f261b28cb8993713f" +dependencies = [ + "bitflags", + "cfg-if", + "core-foundation 0.7.0", + "heim-common", + "heim-derive", + "heim-runtime", + "libc", + "mach", + "widestring", + "winapi 0.3.8", +] + +[[package]] +name = "heim-host" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64b9c8c5f7d1479102a503e3f233241bfb819bf916f02b15e10ba5fbfee7a7f3" +dependencies = [ + "cfg-if", + "heim-common", + "heim-derive", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "platforms", + "winapi 0.3.8", +] + +[[package]] +name = "heim-memory" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d04e9e068b57049062622cf4a182a45f320459e266c39dfc52ca65fc8c178e3a" +dependencies = [ + "cfg-if", + "heim-common", + "heim-derive", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "winapi 0.3.8", +] + +[[package]] +name = "heim-net" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6323430ecae9b74378b37d0078768989c98b570daacec1d899737d90d72a5b" +dependencies = [ + "bitflags", + "cfg-if", + "heim-common", + "heim-derive", + "heim-runtime", + "hex", + "libc", + "macaddr", + "nix 0.16.1", +] + +[[package]] +name = "heim-process" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b20a58b335dc9837088b2c9290822b55c661071cb542f0002e62ed18e8464379" +dependencies = [ + "cfg-if", + "darwin-libproc", + "heim-common", + "heim-cpu", + "heim-derive", + "heim-host", + "heim-net", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "memchr", + "ntapi", + "ordered-float", + "winapi 0.3.8", +] + +[[package]] +name = "heim-runtime" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87591c7b202868445a7da04b3d8f987b0672dd68c3d66e24fe8f759171dd612" +dependencies = [ + "cfg-if", + "futures-channel", + "heim-common", + "lazy_static", + "threadpool", +] + +[[package]] +name = "heim-sensors" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f663425e6ed38a6356ee5751a2ebae421f7449fd0507b7e698c7e1e9fabee0c" +dependencies = [ + "cfg-if", + "heim-common", + "heim-derive", + "heim-runtime", +] + +[[package]] +name = "heim-virt" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd975e9ca42170c0f65102b867e69c3d060027f9bb02fb29cc554f7a0e5680b" +dependencies = [ + "cfg-if", + "heim-common", + "heim-runtime", + "raw-cpuid", +] + [[package]] name = "hermit-abi" version = "0.1.3" @@ -1751,6 +2090,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76cdda6bf525062a0c9e8f14ee2b37935c86b8efb6c8b69b3c83dfb518a914af" + [[package]] name = "hostname" version = "0.1.5" @@ -1779,7 +2124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "http", "tokio-buf", ] @@ -1812,7 +2157,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "futures-cpupool", "h2", "http", @@ -1842,7 +2187,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "hyper", "native-tls", "tokio-io", @@ -1970,11 +2315,22 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +[[package]] +name = "jemalloc-ctl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c502a5ff9dd2924f1ed32ba96e3b65735d837b4bfd978d3161b1702e66aca4b7" +dependencies = [ + "jemalloc-sys", + "libc", + "paste", +] + [[package]] name = "jemalloc-sys" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bef0d4ce37578dfd80b466e3d8324bd9de788e249f1accebb0c472ea4b52bdc" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" dependencies = [ "cc", "fs_extra", @@ -2006,7 +2362,7 @@ version = "14.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe3b688648f1ef5d5072229e2d672ecb92cbff7d1c79bcf3fd5898f3f3df0970" dependencies = [ - "futures", + "futures 0.1.29", "log 0.4.8", "serde", "serde_derive", @@ -2199,6 +2555,21 @@ dependencies = [ "linked-hash-map 0.5.2", ] +[[package]] +name = "macaddr" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee538cb1031f87f970ba28f0e5ebfcdaf63ed1a000a4176b4117537c33d19fb" + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matches" version = "0.1.8" @@ -2221,7 +2592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" dependencies = [ "libc", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2391,7 +2762,7 @@ checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" dependencies = [ "cfg-if", "libc", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2407,6 +2778,19 @@ dependencies = [ "void", ] +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", + "void", +] + [[package]] name = "nodrop" version = "0.1.13" @@ -2423,20 +2807,44 @@ dependencies = [ "version_check", ] +[[package]] +name = "ntapi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26e041cd983acbc087e30fcba770380cfa352d0e392e175b2344ebaf7ea0602" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "num-integer" -version = "0.1.39" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" dependencies = [ + "autocfg 1.0.0", + "num-integer", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.6" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +dependencies = [ + "autocfg 1.0.0", +] [[package]] name = "num_cpus" @@ -2537,13 +2945,22 @@ version = "0.9.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c977d08e1312e2f7e4b86f9ebaa0ed3b19d1daff75fae88bbb88108afbd801fc" dependencies = [ - "autocfg", + "autocfg 0.1.2", "cc", "libc", "pkg-config", "vcpkg", ] +[[package]] +name = "ordered-float" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18869315e81473c951eb56ad5558bbc56978562d3ecfb87abb7a1e944cea4518" +dependencies = [ + "num-traits", +] + [[package]] name = "ordermap" version = "0.3.5" @@ -2556,7 +2973,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2634,7 +3051,7 @@ dependencies = [ "rustc_version", "smallvec", "thread-id", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2649,7 +3066,29 @@ dependencies = [ "redox_syscall", "rustc_version", "smallvec", - "winapi 0.3.7", + "winapi 0.3.8", +] + +[[package]] +name = "paste" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "423a519e1c6e828f1e73b720f9d9ed2fa643dce8a7737fb43235ce0b41eeaa49" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4214c9e912ef61bf42b81ba9a47e8aad1b2ffaf739ab162bf96d1e011f54e6c5" +dependencies = [ + "proc-macro-hack", + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", ] [[package]] @@ -2719,6 +3158,12 @@ dependencies = [ "unicase 1.4.2", ] +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" + [[package]] name = "pkg-config" version = "0.3.14" @@ -2731,6 +3176,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "platforms" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb3b2b1033b8a60b4da6ee470325f887758c95d5320f52f9ce0df055a55940e" + [[package]] name = "ppv-lite86" version = "0.2.5" @@ -2769,6 +3220,12 @@ dependencies = [ "syn 1.0.14", ] +[[package]] +name = "proc-macro-nested" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" + [[package]] name = "proc-macro2" version = "0.4.27" @@ -2870,7 +3327,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2883,7 +3340,7 @@ dependencies = [ "fuchsia-cprng", "libc", "rand_core 0.3.1", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2892,7 +3349,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ - "autocfg", + "autocfg 0.1.2", "libc", "rand_chacha 0.1.1", "rand_core 0.4.0", @@ -2902,7 +3359,7 @@ dependencies = [ "rand_os 0.1.3", "rand_pcg", "rand_xorshift", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -2924,7 +3381,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" dependencies = [ - "autocfg", + "autocfg 0.1.2", "rand_core 0.3.1", ] @@ -2934,7 +3391,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e193067942ef6f485a349a113329140d0ab9e2168ce92274499bb0e9a4190d9d" dependencies = [ - "autocfg", + "autocfg 0.1.2", "c2-chacha", "rand_core 0.5.0", ] @@ -2998,7 +3455,7 @@ checksum = "7b9ea758282efe12823e0d952ddb269d2e1897227e464919a554f2a03ef1b832" dependencies = [ "libc", "rand_core 0.4.0", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3012,7 +3469,7 @@ dependencies = [ "libc", "rand_core 0.4.0", "rdrand", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3031,7 +3488,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "autocfg", + "autocfg 0.1.2", "rand_core 0.4.0", ] @@ -3053,6 +3510,17 @@ dependencies = [ "rand_core 0.5.0", ] +[[package]] +name = "raw-cpuid" +version = "7.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf" +dependencies = [ + "bitflags", + "cc", + "rustc_version", +] + [[package]] name = "rayon" version = "1.2.0" @@ -3138,7 +3606,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3153,7 +3621,7 @@ dependencies = [ "cookie_store", "encoding_rs", "flate2", - "futures", + "futures 0.1.29", "http", "hyper", "hyper-tls", @@ -3199,7 +3667,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3266,7 +3734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f6abf258d99c3c1c5c2131d99d064e94b7b3dd5f416483057f308fea253339" dependencies = [ "lazy_static", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3317,8 +3785,8 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfab8dda0e7a327c696d893df9ffa19cadc4bd195797997f5223cf5831beaf05" dependencies = [ - "core-foundation", - "core-foundation-sys", + "core-foundation 0.5.1", + "core-foundation-sys 0.5.1", "libc", "security-framework-sys", ] @@ -3330,7 +3798,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6696852716b589dff9e886ff83778bb635150168e83afa8ac6b8a78cb82abc" dependencies = [ "MacTypes-sys", - "core-foundation-sys", + "core-foundation-sys 0.5.1", "libc", ] @@ -3616,7 +4084,7 @@ dependencies = [ "rand 0.7.0", "redox_syscall", "remove_dir_all", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3626,7 +4094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926e291c893eed8a1ca9c18e7159cc9008bde470bd9037ca37370e7616fa587b" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "igd", "libc", "log 0.4.8", @@ -3636,7 +4104,7 @@ dependencies = [ "tokio", "tokio-threadpool", "tokio-yamux", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3647,7 +4115,7 @@ checksum = "f639e9adcc3f3d89ec67b65799651bb0e34db476ebd0562caabd960cfcd920b7" dependencies = [ "bs58 0.3.0", "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", "molecule 0.4.0", "openssl", @@ -3705,7 +4173,7 @@ checksum = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" dependencies = [ "libc", "redox_syscall", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3717,6 +4185,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "threadpool" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2f0c90a5f3459330ac8bc0d2f879c693bb7a2f59689c1083fc4ef83834da865" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.1.42" @@ -3725,7 +4202,7 @@ checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" dependencies = [ "libc", "redox_syscall", - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -3754,7 +4231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "mio", "num_cpus", "tokio-codec", @@ -3779,7 +4256,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" dependencies = [ "bytes 0.4.12", "either", - "futures", + "futures 0.1.29", ] [[package]] @@ -3789,7 +4266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "tokio-io", ] @@ -3799,7 +4276,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443" dependencies = [ - "futures", + "futures 0.1.29", "tokio-executor", ] @@ -3810,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27ee0e6db01c5f0b2973824547ce7e637b2ed79b891a9677b0de9bd532b6ac" dependencies = [ "crossbeam-utils", - "futures", + "futures 0.1.29", ] [[package]] @@ -3819,7 +4296,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fe6dc22b08d6993916647d108a1a7d15b9cd29c4f4496c62b92c45b5041b7af" dependencies = [ - "futures", + "futures 0.1.29", "tokio-io", "tokio-threadpool", ] @@ -3831,7 +4308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", ] @@ -3842,7 +4319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af16bfac7e112bea8b0442542161bfc41cbfa4466b580bdda7d18cb88b911ce" dependencies = [ "crossbeam-utils", - "futures", + "futures 0.1.29", "lazy_static", "log 0.4.8", "mio", @@ -3860,7 +4337,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" dependencies = [ - "futures", + "futures 0.1.29", ] [[package]] @@ -3870,7 +4347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2162248ff317e2bc713b261f242b69dbb838b85248ed20bb21df56d60ea4cae7" dependencies = [ "fnv", - "futures", + "futures 0.1.29", ] [[package]] @@ -3880,7 +4357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "iovec", "mio", "tokio-io", @@ -3896,7 +4373,7 @@ dependencies = [ "crossbeam-deque", "crossbeam-queue", "crossbeam-utils", - "futures", + "futures 0.1.29", "log 0.4.8", "num_cpus", "rand 0.6.5", @@ -3911,7 +4388,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2910970404ba6fa78c5539126a9ae2045d62e3713041e447f695f41405a120c6" dependencies = [ "crossbeam-utils", - "futures", + "futures 0.1.29", "slab", "tokio-executor", ] @@ -3923,7 +4400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66268575b80f4a4a710ef83d087fdfeeabdce9b74c797535fbac18a2cb906e92" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", "mio", "tokio-codec", @@ -3938,7 +4415,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.29", "iovec", "libc", "log 0.4.8", @@ -3957,7 +4434,7 @@ checksum = "723c77f5e71061a1363d8402418ac87be5570fe548a638a4eb231fa01d6124b6" dependencies = [ "byteorder", "bytes 0.4.12", - "futures", + "futures 0.1.29", "log 0.4.8", "tokio", ] @@ -4085,6 +4562,17 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece" +[[package]] +name = "uom" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec796ec5f7ac557631709079168286056205c51c60aac33f51764bdc7b8dc4" +dependencies = [ + "num-rational", + "num-traits", + "typenum", +] + [[package]] name = "url" version = "1.7.2" @@ -4173,7 +4661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9658c94fa8b940eab2250bd5a457f9c48b748420d71293b165c8cdbe2f55f71e" dependencies = [ "same-file", - "winapi 0.3.7", + "winapi 0.3.8", "winapi-util", ] @@ -4183,7 +4671,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ - "futures", + "futures 0.1.29", "log 0.4.8", "try-lock", ] @@ -4279,6 +4767,12 @@ dependencies = [ "nom", ] +[[package]] +name = "widestring" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effc0e4ff8085673ea7b9b2e3c73f6bd4d118810c9009ed8f1e16bd96c331db6" + [[package]] name = "winapi" version = "0.2.8" @@ -4287,9 +4781,9 @@ checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -4313,7 +4807,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -4328,7 +4822,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", "winapi-util", ] @@ -4338,7 +4832,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f1f3c6c4d3cab118551b96c476a2caab920701e28875b64a458f2ecb96ec9d" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] @@ -4347,7 +4841,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e" dependencies = [ - "winapi 0.3.7", + "winapi 0.3.8", ] [[package]] diff --git a/ckb-bin/Cargo.toml b/ckb-bin/Cargo.toml index 52e736cc17..d5a5d92955 100644 --- a/ckb-bin/Cargo.toml +++ b/ckb-bin/Cargo.toml @@ -29,9 +29,9 @@ ctrlc = { version = "3.1", features = ["termination"] } ckb-sync = { path = "../sync"} ckb-instrument = { path = "../util/instrument", features = ["progress_bar"] } ckb-build-info = { path = "../util/build-info" } +ckb-memory-tracker = { path = "../util/memory-tracker" } ckb-verification = { path = "../verification" } base64 = "0.10.1" - [features] deadlock_detection = ["ckb-util/deadlock_detection"] diff --git a/ckb-bin/src/subcommand/miner.rs b/ckb-bin/src/subcommand/miner.rs index e087322640..f2cc7d46fc 100644 --- a/ckb-bin/src/subcommand/miner.rs +++ b/ckb-bin/src/subcommand/miner.rs @@ -10,6 +10,8 @@ pub fn miner(args: MinerArgs) -> Result<(), ExitCode> { let mut client = Client::new(new_work_tx, client); let mut miner = Miner::new(args.pow_engine, client.clone(), new_work_rx, &workers); + ckb_memory_tracker::track_current_process(args.memory_tracker.interval); + thread::Builder::new() .name("client".to_string()) .spawn(move || client.poll_block_template()) diff --git a/ckb-bin/src/subcommand/run.rs b/ckb-bin/src/subcommand/run.rs index 80718f3150..4e156504c3 100644 --- a/ckb-bin/src/subcommand/run.rs +++ b/ckb-bin/src/subcommand/run.rs @@ -42,6 +42,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> { // Verify genesis every time starting node verify_genesis(&shared)?; + ckb_memory_tracker::track_current_process(args.config.memory_tracker.interval); + let chain_service = ChainService::new(shared.clone(), table); let chain_controller = chain_service.start(Some("ChainService")); info_target!(crate::LOG_TARGET_MAIN, "ckb version: {}", version); diff --git a/resource/ckb-miner.toml b/resource/ckb-miner.toml index 8385346e37..53db5b9e9c 100644 --- a/resource/ckb-miner.toml +++ b/resource/ckb-miner.toml @@ -39,6 +39,10 @@ dsn = "" # {{ # please leave a way to contact you when we have troubles to reproduce the errors. # org_contact = "" +# [memory_tracker] +# # Seconds between checking the process, 0 is disable, default is 0. +# interval = 600 + [miner.client] rpc_url = "http://127.0.0.1:8114/" # {{ # _ => rpc_url = "http://127.0.0.1:{rpc_port}/" diff --git a/resource/ckb.toml b/resource/ckb.toml index 5d1f2a1f3c..88582a779f 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -42,6 +42,10 @@ dsn = "" # {{ # please leave a way to contact you when we have troubles to reproduce the errors. # org_contact = "" +# [memory_tracker] +# # Seconds between checking the process, 0 is disable, default is 0. +# interval = 600 + [network] listen_addresses = ["/ip4/0.0.0.0/tcp/8115"] # {{ # _ => listen_addresses = ["/ip4/0.0.0.0/tcp/{p2p_port}"] diff --git a/util/app-config/Cargo.toml b/util/app-config/Cargo.toml index e150e328cf..139bcfafe0 100644 --- a/util/app-config/Cargo.toml +++ b/util/app-config/Cargo.toml @@ -24,6 +24,7 @@ ckb-resource = { path = "../../resource"} ckb-store = { path = "../../store" } ckb-network-alert = { path = "../network-alert" } ckb-build-info = { path = "../build-info" } +ckb-memory-tracker = { path = "../memory-tracker" } ckb-indexer = { path = "../../indexer" } ckb-tx-pool = { path = "../../tx-pool" } ckb-notify = { path = "../../notify" } diff --git a/util/app-config/src/app_config.rs b/util/app-config/src/app_config.rs index b04ae7bdff..3841bcc995 100644 --- a/util/app-config/src/app_config.rs +++ b/util/app-config/src/app_config.rs @@ -14,6 +14,7 @@ use ckb_chain_spec::ChainSpec; use ckb_db::DBConfig; use ckb_indexer::IndexerConfig; use ckb_logger::Config as LogConfig; +use ckb_memory_tracker::Config as MemoryTrackerConfig; use ckb_miner::MinerConfig; use ckb_network::NetworkConfig; use ckb_network_alert::config::SignatureConfig as AlertSignatureConfig; @@ -37,6 +38,8 @@ pub struct CKBAppConfig { pub data_dir: PathBuf, pub logger: LogConfig, pub sentry: SentryConfig, + #[serde(default)] + pub memory_tracker: MemoryTrackerConfig, pub chain: ChainConfig, pub block_assembler: Option, @@ -61,6 +64,8 @@ pub struct MinerAppConfig { pub chain: ChainConfig, pub logger: LogConfig, pub sentry: SentryConfig, + #[serde(default)] + pub memory_tracker: MemoryTrackerConfig, pub miner: MinerConfig, } @@ -108,6 +113,13 @@ impl AppConfig { } } + pub fn memory_tracker(&self) -> &MemoryTrackerConfig { + match self { + AppConfig::CKB(config) => &config.memory_tracker, + AppConfig::Miner(config) => &config.memory_tracker, + } + } + pub fn chain_spec(&self) -> Result { let spec_resource = match self { AppConfig::CKB(config) => &config.chain.spec, diff --git a/util/app-config/src/args.rs b/util/app-config/src/args.rs index b00feb2e4b..7bb8870f6f 100644 --- a/util/app-config/src/args.rs +++ b/util/app-config/src/args.rs @@ -1,6 +1,7 @@ use super::app_config::CKBAppConfig; use ckb_chain_spec::consensus::Consensus; use ckb_jsonrpc_types::ScriptHashType; +use ckb_memory_tracker::Config as MemoryTrackerConfig; use ckb_miner::MinerConfig; use ckb_pow::PowEngine; use std::path::PathBuf; @@ -34,6 +35,7 @@ pub struct ProfArgs { pub struct MinerArgs { pub config: MinerConfig, pub pow_engine: Arc, + pub memory_tracker: MemoryTrackerConfig, } pub struct StatsArgs { diff --git a/util/app-config/src/lib.rs b/util/app-config/src/lib.rs index 4b3c574a63..e0e9fea9ea 100644 --- a/util/app-config/src/lib.rs +++ b/util/app-config/src/lib.rs @@ -107,12 +107,14 @@ impl Setup { pub fn miner(self) -> Result { let spec = self.chain_spec()?; + let memory_tracker = self.config.memory_tracker().to_owned(); let config = self.config.into_miner()?; let pow_engine = spec.pow_engine(); Ok(MinerArgs { pow_engine, config: config.miner, + memory_tracker, }) } diff --git a/util/memory-tracker/Cargo.toml b/util/memory-tracker/Cargo.toml new file mode 100644 index 0000000000..60b6e3ad9e --- /dev/null +++ b/util/memory-tracker/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "ckb-memory-tracker" +version = "0.28.1-pre" +authors = ["Nervos Core Dev "] +edition = "2018" +license = "MIT" + +[dependencies] +ckb-logger = { path = "../logger" } +serde = { version = "1.0", features = ["derive"] } + +# TODO Why don't disable this crate by "target.*" in the crates which are dependent on this crate? +# +# I really don't like to write such dirty code. I try a lot. But due to the limit of the "cargo" +# (and a lot of stupid bugs), at last, I have to write these stupid code. +# +# References: +# - [Cargo Issue-1197: Target-specific features](https://github.com/rust-lang/cargo/issues/1197) +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies] +heim = "0.0.9" +futures = "0.3.1" +jemalloc-ctl = "0.3.3" diff --git a/util/memory-tracker/src/config.rs b/util/memory-tracker/src/config.rs new file mode 100644 index 0000000000..c3273373e4 --- /dev/null +++ b/util/memory-tracker/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Config { + pub interval: u64, +} + +impl Default for Config { + fn default() -> Self { + Self { interval: 0 } + } +} diff --git a/util/memory-tracker/src/lib.rs b/util/memory-tracker/src/lib.rs new file mode 100644 index 0000000000..df3fb00997 --- /dev/null +++ b/util/memory-tracker/src/lib.rs @@ -0,0 +1,13 @@ +mod config; +#[cfg_attr( + all(not(target_env = "msvc"), not(target_os = "macos")), + path = "process.rs" +)] +#[cfg_attr( + not(all(not(target_env = "msvc"), not(target_os = "macos"))), + path = "process-mock.rs" +)] +mod process; + +pub use config::Config; +pub use process::track_current_process; diff --git a/util/memory-tracker/src/process-mock.rs b/util/memory-tracker/src/process-mock.rs new file mode 100644 index 0000000000..9f84c3158b --- /dev/null +++ b/util/memory-tracker/src/process-mock.rs @@ -0,0 +1,5 @@ +use ckb_logger::info; + +pub fn track_current_process(_: u64) { + info!("track current process: unsupported"); +} diff --git a/util/memory-tracker/src/process.rs b/util/memory-tracker/src/process.rs new file mode 100644 index 0000000000..8d73e09728 --- /dev/null +++ b/util/memory-tracker/src/process.rs @@ -0,0 +1,105 @@ +use ckb_logger::{error, info, trace}; +use futures::executor::block_on; +use heim::units::information::kibibyte; +use jemalloc_ctl::{epoch, stats}; +use std::{thread, time}; + +macro_rules! je_mib { + ($key:ty) => { + if let Ok(value) = <$key>::mib() { + value + } else { + error!("failed to lookup jemalloc mib for {}", stringify!($key)); + return; + } + }; +} + +macro_rules! mib_read { + ($mib:ident) => { + if let Ok(value) = $mib.read() { + value / 1024 + } else { + error!("failed to read jemalloc stats for {}", stringify!($mib)); + return; + } + }; +} + +pub fn track_current_process(interval: u64) { + if interval == 0 { + info!("track current process: disable"); + } else { + info!("track current process: enable"); + let wait_secs = time::Duration::from_secs(interval); + + let je_epoch = je_mib!(epoch); + // Bytes allocated by the application. + let allocated = je_mib!(stats::allocated); + // Bytes in physically resident data pages mapped by the allocator. + let resident = je_mib!(stats::resident); + // Bytes in active pages allocated by the application. + let active = je_mib!(stats::active); + // Bytes in active extents mapped by the allocator. + let mapped = je_mib!(stats::mapped); + // Bytes in virtual memory mappings that were retained + // rather than being returned to the operating system + let retained = je_mib!(stats::retained); + // Bytes dedicated to jemalloc metadata. + let metadata = je_mib!(stats::metadata); + + if let Err(err) = thread::Builder::new() + .name("MemoryTracker".to_string()) + .spawn(move || { + if let Ok(process) = block_on(heim::process::current()) { + let pid = process.pid(); + loop { + if je_epoch.advance().is_err() { + error!("failed to refresh the jemalloc stats"); + return; + } + if let Ok(memory) = block_on(process.memory()) { + // Resident set size, amount of non-swapped physical memory. + let rss = memory.rss().get::(); + // Virtual memory size, total amount of memory. + let virt = memory.vms().get::(); + + let allocated = mib_read!(allocated); + let resident = mib_read!(resident); + let active = mib_read!(active); + let mapped = mib_read!(mapped); + let retained = mib_read!(retained); + let metadata = mib_read!(metadata); + + trace!( + "CurrentProcess {{ pid: {}, rss: {} KiB, virt: {} KiB, \ + Jemalloc: {{ allocated: {} KiB, resident: {} KiB, \ + active: {} KiB, mapped: {} KiB, retained: {} KiB, \ + metadata: {} KiB }} }}", + pid, + rss, + virt, + allocated, + resident, + active, + mapped, + retained, + metadata + ); + } else { + error!("failed to fetch the memory information about current process"); + } + thread::sleep(wait_secs); + } + } else { + error!("failed to track the currently running program"); + } + }) + { + error!( + "failed to spawn the thread to track current process: {}", + err + ); + } + } +} From cb2bbd9073e1d6bbddc20eb5ab900e154539ab9d Mon Sep 17 00:00:00 2001 From: Boyu Yang Date: Thu, 13 Feb 2020 04:07:46 +0800 Subject: [PATCH 3/3] feat: add a feature to enable jemalloc profiling --- Cargo.lock | 2 + Cargo.toml | 6 ++- Makefile | 4 ++ ckb-bin/src/subcommand/run.rs | 3 +- docs/ckb-debugging.md | 62 ++++++++++++++++++++++++ resource/ckb.toml | 2 +- rpc/Cargo.toml | 1 + rpc/src/config.rs | 5 ++ rpc/src/module/debug.rs | 23 +++++++++ rpc/src/module/mod.rs | 2 + rpc/src/service_builder.rs | 14 ++++-- util/memory-tracker/Cargo.toml | 1 + util/memory-tracker/src/jemalloc-mock.rs | 5 ++ util/memory-tracker/src/jemalloc.rs | 15 ++++++ util/memory-tracker/src/lib.rs | 10 ++++ 15 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 docs/ckb-debugging.md create mode 100644 rpc/src/module/debug.rs create mode 100644 util/memory-tracker/src/jemalloc-mock.rs create mode 100644 util/memory-tracker/src/jemalloc.rs diff --git a/Cargo.lock b/Cargo.lock index eef00c33e1..6885ae5e18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,6 +658,7 @@ dependencies = [ "futures 0.3.4", "heim", "jemalloc-ctl", + "jemalloc-sys", "serde", ] @@ -865,6 +866,7 @@ dependencies = [ "ckb-indexer", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-network", "ckb-network-alert", "ckb-notify", diff --git a/Cargo.toml b/Cargo.toml index 55e8a5478a..3e4a6ff473 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ # Members are ordered by dependencies. Crates at top has fewer dependencies. "util/build-info", "util/logger", + "util/memory-tracker", "util", "util/hash", "util/fixed-hash", @@ -69,9 +70,12 @@ members = [ [profile.release] overflow-checks = true -[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies] +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), not(feature = "profiling")))'.dependencies] jemallocator = { version = "0.3.0" } +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), feature = "profiling"))'.dependencies] +jemallocator = { version = "0.3.0", features = ["profiling"] } [features] default = [] deadlock_detection = ["ckb-bin/deadlock_detection"] +profiling = [] diff --git a/Makefile b/Makefile index da40919bef..47d15f7b45 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,10 @@ check: setup-ckb-test ## Runs all of the compiler's checks. build: ## Build binary with release profile. cargo build ${VERBOSE} --release +.PHONY: build-for-profiling +build-for-profiling: ## Build binary with for profiling. + JEMALLOC_SYS_WITH_MALLOC_CONF="prof:true" cargo build ${VERBOSE} --features "profiling" + .PHONY: prod prod: ## Build binary for production release. RUSTFLAGS="--cfg disable_faketime" cargo build ${VERBOSE} --release diff --git a/ckb-bin/src/subcommand/run.rs b/ckb-bin/src/subcommand/run.rs index 4e156504c3..f964c9c594 100644 --- a/ckb-bin/src/subcommand/run.rs +++ b/ckb-bin/src/subcommand/run.rs @@ -144,7 +144,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> { .enable_experiment(shared.clone()) .enable_integration_test(shared.clone(), network_controller.clone(), chain_controller) .enable_alert(alert_verifier, alert_notifier, network_controller) - .enable_indexer(&args.config.indexer, shared.clone()); + .enable_indexer(&args.config.indexer, shared.clone()) + .enable_debug(); let io_handler = builder.build(); let rpc_server = RpcServer::new(args.config.rpc, io_handler, shared.notify_controller()); diff --git a/docs/ckb-debugging.md b/docs/ckb-debugging.md new file mode 100644 index 0000000000..12532364eb --- /dev/null +++ b/docs/ckb-debugging.md @@ -0,0 +1,62 @@ +# CKB Debugging + +## Memory + +**Only linux versions supported.** + +### Tracking Memory Usage in Logs + +Add the follow configuration into `ckb.toml`: + +```toml +[logger] +filter = "error,ckb-memory-tracker=trace" + +[memory_tracker] +# Seconds between checking the process, 0 is disable, default is 0. +interval = 600 +``` + +### Memory Profiling + +- Compile `ckb` with feature `profiling`. + + ```sh + make build-for-profiling` + ``` + + After compiling, a script named `jeprof` will be generated in `target` direcotry. + + ```sh + find target/ -name "jeprof" + ``` + +- Enable RPC module `Debug` in `ckb.toml`. + + ```toml + [rpc] + modules = ["Debug"] + ``` + +- Run `ckb`. + +- Dump memory usage to a file via call RPC `jemalloc_profiling_dump`. + + ```sh + curl -H 'content-type: application/json' -d '{ "id": 2, "jsonrpc": "2.0", "method": "jemalloc_profiling_dump", "params": [] }' http://localhost:8114 + ``` + + Then, a file named `ckb-jeprof.$TIMESTAMP.heap` will be generated in the working directory of the running `ckb`. + +- Generate a PDF of the call graph. + + **Required**: graphviz and ghostscript + + ```sh + jeprof --show_bytes --pdf target/debug/ckb ckb-jeprof.$TIMESTAMP.heap > call-graph.pdf + ``` + +## References: + +- [JEMALLOC: Use Case: Leak Checking](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking) +- [JEMALLOC: Use Case: Heap Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling) diff --git a/resource/ckb.toml b/resource/ckb.toml index 88582a779f..af046a8f05 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -95,7 +95,7 @@ listen_address = "127.0.0.1:8114" # {{ # Default is 10MiB = 10 * 1024 * 1024 max_request_body_size = 10485760 -# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment"] +# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment", "Debug"] modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment"] # {{ # integration => modules = ["Net", "Pool", "Miner", "Chain", "Experiment", "Stats", "Indexer", "IntegrationTest"] # }} diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index faca28a4ac..7e5b2c7753 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -41,6 +41,7 @@ ckb-error = { path = "../error" } ckb-reward-calculator = { path = "../util/reward-calculator" } ckb-tx-pool = { path = "../tx-pool" } ckb-script = { path = "../script" } +ckb-memory-tracker = { path = "../util/memory-tracker" } [dev-dependencies] reqwest = "0.9.16" diff --git a/rpc/src/config.rs b/rpc/src/config.rs index 5b4342a1c8..7844b9d08b 100644 --- a/rpc/src/config.rs +++ b/rpc/src/config.rs @@ -12,6 +12,7 @@ pub enum Module { IntegrationTest, Alert, Subscription, + Debug, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -69,4 +70,8 @@ impl Config { pub(crate) fn alert_enable(&self) -> bool { self.modules.contains(&Module::Alert) } + + pub(crate) fn debug_enable(&self) -> bool { + self.modules.contains(&Module::Debug) + } } diff --git a/rpc/src/module/debug.rs b/rpc/src/module/debug.rs new file mode 100644 index 0000000000..6f02f26ff4 --- /dev/null +++ b/rpc/src/module/debug.rs @@ -0,0 +1,23 @@ +use jsonrpc_core::Result; +use jsonrpc_derive::rpc; +use std::time; + +#[rpc(server)] +pub trait DebugRpc { + #[rpc(name = "jemalloc_profiling_dump")] + fn jemalloc_profiling_dump(&self) -> Result<()>; +} + +pub(crate) struct DebugRpcImpl {} + +impl DebugRpc for DebugRpcImpl { + fn jemalloc_profiling_dump(&self) -> Result<()> { + let timestamp = time::SystemTime::now() + .duration_since(time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let filename = format!("ckb-jeprof.{}.heap\0", timestamp); + ckb_memory_tracker::jemalloc_profiling_dump(filename); + Ok(()) + } +} diff --git a/rpc/src/module/mod.rs b/rpc/src/module/mod.rs index a7613a86de..e025e5570f 100644 --- a/rpc/src/module/mod.rs +++ b/rpc/src/module/mod.rs @@ -1,5 +1,6 @@ mod alert; mod chain; +mod debug; mod experiment; mod indexer; mod miner; @@ -11,6 +12,7 @@ mod test; pub(crate) use self::alert::{AlertRpc, AlertRpcImpl}; pub(crate) use self::chain::{ChainRpc, ChainRpcImpl}; +pub(crate) use self::debug::{DebugRpc, DebugRpcImpl}; pub(crate) use self::experiment::{ExperimentRpc, ExperimentRpcImpl}; pub(crate) use self::indexer::{IndexerRpc, IndexerRpcImpl}; pub(crate) use self::miner::{MinerRpc, MinerRpcImpl}; diff --git a/rpc/src/service_builder.rs b/rpc/src/service_builder.rs index c81c107411..2cc9ff334d 100644 --- a/rpc/src/service_builder.rs +++ b/rpc/src/service_builder.rs @@ -1,8 +1,9 @@ use crate::config::Config; use crate::module::{ - AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, ExperimentRpc, ExperimentRpcImpl, IndexerRpc, - IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, MinerRpc, MinerRpcImpl, NetworkRpc, - NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, StatsRpcImpl, + AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, DebugRpc, DebugRpcImpl, ExperimentRpc, + ExperimentRpcImpl, IndexerRpc, IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, + MinerRpc, MinerRpcImpl, NetworkRpc, NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, + StatsRpcImpl, }; use crate::IoHandler; use ckb_chain::chain::ChainController; @@ -156,6 +157,13 @@ impl<'a> ServiceBuilder<'a> { self } + pub fn enable_debug(mut self) -> Self { + if self.config.debug_enable() { + self.io_handler.extend_with(DebugRpcImpl {}.to_delegate()); + } + self + } + pub fn build(self) -> IoHandler { let mut io_handler = self.io_handler; io_handler.add_method("ping", |_| futures::future::ok("pong".into())); diff --git a/util/memory-tracker/Cargo.toml b/util/memory-tracker/Cargo.toml index 60b6e3ad9e..96dfe7262c 100644 --- a/util/memory-tracker/Cargo.toml +++ b/util/memory-tracker/Cargo.toml @@ -20,3 +20,4 @@ serde = { version = "1.0", features = ["derive"] } heim = "0.0.9" futures = "0.3.1" jemalloc-ctl = "0.3.3" +jemalloc-sys = "0.3.2" diff --git a/util/memory-tracker/src/jemalloc-mock.rs b/util/memory-tracker/src/jemalloc-mock.rs new file mode 100644 index 0000000000..60a88ae407 --- /dev/null +++ b/util/memory-tracker/src/jemalloc-mock.rs @@ -0,0 +1,5 @@ +use ckb_logger::warn; + +pub fn jemalloc_profiling_dump(_: String) { + warn!("jemalloc profiling dump: unsupported"); +} diff --git a/util/memory-tracker/src/jemalloc.rs b/util/memory-tracker/src/jemalloc.rs new file mode 100644 index 0000000000..250ce3e908 --- /dev/null +++ b/util/memory-tracker/src/jemalloc.rs @@ -0,0 +1,15 @@ +use std::{ffi, mem, ptr}; + +pub fn jemalloc_profiling_dump(mut filename: String) { + let opt_name = "prof.dump"; + let opt_c_name = ffi::CString::new(opt_name).unwrap(); + unsafe { + jemalloc_sys::mallctl( + opt_c_name.as_ptr(), + ptr::null_mut(), + ptr::null_mut(), + &mut filename as *mut _ as *mut _, + mem::size_of::<*mut ffi::c_void>(), + ); + } +} diff --git a/util/memory-tracker/src/lib.rs b/util/memory-tracker/src/lib.rs index df3fb00997..313d4dbd53 100644 --- a/util/memory-tracker/src/lib.rs +++ b/util/memory-tracker/src/lib.rs @@ -1,4 +1,13 @@ mod config; +#[cfg_attr( + all(not(target_env = "msvc"), not(target_os = "macos")), + path = "jemalloc.rs" +)] +#[cfg_attr( + not(all(not(target_env = "msvc"), not(target_os = "macos"))), + path = "jemalloc-mock.rs" +)] +mod jemalloc; #[cfg_attr( all(not(target_env = "msvc"), not(target_os = "macos")), path = "process.rs" @@ -10,4 +19,5 @@ mod config; mod process; pub use config::Config; +pub use jemalloc::jemalloc_profiling_dump; pub use process::track_current_process;