diff --git a/Cargo.lock b/Cargo.lock index eef00c33e1c..6885ae5e180 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,6 +658,7 @@ dependencies = [ "futures 0.3.4", "heim", "jemalloc-ctl", + "jemalloc-sys", "serde", ] @@ -865,6 +866,7 @@ dependencies = [ "ckb-indexer", "ckb-jsonrpc-types", "ckb-logger", + "ckb-memory-tracker", "ckb-network", "ckb-network-alert", "ckb-notify", diff --git a/Cargo.toml b/Cargo.toml index 55e8a5478ad..776b8d6c53f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,9 +69,12 @@ members = [ [profile.release] overflow-checks = true -[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies] +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), not(feature = "profiling")))'.dependencies] jemallocator = { version = "0.3.0" } +[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), feature = "profiling"))'.dependencies] +jemallocator = { version = "0.3.0", features = ["profiling"] } [features] default = [] deadlock_detection = ["ckb-bin/deadlock_detection"] +profiling = [] diff --git a/Makefile b/Makefile index da40919bef0..673f28e4372 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,10 @@ check: setup-ckb-test ## Runs all of the compiler's checks. build: ## Build binary with release profile. cargo build ${VERBOSE} --release +.PHONY: build +build-for-profiling: ## Build binary with for profiling. + JEMALLOC_SYS_WITH_MALLOC_CONF="prof:true" cargo build ${VERBOSE} --features "profiling" + .PHONY: prod prod: ## Build binary for production release. RUSTFLAGS="--cfg disable_faketime" cargo build ${VERBOSE} --release diff --git a/ckb-bin/src/subcommand/run.rs b/ckb-bin/src/subcommand/run.rs index 4e156504c37..f964c9c5940 100644 --- a/ckb-bin/src/subcommand/run.rs +++ b/ckb-bin/src/subcommand/run.rs @@ -144,7 +144,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> { .enable_experiment(shared.clone()) .enable_integration_test(shared.clone(), network_controller.clone(), chain_controller) .enable_alert(alert_verifier, alert_notifier, network_controller) - .enable_indexer(&args.config.indexer, shared.clone()); + .enable_indexer(&args.config.indexer, shared.clone()) + .enable_debug(); let io_handler = builder.build(); let rpc_server = RpcServer::new(args.config.rpc, io_handler, shared.notify_controller()); diff --git a/docs/ckb-debugging.md b/docs/ckb-debugging.md new file mode 100644 index 00000000000..12532364ebf --- /dev/null +++ b/docs/ckb-debugging.md @@ -0,0 +1,62 @@ +# CKB Debugging + +## Memory + +**Only linux versions supported.** + +### Tracking Memory Usage in Logs + +Add the follow configuration into `ckb.toml`: + +```toml +[logger] +filter = "error,ckb-memory-tracker=trace" + +[memory_tracker] +# Seconds between checking the process, 0 is disable, default is 0. +interval = 600 +``` + +### Memory Profiling + +- Compile `ckb` with feature `profiling`. + + ```sh + make build-for-profiling` + ``` + + After compiling, a script named `jeprof` will be generated in `target` direcotry. + + ```sh + find target/ -name "jeprof" + ``` + +- Enable RPC module `Debug` in `ckb.toml`. + + ```toml + [rpc] + modules = ["Debug"] + ``` + +- Run `ckb`. + +- Dump memory usage to a file via call RPC `jemalloc_profiling_dump`. + + ```sh + curl -H 'content-type: application/json' -d '{ "id": 2, "jsonrpc": "2.0", "method": "jemalloc_profiling_dump", "params": [] }' http://localhost:8114 + ``` + + Then, a file named `ckb-jeprof.$TIMESTAMP.heap` will be generated in the working directory of the running `ckb`. + +- Generate a PDF of the call graph. + + **Required**: graphviz and ghostscript + + ```sh + jeprof --show_bytes --pdf target/debug/ckb ckb-jeprof.$TIMESTAMP.heap > call-graph.pdf + ``` + +## References: + +- [JEMALLOC: Use Case: Leak Checking](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking) +- [JEMALLOC: Use Case: Heap Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling) diff --git a/resource/ckb.toml b/resource/ckb.toml index 88582a779fc..af046a8f05a 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -95,7 +95,7 @@ listen_address = "127.0.0.1:8114" # {{ # Default is 10MiB = 10 * 1024 * 1024 max_request_body_size = 10485760 -# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment"] +# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment", "Debug"] modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment"] # {{ # integration => modules = ["Net", "Pool", "Miner", "Chain", "Experiment", "Stats", "Indexer", "IntegrationTest"] # }} diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index faca28a4ace..7e5b2c77534 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -41,6 +41,7 @@ ckb-error = { path = "../error" } ckb-reward-calculator = { path = "../util/reward-calculator" } ckb-tx-pool = { path = "../tx-pool" } ckb-script = { path = "../script" } +ckb-memory-tracker = { path = "../util/memory-tracker" } [dev-dependencies] reqwest = "0.9.16" diff --git a/rpc/src/config.rs b/rpc/src/config.rs index 5b4342a1c82..7844b9d08b5 100644 --- a/rpc/src/config.rs +++ b/rpc/src/config.rs @@ -12,6 +12,7 @@ pub enum Module { IntegrationTest, Alert, Subscription, + Debug, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -69,4 +70,8 @@ impl Config { pub(crate) fn alert_enable(&self) -> bool { self.modules.contains(&Module::Alert) } + + pub(crate) fn debug_enable(&self) -> bool { + self.modules.contains(&Module::Debug) + } } diff --git a/rpc/src/module/debug.rs b/rpc/src/module/debug.rs new file mode 100644 index 00000000000..6f02f26ff42 --- /dev/null +++ b/rpc/src/module/debug.rs @@ -0,0 +1,23 @@ +use jsonrpc_core::Result; +use jsonrpc_derive::rpc; +use std::time; + +#[rpc(server)] +pub trait DebugRpc { + #[rpc(name = "jemalloc_profiling_dump")] + fn jemalloc_profiling_dump(&self) -> Result<()>; +} + +pub(crate) struct DebugRpcImpl {} + +impl DebugRpc for DebugRpcImpl { + fn jemalloc_profiling_dump(&self) -> Result<()> { + let timestamp = time::SystemTime::now() + .duration_since(time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let filename = format!("ckb-jeprof.{}.heap\0", timestamp); + ckb_memory_tracker::jemalloc_profiling_dump(filename); + Ok(()) + } +} diff --git a/rpc/src/module/mod.rs b/rpc/src/module/mod.rs index a7613a86de9..e025e5570f8 100644 --- a/rpc/src/module/mod.rs +++ b/rpc/src/module/mod.rs @@ -1,5 +1,6 @@ mod alert; mod chain; +mod debug; mod experiment; mod indexer; mod miner; @@ -11,6 +12,7 @@ mod test; pub(crate) use self::alert::{AlertRpc, AlertRpcImpl}; pub(crate) use self::chain::{ChainRpc, ChainRpcImpl}; +pub(crate) use self::debug::{DebugRpc, DebugRpcImpl}; pub(crate) use self::experiment::{ExperimentRpc, ExperimentRpcImpl}; pub(crate) use self::indexer::{IndexerRpc, IndexerRpcImpl}; pub(crate) use self::miner::{MinerRpc, MinerRpcImpl}; diff --git a/rpc/src/service_builder.rs b/rpc/src/service_builder.rs index c81c1074110..2cc9ff334db 100644 --- a/rpc/src/service_builder.rs +++ b/rpc/src/service_builder.rs @@ -1,8 +1,9 @@ use crate::config::Config; use crate::module::{ - AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, ExperimentRpc, ExperimentRpcImpl, IndexerRpc, - IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, MinerRpc, MinerRpcImpl, NetworkRpc, - NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, StatsRpcImpl, + AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, DebugRpc, DebugRpcImpl, ExperimentRpc, + ExperimentRpcImpl, IndexerRpc, IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, + MinerRpc, MinerRpcImpl, NetworkRpc, NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, + StatsRpcImpl, }; use crate::IoHandler; use ckb_chain::chain::ChainController; @@ -156,6 +157,13 @@ impl<'a> ServiceBuilder<'a> { self } + pub fn enable_debug(mut self) -> Self { + if self.config.debug_enable() { + self.io_handler.extend_with(DebugRpcImpl {}.to_delegate()); + } + self + } + pub fn build(self) -> IoHandler { let mut io_handler = self.io_handler; io_handler.add_method("ping", |_| futures::future::ok("pong".into())); diff --git a/util/memory-tracker/Cargo.toml b/util/memory-tracker/Cargo.toml index 60b6e3ad9e5..96dfe7262c1 100644 --- a/util/memory-tracker/Cargo.toml +++ b/util/memory-tracker/Cargo.toml @@ -20,3 +20,4 @@ serde = { version = "1.0", features = ["derive"] } heim = "0.0.9" futures = "0.3.1" jemalloc-ctl = "0.3.3" +jemalloc-sys = "0.3.2" diff --git a/util/memory-tracker/src/jemalloc-mock.rs b/util/memory-tracker/src/jemalloc-mock.rs new file mode 100644 index 00000000000..60a88ae407a --- /dev/null +++ b/util/memory-tracker/src/jemalloc-mock.rs @@ -0,0 +1,5 @@ +use ckb_logger::warn; + +pub fn jemalloc_profiling_dump(_: String) { + warn!("jemalloc profiling dump: unsupported"); +} diff --git a/util/memory-tracker/src/jemalloc.rs b/util/memory-tracker/src/jemalloc.rs new file mode 100644 index 00000000000..250ce3e9087 --- /dev/null +++ b/util/memory-tracker/src/jemalloc.rs @@ -0,0 +1,15 @@ +use std::{ffi, mem, ptr}; + +pub fn jemalloc_profiling_dump(mut filename: String) { + let opt_name = "prof.dump"; + let opt_c_name = ffi::CString::new(opt_name).unwrap(); + unsafe { + jemalloc_sys::mallctl( + opt_c_name.as_ptr(), + ptr::null_mut(), + ptr::null_mut(), + &mut filename as *mut _ as *mut _, + mem::size_of::<*mut ffi::c_void>(), + ); + } +} diff --git a/util/memory-tracker/src/lib.rs b/util/memory-tracker/src/lib.rs index df3fb009976..313d4dbd531 100644 --- a/util/memory-tracker/src/lib.rs +++ b/util/memory-tracker/src/lib.rs @@ -1,4 +1,13 @@ mod config; +#[cfg_attr( + all(not(target_env = "msvc"), not(target_os = "macos")), + path = "jemalloc.rs" +)] +#[cfg_attr( + not(all(not(target_env = "msvc"), not(target_os = "macos"))), + path = "jemalloc-mock.rs" +)] +mod jemalloc; #[cfg_attr( all(not(target_env = "msvc"), not(target_os = "macos")), path = "process.rs" @@ -10,4 +19,5 @@ mod config; mod process; pub use config::Config; +pub use jemalloc::jemalloc_profiling_dump; pub use process::track_current_process;