feat: add a feature to enable jemalloc profiling

Cargo.toml

@@ -69,9 +69,12 @@ members = [
 [profile.release]
 overflow-checks = true
 
-[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies]
+[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), not(feature = "profiling")))'.dependencies]
 jemallocator = { version = "0.3.0" }
+[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), feature = "profiling"))'.dependencies]
+jemallocator = { version = "0.3.0", features = ["profiling"] }
 
 [features]
 default = []
 deadlock_detection = ["ckb-bin/deadlock_detection"]
+profiling = []

Makefile

@@ -77,6 +77,10 @@ check: setup-ckb-test ## Runs all of the compiler's checks.
 build: ## Build binary with release profile.
 	cargo build ${VERBOSE} --release
 
+.PHONY: build
+build-for-profiling: ## Build binary with for profiling.
+	JEMALLOC_SYS_WITH_MALLOC_CONF="prof:true" cargo build ${VERBOSE} --features "profiling"
+
 .PHONY: prod
 prod: ## Build binary for production release.
 	RUSTFLAGS="--cfg disable_faketime" cargo build ${VERBOSE} --release

ckb-bin/src/subcommand/run.rs

@@ -144,7 +144,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> {
         .enable_experiment(shared.clone())
         .enable_integration_test(shared.clone(), network_controller.clone(), chain_controller)
         .enable_alert(alert_verifier, alert_notifier, network_controller)
-        .enable_indexer(&args.config.indexer, shared.clone());
+        .enable_indexer(&args.config.indexer, shared.clone())
+        .enable_debug();
     let io_handler = builder.build();
 
     let rpc_server = RpcServer::new(args.config.rpc, io_handler, shared.notify_controller());

docs/ckb-debugging.md

@@ -0,0 +1,62 @@
+# CKB Debugging
+
+## Memory
+
+**Only linux versions supported.**
+
+### Tracking Memory Usage in Logs
+
+Add the follow configuration into `ckb.toml`:
+
+```toml
+[logger]
+filter = "error,ckb-memory-tracker=trace"
+
+[memory_tracker]
+# Seconds between checking the process, 0 is disable, default is 0.
+interval = 600
+```
+
+### Memory Profiling
+
+- Compile `ckb` with feature `profiling`.
+
+  ```sh
+  make build-for-profiling`
+  ```
+
+  After compiling, a script named `jeprof` will be generated in `target` direcotry.
+
+  ```sh
+  find target/ -name "jeprof"
+  ```
+
+- Enable RPC module `Debug` in `ckb.toml`.
+
+  ```toml
+  [rpc]
+  modules = ["Debug"]
+  ```
+
+- Run `ckb`.
+
+- Dump memory usage to a file via call RPC `jemalloc_profiling_dump`.
+
+  ```sh
+  curl -H 'content-type: application/json' -d '{ "id": 2, "jsonrpc": "2.0", "method": "jemalloc_profiling_dump", "params": [] }' http://localhost:8114
+  ```
+
+  Then, a file named `ckb-jeprof.$TIMESTAMP.heap` will be generated in the working directory of the running `ckb`.
+
+- Generate a PDF of the call graph.
+
+  **Required**: graphviz and ghostscript
+
+  ```sh
+  jeprof --show_bytes --pdf target/debug/ckb ckb-jeprof.$TIMESTAMP.heap > call-graph.pdf
+  ```
+
+## References:
+
+- [JEMALLOC: Use Case: Leak Checking](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking)
+- [JEMALLOC: Use Case: Heap Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling)

resource/ckb.toml

@@ -95,7 +95,7 @@ listen_address = "127.0.0.1:8114" # {{
 # Default is 10MiB = 10 * 1024 * 1024
 max_request_body_size = 10485760
 
-# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment"]
+# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment", "Debug"]
 modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment"] # {{
 # integration => modules = ["Net", "Pool", "Miner", "Chain", "Experiment", "Stats", "Indexer", "IntegrationTest"]
 # }}

rpc/Cargo.toml

@@ -41,6 +41,7 @@ ckb-error = { path = "../error" }
 ckb-reward-calculator = { path = "../util/reward-calculator" }
 ckb-tx-pool = { path = "../tx-pool" }
 ckb-script = { path = "../script" }
+ckb-memory-tracker = { path = "../util/memory-tracker" }
 
 [dev-dependencies]
 reqwest = "0.9.16"

rpc/src/config.rs

@@ -12,6 +12,7 @@ pub enum Module {
     IntegrationTest,
     Alert,
     Subscription,
+    Debug,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -69,4 +70,8 @@ impl Config {
     pub(crate) fn alert_enable(&self) -> bool {
         self.modules.contains(&Module::Alert)
     }
+
+    pub(crate) fn debug_enable(&self) -> bool {
+        self.modules.contains(&Module::Debug)
+    }
 }

rpc/src/module/debug.rs

@@ -0,0 +1,23 @@
+use jsonrpc_core::Result;
+use jsonrpc_derive::rpc;
+use std::time;
+
+#[rpc(server)]
+pub trait DebugRpc {
+    #[rpc(name = "jemalloc_profiling_dump")]
+    fn jemalloc_profiling_dump(&self) -> Result<()>;
+}
+
+pub(crate) struct DebugRpcImpl {}
+
+impl DebugRpc for DebugRpcImpl {
+    fn jemalloc_profiling_dump(&self) -> Result<()> {
+        let timestamp = time::SystemTime::now()
+            .duration_since(time::SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        let filename = format!("ckb-jeprof.{}.heap\0", timestamp);
+        ckb_memory_tracker::jemalloc_profiling_dump(filename);
+        Ok(())
+    }
+}

rpc/src/module/mod.rs

@@ -1,5 +1,6 @@
 mod alert;
 mod chain;
+mod debug;
 mod experiment;
 mod indexer;
 mod miner;
@@ -11,6 +12,7 @@ mod test;
 
 pub(crate) use self::alert::{AlertRpc, AlertRpcImpl};
 pub(crate) use self::chain::{ChainRpc, ChainRpcImpl};
+pub(crate) use self::debug::{DebugRpc, DebugRpcImpl};
 pub(crate) use self::experiment::{ExperimentRpc, ExperimentRpcImpl};
 pub(crate) use self::indexer::{IndexerRpc, IndexerRpcImpl};
 pub(crate) use self::miner::{MinerRpc, MinerRpcImpl};

rpc/src/service_builder.rs

@@ -1,8 +1,9 @@
 use crate::config::Config;
 use crate::module::{
-    AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, ExperimentRpc, ExperimentRpcImpl, IndexerRpc,
-    IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, MinerRpc, MinerRpcImpl, NetworkRpc,
-    NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, StatsRpcImpl,
+    AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, DebugRpc, DebugRpcImpl, ExperimentRpc,
+    ExperimentRpcImpl, IndexerRpc, IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl,
+    MinerRpc, MinerRpcImpl, NetworkRpc, NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc,
+    StatsRpcImpl,
 };
 use crate::IoHandler;
 use ckb_chain::chain::ChainController;
@@ -156,6 +157,13 @@ impl<'a> ServiceBuilder<'a> {
         self
     }
 
+    pub fn enable_debug(mut self) -> Self {
+        if self.config.debug_enable() {
+            self.io_handler.extend_with(DebugRpcImpl {}.to_delegate());
+        }
+        self
+    }
+
     pub fn build(self) -> IoHandler {
         let mut io_handler = self.io_handler;
         io_handler.add_method("ping", |_| futures::future::ok("pong".into()));

util/memory-tracker/Cargo.toml

@@ -20,3 +20,4 @@ serde = { version = "1.0", features = ["derive"] }
 heim = "0.0.9"
 futures = "0.3.1"
 jemalloc-ctl = "0.3.3"
+jemalloc-sys = "0.3.2"

util/memory-tracker/src/jemalloc-mock.rs

@@ -0,0 +1,5 @@
+use ckb_logger::warn;
+
+pub fn jemalloc_profiling_dump(_: String) {
+    warn!("jemalloc profiling dump: unsupported");
+}

util/memory-tracker/src/jemalloc.rs

@@ -0,0 +1,15 @@
+use std::{ffi, mem, ptr};
+
+pub fn jemalloc_profiling_dump(mut filename: String) {
+    let opt_name = "prof.dump";
+    let opt_c_name = ffi::CString::new(opt_name).unwrap();
+    unsafe {
+        jemalloc_sys::mallctl(
+            opt_c_name.as_ptr(),
+            ptr::null_mut(),
+            ptr::null_mut(),
+            &mut filename as *mut _ as *mut _,
+            mem::size_of::<*mut ffi::c_void>(),
+        );
+    }
+}

util/memory-tracker/src/lib.rs

@@ -1,4 +1,13 @@
 mod config;
+#[cfg_attr(
+    all(not(target_env = "msvc"), not(target_os = "macos")),
+    path = "jemalloc.rs"
+)]
+#[cfg_attr(
+    not(all(not(target_env = "msvc"), not(target_os = "macos"))),
+    path = "jemalloc-mock.rs"
+)]
+mod jemalloc;
 #[cfg_attr(
     all(not(target_env = "msvc"), not(target_os = "macos")),
     path = "process.rs"
@@ -10,4 +19,5 @@ mod config;
 mod process;
 
 pub use config::Config;
+pub use jemalloc::jemalloc_profiling_dump;
 pub use process::track_current_process;