feat: add a feature to enable jemalloc profiling

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     # Members are ordered by dependencies. Crates at top has fewer dependencies.
     "util/build-info",
     "util/logger",
+    "util/memory-tracker",
     "util",
     "util/hash",
     "util/fixed-hash",
@@ -69,9 +70,12 @@ members = [
 [profile.release]
 overflow-checks = true
 
-[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies]
+[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), not(feature = "profiling")))'.dependencies]
 jemallocator = { version = "0.3.0", features = ["unprefixed_malloc_on_supported_platforms"] }
+[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos"), feature = "profiling"))'.dependencies]
+jemallocator = { version = "0.3.0", features = ["unprefixed_malloc_on_supported_platforms", "profiling"] }
 
 [features]
 default = []
 deadlock_detection = ["ckb-bin/deadlock_detection"]
+profiling = []

Makefile

@@ -77,6 +77,10 @@ check: setup-ckb-test ## Runs all of the compiler's checks.
 build: ## Build binary with release profile.
 	cargo build ${VERBOSE} --release
 
+.PHONY: build-for-profiling
+build-for-profiling: ## Build binary with for profiling.
+	JEMALLOC_SYS_WITH_MALLOC_CONF="prof:true" cargo build ${VERBOSE} --features "profiling"
+
 .PHONY: prod
 prod: ## Build binary for production release.
 	RUSTFLAGS="--cfg disable_faketime" cargo build ${VERBOSE} --release

ckb-bin/Cargo.toml

@@ -29,9 +29,9 @@ ctrlc = { version = "3.1", features = ["termination"] }
 ckb-sync = { path = "../sync"}
 ckb-instrument = { path = "../util/instrument", features = ["progress_bar"] }
 ckb-build-info = { path = "../util/build-info" }
+ckb-memory-tracker = { path = "../util/memory-tracker" }
 ckb-verification = { path = "../verification" }
 base64 = "0.10.1"
 
-
 [features]
 deadlock_detection = ["ckb-util/deadlock_detection"]

ckb-bin/src/subcommand/miner.rs

@@ -10,6 +10,8 @@ pub fn miner(args: MinerArgs) -> Result<(), ExitCode> {
     let mut client = Client::new(new_work_tx, client);
     let mut miner = Miner::new(args.pow_engine, client.clone(), new_work_rx, &workers);
 
+    ckb_memory_tracker::track_current_process(args.memory_tracker.interval);
+
     thread::Builder::new()
         .name("client".to_string())
         .spawn(move || client.poll_block_template())

ckb-bin/src/subcommand/run.rs

@@ -42,6 +42,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> {
     // Verify genesis every time starting node
     verify_genesis(&shared)?;
 
+    ckb_memory_tracker::track_current_process(args.config.memory_tracker.interval);
+
     let chain_service = ChainService::new(shared.clone(), table);
     let chain_controller = chain_service.start(Some("ChainService"));
     info_target!(crate::LOG_TARGET_MAIN, "ckb version: {}", version);
@@ -142,7 +144,8 @@ pub fn run(args: RunArgs, version: Version) -> Result<(), ExitCode> {
         .enable_experiment(shared.clone())
         .enable_integration_test(shared.clone(), network_controller.clone(), chain_controller)
         .enable_alert(alert_verifier, alert_notifier, network_controller)
-        .enable_indexer(&args.config.indexer, shared.clone());
+        .enable_indexer(&args.config.indexer, shared.clone())
+        .enable_debug();
     let io_handler = builder.build();
 
     let rpc_server = RpcServer::new(args.config.rpc, io_handler, shared.notify_controller());

devtools/ci/check-cargotoml.sh

@@ -83,7 +83,7 @@ function search_crate() {
         | wc -l)
     depcnt=$((depcnt + tmpcnt))
     tmpcnt=$({\
-        ${GREP} ${grepopts} "[ (<]\(::\|\)${crate}::" "${source}" \
+        ${GREP} ${grepopts} "\(^\|[ (<]\)\(::\|\)${crate}::" "${source}" \
             || true; }\
         | wc -l)
     depcnt=$((depcnt + tmpcnt))

docs/ckb-debugging.md

@@ -0,0 +1,62 @@
+# CKB Debugging
+
+## Memory
+
+**Only linux versions supported.**
+
+### Tracking Memory Usage in Logs
+
+Add the follow configuration into `ckb.toml`:
+
+```toml
+[logger]
+filter = "error,ckb-memory-tracker=trace"
+
+[memory_tracker]
+# Seconds between checking the process, 0 is disable, default is 0.
+interval = 600
+```
+
+### Memory Profiling
+
+- Compile `ckb` with feature `profiling`.
+
+  ```sh
+  make build-for-profiling`
+  ```
+
+  After compiling, a script named `jeprof` will be generated in `target` direcotry.
+
+  ```sh
+  find target/ -name "jeprof"
+  ```
+
+- Enable RPC module `Debug` in `ckb.toml`.
+
+  ```toml
+  [rpc]
+  modules = ["Debug"]
+  ```
+
+- Run `ckb`.
+
+- Dump memory usage to a file via call RPC `jemalloc_profiling_dump`.
+
+  ```sh
+  curl -H 'content-type: application/json' -d '{ "id": 2, "jsonrpc": "2.0", "method": "jemalloc_profiling_dump", "params": [] }' http://localhost:8114
+  ```
+
+  Then, a file named `ckb-jeprof.$TIMESTAMP.heap` will be generated in the working directory of the running `ckb`.
+
+- Generate a PDF of the call graph.
+
+  **Required**: graphviz and ghostscript
+
+  ```sh
+  jeprof --show_bytes --pdf target/debug/ckb ckb-jeprof.$TIMESTAMP.heap > call-graph.pdf
+  ```
+
+## References:
+
+- [JEMALLOC: Use Case: Leak Checking](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking)
+- [JEMALLOC: Use Case: Heap Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling)

resource/ckb-miner.toml

@@ -39,6 +39,10 @@ dsn = "" # {{
 # please leave a way to contact you when we have troubles to reproduce the errors.
 # org_contact = ""
 
+# [memory_tracker]
+# # Seconds between checking the process, 0 is disable, default is 0.
+# interval = 600
+
 [miner.client]
 rpc_url = "http://127.0.0.1:8114/" # {{
 # _ => rpc_url = "http://127.0.0.1:{rpc_port}/"

resource/ckb.toml

@@ -42,6 +42,10 @@ dsn = "" # {{
 # please leave a way to contact you when we have troubles to reproduce the errors.
 # org_contact = ""
 
+# [memory_tracker]
+# # Seconds between checking the process, 0 is disable, default is 0.
+# interval = 600
+
 [network]
 listen_addresses = ["/ip4/0.0.0.0/tcp/8115"] # {{
 # _ => listen_addresses = ["/ip4/0.0.0.0/tcp/{p2p_port}"]
@@ -91,7 +95,7 @@ listen_address = "127.0.0.1:8114" # {{
 # Default is 10MiB = 10 * 1024 * 1024
 max_request_body_size = 10485760
 
-# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment"]
+# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Indexer", "Experiment", "Debug"]
 modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment"] # {{
 # integration => modules = ["Net", "Pool", "Miner", "Chain", "Experiment", "Stats", "Indexer", "IntegrationTest"]
 # }}

rpc/Cargo.toml

@@ -41,6 +41,7 @@ ckb-error = { path = "../error" }
 ckb-reward-calculator = { path = "../util/reward-calculator" }
 ckb-tx-pool = { path = "../tx-pool" }
 ckb-script = { path = "../script" }
+ckb-memory-tracker = { path = "../util/memory-tracker" }
 
 [dev-dependencies]
 reqwest = "0.9.16"

rpc/src/config.rs

@@ -12,6 +12,7 @@ pub enum Module {
     IntegrationTest,
     Alert,
     Subscription,
+    Debug,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -69,4 +70,8 @@ impl Config {
     pub(crate) fn alert_enable(&self) -> bool {
         self.modules.contains(&Module::Alert)
     }
+
+    pub(crate) fn debug_enable(&self) -> bool {
+        self.modules.contains(&Module::Debug)
+    }
 }

rpc/src/module/debug.rs

@@ -0,0 +1,23 @@
+use jsonrpc_core::Result;
+use jsonrpc_derive::rpc;
+use std::time;
+
+#[rpc(server)]
+pub trait DebugRpc {
+    #[rpc(name = "jemalloc_profiling_dump")]
+    fn jemalloc_profiling_dump(&self) -> Result<()>;
+}
+
+pub(crate) struct DebugRpcImpl {}
+
+impl DebugRpc for DebugRpcImpl {
+    fn jemalloc_profiling_dump(&self) -> Result<()> {
+        let timestamp = time::SystemTime::now()
+            .duration_since(time::SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        let filename = format!("ckb-jeprof.{}.heap\0", timestamp);
+        ckb_memory_tracker::jemalloc_profiling_dump(filename);
+        Ok(())
+    }
+}

rpc/src/module/mod.rs

@@ -1,5 +1,6 @@
 mod alert;
 mod chain;
+mod debug;
 mod experiment;
 mod indexer;
 mod miner;
@@ -11,6 +12,7 @@ mod test;
 
 pub(crate) use self::alert::{AlertRpc, AlertRpcImpl};
 pub(crate) use self::chain::{ChainRpc, ChainRpcImpl};
+pub(crate) use self::debug::{DebugRpc, DebugRpcImpl};
 pub(crate) use self::experiment::{ExperimentRpc, ExperimentRpcImpl};
 pub(crate) use self::indexer::{IndexerRpc, IndexerRpcImpl};
 pub(crate) use self::miner::{MinerRpc, MinerRpcImpl};

rpc/src/service_builder.rs

@@ -1,8 +1,9 @@
 use crate::config::Config;
 use crate::module::{
-    AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, ExperimentRpc, ExperimentRpcImpl, IndexerRpc,
-    IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl, MinerRpc, MinerRpcImpl, NetworkRpc,
-    NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc, StatsRpcImpl,
+    AlertRpc, AlertRpcImpl, ChainRpc, ChainRpcImpl, DebugRpc, DebugRpcImpl, ExperimentRpc,
+    ExperimentRpcImpl, IndexerRpc, IndexerRpcImpl, IntegrationTestRpc, IntegrationTestRpcImpl,
+    MinerRpc, MinerRpcImpl, NetworkRpc, NetworkRpcImpl, PoolRpc, PoolRpcImpl, StatsRpc,
+    StatsRpcImpl,
 };
 use crate::IoHandler;
 use ckb_chain::chain::ChainController;
@@ -172,6 +173,13 @@ impl<'a> ServiceBuilder<'a> {
         self
     }
 
+    pub fn enable_debug(mut self) -> Self {
+        if self.config.debug_enable() {
+            self.io_handler.extend_with(DebugRpcImpl {}.to_delegate());
+        }
+        self
+    }
+
     fn update_disabled_methods<I, M>(&mut self, module: &str, rpc_method: I)
     where
         I: IntoIterator<Item = (String, M)>,

util/app-config/Cargo.toml

@@ -24,6 +24,7 @@ ckb-resource = { path = "../../resource"}
 ckb-store = { path = "../../store" }
 ckb-network-alert = { path = "../network-alert" }
 ckb-build-info = { path = "../build-info" }
+ckb-memory-tracker = { path = "../memory-tracker" }
 ckb-indexer = { path = "../../indexer" }
 ckb-tx-pool = { path = "../../tx-pool" }
 ckb-notify = { path = "../../notify" }

util/app-config/src/app_config.rs

@@ -14,6 +14,7 @@ use ckb_chain_spec::ChainSpec;
 use ckb_db::DBConfig;
 use ckb_indexer::IndexerConfig;
 use ckb_logger::Config as LogConfig;
+use ckb_memory_tracker::Config as MemoryTrackerConfig;
 use ckb_miner::MinerConfig;
 use ckb_network::NetworkConfig;
 use ckb_network_alert::config::SignatureConfig as AlertSignatureConfig;
@@ -37,6 +38,8 @@ pub struct CKBAppConfig {
     pub data_dir: PathBuf,
     pub logger: LogConfig,
     pub sentry: SentryConfig,
+    #[serde(default)]
+    pub memory_tracker: MemoryTrackerConfig,
     pub chain: ChainConfig,
 
     pub block_assembler: Option<BlockAssemblerConfig>,
@@ -61,6 +64,8 @@ pub struct MinerAppConfig {
     pub chain: ChainConfig,
     pub logger: LogConfig,
     pub sentry: SentryConfig,
+    #[serde(default)]
+    pub memory_tracker: MemoryTrackerConfig,
 
     pub miner: MinerConfig,
 }
@@ -108,6 +113,13 @@ impl AppConfig {
         }
     }
 
+    pub fn memory_tracker(&self) -> &MemoryTrackerConfig {
+        match self {
+            AppConfig::CKB(config) => &config.memory_tracker,
+            AppConfig::Miner(config) => &config.memory_tracker,
+        }
+    }
+
     pub fn chain_spec(&self) -> Result<ChainSpec, ExitCode> {
         let spec_resource = match self {
             AppConfig::CKB(config) => &config.chain.spec,

util/app-config/src/args.rs

@@ -1,6 +1,7 @@
 use super::app_config::CKBAppConfig;
 use ckb_chain_spec::consensus::Consensus;
 use ckb_jsonrpc_types::ScriptHashType;
+use ckb_memory_tracker::Config as MemoryTrackerConfig;
 use ckb_miner::MinerConfig;
 use ckb_pow::PowEngine;
 use std::path::PathBuf;
@@ -34,6 +35,7 @@ pub struct ProfArgs {
 pub struct MinerArgs {
     pub config: MinerConfig,
     pub pow_engine: Arc<dyn PowEngine>,
+    pub memory_tracker: MemoryTrackerConfig,
 }
 
 pub struct StatsArgs {

util/app-config/src/lib.rs

@@ -107,12 +107,14 @@ impl Setup {
 
     pub fn miner(self) -> Result<MinerArgs, ExitCode> {
         let spec = self.chain_spec()?;
+        let memory_tracker = self.config.memory_tracker().to_owned();
         let config = self.config.into_miner()?;
         let pow_engine = spec.pow_engine();
 
         Ok(MinerArgs {
             pow_engine,
             config: config.miner,
+            memory_tracker,
         })
     }
 

util/memory-tracker/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "ckb-memory-tracker"
+version = "0.30.1-pre"
+authors = ["Nervos Core Dev <[email protected]>"]
+edition = "2018"
+license = "MIT"
+
+[dependencies]
+ckb-logger = { path = "../logger" }
+serde = { version = "1.0", features = ["derive"] }
+
+# TODO Why don't disable this crate by "target.*" in the crates which are dependent on this crate?
+#
+# I really don't like to write such dirty code. I try a lot. But due to the limit of the "cargo"
+# (and a lot of stupid bugs), at last, I have to write these stupid code.
+#
+# References:
+# - [Cargo Issue-1197: Target-specific features](https://github.com/rust-lang/cargo/issues/1197)
+[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies]
+heim = "0.0.10"
+futures = "0.3.1"
+jemalloc-ctl = "0.3.3"
+jemalloc-sys = "0.3.2"

util/memory-tracker/src/config.rs

@@ -0,0 +1,12 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Config {
+    pub interval: u64,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self { interval: 0 }
+    }
+}

util/memory-tracker/src/jemalloc-mock.rs

@@ -0,0 +1,5 @@
+use ckb_logger::warn;
+
+pub fn jemalloc_profiling_dump(_: String) {
+    warn!("jemalloc profiling dump: unsupported");
+}