diff --git a/Cargo.lock b/Cargo.lock index 4d74d44eee..d9d36aaf5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -739,7 +739,7 @@ dependencies = [ "object_store 1.1.0", "parquet", "parquet_ext", - "pprof", + "pprof 0.10.1", "rand 0.7.3", "serde", "snafu 0.6.10", @@ -1602,6 +1602,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "cpp_demangle" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b446fd40bcc17eddd6a4a78f24315eb90afdb3334999ddfd4909985c47722442" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "cpufeatures" version = "0.2.6" @@ -4988,7 +4997,28 @@ dependencies = [ "once_cell", "parking_lot 0.12.1", "smallvec", - "symbolic-demangle", + "symbolic-demangle 9.2.1", + "tempfile", + "thiserror", +] + +[[package]] +name = "pprof" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" +dependencies = [ + "backtrace", + "cfg-if 1.0.0", + "findshlibs", + "inferno", + "libc", + "log", + "nix 0.26.2", + "once_cell", + "parking_lot 0.12.1", + "smallvec", + "symbolic-demangle 10.2.1", "tempfile", "thiserror", ] @@ -6353,6 +6383,7 @@ dependencies = [ "meta_client", "opensrv-mysql", "paste 1.0.12", + "pprof 0.11.1", "profile", "prom-remote-api", "prometheus 0.12.0", @@ -6991,15 +7022,38 @@ dependencies = [ "uuid 1.3.0", ] +[[package]] +name = "symbolic-common" +version = "10.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid 1.3.0", +] + [[package]] name = "symbolic-demangle" version = "9.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b940a1fdbc72bb3369e38714efe6cd332dbbe46d093cf03d668b9ac390d1ad0" dependencies = [ - "cpp_demangle", + "cpp_demangle 0.3.5", + "rustc-demangle", + "symbolic-common 9.2.1", +] + +[[package]] +name = "symbolic-demangle" +version = "10.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +dependencies = [ + "cpp_demangle 0.4.0", "rustc-demangle", - "symbolic-common", + "symbolic-common 10.2.1", ] [[package]] @@ -7780,8 +7834,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", - "rand 0.3.23", + "cfg-if 1.0.0", + "rand 0.8.5", "static_assertions 1.1.0", ] diff --git a/server/Cargo.toml b/server/Cargo.toml index 062291b800..ba0e26c999 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -35,6 +35,7 @@ logger = { workspace = true } meta_client = { workspace = true } opensrv-mysql = "0.1.0" paste = { workspace = true } +pprof = { version = "0.11.1", features = ["flamegraph"] } profile = { workspace = true } prom-remote-api = { version = "0.2.1", features = ["warp"] } prometheus = { workspace = true } diff --git a/server/src/http.rs b/server/src/http.rs index 96466f3c00..7336771c65 100644 --- a/server/src/http.rs +++ b/server/src/http.rs @@ -3,8 +3,8 @@ //! Http service use std::{ - collections::HashMap, convert::Infallible, error::Error as StdError, net::IpAddr, sync::Arc, - time::Duration, + collections::HashMap, convert::Infallible, error::Error as StdError, fs::File, net::IpAddr, + sync::Arc, thread, time::Duration, }; use analytic_engine::setup::OpenedWals; @@ -194,6 +194,7 @@ impl Service { .or(self.flush_memtable()) .or(self.update_log_level()) .or(self.heap_profile()) + .or(self.cpu_profile()) .or(self.server_config()) .or(self.stats()) } @@ -410,6 +411,40 @@ impl Service { ) } + // GET /debug/cpu_profile/{seconds} + fn cpu_profile( + &self, + ) -> impl Filter + Clone { + warp::path!("debug" / "cpu_profile" / ..) + .and(warp::path::param::()) + .and(warp::get()) + .and(self.with_context()) + .and_then(|duration_sec: u64, ctx: RequestContext| async move { + let handle = ctx.runtime.spawn_blocking(move || -> Result<()> { + let guard = pprof::ProfilerGuardBuilder::default() + .frequency(100) + .blocklist(&["libc", "libgcc", "pthread", "vdso"]) + .build() + .box_err() + .context(Internal)?; + + thread::sleep(Duration::from_secs(duration_sec)); + + let report = guard.report().build().box_err().context(Internal)?; + let file = File::create("/tmp/flamegraph.svg") + .box_err() + .context(Internal)?; + report.flamegraph(file).box_err().context(Internal)?; + Ok(()) + }); + let result = handle.await.context(JoinAsyncTask); + match result { + Ok(_) => Ok("ok"), + Err(e) => Err(reject::custom(e)), + } + }) + } + // GET /debug/config fn server_config( &self,