From 5ce32bfd9e57820c6a54a4bb81a3bc01bc3e4451 Mon Sep 17 00:00:00 2001 From: Daniel Jordon Date: Tue, 10 Dec 2024 15:22:38 -0500 Subject: [PATCH] feat: implement signer metrics exporter (#1084) * Add an initial metrics setup module * Set up metrics from the settings * Add a few counter and histogram metrics --- Cargo.lock | 116 +++++++++++++++++++++--- Cargo.toml | 2 + signer/Cargo.toml | 2 + signer/build.rs | 26 ++++++ signer/src/api/new_block.rs | 8 ++ signer/src/block_observer.rs | 36 +++++++- signer/src/config/default.toml | 9 +- signer/src/config/mod.rs | 29 ++++++ signer/src/lib.rs | 16 +++- signer/src/main.rs | 11 ++- signer/src/message.rs | 15 ++++ signer/src/metrics.rs | 83 +++++++++++++++++ signer/src/transaction_coordinator.rs | 123 +++++++++++++++++++++++--- signer/src/transaction_signer.rs | 71 +++++++++++++-- 14 files changed, 510 insertions(+), 37 deletions(-) create mode 100644 signer/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index a320e3072..1dd26867c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,7 +750,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.5.1", "hyper-util", "itoa 1.0.11", "matchit", @@ -1136,9 +1136,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" dependencies = [ "serde 1.0.203", ] @@ -2098,6 +2098,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2437,6 +2443,9 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -2741,9 +2750,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -2809,7 +2818,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.3.1", + "hyper 1.5.1", "hyper-util", "native-tls", "tokio", @@ -2819,20 +2828,19 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.3.1", + "hyper 1.5.1", "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -3923,6 +3931,50 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3" +dependencies = [ + "ahash", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b6f8152da6d7892ff1b7a1c0fa3f435e92b5918ad67035c3bb432111d9a29b" +dependencies = [ + "base64 0.22.1", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "indexmap 2.7.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b482df36c13dd1869d73d14d28cd4855fbd6cfc32294bee109908a9f4a4ed7" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.2", + "metrics", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -4767,6 +4819,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "powerfmt" version = "0.2.0" @@ -4995,6 +5053,21 @@ dependencies = [ "cc", ] +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "query_map" version = "0.5.0" @@ -5169,6 +5242,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "11.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "rayon" version = "1.10.0" @@ -5335,7 +5417,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.5.1", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -6123,6 +6205,8 @@ dependencies = [ "hex", "include_dir", "libp2p", + "metrics", + "metrics-exporter-prometheus", "mockall 0.12.1", "mockito 1.4.0", "more-asserts", @@ -6172,6 +6256,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" + [[package]] name = "slab" version = "0.4.9" @@ -6980,9 +7070,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index adc6a3d2e..f521f342f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ config = "0.11.0" futures = "0.3.24" hashbrown = "0.14.5" http = "1.1.0" +metrics = "0.24" +metrics-exporter-prometheus = { version = "0.16", default-features = false, features = ["http-listener"] } # This is necessary to compile the AWS Lambda as a lambda. openssl = { version = "0.10.66", features = ["vendored"] } p256k1 = "7.1.0" diff --git a/signer/Cargo.toml b/signer/Cargo.toml index a255ea5eb..28a8a05e7 100644 --- a/signer/Cargo.toml +++ b/signer/Cargo.toml @@ -27,6 +27,8 @@ config = "0.14" futures.workspace = true hashbrown.workspace = true libp2p.workspace = true +metrics.workspace = true +metrics-exporter-prometheus.workspace = true p256k1.workspace = true prost.workspace = true rand.workspace = true diff --git a/signer/build.rs b/signer/build.rs index a1307cf29..68979e64d 100644 --- a/signer/build.rs +++ b/signer/build.rs @@ -1,7 +1,33 @@ fn main() { + set_up_build_info(); // compile_protos(); } +pub fn set_up_build_info() { + let output = std::process::Command::new("rustc") + .arg("--version") + .output() + .expect("Failed to execute rustc"); + + let version = String::from_utf8_lossy(&output.stdout); + + let git_hash = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) + .unwrap_or_default(); + + let env_abi = std::env::var("CARGO_CFG_TARGET_ENV").unwrap(); + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + + // We capture these variables in our binary and use them in the + // build_info metric. + println!("cargo:rustc-env=CARGO_CFG_TARGET_ENV={}", env_abi.trim()); + println!("cargo:rustc-env=CARGO_CFG_TARGET_ARCH={}", arch.trim()); + println!("cargo:rustc-env=GIT_COMMIT={}", git_hash.trim()); + println!("cargo:rustc-env=RUSTC_VERSION={}", version.trim()); +} + pub fn compile_protos() { let workingdir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .parent() diff --git a/signer/src/api/new_block.rs b/signer/src/api/new_block.rs index a72a991c8..78eaa91f3 100644 --- a/signer/src/api/new_block.rs +++ b/signer/src/api/new_block.rs @@ -22,6 +22,8 @@ use std::sync::OnceLock; use crate::context::Context; use crate::emily_client::EmilyInteract; use crate::error::Error; +use crate::metrics::Metrics; +use crate::metrics::STACKS_BLOCKCHAIN; use crate::stacks::events::CompletedDepositEvent; use crate::stacks::events::KeyRotationEvent; use crate::stacks::events::RegistryEvent; @@ -79,6 +81,12 @@ enum UpdateResult { #[tracing::instrument(skip_all, name = "new-block")] pub async fn new_block_handler(state: State>, body: String) -> StatusCode { tracing::debug!("received a new block event from stacks-core"); + metrics::counter!( + Metrics::BlocksObservedTotal, + "blockchain" => STACKS_BLOCKCHAIN, + ) + .increment(1); + let api = state.0; let registry_address = SBTC_REGISTRY_IDENTIFIER.get_or_init(|| { diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index 91fa94478..45ad39313 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -28,6 +28,8 @@ use crate::context::SbtcLimits; use crate::context::SignerEvent; use crate::emily_client::EmilyInteract; use crate::error::Error; +use crate::metrics::Metrics; +use crate::metrics::BITCOIN_BLOCKCHAIN; use crate::stacks::api::StacksInteract; use crate::stacks::api::TenureBlocks; use crate::storage; @@ -137,6 +139,11 @@ where match poll.await { Ok(Some(Ok(block_hash))) => { tracing::info!("observed new bitcoin block from stream"); + metrics::counter!( + Metrics::BlocksObservedTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + ) + .increment(1); let next_blocks = match self.next_blocks_to_process(block_hash).await { Ok(blocks) => blocks, @@ -203,17 +210,31 @@ impl BlockObserver { #[tracing::instrument(skip_all)] pub async fn load_requests(&self, requests: &[CreateDepositRequest]) -> Result<(), Error> { let mut deposit_requests = Vec::new(); + let bitcoin_client = self.context.get_bitcoin_client(); + for request in requests { let deposit = request - .validate(&self.context.get_bitcoin_client()) + .validate(&bitcoin_client) .await .inspect_err(|error| tracing::warn!(%error, "could not validate deposit request")); // We log the error above, so we just need to extract the // deposit now. - if let Ok(Some(deposit)) = deposit { - deposit_requests.push(deposit); - } + let deposit_status = match deposit { + Ok(Some(deposit)) => { + deposit_requests.push(deposit); + "success" + } + Ok(None) => "unconfirmed", + Err(_) => "failed", + }; + + metrics::counter!( + Metrics::DepositRequestsTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "status" => deposit_status, + ) + .increment(1); } self.store_deposit_requests(deposit_requests).await?; @@ -435,6 +456,13 @@ impl BlockObserver { for prevout in tx_info.to_inputs(&signer_script_pubkeys) { db.write_tx_prevout(&prevout).await?; + if prevout.prevout_type == model::TxPrevoutType::Deposit { + metrics::counter!( + Metrics::DepositsSweptTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + ) + .increment(1); + } } for output in tx_info.to_outputs(&signer_script_pubkeys) { diff --git a/signer/src/config/default.toml b/signer/src/config/default.toml index e2d2f7bc6..44dcb2dd9 100644 --- a/signer/src/config/default.toml +++ b/signer/src/config/default.toml @@ -184,6 +184,13 @@ dkg_max_duration = 120 # Environment: SIGNER_SIGNER__DKG_BEGIN_PAUSE # dkg_begin_pause = 10 +# When defined, this field sets the scrape endpoint as an IPv4 or IPv6 +# socket address for exporting metrics for Prometheus. +# +# Required: false +# Environment: SIGNER_SIGNER__PROMETHEUS_EXPORTER_ENDPOINT +# prometheus_exporter_endpoint = "[::]:9184" + # !! ============================================================================== # !! Stacks Event Observer Configuration # !! @@ -272,4 +279,4 @@ public_endpoints = [] # Default: false # Required: false # Environment: SIGNER_SIGNER__P2P__ENABLE_MDNS -enable_mdns = true \ No newline at end of file +enable_mdns = true diff --git a/signer/src/config/mod.rs b/signer/src/config/mod.rs index 81a60d3cd..d21969c68 100644 --- a/signer/src/config/mod.rs +++ b/signer/src/config/mod.rs @@ -209,6 +209,8 @@ pub struct SignerConfig { /// The postgres database endpoint #[serde(deserialize_with = "url_deserializer_single")] pub db_endpoint: Url, + /// The scrape endpoint for exporting metrics for Prometheus. + pub prometheus_exporter_endpoint: Option, /// The public keys of the signer sit during the bootstrapping phase of /// the signers. pub bootstrap_signing_set: Vec, @@ -492,6 +494,7 @@ mod tests { assert_eq!(settings.signer.bootstrap_signatures_required, 2); assert_eq!(settings.signer.bitcoin_block_horizon, 1500); assert_eq!(settings.signer.context_window, 1000); + assert!(settings.signer.prometheus_exporter_endpoint.is_none()); assert_eq!( settings.signer.bitcoin_presign_request_max_duration, Duration::from_secs(30) @@ -623,6 +626,32 @@ mod tests { assert_eq!(settings.signer.network, NetworkKind::Regtest); } + #[test] + fn prometheus_exporter_endpoint_with_environment() { + clear_env(); + + std::env::set_var("SIGNER_SIGNER__PROMETHEUS_EXPORTER_ENDPOINT", "[::]:9851"); + + let settings = Settings::new_from_default_config().unwrap(); + let endpoint = settings.signer.prometheus_exporter_endpoint.unwrap(); + + assert!(endpoint.ip().is_unspecified()); + assert!(endpoint.is_ipv6()); + assert_eq!(endpoint.port(), 9851); + + std::env::set_var( + "SIGNER_SIGNER__PROMETHEUS_EXPORTER_ENDPOINT", + "0.0.0.0:9852", + ); + + let settings = Settings::new_from_default_config().unwrap(); + let endpoint = settings.signer.prometheus_exporter_endpoint.unwrap(); + + assert!(endpoint.ip().is_unspecified()); + assert!(endpoint.is_ipv4()); + assert_eq!(endpoint.port(), 9852); + } + #[test] fn default_config_toml_loads_with_environment() { clear_env(); diff --git a/signer/src/lib.rs b/signer/src/lib.rs index 45b44ddb7..508d235f9 100644 --- a/signer/src/lib.rs +++ b/signer/src/lib.rs @@ -19,6 +19,7 @@ pub mod error; pub mod keys; pub mod logging; pub mod message; +pub mod metrics; pub mod network; pub mod proto; pub mod request_decider; @@ -53,7 +54,7 @@ const MAX_KEYS: u16 = 128; /// some "time". Right now this "time", the locktime, can only be /// denominated in bitcoin blocks. Once locktime number of blocks have been /// added to the blockchain after the deposit has been confirmed, the -/// depositer can reclaim the deposit transaction. Signers will not attempt +/// depositor can reclaim the deposit transaction. Signers will not attempt /// to sweep in the deposited funds if the number of blocks left is less /// than or equal to this value. /// @@ -76,3 +77,16 @@ pub const MAX_REORG_BLOCK_COUNT: i64 = 10; /// The maximum number of sweep transactions that the signers can confirm /// per block. pub const MAX_TX_PER_BITCOIN_BLOCK: i64 = 25; + +/// These are all build info variables. Many of them are set in build.rs. + +/// The name of the binary that is being run, +pub const PACKAGE_NAME: &str = env!("CARGO_PKG_NAME"); +/// The target environment ABI of the signer binary build. +pub const TARGET_ENV_ABI: &str = env!("CARGO_CFG_TARGET_ENV"); +/// The CPU target architecture of the signer binary build. +pub const TARGET_ARCH: &str = env!("CARGO_CFG_TARGET_ARCH"); +/// The version of rustc used to build the signer binary. +pub const RUSTC_VERSION: &str = env!("RUSTC_VERSION"); +/// The git sha that the binary was built from. +pub const GIT_COMMIT: &str = env!("GIT_COMMIT"); diff --git a/signer/src/main.rs b/signer/src/main.rs index a6126e209..910bf92dc 100644 --- a/signer/src/main.rs +++ b/signer/src/main.rs @@ -69,10 +69,19 @@ async fn main() -> Result<(), Box> { // Configure the binary's stdout/err output based on the provided output format. let pretty = matches!(args.output_format, Some(LogOutputFormat::Pretty)); - signer::logging::setup_logging("", pretty); + signer::logging::setup_logging("info,signer=debug", pretty); + + tracing::info!( + rust_version = signer::RUSTC_VERSION, + revision = signer::GIT_COMMIT, + arch = signer::TARGET_ARCH, + env_abi = signer::TARGET_ENV_ABI, + "starting the sBTC signer", + ); // Load the configuration file and/or environment variables. let settings = Settings::new(args.config)?; + signer::metrics::setup_metrics(settings.signer.prometheus_exporter_endpoint); // Open a connection to the signer db. let db = PgStore::connect(settings.signer.db_endpoint.as_str()).await?; diff --git a/signer/src/message.rs b/signer/src/message.rs index 7d7df6fdd..7aff05b98 100644 --- a/signer/src/message.rs +++ b/signer/src/message.rs @@ -5,6 +5,7 @@ use secp256k1::ecdsa::RecoverableSignature; use crate::bitcoin::utxo::Fees; use crate::bitcoin::validation::TxRequestIds; use crate::keys::PublicKey; +use crate::stacks::contracts::ContractCall; use crate::stacks::contracts::StacksTx; use crate::storage::model::BitcoinBlockHash; use crate::storage::model::StacksTxId; @@ -189,6 +190,20 @@ pub struct StacksTransactionSignRequest { pub txid: blockstack_lib::burnchains::Txid, } +impl StacksTransactionSignRequest { + /// Return the kind of transaction that that is being asked to be + /// signed. + pub fn tx_kind(&self) -> &'static str { + match &self.contract_tx { + StacksTx::ContractCall(ContractCall::CompleteDepositV1(_)) => "complete-deposit", + StacksTx::ContractCall(ContractCall::AcceptWithdrawalV1(_)) => "accept-withdrawal", + StacksTx::ContractCall(ContractCall::RejectWithdrawalV1(_)) => "reject-withdrawal", + StacksTx::ContractCall(ContractCall::RotateKeysV1(_)) => "rotate-keys", + StacksTx::SmartContract(_) => "smart-contract-deployment", + } + } +} + /// Represents a signature of a Stacks transaction. #[derive(Debug, Clone, PartialEq)] pub struct StacksTransactionSignature { diff --git a/signer/src/metrics.rs b/signer/src/metrics.rs new file mode 100644 index 000000000..a491b59d1 --- /dev/null +++ b/signer/src/metrics.rs @@ -0,0 +1,83 @@ +//! A module for setting up metrics in the APP +//! + +use std::net::SocketAddr; + +use metrics_exporter_prometheus::PrometheusBuilder; + +/// The buckets used for metric histograms +const METRIC_BUCKETS: [f64; 9] = [1e-4, 1e-3, 1e-2, 0.1, 0.5, 1.0, 5.0, 20.0, f64::INFINITY]; + +/// The quantiles to use when rendering histograms +const METRIC_QUANTILES: [f64; 8] = [0.0, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 1.0]; + +/// All metrics captured in this crate +#[derive(strum::IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +pub enum Metrics { + /// The metric for the total number of submitted transactions. + TransactionsSubmittedTotal, + /// The metric for the total number of deposit requests that have been + /// swept. + DepositsSweptTotal, + /// The metric for the total number of observed bitcoin or stacks + /// blocks. We use a label to distinguish ¡between the two. Note that + /// this only includes bitcoin blocks observed over the ZeroMQ + /// interface and stacks blocks observed from the event observer. + BlocksObservedTotal, + /// The number of deposit requests processed from Emily. This includes + /// duplicates. + DepositRequestsTotal, + /// The total number of signing rounds that have completed + /// successfully. This includes WSTS and "regular" multi-sig signing + /// rounds on stacks. We use a label to distinguish between the two. + SigningRoundsCompletedTotal, + /// The total number of tenures that this signer has served as + /// coordinator. + CoordinatorTenuresTotal, + /// The total number of sign requests received from the signer. + SignRequestsTotal, + /// The amount of time it took to complete a signing round in seconds. + /// This includes WSTS and "regular" multi-sig signing rounds on + /// stacks. We use a label to distinguish between the two. + SigningRoundDurationSeconds, + /// The amount of time, in seconds for running bitcoin or stacks + /// validation. + ValidationDurationSeconds, +} + +impl From for metrics::KeyName { + fn from(value: Metrics) -> Self { + metrics::KeyName::from_const_str(value.into()) + } +} + +/// Label for bitcoin blockchain based metrics +pub const BITCOIN_BLOCKCHAIN: &str = "bitcoin"; + +/// Label for stacks blockchain based metrics. +pub const STACKS_BLOCKCHAIN: &str = "stacks"; + +/// Set up a prometheus exporter for metrics. +pub fn setup_metrics(prometheus_exporter_endpoint: Option) { + if let Some(addr) = prometheus_exporter_endpoint { + PrometheusBuilder::new() + .with_http_listener(addr) + .add_global_label("app", crate::PACKAGE_NAME) + .set_buckets(&METRIC_BUCKETS) + .expect("received an empty slice of metric buckets") + .set_quantiles(&METRIC_QUANTILES) + .expect("received an empty slice of metric quantiles") + .install() + .expect("could not install the prometheus server"); + } + + metrics::gauge!( + "build_info", + "rust_version" => crate::RUSTC_VERSION, + "revision" => crate::GIT_COMMIT, + "arch" => crate::TARGET_ARCH, + "env_abi" => crate::TARGET_ENV_ABI, + ) + .set(1.0); +} diff --git a/signer/src/transaction_coordinator.rs b/signer/src/transaction_coordinator.rs index 2dff46e82..3c096948b 100644 --- a/signer/src/transaction_coordinator.rs +++ b/signer/src/transaction_coordinator.rs @@ -38,6 +38,9 @@ use crate::message::BitcoinPreSignRequest; use crate::message::Payload; use crate::message::SignerMessage; use crate::message::StacksTransactionSignRequest; +use crate::metrics::Metrics; +use crate::metrics::BITCOIN_BLOCKCHAIN; +use crate::metrics::STACKS_BLOCKCHAIN; use crate::network; use crate::signature::TaprootSignature; use crate::stacks::api::FeePriority; @@ -301,6 +304,7 @@ where } tracing::debug!("we are the coordinator, we may need to coordinate DKG"); + metrics::counter!(Metrics::CoordinatorTenuresTotal).increment(1); // If Self::get_signer_set_and_aggregate_key did not return an // aggregate key, then we know that we have not run DKG yet. Since // we are the coordinator, we should coordinate DKG. @@ -477,12 +481,37 @@ where Ok(()) }; + let instant = std::time::Instant::now(); + // Wait for the future to complete with a timeout - tokio::time::timeout(self.bitcoin_presign_request_max_duration, future) + let res = tokio::time::timeout(self.bitcoin_presign_request_max_duration, future) .await .map_err(|_| { Error::CoordinatorTimeout(self.bitcoin_presign_request_max_duration.as_secs()) - })? + }); + + let status = match &res { + Ok(Ok(_)) => "success", + Ok(Err(_)) => "failure", + Err(_) => "timeout", + }; + + metrics::histogram!( + Metrics::SigningRoundDurationSeconds, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep-presign", + "status" => status, + ) + .record(instant.elapsed()); + metrics::counter!( + Metrics::SignRequestsTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep-presign-broadcast", + "status" => status, + ) + .increment(1); + + res? } /// Construct and coordinate WSTS signing rounds for sBTC transactions on Bitcoin, @@ -635,9 +664,10 @@ where let process_request_fut = self.process_sign_request(sign_request, chain_tip, multi_tx, &wallet); - match process_request_fut.await { + let status = match process_request_fut.await { Ok(txid) => { - tracing::info!(%txid, "successfully submitted complete-deposit transaction") + tracing::info!(%txid, "successfully submitted complete-deposit transaction"); + "success" } Err(error) => { tracing::warn!( @@ -647,8 +677,16 @@ where "could not process the stacks sign request for a deposit" ); wallet.set_nonce(wallet.get_nonce().saturating_sub(1)); + "failure" } - } + }; + + metrics::counter!( + Metrics::TransactionsSubmittedTotal, + "blockchain" => STACKS_BLOCKCHAIN, + "status" => status, + ) + .increment(1); } Ok(()) @@ -703,11 +741,31 @@ where multi_tx: MultisigTx, wallet: &SignerWallet, ) -> Result { + let kind = sign_request.tx_kind(); + + let instant = std::time::Instant::now(); let tx = self .sign_stacks_transaction(sign_request, multi_tx, chain_tip, wallet) - .await?; + .await; + + let status = if tx.is_ok() { "success" } else { "failure" }; + + metrics::histogram!( + Metrics::SigningRoundDurationSeconds, + "blockchain" => STACKS_BLOCKCHAIN, + "kind" => kind, + "status" => status, + ) + .record(instant.elapsed()); + metrics::counter!( + Metrics::SigningRoundsCompletedTotal, + "blockchain" => STACKS_BLOCKCHAIN, + "kind" => kind, + "status" => status, + ) + .increment(1); - match self.context.get_stacks_client().submit_tx(&tx).await { + match self.context.get_stacks_client().submit_tx(&tx?).await { Ok(SubmitTxResponse::Acceptance(txid)) => Ok(txid.into()), Ok(SubmitTxResponse::Rejection(err)) => Err(err.into()), Err(err) => Err(err), @@ -867,7 +925,7 @@ where let msg = sighashes.signers.to_raw_hash().to_byte_array(); let txid = transaction.tx.compute_txid(); - + let instant = std::time::Instant::now(); let signature = self .coordinate_signing_round( bitcoin_chain_tip, @@ -878,6 +936,20 @@ where ) .await?; + metrics::histogram!( + Metrics::SigningRoundDurationSeconds, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep", + ) + .record(instant.elapsed()); + + metrics::counter!( + Metrics::SigningRoundsCompletedTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep", + ) + .increment(1); + let signer_witness = bitcoin::Witness::p2tr_key_spend(&signature.into()); let mut deposit_witness = Vec::new(); @@ -894,6 +966,7 @@ where ) .await?; + let instant = std::time::Instant::now(); let signature = self .coordinate_signing_round( bitcoin_chain_tip, @@ -904,6 +977,19 @@ where ) .await?; + metrics::histogram!( + Metrics::SigningRoundDurationSeconds, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep", + ) + .record(instant.elapsed()); + metrics::counter!( + Metrics::SigningRoundsCompletedTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep", + ) + .increment(1); + let witness = deposit.construct_witness_data(signature.into()); deposit_witness.push(witness); @@ -924,14 +1010,27 @@ where tracing::info!("broadcasting bitcoin transaction"); // Broadcast the transaction to the Bitcoin network. - self.context + let response = self + .context .get_bitcoin_client() .broadcast_transaction(&transaction.tx) - .await?; + .await; - tracing::info!("bitcoin transaction accepted by bitcoin-core"); + let status = if response.is_ok() { + tracing::info!("bitcoin transaction accepted by bitcoin-core"); + "success" + } else { + "failure" + }; + metrics::counter!(crate::metrics::Metrics::ValidationDurationSeconds).increment(1); + metrics::counter!( + Metrics::TransactionsSubmittedTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "status" => status, + ) + .increment(1); - Ok(()) + response } #[tracing::instrument(skip_all)] diff --git a/signer/src/transaction_signer.rs b/signer/src/transaction_signer.rs index e2dfaf35a..020435f29 100644 --- a/signer/src/transaction_signer.rs +++ b/signer/src/transaction_signer.rs @@ -24,6 +24,9 @@ use crate::keys::PublicKey; use crate::message; use crate::message::BitcoinPreSignAck; use crate::message::StacksTransactionSignRequest; +use crate::metrics::Metrics; +use crate::metrics::BITCOIN_BLOCKCHAIN; +use crate::metrics::STACKS_BLOCKCHAIN; use crate::network; use crate::stacks::contracts::AsContractCall as _; use crate::stacks::contracts::ContractCall; @@ -250,8 +253,32 @@ where } (message::Payload::BitcoinPreSignRequest(requests), _, _) => { - self.handle_bitcoin_pre_sign_request(requests, &msg.bitcoin_chain_tip) - .await?; + let instant = std::time::Instant::now(); + let pre_validation_status = self + .handle_bitcoin_pre_sign_request(requests, &msg.bitcoin_chain_tip) + .await; + + let status = if pre_validation_status.is_ok() { + "success" + } else { + "failure" + }; + metrics::histogram!( + Metrics::ValidationDurationSeconds, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep-presign", + "status" => status, + ) + .record(instant.elapsed()); + + metrics::counter!( + Metrics::SignRequestsTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep-presign", + "status" => status, + ) + .increment(1); + pre_validation_status?; } // Message types ignored by the transaction signer (message::Payload::StacksTransactionSignature(_), _, _) @@ -427,8 +454,25 @@ where bitcoin_chain_tip: &model::BitcoinBlockHash, origin_public_key: &PublicKey, ) -> Result<(), Error> { - self.assert_valid_stacks_tx_sign_request(request, bitcoin_chain_tip, origin_public_key) - .await?; + let instant = std::time::Instant::now(); + let validation_status = self + .assert_valid_stacks_tx_sign_request(request, bitcoin_chain_tip, origin_public_key) + .await; + + metrics::histogram!( + Metrics::ValidationDurationSeconds, + "blockchain" => STACKS_BLOCKCHAIN, + "kind" => request.tx_kind(), + ) + .record(instant.elapsed()); + metrics::counter!( + Metrics::SignRequestsTotal, + "blockchain" => STACKS_BLOCKCHAIN, + "kind" => request.tx_kind(), + "status" => if validation_status.is_ok() { "success" } else { "failed" }, + ) + .increment(1); + validation_status?; // We need to set the nonce in order to get the exact transaction // that we need to sign. @@ -621,7 +665,24 @@ where } let db = self.context.get_storage(); - Self::validate_bitcoin_sign_request(&db, &request.message).await?; + let sig_hash = &request.message; + let validation_outcome = Self::validate_bitcoin_sign_request(&db, sig_hash).await; + + let validation_status = match &validation_outcome { + Ok(()) => "success", + Err(Error::SigHashConversion(_)) => "improper-sighash", + Err(Error::UnknownSigHash(_)) => "unknown-sighash", + Err(Error::InvalidSigHash(_)) => "invalid-sighash", + Err(_) => "unexpected-failure", + }; + + metrics::counter!( + Metrics::SignRequestsTotal, + "blockchain" => BITCOIN_BLOCKCHAIN, + "kind" => "sweep", + "status" => validation_status, + ) + .increment(1); if !self.wsts_state_machines.contains_key(&msg.txid) { let (maybe_aggregate_key, _) = self