From 07b28b2d5de0defea28695b045383077e859bc1c Mon Sep 17 00:00:00 2001 From: sadhan Date: Mon, 5 Sep 2022 22:37:07 -0700 Subject: [PATCH 1/3] Add ability to run stress for a finite duration or count --- Cargo.lock | 97 +++++++++++++++---- crates/sui-benchmark/Cargo.toml | 4 + .../sui-benchmark/src/drivers/bench_driver.rs | 8 +- .../src/workloads/shared_counter.rs | 3 +- .../src/workloads/transfer_object.rs | 5 +- 5 files changed, 89 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d51f38ed62651..fc74ed15389d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,7 +195,7 @@ dependencies = [ "ark-serialize", "ark-std", "derivative", - "num-bigint", + "num-bigint 0.4.3", "num-traits 0.2.15", "paste", "rayon", @@ -219,7 +219,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" dependencies = [ - "num-bigint", + "num-bigint 0.4.3", "num-traits 0.2.15", "quote 1.0.21", "syn 1.0.99", @@ -3890,7 +3890,7 @@ dependencies = [ "dirs-next", "hex", "move-core-types", - "num-bigint", + "num-bigint 0.4.3", "once_cell", "serde 1.0.144", "sha2 0.9.9", @@ -3917,7 +3917,7 @@ dependencies = [ "move-ir-to-bytecode", "move-ir-types", "move-symbol-pool", - "num-bigint", + "num-bigint 0.4.3", "once_cell", "petgraph 0.5.1", "regex", @@ -3991,7 +3991,7 @@ dependencies = [ "log", "move-compiler", "move-model", - "num", + "num 0.4.0", "once_cell", "regex", "serde 1.0.144", @@ -4096,7 +4096,7 @@ dependencies = [ "move-disassembler", "move-ir-types", "move-symbol-pool", - "num", + "num 0.4.0", "once_cell", "regex", "serde 1.0.144", @@ -4164,7 +4164,7 @@ dependencies = [ "move-model", "move-prover-boogie-backend", "move-stackless-bytecode", - "num", + "num 0.4.0", "once_cell", "pretty", "rand 0.8.5", @@ -4192,7 +4192,7 @@ dependencies = [ "move-core-types", "move-model", "move-stackless-bytecode", - "num", + "num 0.4.0", "once_cell", "pretty", "rand 0.8.5", @@ -4249,7 +4249,7 @@ dependencies = [ "move-ir-to-bytecode", "move-model", "move-read-write-set-types", - "num", + "num 0.4.0", "once_cell", "paste", "petgraph 0.5.1", @@ -4270,7 +4270,7 @@ dependencies = [ "move-core-types", "move-model", "move-stackless-bytecode", - "num", + "num 0.4.0", "serde 1.0.144", ] @@ -4791,17 +4791,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-bigint 0.2.6", + "num-complex 0.2.4", + "num-integer", + "num-iter", + "num-rational 0.2.4", + "num-traits 0.2.15", +] + [[package]] name = "num" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" dependencies = [ - "num-bigint", - "num-complex", + "num-bigint 0.4.3", + "num-complex 0.4.2", "num-integer", "num-iter", - "num-rational", + "num-rational 0.4.1", + "num-traits 0.2.15", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.1.0", + "num-integer", "num-traits 0.2.15", ] @@ -4816,6 +4841,16 @@ dependencies = [ "num-traits 0.2.15", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg 1.1.0", + "num-traits 0.2.15", +] + [[package]] name = "num-complex" version = "0.4.2" @@ -4846,6 +4881,18 @@ dependencies = [ "num-traits 0.2.15", ] +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg 1.1.0", + "num-bigint 0.2.6", + "num-integer", + "num-traits 0.2.15", +] + [[package]] name = "num-rational" version = "0.4.1" @@ -4853,7 +4900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg 1.1.0", - "num-bigint", + "num-bigint 0.4.3", "num-integer", "num-traits 0.2.15", ] @@ -5201,6 +5248,17 @@ dependencies = [ "regex", ] +[[package]] +name = "parse_duration" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d" +dependencies = [ + "lazy_static 1.4.0", + "num 0.2.1", + "regex", +] + [[package]] name = "paste" version = "1.0.8" @@ -7433,6 +7491,7 @@ dependencies = [ "multiaddr", "node", "num_cpus", + "parse_duration", "prometheus", "rand 0.8.5", "rand_distr", @@ -10084,12 +10143,12 @@ dependencies = [ "nom 6.1.2", "nom 7.1.1", "normalize-line-endings", - "num", - "num-bigint", - "num-complex", + "num 0.4.0", + "num-bigint 0.4.3", + "num-complex 0.4.2", "num-integer", "num-iter", - "num-rational", + "num-rational 0.4.1", "num-traits 0.1.43", "num-traits 0.2.15", "num_cpus", @@ -10667,7 +10726,7 @@ dependencies = [ "mysten-util-mem-derive", "nom 7.1.1", "normalize-line-endings", - "num-bigint", + "num-bigint 0.4.3", "num-integer", "num-traits 0.2.15", "num_cpus", diff --git a/crates/sui-benchmark/Cargo.toml b/crates/sui-benchmark/Cargo.toml index c36f6f4f8c395..b7431fe37e252 100644 --- a/crates/sui-benchmark/Cargo.toml +++ b/crates/sui-benchmark/Cargo.toml @@ -31,7 +31,11 @@ rand = "0.8.5" base64 = "0.13.0" rand_distr = "0.4.3" indicatif = "0.17.0" +<<<<<<< HEAD duration-str = "0.4.0" +======= +parse_duration = "2.1.1" +>>>>>>> 61b0be6a9 (Add ability to run stress for a finite duration or count) bcs = "0.1.3" sui-core = { path = "../sui-core" } diff --git a/crates/sui-benchmark/src/drivers/bench_driver.rs b/crates/sui-benchmark/src/drivers/bench_driver.rs index 86ee6cf8835ed..d3d0c7662acae 100644 --- a/crates/sui-benchmark/src/drivers/bench_driver.rs +++ b/crates/sui-benchmark/src/drivers/bench_driver.rs @@ -110,10 +110,9 @@ enum NextOp { Retry(RetryType), } -async fn start_benchmark(pb: Arc) -> &'static Instant { +async fn print_and_start_benchmark() -> &'static Instant { static ONCE: OnceCell = OnceCell::const_new(); ONCE.get_or_init(|| async move { - pb.finish(); eprintln!("Starting benchmark!"); Instant::now() }) @@ -211,7 +210,6 @@ impl Driver<()> for BenchDriver { let stat_delay_micros = 1_000_000 * self.stat_collection_interval; let metrics = Arc::new(BenchMetrics::new(registry)); let barrier = Arc::new(Barrier::new(num_workers as usize)); - let pb = Arc::new(ProgressBar::new(num_workers)); eprintln!("Setting up workers..."); let progress = Arc::new(match run_duration { Interval::Count(count) => ProgressBar::new(count), @@ -222,7 +220,6 @@ impl Driver<()> for BenchDriver { let request_delay_micros = 1_000_000 / worker.target_qps; let mut free_pool = worker.payload; let progress = progress.clone(); - let pb = pb.clone(); let tx_cloned = tx.clone(); let cloned_barrier = barrier.clone(); let metrics_cloned = metrics.clone(); @@ -231,9 +228,8 @@ impl Driver<()> for BenchDriver { QuorumDriverHandler::new(aggregator.clone(), QuorumDriverMetrics::new_for_tests()); let qd = quorum_driver_handler.clone_quorum_driver(); let runner = tokio::spawn(async move { - pb.inc(1); cloned_barrier.wait().await; - let start_time = start_benchmark(pb).await; + let start_time = print_and_start_benchmark().await; let mut num_success = 0; let mut num_error = 0; let mut min_latency = Duration::MAX; diff --git a/crates/sui-benchmark/src/workloads/shared_counter.rs b/crates/sui-benchmark/src/workloads/shared_counter.rs index d2bc7668be574..aaad1e48da63e 100644 --- a/crates/sui-benchmark/src/workloads/shared_counter.rs +++ b/crates/sui-benchmark/src/workloads/shared_counter.rs @@ -19,6 +19,7 @@ use test_utils::messages::{make_counter_create_transaction, make_counter_increme use test_utils::{ messages::create_publish_move_package_transaction, transaction::parse_package_ref, }; +use tracing::log::info; pub struct SharedCounterTestPayload { package_ref: ObjectRef, @@ -152,7 +153,7 @@ impl Workload for SharedCounterWorkload { } } // create counters using gas objects we created above - eprintln!("Creating shared counters, this may take a while.."); + info!("Creating shared counters, this may take a while.."); let futures = counters_gas .into_iter() .map(|(sender, keypair, gas)| async move { diff --git a/crates/sui-benchmark/src/workloads/transfer_object.rs b/crates/sui-benchmark/src/workloads/transfer_object.rs index d1d085d624fa1..38b285f7db2db 100644 --- a/crates/sui-benchmark/src/workloads/transfer_object.rs +++ b/crates/sui-benchmark/src/workloads/transfer_object.rs @@ -16,6 +16,7 @@ use sui_types::{ }; use test_utils::messages::make_transfer_object_transaction; +use tracing::log::info; use super::workload::{ get_latest, transfer_sui_for_testing, Gas, Payload, Workload, WorkloadType, MAX_GAS_FOR_TESTING, @@ -128,7 +129,7 @@ impl Workload for TransferObjectWorkload { .choose(&mut rand::thread_rng()) .unwrap(); // create as many gas objects as there are number of transfer objects times number of accounts - eprintln!("Creating enough gas to transfer objects.."); + info!("Creating enough gas to transfer objects.."); let mut transfer_gas: Vec> = vec![]; for _i in 0..count { let mut account_transfer_gas = vec![]; @@ -148,7 +149,7 @@ impl Workload for TransferObjectWorkload { } transfer_gas.push(account_transfer_gas); } - eprintln!("Creating objects to transfer.."); + info!("Creating objects to transfer.."); // create transfer objects with 1 SUI value each let mut transfer_objects: Vec = vec![]; for _i in 0..count { From 0c7f64ad18468405001517597265639215593b47 Mon Sep 17 00:00:00 2001 From: sadhan Date: Mon, 5 Sep 2022 22:37:07 -0700 Subject: [PATCH 2/3] Add ability to run stress for a finite duration or count --- crates/sui-benchmark/src/workloads/transfer_object.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/sui-benchmark/src/workloads/transfer_object.rs b/crates/sui-benchmark/src/workloads/transfer_object.rs index 38b285f7db2db..aed24e379e6d8 100644 --- a/crates/sui-benchmark/src/workloads/transfer_object.rs +++ b/crates/sui-benchmark/src/workloads/transfer_object.rs @@ -131,9 +131,6 @@ impl Workload for TransferObjectWorkload { // create as many gas objects as there are number of transfer objects times number of accounts info!("Creating enough gas to transfer objects.."); let mut transfer_gas: Vec> = vec![]; - for _i in 0..count { - let mut account_transfer_gas = vec![]; - for (owner, _) in self.transfer_keypairs.iter() { if let Some((updated, minted)) = transfer_sui_for_testing( (primary_gas_ref, Owner::AddressOwner(self.test_gas_owner)), &self.test_gas_keypair, From 798d72ad2814463d232c362f71b9d103a1f95366 Mon Sep 17 00:00:00 2001 From: sadhan Date: Wed, 7 Sep 2022 23:21:58 -0700 Subject: [PATCH 3/3] Add ability to store benchmark results and compare with a previous run --- Cargo.lock | 137 ++++---- crates/sui-benchmark/Cargo.toml | 7 +- crates/sui-benchmark/src/bin/stress.rs | 48 ++- .../sui-benchmark/src/drivers/bench_driver.rs | 150 +++++---- crates/sui-benchmark/src/drivers/mod.rs | 304 +++++++++++++++++- .../src/workloads/shared_counter.rs | 3 +- .../src/workloads/transfer_object.rs | 8 +- .../unit_tests/authority_aggregator_tests.rs | 4 +- crates/workspace-hack/Cargo.toml | 10 +- 9 files changed, 507 insertions(+), 164 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc74ed15389d2..0bc8739fb8710 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,7 +195,7 @@ dependencies = [ "ark-serialize", "ark-std", "derivative", - "num-bigint 0.4.3", + "num-bigint", "num-traits 0.2.15", "paste", "rayon", @@ -219,7 +219,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" dependencies = [ - "num-bigint 0.4.3", + "num-bigint", "num-traits 0.2.15", "quote 1.0.21", "syn 1.0.99", @@ -1153,6 +1153,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "comfy-table" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85914173c2f558d61613bfbbf1911f14e630895087a7ed2fafc0f5319e1536e7" +dependencies = [ + "crossterm 0.25.0", + "strum", + "strum_macros", + "unicode-width", +] + [[package]] name = "config" version = "0.1.0" @@ -1291,6 +1303,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "criterion" version = "0.3.5" @@ -2325,6 +2346,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flexstr" version = "0.9.2" @@ -2798,7 +2829,11 @@ version = "7.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea9fe3952d32674a14e0975009a3547af9ea364995b5ec1add2e23c2ae523ab" dependencies = [ + "base64", "byteorder", + "crossbeam-channel", + "flate2", + "nom 7.1.1", "num-traits 0.2.15", ] @@ -3890,7 +3925,7 @@ dependencies = [ "dirs-next", "hex", "move-core-types", - "num-bigint 0.4.3", + "num-bigint", "once_cell", "serde 1.0.144", "sha2 0.9.9", @@ -3917,7 +3952,7 @@ dependencies = [ "move-ir-to-bytecode", "move-ir-types", "move-symbol-pool", - "num-bigint 0.4.3", + "num-bigint", "once_cell", "petgraph 0.5.1", "regex", @@ -3991,7 +4026,7 @@ dependencies = [ "log", "move-compiler", "move-model", - "num 0.4.0", + "num", "once_cell", "regex", "serde 1.0.144", @@ -4096,7 +4131,7 @@ dependencies = [ "move-disassembler", "move-ir-types", "move-symbol-pool", - "num 0.4.0", + "num", "once_cell", "regex", "serde 1.0.144", @@ -4164,7 +4199,7 @@ dependencies = [ "move-model", "move-prover-boogie-backend", "move-stackless-bytecode", - "num 0.4.0", + "num", "once_cell", "pretty", "rand 0.8.5", @@ -4192,7 +4227,7 @@ dependencies = [ "move-core-types", "move-model", "move-stackless-bytecode", - "num 0.4.0", + "num", "once_cell", "pretty", "rand 0.8.5", @@ -4249,7 +4284,7 @@ dependencies = [ "move-ir-to-bytecode", "move-model", "move-read-write-set-types", - "num 0.4.0", + "num", "once_cell", "paste", "petgraph 0.5.1", @@ -4270,7 +4305,7 @@ dependencies = [ "move-core-types", "move-model", "move-stackless-bytecode", - "num 0.4.0", + "num", "serde 1.0.144", ] @@ -4791,42 +4826,17 @@ dependencies = [ "winapi", ] -[[package]] -name = "num" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" -dependencies = [ - "num-bigint 0.2.6", - "num-complex 0.2.4", - "num-integer", - "num-iter", - "num-rational 0.2.4", - "num-traits 0.2.15", -] - [[package]] name = "num" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" dependencies = [ - "num-bigint 0.4.3", - "num-complex 0.4.2", + "num-bigint", + "num-complex", "num-integer", "num-iter", - "num-rational 0.4.1", - "num-traits 0.2.15", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.1.0", - "num-integer", + "num-rational", "num-traits 0.2.15", ] @@ -4841,16 +4851,6 @@ dependencies = [ "num-traits 0.2.15", ] -[[package]] -name = "num-complex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" -dependencies = [ - "autocfg 1.1.0", - "num-traits 0.2.15", -] - [[package]] name = "num-complex" version = "0.4.2" @@ -4881,18 +4881,6 @@ dependencies = [ "num-traits 0.2.15", ] -[[package]] -name = "num-rational" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" -dependencies = [ - "autocfg 1.1.0", - "num-bigint 0.2.6", - "num-integer", - "num-traits 0.2.15", -] - [[package]] name = "num-rational" version = "0.4.1" @@ -4900,7 +4888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg 1.1.0", - "num-bigint 0.4.3", + "num-bigint", "num-integer", "num-traits 0.2.15", ] @@ -5248,17 +5236,6 @@ dependencies = [ "regex", ] -[[package]] -name = "parse_duration" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d" -dependencies = [ - "lazy_static 1.4.0", - "num 0.2.1", - "regex", -] - [[package]] name = "paste" version = "1.0.8" @@ -7481,9 +7458,11 @@ dependencies = [ "base64", "bcs", "clap 3.2.17", + "comfy-table", "crossterm 0.23.2", "duration-str", "futures", + "hdrhistogram", "indicatif", "jemalloc-ctl", "jemallocator", @@ -7491,7 +7470,6 @@ dependencies = [ "multiaddr", "node", "num_cpus", - "parse_duration", "prometheus", "rand 0.8.5", "rand_distr", @@ -9853,6 +9831,7 @@ dependencies = [ "colored", "colored-diff", "combine", + "comfy-table", "config 0.1.0", "config 0.11.0", "consensus", @@ -9865,6 +9844,7 @@ dependencies = [ "cpufeatures", "crc", "crc-catalog", + "crc32fast", "criterion", "criterion-plot", "crossbeam", @@ -9954,6 +9934,7 @@ dependencies = [ "fiat-crypto", "fixedbitset 0.2.0", "fixedbitset 0.4.1", + "flate2", "flexstr", "float-cmp", "flume", @@ -10143,12 +10124,12 @@ dependencies = [ "nom 6.1.2", "nom 7.1.1", "normalize-line-endings", - "num 0.4.0", - "num-bigint 0.4.3", - "num-complex 0.4.2", + "num", + "num-bigint", + "num-complex", "num-integer", "num-iter", - "num-rational 0.4.1", + "num-rational", "num-traits 0.1.43", "num-traits 0.2.15", "num_cpus", @@ -10726,7 +10707,7 @@ dependencies = [ "mysten-util-mem-derive", "nom 7.1.1", "normalize-line-endings", - "num-bigint 0.4.3", + "num-bigint", "num-integer", "num-traits 0.2.15", "num_cpus", diff --git a/crates/sui-benchmark/Cargo.toml b/crates/sui-benchmark/Cargo.toml index b7431fe37e252..6e168b721a834 100644 --- a/crates/sui-benchmark/Cargo.toml +++ b/crates/sui-benchmark/Cargo.toml @@ -31,12 +31,9 @@ rand = "0.8.5" base64 = "0.13.0" rand_distr = "0.4.3" indicatif = "0.17.0" -<<<<<<< HEAD duration-str = "0.4.0" -======= -parse_duration = "2.1.1" ->>>>>>> 61b0be6a9 (Add ability to run stress for a finite duration or count) - +hdrhistogram = "7.5.1" +comfy-table = "6.1.0" bcs = "0.1.3" sui-core = { path = "../sui-core" } sui-config = { path = "../sui-config" } diff --git a/crates/sui-benchmark/src/bin/stress.rs b/crates/sui-benchmark/src/bin/stress.rs index eea6c2f200c99..b1728802c0d95 100644 --- a/crates/sui-benchmark/src/bin/stress.rs +++ b/crates/sui-benchmark/src/bin/stress.rs @@ -14,6 +14,8 @@ use std::time::Duration; use strum_macros::EnumString; use sui_benchmark::drivers::bench_driver::BenchDriver; use sui_benchmark::drivers::driver::Driver; +use sui_benchmark::drivers::BenchmarkCmp; +use sui_benchmark::drivers::BenchmarkStats; use sui_benchmark::drivers::Interval; use sui_benchmark::workloads::shared_counter::SharedCounterWorkload; use sui_benchmark::workloads::transfer_object::TransferObjectWorkload; @@ -125,6 +127,12 @@ struct Opts { /// "10000" #[clap(long, global = true, default_value = "unbounded")] pub run_duration: Interval, + /// Path where benchmark stats is stored + #[clap(long, default_value = "/tmp/bench_result", global = true)] + pub benchmark_stats_path: String, + /// Path where previous benchmark stats is stored to use for comparison + #[clap(long, default_value = "", global = true)] + pub compare_with: String, } #[derive(Debug, Clone, Parser, Eq, PartialEq, EnumString)] @@ -322,10 +330,10 @@ fn make_transfer_object_workload( /// --num-server-threads 10 \ /// --num-transfer-accounts 2 \ /// bench \ -/// --target-qps 20 \ +/// --target-qps 100 \ /// --in-flight-ratio 2 \ -/// --shared-counter 10 \ -/// --transfer-object 10``` +/// --shared-counter 50 \ +/// --transfer-object 50``` /// To point the traffic to an already running cluster, /// use it something like: /// ```cargo run --release --package sui-benchmark --bin stress -- --num-client-threads 12 \ @@ -334,10 +342,10 @@ fn make_transfer_object_workload( /// --primary-gas-id 0x59931dcac57ba20d75321acaf55e8eb5a2c47e9f \ /// --gateway-config-path /tmp/gateway.yaml \ /// --keystore-path /tmp/sui.keystore bench \ -/// --target-qps 1 \ +/// --target-qps 100 \ /// --in-flight-ratio 2 \ -/// --shared-counter 10 \ -/// --transfer-object 10``` +/// --shared-counter 50 \ +/// --transfer-object 50``` #[tokio::main] async fn main() -> Result<()> { let mut config = telemetry_subscribers::TelemetryConfig::new("stress"); @@ -501,6 +509,8 @@ async fn main() -> Result<()> { .worker_threads(opts.num_client_threads as usize) .build() .unwrap(); + let prev_benchmark_stats_path = opts.compare_with.clone(); + let curr_benchmark_stats_path = opts.benchmark_stats_path.clone(); let handle = std::thread::spawn(move || { client_runtime.block_on(async move { let committee = GatewayState::make_committee(&gateway_config).unwrap(); @@ -584,7 +594,7 @@ async fn main() -> Result<()> { workloads }; let interval = opts.run_duration; - // We only show the progress in stderr + // We only show continuous progress in stderr // if benchmark is running in unbounded mode, // otherwise summarized benchmark results are // published in the end @@ -601,6 +611,28 @@ async fn main() -> Result<()> { if let Err(err) = joined { Err(anyhow!("Failed to join client runtime: {:?}", err)) } else { - joined.unwrap() + let stats: BenchmarkStats = joined.unwrap().unwrap(); + let table = stats.to_table(); + eprintln!("Benchmark Report:"); + eprintln!("{}", table); + if !prev_benchmark_stats_path.is_empty() { + let data = std::fs::read_to_string(&prev_benchmark_stats_path)?; + let prev_stats: BenchmarkStats = serde_json::from_str(&data)?; + let cmp = BenchmarkCmp { + new: &stats, + old: &prev_stats, + }; + let cmp_table = cmp.to_table(); + eprintln!( + "Benchmark Comparison Report[{}]:", + prev_benchmark_stats_path + ); + eprintln!("{}", cmp_table); + } + if !curr_benchmark_stats_path.is_empty() { + let serialized = serde_json::to_string(&stats)?; + std::fs::write(curr_benchmark_stats_path, serialized)?; + } + Ok(()) } } diff --git a/crates/sui-benchmark/src/drivers/bench_driver.rs b/crates/sui-benchmark/src/drivers/bench_driver.rs index d3d0c7662acae..f01d6f5502ee9 100644 --- a/crates/sui-benchmark/src/drivers/bench_driver.rs +++ b/crates/sui-benchmark/src/drivers/bench_driver.rs @@ -8,6 +8,7 @@ use futures::future::BoxFuture; use futures::FutureExt; use futures::{stream::FuturesUnordered, StreamExt}; use indicatif::ProgressBar; +use indicatif::ProgressStyle; use prometheus::register_gauge_vec_with_registry; use prometheus::register_histogram_vec_with_registry; use prometheus::register_int_counter_vec_with_registry; @@ -19,6 +20,7 @@ use sui_core::authority_aggregator::AuthorityAggregator; use tokio::sync::OnceCell; use crate::drivers::driver::Driver; +use crate::drivers::HistogramWrapper; use crate::workloads::workload::Payload; use crate::workloads::workload::WorkloadInfo; use std::collections::{BTreeMap, VecDeque}; @@ -36,6 +38,7 @@ use tokio::time; use tokio::time::Instant; use tracing::{debug, error}; +use super::BenchmarkStats; use super::Interval; pub struct BenchMetrics { pub num_success: IntCounterVec, @@ -94,14 +97,10 @@ impl BenchMetrics { struct Stats { pub id: usize, - pub num_success: u64, - pub num_error: u64, pub num_no_gas: u64, pub num_submitted: u64, pub num_in_flight: u64, - pub min_latency: Duration, - pub max_latency: Duration, - pub duration: Duration, + pub bench_stats: BenchmarkStats, } type RetryType = Box<(TransactionEnvelope, Box)>; @@ -146,7 +145,7 @@ impl BenchDriver { Interval::Count(count) => { progress_bar.inc(1); if progress_bar.position() >= count { - progress_bar.finish(); + progress_bar.finish_and_clear(); } } Interval::Time(Duration::MAX) => progress_bar.inc(1), @@ -154,7 +153,7 @@ impl BenchDriver { let elapsed_secs = (Instant::now() - start_time).as_secs(); progress_bar.set_position(std::cmp::min(duration.as_secs(), elapsed_secs)); if progress_bar.position() >= duration.as_secs() { - progress_bar.finish(); + progress_bar.finish_and_clear(); } } } @@ -188,7 +187,7 @@ impl BenchDriver { } #[async_trait] -impl Driver<()> for BenchDriver { +impl Driver for BenchDriver { async fn run( &self, workloads: Vec, @@ -196,7 +195,7 @@ impl Driver<()> for BenchDriver { registry: &Registry, show_progress: bool, run_duration: Interval, - ) -> Result<(), anyhow::Error> { + ) -> Result { let mut tasks = Vec::new(); let (tx, mut rx) = tokio::sync::mpsc::channel(100); let mut bench_workers = vec![]; @@ -212,9 +211,17 @@ impl Driver<()> for BenchDriver { let barrier = Arc::new(Barrier::new(num_workers as usize)); eprintln!("Setting up workers..."); let progress = Arc::new(match run_duration { - Interval::Count(count) => ProgressBar::new(count), + Interval::Count(count) => ProgressBar::new(count) + .with_prefix("Running benchmark(count):") + .with_style( + ProgressStyle::with_template("{prefix}: {wide_bar} {pos}/{len}").unwrap(), + ), Interval::Time(Duration::MAX) => ProgressBar::hidden(), - Interval::Time(duration) => ProgressBar::new(duration.as_secs()), + Interval::Time(duration) => ProgressBar::new(duration.as_secs()) + .with_prefix("Running benchmark(duration):") + .with_style( + ProgressStyle::with_template("{prefix}: {wide_bar} {pos}/{len}").unwrap(), + ), }); for (i, worker) in bench_workers.into_iter().enumerate() { let request_delay_micros = 1_000_000 / worker.target_qps; @@ -232,11 +239,11 @@ impl Driver<()> for BenchDriver { let start_time = print_and_start_benchmark().await; let mut num_success = 0; let mut num_error = 0; - let mut min_latency = Duration::MAX; - let mut max_latency = Duration::ZERO; let mut num_no_gas = 0; let mut num_in_flight: u64 = 0; let mut num_submitted = 0; + let mut latency_histogram = + hdrhistogram::Histogram::::new_with_max(100000, 2).unwrap(); let mut request_interval = time::interval(Duration::from_micros(request_delay_micros)); request_interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); @@ -244,21 +251,25 @@ impl Driver<()> for BenchDriver { let mut futures: FuturesUnordered> = FuturesUnordered::new(); let mut retry_queue: VecDeque = VecDeque::new(); - + let mut stat_start_time: Instant = Instant::now(); loop { tokio::select! { + _ = tokio::signal::ctrl_c() => { + break; + } _ = stat_interval.tick() => { if tx_cloned .try_send(Stats { id: i as usize, - num_success, - num_error, - min_latency, - max_latency, num_no_gas, num_in_flight, num_submitted, - duration: Duration::from_micros(stat_delay_micros), + bench_stats: BenchmarkStats { + duration: stat_start_time.elapsed(), + num_error, + num_success, + latency_ms: HistogramWrapper {histogram: latency_histogram.clone()}, + }, }) .is_err() { @@ -268,8 +279,8 @@ impl Driver<()> for BenchDriver { num_error = 0; num_no_gas = 0; num_submitted = 0; - min_latency = Duration::MAX; - max_latency = Duration::ZERO; + stat_start_time = Instant::now(); + latency_histogram.reset(); } _ = request_interval.tick() => { @@ -380,12 +391,7 @@ impl Driver<()> for BenchDriver { num_success += 1; num_in_flight -= 1; free_pool.push(new_payload); - if latency > max_latency { - max_latency = latency; - } - if latency < min_latency { - min_latency = latency; - } + latency_histogram.record(latency.as_millis().try_into().unwrap()).unwrap(); BenchDriver::update_progress(*start_time, run_duration, progress.clone()); if progress.is_finished() { break; @@ -399,51 +405,74 @@ impl Driver<()> for BenchDriver { } } } + // send stats one last time + if tx_cloned + .try_send(Stats { + id: i as usize, + num_no_gas, + num_in_flight, + num_submitted, + bench_stats: BenchmarkStats { + duration: stat_start_time.elapsed(), + num_error, + num_success, + latency_ms: HistogramWrapper { + histogram: latency_histogram, + }, + }, + }) + .is_err() + { + debug!("Failed to update stat!"); + } }); tasks.push(runner); } - tasks.push(tokio::spawn(async move { + let stat_task = tokio::spawn(async move { + let mut benchmark_stat = BenchmarkStats { + duration: Duration::ZERO, + num_error: 0, + num_success: 0, + latency_ms: HistogramWrapper { + histogram: hdrhistogram::Histogram::::new_with_max(100000, 2).unwrap(), + }, + }; let mut stat_collection: BTreeMap = BTreeMap::new(); let mut counter = 0; - let mut stat = "".to_string(); - while let Some(s @ Stats { - id, - num_success: _, - num_error: _, - min_latency: _, - max_latency: _, - num_no_gas: _, - num_in_flight: _, - num_submitted: _, - duration - }) = rx.recv().await { - stat_collection.insert(id, s); + let mut stat; + let start = Instant::now(); + while let Some( + sample_stat @ Stats { + id, + num_no_gas: _, + num_in_flight: _, + num_submitted: _, + bench_stats: _, + }, + ) = rx.recv().await + { + benchmark_stat.update(start.elapsed(), &sample_stat.bench_stats); + stat_collection.insert(id, sample_stat); let mut total_qps: f32 = 0.0; let mut num_success: u64 = 0; let mut num_error: u64 = 0; - let mut min_latency: Duration = Duration::MAX; - let mut max_latency: Duration = Duration::ZERO; + let mut latency_histogram = + hdrhistogram::Histogram::::new_with_max(100000, 2).unwrap(); let mut num_in_flight: u64 = 0; let mut num_submitted: u64 = 0; let mut num_no_gas = 0; for (_, v) in stat_collection.iter() { - total_qps += v.num_success as f32 / duration.as_secs() as f32; - num_success += v.num_success; - num_error += v.num_error; + total_qps += + v.bench_stats.num_success as f32 / v.bench_stats.duration.as_secs() as f32; + num_success += v.bench_stats.num_success; + num_error += v.bench_stats.num_error; num_no_gas += v.num_no_gas; num_submitted += v.num_submitted; num_in_flight += v.num_in_flight; - min_latency = if v.min_latency < min_latency { - v.min_latency - } else { - min_latency - }; - max_latency = if v.max_latency > max_latency { - v.max_latency - } else { - max_latency - }; + latency_histogram + .add(&v.bench_stats.latency_ms.histogram) + .unwrap(); } let denom = num_success + num_error; let _error_rate = if denom > 0 { @@ -453,16 +482,17 @@ impl Driver<()> for BenchDriver { }; counter += 1; if counter % num_workers == 0 { - stat = format!("Throughput = {}, min_latency_ms = {}, max_latency_ms = {}, num_success = {}, num_error = {}, no_gas = {}, submitted = {}, in_flight = {}", total_qps, min_latency.as_millis(), max_latency.as_millis(), num_success, num_error, num_no_gas, num_submitted, num_in_flight); + stat = format!("Throughput = {}, latency_ms(min/p50/p99/max) = {}/{}/{}/{}, num_success = {}, num_error = {}, no_gas = {}, submitted = {}, in_flight = {}", total_qps, latency_histogram.min(), latency_histogram.value_at_quantile(0.5), latency_histogram.value_at_quantile(0.99), latency_histogram.max(), num_success, num_error, num_no_gas, num_submitted, num_in_flight); if show_progress { eprintln!("{}", stat); } } } - eprintln!("{}", stat); - })); + benchmark_stat + }); drop(tx); let _res: Vec<_> = try_join_all(tasks).await.unwrap().into_iter().collect(); - Ok(()) + let benchmark_stat = stat_task.await.unwrap(); + Ok(benchmark_stat) } } diff --git a/crates/sui-benchmark/src/drivers/mod.rs b/crates/sui-benchmark/src/drivers/mod.rs index 64f35d6479192..625e0c1fa2ae2 100644 --- a/crates/sui-benchmark/src/drivers/mod.rs +++ b/crates/sui-benchmark/src/drivers/mod.rs @@ -1,15 +1,15 @@ // Copyright (c) 2022, Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::str::FromStr; - use duration_str::parse; -use serde::{Deserialize, Serialize}; +use std::{str::FromStr, time::Duration}; pub mod bench_driver; pub mod driver; +use comfy_table::{Cell, Color, ContentArrangement, Row, Table}; +use hdrhistogram::{serialization::Serializer, Histogram}; -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] pub enum Interval { Count(u64), Time(tokio::time::Duration), @@ -36,3 +36,299 @@ impl FromStr for Interval { } } } + +// wrapper which implements serde +#[allow(dead_code)] +pub struct HistogramWrapper { + histogram: Histogram, +} + +impl serde::Serialize for HistogramWrapper { + fn serialize(&self, serializer: S) -> Result { + let mut vec = Vec::new(); + hdrhistogram::serialization::V2Serializer::new() + .serialize(&self.histogram, &mut vec) + .map_err(|e| serde::ser::Error::custom(e.to_string()))?; + serializer.serialize_bytes(&vec) + } +} + +impl<'de> serde::Deserialize<'de> for HistogramWrapper { + fn deserialize>(deserializer: D) -> Result { + let vec: Vec = serde::Deserialize::deserialize(deserializer)?; + let histogram: Histogram = hdrhistogram::serialization::Deserializer::new() + .deserialize(&mut &vec[..]) + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + Ok(HistogramWrapper { histogram }) + } +} + +/// Stores the final statistics of the test run. +#[derive(serde::Serialize, serde::Deserialize)] +pub struct BenchmarkStats { + pub duration: Duration, + pub num_error: u64, + pub num_success: u64, + pub latency_ms: HistogramWrapper, +} + +impl BenchmarkStats { + pub fn update(&mut self, duration: Duration, sample_stat: &BenchmarkStats) { + self.duration = duration; + self.num_error += sample_stat.num_error; + self.num_success += sample_stat.num_success; + self.latency_ms + .histogram + .add(&sample_stat.latency_ms.histogram) + .unwrap(); + } + pub fn to_table(&self) -> Table { + let mut table = Table::new(); + table + .set_content_arrangement(ContentArrangement::Dynamic) + .set_width(200) + .set_header(vec![ + "duration(s)", + "tps", + "error%", + "min", + "p25", + "p50", + "p75", + "p90", + "p99", + "p99.9", + "max", + ]); + let mut row = Row::new(); + row.add_cell(Cell::new(self.duration.as_secs())); + row.add_cell(Cell::new(self.num_success / self.duration.as_secs())); + row.add_cell(Cell::new( + self.num_error / (self.num_error + self.num_success), + )); + row.add_cell(Cell::new(self.latency_ms.histogram.min())); + row.add_cell(Cell::new(self.latency_ms.histogram.value_at_quantile(0.25))); + row.add_cell(Cell::new(self.latency_ms.histogram.value_at_quantile(0.5))); + row.add_cell(Cell::new(self.latency_ms.histogram.value_at_quantile(0.75))); + row.add_cell(Cell::new(self.latency_ms.histogram.value_at_quantile(0.9))); + row.add_cell(Cell::new(self.latency_ms.histogram.value_at_quantile(0.99))); + row.add_cell(Cell::new( + self.latency_ms.histogram.value_at_quantile(0.999), + )); + row.add_cell(Cell::new(self.latency_ms.histogram.max())); + table.add_row(row); + table + } +} + +/// A comparison between an old and a new benchmark. +/// All differences are reported in terms of measuring improvements +/// (negative) or regressions (positive). That is, if an old benchmark +/// is slower than a new benchmark, then the difference is negative. +/// Conversely, if an old benchmark is faster than a new benchmark, +/// then the difference is positive. +#[derive(Clone, Debug)] +pub struct Comparison { + pub name: String, + pub old_value: String, + pub new_value: String, + pub diff: i64, + pub diff_ratio: f64, + pub speedup: f64, +} + +pub struct BenchmarkCmp<'a> { + pub new: &'a BenchmarkStats, + pub old: &'a BenchmarkStats, +} + +impl BenchmarkCmp<'_> { + pub fn to_table(&self) -> Table { + let mut table = Table::new(); + table.set_header(vec!["name", "old", "new", "diff", "diff_ratio", "speedup"]); + for cmp in self.all_cmps() { + let diff_ratio = format!("{:.2}%", cmp.diff_ratio * 100f64); + let speedup = format!("{:.2}x", cmp.speedup); + let diff = format!("{:.2}", cmp.diff); + let mut row = Row::new(); + row.add_cell(Cell::new(cmp.name)); + row.add_cell(Cell::new(cmp.old_value)); + row.add_cell(Cell::new(cmp.new_value)); + if cmp.speedup >= 1.0 { + row.add_cell(Cell::new(diff).fg(Color::Green)); + row.add_cell(Cell::new(diff_ratio).fg(Color::Green)); + row.add_cell(Cell::new(speedup).fg(Color::Green)); + } else { + row.add_cell(Cell::new(diff).fg(Color::Red)); + row.add_cell(Cell::new(diff_ratio).fg(Color::Red)); + row.add_cell(Cell::new(speedup).fg(Color::Red)); + } + table.add_row(row); + } + table + } + pub fn all_cmps(&self) -> Vec { + vec![ + self.cmp_tps(), + self.cmp_error_rate(), + self.cmp_min_latency(), + self.cmp_p25_latency(), + self.cmp_p50_latency(), + self.cmp_p75_latency(), + self.cmp_p90_latency(), + self.cmp_p99_latency(), + self.cmp_p999_latency(), + self.cmp_max_latency(), + ] + } + pub fn cmp_tps(&self) -> Comparison { + let old_tps = self.old.num_success / self.old.duration.as_secs(); + let new_tps = self.new.num_success / self.new.duration.as_secs(); + let diff = new_tps as i64 - old_tps as i64; + let diff_ratio = diff as f64 / old_tps as f64; + let speedup = 1.0 + diff_ratio; + Comparison { + name: "tps".to_string(), + old_value: format!("{:.2}", old_tps), + new_value: format!("{:.2}", new_tps), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_error_rate(&self) -> Comparison { + let old_error_rate = self.old.num_error / (self.old.num_error + self.old.num_success); + let new_error_rate = self.new.num_error / (self.new.num_error + self.new.num_success); + let diff = new_error_rate as i64 - old_error_rate as i64; + let diff_ratio = diff as f64 / old_error_rate as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "error_rate".to_string(), + old_value: format!("{:.2}", old_error_rate), + new_value: format!("{:.2}", new_error_rate), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_min_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.min() as i64; + let new = self.new.latency_ms.histogram.min() as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "min_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_p25_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.value_at_quantile(0.25) as i64; + let new = self.new.latency_ms.histogram.value_at_quantile(0.25) as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "p25_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_p50_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.value_at_quantile(0.5) as i64; + let new = self.new.latency_ms.histogram.value_at_quantile(0.5) as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "p50_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_p75_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.value_at_quantile(0.75) as i64; + let new = self.new.latency_ms.histogram.value_at_quantile(0.75) as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "p75_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_p90_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.value_at_quantile(0.9) as i64; + let new = self.new.latency_ms.histogram.value_at_quantile(0.9) as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "p90_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_p99_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.value_at_quantile(0.99) as i64; + let new = self.new.latency_ms.histogram.value_at_quantile(0.99) as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "p99_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_p999_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.value_at_quantile(0.999) as i64; + let new = self.new.latency_ms.histogram.value_at_quantile(0.999) as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "p999_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } + pub fn cmp_max_latency(&self) -> Comparison { + let old = self.old.latency_ms.histogram.max() as i64; + let new = self.new.latency_ms.histogram.max() as i64; + let diff = new - old; + let diff_ratio = diff as f64 / old as f64; + let speedup = 1.0 / (1.0 + diff_ratio); + Comparison { + name: "max_latency".to_string(), + old_value: format!("{:.2}", old), + new_value: format!("{:.2}", new), + diff, + diff_ratio, + speedup, + } + } +} diff --git a/crates/sui-benchmark/src/workloads/shared_counter.rs b/crates/sui-benchmark/src/workloads/shared_counter.rs index aaad1e48da63e..d2bc7668be574 100644 --- a/crates/sui-benchmark/src/workloads/shared_counter.rs +++ b/crates/sui-benchmark/src/workloads/shared_counter.rs @@ -19,7 +19,6 @@ use test_utils::messages::{make_counter_create_transaction, make_counter_increme use test_utils::{ messages::create_publish_move_package_transaction, transaction::parse_package_ref, }; -use tracing::log::info; pub struct SharedCounterTestPayload { package_ref: ObjectRef, @@ -153,7 +152,7 @@ impl Workload for SharedCounterWorkload { } } // create counters using gas objects we created above - info!("Creating shared counters, this may take a while.."); + eprintln!("Creating shared counters, this may take a while.."); let futures = counters_gas .into_iter() .map(|(sender, keypair, gas)| async move { diff --git a/crates/sui-benchmark/src/workloads/transfer_object.rs b/crates/sui-benchmark/src/workloads/transfer_object.rs index aed24e379e6d8..d1d085d624fa1 100644 --- a/crates/sui-benchmark/src/workloads/transfer_object.rs +++ b/crates/sui-benchmark/src/workloads/transfer_object.rs @@ -16,7 +16,6 @@ use sui_types::{ }; use test_utils::messages::make_transfer_object_transaction; -use tracing::log::info; use super::workload::{ get_latest, transfer_sui_for_testing, Gas, Payload, Workload, WorkloadType, MAX_GAS_FOR_TESTING, @@ -129,8 +128,11 @@ impl Workload for TransferObjectWorkload { .choose(&mut rand::thread_rng()) .unwrap(); // create as many gas objects as there are number of transfer objects times number of accounts - info!("Creating enough gas to transfer objects.."); + eprintln!("Creating enough gas to transfer objects.."); let mut transfer_gas: Vec> = vec![]; + for _i in 0..count { + let mut account_transfer_gas = vec![]; + for (owner, _) in self.transfer_keypairs.iter() { if let Some((updated, minted)) = transfer_sui_for_testing( (primary_gas_ref, Owner::AddressOwner(self.test_gas_owner)), &self.test_gas_keypair, @@ -146,7 +148,7 @@ impl Workload for TransferObjectWorkload { } transfer_gas.push(account_transfer_gas); } - info!("Creating objects to transfer.."); + eprintln!("Creating objects to transfer.."); // create transfer objects with 1 SUI value each let mut transfer_objects: Vec = vec![]; for _i in 0..count { diff --git a/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs b/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs index b6137aea971fb..2263764ac5b33 100644 --- a/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs @@ -433,7 +433,7 @@ async fn execute_transaction_with_fault_configs( /// we spawn a tokio task on the server, client timing out and /// terminating the connection does not stop server from completing /// execution on its side -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn test_quorum_map_and_reduce_timeout() { let build_config = BuildConfig::default(); let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -460,7 +460,7 @@ async fn test_quorum_map_and_reduce_timeout() { certified_effects, Err(SuiError::QuorumFailedToExecuteCertificate { .. }) )); - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; let tx_info = TransactionInfoRequest { transaction_digest: *tx.digest(), }; diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index cd8471651a5a5..b7591c7471e1f 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -99,6 +99,7 @@ collectable = { version = "0.0.2", default-features = false } colored = { version = "2", default-features = false } colored-diff = { version = "0.2", default-features = false } combine = { version = "4", features = ["alloc", "bytes", "std"] } +comfy-table = { version = "6", features = ["crossterm", "tty"] } config-df9ed7b29d513bb1 = { package = "config", git = "https://github.com/MystenLabs/narwhal", rev = "aa0cc2d693572d197531a1054e0e3e417d2b7404", default-features = false } config-a6292c17cd707f01 = { package = "config", version = "0.11", features = ["hjson", "ini", "json", "rust-ini", "serde-hjson", "serde_json", "toml", "yaml", "yaml-rust"] } consensus = { git = "https://github.com/MystenLabs/narwhal", rev = "aa0cc2d693572d197531a1054e0e3e417d2b7404", features = ["benchmark", "rand"] } @@ -108,6 +109,7 @@ constant_time_eq = { version = "0.1", default-features = false } core2 = { version = "0.4", default-features = false, features = ["alloc"] } crc = { version = "3", default-features = false } crc-catalog = { version = "2", default-features = false } +crc32fast = { version = "1", features = ["std"] } criterion = { version = "0.3", features = ["cargo_bench_support"] } criterion-plot = { version = "0.4", default-features = false } crossbeam = { version = "0.8", features = ["alloc", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", "crossbeam-queue", "std"] } @@ -178,6 +180,7 @@ ff = { version = "0.12", default-features = false } fiat-crypto = { version = "0.1", features = ["std"] } fixedbitset-6f8ce4dd05d13bba = { package = "fixedbitset", version = "0.2", default-features = false } fixedbitset-9fbad63c4bcf4a8f = { package = "fixedbitset", version = "0.4", default-features = false } +flate2 = { version = "1", features = ["miniz_oxide", "rust_backend"] } flexstr = { version = "0.9", features = ["std"] } float-cmp = { version = "0.9", features = ["num-traits", "ratio"] } flume = { version = "0.10", default-features = false, features = ["async", "futures-core", "futures-sink", "pin-project"] } @@ -217,7 +220,7 @@ half = { version = "1", default-features = false } hashbrown-a6292c17cd707f01 = { package = "hashbrown", version = "0.11", features = ["ahash", "inline-more"] } hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["ahash", "inline-more", "raw"] } hashlink = { version = "0.8", default-features = false } -hdrhistogram = { version = "7", default-features = false } +hdrhistogram = { version = "7", features = ["base64", "crossbeam-channel", "flate2", "nom", "serialization", "sync"] } heck-468e82937335b1c9 = { package = "heck", version = "0.3", default-features = false } hex = { version = "0.4", features = ["alloc", "std"] } hkdf = { version = "0.12", default-features = false, features = ["std"] } @@ -719,6 +722,7 @@ collectable = { version = "0.0.2", default-features = false } colored = { version = "2", default-features = false } colored-diff = { version = "0.2", default-features = false } combine = { version = "4", features = ["alloc", "bytes", "std"] } +comfy-table = { version = "6", features = ["crossterm", "tty"] } config-df9ed7b29d513bb1 = { package = "config", git = "https://github.com/MystenLabs/narwhal", rev = "aa0cc2d693572d197531a1054e0e3e417d2b7404", default-features = false } config-a6292c17cd707f01 = { package = "config", version = "0.11", features = ["hjson", "ini", "json", "rust-ini", "serde-hjson", "serde_json", "toml", "yaml", "yaml-rust"] } consensus = { git = "https://github.com/MystenLabs/narwhal", rev = "aa0cc2d693572d197531a1054e0e3e417d2b7404", features = ["benchmark", "rand"] } @@ -728,6 +732,7 @@ constant_time_eq = { version = "0.1", default-features = false } core2 = { version = "0.4", default-features = false, features = ["alloc"] } crc = { version = "3", default-features = false } crc-catalog = { version = "2", default-features = false } +crc32fast = { version = "1", features = ["std"] } criterion = { version = "0.3", features = ["cargo_bench_support"] } criterion-plot = { version = "0.4", default-features = false } crossbeam = { version = "0.8", features = ["alloc", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", "crossbeam-queue", "std"] } @@ -810,6 +815,7 @@ ff = { version = "0.12", default-features = false } fiat-crypto = { version = "0.1", features = ["std"] } fixedbitset-6f8ce4dd05d13bba = { package = "fixedbitset", version = "0.2", default-features = false } fixedbitset-9fbad63c4bcf4a8f = { package = "fixedbitset", version = "0.4", default-features = false } +flate2 = { version = "1", features = ["miniz_oxide", "rust_backend"] } flexstr = { version = "0.9", features = ["std"] } float-cmp = { version = "0.9", features = ["num-traits", "ratio"] } flume = { version = "0.10", default-features = false, features = ["async", "futures-core", "futures-sink", "pin-project"] } @@ -850,7 +856,7 @@ half = { version = "1", default-features = false } hashbrown-a6292c17cd707f01 = { package = "hashbrown", version = "0.11", features = ["ahash", "inline-more"] } hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["ahash", "inline-more", "raw"] } hashlink = { version = "0.8", default-features = false } -hdrhistogram = { version = "7", default-features = false } +hdrhistogram = { version = "7", features = ["base64", "crossbeam-channel", "flate2", "nom", "serialization", "sync"] } heck-468e82937335b1c9 = { package = "heck", version = "0.3", default-features = false } heck-9fbad63c4bcf4a8f = { package = "heck", version = "0.4", features = ["unicode", "unicode-segmentation"] } hex = { version = "0.4", features = ["alloc", "std"] }