diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index d7a842d89..8bdc83be2 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use super::benchmark_result::BenchmarkResult; -use crate::util::units::Scalar; +use crate::{options::SortOrder, util::units::Scalar}; #[derive(Debug)] pub struct BenchmarkResultWithRelativeSpeed<'a> { @@ -25,8 +25,9 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], fastest: &'a BenchmarkResult, + sort_order: SortOrder, ) -> Vec> { - results + let mut results: Vec<_> = results .iter() .map(|result| { let is_fastest = result == fastest; @@ -61,11 +62,21 @@ fn compute_relative_speeds<'a>( is_fastest, } }) - .collect() + .collect(); + + match sort_order { + SortOrder::Command => {} + SortOrder::MeanTime => { + results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result)); + } + } + + results } pub fn compute_with_check( results: &[BenchmarkResult], + sort_order: SortOrder, ) -> Option> { let fastest = fastest_of(results); @@ -73,14 +84,17 @@ pub fn compute_with_check( return None; } - Some(compute_relative_speeds(results, fastest)) + Some(compute_relative_speeds(results, fastest, sort_order)) } /// Same as compute_with_check, potentially resulting in relative speeds of infinity -pub fn compute(results: &[BenchmarkResult]) -> Vec { +pub fn compute( + results: &[BenchmarkResult], + sort_order: SortOrder, +) -> Vec { let fastest = fastest_of(results); - compute_relative_speeds(results, fastest) + compute_relative_speeds(results, fastest, sort_order) } #[cfg(test)] @@ -113,7 +127,7 @@ fn test_compute_relative_speed() { create_result("cmd3", 5.0), ]; - let annotated_results = compute_with_check(&results).unwrap(); + let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap(); assert_relative_eq!(1.5, annotated_results[0].relative_speed); assert_relative_eq!(1.0, annotated_results[1].relative_speed); @@ -124,7 +138,7 @@ fn test_compute_relative_speed() { fn test_compute_relative_speed_for_zero_times() { let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)]; - let annotated_results = compute_with_check(&results); + let annotated_results = compute_with_check(&results, SortOrder::Command); assert!(annotated_results.is_none()); } diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index ced48338a..017a286fa 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -6,7 +6,7 @@ use super::{relative_speed, Benchmark}; use crate::command::Commands; use crate::export::ExportManager; -use crate::options::{ExecutorKind, Options, OutputStyleOption}; +use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder}; use anyhow::Result; @@ -46,7 +46,11 @@ impl<'a> Scheduler<'a> { // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. - self.export_manager.write_results(&self.results, true)?; + self.export_manager.write_results( + &self.results, + self.options.sort_order_exports, + true, + )?; } Ok(()) @@ -61,29 +65,55 @@ impl<'a> Scheduler<'a> { return; } - if let Some(mut annotated_results) = relative_speed::compute_with_check(&self.results) { - annotated_results.sort_by(|l, r| relative_speed::compare_mean_time(l.result, r.result)); - - let fastest = &annotated_results[0]; - let others = &annotated_results[1..]; - - println!("{}", "Summary".bold()); - println!( - " {} ran", - fastest.result.command_with_unused_parameters.cyan() - ); - - for item in others { - println!( - "{}{} times faster than {}", - format!("{:8.2}", item.relative_speed).bold().green(), - if let Some(stddev) = item.relative_speed_stddev { - format!(" ± {}", format!("{:.2}", stddev).green()) - } else { - "".into() - }, - &item.result.command_with_unused_parameters.magenta() - ); + if let Some(annotated_results) = relative_speed::compute_with_check( + &self.results, + self.options.sort_order_speed_comparison, + ) { + match self.options.sort_order_speed_comparison { + SortOrder::MeanTime => { + println!("{}", "Summary".bold()); + + let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap(); + let others = annotated_results.iter().filter(|r| !r.is_fastest); + + println!( + " {} ran", + fastest.result.command_with_unused_parameters.cyan() + ); + + for item in others { + println!( + "{}{} times faster than {}", + format!("{:8.2}", item.relative_speed).bold().green(), + if let Some(stddev) = item.relative_speed_stddev { + format!(" ± {}", format!("{:.2}", stddev).green()) + } else { + "".into() + }, + &item.result.command_with_unused_parameters.magenta() + ); + } + } + SortOrder::Command => { + println!("{}", "Relative speed comparison".bold()); + + for item in annotated_results { + println!( + " {}{} {}", + format!("{:10.2}", item.relative_speed).bold().green(), + if item.is_fastest { + " ".into() + } else { + if let Some(stddev) = item.relative_speed_stddev { + format!(" ± {}", format!("{:5.2}", stddev).green()) + } else { + " ".into() + } + }, + &item.result.command_with_unused_parameters, + ); + } + } } } else { eprintln!( @@ -99,6 +129,7 @@ impl<'a> Scheduler<'a> { } pub fn final_export(&self) -> Result<()> { - self.export_manager.write_results(&self.results, false) + self.export_manager + .write_results(&self.results, self.options.sort_order_exports, false) } } diff --git a/src/cli.rs b/src/cli.rs index b4f814ef5..806325340 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -160,21 +160,6 @@ fn build_command() -> Command { possible parameter combinations.\n" ), ) - .arg( - Arg::new("style") - .long("style") - .action(ArgAction::Set) - .value_name("TYPE") - .value_parser(["auto", "basic", "full", "nocolor", "color", "none"]) - .help( - "Set output style type (default: auto). Set this to 'basic' to disable output \ - coloring and interactive elements. Set it to 'full' to enable all effects \ - even if no interactive terminal was detected. Set this to 'nocolor' to \ - keep the interactive output without any colors. Set this to 'color' to keep \ - the colors without any interactive output. Set this to 'none' to disable all \ - the output of the tool.", - ), - ) .arg( Arg::new("shell") .long("shell") @@ -204,6 +189,38 @@ fn build_command() -> Command { .short('i') .help("Ignore non-zero exit codes of the benchmarked programs."), ) + .arg( + Arg::new("style") + .long("style") + .action(ArgAction::Set) + .value_name("TYPE") + .value_parser(["auto", "basic", "full", "nocolor", "color", "none"]) + .help( + "Set output style type (default: auto). Set this to 'basic' to disable output \ + coloring and interactive elements. Set it to 'full' to enable all effects \ + even if no interactive terminal was detected. Set this to 'nocolor' to \ + keep the interactive output without any colors. Set this to 'color' to keep \ + the colors without any interactive output. Set this to 'none' to disable all \ + the output of the tool.", + ), + ) + .arg( + Arg::new("sort") + .long("sort") + .action(ArgAction::Set) + .value_name("METHOD") + .value_parser(["auto", "command", "mean-time"]) + .default_value("auto") + .hide_default_value(true) + .help( + "Specify the sort order of the speed comparison summary and the exported tables for \ + markup formats (Markdown, AsciiDoc, org-mode):\n \ + * 'auto' (default): the speed comparison will be ordered by time and\n \ + the markup tables will be ordered by command (input order).\n \ + * 'command': order benchmarks in the way they were specified\n \ + * 'mean-time': order benchmarks by mean runtime\n" + ), + ) .arg( Arg::new("time-unit") .long("time-unit") diff --git a/src/export/asciidoc.rs b/src/export/asciidoc.rs index 9133d6c6a..c95661726 100644 --- a/src/export/asciidoc.rs +++ b/src/export/asciidoc.rs @@ -76,6 +76,9 @@ fn cfg_test_table_header(unit_short_name: &str) -> String { ) } +#[cfg(test)] +use crate::options::SortOrder; + #[cfg(test)] use crate::util::units::Unit; @@ -132,8 +135,12 @@ fn test_asciidoc_format_s() { }, ]; - let actual = - String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap(); + let actual = String::from_utf8( + exporter + .serialize(&results, Some(Unit::Second), SortOrder::Command) + .unwrap(), + ) + .unwrap(); let expect = format!( "{} | `FOO=1 BAR=2 command \\| 1` @@ -205,7 +212,7 @@ fn test_asciidoc_format_ms() { let actual = String::from_utf8( exporter - .serialize(&results, Some(Unit::MilliSecond)) + .serialize(&results, Some(Unit::MilliSecond), SortOrder::Command) .unwrap(), ) .unwrap(); diff --git a/src/export/csv.rs b/src/export/csv.rs index c846a9508..b91c8de5f 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -4,6 +4,7 @@ use csv::WriterBuilder; use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; +use crate::options::SortOrder; use crate::util::units::Unit; use anyhow::Result; @@ -12,7 +13,12 @@ use anyhow::Result; pub struct CsvExporter {} impl Exporter for CsvExporter { - fn serialize(&self, results: &[BenchmarkResult], _unit: Option) -> Result> { + fn serialize( + &self, + results: &[BenchmarkResult], + _unit: Option, + _sort_order: SortOrder, + ) -> Result> { let mut writer = WriterBuilder::new().from_writer(vec![]); { @@ -105,8 +111,12 @@ fn test_csv() { FOO=one BAR=seven command | 2,11,12,11,13,14,15,16.5,seven,one\n\ ", ); - let gens = - String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap(); + let gens = String::from_utf8( + exporter + .serialize(&results, Some(Unit::Second), SortOrder::Command) + .unwrap(), + ) + .unwrap(); assert_eq!(exps, gens); } diff --git a/src/export/json.rs b/src/export/json.rs index de4720444..d2542c1ca 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -3,6 +3,7 @@ use serde_json::to_vec_pretty; use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; +use crate::options::SortOrder; use crate::util::units::Unit; use anyhow::Result; @@ -16,7 +17,12 @@ struct HyperfineSummary<'a> { pub struct JsonExporter {} impl Exporter for JsonExporter { - fn serialize(&self, results: &[BenchmarkResult], _unit: Option) -> Result> { + fn serialize( + &self, + results: &[BenchmarkResult], + _unit: Option, + _sort_order: SortOrder, + ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { results }); if let Ok(ref mut content) = output { content.push(b'\n'); diff --git a/src/export/markdown.rs b/src/export/markdown.rs index ba6623b45..6ad52896c 100644 --- a/src/export/markdown.rs +++ b/src/export/markdown.rs @@ -28,6 +28,9 @@ impl MarkupExporter for MarkdownExporter { } } +#[cfg(test)] +use crate::options::SortOrder; + /// Check Markdown-based data row formatting #[test] fn test_markdown_formatter_table_data() { @@ -99,7 +102,12 @@ fn test_markdown_format_ms() { }, ]; - let actual = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap(); + let actual = String::from_utf8( + exporter + .serialize(&timing_results, None, SortOrder::Command) + .unwrap(), + ) + .unwrap(); let expect = format!( "{}\ | `sleep 0.1` | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 | @@ -151,7 +159,12 @@ fn test_markdown_format_s() { }, ]; - let actual = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap(); + let actual = String::from_utf8( + exporter + .serialize(&timing_results, None, SortOrder::Command) + .unwrap(), + ) + .unwrap(); let expect = format!( "{}\ | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 | @@ -205,7 +218,7 @@ fn test_markdown_format_time_unit_s() { let actual = String::from_utf8( exporter - .serialize(&timing_results, Some(Unit::Second)) + .serialize(&timing_results, Some(Unit::Second), SortOrder::Command) .unwrap(), ) .unwrap(); @@ -263,7 +276,7 @@ fn test_markdown_format_time_unit_ms() { let actual = String::from_utf8( exporter - .serialize(&timing_results, Some(Unit::MilliSecond)) + .serialize(&timing_results, Some(Unit::MilliSecond), SortOrder::Command) .unwrap(), ) .unwrap(); diff --git a/src/export/markup.rs b/src/export/markup.rs index c789f3abf..88bcaa90d 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -1,5 +1,6 @@ use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed; use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed}; +use crate::options::SortOrder; use crate::output::format::format_duration_value; use crate::util::units::Unit; @@ -105,9 +106,14 @@ fn determine_unit_from_results(results: &[BenchmarkResult]) -> Unit { } impl Exporter for T { - fn serialize(&self, results: &[BenchmarkResult], unit: Option) -> Result> { + fn serialize( + &self, + results: &[BenchmarkResult], + unit: Option, + sort_order: SortOrder, + ) -> Result> { let unit = unit.unwrap_or_else(|| determine_unit_from_results(results)); - let entries = relative_speed::compute(results); + let entries = relative_speed::compute(results, sort_order); let table = self.table_results(&entries, unit); Ok(table.as_bytes().to_vec()) diff --git a/src/export/mod.rs b/src/export/mod.rs index 556409c69..c240f38a1 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -15,6 +15,7 @@ use self::markdown::MarkdownExporter; use self::orgmode::OrgmodeExporter; use crate::benchmark::benchmark_result::BenchmarkResult; +use crate::options::SortOrder; use crate::util::units::Unit; use anyhow::{Context, Result}; @@ -42,7 +43,12 @@ pub enum ExportType { /// Interface for different exporters. trait Exporter { /// Export the given entries in the serialized form. - fn serialize(&self, results: &[BenchmarkResult], unit: Option) -> Result>; + fn serialize( + &self, + results: &[BenchmarkResult], + unit: Option, + sort_order: SortOrder, + ) -> Result>; } pub enum ExportTarget { @@ -116,9 +122,12 @@ impl ExportManager { /// results are written to all file targets (to always have them up to date, even /// if a benchmark fails). In the latter case, we only print to stdout targets (in /// order not to clutter the output of hyperfine with intermediate results). - pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> { + pub fn write_results(&self, results: &[BenchmarkResult], sort_order: SortOrder, intermediate: bool) -> Result<()> { for e in &self.exporters { - let content = || e.exporter.serialize(results, self.time_unit); + let content = || { + e.exporter + .serialize(results, self.time_unit, sort_order) + }; match e.target { ExportTarget::File(ref filename) => { diff --git a/src/export/orgmode.rs b/src/export/orgmode.rs index 8ec879299..dabc62d19 100644 --- a/src/export/orgmode.rs +++ b/src/export/orgmode.rs @@ -22,6 +22,9 @@ impl MarkupExporter for OrgmodeExporter { } } +#[cfg(test)] +use crate::options::SortOrder; + /// Check Emacs org-mode data row formatting #[test] fn test_orgmode_formatter_table_data() { @@ -105,7 +108,12 @@ fn test_orgmode_format_ms() { }, ]; - let actual = String::from_utf8(exporter.serialize(&results, None).unwrap()).unwrap(); + let actual = String::from_utf8( + exporter + .serialize(&results, None, SortOrder::Command) + .unwrap(), + ) + .unwrap(); let expect = format!( "{}\ | =sleep 0.1= | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 | @@ -159,8 +167,12 @@ fn test_orgmode_format_s() { }, ]; - let actual = - String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap(); + let actual = String::from_utf8( + exporter + .serialize(&results, Some(Unit::Second), SortOrder::Command) + .unwrap(), + ) + .unwrap(); let expect = format!( "{}\ | =sleep 2= | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 | diff --git a/src/options.rs b/src/options.rs index 15fb68ef7..b630f7819 100644 --- a/src/options.rs +++ b/src/options.rs @@ -95,6 +95,12 @@ pub enum OutputStyleOption { Disabled, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SortOrder { + Command, + MeanTime, +} + /// Bounds for the number of benchmark runs pub struct RunBounds { /// Minimum number of benchmark runs @@ -210,6 +216,12 @@ pub struct Options { /// What color mode to use for the terminal output pub output_style: OutputStyleOption, + /// How to order benchmarks in the relative speed comparison + pub sort_order_speed_comparison: SortOrder, + + /// How to order benchmarks in the markup format exports + pub sort_order_exports: SortOrder, + /// Determines how we run commands pub executor_kind: ExecutorKind, @@ -234,6 +246,8 @@ impl Default for Options { setup_command: None, cleanup_command: None, output_style: OutputStyleOption::Full, + sort_order_speed_comparison: SortOrder::MeanTime, + sort_order_exports: SortOrder::Command, executor_kind: ExecutorKind::default(), command_output_policy: CommandOutputPolicy::Null, time_unit: None, @@ -346,6 +360,13 @@ impl Options { OutputStyleOption::Disabled => {} }; + (options.sort_order_speed_comparison, options.sort_order_exports) = match matches.get_one::("sort").map(|s| s.as_str()) { + None | Some("auto") => (SortOrder::MeanTime, SortOrder::Command), + Some("command") => (SortOrder::Command, SortOrder::Command), + Some("mean-time") => (SortOrder::MeanTime, SortOrder::MeanTime), + Some(_) => unreachable!("Unknown sort order"), + }; + options.executor_kind = if matches.get_flag("no-shell") { ExecutorKind::Raw } else {