Add new --sort option

This adds a new `--sort` option to choose the way in which the results
in the speed comparison and the markup exports are ordered.

closes #614
closes #601

src/benchmark/relative_speed.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
 
 use super::benchmark_result::BenchmarkResult;
-use crate::util::units::Scalar;
+use crate::{options::SortOrder, util::units::Scalar};
 
 #[derive(Debug)]
 pub struct BenchmarkResultWithRelativeSpeed<'a> {
@@ -25,8 +25,9 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
 fn compute_relative_speeds<'a>(
     results: &'a [BenchmarkResult],
     fastest: &'a BenchmarkResult,
+    sort_order: SortOrder,
 ) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
-    results
+    let mut results: Vec<_> = results
         .iter()
         .map(|result| {
             let is_fastest = result == fastest;
@@ -61,26 +62,39 @@ fn compute_relative_speeds<'a>(
                 is_fastest,
             }
         })
-        .collect()
+        .collect();
+
+    match sort_order {
+        SortOrder::Command => {}
+        SortOrder::MeanTime => {
+            results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result));
+        }
+    }
+
+    results
 }
 
 pub fn compute_with_check(
     results: &[BenchmarkResult],
+    sort_order: SortOrder,
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
     let fastest = fastest_of(results);
 
     if fastest.mean == 0.0 {
         return None;
     }
 
-    Some(compute_relative_speeds(results, fastest))
+    Some(compute_relative_speeds(results, fastest, sort_order))
 }
 
 /// Same as compute_with_check, potentially resulting in relative speeds of infinity
-pub fn compute(results: &[BenchmarkResult]) -> Vec<BenchmarkResultWithRelativeSpeed> {
+pub fn compute(
+    results: &[BenchmarkResult],
+    sort_order: SortOrder,
+) -> Vec<BenchmarkResultWithRelativeSpeed> {
     let fastest = fastest_of(results);
 
-    compute_relative_speeds(results, fastest)
+    compute_relative_speeds(results, fastest, sort_order)
 }
 
 #[cfg(test)]
@@ -113,7 +127,7 @@ fn test_compute_relative_speed() {
         create_result("cmd3", 5.0),
     ];
 
-    let annotated_results = compute_with_check(&results).unwrap();
+    let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap();
 
     assert_relative_eq!(1.5, annotated_results[0].relative_speed);
     assert_relative_eq!(1.0, annotated_results[1].relative_speed);
@@ -124,7 +138,7 @@ fn test_compute_relative_speed() {
 fn test_compute_relative_speed_for_zero_times() {
     let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
 
-    let annotated_results = compute_with_check(&results);
+    let annotated_results = compute_with_check(&results, SortOrder::Command);
 
     assert!(annotated_results.is_none());
 }

src/benchmark/scheduler.rs

@@ -6,7 +6,7 @@ use super::{relative_speed, Benchmark};
 
 use crate::command::Commands;
 use crate::export::ExportManager;
-use crate::options::{ExecutorKind, Options, OutputStyleOption};
+use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
 
 use anyhow::Result;
 
@@ -46,7 +46,11 @@ impl<'a> Scheduler<'a> {
 
             // We export results after each individual benchmark, because
             // we would risk losing them if a later benchmark fails.
-            self.export_manager.write_results(&self.results, true)?;
+            self.export_manager.write_results(
+                &self.results,
+                self.options.sort_order_exports,
+                true,
+            )?;
         }
 
         Ok(())
@@ -61,29 +65,55 @@ impl<'a> Scheduler<'a> {
             return;
         }
 
-        if let Some(mut annotated_results) = relative_speed::compute_with_check(&self.results) {
-            annotated_results.sort_by(|l, r| relative_speed::compare_mean_time(l.result, r.result));
-
-            let fastest = &annotated_results[0];
-            let others = &annotated_results[1..];
-
-            println!("{}", "Summary".bold());
-            println!(
-                "  {} ran",
-                fastest.result.command_with_unused_parameters.cyan()
-            );
-
-            for item in others {
-                println!(
-                    "{}{} times faster than {}",
-                    format!("{:8.2}", item.relative_speed).bold().green(),
-                    if let Some(stddev) = item.relative_speed_stddev {
-                        format!(" ± {}", format!("{:.2}", stddev).green())
-                    } else {
-                        "".into()
-                    },
-                    &item.result.command_with_unused_parameters.magenta()
-                );
+        if let Some(annotated_results) = relative_speed::compute_with_check(
+            &self.results,
+            self.options.sort_order_speed_comparison,
+        ) {
+            match self.options.sort_order_speed_comparison {
+                SortOrder::MeanTime => {
+                    println!("{}", "Summary".bold());
+
+                    let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
+                    let others = annotated_results.iter().filter(|r| !r.is_fastest);
+
+                    println!(
+                        "  {} ran",
+                        fastest.result.command_with_unused_parameters.cyan()
+                    );
+
+                    for item in others {
+                        println!(
+                            "{}{} times faster than {}",
+                            format!("{:8.2}", item.relative_speed).bold().green(),
+                            if let Some(stddev) = item.relative_speed_stddev {
+                                format!(" ± {}", format!("{:.2}", stddev).green())
+                            } else {
+                                "".into()
+                            },
+                            &item.result.command_with_unused_parameters.magenta()
+                        );
+                    }
+                }
+                SortOrder::Command => {
+                    println!("{}", "Relative speed comparison".bold());
+
+                    for item in annotated_results {
+                        println!(
+                            "  {}{}  {}",
+                            format!("{:10.2}", item.relative_speed).bold().green(),
+                            if item.is_fastest {
+                                "        ".into()
+                            } else {
+                                if let Some(stddev) = item.relative_speed_stddev {
+                                    format!(" ± {}", format!("{:5.2}", stddev).green())
+                                } else {
+                                    "        ".into()
+                                }
+                            },
+                            &item.result.command_with_unused_parameters,
+                        );
+                    }
+                }
             }
         } else {
             eprintln!(
@@ -99,6 +129,7 @@ impl<'a> Scheduler<'a> {
     }
 
     pub fn final_export(&self) -> Result<()> {
-        self.export_manager.write_results(&self.results, false)
+        self.export_manager
+            .write_results(&self.results, self.options.sort_order_exports, false)
     }
 }

src/cli.rs

@@ -160,21 +160,6 @@ fn build_command() -> Command {
                      possible parameter combinations.\n"
                 ),
         )
-        .arg(
-            Arg::new("style")
-                .long("style")
-                .action(ArgAction::Set)
-                .value_name("TYPE")
-                .value_parser(["auto", "basic", "full", "nocolor", "color", "none"])
-                .help(
-                    "Set output style type (default: auto). Set this to 'basic' to disable output \
-                     coloring and interactive elements. Set it to 'full' to enable all effects \
-                     even if no interactive terminal was detected. Set this to 'nocolor' to \
-                     keep the interactive output without any colors. Set this to 'color' to keep \
-                     the colors without any interactive output. Set this to 'none' to disable all \
-                     the output of the tool.",
-                ),
-        )
         .arg(
             Arg::new("shell")
                 .long("shell")
@@ -204,6 +189,38 @@ fn build_command() -> Command {
                 .short('i')
                 .help("Ignore non-zero exit codes of the benchmarked programs."),
         )
+        .arg(
+            Arg::new("style")
+                .long("style")
+                .action(ArgAction::Set)
+                .value_name("TYPE")
+                .value_parser(["auto", "basic", "full", "nocolor", "color", "none"])
+                .help(
+                    "Set output style type (default: auto). Set this to 'basic' to disable output \
+                     coloring and interactive elements. Set it to 'full' to enable all effects \
+                     even if no interactive terminal was detected. Set this to 'nocolor' to \
+                     keep the interactive output without any colors. Set this to 'color' to keep \
+                     the colors without any interactive output. Set this to 'none' to disable all \
+                     the output of the tool.",
+                ),
+        )
+        .arg(
+            Arg::new("sort")
+            .long("sort")
+            .action(ArgAction::Set)
+            .value_name("METHOD")
+            .value_parser(["auto", "command", "mean-time"])
+            .default_value("auto")
+            .hide_default_value(true)
+            .help(
+                "Specify the sort order of the speed comparison summary and the exported tables for \
+                 markup formats (Markdown, AsciiDoc, org-mode):\n  \
+                   * 'auto' (default): the speed comparison will be ordered by time and\n    \
+                     the markup tables will be ordered by command (input order).\n  \
+                   * 'command': order benchmarks in the way they were specified\n  \
+                   * 'mean-time': order benchmarks by mean runtime\n"
+            ),
+        )
         .arg(
             Arg::new("time-unit")
                 .long("time-unit")

src/export/asciidoc.rs

@@ -76,6 +76,9 @@ fn cfg_test_table_header(unit_short_name: &str) -> String {
     )
 }
 
+#[cfg(test)]
+use crate::options::SortOrder;
+
 #[cfg(test)]
 use crate::util::units::Unit;
 
@@ -132,8 +135,12 @@ fn test_asciidoc_format_s() {
         },
     ];
 
-    let actual =
-        String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap();
+    let actual = String::from_utf8(
+        exporter
+            .serialize(&results, Some(Unit::Second), SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
     let expect = format!(
         "{}
 | `FOO=1 BAR=2 command \\| 1` 
@@ -205,7 +212,7 @@ fn test_asciidoc_format_ms() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, Some(Unit::MilliSecond))
+            .serialize(&results, Some(Unit::MilliSecond), SortOrder::Command)
             .unwrap(),
     )
     .unwrap();

src/export/csv.rs

@@ -4,6 +4,7 @@ use csv::WriterBuilder;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::options::SortOrder;
 use crate::util::units::Unit;
 
 use anyhow::Result;
@@ -12,7 +13,12 @@ use anyhow::Result;
 pub struct CsvExporter {}
 
 impl Exporter for CsvExporter {
-    fn serialize(&self, results: &[BenchmarkResult], _unit: Option<Unit>) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        _unit: Option<Unit>,
+        _sort_order: SortOrder,
+    ) -> Result<Vec<u8>> {
         let mut writer = WriterBuilder::new().from_writer(vec![]);
 
         {
@@ -105,8 +111,12 @@ fn test_csv() {
         FOO=one BAR=seven command | 2,11,12,11,13,14,15,16.5,seven,one\n\
         ",
     );
-    let gens =
-        String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap();
+    let gens = String::from_utf8(
+        exporter
+            .serialize(&results, Some(Unit::Second), SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
 
     assert_eq!(exps, gens);
 }

src/export/json.rs

@@ -3,6 +3,7 @@ use serde_json::to_vec_pretty;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::options::SortOrder;
 use crate::util::units::Unit;
 
 use anyhow::Result;
@@ -16,7 +17,12 @@ struct HyperfineSummary<'a> {
 pub struct JsonExporter {}
 
 impl Exporter for JsonExporter {
-    fn serialize(&self, results: &[BenchmarkResult], _unit: Option<Unit>) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        _unit: Option<Unit>,
+        _sort_order: SortOrder,
+    ) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary { results });
         if let Ok(ref mut content) = output {
             content.push(b'\n');