Multiple output policies

doc/hyperfine.1

@@ -290,6 +290,11 @@ Don't redirect the output at all (same as \&'\-\-show\-output').
 .IP "<FILE>"
 Write the output to the given file.
 .RE
+.IP
+This option can be specified once for all commands or multiple times,
+once for each command. Note: If you want to log the output of each and
+every iteration, you can use a shell redirection and the $HYPERFINE_ITERATION
+environment variable: 'my-command > output-${HYPERFINE_ITERATION}.log'
 .HP
 \fB\-\-input\fR \fIWHERE\fP
 .IP

src/benchmark/executor.rs

@@ -39,6 +39,7 @@ pub trait Executor {
         command: &Command<'_>,
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
+        output_policy: &CommandOutputPolicy,
     ) -> Result<(TimingResult, ExitStatus)>;
 
     /// Perform a calibration of this executor. For example,
@@ -116,13 +117,14 @@ impl Executor for RawExecutor<'_> {
         command: &Command<'_>,
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
+        output_policy: &CommandOutputPolicy,
     ) -> Result<(TimingResult, ExitStatus)> {
         let result = run_command_and_measure_common(
             command.get_command()?,
             iteration,
             command_failure_action.unwrap_or(self.options.command_failure_action),
             &self.options.command_input_policy,
-            &self.options.command_output_policy,
+            output_policy,
             &command.get_command_line(),
         )?;
 
@@ -167,6 +169,7 @@ impl Executor for ShellExecutor<'_> {
         command: &Command<'_>,
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
+        output_policy: &CommandOutputPolicy,
     ) -> Result<(TimingResult, ExitStatus)> {
         let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
         let mut command_builder = self.shell.command();
@@ -185,7 +188,7 @@ impl Executor for ShellExecutor<'_> {
             iteration,
             command_failure_action.unwrap_or(self.options.command_failure_action),
             &self.options.command_input_policy,
-            &self.options.command_output_policy,
+            output_policy,
             &command.get_command_line(),
         )?;
 
@@ -229,6 +232,7 @@ impl Executor for ShellExecutor<'_> {
                 &Command::new(None, ""),
                 BenchmarkIteration::NonBenchmarkRun,
                 None,
+                &CommandOutputPolicy::Null,
             );
 
             match res {
@@ -300,6 +304,7 @@ impl Executor for MockExecutor {
         command: &Command<'_>,
         _iteration: BenchmarkIteration,
         _command_failure_action: Option<CmdFailureAction>,
+        _output_policy: &CommandOutputPolicy,
     ) -> Result<(TimingResult, ExitStatus)> {
         #[cfg(unix)]
         let status = {

src/benchmark/mod.rs

@@ -8,7 +8,9 @@ use std::cmp;
 
 use crate::benchmark::executor::BenchmarkIteration;
 use crate::command::Command;
-use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption};
+use crate::options::{
+    CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
+};
 use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
 use crate::output::format::{format_duration, format_duration_unit};
 use crate::output::progress_bar::get_progress_bar;
@@ -56,12 +58,14 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         error_output: &'static str,
+        output_policy: &CommandOutputPolicy,
     ) -> Result<TimingResult> {
         self.executor
             .run_command_and_measure(
                 command,
                 executor::BenchmarkIteration::NonBenchmarkRun,
                 Some(CmdFailureAction::RaiseError),
+                output_policy,
             )
             .map(|r| r.0)
             .map_err(|_| anyhow!(error_output))
@@ -71,6 +75,7 @@ impl<'a> Benchmark<'a> {
     fn run_setup_command(
         &self,
         parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
+        output_policy: &CommandOutputPolicy,
     ) -> Result<TimingResult> {
         let command = self
             .options
@@ -82,7 +87,7 @@ impl<'a> Benchmark<'a> {
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
         Ok(command
-            .map(|cmd| self.run_intermediate_command(&cmd, error_output))
+            .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
             .transpose()?
             .unwrap_or_default())
     }
@@ -91,6 +96,7 @@ impl<'a> Benchmark<'a> {
     fn run_cleanup_command(
         &self,
         parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
+        output_policy: &CommandOutputPolicy,
     ) -> Result<TimingResult> {
         let command = self
             .options
@@ -102,25 +108,33 @@ impl<'a> Benchmark<'a> {
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
         Ok(command
-            .map(|cmd| self.run_intermediate_command(&cmd, error_output))
+            .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
             .transpose()?
             .unwrap_or_default())
     }
 
     /// Run the command specified by `--prepare`.
-    fn run_preparation_command(&self, command: &Command<'_>) -> Result<TimingResult> {
+    fn run_preparation_command(
+        &self,
+        command: &Command<'_>,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<TimingResult> {
         let error_output = "The preparation command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
-        self.run_intermediate_command(command, error_output)
+        self.run_intermediate_command(command, error_output, output_policy)
     }
 
     /// Run the command specified by `--conclude`.
-    fn run_conclusion_command(&self, command: &Command<'_>) -> Result<TimingResult> {
+    fn run_conclusion_command(
+        &self,
+        command: &Command<'_>,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<TimingResult> {
         let error_output = "The conclusion command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
-        self.run_intermediate_command(command, error_output)
+        self.run_intermediate_command(command, error_output, output_policy)
     }
 
     /// Run the benchmark for a single command
@@ -140,6 +154,8 @@ impl<'a> Benchmark<'a> {
         let mut exit_codes: Vec<Option<i32>> = vec![];
         let mut all_succeeded = true;
 
+        let output_policy = &self.options.command_output_policies[self.number];
+
         let preparation_command = self.options.preparation_command.as_ref().map(|values| {
             let preparation_command = if values.len() == 1 {
                 &values[0]
@@ -152,10 +168,11 @@ impl<'a> Benchmark<'a> {
                 self.command.get_parameters().iter().cloned(),
             )
         });
+
         let run_preparation_command = || {
             preparation_command
                 .as_ref()
-                .map(|cmd| self.run_preparation_command(cmd))
+                .map(|cmd| self.run_preparation_command(cmd, output_policy))
                 .transpose()
         };
 
@@ -174,11 +191,11 @@ impl<'a> Benchmark<'a> {
         let run_conclusion_command = || {
             conclusion_command
                 .as_ref()
-                .map(|cmd| self.run_conclusion_command(cmd))
+                .map(|cmd| self.run_conclusion_command(cmd, output_policy))
                 .transpose()
         };
 
-        self.run_setup_command(self.command.get_parameters().iter().cloned())?;
+        self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
 
         // Warmup phase
         if self.options.warmup_count > 0 {
@@ -198,6 +215,7 @@ impl<'a> Benchmark<'a> {
                     self.command,
                     BenchmarkIteration::Warmup(i),
                     None,
+                    output_policy,
                 )?;
                 let _ = run_conclusion_command()?;
                 if let Some(bar) = progress_bar.as_ref() {
@@ -229,6 +247,7 @@ impl<'a> Benchmark<'a> {
             self.command,
             BenchmarkIteration::Benchmark(0),
             None,
+            output_policy,
         )?;
         let success = status.success();
 
@@ -289,6 +308,7 @@ impl<'a> Benchmark<'a> {
                 self.command,
                 BenchmarkIteration::Benchmark(i + 1),
                 None,
+                output_policy,
             )?;
             let success = status.success();
 
@@ -418,7 +438,7 @@ impl<'a> Benchmark<'a> {
             println!(" ");
         }
 
-        self.run_cleanup_command(self.command.get_parameters().iter().cloned())?;
+        self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
 
         Ok(BenchmarkResult {
             command: self.command.get_name(),

src/cli.rs

@@ -327,7 +327,7 @@ fn build_command() -> Command {
             Arg::new("output")
                 .long("output")
                 .conflicts_with("show-output")
-                .action(ArgAction::Set)
+                .action(ArgAction::Append)
                 .value_name("WHERE")
                 .help(
                     "Control where the output of the benchmark is redirected. Note \
@@ -343,7 +343,11 @@ fn build_command() -> Command {
                      \n  \
                        inherit:  Don't redirect the output at all (same as '--show-output').\n\
                      \n  \
-                       <FILE>:   Write the output to the given file.",
+                       <FILE>:   Write the output to the given file.\n\n\
+                    This option can be specified once for all commands or multiple times, once for \
+                    each command. Note: If you want to log the output of each and every iteration, \
+                    you can use a shell redirection and the '$HYPERFINE_ITERATION' environment variable:\n    \
+                    hyperfine 'my-command > output-${HYPERFINE_ITERATION}.log'\n\n",
                 ),
         )
         .arg(

src/command.rs

@@ -251,8 +251,8 @@ impl<'a> Commands<'a> {
         self.0.iter()
     }
 
-    pub fn num_commands(&self) -> usize {
-        self.0.len()
+    pub fn num_commands(&self, has_reference_command: bool) -> usize {
+        self.0.len() + if has_reference_command { 1 } else { 0 }
     }
 
     /// Finds all the strings that appear multiple times in the input iterator, returning them in

src/main.rs

@@ -32,7 +32,7 @@ fn run() -> Result<()> {
     colored::control::set_virtual_terminal(true).unwrap();
 
     let cli_arguments = get_cli_arguments(env::args_os());
-    let options = Options::from_cli_arguments(&cli_arguments)?;
+    let mut options = Options::from_cli_arguments(&cli_arguments)?;
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
     let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.time_unit)?;
 

src/options.rs

@@ -234,8 +234,8 @@ pub struct Options {
     /// Where input to the benchmarked command comes from
     pub command_input_policy: CommandInputPolicy,
 
-    /// What to do with the output of the benchmarked command
-    pub command_output_policy: CommandOutputPolicy,
+    /// What to do with the output of the benchmarked commands
+    pub command_output_policies: Vec<CommandOutputPolicy>,
 
     /// Which time unit to use when displaying results
     pub time_unit: Option<Unit>,
@@ -257,7 +257,7 @@ impl Default for Options {
             sort_order_speed_comparison: SortOrder::MeanTime,
             sort_order_exports: SortOrder::Command,
             executor_kind: ExecutorKind::default(),
-            command_output_policy: CommandOutputPolicy::Null,
+            command_output_policies: vec![CommandOutputPolicy::Null],
             time_unit: None,
             command_input_policy: CommandInputPolicy::Null,
         }
@@ -320,23 +320,28 @@ impl Options {
 
         options.cleanup_command = matches.get_one::<String>("cleanup").map(String::from);
 
-        options.command_output_policy = if matches.get_flag("show-output") {
-            CommandOutputPolicy::Inherit
-        } else if let Some(output) = matches.get_one::<String>("output").map(|s| s.as_str()) {
-            match output {
-                "null" => CommandOutputPolicy::Null,
-                "pipe" => CommandOutputPolicy::Pipe,
-                "inherit" => CommandOutputPolicy::Inherit,
-                arg => {
-                    let path = PathBuf::from(arg);
-                    if path.components().count() <= 1 {
-                        return Err(OptionsError::UnknownOutputPolicy(arg.to_string()));
+        options.command_output_policies = if matches.get_flag("show-output") {
+            vec![CommandOutputPolicy::Inherit]
+        } else if let Some(output_values) = matches.get_many::<String>("output") {
+            let mut policies = vec![];
+            for value in output_values {
+                let policy = match value.as_str() {
+                    "null" => CommandOutputPolicy::Null,
+                    "pipe" => CommandOutputPolicy::Pipe,
+                    "inherit" => CommandOutputPolicy::Inherit,
+                    arg => {
+                        let path = PathBuf::from(arg);
+                        if path.components().count() <= 1 {
+                            return Err(OptionsError::UnknownOutputPolicy(arg.to_string()));
+                        }
+                        CommandOutputPolicy::File(path)
                     }
-                    CommandOutputPolicy::File(path)
-                }
+                };
+                policies.push(policy);
             }
+            policies
         } else {
-            CommandOutputPolicy::Null
+            vec![CommandOutputPolicy::Null]
         };
 
         options.output_style = match matches.get_one::<String>("style").map(|s| s.as_str()) {
@@ -346,7 +351,10 @@ impl Options {
             Some("color") => OutputStyleOption::Color,
             Some("none") => OutputStyleOption::Disabled,
             _ => {
-                if options.command_output_policy == CommandOutputPolicy::Inherit
+                if options
+                    .command_output_policies
+                    .iter()
+                    .any(|policy| *policy == CommandOutputPolicy::Inherit)
                     || !io::stdout().is_terminal()
                 {
                     OutputStyleOption::Basic
@@ -436,26 +444,34 @@ impl Options {
         Ok(options)
     }
 
-    pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
-        let num_commands = commands.num_commands()
-            + if self.reference_command.is_some() {
-                1
-            } else {
-                0
-            };
+    pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> {
+        let has_reference_command = self.reference_command.is_some();
+        let num_commands = commands.num_commands(has_reference_command);
+
         if let Some(preparation_command) = &self.preparation_command {
             ensure!(
                 preparation_command.len() <= 1 || num_commands == preparation_command.len(),
-                "The '--prepare' option has to be provided just once or N times, where N is the \
-             number of benchmark commands including a potential reference."
+                "The '--prepare' option has to be provided just once or N times, where N={num_commands} is the \
+                 number of benchmark commands (including a potential reference)."
             );
         }
 
         if let Some(conclusion_command) = &self.conclusion_command {
             ensure!(
                 conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
-                "The '--conclude' option has to be provided just once or N times, where N is the \
-             number of benchmark commands including a potential reference."
+                "The '--conclude' option has to be provided just once or N times, where N={num_commands} is the \
+                 number of benchmark commands (including a potential reference)."
+            );
+        }
+
+        if self.command_output_policies.len() == 1 {
+            self.command_output_policies =
+                vec![self.command_output_policies[0].clone(); num_commands];
+        } else {
+            ensure!(
+                self.command_output_policies.len() == num_commands,
+                "The '--output' option has to be provided just once or N times, where N={num_commands} is the \
+                 number of benchmark commands (including a potential reference)."
             );
         }