diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index 73f8e20f4..00fa3521b 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -290,6 +290,11 @@ Don't redirect the output at all (same as \&'\-\-show\-output'). .IP "" Write the output to the given file. .RE +.IP +This option can be specified once for all commands or multiple times, +once for each command. Note: If you want to log the output of each and +every iteration, you can use a shell redirection and the $HYPERFINE_ITERATION +environment variable: 'my-command > output-${HYPERFINE_ITERATION}.log' .HP \fB\-\-input\fR \fIWHERE\fP .IP diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index bf30be1f3..42af62231 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -39,6 +39,7 @@ pub trait Executor { command: &Command<'_>, iteration: BenchmarkIteration, command_failure_action: Option, + output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)>; /// Perform a calibration of this executor. For example, @@ -116,13 +117,14 @@ impl Executor for RawExecutor<'_> { command: &Command<'_>, iteration: BenchmarkIteration, command_failure_action: Option, + output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)> { let result = run_command_and_measure_common( command.get_command()?, iteration, command_failure_action.unwrap_or(self.options.command_failure_action), &self.options.command_input_policy, - &self.options.command_output_policy, + output_policy, &command.get_command_line(), )?; @@ -167,6 +169,7 @@ impl Executor for ShellExecutor<'_> { command: &Command<'_>, iteration: BenchmarkIteration, command_failure_action: Option, + output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)> { let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let mut command_builder = self.shell.command(); @@ -185,7 +188,7 @@ impl Executor for ShellExecutor<'_> { iteration, command_failure_action.unwrap_or(self.options.command_failure_action), &self.options.command_input_policy, - &self.options.command_output_policy, + output_policy, &command.get_command_line(), )?; @@ -229,6 +232,7 @@ impl Executor for ShellExecutor<'_> { &Command::new(None, ""), BenchmarkIteration::NonBenchmarkRun, None, + &CommandOutputPolicy::Null, ); match res { @@ -300,6 +304,7 @@ impl Executor for MockExecutor { command: &Command<'_>, _iteration: BenchmarkIteration, _command_failure_action: Option, + _output_policy: &CommandOutputPolicy, ) -> Result<(TimingResult, ExitStatus)> { #[cfg(unix)] let status = { diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 1d41c64d7..2cca85736 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,7 +8,9 @@ use std::cmp; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; -use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption}; +use crate::options::{ + CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, +}; use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; use crate::output::format::{format_duration, format_duration_unit}; use crate::output::progress_bar::get_progress_bar; @@ -56,12 +58,14 @@ impl<'a> Benchmark<'a> { &self, command: &Command<'_>, error_output: &'static str, + output_policy: &CommandOutputPolicy, ) -> Result { self.executor .run_command_and_measure( command, executor::BenchmarkIteration::NonBenchmarkRun, Some(CmdFailureAction::RaiseError), + output_policy, ) .map(|r| r.0) .map_err(|_| anyhow!(error_output)) @@ -71,6 +75,7 @@ impl<'a> Benchmark<'a> { fn run_setup_command( &self, parameters: impl IntoIterator>, + output_policy: &CommandOutputPolicy, ) -> Result { let command = self .options @@ -82,7 +87,7 @@ impl<'a> Benchmark<'a> { Append ' || true' to the command if you are sure that this can be ignored."; Ok(command - .map(|cmd| self.run_intermediate_command(&cmd, error_output)) + .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy)) .transpose()? .unwrap_or_default()) } @@ -91,6 +96,7 @@ impl<'a> Benchmark<'a> { fn run_cleanup_command( &self, parameters: impl IntoIterator>, + output_policy: &CommandOutputPolicy, ) -> Result { let command = self .options @@ -102,25 +108,33 @@ impl<'a> Benchmark<'a> { Append ' || true' to the command if you are sure that this can be ignored."; Ok(command - .map(|cmd| self.run_intermediate_command(&cmd, error_output)) + .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy)) .transpose()? .unwrap_or_default()) } /// Run the command specified by `--prepare`. - fn run_preparation_command(&self, command: &Command<'_>) -> Result { + fn run_preparation_command( + &self, + command: &Command<'_>, + output_policy: &CommandOutputPolicy, + ) -> Result { let error_output = "The preparation command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; - self.run_intermediate_command(command, error_output) + self.run_intermediate_command(command, error_output, output_policy) } /// Run the command specified by `--conclude`. - fn run_conclusion_command(&self, command: &Command<'_>) -> Result { + fn run_conclusion_command( + &self, + command: &Command<'_>, + output_policy: &CommandOutputPolicy, + ) -> Result { let error_output = "The conclusion command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; - self.run_intermediate_command(command, error_output) + self.run_intermediate_command(command, error_output, output_policy) } /// Run the benchmark for a single command @@ -140,6 +154,8 @@ impl<'a> Benchmark<'a> { let mut exit_codes: Vec> = vec![]; let mut all_succeeded = true; + let output_policy = &self.options.command_output_policies[self.number]; + let preparation_command = self.options.preparation_command.as_ref().map(|values| { let preparation_command = if values.len() == 1 { &values[0] @@ -152,10 +168,11 @@ impl<'a> Benchmark<'a> { self.command.get_parameters().iter().cloned(), ) }); + let run_preparation_command = || { preparation_command .as_ref() - .map(|cmd| self.run_preparation_command(cmd)) + .map(|cmd| self.run_preparation_command(cmd, output_policy)) .transpose() }; @@ -174,11 +191,11 @@ impl<'a> Benchmark<'a> { let run_conclusion_command = || { conclusion_command .as_ref() - .map(|cmd| self.run_conclusion_command(cmd)) + .map(|cmd| self.run_conclusion_command(cmd, output_policy)) .transpose() }; - self.run_setup_command(self.command.get_parameters().iter().cloned())?; + self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?; // Warmup phase if self.options.warmup_count > 0 { @@ -198,6 +215,7 @@ impl<'a> Benchmark<'a> { self.command, BenchmarkIteration::Warmup(i), None, + output_policy, )?; let _ = run_conclusion_command()?; if let Some(bar) = progress_bar.as_ref() { @@ -229,6 +247,7 @@ impl<'a> Benchmark<'a> { self.command, BenchmarkIteration::Benchmark(0), None, + output_policy, )?; let success = status.success(); @@ -289,6 +308,7 @@ impl<'a> Benchmark<'a> { self.command, BenchmarkIteration::Benchmark(i + 1), None, + output_policy, )?; let success = status.success(); @@ -418,7 +438,7 @@ impl<'a> Benchmark<'a> { println!(" "); } - self.run_cleanup_command(self.command.get_parameters().iter().cloned())?; + self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?; Ok(BenchmarkResult { command: self.command.get_name(), diff --git a/src/cli.rs b/src/cli.rs index 6684ebc86..5dbc56799 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -327,7 +327,7 @@ fn build_command() -> Command { Arg::new("output") .long("output") .conflicts_with("show-output") - .action(ArgAction::Set) + .action(ArgAction::Append) .value_name("WHERE") .help( "Control where the output of the benchmark is redirected. Note \ @@ -343,7 +343,11 @@ fn build_command() -> Command { \n \ inherit: Don't redirect the output at all (same as '--show-output').\n\ \n \ - : Write the output to the given file.", + : Write the output to the given file.\n\n\ + This option can be specified once for all commands or multiple times, once for \ + each command. Note: If you want to log the output of each and every iteration, \ + you can use a shell redirection and the '$HYPERFINE_ITERATION' environment variable:\n \ + hyperfine 'my-command > output-${HYPERFINE_ITERATION}.log'\n\n", ), ) .arg( diff --git a/src/command.rs b/src/command.rs index 50fa906aa..c0ae4975b 100644 --- a/src/command.rs +++ b/src/command.rs @@ -251,8 +251,8 @@ impl<'a> Commands<'a> { self.0.iter() } - pub fn num_commands(&self) -> usize { - self.0.len() + pub fn num_commands(&self, has_reference_command: bool) -> usize { + self.0.len() + if has_reference_command { 1 } else { 0 } } /// Finds all the strings that appear multiple times in the input iterator, returning them in diff --git a/src/main.rs b/src/main.rs index a75d4bd07..79acc0bbc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,7 +32,7 @@ fn run() -> Result<()> { colored::control::set_virtual_terminal(true).unwrap(); let cli_arguments = get_cli_arguments(env::args_os()); - let options = Options::from_cli_arguments(&cli_arguments)?; + let mut options = Options::from_cli_arguments(&cli_arguments)?; let commands = Commands::from_cli_arguments(&cli_arguments)?; let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.time_unit)?; diff --git a/src/options.rs b/src/options.rs index 72b1e1b0a..c9b0e44eb 100644 --- a/src/options.rs +++ b/src/options.rs @@ -234,8 +234,8 @@ pub struct Options { /// Where input to the benchmarked command comes from pub command_input_policy: CommandInputPolicy, - /// What to do with the output of the benchmarked command - pub command_output_policy: CommandOutputPolicy, + /// What to do with the output of the benchmarked commands + pub command_output_policies: Vec, /// Which time unit to use when displaying results pub time_unit: Option, @@ -257,7 +257,7 @@ impl Default for Options { sort_order_speed_comparison: SortOrder::MeanTime, sort_order_exports: SortOrder::Command, executor_kind: ExecutorKind::default(), - command_output_policy: CommandOutputPolicy::Null, + command_output_policies: vec![CommandOutputPolicy::Null], time_unit: None, command_input_policy: CommandInputPolicy::Null, } @@ -320,23 +320,28 @@ impl Options { options.cleanup_command = matches.get_one::("cleanup").map(String::from); - options.command_output_policy = if matches.get_flag("show-output") { - CommandOutputPolicy::Inherit - } else if let Some(output) = matches.get_one::("output").map(|s| s.as_str()) { - match output { - "null" => CommandOutputPolicy::Null, - "pipe" => CommandOutputPolicy::Pipe, - "inherit" => CommandOutputPolicy::Inherit, - arg => { - let path = PathBuf::from(arg); - if path.components().count() <= 1 { - return Err(OptionsError::UnknownOutputPolicy(arg.to_string())); + options.command_output_policies = if matches.get_flag("show-output") { + vec![CommandOutputPolicy::Inherit] + } else if let Some(output_values) = matches.get_many::("output") { + let mut policies = vec![]; + for value in output_values { + let policy = match value.as_str() { + "null" => CommandOutputPolicy::Null, + "pipe" => CommandOutputPolicy::Pipe, + "inherit" => CommandOutputPolicy::Inherit, + arg => { + let path = PathBuf::from(arg); + if path.components().count() <= 1 { + return Err(OptionsError::UnknownOutputPolicy(arg.to_string())); + } + CommandOutputPolicy::File(path) } - CommandOutputPolicy::File(path) - } + }; + policies.push(policy); } + policies } else { - CommandOutputPolicy::Null + vec![CommandOutputPolicy::Null] }; options.output_style = match matches.get_one::("style").map(|s| s.as_str()) { @@ -346,7 +351,10 @@ impl Options { Some("color") => OutputStyleOption::Color, Some("none") => OutputStyleOption::Disabled, _ => { - if options.command_output_policy == CommandOutputPolicy::Inherit + if options + .command_output_policies + .iter() + .any(|policy| *policy == CommandOutputPolicy::Inherit) || !io::stdout().is_terminal() { OutputStyleOption::Basic @@ -436,26 +444,34 @@ impl Options { Ok(options) } - pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> { - let num_commands = commands.num_commands() - + if self.reference_command.is_some() { - 1 - } else { - 0 - }; + pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> { + let has_reference_command = self.reference_command.is_some(); + let num_commands = commands.num_commands(has_reference_command); + if let Some(preparation_command) = &self.preparation_command { ensure!( preparation_command.len() <= 1 || num_commands == preparation_command.len(), - "The '--prepare' option has to be provided just once or N times, where N is the \ - number of benchmark commands including a potential reference." + "The '--prepare' option has to be provided just once or N times, where N={num_commands} is the \ + number of benchmark commands (including a potential reference)." ); } if let Some(conclusion_command) = &self.conclusion_command { ensure!( conclusion_command.len() <= 1 || num_commands == conclusion_command.len(), - "The '--conclude' option has to be provided just once or N times, where N is the \ - number of benchmark commands including a potential reference." + "The '--conclude' option has to be provided just once or N times, where N={num_commands} is the \ + number of benchmark commands (including a potential reference)." + ); + } + + if self.command_output_policies.len() == 1 { + self.command_output_policies = + vec![self.command_output_policies[0].clone(); num_commands]; + } else { + ensure!( + self.command_output_policies.len() == num_commands, + "The '--output' option has to be provided just once or N times, where N={num_commands} is the \ + number of benchmark commands (including a potential reference)." ); }