From 8c14c6adba282542fd0aee1a82436732f2142308 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Wed, 21 Feb 2024 20:38:10 +0100 Subject: [PATCH 1/4] feat: upload execution logs with the profile --- Cargo.lock | 212 ++++++++++++++------- Cargo.toml | 3 +- src/app.rs | 14 +- src/ci_provider/buildkite/logger.rs | 37 +++- src/ci_provider/buildkite/provider.rs | 6 +- src/ci_provider/github_actions/logger.rs | 25 ++- src/ci_provider/github_actions/provider.rs | 12 +- src/ci_provider/provider.rs | 6 +- src/logger.rs | 34 ++++ src/main.rs | 4 +- 10 files changed, 252 insertions(+), 101 deletions(-) create mode 100644 src/logger.rs diff --git a/Cargo.lock b/Cargo.lock index 7aeda46..887d449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,7 +254,6 @@ dependencies = [ "async-compression", "base64", "clap", - "env_logger", "insta", "itertools", "lazy_static", @@ -268,7 +267,9 @@ dependencies = [ "serde", "serde_json", "sha256", + "simplelog", "temp-env", + "tempfile", "tokio", "tokio-tar", "url", @@ -336,6 +337,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -367,19 +377,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "env_logger" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -388,12 +385,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -609,12 +606,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" - [[package]] name = "hex" version = "0.4.3" @@ -655,12 +646,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "hyper" version = "0.14.27" @@ -785,17 +770,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" -[[package]] -name = "is-terminal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" -dependencies = [ - "hermit-abi", - "rustix", - "windows-sys 0.48.0", -] - [[package]] name = "itertools" version = "0.11.0" @@ -828,9 +802,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "linked-hash-map" @@ -840,9 +814,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -926,6 +900,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.17" @@ -1121,6 +1101,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1329,15 +1315,15 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1458,6 +1444,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" +[[package]] +name = "simplelog" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" +dependencies = [ + "log", + "time", +] + [[package]] name = "slab" version = "0.4.9" @@ -1552,24 +1548,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.4.1", "rustix", - "windows-sys 0.48.0", -] - -[[package]] -name = "termcolor" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" -dependencies = [ - "winapi-util", + "windows-sys 0.52.0", ] [[package]] @@ -1592,6 +1578,37 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1933,15 +1950,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -1975,6 +1983,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -2005,6 +2022,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -2017,6 +2049,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -2029,6 +2067,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -2041,6 +2085,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -2053,6 +2103,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -2065,6 +2121,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -2077,6 +2139,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -2089,6 +2157,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" diff --git a/Cargo.toml b/Cargo.toml index 3a62030..f6c3f0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ publish = false [dependencies] anyhow = "1.0.75" clap = { version = "4.4.8", features = ["derive", "env"] } -env_logger = "0.10.1" itertools = "0.11.0" lazy_static = "1.4.0" log = "0.4.20" @@ -31,6 +30,8 @@ tokio-tar = "0.3.1" md5 = "0.7.0" base64 = "0.21.0" async-compression = { version = "0.4.5", features = ["tokio", "gzip"] } +simplelog = { version = "0.12.1", default-features = false } +tempfile = "3.10.0" [dev-dependencies] temp-env = { version = "0.3.6", features = ["async_closure"] } diff --git a/src/app.rs b/src/app.rs index a81c1d5..29ca3b1 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,6 +1,4 @@ -use std::env; - -use crate::{ci_provider, config::Config, prelude::*, runner, uploader, VERSION}; +use crate::{ci_provider, config::Config, logger::Logger, prelude::*, runner, uploader, VERSION}; use clap::Parser; fn show_banner() { @@ -82,20 +80,16 @@ pub async fn run() -> Result<()> { let args = AppArgs::parse(); let config = Config::try_from(args)?; let provider = ci_provider::get_provider(&config)?; - - let log_level = env::var("CODSPEED_LOG") - .ok() - .and_then(|log_level| log_level.parse::().ok()) - .unwrap_or(log::LevelFilter::Info); - log::set_max_level(log_level); - provider.setup_logger()?; + let logger = Logger::new(&provider)?; show_banner(); debug!("config: {:#?}", config); let run_data = runner::run(&config).await?; + if !config.skip_upload { start_group!("Upload the results"); + logger.persist_log_to_profile_folder(&run_data)?; uploader::upload(&config, provider, &run_data).await?; end_group!(); } diff --git a/src/ci_provider/buildkite/logger.rs b/src/ci_provider/buildkite/logger.rs index 7855f82..33462a9 100644 --- a/src/ci_provider/buildkite/logger.rs +++ b/src/ci_provider/buildkite/logger.rs @@ -1,11 +1,24 @@ -use log::*; - use crate::ci_provider::logger::{get_group_event, GroupEvent}; +use log::*; +use simplelog::SharedLogger; +use std::{env, io::Write}; /// A logger that prints logs in the format expected by Buildkite /// /// See https://buildkite.com/docs/pipelines/managing-log-output -pub struct BuildkiteLogger; +pub struct BuildkiteLogger { + log_level: LevelFilter, +} + +impl BuildkiteLogger { + pub fn new() -> Self { + let log_level = env::var("CODSPEED_LOG") + .ok() + .and_then(|log_level| log_level.parse::().ok()) + .unwrap_or(log::LevelFilter::Info); + Self { log_level } + } +} impl Log for BuildkiteLogger { fn enabled(&self, _metadata: &Metadata) -> bool { @@ -49,5 +62,21 @@ impl Log for BuildkiteLogger { } } - fn flush(&self) {} + fn flush(&self) { + std::io::stdout().flush().unwrap(); + } +} + +impl SharedLogger for BuildkiteLogger { + fn level(&self) -> LevelFilter { + self.log_level + } + + fn config(&self) -> Option<&simplelog::Config> { + None + } + + fn as_log(self: Box) -> Box { + Box::new(*self) + } } diff --git a/src/ci_provider/buildkite/provider.rs b/src/ci_provider/buildkite/provider.rs index 8283008..cd935cf 100644 --- a/src/ci_provider/buildkite/provider.rs +++ b/src/ci_provider/buildkite/provider.rs @@ -2,6 +2,7 @@ use std::env; use lazy_static::lazy_static; use regex::Regex; +use simplelog::SharedLogger; use crate::{ ci_provider::{ @@ -125,9 +126,8 @@ impl CIProviderDetector for BuildkiteProvider { } impl CIProvider for BuildkiteProvider { - fn setup_logger(&self) -> Result<()> { - log::set_logger(&BuildkiteLogger)?; - Ok(()) + fn get_logger(&self) -> Box { + Box::new(BuildkiteLogger::new()) } fn get_provider_name(&self) -> &'static str { diff --git a/src/ci_provider/github_actions/logger.rs b/src/ci_provider/github_actions/logger.rs index 60eacb9..f7e6152 100644 --- a/src/ci_provider/github_actions/logger.rs +++ b/src/ci_provider/github_actions/logger.rs @@ -1,6 +1,7 @@ -use log::*; - use crate::ci_provider::logger::{get_group_event, GroupEvent}; +use log::*; +use simplelog::SharedLogger; +use std::io::Write; /// A logger that prints logs in the format expected by GitHub Actions, with grouping support. /// @@ -41,5 +42,23 @@ impl Log for GithubActionLogger { lines.for_each(|line| println!("{}{}", prefix, line)); } - fn flush(&self) {} + fn flush(&self) { + std::io::stdout().flush().unwrap(); + } +} + +impl SharedLogger for GithubActionLogger { + fn level(&self) -> LevelFilter { + // since TRACE and DEBUG use ::debug::, we always enable them and let GitHub handle the filtering + // thanks to https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging#enabling-step-debug-logging + LevelFilter::Trace + } + + fn config(&self) -> Option<&simplelog::Config> { + None + } + + fn as_log(self: Box) -> Box { + Box::new(*self) + } } diff --git a/src/ci_provider/github_actions/provider.rs b/src/ci_provider/github_actions/provider.rs index 8e18585..efc9d32 100644 --- a/src/ci_provider/github_actions/provider.rs +++ b/src/ci_provider/github_actions/provider.rs @@ -1,8 +1,8 @@ -use std::{env, fs}; - use lazy_static::lazy_static; use regex::Regex; use serde_json::Value; +use simplelog::SharedLogger; +use std::{env, fs}; use crate::{ ci_provider::{ @@ -116,12 +116,8 @@ impl CIProviderDetector for GitHubActionsProvider { } impl CIProvider for GitHubActionsProvider { - fn setup_logger(&self) -> Result<()> { - log::set_logger(&GithubActionLogger)?; - // since TRACE and DEBUG use ::debug::, we always enable them and let GitHub handle the filtering - // thanks to https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging#enabling-step-debug-logging - log::set_max_level(log::LevelFilter::Trace); - Ok(()) + fn get_logger(&self) -> Box { + Box::new(GithubActionLogger) } fn get_provider_name(&self) -> &'static str { diff --git a/src/ci_provider/provider.rs b/src/ci_provider/provider.rs index 35b8ada..4f411bd 100644 --- a/src/ci_provider/provider.rs +++ b/src/ci_provider/provider.rs @@ -1,3 +1,5 @@ +use simplelog::SharedLogger; + use crate::config::Config; use crate::prelude::*; use crate::uploader::{Runner, UploadMetadata}; @@ -11,8 +13,8 @@ pub trait CIProviderDetector { /// `CIProvider` is a trait that defines the necessary methods for a continuous integration provider. pub trait CIProvider { - /// Registers the logger for the CI provider. - fn setup_logger(&self) -> Result<()>; + /// Returns the logger for the CI provider. + fn get_logger(&self) -> Box; /// Returns the name of the CI provider. /// diff --git a/src/logger.rs b/src/logger.rs new file mode 100644 index 0000000..fb859f9 --- /dev/null +++ b/src/logger.rs @@ -0,0 +1,34 @@ +use crate::runner::RunData; +use crate::{ci_provider::CIProvider, prelude::*}; +use log::LevelFilter; +use simplelog::{CombinedLogger, WriteLogger}; +use std::fs::copy; +use std::path::PathBuf; +use tempfile::NamedTempFile; + +pub struct Logger { + log_file_path: PathBuf, +} + +impl Logger { + #[allow(clippy::borrowed_box)] + pub fn new(provider: &Box) -> Result { + let provider_logger = provider.get_logger(); + let log_file = NamedTempFile::new().context("Failed to create log file")?; + let log_file_path = log_file.path().to_path_buf(); + let file_logger = + WriteLogger::new(LevelFilter::Trace, simplelog::Config::default(), log_file); + CombinedLogger::init(vec![provider_logger, file_logger]) + .context("Failed to init logger")?; + Ok(Self { log_file_path }) + } + + pub fn persist_log_to_profile_folder(&self, run_data: &RunData) -> Result<()> { + let profile_folder = run_data.profile_folder.clone(); + let dest_log_file_path = profile_folder.join("runner.log"); + debug!("Persisting log file to {}", dest_log_file_path.display()); + log::logger().flush(); + copy(&self.log_file_path, dest_log_file_path).context("Failed to copy log file")?; + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index ce75316..51e4233 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,14 +3,16 @@ mod ci_provider; mod config; mod helpers; mod instruments; +mod logger; mod prelude; mod request_client; mod runner; mod uploader; -use log::log_enabled; use prelude::*; +use log::log_enabled; + pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const MONGODB_TRACER_VERSION: &str = "cs-mongo-tracer-v0.2.0"; pub const VALGRIND_CODSPEED_VERSION: &str = "3.21.0-0codspeed1"; From 5ee21e0b31f01a4986930f1d0c573de9c604e5ff Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Thu, 22 Feb 2024 11:02:20 +0100 Subject: [PATCH 2/4] feat: include the execution output in the logs --- src/ci_provider/buildkite/logger.rs | 8 ++++- src/ci_provider/github_actions/logger.rs | 8 ++++- src/ci_provider/logger.rs | 7 ++++ src/logger.rs | 9 +++-- src/runner/mod.rs | 1 + src/runner/valgrind.rs | 44 ++++++++++++++++++++++-- 6 files changed, 70 insertions(+), 7 deletions(-) diff --git a/src/ci_provider/buildkite/logger.rs b/src/ci_provider/buildkite/logger.rs index 33462a9..188c470 100644 --- a/src/ci_provider/buildkite/logger.rs +++ b/src/ci_provider/buildkite/logger.rs @@ -1,4 +1,6 @@ -use crate::ci_provider::logger::{get_group_event, GroupEvent}; +use crate::ci_provider::logger::{ + get_group_event, should_provider_logger_handle_record, GroupEvent, +}; use log::*; use simplelog::SharedLogger; use std::{env, io::Write}; @@ -26,6 +28,10 @@ impl Log for BuildkiteLogger { } fn log(&self, record: &Record) { + if !should_provider_logger_handle_record(record) { + return; + } + let level = record.level(); let message = record.args(); diff --git a/src/ci_provider/github_actions/logger.rs b/src/ci_provider/github_actions/logger.rs index f7e6152..59f35c0 100644 --- a/src/ci_provider/github_actions/logger.rs +++ b/src/ci_provider/github_actions/logger.rs @@ -1,4 +1,6 @@ -use crate::ci_provider::logger::{get_group_event, GroupEvent}; +use crate::ci_provider::logger::{ + get_group_event, should_provider_logger_handle_record, GroupEvent, +}; use log::*; use simplelog::SharedLogger; use std::io::Write; @@ -14,6 +16,10 @@ impl Log for GithubActionLogger { } fn log(&self, record: &Record) { + if !should_provider_logger_handle_record(record) { + return; + } + let level = record.level(); let message = record.args(); diff --git a/src/ci_provider/logger.rs b/src/ci_provider/logger.rs index 63afd64..ebd6193 100644 --- a/src/ci_provider/logger.rs +++ b/src/ci_provider/logger.rs @@ -1,3 +1,5 @@ +use crate::runner::VALGRIND_EXECUTION_TARGET; + /// This target is used exclusively to handle group events. pub const GROUP_TARGET: &str = "codspeed::group"; pub const OPENED_GROUP_TARGET: &str = "codspeed::group::opened"; @@ -71,3 +73,8 @@ pub(super) fn get_group_event(record: &log::Record) -> Option { _ => None, } } + +pub(super) fn should_provider_logger_handle_record(record: &log::Record) -> bool { + // Provider logger should handle all records except the ones from the valgrind execution target + record.target() != VALGRIND_EXECUTION_TARGET +} diff --git a/src/logger.rs b/src/logger.rs index fb859f9..beeed01 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -1,3 +1,4 @@ +use crate::ci_provider::logger::{GROUP_TARGET, OPENED_GROUP_TARGET}; use crate::runner::RunData; use crate::{ci_provider::CIProvider, prelude::*}; use log::LevelFilter; @@ -16,8 +17,12 @@ impl Logger { let provider_logger = provider.get_logger(); let log_file = NamedTempFile::new().context("Failed to create log file")?; let log_file_path = log_file.path().to_path_buf(); - let file_logger = - WriteLogger::new(LevelFilter::Trace, simplelog::Config::default(), log_file); + let file_logger_config = simplelog::ConfigBuilder::new() + // Groups are not logged to the file + .add_filter_ignore_str(GROUP_TARGET) + .add_filter_ignore_str(OPENED_GROUP_TARGET) + .build(); + let file_logger = WriteLogger::new(LevelFilter::Trace, file_logger_config, log_file); CombinedLogger::init(vec![provider_logger, file_logger]) .context("Failed to init logger")?; Ok(Self { log_file_path }) diff --git a/src/runner/mod.rs b/src/runner/mod.rs index 9f34d35..ecd9468 100644 --- a/src/runner/mod.rs +++ b/src/runner/mod.rs @@ -6,3 +6,4 @@ mod valgrind; pub use self::run::RunData; pub use run::run; +pub use valgrind::VALGRIND_EXECUTION_TARGET; diff --git a/src/runner/valgrind.rs b/src/runner/valgrind.rs index e49164f..0877c80 100644 --- a/src/runner/valgrind.rs +++ b/src/runner/valgrind.rs @@ -4,10 +4,12 @@ use crate::prelude::*; use crate::runner::helpers::ignored_objects_path::get_objects_path_to_ignore; use crate::runner::helpers::introspected_node::setup_introspected_node; use lazy_static::lazy_static; -use std::env; use std::fs::canonicalize; +use std::io::{Read, Write}; use std::path::Path; +use std::process::ExitStatus; use std::{collections::HashMap, env::consts::ARCH, process::Command}; +use std::{env, thread}; lazy_static! { static ref BASE_INJECTED_ENV: HashMap<&'static str, String> = { @@ -54,6 +56,43 @@ fn get_bench_command(config: &Config) -> String { .replace("cargo codspeed", "cargo-codspeed") } +pub const VALGRIND_EXECUTION_TARGET: &str = "valgrind::execution"; + +fn run_command_with_log_pipe(mut cmd: Command) -> Result { + fn log_tee( + mut reader: impl Read, + mut writer: impl Write, + log_prefix: Option<&str>, + ) -> Result<()> { + let prefix = log_prefix.unwrap_or(""); + let mut buffer = [0; 1024]; + loop { + let bytes_read = reader.read(&mut buffer)?; + if bytes_read == 0 { + break; + } + writer.write_all(&buffer[..bytes_read])?; + trace!(target: VALGRIND_EXECUTION_TARGET, "{}{}", prefix, String::from_utf8_lossy(&buffer[..bytes_read])); + } + Ok(()) + } + + let mut process = cmd + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("failed to spawn the process")?; + let stdout = process.stdout.take().expect("unable to get stdout"); + let stderr = process.stderr.take().expect("unable to get stderr"); + thread::spawn(move || { + log_tee(stdout, std::io::stdout(), None).unwrap(); + }); + thread::spawn(move || { + log_tee(stderr, std::io::stderr(), Some("[stderr]")).unwrap(); + }); + process.wait().context("failed to wait for the process") +} + pub fn measure( config: &Config, profile_folder: &Path, @@ -102,8 +141,7 @@ pub fn measure( } debug!("cmd: {:?}", cmd); - let status = cmd - .status() + let status = run_command_with_log_pipe(cmd) .map_err(|e| anyhow!("failed to execute the benchmark process. {}", e))?; if !status.success() { bail!("failed to execute the benchmark process"); From eede9103843e2cf09049b6f0a9c6121da644b567 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Thu, 22 Feb 2024 11:33:42 +0100 Subject: [PATCH 3/4] chore: enforce tag signing with cargo release --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index f6c3f0c..dec3d28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,10 @@ temp-env = { version = "0.3.6", features = ["async_closure"] } insta = { version = "1.29.0", features = ["json", "redactions"] } +[workspace.metadata.release] +sign-tag = true +sign-commit = true + [profile.dist] inherits = "release" lto = "thin" From e614b72140e0429c2461a0e1b88a3c4bfffb4722 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Thu, 22 Feb 2024 11:50:01 +0100 Subject: [PATCH 4/4] fix: properly handle log levels with buildkite --- src/ci_provider/buildkite/logger.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ci_provider/buildkite/logger.rs b/src/ci_provider/buildkite/logger.rs index 188c470..622deb6 100644 --- a/src/ci_provider/buildkite/logger.rs +++ b/src/ci_provider/buildkite/logger.rs @@ -48,6 +48,9 @@ impl Log for BuildkiteLogger { return; } + if level > self.log_level { + return; + } // there is no support for log levels in Buildkite, so we print the level in the message match level { Level::Error => {