From a6c5a637d43a54d4fee36b655b0626389d0b179a Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 15:22:30 -0400 Subject: [PATCH 01/10] WIP --- src/harness/mod.rs | 80 ++++++++++++++++++++++++++++++++++++++-------- src/report.rs | 20 ++++++++---- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/src/harness/mod.rs b/src/harness/mod.rs index ae8379a..01f53da 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -262,31 +262,85 @@ impl TestHarness { WriteImpl::HarnessCallback => { // Do nothing, implicit default } - WriteImpl::Print => { + other => { output.push_str(separator); - output.push_str("print"); - } - WriteImpl::Assert => { - output.push_str(separator); - output.push_str("assert"); - } - WriteImpl::Noop => { - output.push_str(separator); - output.push_str("noop"); + output.push_str(&other.to_string()) } } match val_generator { ValueGeneratorKind::Graffiti => { // Do nothing, implicit default } - ValueGeneratorKind::Random { seed } => { + other => { output.push_str(separator); - output.push_str(&format!("random{seed}")); + output.push_str(&other.to_string()) } } output } + pub fn parse_test_key(&self, input: &str) -> Result { + let separator = "::"; + let parts = input.split(separator).collect::>(); + + let [test, rest @ ..] = &parts[..] else { + todo!(); + }; + let mut key = TestKey { + test: test.to_string(), + caller: String::new(), + callee: String::new(), + options: TestOptions { + convention: CallingConvention::C, + repr: LangRepr::C, + functions: FunctionSelector::All, + val_writer: WriteImpl::HarnessCallback, + val_generator: ValueGeneratorKind::Graffiti, + }, + }; + for part in rest { + // pairs + if let Some((caller, callee)) = part.split_once("_calls_") { + key.caller = caller.to_owned(); + key.callee = callee.to_owned(); + continue; + } + if let Some(caller) = part.strip_suffix("_caller") { + key.caller = caller.to_owned(); + continue; + } + if let Some(callee) = part.strip_suffix("_callee") { + key.callee = callee.to_owned(); + continue; + } + + // repr + if let Some(repr) = part.strip_prefix("repr_") { + key.options.repr = repr.parse()?; + continue; + } + + // conv + if let Some(conv) = part.strip_prefix("conv_") { + key.options.convention = conv.parse()?; + continue; + } + // generator + if let Ok(val_generator) = part.parse() { + key.options.val_generator = val_generator; + continue; + } + // writer + if let Ok(val_writer) = part.parse() { + key.options.val_writer = val_writer; + continue; + } + + return Err(format!("unknown testkey part: {part}")) + } + Ok(key) + } + /// The name of a test for pretty-printing. pub fn full_test_name(&self, key: &TestKey) -> String { self.base_id(key, None, "::") @@ -297,4 +351,4 @@ impl TestHarness { let base = self.full_test_name(key); format!("{base}::{func_name}") } -} +} \ No newline at end of file diff --git a/src/report.rs b/src/report.rs index 958ab43..1865e80 100644 --- a/src/report.rs +++ b/src/report.rs @@ -55,6 +55,15 @@ pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolc result } +struct ExpectFile { + targets: IndexMap, +} +struct ExpectEntry { + tests: IndexMap +} + +R + impl Serialize for BuildError { fn serialize(&self, serializer: S) -> Result where @@ -209,7 +218,7 @@ impl TestKey { } } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct TestRules { pub run: TestRunMode, pub check: TestCheckMode, @@ -218,9 +227,8 @@ pub struct TestRules { /// How far the test should be executed /// /// Each case implies all the previous cases. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize)] -#[allow(dead_code)] - +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum TestRunMode { /// Don't run the test at all (marked as skipped) Skip, @@ -240,7 +248,7 @@ pub enum TestRunMode { /// /// Tests that are Skipped ignore this. #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize)] -#[allow(dead_code)] +#[serde(rename_all = "kebab-case")] pub enum TestCheckMode { /// The test must successfully complete this phase, /// whatever happens after that is gravy. @@ -252,7 +260,7 @@ pub enum TestCheckMode { Busted(TestRunMode), /// The test is flakey and random but we want to run it anyway, /// so accept whatever result we get as ok. - Random, + Random(bool), } #[derive(Debug, Serialize)] From 0afa74251d7be862b43d0e7a117e2fc5461598d2 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 19:16:10 -0400 Subject: [PATCH 02/10] feat: runtime test expectation file fixes #9 --- Cargo.lock | 55 +++++++++- Cargo.toml | 6 +- abi-cafe-rules-example.toml | 28 +++++ src/cli.rs | 9 ++ src/error.rs | 2 + src/files.rs | 1 + src/harness/build.rs | 2 +- src/harness/check.rs | 1 - src/harness/mod.rs | 61 ++++++----- src/harness/read.rs | 15 +++ src/{ => harness}/report.rs | 213 +++++++++++++++++++++++++++--------- src/harness/run.rs | 2 +- src/harness/test.rs | 6 +- src/main.rs | 7 +- 14 files changed, 314 insertions(+), 94 deletions(-) create mode 100644 abi-cafe-rules-example.toml rename src/{ => harness}/report.rs (74%) diff --git a/Cargo.lock b/Cargo.lock index e644eed..c3f3c63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "toml 0.8.14", "tracing", "tracing-subscriber", ] @@ -169,7 +170,7 @@ checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996" dependencies = [ "semver", "serde", - "toml", + "toml 0.5.11", "url", ] @@ -841,6 +842,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1048,6 +1058,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1331,3 +1375,12 @@ name = "windows_x86_64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] diff --git a/Cargo.toml b/Cargo.toml index ad8d6c9..8d34963 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ cc.workspace = true clap.workspace = true console.workspace = true kdl.workspace = true +include_dir.workspace = true libloading.workspace = true linked-hash-map.workspace = true miette.workspace = true @@ -32,7 +33,8 @@ thiserror.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -include_dir = "0.7.4" +toml.workspace = true + [build-dependencies] built.workspace = true @@ -60,6 +62,7 @@ camino = { version = "1.1.7", features = ["serde1"] } cc = { version = "1.1.0" } clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] } console = "0.15.8" +include_dir = "0.7.4" kdl = "4.6.0" libloading = "0.7.3" linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] } @@ -73,6 +76,7 @@ serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.83" thiserror = "1.0.30" tokio = { version = "1.37.0", features = ["full", "tracing"] } +toml = "0.8.14" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } # build diff --git a/abi-cafe-rules-example.toml b/abi-cafe-rules-example.toml new file mode 100644 index 0000000..6d35176 --- /dev/null +++ b/abi-cafe-rules-example.toml @@ -0,0 +1,28 @@ +# Here are some example annotations for test expecations + +# this test fails on this platform, with this toolchain pairing +[targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"] +fail = "check" + +# this test has random results on this platform, whenever rustc is the caller (callee also supported) +[targets.x86_64-pc-windows-msvc."simple::rustc_caller"] +random = true + +# whenever this test involves cc, only link it, and expect linking to fail +[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] +run = "link" +fail = "link" + +# any repr(c) version of this test fails to run +[targets.x86_64-unknown-linux-gnu."simple::repr_c"] +busted = "run" + +# for this pairing, with the rust calling convention, only generate the test, and expect it to work +[targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"] +run = "generate" +pass = "generate" + +# can match all tests with leading :: +[targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"] +run = "generate" +pass = "generate" diff --git a/src/cli.rs b/src/cli.rs index 0e08daf..6c658c0 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -130,6 +130,12 @@ struct Cli { #[clap(long)] add_tests: Option, + /// Add the test expectations at the given path + /// + /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir) + #[clap(long)] + rules: Option, + /// disable the builtin tests /// /// See also `--add-tests` @@ -154,6 +160,7 @@ pub fn make_app() -> Config { output_format, add_rustc_codegen_backend, add_tests, + rules, disable_builtin_tests, // unimplemented select_vals: _, @@ -228,12 +235,14 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`. let out_dir = target_dir.join("temp"); let generated_src_dir = target_dir.join("generated_impls"); let runtime_test_input_dir = add_tests; + let runtime_rules_file = rules.unwrap_or_else(|| "abi-cafe-rules.toml".into()); let paths = Paths { target_dir, out_dir, generated_src_dir, runtime_test_input_dir, + runtime_rules_file, }; Config { output_format, diff --git a/src/error.rs b/src/error.rs index 723da86..f007ec5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,8 @@ pub enum GenerateError { #[error(transparent)] #[diagnostic(transparent)] KdlScriptError(#[from] kdl_script::KdlScriptError), + #[error(transparent)] + TomlError(#[from] toml::de::Error), /// Used to signal we just skipped it #[error("")] Skipped, diff --git a/src/files.rs b/src/files.rs index f5108b1..669b6db 100644 --- a/src/files.rs +++ b/src/files.rs @@ -13,6 +13,7 @@ pub struct Paths { pub out_dir: Utf8PathBuf, pub generated_src_dir: Utf8PathBuf, pub runtime_test_input_dir: Option, + pub runtime_rules_file: Utf8PathBuf, } impl Paths { pub fn harness_dylib_main_file(&self) -> Utf8PathBuf { diff --git a/src/harness/build.rs b/src/harness/build.rs index ed70068..4d80b13 100644 --- a/src/harness/build.rs +++ b/src/harness/build.rs @@ -5,8 +5,8 @@ use camino::Utf8Path; use tracing::info; use crate::error::*; +use crate::harness::report::*; use crate::harness::test::*; -use crate::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/check.rs b/src/harness/check.rs index 38cf7f5..8bc6cbf 100644 --- a/src/harness/check.rs +++ b/src/harness/check.rs @@ -5,7 +5,6 @@ use kdl_script::types::Ty; use tracing::{error, info}; use crate::error::*; -use crate::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/mod.rs b/src/harness/mod.rs index 01f53da..49bab66 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -17,11 +17,12 @@ mod build; mod check; mod generate; mod read; +pub mod report; mod run; pub mod test; pub mod vals; -pub use read::{find_tests, spawn_read_test}; +pub use read::{find_test_rules, find_tests, spawn_read_test}; pub use run::TestBuffer; pub type Memoized = Mutex>>>; @@ -30,6 +31,7 @@ pub struct TestHarness { paths: Paths, toolchains: Toolchains, tests: SortedMap>, + test_rules: ExpectFile, tests_with_vals: Memoized<(TestId, ValueGeneratorKind), Arc>, tests_with_toolchain: Memoized<(TestId, ValueGeneratorKind, ToolchainId), Arc>, @@ -39,11 +41,12 @@ pub struct TestHarness { } impl TestHarness { - pub fn new(tests: SortedMap>, cfg: &Config) -> Self { + pub fn new(test_rules: ExpectFile, tests: SortedMap>, cfg: &Config) -> Self { let toolchains = toolchains::create_toolchains(cfg); Self { paths: cfg.paths.clone(), tests, + test_rules, toolchains, tests_with_vals: Default::default(), tests_with_toolchain: Default::default(), @@ -109,13 +112,6 @@ impl TestHarness { .clone(); Ok(output) } - pub fn get_test_rules(&self, test_key: &TestKey) -> TestRules { - let caller = self.toolchains[&test_key.caller].clone(); - let callee = self.toolchains[&test_key.callee].clone(); - - get_test_rules(test_key, &*caller, &*callee) - } - pub fn spawn_test( self: Arc, rt: &tokio::runtime::Runtime, @@ -279,64 +275,69 @@ impl TestHarness { output } - pub fn parse_test_key(&self, input: &str) -> Result { + pub fn parse_test_pattern(&self, input: &str) -> Result { let separator = "::"; let parts = input.split(separator).collect::>(); let [test, rest @ ..] = &parts[..] else { todo!(); }; - let mut key = TestKey { - test: test.to_string(), - caller: String::new(), - callee: String::new(), - options: TestOptions { - convention: CallingConvention::C, - repr: LangRepr::C, - functions: FunctionSelector::All, - val_writer: WriteImpl::HarnessCallback, - val_generator: ValueGeneratorKind::Graffiti, + let mut key = TestKeyPattern { + test: (!test.is_empty()).then(|| test.to_string()), + caller: None, + callee: None, + toolchain: None, + options: TestOptionsPattern { + convention: None, + repr: None, + functions: None, + val_writer: None, + val_generator: None, }, }; for part in rest { // pairs if let Some((caller, callee)) = part.split_once("_calls_") { - key.caller = caller.to_owned(); - key.callee = callee.to_owned(); + key.caller = Some(caller.to_owned()); + key.callee = Some(callee.to_owned()); continue; } if let Some(caller) = part.strip_suffix("_caller") { - key.caller = caller.to_owned(); + key.caller = Some(caller.to_owned()); continue; } if let Some(callee) = part.strip_suffix("_callee") { - key.callee = callee.to_owned(); + key.callee = Some(callee.to_owned()); + continue; + } + if let Some(toolchain) = part.strip_suffix("_toolchain") { + key.toolchain = Some(toolchain.to_owned()); continue; } // repr if let Some(repr) = part.strip_prefix("repr_") { - key.options.repr = repr.parse()?; + key.options.repr = Some(repr.parse()?); continue; } // conv if let Some(conv) = part.strip_prefix("conv_") { - key.options.convention = conv.parse()?; + key.options.convention = Some(conv.parse()?); continue; } // generator if let Ok(val_generator) = part.parse() { - key.options.val_generator = val_generator; + key.options.val_generator = Some(val_generator); continue; } // writer if let Ok(val_writer) = part.parse() { - key.options.val_writer = val_writer; + key.options.val_writer = Some(val_writer); continue; } - return Err(format!("unknown testkey part: {part}")) + return Err(format!("unknown testkey part: {part}")); } Ok(key) } @@ -351,4 +352,4 @@ impl TestHarness { let base = self.full_test_name(key); format!("{base}::{func_name}") } -} \ No newline at end of file +} diff --git a/src/harness/read.rs b/src/harness/read.rs index a05fc95..cfd56f5 100644 --- a/src/harness/read.rs +++ b/src/harness/read.rs @@ -32,6 +32,21 @@ impl Pathish { } } +pub fn find_test_rules(cfg: &Config) -> Result { + let rules = find_test_rules_runtime(&cfg.paths.runtime_rules_file)?; + Ok(rules) +} + +pub fn find_test_rules_runtime(rule_file: &Utf8Path) -> Result { + if rule_file.exists() { + let data = read_runtime_file_to_string(rule_file)?; + let rules = toml::from_str(&data)?; + Ok(rules) + } else { + Ok(ExpectFile::default()) + } +} + pub fn find_tests(cfg: &Config) -> Result, GenerateError> { let mut tests = find_tests_runtime(cfg.paths.runtime_test_input_dir.as_deref())?; let mut more_tests = find_tests_static(cfg.disable_builtin_tests)?; diff --git a/src/report.rs b/src/harness/report.rs similarity index 74% rename from src/report.rs rename to src/harness/report.rs index 1865e80..461c29f 100644 --- a/src/report.rs +++ b/src/harness/report.rs @@ -1,68 +1,83 @@ use camino::Utf8PathBuf; use console::Style; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::json; use crate::error::*; use crate::harness::test::*; -use crate::toolchains::*; use crate::*; /// These are the builtin test-expectations, edit these if there are new rules! -#[allow(unused_variables)] -pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolchain) -> TestRules { - use TestCheckMode::*; - use TestRunMode::*; - - // By default, require tests to run completely and pass - let mut result = TestRules { - run: Check, - check: Pass(Check), - }; +impl TestHarness { + #[allow(unused_variables)] + pub fn get_test_rules(&self, key: &TestKey) -> TestRules { + let caller = self.toolchain_by_test_key(key, CallSide::Caller); + let callee = self.toolchain_by_test_key(key, CallSide::Caller); - // Now apply specific custom expectations for platforms/suites - let is_c = caller.lang() == "c" || callee.lang() == "c"; - let is_rust = caller.lang() == "rust" || callee.lang() == "rust"; - let is_rust_and_c = is_c && is_rust; - - // i128 types are fake on windows so this is all random garbage that might - // not even compile, but that datapoint is a little interesting/useful - // so let's keep running them and just ignore the result for now. - // - // Anyone who cares about this situation more can make the expectations more precise. - if cfg!(windows) && (test.test == "i128" || test.test == "u128") { - result.check = Random; - } + use TestCheckMode::*; + use TestRunMode::*; - // CI GCC is too old to support `_Float16`. - if cfg!(all(target_arch = "x86_64", target_os = "linux")) && is_c && test.test == "f16" { - result.check = Random; - } + // By default, require tests to run completely and pass + let mut result = TestRules { + run: Check, + check: Pass(Check), + }; - // FIXME: investigate why this is failing to build - if cfg!(windows) && is_c && (test.test == "EmptyStruct" || test.test == "EmptyStructInside") { - result.check = Busted(Build); - } + // Now apply specific custom expectations for platforms/suites + let is_c = caller.lang() == "c" || callee.lang() == "c"; + let is_rust = caller.lang() == "rust" || callee.lang() == "rust"; + let is_rust_and_c = is_c && is_rust; + + // i128 types are fake on windows so this is all random garbage that might + // not even compile, but that datapoint is a little interesting/useful + // so let's keep running them and just ignore the result for now. + // + // Anyone who cares about this situation more can make the expectations more precise. + if cfg!(windows) && (key.test == "i128" || key.test == "u128") { + result.check = Random(true); + } - // - // - // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + // CI GCC is too old to support `_Float16`. + if cfg!(all(target_arch = "x86_64", target_os = "linux")) && is_c && key.test == "f16" { + result.check = Random(true); + } - // END OF VENDOR RESERVED AREA - // - // + // FIXME: investigate why this is failing to build + if cfg!(windows) && is_c && (key.test == "EmptyStruct" || key.test == "EmptyStructInside") { + result.check = Busted(Build); + } - result + // + // + // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + + // END OF VENDOR RESERVED AREA + // + // + + if let Some(rules) = self.test_rules.targets.get(built_info::TARGET) { + for (pattern, rules) in rules { + let pat = self + .parse_test_pattern(pattern) + .expect("failed to parse test pattern"); + if pat.matches(key) { + if let Some(run) = rules.run.clone() { + result.run = run; + } + if let Some(check) = rules.check.clone() { + result.check = check; + } + } + } + } + result + } } -struct ExpectFile { - targets: IndexMap, +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ExpectFile { + pub targets: SortedMap>, } -struct ExpectEntry { - tests: IndexMap -} - -R impl Serialize for BuildError { fn serialize(&self, serializer: S) -> Result @@ -156,7 +171,7 @@ pub fn report_test(results: TestRunResults) -> TestReport { .map(|r| !r.all_passed) .unwrap_or(false), }, - TestCheckMode::Random => true, + TestCheckMode::Random(_) => true, }; if passed { if matches!(results.rules.check, TestCheckMode::Busted(_)) { @@ -209,6 +224,23 @@ pub struct TestKey { pub callee: ToolchainId, pub options: TestOptions, } + +#[derive(Debug, Clone)] +pub struct TestKeyPattern { + pub test: Option, + pub caller: Option, + pub callee: Option, + pub toolchain: Option, + pub options: TestOptionsPattern, +} +#[derive(Debug, Clone)] +pub struct TestOptionsPattern { + pub convention: Option, + pub functions: Option, + pub val_writer: Option, + pub val_generator: Option, + pub repr: Option, +} impl TestKey { pub(crate) fn toolchain_id(&self, call_side: CallSide) -> &str { match call_side { @@ -218,12 +250,88 @@ impl TestKey { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +impl TestKeyPattern { + fn matches(&self, key: &TestKey) -> bool { + let TestKeyPattern { + test, + caller, + callee, + toolchain, + options: + TestOptionsPattern { + convention, + functions, + val_writer, + val_generator, + repr, + }, + } = self; + + if let Some(test) = test { + if test != &key.test { + return false; + } + } + + if let Some(caller) = caller { + if caller != &key.caller { + return false; + } + } + if let Some(callee) = callee { + if callee != &key.callee { + return false; + } + } + if let Some(toolchain) = toolchain { + if toolchain != &key.caller && toolchain != &key.callee { + return false; + } + } + + if let Some(convention) = convention { + if convention != &key.options.convention { + return false; + } + } + if let Some(functions) = functions { + if functions != &key.options.functions { + return false; + } + } + if let Some(val_writer) = val_writer { + if val_writer != &key.options.val_writer { + return false; + } + } + if let Some(val_generator) = val_generator { + if val_generator != &key.options.val_generator { + return false; + } + } + if let Some(repr) = repr { + if repr != &key.options.repr { + return false; + } + } + + true + } +} + +#[derive(Debug, Clone, Serialize)] pub struct TestRules { pub run: TestRunMode, + #[serde(flatten)] pub check: TestCheckMode, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestRulesPattern { + pub run: Option, + #[serde(flatten)] + pub check: Option, +} /// How far the test should be executed /// /// Each case implies all the previous cases. @@ -247,7 +355,7 @@ pub enum TestRunMode { /// To what level of correctness should the test be graded? /// /// Tests that are Skipped ignore this. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize)] +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum TestCheckMode { /// The test must successfully complete this phase, @@ -315,6 +423,7 @@ pub struct CheckOutput { } #[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] pub enum TestConclusion { Skipped, Passed, @@ -350,7 +459,7 @@ impl FullReport { } (Passed, Pass(_)) => write!(f, "passed")?, - (Passed, Random) => write!(f, "passed (random, result ignored)")?, + (Passed, Random(_)) => write!(f, "passed (random, result ignored)")?, (Passed, Fail(_)) => write!(f, "passed (failed as expected)")?, (Failed, Pass(_)) => { @@ -374,7 +483,7 @@ impl FullReport { writeln!(f, " {}", red.apply_to(err))?; } } - (Failed, Random) => { + (Failed, Random(_)) => { write!(f, "{}", red.apply_to("failed!? (failed but random!?)"))? } (Failed, Fail(_)) => { diff --git a/src/harness/run.rs b/src/harness/run.rs index cf7df25..e2e31e2 100644 --- a/src/harness/run.rs +++ b/src/harness/run.rs @@ -6,7 +6,7 @@ use serde::Serialize; use tracing::info; use crate::error::*; -use crate::report::*; +use crate::harness::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/test.rs b/src/harness/test.rs index f13642a..cce0d39 100644 --- a/src/harness/test.rs +++ b/src/harness/test.rs @@ -114,19 +114,19 @@ impl FunctionSelector { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] pub enum FunctionSelector { All, One { idx: FuncIdx, args: ArgSelector }, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] pub enum ArgSelector { All, One { idx: usize, vals: ValSelector }, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] pub enum ValSelector { All, One { idx: usize }, diff --git a/src/main.rs b/src/main.rs index e0e7849..f85d9ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,17 +6,15 @@ mod harness; mod log; mod toolchains; -mod report; - use error::*; use files::Paths; +use harness::report::*; use harness::test::*; use harness::vals::*; use harness::*; use toolchains::*; use kdl_script::parse::LangRepr; -use report::*; use std::error::Error; use std::process::Command; use std::sync::Arc; @@ -92,6 +90,7 @@ fn main() -> Result<(), Box> { let _handle = rt.enter(); // Grab all the tests + let test_rules = harness::find_test_rules(&cfg)?; let test_sources = harness::find_tests(&cfg)?; let read_tasks = test_sources .into_iter() @@ -119,7 +118,7 @@ fn main() -> Result<(), Box> { } debug!("loaded tests!"); - let harness = Arc::new(TestHarness::new(tests, &cfg)); + let harness = Arc::new(TestHarness::new(test_rules, tests, &cfg)); debug!("initialized test harness!"); // Run the tests From 89127dc45a04a05b0c70f72f449e3ee5eb42bfbd Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 20:15:01 -0400 Subject: [PATCH 03/10] fix --- src/harness/report.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/harness/report.rs b/src/harness/report.rs index 461c29f..eaf8c1c 100644 --- a/src/harness/report.rs +++ b/src/harness/report.rs @@ -12,7 +12,7 @@ impl TestHarness { #[allow(unused_variables)] pub fn get_test_rules(&self, key: &TestKey) -> TestRules { let caller = self.toolchain_by_test_key(key, CallSide::Caller); - let callee = self.toolchain_by_test_key(key, CallSide::Caller); + let callee = self.toolchain_by_test_key(key, CallSide::Callee); use TestCheckMode::*; use TestRunMode::*; From e814e894d43844c4f23780ef4c8594cad0480031 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 21:29:07 -0400 Subject: [PATCH 04/10] feat: migrate builtin test expectations to static file --- include/harness/abi-cafe-rules.toml | 19 +++ kdl-script/src/parse.rs | 2 +- src/files.rs | 6 +- src/harness/mod.rs | 71 +-------- src/harness/read.rs | 11 +- src/harness/report.rs | 224 +++++++++++++++++++++------- src/harness/test.rs | 10 +- src/toolchains/mod.rs | 1 + 8 files changed, 211 insertions(+), 133 deletions(-) create mode 100644 include/harness/abi-cafe-rules.toml diff --git a/include/harness/abi-cafe-rules.toml b/include/harness/abi-cafe-rules.toml new file mode 100644 index 0000000..7636a5c --- /dev/null +++ b/include/harness/abi-cafe-rules.toml @@ -0,0 +1,19 @@ +# i128 types are fake on windows so this is all random garbage that might +# not even compile, but that datapoint is a little interesting/useful +# so let's keep running them and just ignore the result for now. +# +# Anyone who cares about this situation more can make the expectations more precise. +[targets.x86_64-pc-windows-msvc."i128::cc_toolchain"] +random = true +[targets.x86_64-pc-windows-msvc."u128::cc_toolchain"] +random = true + +# FIXME: investigate why this is failing to build +[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] +busted = "build" +[targets.x86_64-pc-windows-msvc."EmptyStructInside::cc_toolchain"] +busted = "build" + +# CI GCC is too old to support _Float16 +[targets.x86_64-unknown-liinux-gnu."f16::conv_c"] +random = true diff --git a/kdl-script/src/parse.rs b/kdl-script/src/parse.rs index 5b336b2..f26d6a4 100644 --- a/kdl-script/src/parse.rs +++ b/kdl-script/src/parse.rs @@ -218,7 +218,7 @@ pub enum Repr { Transparent, } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, serde::Serialize)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)] pub enum LangRepr { Rust, C, diff --git a/src/files.rs b/src/files.rs index 669b6db..982c6c1 100644 --- a/src/files.rs +++ b/src/files.rs @@ -90,12 +90,16 @@ pub fn load_file(file: &File) -> String { clean_newlines(string) } -pub fn tests() -> &'static Dir<'static> { +pub fn static_tests() -> &'static Dir<'static> { INCLUDES .get_dir("tests") .expect("includes didn't contain ./test") } +pub fn static_rules() -> String { + get_file("harness/abi-cafe-rules.toml") +} + fn clean_newlines(input: &str) -> String { input.replace('\r', "") } diff --git a/src/harness/mod.rs b/src/harness/mod.rs index 49bab66..d3be307 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -31,7 +31,7 @@ pub struct TestHarness { paths: Paths, toolchains: Toolchains, tests: SortedMap>, - test_rules: ExpectFile, + test_rules: Vec, tests_with_vals: Memoized<(TestId, ValueGeneratorKind), Arc>, tests_with_toolchain: Memoized<(TestId, ValueGeneratorKind, ToolchainId), Arc>, @@ -41,7 +41,7 @@ pub struct TestHarness { } impl TestHarness { - pub fn new(test_rules: ExpectFile, tests: SortedMap>, cfg: &Config) -> Self { + pub fn new(test_rules: Vec, tests: SortedMap>, cfg: &Config) -> Self { let toolchains = toolchains::create_toolchains(cfg); Self { paths: cfg.paths.clone(), @@ -275,73 +275,6 @@ impl TestHarness { output } - pub fn parse_test_pattern(&self, input: &str) -> Result { - let separator = "::"; - let parts = input.split(separator).collect::>(); - - let [test, rest @ ..] = &parts[..] else { - todo!(); - }; - let mut key = TestKeyPattern { - test: (!test.is_empty()).then(|| test.to_string()), - caller: None, - callee: None, - toolchain: None, - options: TestOptionsPattern { - convention: None, - repr: None, - functions: None, - val_writer: None, - val_generator: None, - }, - }; - for part in rest { - // pairs - if let Some((caller, callee)) = part.split_once("_calls_") { - key.caller = Some(caller.to_owned()); - key.callee = Some(callee.to_owned()); - continue; - } - if let Some(caller) = part.strip_suffix("_caller") { - key.caller = Some(caller.to_owned()); - continue; - } - if let Some(callee) = part.strip_suffix("_callee") { - key.callee = Some(callee.to_owned()); - continue; - } - if let Some(toolchain) = part.strip_suffix("_toolchain") { - key.toolchain = Some(toolchain.to_owned()); - continue; - } - - // repr - if let Some(repr) = part.strip_prefix("repr_") { - key.options.repr = Some(repr.parse()?); - continue; - } - - // conv - if let Some(conv) = part.strip_prefix("conv_") { - key.options.convention = Some(conv.parse()?); - continue; - } - // generator - if let Ok(val_generator) = part.parse() { - key.options.val_generator = Some(val_generator); - continue; - } - // writer - if let Ok(val_writer) = part.parse() { - key.options.val_writer = Some(val_writer); - continue; - } - - return Err(format!("unknown testkey part: {part}")); - } - Ok(key) - } - /// The name of a test for pretty-printing. pub fn full_test_name(&self, key: &TestKey) -> String { self.base_id(key, None, "::") diff --git a/src/harness/read.rs b/src/harness/read.rs index cfd56f5..62e5ee0 100644 --- a/src/harness/read.rs +++ b/src/harness/read.rs @@ -32,8 +32,15 @@ impl Pathish { } } -pub fn find_test_rules(cfg: &Config) -> Result { +pub fn find_test_rules(cfg: &Config) -> Result, GenerateError> { + let static_rules = find_test_rules_static()?; let rules = find_test_rules_runtime(&cfg.paths.runtime_rules_file)?; + Ok(vec![static_rules, rules]) +} + +pub fn find_test_rules_static() -> Result { + let data = files::static_rules(); + let rules = toml::from_str(&data)?; Ok(rules) } @@ -92,7 +99,7 @@ pub fn find_tests_static( return Ok(tests); } - let mut dirs = vec![crate::files::tests()]; + let mut dirs = vec![crate::files::static_tests()]; while let Some(dir) = dirs.pop() { for entry in dir.entries() { // If it's a dir, add it to the working set diff --git a/src/harness/report.rs b/src/harness/report.rs index eaf8c1c..5302d32 100644 --- a/src/harness/report.rs +++ b/src/harness/report.rs @@ -11,9 +11,6 @@ use crate::*; impl TestHarness { #[allow(unused_variables)] pub fn get_test_rules(&self, key: &TestKey) -> TestRules { - let caller = self.toolchain_by_test_key(key, CallSide::Caller); - let callee = self.toolchain_by_test_key(key, CallSide::Callee); - use TestCheckMode::*; use TestRunMode::*; @@ -23,30 +20,6 @@ impl TestHarness { check: Pass(Check), }; - // Now apply specific custom expectations for platforms/suites - let is_c = caller.lang() == "c" || callee.lang() == "c"; - let is_rust = caller.lang() == "rust" || callee.lang() == "rust"; - let is_rust_and_c = is_c && is_rust; - - // i128 types are fake on windows so this is all random garbage that might - // not even compile, but that datapoint is a little interesting/useful - // so let's keep running them and just ignore the result for now. - // - // Anyone who cares about this situation more can make the expectations more precise. - if cfg!(windows) && (key.test == "i128" || key.test == "u128") { - result.check = Random(true); - } - - // CI GCC is too old to support `_Float16`. - if cfg!(all(target_arch = "x86_64", target_os = "linux")) && is_c && key.test == "f16" { - result.check = Random(true); - } - - // FIXME: investigate why this is failing to build - if cfg!(windows) && is_c && (key.test == "EmptyStruct" || key.test == "EmptyStructInside") { - result.check = Busted(Build); - } - // // // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES @@ -55,28 +28,35 @@ impl TestHarness { // // - if let Some(rules) = self.test_rules.targets.get(built_info::TARGET) { - for (pattern, rules) in rules { - let pat = self - .parse_test_pattern(pattern) - .expect("failed to parse test pattern"); - if pat.matches(key) { - if let Some(run) = rules.run.clone() { - result.run = run; - } - if let Some(check) = rules.check.clone() { - result.check = check; + for expect_file in &self.test_rules { + let rulesets = [ + expect_file.targets.get("*"), + expect_file.targets.get(built_info::TARGET), + ]; + for rules in rulesets { + let Some(rules) = rules else { + continue; + }; + for (pattern, rules) in rules { + if pattern.matches(key) { + if let Some(run) = rules.run.clone() { + result.run = run; + } + if let Some(check) = rules.check.clone() { + result.check = check; + } } } } } + result } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct ExpectFile { - pub targets: SortedMap>, + pub targets: SortedMap>, } impl Serialize for BuildError { @@ -225,7 +205,7 @@ pub struct TestKey { pub options: TestOptions, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct TestKeyPattern { pub test: Option, pub caller: Option, @@ -233,11 +213,9 @@ pub struct TestKeyPattern { pub toolchain: Option, pub options: TestOptionsPattern, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct TestOptionsPattern { pub convention: Option, - pub functions: Option, - pub val_writer: Option, pub val_generator: Option, pub repr: Option, } @@ -260,8 +238,6 @@ impl TestKeyPattern { options: TestOptionsPattern { convention, - functions, - val_writer, val_generator, repr, }, @@ -294,16 +270,6 @@ impl TestKeyPattern { return false; } } - if let Some(functions) = functions { - if functions != &key.options.functions { - return false; - } - } - if let Some(val_writer) = val_writer { - if val_writer != &key.options.val_writer { - return false; - } - } if let Some(val_generator) = val_generator { if val_generator != &key.options.val_generator { return false; @@ -319,6 +285,154 @@ impl TestKeyPattern { } } +impl std::str::FromStr for TestKeyPattern { + type Err = String; + + fn from_str(input: &str) -> Result { + let separator = "::"; + let parts = input.split(separator).collect::>(); + + let mut key = TestKeyPattern { + test: None, + caller: None, + callee: None, + toolchain: None, + options: TestOptionsPattern { + convention: None, + repr: None, + val_generator: None, + }, + }; + + let [test, rest @ ..] = &parts[..] else { + return Ok(key); + }; + key.test = (!test.is_empty()).then(|| test.to_string()); + + for part in rest { + // pairs + if let Some((caller, callee)) = part.split_once("_calls_") { + key.caller = Some(caller.to_owned()); + key.callee = Some(callee.to_owned()); + continue; + } + if let Some(caller) = part.strip_suffix("_caller") { + key.caller = Some(caller.to_owned()); + continue; + } + if let Some(callee) = part.strip_suffix("_callee") { + key.callee = Some(callee.to_owned()); + continue; + } + if let Some(toolchain) = part.strip_suffix("_toolchain") { + key.toolchain = Some(toolchain.to_owned()); + continue; + } + + // repr + if let Some(repr) = part.strip_prefix("repr_") { + key.options.repr = Some(repr.parse()?); + continue; + } + + // conv + if let Some(conv) = part.strip_prefix("conv_") { + key.options.convention = Some(conv.parse()?); + continue; + } + // generator + if let Ok(val_generator) = part.parse() { + key.options.val_generator = Some(val_generator); + continue; + } + + return Err(format!("unknown testkey part: {part}")); + } + Ok(key) + } +} +impl std::fmt::Display for TestKeyPattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let TestKeyPattern { + test, + caller, + callee, + toolchain, + options: TestOptionsPattern { + convention, + val_generator, + repr, + }, + } = self; + let separator = "::"; + let mut output = String::new(); + if let Some(test) = test { + output.push_str(test); + } + if let Some(convention) = convention { + output.push_str(separator); + output.push_str(&format!("conv_{convention}")); + } + if let Some(convention) = convention { + output.push_str(separator); + output.push_str(&format!("conv_{convention}")); + } + if let Some(repr) = repr { + output.push_str(separator); + output.push_str(&format!("repr_{repr}")); + } + if let Some(toolchain) = toolchain { + output.push_str(separator); + output.push_str(&format!("{toolchain}_toolchain")); + } + match (caller, callee) { + (Some(caller), Some(callee)) => { + output.push_str(separator); + output.push_str(caller); + output.push_str("_calls_"); + output.push_str(callee); + } + (Some(caller), None) => { + output.push_str(separator); + output.push_str(caller); + output.push_str("_caller"); + } + (None, Some(callee)) => { + output.push_str(separator); + output.push_str(callee); + output.push_str("_callee"); + } + (None, None) => { + // Noting + } + } + output.push_str(separator); + if let Some(val_generator) = val_generator { + output.push_str(separator); + output.push_str(&val_generator.to_string()); + } + output.fmt(f) + } +} +impl<'de> Deserialize<'de> for TestKeyPattern { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de> + { + use serde::de::Error; + let input = String::deserialize(deserializer)?; + input.parse().map_err(|e| D::Error::custom(e)) + } +} +impl Serialize for TestKeyPattern { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer { + self.to_string().serialize(serializer) + } +} + + #[derive(Debug, Clone, Serialize)] pub struct TestRules { pub run: TestRunMode, diff --git a/src/harness/test.rs b/src/harness/test.rs index cce0d39..05f4327 100644 --- a/src/harness/test.rs +++ b/src/harness/test.rs @@ -114,19 +114,19 @@ impl FunctionSelector { } } -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum FunctionSelector { All, One { idx: FuncIdx, args: ArgSelector }, } -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum ArgSelector { All, One { idx: usize, vals: ValSelector }, } -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum ValSelector { All, One { idx: usize }, @@ -202,7 +202,7 @@ impl std::ops::Deref for TestImpl { } } -#[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum WriteImpl { HarnessCallback, Assert, @@ -275,7 +275,7 @@ impl TestWithToolchain { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize, serde::Serialize)] #[serde(rename = "lowercase")] pub enum CallingConvention { /// The platform's default C convention (cdecl?) diff --git a/src/toolchains/mod.rs b/src/toolchains/mod.rs index 69cd852..01c7115 100644 --- a/src/toolchains/mod.rs +++ b/src/toolchains/mod.rs @@ -23,6 +23,7 @@ const C_TOOLCHAINS: &[&str] = &[TOOLCHAIN_CC, TOOLCHAIN_GCC, TOOLCHAIN_CLANG, TO /// A compiler/language toolchain! pub trait Toolchain { + #[allow(dead_code)] fn lang(&self) -> &'static str; fn src_ext(&self) -> &'static str; fn pun_env(&self) -> Arc; From d7bb583c22881a4bd507e11f94e20c98078e4312 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 21:30:53 -0400 Subject: [PATCH 05/10] fix: typo --- include/harness/abi-cafe-rules.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/harness/abi-cafe-rules.toml b/include/harness/abi-cafe-rules.toml index 7636a5c..77a8634 100644 --- a/include/harness/abi-cafe-rules.toml +++ b/include/harness/abi-cafe-rules.toml @@ -15,5 +15,5 @@ busted = "build" busted = "build" # CI GCC is too old to support _Float16 -[targets.x86_64-unknown-liinux-gnu."f16::conv_c"] +[targets.x86_64-unknown-linux-gnu."f16::conv_c"] random = true From 03b8ea339d22fbc50129aff98372a8dc7d110ca6 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 21:32:11 -0400 Subject: [PATCH 06/10] fmt --- src/harness/mod.rs | 6 +++++- src/harness/report.rs | 19 ++++++++++--------- src/harness/test.rs | 4 +++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/harness/mod.rs b/src/harness/mod.rs index d3be307..c06f0a0 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -41,7 +41,11 @@ pub struct TestHarness { } impl TestHarness { - pub fn new(test_rules: Vec, tests: SortedMap>, cfg: &Config) -> Self { + pub fn new( + test_rules: Vec, + tests: SortedMap>, + cfg: &Config, + ) -> Self { let toolchains = toolchains::create_toolchains(cfg); Self { paths: cfg.paths.clone(), diff --git a/src/harness/report.rs b/src/harness/report.rs index 5302d32..32454c8 100644 --- a/src/harness/report.rs +++ b/src/harness/report.rs @@ -358,11 +358,12 @@ impl std::fmt::Display for TestKeyPattern { caller, callee, toolchain, - options: TestOptionsPattern { - convention, - val_generator, - repr, - }, + options: + TestOptionsPattern { + convention, + val_generator, + repr, + }, } = self; let separator = "::"; let mut output = String::new(); @@ -417,22 +418,22 @@ impl std::fmt::Display for TestKeyPattern { impl<'de> Deserialize<'de> for TestKeyPattern { fn deserialize(deserializer: D) -> Result where - D: serde::Deserializer<'de> + D: serde::Deserializer<'de>, { use serde::de::Error; let input = String::deserialize(deserializer)?; - input.parse().map_err(|e| D::Error::custom(e)) + input.parse().map_err(D::Error::custom) } } impl Serialize for TestKeyPattern { fn serialize(&self, serializer: S) -> Result where - S: serde::Serializer { + S: serde::Serializer, + { self.to_string().serialize(serializer) } } - #[derive(Debug, Clone, Serialize)] pub struct TestRules { pub run: TestRunMode, diff --git a/src/harness/test.rs b/src/harness/test.rs index 05f4327..64a041a 100644 --- a/src/harness/test.rs +++ b/src/harness/test.rs @@ -275,7 +275,9 @@ impl TestWithToolchain { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize, serde::Serialize)] +#[derive( + Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize, serde::Serialize, +)] #[serde(rename = "lowercase")] pub enum CallingConvention { /// The platform's default C convention (cdecl?) From a6bb706cb1668291f9e39badf0d6f731afcfa27f Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 21:54:07 -0400 Subject: [PATCH 07/10] fix: make vendor god --- src/harness/report.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/harness/report.rs b/src/harness/report.rs index 32454c8..5331bf1 100644 --- a/src/harness/report.rs +++ b/src/harness/report.rs @@ -20,14 +20,6 @@ impl TestHarness { check: Pass(Check), }; - // - // - // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES - - // END OF VENDOR RESERVED AREA - // - // - for expect_file in &self.test_rules { let rulesets = [ expect_file.targets.get("*"), @@ -50,6 +42,14 @@ impl TestHarness { } } + // + // + // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + + // END OF VENDOR RESERVED AREA + // + // + result } } From 1297b82433f0a64bfa2fa421139dd4ee8f018401 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sat, 13 Jul 2024 22:01:53 -0400 Subject: [PATCH 08/10] feat: add --disable-builtin-rules --- src/cli.rs | 11 +++++++++++ src/harness/read.rs | 12 ++++++++---- src/main.rs | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 6c658c0..c041ec9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -133,6 +133,9 @@ struct Cli { /// Add the test expectations at the given path /// /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir) + /// + /// Note that there are already builtin rules (disabled with `--disable-builtin-riles`), + /// and it would be nice for rules to be upstreamed so everyone can benefit! #[clap(long)] rules: Option, @@ -142,6 +145,12 @@ struct Cli { #[clap(long)] disable_builtin_tests: bool, + /// disable the builtin rules + /// + /// See also `--add-rules` + #[clap(long)] + disable_builtin_rules: bool, + /// deprecated, does nothing (we always procgen now) #[clap(long, hide = true)] procgen_tests: bool, @@ -162,6 +171,7 @@ pub fn make_app() -> Config { add_tests, rules, disable_builtin_tests, + disable_builtin_rules, // unimplemented select_vals: _, key: _, @@ -257,6 +267,7 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`. run_selections, minimizing_write_impl, disable_builtin_tests, + disable_builtin_rules, paths, } } diff --git a/src/harness/read.rs b/src/harness/read.rs index 62e5ee0..9b56d21 100644 --- a/src/harness/read.rs +++ b/src/harness/read.rs @@ -33,14 +33,18 @@ impl Pathish { } pub fn find_test_rules(cfg: &Config) -> Result, GenerateError> { - let static_rules = find_test_rules_static()?; + let static_rules = find_test_rules_static(cfg.disable_builtin_rules)?; let rules = find_test_rules_runtime(&cfg.paths.runtime_rules_file)?; Ok(vec![static_rules, rules]) } -pub fn find_test_rules_static() -> Result { - let data = files::static_rules(); - let rules = toml::from_str(&data)?; +pub fn find_test_rules_static(disable_builtin_rules: bool) -> Result { + let rules = if disable_builtin_rules { + ExpectFile::default() + } else { + let data = files::static_rules(); + toml::from_str(&data)? + }; Ok(rules) } diff --git a/src/main.rs b/src/main.rs index f85d9ef..9c3b01f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -74,6 +74,7 @@ pub struct Config { pub minimizing_write_impl: WriteImpl, pub rustc_codegen_backends: Vec<(String, String)>, pub disable_builtin_tests: bool, + pub disable_builtin_rules: bool, pub paths: Paths, } From 88632cc42caa7e37caba1db34a71c24739a62d37 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sun, 14 Jul 2024 12:14:27 -0400 Subject: [PATCH 09/10] feat: suggest abi-cafe-rules.toml keys --- Cargo.lock | 3 + Cargo.toml | 4 +- abi-cafe-rules-example.toml | 28 ----- include/harness/abi-cafe-rules.toml | 41 +++++++ src/cli.rs | 2 +- src/error.rs | 28 ++--- src/harness/check.rs | 18 ++- src/harness/report.rs | 119 ++++++++++++-------- src/harness/test.rs | 2 +- src/main.rs | 165 ++++++++++++++++------------ 10 files changed, 246 insertions(+), 164 deletions(-) delete mode 100644 abi-cafe-rules-example.toml diff --git a/Cargo.lock b/Cargo.lock index c3f3c63..8823fdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,7 @@ dependencies = [ "clap", "console", "include_dir", + "indexmap", "kdl", "kdl-script", "libloading", @@ -355,6 +356,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] @@ -1064,6 +1066,7 @@ version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ + "indexmap", "serde", "serde_spanned", "toml_datetime", diff --git a/Cargo.toml b/Cargo.toml index 8d34963..dea5f5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ clap.workspace = true console.workspace = true kdl.workspace = true include_dir.workspace = true +indexmap.workspace = true libloading.workspace = true linked-hash-map.workspace = true miette.workspace = true @@ -63,6 +64,7 @@ cc = { version = "1.1.0" } clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] } console = "0.15.8" include_dir = "0.7.4" +indexmap = { version = "2.2.6", features = ["serde"] } kdl = "4.6.0" libloading = "0.7.3" linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] } @@ -76,7 +78,7 @@ serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.83" thiserror = "1.0.30" tokio = { version = "1.37.0", features = ["full", "tracing"] } -toml = "0.8.14" +toml = { version = "0.8.14", features = ["preserve_order"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } # build diff --git a/abi-cafe-rules-example.toml b/abi-cafe-rules-example.toml deleted file mode 100644 index 6d35176..0000000 --- a/abi-cafe-rules-example.toml +++ /dev/null @@ -1,28 +0,0 @@ -# Here are some example annotations for test expecations - -# this test fails on this platform, with this toolchain pairing -[targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"] -fail = "check" - -# this test has random results on this platform, whenever rustc is the caller (callee also supported) -[targets.x86_64-pc-windows-msvc."simple::rustc_caller"] -random = true - -# whenever this test involves cc, only link it, and expect linking to fail -[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] -run = "link" -fail = "link" - -# any repr(c) version of this test fails to run -[targets.x86_64-unknown-linux-gnu."simple::repr_c"] -busted = "run" - -# for this pairing, with the rust calling convention, only generate the test, and expect it to work -[targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"] -run = "generate" -pass = "generate" - -# can match all tests with leading :: -[targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"] -run = "generate" -pass = "generate" diff --git a/include/harness/abi-cafe-rules.toml b/include/harness/abi-cafe-rules.toml index 77a8634..29b29ce 100644 --- a/include/harness/abi-cafe-rules.toml +++ b/include/harness/abi-cafe-rules.toml @@ -17,3 +17,44 @@ busted = "build" # CI GCC is too old to support _Float16 [targets.x86_64-unknown-linux-gnu."f16::conv_c"] random = true + + + +# +# +# Here are some example annotations for test expecations +# +# + +# this test fails on this platform, with this toolchain pairing +# +# [targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"] +# fail = "check" + +# this test has random results on this platform, whenever rustc is the caller (callee also supported) +# +# [targets.x86_64-pc-windows-msvc."simple::rustc_caller"] +# random = true + +# whenever this test involves cc, only link it, and expect linking to fail +# +# [targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] +# run = "link" +# fail = "link" + +# any repr(c) version of this test fails to run +# +# [targets.x86_64-unknown-linux-gnu."simple::repr_c"] +# busted = "run" + +# for this pairing, with the rust calling convention, only generate the test, and expect it to work +# +# [targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"] +# run = "generate" +# pass = "generate" + +# can match all tests with leading :: +# +# [targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"] +# run = "generate" +# pass = "generate" diff --git a/src/cli.rs b/src/cli.rs index c041ec9..445c2cf 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -134,7 +134,7 @@ struct Cli { /// /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir) /// - /// Note that there are already builtin rules (disabled with `--disable-builtin-riles`), + /// Note that there are already builtin rules (disabled with `--disable-builtin-rules`), /// and it would be nice for rules to be upstreamed so everyone can benefit! #[clap(long)] rules: Option, diff --git a/src/error.rs b/src/error.rs index f007ec5..ec2cf14 100644 --- a/src/error.rs +++ b/src/error.rs @@ -74,13 +74,13 @@ pub enum BuildError { #[derive(Debug, thiserror::Error, Diagnostic)] pub enum CheckFailure { #[error( - " func {func_name}'s values differed - values (native-endian hex bytes): - expect: {} - caller: {} - callee: {} - the value was {val_path}: {val_ty_name} - whose arg was {arg_name}: {arg_ty_name}", + " func {func_name}'s values differed + values (native-endian hex bytes): + expect: {} + caller: {} + callee: {} + the value was {val_path}: {val_ty_name} + whose arg was {arg_name}: {arg_ty_name}", fmt_bytes(expected), fmt_bytes(caller), fmt_bytes(callee) @@ -99,13 +99,13 @@ pub enum CheckFailure { callee: Vec, }, #[error( - " func {func_name}'s value had unexpected variant - values: - expect: {expected} - caller: {caller} - callee: {callee} - the value was {val_path}: {val_ty_name} - whose arg was {arg_name}: {arg_ty_name}" + " func {func_name}'s value had unexpected variant + values: + expect: {expected} + caller: {caller} + callee: {callee} + the value was {val_path}: {val_ty_name} + whose arg was {arg_name}: {arg_ty_name}" )] TagMismatch { func_idx: usize, diff --git a/src/harness/check.rs b/src/harness/check.rs index 8bc6cbf..88d2de4 100644 --- a/src/harness/check.rs +++ b/src/harness/check.rs @@ -28,7 +28,7 @@ impl TestHarness { // Start peeling back the layers of the buffers. // funcs (subtests) -> vals (args/returns) -> fields -> bytes - let mut results: Vec> = Vec::new(); + let mut results: Vec = Vec::new(); // `Run` already checks that this length is congruent with all the inputs/outputs Vecs let expected_funcs = key.options.functions.active_funcs(&test.types); @@ -53,7 +53,10 @@ impl TestHarness { let caller_val = caller_func.vals.get(val_idx).unwrap_or(&empty_val); let callee_val = callee_func.vals.get(val_idx).unwrap_or(&empty_val); if let Err(e) = self.check_val(&test, expected_val, caller_val, callee_val) { - results.push(Err(e)); + results.push(SubtestDetails { + result: Err(e), + minimized: None, + }); // FIXME: now that each value is absolutely indexed, // we should be able to check all the values independently // and return all errors. However the first one is the most @@ -63,7 +66,10 @@ impl TestHarness { } // If we got this far then the test passes - results.push(Ok(())); + results.push(SubtestDetails { + result: Ok(()), + minimized: None, + }); } // Report the results of each subtest @@ -77,12 +83,12 @@ impl TestHarness { .map(|func_id| self.full_subtest_name(key, &test.types.realize_func(func_id).name)) .collect::>(); let max_name_len = names.iter().fold(0, |max, name| max.max(name.len())); - let num_passed = results.iter().filter(|r| r.is_ok()).count(); + let num_passed = results.iter().filter(|t| t.result.is_ok()).count(); let all_passed = num_passed == results.len(); if !all_passed { - for (subtest_name, result) in names.iter().zip(&results) { - match result { + for (subtest_name, subtest) in names.iter().zip(&results) { + match &subtest.result { Ok(()) => { info!("Test {subtest_name:width$} passed", width = max_name_len); } diff --git a/src/harness/report.rs b/src/harness/report.rs index 5331bf1..aca4867 100644 --- a/src/harness/report.rs +++ b/src/harness/report.rs @@ -31,10 +31,10 @@ impl TestHarness { }; for (pattern, rules) in rules { if pattern.matches(key) { - if let Some(run) = rules.run.clone() { + if let Some(run) = rules.run { result.run = run; } - if let Some(check) = rules.check.clone() { + if let Some(check) = rules.check { result.check = check; } } @@ -56,7 +56,8 @@ impl TestHarness { #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct ExpectFile { - pub targets: SortedMap>, + #[serde(default)] + pub targets: IndexMap>, } impl Serialize for BuildError { @@ -127,33 +128,12 @@ pub fn report_test(results: TestRunResults) -> TestReport { Skipped } else { let passed = match &results.rules.check { - TestCheckMode::Pass(must_pass) => match must_pass { - Skip => true, - Generate => results.source.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Build => results.build.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Link => results.link.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Run => results.run.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Check => results - .check - .as_ref() - .map(|r| r.all_passed) - .unwrap_or(false), - }, - TestCheckMode::Fail(must_fail) | TestCheckMode::Busted(must_fail) => match must_fail { - Skip => true, - Generate => results.source.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Build => results.build.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Link => results.link.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Run => results.run.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Check => results - .check - .as_ref() - .map(|r| !r.all_passed) - .unwrap_or(false), - }, - TestCheckMode::Random(_) => true, + TestCheckMode::Pass(must_pass) => success_at_step(&results, must_pass, true), + TestCheckMode::Fail(must_fail) => success_at_step(&results, must_fail, false), + TestCheckMode::Busted(must_fail) => success_at_step(&results, must_fail, false), + TestCheckMode::Random(_) => Some(true), }; - if passed { + if passed.unwrap_or(false) { if matches!(results.rules.check, TestCheckMode::Busted(_)) { TestConclusion::Busted } else { @@ -163,18 +143,47 @@ pub fn report_test(results: TestRunResults) -> TestReport { TestConclusion::Failed } }; + + // Compute what the annotation *could* be to make CI green + let did_pass = success_at_step(&results, &results.ran_to, true).unwrap_or(false); + let could_be = TestRulesPattern { + run: if results.rules.run != TestRunMode::Check { + Some(results.rules.run) + } else { + None + }, + check: if did_pass { + Some(TestCheckMode::Pass(results.rules.run)) + } else { + Some(TestCheckMode::Busted(results.rules.run)) + }, + }; TestReport { key: results.key.clone(), - rules: results.rules.clone(), + rules: results.rules, conclusion, + could_be, results, } } +fn success_at_step(results: &TestRunResults, step: &TestRunMode, wants_pass: bool) -> Option { + use TestRunMode::*; + let res = match step { + Skip => return Some(true), + Generate => results.source.as_ref().map(|r| r.is_ok()), + Build => results.build.as_ref().map(|r| r.is_ok()), + Link => results.link.as_ref().map(|r| r.is_ok()), + Run => results.run.as_ref().map(|r| r.is_ok()), + Check => results.check.as_ref().map(|r| r.all_passed), + }; + res.map(|res| res == wants_pass) +} + #[derive(Debug, Serialize)] pub struct FullReport { pub summary: TestSummary, - pub config: TestConfig, + pub possible_rules: Option, pub tests: Vec, } @@ -184,10 +193,9 @@ pub struct TestReport { pub rules: TestRules, pub results: TestRunResults, pub conclusion: TestConclusion, + pub could_be: TestRulesPattern, } -#[derive(Debug, Serialize)] -pub struct TestConfig {} #[derive(Debug, Serialize)] pub struct TestSummary { pub num_tests: u64, @@ -205,7 +213,7 @@ pub struct TestKey { pub options: TestOptions, } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct TestKeyPattern { pub test: Option, pub caller: Option, @@ -213,7 +221,7 @@ pub struct TestKeyPattern { pub toolchain: Option, pub options: TestOptionsPattern, } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct TestOptionsPattern { pub convention: Option, pub val_generator: Option, @@ -407,7 +415,6 @@ impl std::fmt::Display for TestKeyPattern { // Noting } } - output.push_str(separator); if let Some(val_generator) = val_generator { output.push_str(separator); output.push_str(&val_generator.to_string()); @@ -434,7 +441,7 @@ impl Serialize for TestKeyPattern { } } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Copy, Serialize)] pub struct TestRules { pub run: TestRunMode, #[serde(flatten)] @@ -450,7 +457,7 @@ pub struct TestRulesPattern { /// How far the test should be executed /// /// Each case implies all the previous cases. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum TestRunMode { /// Don't run the test at all (marked as skipped) @@ -470,7 +477,7 @@ pub enum TestRunMode { /// To what level of correctness should the test be graded? /// /// Tests that are Skipped ignore this. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum TestCheckMode { /// The test must successfully complete this phase, @@ -534,7 +541,13 @@ pub struct LinkOutput { pub struct CheckOutput { pub all_passed: bool, pub subtest_names: Vec, - pub subtest_checks: Vec>, + pub subtest_checks: Vec, +} + +#[derive(Debug, Serialize)] +pub struct SubtestDetails { + pub result: Result<(), CheckFailure>, + pub minimized: Option, } #[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] @@ -615,7 +628,8 @@ impl FullReport { } } - let be_detailed = test.results.ran_to >= TestRunMode::Check; + let be_detailed = test.results.ran_to >= TestRunMode::Check + && test.conclusion != TestConclusion::Busted; if !be_detailed { writeln!(f)?; continue; @@ -624,7 +638,7 @@ impl FullReport { continue; }; let sub_results = &check_result.subtest_checks; - let num_passed = sub_results.iter().filter(|r| r.is_ok()).count(); + let num_passed = sub_results.iter().filter(|t| t.result.is_ok()).count(); writeln!(f, " ({num_passed:>3}/{:<3} passed)", sub_results.len())?; // If all the subtests pass, don't bother with a breakdown. @@ -636,11 +650,16 @@ impl FullReport { .subtest_names .iter() .fold(0, |max, name| max.max(name.len())); - for (subtest_name, result) in check_result.subtest_names.iter().zip(sub_results.iter()) + for (subtest_name, subtest) in check_result.subtest_names.iter().zip(sub_results.iter()) { write!(f, " {:width$} ", subtest_name, width = max_name_len)?; - if let Err(e) = result { + if let Err(e) = &subtest.result { writeln!(f, "{}", red.apply_to("failed!"))?; + if let Some(minimized) = &subtest.minimized { + writeln!(f, " {}", blue.apply_to("minimized to:"))?; + writeln!(f, " caller: {}", blue.apply_to(&minimized.caller_src))?; + writeln!(f, " callee: {}", blue.apply_to(&minimized.callee_src))?; + } writeln!(f, "{}", red.apply_to(e))?; } else { writeln!(f)?; @@ -652,7 +671,7 @@ impl FullReport { let summary_style = if self.summary.num_failed > 0 { red } else if self.summary.num_busted > 0 { - blue + blue.clone() } else { green }; @@ -665,6 +684,16 @@ impl FullReport { self.summary.num_skipped ); writeln!(f, "{}", summary_style.apply_to(summary),)?; + if let Some(rules) = &self.possible_rules { + writeln!(f)?; + writeln!( + f, + "{}", + blue.apply_to("(experimental) adding this to your abi-cafe-rules.toml might help:") + )?; + let toml = toml::to_string_pretty(rules).expect("failed to serialize possible rules!?"); + writeln!(f, "{}", toml)?; + } Ok(()) } diff --git a/src/harness/test.rs b/src/harness/test.rs index 64a041a..90884b1 100644 --- a/src/harness/test.rs +++ b/src/harness/test.rs @@ -276,7 +276,7 @@ impl TestWithToolchain { } #[derive( - Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize, serde::Serialize, + Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize, serde::Serialize, )] #[serde(rename = "lowercase")] pub enum CallingConvention { diff --git a/src/main.rs b/src/main.rs index 9c3b01f..46634eb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ use harness::report::*; use harness::test::*; use harness::vals::*; use harness::*; +use indexmap::IndexMap; use toolchains::*; use kdl_script::parse::LangRepr; @@ -123,8 +124,6 @@ fn main() -> Result<(), Box> { debug!("initialized test harness!"); // Run the tests - use TestConclusion::*; - let mut tasks = vec![]; // The cruel bastard that is combinatorics... THE GOD LOOPS @@ -161,11 +160,7 @@ fn main() -> Result<(), Box> { }, }; let rules = harness.get_test_rules(&test_key); - let task = harness.clone().spawn_test( - &rt, - rules.clone(), - test_key.clone(), - ); + let task = harness.clone().spawn_test(&rt, rules, test_key.clone()); tasks.push(task); } @@ -185,6 +180,34 @@ fn main() -> Result<(), Box> { .collect::>(); // Compute the final report + let mut full_report = compute_final_report(&cfg, &harness, reports); + + if full_report.failed() { + generate_minimized_failures(&cfg, &harness, &rt, &mut full_report); + } + + let mut output = std::io::stdout(); + match cfg.output_format { + OutputFormat::Human => full_report.print_human(&harness, &mut output)?, + OutputFormat::Json => full_report.print_json(&harness, &mut output)?, + OutputFormat::RustcJson => full_report.print_rustc_json(&harness, &mut output)?, + } + + if full_report.failed() { + Err(TestsFailed {})?; + } + Ok(()) +} + +fn compute_final_report( + _cfg: &Config, + harness: &Arc, + reports: Vec, +) -> FullReport { + use TestConclusion::*; + + let mut expects = IndexMap::::new(); + let mut num_tests = 0; let mut num_passed = 0; let mut num_busted = 0; @@ -196,11 +219,25 @@ fn main() -> Result<(), Box> { Busted => num_busted += 1, Skipped => num_skipped += 1, Passed => num_passed += 1, - Failed => num_failed += 1, + Failed => { + num_failed += 1; + let pattern = harness.base_id(&report.key, None, "::"); + if let Ok(pattern) = pattern.parse() { + expects.insert(pattern, report.could_be.clone()); + } + } } } - let full_report = FullReport { + let possible_rules = if expects.is_empty() { + None + } else { + Some(ExpectFile { + targets: IndexMap::from_iter([(built_info::TARGET.to_owned(), expects)]), + }) + }; + + FullReport { summary: TestSummary { num_tests, num_passed, @@ -208,78 +245,70 @@ fn main() -> Result<(), Box> { num_failed, num_skipped, }, - // FIXME: put in a bunch of metadata here? - config: TestConfig {}, + possible_rules, tests: reports, - }; - - let mut output = std::io::stdout(); - match cfg.output_format { - OutputFormat::Human => full_report.print_human(&harness, &mut output)?, - OutputFormat::Json => full_report.print_json(&harness, &mut output)?, - OutputFormat::RustcJson => full_report.print_rustc_json(&harness, &mut output)?, - } - - if full_report.failed() { - generate_minimized_failures(&cfg, &harness, &rt, &full_report); - Err(TestsFailed {})?; } - Ok(()) } fn generate_minimized_failures( cfg: &Config, harness: &Arc, rt: &tokio::runtime::Runtime, - reports: &FullReport, + reports: &mut FullReport, ) { - info!("rerunning failures"); - let tasks = reports.tests.iter().flat_map(|report| { + info!("minimizing failures..."); + let mut tasks = vec![]; + for (test_idx, report) in reports.tests.iter().enumerate() { let Some(check) = report.results.check.as_ref() else { - return vec![]; + continue; }; - check - .subtest_checks - .iter() - .filter_map(|func_result| { - let Err(failure) = func_result else { - return None; - }; - let functions = match *failure { - CheckFailure::ValMismatch { - func_idx, - arg_idx, - val_idx, - .. - } - | CheckFailure::TagMismatch { - func_idx, - arg_idx, - val_idx, - .. - } => FunctionSelector::One { - idx: func_idx, - args: ArgSelector::One { - idx: arg_idx, - vals: ValSelector::One { idx: val_idx }, - }, + // FIXME: certainly classes of run failure could also be minimized, + // because we have information indicating there was an error in a specific func! + for (subtest_idx, subtest) in check.subtest_checks.iter().enumerate() { + let Err(failure) = &subtest.result else { + continue; + }; + + let functions = match *failure { + CheckFailure::ValMismatch { + func_idx, + arg_idx, + val_idx, + .. + } + | CheckFailure::TagMismatch { + func_idx, + arg_idx, + val_idx, + .. + } => FunctionSelector::One { + idx: func_idx, + args: ArgSelector::One { + idx: arg_idx, + vals: ValSelector::One { idx: val_idx }, }, - }; + }, + }; - let mut test_key = report.key.clone(); - test_key.options.functions = functions; - test_key.options.val_writer = cfg.minimizing_write_impl; - let mut rules = report.rules.clone(); - rules.run = TestRunMode::Generate; + let mut test_key = report.key.clone(); + test_key.options.functions = functions; + test_key.options.val_writer = cfg.minimizing_write_impl; + let mut rules = report.rules; + rules.run = TestRunMode::Generate; - let task = harness.clone().spawn_test(rt, rules, test_key); - Some(task) - }) - .collect() - }); + let task = harness.clone().spawn_test(rt, rules, test_key); + tasks.push((test_idx, subtest_idx, task)); + } + } - let _results = tasks - .into_iter() - .map(|task| rt.block_on(task).expect("failed to join task")) - .collect::>(); + for (test_idx, subtest_idx, task) in tasks { + let results = rt.block_on(task).expect("failed to join task"); + reports.tests[test_idx] + .results + .check + .as_mut() + .unwrap() + .subtest_checks[subtest_idx] + .minimized = results.source.and_then(|r| r.ok()); + } } From 8c28daf6d99dd3902bb77fe231efb89bf7031f25 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Sun, 14 Jul 2024 12:32:09 -0400 Subject: [PATCH 10/10] fix: don't emit conv:: twice --- src/harness/report.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/harness/report.rs b/src/harness/report.rs index aca4867..847f31e 100644 --- a/src/harness/report.rs +++ b/src/harness/report.rs @@ -382,10 +382,6 @@ impl std::fmt::Display for TestKeyPattern { output.push_str(separator); output.push_str(&format!("conv_{convention}")); } - if let Some(convention) = convention { - output.push_str(separator); - output.push_str(&format!("conv_{convention}")); - } if let Some(repr) = repr { output.push_str(separator); output.push_str(&format!("repr_{repr}"));