feat: runtime test expectation file

fixes #9

Cargo.toml

@@ -20,6 +20,7 @@ cc.workspace = true
 clap.workspace = true
 console.workspace = true
 kdl.workspace = true
+include_dir.workspace = true
 libloading.workspace = true
 linked-hash-map.workspace = true
 miette.workspace = true
@@ -32,7 +33,8 @@ thiserror.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true
-include_dir = "0.7.4"
+toml.workspace = true
+
 
 [build-dependencies]
 built.workspace = true
@@ -60,6 +62,7 @@ camino = { version = "1.1.7", features = ["serde1"] }
 cc = { version = "1.1.0" }
 clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] }
 console = "0.15.8"
+include_dir = "0.7.4"
 kdl = "4.6.0"
 libloading = "0.7.3"
 linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] }
@@ -73,6 +76,7 @@ serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.83"
 thiserror = "1.0.30"
 tokio = { version = "1.37.0", features = ["full", "tracing"] }
+toml = "0.8.14"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 # build

abi-cafe-rules-example.toml

@@ -0,0 +1,28 @@
+# Here are some example annotations for test expecations
+
+# this test fails on this platform, with this toolchain pairing
+[targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"]
+fail = "check"
+
+# this test has random results on this platform, whenever rustc is the caller (callee also supported)
+[targets.x86_64-pc-windows-msvc."simple::rustc_caller"]
+random = true
+
+# whenever this test involves cc, only link it, and expect linking to fail
+[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"]
+run = "link"
+fail = "link"
+
+# any repr(c) version of this test fails to run
+[targets.x86_64-unknown-linux-gnu."simple::repr_c"]
+busted = "run"
+
+# for this pairing, with the rust calling convention, only generate the test, and expect it to work
+[targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"]
+run = "generate"
+pass = "generate"
+
+# can match all tests with leading ::
+[targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"]
+run = "generate"
+pass = "generate"

src/cli.rs

@@ -130,6 +130,12 @@ struct Cli {
     #[clap(long)]
     add_tests: Option<Utf8PathBuf>,
 
+    /// Add the test expectations at the given path
+    ///
+    /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir)
+    #[clap(long)]
+    rules: Option<Utf8PathBuf>,
+
     /// disable the builtin tests
     ///
     /// See also `--add-tests`
@@ -154,6 +160,7 @@ pub fn make_app() -> Config {
         output_format,
         add_rustc_codegen_backend,
         add_tests,
+        rules,
         disable_builtin_tests,
         // unimplemented
         select_vals: _,
@@ -228,12 +235,14 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`.
     let out_dir = target_dir.join("temp");
     let generated_src_dir = target_dir.join("generated_impls");
     let runtime_test_input_dir = add_tests;
+    let runtime_rules_file = rules.unwrap_or_else(|| "abi-cafe-rules.toml".into());
 
     let paths = Paths {
         target_dir,
         out_dir,
         generated_src_dir,
         runtime_test_input_dir,
+        runtime_rules_file,
     };
     Config {
         output_format,

src/error.rs

@@ -26,6 +26,8 @@ pub enum GenerateError {
     #[error(transparent)]
     #[diagnostic(transparent)]
     KdlScriptError(#[from] kdl_script::KdlScriptError),
+    #[error(transparent)]
+    TomlError(#[from] toml::de::Error),
     /// Used to signal we just skipped it
     #[error("<skipped>")]
     Skipped,

src/files.rs

@@ -13,6 +13,7 @@ pub struct Paths {
     pub out_dir: Utf8PathBuf,
     pub generated_src_dir: Utf8PathBuf,
     pub runtime_test_input_dir: Option<Utf8PathBuf>,
+    pub runtime_rules_file: Utf8PathBuf,
 }
 impl Paths {
     pub fn harness_dylib_main_file(&self) -> Utf8PathBuf {

src/harness/build.rs

@@ -5,8 +5,8 @@ use camino::Utf8Path;
 use tracing::info;
 
 use crate::error::*;
+use crate::harness::report::*;
 use crate::harness::test::*;
-use crate::report::*;
 use crate::*;
 
 impl TestHarness {

src/harness/check.rs

@@ -5,7 +5,6 @@ use kdl_script::types::Ty;
 use tracing::{error, info};
 
 use crate::error::*;
-use crate::report::*;
 use crate::*;
 
 impl TestHarness {

src/harness/mod.rs

@@ -17,11 +17,12 @@ mod build;
 mod check;
 mod generate;
 mod read;
+pub mod report;
 mod run;
 pub mod test;
 pub mod vals;
 
-pub use read::{find_tests, spawn_read_test};
+pub use read::{find_test_rules, find_tests, spawn_read_test};
 pub use run::TestBuffer;
 
 pub type Memoized<K, V> = Mutex<SortedMap<K, Arc<OnceCell<V>>>>;
@@ -30,6 +31,7 @@ pub struct TestHarness {
     paths: Paths,
     toolchains: Toolchains,
     tests: SortedMap<TestId, Arc<Test>>,
+    test_rules: ExpectFile,
     tests_with_vals: Memoized<(TestId, ValueGeneratorKind), Arc<TestWithVals>>,
     tests_with_toolchain:
         Memoized<(TestId, ValueGeneratorKind, ToolchainId), Arc<TestWithToolchain>>,
@@ -39,11 +41,12 @@ pub struct TestHarness {
 }
 
 impl TestHarness {
-    pub fn new(tests: SortedMap<TestId, Arc<Test>>, cfg: &Config) -> Self {
+    pub fn new(test_rules: ExpectFile, tests: SortedMap<TestId, Arc<Test>>, cfg: &Config) -> Self {
         let toolchains = toolchains::create_toolchains(cfg);
         Self {
             paths: cfg.paths.clone(),
             tests,
+            test_rules,
             toolchains,
             tests_with_vals: Default::default(),
             tests_with_toolchain: Default::default(),
@@ -109,13 +112,6 @@ impl TestHarness {
             .clone();
         Ok(output)
     }
-    pub fn get_test_rules(&self, test_key: &TestKey) -> TestRules {
-        let caller = self.toolchains[&test_key.caller].clone();
-        let callee = self.toolchains[&test_key.callee].clone();
-
-        get_test_rules(test_key, &*caller, &*callee)
-    }
-
     pub fn spawn_test(
         self: Arc<Self>,
         rt: &tokio::runtime::Runtime,
@@ -279,64 +275,69 @@ impl TestHarness {
         output
     }
 
-    pub fn parse_test_key(&self, input: &str) -> Result<TestKey, String> {
+    pub fn parse_test_pattern(&self, input: &str) -> Result<TestKeyPattern, String> {
         let separator = "::";
         let parts = input.split(separator).collect::<Vec<_>>();
 
         let [test, rest @ ..] = &parts[..] else {
             todo!();
         };
-        let mut key = TestKey {
-            test: test.to_string(),
-            caller: String::new(),
-            callee: String::new(),
-            options: TestOptions {
-                convention: CallingConvention::C,
-                repr: LangRepr::C,
-                functions: FunctionSelector::All,
-                val_writer: WriteImpl::HarnessCallback,
-                val_generator: ValueGeneratorKind::Graffiti,
+        let mut key = TestKeyPattern {
+            test: (!test.is_empty()).then(|| test.to_string()),
+            caller: None,
+            callee: None,
+            toolchain: None,
+            options: TestOptionsPattern {
+                convention: None,
+                repr: None,
+                functions: None,
+                val_writer: None,
+                val_generator: None,
             },
         };
         for part in rest {
             // pairs
             if let Some((caller, callee)) = part.split_once("_calls_") {
-                key.caller = caller.to_owned();
-                key.callee = callee.to_owned();
+                key.caller = Some(caller.to_owned());
+                key.callee = Some(callee.to_owned());
                 continue;
             }
             if let Some(caller) = part.strip_suffix("_caller") {
-                key.caller = caller.to_owned();
+                key.caller = Some(caller.to_owned());
                 continue;
             }
             if let Some(callee) = part.strip_suffix("_callee") {
-                key.callee = callee.to_owned();
+                key.callee = Some(callee.to_owned());
+                continue;
+            }
+            if let Some(toolchain) = part.strip_suffix("_toolchain") {
+                key.toolchain = Some(toolchain.to_owned());
                 continue;
             }
 
             // repr
             if let Some(repr) = part.strip_prefix("repr_") {
-                key.options.repr = repr.parse()?;
+                key.options.repr = Some(repr.parse()?);
                 continue;
             }
 
             // conv
             if let Some(conv) = part.strip_prefix("conv_") {
-                key.options.convention = conv.parse()?;
+                key.options.convention = Some(conv.parse()?);
                 continue;
             }
             // generator
             if let Ok(val_generator) = part.parse() {
-                key.options.val_generator = val_generator;
+                key.options.val_generator = Some(val_generator);
                 continue;
             }
             // writer
             if let Ok(val_writer) = part.parse() {
-                key.options.val_writer = val_writer;
+                key.options.val_writer = Some(val_writer);
                 continue;
             }
 
-            return Err(format!("unknown testkey part: {part}"))
+            return Err(format!("unknown testkey part: {part}"));
         }
         Ok(key)
     }
@@ -351,4 +352,4 @@ impl TestHarness {
         let base = self.full_test_name(key);
         format!("{base}::{func_name}")
     }
-}
+}

src/harness/read.rs

@@ -32,6 +32,21 @@ impl Pathish {
     }
 }
 
+pub fn find_test_rules(cfg: &Config) -> Result<ExpectFile, GenerateError> {
+    let rules = find_test_rules_runtime(&cfg.paths.runtime_rules_file)?;
+    Ok(rules)
+}
+
+pub fn find_test_rules_runtime(rule_file: &Utf8Path) -> Result<ExpectFile, GenerateError> {
+    if rule_file.exists() {
+        let data = read_runtime_file_to_string(rule_file)?;
+        let rules = toml::from_str(&data)?;
+        Ok(rules)
+    } else {
+        Ok(ExpectFile::default())
+    }
+}
+
 pub fn find_tests(cfg: &Config) -> Result<SortedMap<TestId, TestFile>, GenerateError> {
     let mut tests = find_tests_runtime(cfg.paths.runtime_test_input_dir.as_deref())?;
     let mut more_tests = find_tests_static(cfg.disable_builtin_tests)?;