feat: suggest abi-cafe-rules.toml keys

Cargo.toml

@@ -21,6 +21,7 @@ clap.workspace = true
 console.workspace = true
 kdl.workspace = true
 include_dir.workspace = true
+indexmap.workspace = true
 libloading.workspace = true
 linked-hash-map.workspace = true
 miette.workspace = true
@@ -63,6 +64,7 @@ cc = { version = "1.1.0" }
 clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] }
 console = "0.15.8"
 include_dir = "0.7.4"
+indexmap = { version = "2.2.6", features = ["serde"] }
 kdl = "4.6.0"
 libloading = "0.7.3"
 linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] }
@@ -76,7 +78,7 @@ serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.83"
 thiserror = "1.0.30"
 tokio = { version = "1.37.0", features = ["full", "tracing"] }
-toml = "0.8.14"
+toml = { version = "0.8.14", features = ["preserve_order"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 # build

include/harness/abi-cafe-rules.toml

@@ -17,3 +17,44 @@ busted = "build"
 # CI GCC is too old to support _Float16
 [targets.x86_64-unknown-linux-gnu."f16::conv_c"]
 random = true
+
+
+
+#
+#
+# Here are some example annotations for test expecations
+#
+#
+
+# this test fails on this platform, with this toolchain pairing
+#
+# [targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"]
+# fail = "check"
+
+# this test has random results on this platform, whenever rustc is the caller (callee also supported)
+#
+# [targets.x86_64-pc-windows-msvc."simple::rustc_caller"]
+# random = true
+
+# whenever this test involves cc, only link it, and expect linking to fail
+#
+# [targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"]
+# run = "link"
+# fail = "link"
+
+# any repr(c) version of this test fails to run
+#
+# [targets.x86_64-unknown-linux-gnu."simple::repr_c"]
+# busted = "run"
+
+# for this pairing, with the rust calling convention, only generate the test, and expect it to work
+#
+# [targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"]
+# run = "generate"
+# pass = "generate"
+
+# can match all tests with leading ::
+#
+# [targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"]
+# run = "generate"
+# pass = "generate"

src/cli.rs

@@ -134,7 +134,7 @@ struct Cli {
     ///
     /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir)
     ///
-    /// Note that there are already builtin rules (disabled with `--disable-builtin-riles`),
+    /// Note that there are already builtin rules (disabled with `--disable-builtin-rules`),
     /// and it would be nice for rules to be upstreamed so everyone can benefit!
     #[clap(long)]
     rules: Option<Utf8PathBuf>,

src/error.rs

@@ -74,13 +74,13 @@ pub enum BuildError {
 #[derive(Debug, thiserror::Error, Diagnostic)]
 pub enum CheckFailure {
     #[error(
-        "  func {func_name}'s values differed
-    values (native-endian hex bytes):
-      expect: {}
-      caller: {}
-      callee: {}
-    the value was {val_path}: {val_ty_name}
-    whose arg was {arg_name}: {arg_ty_name}",
+        "    func {func_name}'s values differed
+      values (native-endian hex bytes):
+        expect: {}
+        caller: {}
+        callee: {}
+      the value was {val_path}: {val_ty_name}
+      whose arg was {arg_name}: {arg_ty_name}",
         fmt_bytes(expected),
         fmt_bytes(caller),
         fmt_bytes(callee)
@@ -99,13 +99,13 @@ pub enum CheckFailure {
         callee: Vec<u8>,
     },
     #[error(
-        "  func {func_name}'s value had unexpected variant
-    values:
-      expect: {expected}
-      caller: {caller}
-      callee: {callee}
-    the value was {val_path}: {val_ty_name}
-    whose arg was {arg_name}: {arg_ty_name}"
+        "    func {func_name}'s value had unexpected variant
+      values:
+        expect: {expected}
+        caller: {caller}
+        callee: {callee}
+      the value was {val_path}: {val_ty_name}
+      whose arg was {arg_name}: {arg_ty_name}"
     )]
     TagMismatch {
         func_idx: usize,

src/harness/check.rs

@@ -28,7 +28,7 @@ impl TestHarness {
         // Start peeling back the layers of the buffers.
         // funcs (subtests) -> vals (args/returns) -> fields -> bytes
 
-        let mut results: Vec<Result<(), CheckFailure>> = Vec::new();
+        let mut results: Vec<SubtestDetails> = Vec::new();
 
         // `Run` already checks that this length is congruent with all the inputs/outputs Vecs
         let expected_funcs = key.options.functions.active_funcs(&test.types);
@@ -53,7 +53,10 @@ impl TestHarness {
                 let caller_val = caller_func.vals.get(val_idx).unwrap_or(&empty_val);
                 let callee_val = callee_func.vals.get(val_idx).unwrap_or(&empty_val);
                 if let Err(e) = self.check_val(&test, expected_val, caller_val, callee_val) {
-                    results.push(Err(e));
+                    results.push(SubtestDetails {
+                        result: Err(e),
+                        minimized: None,
+                    });
                     // FIXME: now that each value is absolutely indexed,
                     // we should be able to check all the values independently
                     // and return all errors. However the first one is the most
@@ -63,7 +66,10 @@ impl TestHarness {
             }
 
             // If we got this far then the test passes
-            results.push(Ok(()));
+            results.push(SubtestDetails {
+                result: Ok(()),
+                minimized: None,
+            });
         }
 
         // Report the results of each subtest
@@ -77,12 +83,12 @@ impl TestHarness {
             .map(|func_id| self.full_subtest_name(key, &test.types.realize_func(func_id).name))
             .collect::<Vec<_>>();
         let max_name_len = names.iter().fold(0, |max, name| max.max(name.len()));
-        let num_passed = results.iter().filter(|r| r.is_ok()).count();
+        let num_passed = results.iter().filter(|t| t.result.is_ok()).count();
         let all_passed = num_passed == results.len();
 
         if !all_passed {
-            for (subtest_name, result) in names.iter().zip(&results) {
-                match result {
+            for (subtest_name, subtest) in names.iter().zip(&results) {
+                match &subtest.result {
                     Ok(()) => {
                         info!("Test {subtest_name:width$} passed", width = max_name_len);
                     }