Replace custom "test harnesses" with libtest-mimic (#1450)

* Replace custom "test harnesses" with `libtest-mimic`

There's a number of tests suites in this repository which are based on
"drop a file here and it's a test now". Up to now these have been
powered by a custom "test harness" copied around which implements at
filtering mechanism but little else. The `libtest-mimic` crate provides
a `test`-crate-lookalike API which enables dynamically building the list
of tests at program start time (aka read the filesystem) and then
afterwards looks and feels similar to the `test` crate's CLI and UX.

This enables a nicer dev experience when running these tests and
additionally enables using runners like `cargo nextest` with this
repository.

* Don't be so fancy about running a single test

* Disable threads in CLI test

Author: alexcrichton

Cargo.lock

@@ -56,6 +56,7 @@ url = "2.0.0"
 pretty_assertions = "1.3.0"
 semver = "1.0.0"
 smallvec = "1.11.1"
+libtest-mimic = "0.7.0"
 
 wasm-compose = { version = "0.201.0", path = "crates/wasm-compose" }
 wasm-encoder = { version = "0.201.0", path = "crates/wasm-encoder" }
@@ -136,6 +137,7 @@ tempfile = "3.1"
 diff = "0.1"
 wast = { path = 'crates/wast' }
 pretty_assertions = { workspace = true }
+libtest-mimic = { workspace = true }
 
 [[test]]
 name = "cli"

Cargo.toml

@@ -24,7 +24,7 @@ bumpalo = "3.14.0"
 
 [dev-dependencies]
 anyhow = { workspace = true }
-rayon = { workspace = true }
+libtest-mimic = { workspace = true }
 wasmparser = { path = "../wasmparser" }
 wat = { path = "../wat" }
 

crates/wast/Cargo.toml

@@ -4,46 +4,28 @@
 //! Use `BLESS=1` in the environment to auto-update `*.err` files. Be sure to
 //! look at the diff!
 
-use rayon::prelude::*;
+use libtest_mimic::{Arguments, Trial};
 use std::env;
 use std::path::{Path, PathBuf};
 
 fn main() {
     let mut tests = Vec::new();
     find_tests("tests/parse-fail".as_ref(), &mut tests);
-    let filter = std::env::args().nth(1);
-
     let bless = env::var("BLESS").is_ok();
-    let tests = tests
-        .iter()
-        .filter(|test| {
-            if let Some(filter) = &filter {
-                if let Some(s) = test.file_name().and_then(|s| s.to_str()) {
-                    if !s.contains(filter) {
-                        return false;
-                    }
-                }
-            }
-            true
-        })
-        .collect::<Vec<_>>();
-
-    println!("running {} tests\n", tests.len());
-
-    let errors = tests
-        .par_iter()
-        .filter_map(|test| run_test(test, bless).err())
-        .collect::<Vec<_>>();
 
-    if !errors.is_empty() {
-        for msg in errors.iter() {
-            eprintln!("{}", msg);
-        }
-
-        panic!("{} tests failed", errors.len())
+    let mut trials = Vec::new();
+    for test in tests {
+        let trial = Trial::test(format!("{test:?}"), move || {
+            run_test(&test, bless).map_err(|e| format!("{e:?}").into())
+        });
+        trials.push(trial);
     }
 
-    println!("test result: ok. {} passed\n", tests.len());
+    let mut args = Arguments::from_args();
+    if cfg!(target_family = "wasm") && !cfg!(target_feature = "atomics") {
+        args.test_threads = Some(1);
+    }
+    libtest_mimic::run(&args, trials).exit();
 }
 
 fn run_test(test: &Path, bless: bool) -> anyhow::Result<()> {

crates/wast/tests/parse-fail.rs

@@ -44,11 +44,11 @@ decoding = ['dep:wasmparser']
 wat = ['decoding', 'dep:wat']
 
 [dev-dependencies]
-rayon = "1"
 env_logger = { workspace = true }
 pretty_assertions = { workspace = true }
 serde_json = { workspace = true }
 wit-parser = { path = '.', features = ['serde', 'wat'] }
+libtest-mimic = { workspace = true }
 
 [[test]]
 name = "all"

crates/wit-parser/Cargo.toml

@@ -8,61 +8,36 @@
 //!     cargo test --test all foo.wit
 
 use anyhow::{bail, Context, Result};
+use libtest_mimic::{Arguments, Trial};
 use pretty_assertions::StrComparison;
-use rayon::prelude::*;
 use std::env;
 use std::ffi::OsStr;
 use std::fs;
 use std::io;
 use std::path::{Path, PathBuf};
 use std::str;
-use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
 use wit_parser::*;
 
 fn main() {
     env_logger::init();
     let tests = find_tests();
-    let filter = std::env::args().nth(1);
 
-    let tests = tests
-        .par_iter()
-        .filter_map(|test| {
-            if let Some(filter) = &filter {
-                if let Some(s) = test.to_str() {
-                    if !s.contains(filter) {
-                        return None;
-                    }
-                }
-            }
-            Some(test)
-        })
-        .collect::<Vec<_>>();
-
-    println!("running {} test files\n", tests.len());
-
-    let ntests = AtomicUsize::new(0);
-    let errors = tests
-        .par_iter()
-        .filter_map(|test| {
-            Runner { ntests: &ntests }
-                .run(test)
+    let mut trials = Vec::new();
+    for test in tests {
+        let trial = Trial::test(format!("{test:?}"), move || {
+            Runner {}
+                .run(&test)
                 .context(format!("test {:?} failed", test))
-                .err()
-        })
-        .collect::<Vec<_>>();
-
-    if !errors.is_empty() {
-        for msg in errors.iter() {
-            eprintln!("{:?}", msg);
-        }
-
-        panic!("{} tests failed", errors.len())
+                .map_err(|e| format!("{e:?}").into())
+        });
+        trials.push(trial);
     }
 
-    println!(
-        "test result: ok. {} directives passed\n",
-        ntests.load(SeqCst)
-    );
+    let mut args = Arguments::from_args();
+    if cfg!(target_family = "wasm") && !cfg!(target_feature = "atomics") {
+        args.test_threads = Some(1);
+    }
+    libtest_mimic::run(&args, trials).exit();
 }
 
 /// Recursively finds all tests in a whitelisted set of directories which we
@@ -95,11 +70,9 @@ fn find_tests() -> Vec<PathBuf> {
     }
 }
 
-struct Runner<'a> {
-    ntests: &'a AtomicUsize,
-}
+struct Runner {}
 
-impl Runner<'_> {
+impl Runner {
     fn run(&mut self, test: &Path) -> Result<()> {
         let mut resolve = Resolve::new();
         let result = resolve.push_path(test);
@@ -164,13 +137,8 @@ impl Runner<'_> {
                 );
             }
         }
-        self.bump_ntests();
         Ok(())
     }
-
-    fn bump_ntests(&self) {
-        self.ntests.fetch_add(1, SeqCst);
-    }
 }
 
 fn normalize(s: &str, extension: &str) -> String {

crates/wit-parser/tests/all.rs

@@ -26,59 +26,36 @@
 //! to look at the diff!
 
 use anyhow::{anyhow, bail, Context, Result};
+use libtest_mimic::{Arguments, Trial};
 use pretty_assertions::StrComparison;
-use rayon::prelude::*;
 use std::env;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Output, Stdio};
 
 fn main() {
-    // This test suite can't run on wasm since it involves spawning
-    // subprocesses.
-    if cfg!(target_family = "wasm") {
-        return;
-    }
-
     let mut tests = Vec::new();
     find_tests("tests/cli".as_ref(), &mut tests);
-    let filter = std::env::args().nth(1);
-
     let bless = env::var("BLESS").is_ok();
-    let tests = tests
-        .iter()
-        .filter(|test| {
-            if let Some(filter) = &filter {
-                if let Some(s) = test.file_name().and_then(|s| s.to_str()) {
-                    if !s.contains(filter) {
-                        return false;
-                    }
-                }
-            }
-            true
-        })
-        .collect::<Vec<_>>();
-
-    println!("running {} tests\n", tests.len());
 
-    let errors = tests
-        .par_iter()
-        .filter_map(|test| {
-            run_test(test, bless)
+    let mut trials = Vec::new();
+    for test in tests {
+        let trial = Trial::test(format!("{test:?}"), move || {
+            run_test(&test, bless)
                 .with_context(|| format!("failed test {test:?}"))
-                .err()
+                .map_err(|e| format!("{e:?}").into())
         })
-        .collect::<Vec<_>>();
-
-    if !errors.is_empty() {
-        for msg in errors.iter() {
-            eprintln!("{:?}", msg);
-        }
-
-        panic!("{} tests failed", errors.len())
+        // This test suite can't run on wasm since it involves spawning
+        // subprocesses.
+        .with_ignored_flag(cfg!(target_family = "wasm"));
+        trials.push(trial);
     }
 
-    println!("test result: ok. {} passed\n", tests.len());
+    let mut args = Arguments::from_args();
+    if cfg!(target_family = "wasm") && !cfg!(target_feature = "atomics") {
+        args.test_threads = Some(1);
+    }
+    libtest_mimic::run(&args, trials).exit();
 }
 
 fn wasm_tools_exe() -> Command {