From a26d3ee41caf4a8630f5bba05712d3bca39a6c06 Mon Sep 17 00:00:00 2001 From: jyviko <2501648+jyviko@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:59:22 -0400 Subject: [PATCH] xtask: known-failing grades a nextest junit report against tests/KNOWN_FAILING_TESTS `cargo xtask known-failing --junit PATH` reads nextest's junit (a file, or every .xml under a directory, so all of a run's junit artifacts at once) and fails on a test that failed and the list does not name, and on a listed test that passed (FIXED: take its line off, as the kernel rig's gate does). A listed test that did not run is neither, since a pull request tests only the packages it touches. The list has the rig's KNOWN_FAILING format (` `, sorted, the rig's own parser), with `` nextest's `::`; `cargo xtask lists` checks it is sorted. The list starts empty. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + tests/KNOWN_FAILING_TESTS | 10 ++ xtask/Cargo.toml | 1 + xtask/src/kernel_cases.rs | 2 +- xtask/src/known_failing.rs | 267 +++++++++++++++++++++++++++++++++++++ xtask/src/lists.rs | 5 +- xtask/src/main.rs | 11 +- 7 files changed, 294 insertions(+), 3 deletions(-) create mode 100644 tests/KNOWN_FAILING_TESTS create mode 100644 xtask/src/known_failing.rs diff --git a/Cargo.lock b/Cargo.lock index 3793afab4..f8d2d87f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5725,6 +5725,7 @@ name = "xtask" version = "0.24.0" dependencies = [ "proc-macro2", + "quick-xml", "serde", "serde_json", "sha2 0.10.9", diff --git a/tests/KNOWN_FAILING_TESTS b/tests/KNOWN_FAILING_TESTS new file mode 100644 index 000000000..2aaf3e407 --- /dev/null +++ b/tests/KNOWN_FAILING_TESTS @@ -0,0 +1,10 @@ +# The cargo tests that fail today, one a line: ` `, with the issue that +# tracks it when there is one. Sorted. `` is nextest's `::`, as the +# junit report names it. CI's test jobs (`cargo xtask known-failing`) fail on a test that fails +# and is not listed here, and on a listed test that passes: a fix removes its line in the same +# pull request. The tests stay un-ignored, so every run shows whether they still fail. +# Refresh (after an SMLib pin bump): download the junit artifacts of develop's latest CI run, one +# directory each (`gh run download -R tangency-labs/solidcraft -n -D /` +# for junit and junit-gpu-1..4), run `cargo xtask known-failing --junit `, then drop each +# FIXED line and add each REGRESSION line with its cause, from the test's failure message. +# Baseline: none yet. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 3715ecaf2..2f95f5505 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -7,6 +7,7 @@ publish = false [dependencies] proc-macro2 = { workspace = true } +quick-xml = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } diff --git a/xtask/src/kernel_cases.rs b/xtask/src/kernel_cases.rs index 9e94eedec..e931beb31 100644 --- a/xtask/src/kernel_cases.rs +++ b/xtask/src/kernel_cases.rs @@ -372,7 +372,7 @@ pub fn run(rest: &[&str]) -> Result<(), String> { /// The cases a known-failing baseline lists, with their causes: one per line, ` `, /// the case's directory relative to the baseline's own; `#` starts a comment line. The list is /// sorted with no case twice, and every case has a cause. -fn known_failing(text: &str) -> Result, String> { +pub(crate) fn known_failing(text: &str) -> Result, String> { let mut out: Vec<(&str, &str)> = Vec::new(); for (n, line) in text.lines().enumerate() { let line = line.trim(); diff --git a/xtask/src/known_failing.rs b/xtask/src/known_failing.rs new file mode 100644 index 000000000..b85b3a0c9 --- /dev/null +++ b/xtask/src/known_failing.rs @@ -0,0 +1,267 @@ +//! `cargo xtask known-failing --junit PATH... [--list FILE]`: grades a nextest run (its junit +//! report) against the tests known to fail today, `tests/KNOWN_FAILING_TESTS`. CI's test jobs +//! run it after nextest, in place of failing on any failed test. It fails on a test that failed +//! and is not listed, and on a listed test that passed (fixed: its line comes off in the same +//! pull request); a listed test that did not run (a pull request tests only the packages it +//! touches) is neither. The list has the kernel rig's format (`kernel_cases::known_failing`): +//! one test a line, ` `, sorted, with `` nextest's +//! `::` (`solidcraft-kernel::tests::...`, or +//! `solidcraft-cli::action_matrix::action_matrix_sample` for an integration test binary). + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use quick_xml::Reader; +use quick_xml::events::{BytesStart, Event}; + +pub const LIST: &str = "tests/KNOWN_FAILING_TESTS"; + +pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> { + let mut reports = Vec::new(); + let mut list = root.join(LIST); + let mut args = rest.iter(); + while let Some(arg) = args.next() { + match *arg { + "--junit" => reports.push(PathBuf::from(args.next().ok_or("--junit needs a junit file or a directory of them")?)), + "--list" => list = PathBuf::from(args.next().ok_or("--list needs a file")?), + other => return Err(format!("known-failing: unknown argument `{other}`")), + } + } + if reports.is_empty() { + return Err("known-failing: give the nextest junit report with --junit FILE or DIR".into()); + } + let mut files = Vec::new(); + for path in &reports { + xml_files(path, &mut files)?; + } + if files.is_empty() { + return Err(format!( + "known-failing: no junit (.xml) file in {}", + reports.iter().map(|p| p.display().to_string()).collect::>().join(", ") + )); + } + let mut outcomes = BTreeMap::new(); + for file in &files { + let text = std::fs::read_to_string(file).map_err(|e| format!("{}: {e}", file.display()))?; + add(&mut outcomes, &text).map_err(|e| format!("{}: {e}", file.display()))?; + } + let text = std::fs::read_to_string(&list).map_err(|e| format!("{}: {e}", list.display()))?; + println!("{}", grade(&text, &outcomes).map_err(|e| format!("{}: {e}", list.display()))?); + Ok(()) +} + +/// The tests of one junit report into `all`. A test in two reports failed if it failed in either. +fn add(all: &mut BTreeMap, xml: &str) -> Result<(), String> { + for (test, failed) in junit_outcomes(xml)? { + *all.entry(test).or_insert(false) |= failed; + } + Ok(()) +} + +/// `cargo xtask lists`: the list is sorted, each test once, with a cause each. +pub fn check_list(root: &Path) -> Result<(), String> { + let file = root.join(LIST); + let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?; + crate::kernel_cases::known_failing(&text).map(|_| ()).map_err(|e| format!("{LIST}: {e}")) +} + +/// `path` itself, or every `.xml` file under it (sorted), into `out`. +fn xml_files(path: &Path, out: &mut Vec) -> Result<(), String> { + if path.is_file() { + out.push(path.to_path_buf()); + return Ok(()); + } + let entries = std::fs::read_dir(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut paths: Vec = entries.filter_map(Result::ok).map(|e| e.path()).collect(); + paths.sort(); + for p in paths { + if p.is_dir() { + xml_files(&p, out)?; + } else if p.extension().is_some_and(|x| x == "xml") { + out.push(p); + } + } + Ok(()) +} + +/// Every test a junit report ran, as `::` with whether it failed (a `failure` +/// or `error` in its case). A skipped case did not run and is left out; a case that passed on a +/// retry (nextest's `flakyFailure`) passed. +fn junit_outcomes(xml: &str) -> Result, String> { + let mut reader = Reader::from_str(xml); + let mut suite: Option = None; + // The open test case: its id, whether it failed, whether it was skipped. + let mut case: Option<(String, bool, bool)> = None; + let mut out = Vec::new(); + loop { + let event = reader.read_event().map_err(|e| format!("not a junit report: {e}"))?; + let empty = matches!(event, Event::Empty(_)); + match event { + Event::Start(e) | Event::Empty(e) => match e.name().as_ref() { + b"testsuite" if !empty => suite = Some(attribute(&e, "name")?), + b"testcase" => { + let name = attribute(&e, "name")?; + let suite = suite.as_deref().ok_or_else(|| format!("test case {name} outside a testsuite"))?; + let id = format!("{suite}::{name}"); + if empty { + out.push((id, false)); + } else { + case = Some((id, false, false)); + } + } + b"failure" | b"error" => { + if let Some(c) = case.as_mut() { + c.1 = true; + } + } + b"skipped" => { + if let Some(c) = case.as_mut() { + c.2 = true; + } + } + _ => {} + }, + Event::End(e) => match e.name().as_ref() { + b"testcase" => { + if let Some((id, failed, skipped)) = case.take() + && !skipped + { + out.push((id, failed)); + } + } + b"testsuite" => suite = None, + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + Ok(out) +} + +fn attribute(e: &BytesStart, name: &str) -> Result { + let tag = String::from_utf8_lossy(e.name().as_ref()).into_owned(); + let a = e.try_get_attribute(name).map_err(|err| format!("<{tag}>: {err}"))?.ok_or_else(|| format!("<{tag}> has no {name}"))?; + a.normalized_value(quick_xml::XmlVersion::Implicit1_0).map(|v| v.into_owned()).map_err(|err| format!("<{tag} {name}>: {err}")) +} + +/// The gate: every test that ran passes except those `list` names. A failed test the list does +/// not name is a regression; a listed test that passed is fixed and must leave the list, so the +/// list only shrinks. A listed test that did not run is neither. +fn grade(list: &str, outcomes: &BTreeMap) -> Result { + let listed = crate::kernel_cases::known_failing(list)?; + let is_listed = |test: &str| listed.iter().any(|(t, _)| *t == test); + let mut problems = Vec::new(); + let mut known = 0; + for (test, failed) in outcomes { + match (is_listed(test), *failed) { + (false, true) => problems.push(format!("REGRESSION, failed and not on the known-failing list: {test}")), + (true, false) => problems.push(format!("FIXED, remove it from the known-failing list: {test}")), + (true, true) => known += 1, + (false, false) => {} + } + } + let not_run = listed.iter().filter(|(t, _)| !outcomes.contains_key(*t)).count(); + let summary = format!( + "known-failing: {} ran, {} pass, {known} known failing, {not_run} listed did not run here", + outcomes.len(), + outcomes.values().filter(|f| !**f).count() + ); + if problems.is_empty() { Ok(format!("{summary}; no regression")) } else { Err(format!("{summary}\n {}", problems.join("\n "))) } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::{add, grade, junit_outcomes}; + + /// A nextest report in miniature: a library and an integration test binary, a pass, a + /// failure, an error, a skipped case and a pass on a retry. + const REPORT: &str = r#" + + + + + panicked: 1 vs 2 + running 1 test + + + + + + + + + + + + + +"#; + + fn outcomes(xml: &str) -> BTreeMap { + junit_outcomes(xml).unwrap().into_iter().collect() + } + + #[test] + fn a_junit_report_gives_each_test_that_ran_with_whether_it_failed() { + let got: Vec<(String, bool)> = junit_outcomes(REPORT).unwrap(); + let want = [ + ("solidcraft-kernel::tests::passes", false), + ("solidcraft-kernel::tests::fails", true), + ("solidcraft-kernel::tests::times_out", true), + ("solidcraft-cli::action_matrix::action_matrix_sample", false), + ("solidcraft-cli::action_matrix::a__case", false), + ]; + assert_eq!(got, want.map(|(t, f)| (t.to_string(), f))); + assert!(junit_outcomes("").unwrap_err().contains("outside a testsuite")); + assert!(junit_outcomes("").unwrap_err().contains("has no name")); + assert!(junit_outcomes("").unwrap_err().contains("not a junit report")); + } + + #[test] + fn the_gate_passes_listed_failures_and_fails_on_a_new_failure_or_a_fixed_test() { + let list = "# today\n\ + solidcraft-kernel::tests::fails SMLib status 1001\n\ + solidcraft-kernel::tests::not_run_here volume 1e-4 off (#2422)\n\ + solidcraft-kernel::tests::times_out SMLib hangs on the corner\n"; + let ok = grade(list, &outcomes(REPORT)).unwrap(); + assert_eq!(ok, "known-failing: 5 ran, 3 pass, 2 known failing, 1 listed did not run here; no regression"); + + let fixed = REPORT.replace("", ""); + let err = grade(list, &outcomes(&fixed)).unwrap_err(); + assert!(err.contains("FIXED, remove it from the known-failing list: solidcraft-kernel::tests::times_out"), "{err}"); + assert!(err.starts_with("known-failing: 5 ran, 4 pass, 1 known failing"), "{err}"); + + let shorter = list.replace("solidcraft-kernel::tests::fails SMLib status 1001\n", ""); + let err = grade(&shorter, &outcomes(REPORT)).unwrap_err(); + assert!(err.contains("REGRESSION, failed and not on the known-failing list: solidcraft-kernel::tests::fails"), "{err}"); + assert!(!err.contains("FIXED"), "{err}"); + } + + #[test] + fn a_test_failing_in_any_report_failed() { + let passed = REPORT.replace( + "panicked: 1 vs 2", + "", + ); + for reports in [[REPORT, passed.as_str()], [passed.as_str(), REPORT]] { + let mut all = BTreeMap::new(); + for r in reports { + add(&mut all, r).unwrap(); + } + assert_eq!(all.get("solidcraft-kernel::tests::fails"), Some(&true)); + assert_eq!(all.get("solidcraft-kernel::tests::passes"), Some(&false)); + } + assert_eq!(outcomes(&passed).get("solidcraft-kernel::tests::fails"), Some(&false)); + } + + #[test] + fn the_list_is_sorted_with_a_cause_each() { + let none = BTreeMap::new(); + assert!(grade("b x\na y\n", &none).unwrap_err().contains("out of order")); + assert!(grade("a x\na y\n", &none).unwrap_err().contains("a repeat of")); + assert!(grade("a\n", &none).unwrap_err().contains("gives no cause")); + } +} diff --git a/xtask/src/lists.rs b/xtask/src/lists.rs index 7f4054013..30311966c 100644 --- a/xtask/src/lists.rs +++ b/xtask/src/lists.rs @@ -5,6 +5,8 @@ //! //! - The message catalogs (`assets/locales/*.json`): one message per line, sorted by id, no //! duplicate ids, as `serde_json` pretty-prints them. +//! - `tests/KNOWN_FAILING_TESTS`: one test a line with its cause, sorted, each test once +//! (checked only; `cargo xtask known-failing` reads it). use std::path::Path; @@ -39,7 +41,8 @@ pub fn run(root: &Path, fix: bool) -> Result<(), String> { } crate::pending::check_locales(root)?; crate::pending::check_attribution(root)?; - println!("lists: message catalogs are sorted; pending locale and attribution files fold cleanly"); + crate::known_failing::check_list(root)?; + println!("lists: message catalogs and the known-failing tests are sorted; pending locale and attribution files fold cleanly"); Ok(()) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index d4ae31264..2807783ab 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -12,6 +12,7 @@ mod i18n; mod icons; mod kernel_cases; mod kernel_ladder; +mod known_failing; mod layers; mod licences; mod lists; @@ -68,6 +69,12 @@ commands: --coverage: instead of running cases, fail on a registry command (the CLI's `commands`, planned ones aside) that no recipe.json under tests/kernel-cases/ uses and tests/kernel-cases/EXEMPT does not list, and on a stale EXEMPT line + known-failing --junit PATH... [--list FILE] + grade a nextest run's junit report (a file, or every .xml under a directory) + against the tests known to fail today, tests/KNOWN_FAILING_TESTS (FILE): fails + on a test that failed and is not listed, and on a listed test that passed + (fixed: take its line off); a listed test that did not run is neither. CI's + test jobs run it after nextest action-matrix [--bless] [--results FILE] [--from DIR] try every ordinary action on every face, edge, vertex and body of the design corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it; @@ -91,7 +98,8 @@ commands: into ATTRIBUTION.md, and delete them (the Generated files workflow runs this on main; pull requests add the pending files instead of editing the shared ones) lists [--fix] check that files many changes add to are sorted one item per line (the - message catalogs in assets/locales); --fix sorts them + message catalogs in assets/locales, tests/KNOWN_FAILING_TESTS); --fix sorts + the catalogs skills check that AGENTS.md and skills/*/SKILL.md cite files, `cargo xtask` commands and skills that exist, that the router table lists exactly the skills there are, and that AGENTS.md and new skills stay short @@ -126,6 +134,7 @@ fn main() -> ExitCode { Some("features") => features::run(&root()), Some("i18n") => i18n::run(&root(), &rest), Some("kernel-cases") => kernel_cases::run(&rest), + Some("known-failing") => known_failing::run(&root(), &rest), Some("oracle") => cmd_oracle(rest.iter().position(|a| *a == "--cli").and_then(|i| rest.get(i + 1)).copied()), Some("action-matrix") => { let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied();