Files
solidcraft/xtask/src/kernel_cases.rs
T
81e55b5443 New kernel cases go to solidcore's corpus; the rig's case list only shrinks (#11646)
* New kernel cases go to solidcore's corpus; tests/kernel-cases/ only shrinks

tests/kernel-cases/CASES lists every rig and ladder case there is today. `cargo xtask lists`
fails on a case directory it does not list, saying where a kernel case goes instead
(solidcore's crates/solidcore/tests/kernel_cases/, exported with `cargo xtask kernel-cases
--export` or written by hand), and on a listed case that is gone. AGENTS.md, CONTRIBUTING.md,
the tests skill, the rig README and EXEMPT say the same; a new command gets an EXEMPT line
naming where it is tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ

* CONTRIBUTING: the kernel-cases paragraph ends before the naming corpus one

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ

---------

Co-authored-by: jyviko <2501648+jyviko@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 15:42:09 +00:00

827 lines
41 KiB
Rust

//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases`, one process per case
//! (so a crash or a hang is that case's result, not the run's), and writes its report,
//! `docs/kernel-cases.md`: results per case, coverage per area and per source, and the issues and
//! PRs looked at and skipped. Each case is graded against its own answer.
//!
//! The run's output folder (`--out`, default `target/kernel-cases`) holds `results.json`, every
//! case's full result, and `dumps/<area>-<case>/`, the inputs of each kernel call a case's run
//! failed in (`solidcraft-cli kernel-cases --dumps`), for replay.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use serde_json::{Value, json};
use crate::{cargo, root};
const DEFAULT_TIMEOUT_S: u64 = 180;
/// How many cases ended in each status.
type StatusCounts = BTreeMap<String, usize>;
pub(crate) struct Options {
cli: Option<String>,
area: Option<String>,
timeout: Duration,
pub(crate) jobs: usize,
/// The run's output folder: `results.json` and the kernel's dumps (`dumps/`).
out: PathBuf,
/// `--known-failing FILE`: run as a regression gate against that baseline (see [`gate`]).
known_failing: Option<PathBuf>,
/// `--validation MODE`: when SMLib's full check runs on the cases' features (each_step,
/// each_feature, off; the CLI's default, each_feature, when not given).
validation: Option<String>,
/// `--export DIR`: each case's corpus entry for solidcore's rig into DIR (`solidcraft-cli
/// kernel-cases --export`), then DIR/manifest.json; no report is written.
pub(crate) export: Option<PathBuf>,
}
/// The run's options; `out` is its output folder unless `--out` gives one.
pub(crate) fn options(rest: &[&str], out: &str) -> Result<Options, String> {
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string());
let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
let out = arg("--out").map_or_else(|| root().join(out), PathBuf::from);
if rest.contains(&"--route") || rest.contains(&"--kernel") {
return Err("--route and --kernel are gone: the cases run on solidcore, the only kernel".into());
}
let known_failing = arg("--known-failing").map(|p| {
let p = PathBuf::from(p);
if p.is_absolute() { p } else { root().join(p) }
});
let validation = arg("--validation");
if let Some(v) = validation.as_deref().filter(|v| !["each_step", "each_feature", "off"].contains(v)) {
return Err(format!("--validation must be each_step, each_feature or off, not `{v}`"));
}
let export = arg("--export").map(|p| {
let p = PathBuf::from(p);
if p.is_absolute() { p } else { root().join(p) }
});
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing, validation, export })
}
/// Every case's full result as JSON, into the run's output folder.
pub(crate) fn save_results(o: &Options, results: &[Value]) -> Result<(), String> {
save_json(&o.out, "results.json", results)
}
/// `value` as pretty JSON into `dir/name`.
fn save_json<T: serde::Serialize + ?Sized>(dir: &Path, name: &str, value: &T) -> Result<(), String> {
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let path = dir.join(name);
let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?;
std::fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))
}
pub(crate) fn read(path: &Path) -> Value {
std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or(Value::Null)
}
/// Every case directory, sorted by area then id.
fn cases(dir: &Path, only: Option<&str>) -> Result<Vec<PathBuf>, String> {
let mut out = Vec::new();
for area in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() {
let area = area.path();
if !area.is_dir() || only.is_some_and(|a| area.file_name().is_none_or(|n| n != a)) {
continue;
}
for case in std::fs::read_dir(&area).map_err(|e| format!("{}: {e}", area.display()))?.flatten() {
let case = case.path();
if case.join("recipe.json").is_file() && case.join("expected.json").is_file() {
out.push(case);
}
}
}
out.sort();
Ok(out)
}
/// One case, in its own process, killed after the timeout; `path` is the case's directory under
/// `tests/kernel-cases/`.
fn run_one(o: &Options, case: &Path) -> Value {
let mut r = run_case(o, case);
let base = root().join("tests/kernel-cases");
r["path"] = json!(case.strip_prefix(&base).unwrap_or(case).to_string_lossy().replace('\\', "/"));
r
}
/// Waits for `child` for at most `timeout` while reading its stdout and stderr, so a child that
/// writes more than the pipe buffer (64 KiB on Linux) is not blocked in `write` and taken for a
/// hang. `None`: the child was still running at the timeout and has been killed.
fn wait_draining(mut child: std::process::Child, timeout: Duration) -> Result<Option<std::process::Output>, String> {
fn drain<R: std::io::Read + Send + 'static>(pipe: Option<R>) -> std::thread::JoinHandle<Vec<u8>> {
std::thread::spawn(move || {
let mut buf = Vec::new();
if let Some(mut p) = pipe {
let _ = p.read_to_end(&mut buf);
}
buf
})
}
let (stdout, stderr) = (drain(child.stdout.take()), drain(child.stderr.take()));
let start = Instant::now();
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) if start.elapsed() > timeout => {
let _ = child.kill();
let _ = child.wait();
// The readers are left to finish on their own: a grandchild (`cargo run`'s CLI)
// may still hold the pipes open.
return Ok(None);
}
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
Err(e) => return Err(format!("wait: {e}")),
}
};
let join = |h: std::thread::JoinHandle<Vec<u8>>| h.join().map_err(|_| "output: a pipe reader stopped".to_string());
Ok(Some(std::process::Output { status, stdout: join(stdout)?, stderr: join(stderr)? }))
}
fn run_case(o: &Options, case: &Path) -> Value {
let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf();
let mut cmd = match &o.cli {
Some(bin) => Command::new(bin),
None => {
let mut c = cargo();
c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--"]);
c
}
};
let mut dumps = std::ffi::OsString::from("--dumps=");
dumps.push(o.out.join("dumps"));
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps);
if let Some(v) = &o.validation {
cmd.arg(format!("--validation={v}"));
}
if let Some(d) = &o.export {
// This run's entry replaces the last one's; a case that hangs or crashes now leaves none,
// so the manifest marks it, never an entry from an earlier run.
if let Ok(under) = case.strip_prefix(root().join("tests/kernel-cases")) {
// Case ids hold dots (`m5x0.7`): the names are built whole, never by extension.
let entry = d.join(under);
let id = entry.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
for stale in [entry.with_file_name(format!("{id}.json")), entry.with_file_name(format!("{id}.bodies.json.gz"))] {
if let Err(e) = std::fs::remove_file(&stale)
&& e.kind() != std::io::ErrorKind::NotFound
{
eprintln!("kernel-cases: {}: {e}", stale.display());
}
}
}
let mut export = std::ffi::OsString::from("--export=");
export.push(d);
cmd.arg(export);
}
cmd.arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped());
let base = |status: &str, error: String| {
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
json!({"case": id, "area": area, "status": status, "error": error})
};
let child = match cmd.spawn() {
Ok(c) => c,
Err(e) => return base("fail", format!("solidcraft-cli: {e}")),
};
// A case may need longer than the run's timeout (`timeout_s`, at most 30 min).
let timeout = read(&case.join("expected.json"))["timeout_s"].as_u64().map_or(o.timeout, |s| Duration::from_secs(s.min(1800)).max(o.timeout));
let start = Instant::now();
let out = match wait_draining(child, timeout) {
Ok(Some(out)) => out,
Ok(None) => return base("hang", format!("no result after {} s", timeout.as_secs())),
Err(e) => return base("fail", e),
};
match serde_json::from_slice::<Value>(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) {
Some(mut r) => {
// `seconds` is the commands' own time, measured inside the case's process.
r["process_seconds"] = json!(start.elapsed().as_secs_f64());
r
}
// No JSON: the process died (a native crash, an abort).
None => {
let err = String::from_utf8_lossy(&out.stderr);
let tail: String = err.lines().rev().take(3).collect::<Vec<_>>().into_iter().rev().collect::<Vec<_>>().join(" / ");
base("crash", format!("process exited ({}) without a result: {tail}", out.status))
}
}
}
/// Every case, one job pool for them all; the results in order.
pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
// Without --cli each case is `cargo run`: build the CLI once first, so the first cases don't
// spend their timeout compiling it.
if o.cli.is_none() {
let mut c = cargo();
c.args(["build", "-q", "--release", "-p", "solidcraft-cli"]);
if let Err(e) = c.status().map_err(|e| e.to_string()).and_then(|s| if s.success() { Ok(()) } else { Err(s.to_string()) }) {
eprintln!("kernel-cases: building solidcraft-cli failed ({e}); each case will say why");
}
}
let next = Arc::new(Mutex::new(0usize));
let results = Arc::new(Mutex::new(vec![Value::Null; list.len()]));
std::thread::scope(|s| {
for _ in 0..o.jobs.min(list.len().max(1)) {
let (next, results) = (Arc::clone(&next), Arc::clone(&results));
s.spawn(move || {
loop {
let i = {
let Ok(mut n) = next.lock() else { return };
let i = *n;
*n += 1;
i
};
let Some(case) = list.get(i) else { return };
let r = run_one(o, case);
println!("{:<11} {}", r["status"].as_str().unwrap_or("?").to_uppercase(), case.strip_prefix(root()).unwrap_or(case).display());
if let Ok(mut all) = results.lock()
&& let Some(slot) = all.get_mut(i)
{
*slot = r;
}
}
});
}
});
Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default()
}
/// What went wrong in a result: its failed checks, then its error.
fn notes(r: &Value) -> Vec<String> {
let mut notes: Vec<String> = r["checks"]
.as_array()
.into_iter()
.flatten()
.filter(|c| !c["ok"].as_bool().unwrap_or(false))
.map(|c| format!("{} {} (want {})", c["check"].as_str().unwrap_or(""), short(&c["got"]), short(&c["want"])))
.collect();
if let Some(e) = r["error"].as_str() {
notes.push(e.chars().take(160).collect());
}
notes
}
pub(crate) fn cell(r: Option<&Value>) -> String {
let Some(r) = r else { return String::new() };
let status = r["status"].as_str().unwrap_or("?");
let notes = notes(r);
let s = match status {
"pass" => "pass".to_string(),
"unsupported" => "not supported yet".to_string(),
other => format!("**{}**", other.to_uppercase()),
};
if notes.is_empty() { s } else { format!("{s}: {}", notes.join("; ")).replace('|', "/").replace('\n', " ") }
}
fn short(v: &Value) -> String {
match v {
Value::Number(n) => n.as_f64().map(|x| format!("{x:.6}").trim_end_matches('0').trim_end_matches('.').to_string()).unwrap_or_default(),
Value::String(s) => s.chars().take(100).collect(),
Value::Array(a) => format!("[{}]", a.iter().map(short).collect::<Vec<_>>().join(", ")),
other => other.to_string(),
}
}
fn sources_of(want: &Value) -> Vec<String> {
want["sources"].as_array().into_iter().flatten().filter_map(|s| s.as_str().map(str::to_string)).collect()
}
/// The number in `owner/repo#123`, for sorting.
fn source_number(s: &str) -> u64 {
s.rsplit('#').next().and_then(|n| n.parse().ok()).unwrap_or(u64::MAX)
}
/// `tangency-labs/solidcraft#123` as a link.
fn link(source: &str) -> String {
match source.split_once('#') {
Some((repo, n)) if repo.contains('/') => format!("[{source}](https://github.com/{repo}/issues/{n})"),
_ => source.to_string(),
}
}
/// Coverage by kind of source (issue, merged PR, a frozen truck-geometry PR...), from
/// `sources.json`: how many were looked at, how many became cases, how many were skipped.
fn source_kinds(dir: &Path, cited: &BTreeMap<String, Vec<String>>) -> Result<String, String> {
let kinds = read(&dir.join("sources.json"));
let Some(kinds) = kinds.as_object() else { return Ok(String::new()) };
let cited: std::collections::BTreeSet<u64> = cited.keys().map(|s| source_number(s)).collect();
let mut table: BTreeMap<String, (usize, usize)> = BTreeMap::new();
for (n, kind) in kinds {
let row = table.entry(kind.as_str().unwrap_or("?").to_string()).or_default();
if n.parse::<u64>().is_ok_and(|n| cited.contains(&n)) {
row.0 += 1;
} else {
row.1 += 1;
}
}
let mut md = String::from("\n## Coverage by source\n\n| Source | Looked at | Became cases | Skipped |\n|---|---|---|---|\n");
let (mut a, mut b) = (0, 0);
for (kind, (cases, skipped)) in &table {
md += &format!("| {kind} | {} | {cases} | {skipped} |\n", cases + skipped);
a += cases;
b += skipped;
}
md += &format!("| **all** | {} | {a} | {b} |\n", a + b);
Ok(md)
}
/// The run's report into `to` (relative to the repository root, or absolute).
fn write_report(dir: &Path, list: &[PathBuf], results: &[Value], to: &Path) -> Result<String, String> {
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
let by_case: BTreeMap<(String, String), &Value> = results.iter().map(|r| (key(r), r)).collect();
// Coverage: per area, cases and their statuses; per source, its cases.
let mut areas: BTreeMap<String, (usize, StatusCounts)> = BTreeMap::new();
let mut per_source: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut kinds: BTreeMap<String, usize> = BTreeMap::new();
let mut rows = String::new();
for case in list {
let want = read(&case.join("expected.json"));
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let entry = areas.entry(area.clone()).or_default();
entry.0 += 1;
let run = by_case.get(&(area.clone(), id.clone())).copied();
let status = run.and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
*entry.1.entry(status).or_default() += 1;
let sources = sources_of(&want);
for s in &sources {
per_source.entry(s.clone()).or_default().push(format!("{area}/{id}"));
}
*kinds.entry(want["source"].as_str().unwrap_or("?").split_whitespace().next().unwrap_or("?").to_string()).or_default() += 1;
rows += &format!(
"| {area}/{id} | {} | {} | {} | {} |\n",
sources.iter().map(|s| link(s)).collect::<Vec<_>>().join(" "),
want["what"].as_str().unwrap_or("").replace('|', "/"),
want["source"].as_str().unwrap_or("?"),
cell(run)
);
}
let mut md = String::from(
"# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and the kernel's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nThe cases run on solidcore, SolidCraft's kernel. A failing case stays failing until the kernel is fixed; there is no workaround on SolidCraft's side. A failed kernel call leaves its inputs for replay in the run's output folder (`target/kernel-cases/dumps/<area>-<case>/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format. Every area is also a regression gate: CI's Kernel rig workflow runs them on every pull request that touches the kernel and fails on a case that fails and its area's `KNOWN_FAILING` (such as [`fillets/KNOWN_FAILING`](../tests/kernel-cases/fillets/KNOWN_FAILING)) does not list, or a listed case that passes.\n\n",
);
md += "## Coverage\n\n| Area | Cases | Results |\n|---|---|---|\n";
for (area, (n, statuses)) in &areas {
md += &format!("| {area} | {n} | {} |\n", statuses.iter().map(|(s, c)| format!("{c} {s}")).collect::<Vec<_>>().join(", "));
}
md += &format!(
"\n{} cases from {} sources; answers: {}.\n\n## Results\n\n| Case | Sources | What went wrong | Answer | Result |\n|---|---|---|---|---|\n",
list.len(),
per_source.len(),
kinds.iter().map(|(k, v)| format!("{v} {k}")).collect::<Vec<_>>().join(", ")
);
md += &rows;
md += "\n## Sources\n\n| Source | Cases |\n|---|---|\n";
for (s, cs) in &per_source {
md += &format!("| {} | {} |\n", link(s), cs.join(", "));
}
md += &source_kinds(dir, &per_source)?;
// What was looked at and skipped, area by area (`<area>/skipped.json`).
md += "\n## Looked at and skipped\n\n";
let mut skipped = 0;
let mut area_dirs: Vec<PathBuf> = std::fs::read_dir(dir).map_err(|e| e.to_string())?.flatten().map(|e| e.path()).filter(|p| p.is_dir()).collect();
area_dirs.sort();
for a in area_dirs {
// skipped.json, or several (skipped-<part>.json) where people triage an area in parts.
let mut files: Vec<PathBuf> = std::fs::read_dir(&a)
.map_err(|e| e.to_string())?
.flatten()
.map(|e| e.path())
.filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with("skipped") && n.ends_with(".json")))
.collect();
files.sort();
let mut items: Vec<Value> = files.iter().flat_map(|f| read(f).as_array().cloned().unwrap_or_default()).collect();
if items.is_empty() {
continue;
}
items.sort_by_key(|it| source_number(it["source"].as_str().unwrap_or("")));
let items = &items;
md += &format!("### {}\n\n| Source | Why skipped |\n|---|---|\n", a.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default());
for it in items {
skipped += 1;
md += &format!("| {} | {} |\n", link(it["source"].as_str().unwrap_or("?")), it["why"].as_str().unwrap_or("").replace('|', "/"));
}
md += "\n";
}
if skipped == 0 {
md += "None yet.\n";
}
let path = root().join(to);
let shown = path.strip_prefix(root()).unwrap_or(&path).display().to_string();
std::fs::write(&path, md).map_err(|e| format!("{shown}: {e}"))?;
let pass = results.iter().filter(|r| r["status"] == "pass").count();
Ok(format!("kernel-cases: {pass}/{} pass → {shown} ({skipped} skipped sources listed)", list.len()))
}
pub fn run(rest: &[&str]) -> Result<(), String> {
if rest.contains(&"--ladder") {
return crate::kernel_ladder::run(rest);
}
let o = options(rest, "target/kernel-cases")?;
let dir = root().join("tests/kernel-cases");
if rest.contains(&"--coverage") {
println!("{}", run_coverage(&o, &dir)?);
return Ok(());
}
let list = cases(&dir, o.area.as_deref())?;
if list.is_empty() {
return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
}
let results = run_all(&o, &list);
save_results(&o, &results)?;
if let Some(d) = &o.export {
println!("{}", crate::kernel_corpus::manifest(d, &results, None)?);
return Ok(());
}
if let Some(file) = &o.known_failing {
// A gate run grades the run against its baseline, and writes no report; its results.json
// is still saved above.
let text = std::fs::read_to_string(file).map_err(|e| format!("{}: {e}", file.display()))?;
let base = file.parent().unwrap_or(&dir);
let paths: Vec<String> = list.iter().map(|c| c.strip_prefix(base).unwrap_or(c).to_string_lossy().replace('\\', "/")).collect();
let statuses: Vec<&str> = results.iter().map(|r| r["status"].as_str().unwrap_or("?")).collect();
println!("{}", gate(&text, &paths, &statuses).map_err(|e| format!("{}: {e}", file.display()))?);
return Ok(());
}
println!("{}", write_report(&dir, &list, &results, Path::new("docs/kernel-cases.md"))?);
Ok(())
}
/// The cases a known-failing baseline lists, with their causes: one per line, `<case> <cause>`,
/// the case's directory relative to the baseline's own; `#` starts a comment line. The list is
/// sorted with no case twice, and every case has a cause.
pub(crate) fn known_failing(text: &str) -> Result<Vec<(&str, &str)>, String> {
let mut out: Vec<(&str, &str)> = Vec::new();
for (n, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let (case, cause) = line.split_once(char::is_whitespace).map_or((line, ""), |(c, w)| (c, w.trim()));
if cause.is_empty() {
return Err(format!("line {}: {case} gives no cause (`<case> <why it fails today>`)", n + 1));
}
if let Some((last, _)) = out.last()
&& *last >= case
{
return Err(format!(
"line {}: {case} is {} {last}: keep the list sorted, each case once",
n + 1,
if *last == case { "a repeat of" } else { "out of order after" }
));
}
out.push((case, cause));
}
Ok(out)
}
/// The regression gate: `cases` (paths relative to the baseline's directory) ran with
/// `statuses`; every case passes except those the baseline `text` lists. A case that fails and
/// is not listed is a regression; a listed case that passes is fixed and must leave the list, so
/// the list only shrinks. Either fails the gate, as does a listed case that is not in the set.
fn gate(text: &str, cases: &[String], statuses: &[&str]) -> Result<String, String> {
let listed = known_failing(text)?;
let mut problems = Vec::new();
for (case, _) in &listed {
if !cases.iter().any(|c| c == case) {
problems.push(format!("listed but not a case of this set: {case}"));
}
}
let mut failing = 0;
for (case, status) in cases.iter().zip(statuses) {
match (listed.iter().find(|(c, _)| c == case), *status == "pass") {
(None, false) => problems.push(format!("REGRESSION ({status}): {case}")),
(Some(_), true) => problems.push(format!("FIXED, remove it from the known-failing list: {case}")),
(Some(_), false) => failing += 1,
(None, true) => {}
}
}
if problems.is_empty() {
Ok(format!("kernel-cases gate: {} pass, {failing} known failing, no regression", cases.len() - failing))
} else {
Err(format!("kernel-cases gate:\n {}", problems.join("\n ")))
}
}
/// The commands of the registry (`solidcraft-cli commands`): each id, with whether it is only
/// planned (registered, not built yet).
fn registry(o: &Options) -> Result<Vec<(String, bool)>, String> {
let mut cmd = match &o.cli {
Some(bin) => Command::new(bin),
None => {
let mut c = cargo();
c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--"]);
c
}
};
let out = cmd.current_dir(root()).arg("commands").output().map_err(|e| format!("solidcraft-cli: {e}"))?;
if !out.status.success() {
return Err(format!("solidcraft-cli commands failed:\n{}", String::from_utf8_lossy(&out.stderr)));
}
let list: Value = serde_json::from_slice(&out.stdout).map_err(|e| format!("commands JSON: {e}"))?;
let list = list.as_array().ok_or("commands JSON: not a list")?;
Ok(list.iter().filter_map(|c| Some((c["id"].as_str()?.to_string(), c["planned"].as_str().is_some_and(|p| !p.is_empty())))).collect())
}
/// Every command id the command scripts (`recipe.json`) under `dir` run, at any depth.
fn used_commands(dir: &Path) -> Result<Vec<String>, String> {
let mut out = Vec::new();
let mut todo = vec![dir.to_path_buf()];
while let Some(d) = todo.pop() {
for e in std::fs::read_dir(&d).map_err(|e| format!("{}: {e}", d.display()))?.flatten() {
let p = e.path();
if p.is_dir() {
todo.push(p);
} else if p.file_name().is_some_and(|n| n == "recipe.json") {
let text = std::fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
out.extend(recipe_commands(&text).map_err(|e| format!("{}: {e}", p.display()))?);
}
}
}
out.sort();
out.dedup();
Ok(out)
}
/// The command ids a recipe runs: each `commands[].command` of a command script. An oracle
/// recipe (no `commands`) names features, not commands, and counts none.
fn recipe_commands(text: &str) -> Result<Vec<String>, String> {
let v: Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
let Some(list) = v.get("commands") else { return Ok(Vec::new()) };
let list = list.as_array().ok_or("`commands` is not a list")?;
list.iter()
.enumerate()
.map(|(i, c)| c["command"].as_str().map(str::to_string).ok_or_else(|| format!("commands[{i}] has no `command` id")))
.collect()
}
/// One line of `tests/kernel-cases/EXEMPT`: a command with no case (its reason is for readers).
#[derive(Debug, PartialEq)]
struct Exempt<'a> {
id: &'a str,
/// Listed under `[to cover]`: a geometry command still waiting for its case.
to_cover: bool,
}
/// The section header of the commands that make geometry but have no case yet.
const TO_COVER: &str = "[to cover]";
/// The commands `tests/kernel-cases/EXEMPT` exempts from the coverage check: one per line,
/// `<id> <why it has no case>`; `#` starts a comment line. The lines before `[to cover]` have no
/// geometry to grade; those after it make geometry and wait for a case, so that section only
/// shrinks. Each section is sorted, with no id twice in the file, and every line gives a reason.
fn exempt(text: &str) -> Result<Vec<Exempt<'_>>, String> {
let mut out: Vec<Exempt> = Vec::new();
let mut to_cover = false;
let mut last: Option<&str> = None;
for (n, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
if line == TO_COVER {
if to_cover {
return Err(format!("line {}: a second {TO_COVER} section", n + 1));
}
to_cover = true;
last = None;
continue;
}
let (id, why) = line.split_once(char::is_whitespace).map_or((line, ""), |(c, w)| (c, w.trim()));
if why.is_empty() {
return Err(format!("line {}: {id} gives no reason (`<id> <why it has no case>`)", n + 1));
}
if out.iter().any(|e| e.id == id) {
return Err(format!("line {}: {id} is listed twice", n + 1));
}
if let Some(prev) = last
&& prev > id
{
return Err(format!("line {}: {id} is out of order after {prev}: keep each section sorted", n + 1));
}
last = Some(id);
out.push(Exempt { id, to_cover });
}
Ok(out)
}
/// The coverage check: every built command of `registry` (id, planned) is in `used` (run by a
/// case) or exempted by `exempt_text`; an exemption names a registered command that no case
/// uses, so a case that comes removes its line.
fn coverage(registry: &[(String, bool)], used: &[String], exempt_text: &str) -> Result<String, String> {
let listed = exempt(exempt_text)?;
let mut problems = Vec::new();
for e in &listed {
if !registry.iter().any(|(id, _)| id == e.id) {
problems.push(format!("EXEMPT lists {}, which is not a registry command: remove its line", e.id));
} else if used.iter().any(|u| u == e.id) {
problems.push(format!("EXEMPT lists {}, which a case now uses: remove its line", e.id));
}
}
let mut covered = 0;
for (id, _) in registry.iter().filter(|(_, planned)| !planned) {
if used.iter().any(|u| u == id) {
covered += 1;
} else if !listed.iter().any(|e| e.id == id) {
problems.push(format!("NO CASE: {id} (tests/kernel-cases/ takes no new cases: add a line to EXEMPT naming where it is tested)"));
}
}
if !problems.is_empty() {
return Err(format!("kernel-cases coverage:\n {}", problems.join("\n ")));
}
let to_cover = listed.iter().filter(|e| e.to_cover).count();
Ok(format!("kernel-cases coverage: {covered} commands have a case, {} have no geometry, {to_cover} still to cover", listed.len() - to_cover))
}
fn run_coverage(o: &Options, dir: &Path) -> Result<String, String> {
let file = dir.join("EXEMPT");
let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
coverage(&registry(o)?, &used_commands(dir)?, &text).map_err(|e| format!("{}: {e}", file.display()))
}
/// The committed list of the rig's cases. New kernel cases go to solidcore's corpus, so this
/// list only shrinks: `cargo xtask lists` fails on a case it does not name, and on a line whose
/// case is gone (the pull request that deletes a case removes its line).
pub const CASES_LIST: &str = "tests/kernel-cases/CASES";
/// Where a new kernel case goes instead, for the check's failure message.
const NEW_CASES_GO: &str = "tests/kernel-cases/ takes no new cases. A kernel case goes to solidcore's corpus, \
crates/solidcore/tests/kernel_cases/<area>/ in tangency-labs/solidcore: draft it here as a recipe, export it with \
`cargo xtask kernel-cases --area <area> --export DIR` and add DIR/<area>/<case>.json (and its .bodies.json.gz) there, \
or write the entry by hand as that folder's README describes; then delete the draft. A case about a command, the UI \
or a product rule is a test in its area's existing test file here (CONTRIBUTING.md, \"Tests\").";
/// `cargo xtask lists`: every case directory under `tests/kernel-cases/` is in [`CASES_LIST`],
/// and every line there is a case.
pub fn check_cases_list(root: &Path) -> Result<(), String> {
let dir = root.join("tests/kernel-cases");
let mut found = Vec::new();
case_dirs(&dir, &dir, &mut found)?;
let file = root.join(CASES_LIST);
let text = std::fs::read_to_string(&file).map_err(|e| format!("{CASES_LIST}: {e}"))?;
cases_listed(&text, &found).map_err(|e| format!("{CASES_LIST}: {e}"))
}
/// Every directory under `dir` that holds a `recipe.json` (a rig case or a ladder case), as its
/// path relative to `base` with `/` separators.
fn case_dirs(dir: &Path, base: &Path, out: &mut Vec<String>) -> Result<(), String> {
for entry in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() {
let path = entry.path();
if !path.is_dir() {
continue;
}
if path.join("recipe.json").is_file() {
out.push(path.strip_prefix(base).unwrap_or(&path).to_string_lossy().replace('\\', "/"));
}
case_dirs(&path, base, out)?;
}
Ok(())
}
/// The list `text` (one case a line, sorted, none twice; `#` starts a comment) against the cases
/// `found`: fails on a case it does not list, and on a listed case that is not there.
fn cases_listed(text: &str, found: &[String]) -> Result<(), String> {
let mut listed: Vec<&str> = Vec::new();
for (n, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
if let Some(prev) = listed.last()
&& *prev >= line
{
let what = if *prev == line { "a repeat" } else { "out of order" };
return Err(format!("line {}: {line} is {what} after {prev}: keep the list sorted, one case a line", n + 1));
}
listed.push(line);
}
let mut new: Vec<&str> = found.iter().map(String::as_str).filter(|c| listed.binary_search(c).is_err()).collect();
new.sort_unstable();
let gone: Vec<&str> = listed.iter().copied().filter(|l| !found.iter().any(|c| c == l)).collect();
let mut problems = Vec::new();
if !new.is_empty() {
problems.push(format!("NEW CASE: {}\n{NEW_CASES_GO}", new.join(", ")));
}
if !gone.is_empty() {
problems.push(format!("listed but not a case (remove its line): {}", gone.join(", ")));
}
if problems.is_empty() { Ok(()) } else { Err(problems.join("\n")) }
}
#[cfg(test)]
mod tests {
use super::*;
/// A child that writes more than the pipe buffer still finishes, with all its output.
#[cfg(unix)]
#[test]
fn a_child_writing_more_than_the_pipe_buffer_is_not_a_hang() {
let child = Command::new("sh")
.args(["-c", "head -c 200000 /dev/zero; head -c 100000 /dev/zero >&2"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let out = wait_draining(child, Duration::from_secs(60)).unwrap().unwrap();
assert!(out.status.success());
assert_eq!((out.stdout.len(), out.stderr.len()), (200_000, 100_000));
let slow = Command::new("sh").args(["-c", "sleep 30"]).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
let t = Instant::now();
assert!(wait_draining(slow, Duration::from_millis(200)).unwrap().is_none());
assert!(t.elapsed() < Duration::from_secs(20));
}
/// `--route`, the old per-family run, and `--kernel`, the old choice of kernel, are no options:
/// passed anyway, they are refused rather than run as something else.
#[test]
fn route_and_kernel_are_refused() {
assert!(options(&["--route", "boolean=solidcore"], "x").is_err());
assert!(options(&["--kernel", "smlib"], "x").is_err());
assert!(options(&["--kernel", "solidcore", "--known-failing", "/k"], "x").is_err());
let o = options(&["--known-failing", "/k", "--out", "/r"], "x").unwrap();
assert_eq!((o.known_failing, o.out), (Some(PathBuf::from("/k")), PathBuf::from("/r")));
}
fn gate_on(text: &str, cases: &[(&str, &str)]) -> Result<String, String> {
let names: Vec<String> = cases.iter().map(|(c, _)| c.to_string()).collect();
let statuses: Vec<&str> = cases.iter().map(|(_, s)| *s).collect();
gate(text, &names, &statuses)
}
#[test]
fn the_gate_fails_on_a_regression_and_on_a_fixed_case_still_listed() {
let list = "# today\na-case SMLib refuses the corner\nb-case G1 at the corner point\n";
assert!(gate_on(list, &[("a-case", "fail"), ("b-case", "crash"), ("c-case", "pass")]).is_ok());
let regression = gate_on(list, &[("a-case", "fail"), ("b-case", "fail"), ("c-case", "hang")]).unwrap_err();
assert!(regression.contains("REGRESSION (hang): c-case"), "{regression}");
let fixed = gate_on(list, &[("a-case", "pass"), ("b-case", "fail"), ("c-case", "pass")]).unwrap_err();
assert!(fixed.contains("FIXED, remove it from the known-failing list: a-case"), "{fixed}");
let stale = gate_on(list, &[("a-case", "fail"), ("c-case", "pass")]).unwrap_err();
assert!(stale.contains("not a case of this set: b-case"), "{stale}");
}
#[test]
fn the_known_failing_list_is_sorted_with_a_cause_each() {
assert!(gate_on("b x\na y\n", &[("a", "fail"), ("b", "fail")]).unwrap_err().contains("out of order"));
assert!(gate_on("a x\na y\n", &[("a", "fail")]).unwrap_err().contains("a repeat of"));
assert!(gate_on("a\n", &[("a", "fail")]).unwrap_err().contains("gives no cause"));
}
#[test]
fn a_recipe_names_the_commands_of_its_script() {
let script = r#"{"base": "../a", "commands": [{"command": "solid.box", "params": {}}, {"command": "solid.fillet"}]}"#;
assert_eq!(recipe_commands(script).unwrap(), ["solid.box", "solid.fillet"]);
assert!(recipe_commands(r#"{"id": "01-box", "features": []}"#).unwrap().is_empty());
assert!(recipe_commands(r#"{"commands": [{"params": {}}]}"#).unwrap_err().contains("no `command` id"));
assert!(recipe_commands(r#"{"commands": {}}"#).unwrap_err().contains("not a list"));
}
#[test]
fn the_exempt_list_has_two_sorted_sections_with_a_reason_each() {
let text = "# header\nview.fit camera only\nview.top camera only\n\n[to cover]\nsolid.cone surfaces area\n";
assert_eq!(
exempt(text).unwrap(),
[Exempt { id: "view.fit", to_cover: false }, Exempt { id: "view.top", to_cover: false }, Exempt { id: "solid.cone", to_cover: true },]
);
assert!(exempt("view.top x\nview.fit x\n").unwrap_err().contains("out of order"));
assert!(exempt("view.fit x\n[to cover]\nview.fit y\n").unwrap_err().contains("listed twice"));
assert!(exempt("view.fit\n").unwrap_err().contains("gives no reason"));
assert!(exempt("[to cover]\n[to cover]\n").unwrap_err().contains("second"));
}
#[test]
fn coverage_fails_on_a_command_with_no_case_and_on_a_stale_exemption() {
let registry: Vec<(String, bool)> =
[("solid.box", false), ("solid.cone", false), ("view.fit", false), ("surface.fillet", true)].map(|(i, p)| (i.to_string(), p)).to_vec();
let used = ["solid.box".to_string()];
let list = "view.fit camera only\n[to cover]\nsolid.cone surfaces area\n";
let ok = coverage(&registry, &used, list).unwrap();
assert!(ok.contains("1 commands have a case, 1 have no geometry, 1 still to cover"), "{ok}");
let missing = coverage(&registry, &used, "view.fit camera only\n").unwrap_err();
assert!(missing.contains("NO CASE: solid.cone"), "{missing}");
assert!(!missing.contains("surface.fillet"), "a planned command needs no case: {missing}");
let covered = coverage(&registry, &["solid.box".into(), "solid.cone".into()], list).unwrap_err();
assert!(covered.contains("EXEMPT lists solid.cone, which a case now uses"), "{covered}");
let gone = coverage(&registry, &used, "view.fit camera only\nview.gone removed\n[to cover]\nsolid.cone x\n").unwrap_err();
assert!(gone.contains("view.gone, which is not a registry command"), "{gone}");
}
/// The case list only shrinks: a new case directory fails with where kernel cases go now, a
/// deleted case still listed fails, and the list stays sorted with no case twice.
#[test]
fn the_case_list_fails_on_a_new_case_and_on_a_gone_one() {
let found = |cases: &[&str]| cases.iter().map(|c| c.to_string()).collect::<Vec<_>>();
let list = "# header\nbooleans/1-a\nladder/01-tier/001-box\n";
assert!(cases_listed(list, &found(&["ladder/01-tier/001-box", "booleans/1-a"])).is_ok());
let new = cases_listed(list, &found(&["booleans/1-a", "booleans/2-b", "ladder/01-tier/001-box"])).unwrap_err();
assert!(new.contains("NEW CASE: booleans/2-b"), "{new}");
assert!(new.contains("crates/solidcore/tests/kernel_cases/"), "{new}");
let gone = cases_listed(list, &found(&["booleans/1-a"])).unwrap_err();
assert!(gone.contains("remove its line): ladder/01-tier/001-box"), "{gone}");
assert!(cases_listed("b\na\n", &found(&["a", "b"])).unwrap_err().contains("out of order"));
assert!(cases_listed("a\na\n", &found(&["a"])).unwrap_err().contains("a repeat"));
}
}