diff --git a/tests/kernel-cases/README.md b/tests/kernel-cases/README.md index cc0ced540..cef8d505b 100644 --- a/tests/kernel-cases/README.md +++ b/tests/kernel-cases/README.md @@ -47,6 +47,9 @@ Each case has: files the case writes (an export). - `expected.json`: the right answer, graded by `solidcraft-cli kernel-cases`. +A script may start with `"base": ""` (relative to its own directory): that case's +commands, and its own base's, run first. The ladder's cases build on each other this way. + Each area also has `skipped.json` (or several, `skipped-.json`, where an area is triaged in parts): every issue and PR looked at that did not become a case, with one line each saying why. @@ -81,11 +84,38 @@ parts): every issue and PR looked at that did not become a case, with one line e | `body_count`, `faces`, `edges`, `vertices` | Exact counts, where the geometry fixes them. Faces and edges count whole analytic surfaces and curves once (a full cylinder side is one face). | | `volume_mm3`, `area_mm2`, `center_of_mass`, `bbox` | As in `tests/oracle`: totals over every body, relative tolerance 1e-3 for volume and area, 1e-3 of the box diagonal for positions. | | `tolerance` | Optional overrides: `rel`, `abs_mm`. | +| `missing` | A capability SolidCraft has no command or parameter for yet (`"solid.thread has no start count"`): the case is reported **not supported yet** without running, so the gap stays visible. | + +## The complexity ladder + +`ladder/` orders kernel operations by complexity instead of by where they went wrong: tiers from +primitives through degenerate booleans, analytic and spline profiles, holes and taps, threads, +fillets, chamfers, shells, patterns, to whole parts. Later tiers build on earlier tiers' shapes (a +running part grows one tier at a time through `base`), so the last case carries everything. + +- `ladder///`: cases in the format above, each a test with its own answer. +- `ladder/ladder.json`: the tiers in order and, per tier, its cases in order (paths under + `tests/kernel-cases/`, so a tier may also list a rig case from another area). +- `ladder/golden.json`: per case and kernel the status, the commands' time and what the kernel + made (counts, volume, area). Runs compare with it and flag slowdowns (over twice the time and + 0.25 s more) and changed counts or statuses. It is re-recorded only with a reason. +- `tools/kernel-ladder/cases.py` writes the recipes and answers (standard-library Python; each + `method` gives the derivation). + +```sh +cargo xtask kernel-cases --ladder --kernel both --cli target/release/solidcraft-cli +cargo xtask kernel-cases --ladder --kernel both --cli target/release/solidcraft-cli --record "why" +``` + +`--record` measures one case at a time, rewrites the golden and sorts each tier by the first +kernel's time, fastest first; commit both files. The report, `docs/kernel-ladder.md`, gives per +tier and kernel the counts per status and the **frontier**: the first case in ladder order the +kernel fails. ## Results Per case and kernel: **pass**; **fail** with the numbers; **not supported yet** (the kernel says -so); **crash** (a panic caught by the guards, or the process died); **hang** (no result before +so, or the case names a `missing` capability); **crash** (a panic caught by the guards, or the process died); **hang** (no result before the timeout). ## Exporting to solidcore diff --git a/xtask/src/kernel_cases.rs b/xtask/src/kernel_cases.rs index 21e9f9c4d..c046c7069 100644 --- a/xtask/src/kernel_cases.rs +++ b/xtask/src/kernel_cases.rs @@ -20,17 +20,17 @@ const DEFAULT_TIMEOUT_S: u64 = 180; /// Per kernel, how many cases ended in each status. type StatusCounts = BTreeMap>; -struct Options { - kernels: Vec, +pub(crate) struct Options { + pub(crate) kernels: Vec, cli: Option, area: Option, timeout: Duration, - jobs: usize, + pub(crate) jobs: usize, /// Where to keep every case's full result as JSON. results: Option, } -fn options(rest: &[&str]) -> Result { +pub(crate) fn options(rest: &[&str]) -> Result { let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string()); let kernels = match arg("--kernel").as_deref() { None | Some("default") => vec!["default".to_string()], @@ -43,7 +43,7 @@ fn options(rest: &[&str]) -> Result { Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, results: arg("--results") }) } -fn read(path: &Path) -> Value { +pub(crate) fn read(path: &Path) -> Value { std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or(Value::Null) } @@ -66,8 +66,16 @@ fn cases(dir: &Path, only: Option<&str>) -> Result, String> { Ok(out) } -/// One case on one kernel, in its own process, killed after the timeout. +/// One case on one kernel, in its own process, killed after the timeout; `path` is the case's +/// directory under `tests/kernel-cases/`. fn run_one(o: &Options, kernel: &str, case: &Path) -> Value { + let mut r = run_case(o, kernel, case); + let base = root().join("tests/kernel-cases"); + r["path"] = json!(case.strip_prefix(&base).unwrap_or(case).to_string_lossy().replace('\\', "/")); + r +} + +fn run_case(o: &Options, kernel: &str, case: &Path) -> Value { let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf(); let mut cmd = match &o.cli { Some(bin) => Command::new(bin), @@ -111,7 +119,8 @@ fn run_one(o: &Options, kernel: &str, case: &Path) -> Value { }; match serde_json::from_slice::(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) { Some(mut r) => { - r["seconds"] = json!(start.elapsed().as_secs_f64()); + // `seconds` is the commands' own time, measured inside the case's process. + r["process_seconds"] = json!(start.elapsed().as_secs_f64()); r } // No JSON: the process died (a native crash, an abort). @@ -123,7 +132,7 @@ fn run_one(o: &Options, kernel: &str, case: &Path) -> Value { } } -fn run_all(o: &Options, list: &[PathBuf]) -> Vec { +pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec { let jobs: Vec<(String, PathBuf)> = o.kernels.iter().flat_map(|k| list.iter().map(move |c| (k.clone(), c.clone()))).collect(); let next = Arc::new(Mutex::new(0usize)); let results = Arc::new(Mutex::new(vec![Value::Null; jobs.len()])); @@ -157,7 +166,7 @@ fn run_all(o: &Options, list: &[PathBuf]) -> Vec { Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default() } -fn cell(r: Option<&Value>) -> String { +pub(crate) fn cell(r: Option<&Value>) -> String { let Some(r) = r else { return String::new() }; let status = r["status"].as_str().unwrap_or("?"); let mut notes: Vec = r["checks"] @@ -318,6 +327,9 @@ fn write_report(o: &Options, dir: &Path, list: &[PathBuf], results: &[Value]) -> } pub fn run(rest: &[&str]) -> Result<(), String> { + if rest.contains(&"--ladder") { + return crate::kernel_ladder::run(rest); + } let o = options(rest)?; let dir = root().join("tests/kernel-cases"); let list = cases(&dir, o.area.as_deref())?; diff --git a/xtask/src/kernel_ladder.rs b/xtask/src/kernel_ladder.rs new file mode 100644 index 000000000..97f23a211 --- /dev/null +++ b/xtask/src/kernel_ladder.rs @@ -0,0 +1,316 @@ +//! `cargo xtask kernel-cases --ladder [--record REASON]`: the kernel complexity ladder. +//! +//! `tests/kernel-cases/ladder/ladder.json` lists tiers in order of complexity and, per tier, its +//! cases in order (paths under `tests/kernel-cases/`, so a tier may also list rig cases from other +//! areas). Each case is a test like any rig case, graded against its own `expected.json`. On top, +//! `tests/kernel-cases/ladder/golden.json` records per case and kernel the status, the commands' +//! time and what the kernel made (counts, volume, area): the run compares with it and flags +//! slowdowns and changes. `--record REASON` re-measures the golden (one case at a time unless +//! `--jobs` says otherwise, so the times are not shared with other cases) and re-sorts each tier +//! by the first kernel's time, fastest first; both files are committed. +//! +//! The report, `docs/kernel-ladder.md`, gives per tier and kernel the counts per status and the +//! robustness frontier: the first case, in ladder order, the kernel fails. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use serde_json::{Value, json}; + +use crate::kernel_cases::{Options, cell, options, read, run_all}; +use crate::root; + +/// A case is flagged slower when it takes more than this many times its golden time… +const SLOWDOWN_FACTOR: f64 = 2.0; +/// …and at least this many seconds more (timer noise on fast cases is not a slowdown). +const SLOWDOWN_MIN_S: f64 = 0.25; + +struct Tier { + tier: String, + what: String, + cases: Vec, +} + +fn manifest_path() -> PathBuf { + root().join("tests/kernel-cases/ladder/ladder.json") +} + +fn golden_path() -> PathBuf { + root().join("tests/kernel-cases/ladder/golden.json") +} + +fn tiers(manifest: &Value) -> Result, String> { + let list = manifest["tiers"].as_array().ok_or("ladder.json: `tiers` must be an array")?; + let mut out = Vec::new(); + for t in list { + let tier = t["tier"].as_str().ok_or("ladder.json: a tier without `tier`")?.to_string(); + let cases = t["cases"] + .as_array() + .ok_or_else(|| format!("ladder.json: tier {tier} without `cases`"))? + .iter() + .map(|c| c.as_str().map(str::to_string).ok_or_else(|| format!("ladder.json: tier {tier}: a case that is not a path"))) + .collect::, _>>()?; + out.push(Tier { tier, what: t["what"].as_str().unwrap_or("").to_string(), cases }); + } + Ok(out) +} + +/// Every case directory under `ladder/` is listed once in the manifest, and every listed case +/// exists: a case left out would never run. +fn check_listed(tiers: &[Tier]) -> Result<(), String> { + let base = root().join("tests/kernel-cases"); + let mut listed: BTreeMap<&str, usize> = BTreeMap::new(); + for t in tiers { + for c in &t.cases { + *listed.entry(c.as_str()).or_default() += 1; + let d = base.join(c); + if !(d.join("recipe.json").is_file() && d.join("expected.json").is_file()) { + return Err(format!("ladder.json: {c} has no recipe.json and expected.json")); + } + } + } + if let Some((c, _)) = listed.iter().find(|(_, n)| **n > 1) { + return Err(format!("ladder.json: {c} is listed more than once")); + } + let ladder = base.join("ladder"); + for tier in std::fs::read_dir(&ladder).map_err(|e| format!("{}: {e}", ladder.display()))?.flatten() { + if !tier.path().is_dir() { + continue; + } + for case in std::fs::read_dir(tier.path()).map_err(|e| e.to_string())?.flatten() { + let rel = case.path().strip_prefix(&base).map(|p| p.to_string_lossy().replace('\\', "/")).unwrap_or_default(); + if case.path().join("recipe.json").is_file() && !listed.contains_key(rel.as_str()) { + return Err(format!("ladder.json: {rel} is not listed in any tier")); + } + } + } + Ok(()) +} + +fn rel_of(r: &Value) -> String { + r["path"].as_str().unwrap_or("").to_string() +} + +/// The golden entry for one result: status, the commands' time, what was made. +fn golden_entry(r: &Value) -> Value { + let mut g = json!({"status": r["status"], "seconds": r["seconds"]}); + if let Some(m) = r.get("measured").filter(|m| m.is_object()) { + g["measured"] = m.clone(); + } + g +} + +/// What changed against the golden: a slowdown, a status change, other counts. +fn flags(r: &Value, g: Option<&Value>) -> Vec { + let Some(g) = g else { return vec!["no golden".into()] }; + let mut out = Vec::new(); + if g["status"] != r["status"] { + out.push(format!("status was {}", g["status"].as_str().unwrap_or("?"))); + } + if let (Some(now), Some(then)) = (r["seconds"].as_f64(), g["seconds"].as_f64()) + && now > then * SLOWDOWN_FACTOR + && now - then > SLOWDOWN_MIN_S + { + out.push(format!("slower: {now:.2} s (golden {then:.2} s)")); + } + for k in ["body_count", "faces", "edges", "vertices"] { + let (now, then) = (&r["measured"][k], &g["measured"][k]); + if !now.is_null() && !then.is_null() && now != then { + out.push(format!("{k} {now} (golden {then})")); + } + } + out +} + +fn secs(v: &Value) -> String { + v.as_f64().map(|s| format!("{s:.3}")).unwrap_or_else(|| "–".into()) +} + +fn write_report(o: &Options, tiers: &[Tier], results: &BTreeMap<(String, String), Value>, golden: &Value) -> Result { + let mut md = String::from( + "# Kernel complexity ladder\n\nGenerated by `cargo xtask kernel-cases --ladder`. The ladder in [`tests/kernel-cases/ladder/`](../tests/kernel-cases/ladder/) orders kernel operations by complexity: each tier builds on the shapes of the tiers before it, so the last case carries everything. Every case is a test with its own right answer (analysis or invariants, as in the [regression rig](kernel-cases.md)); the golden (`tests/kernel-cases/ladder/golden.json`) records per kernel its time and what it made, orders each tier by time and flags slowdowns.\n\nThe **frontier** is the first case of a tier, in ladder order, that the kernel fails (fail, crash or hang).\n\n## Tiers\n\n| Tier | What | Cases |", + ); + for k in &o.kernels { + md += &format!(" {k} | {k} frontier |"); + } + md += "\n|---|---|---|"; + md += &"---|---|".repeat(o.kernels.len()); + md += "\n"; + let mut rows = String::new(); + let mut totals: BTreeMap> = BTreeMap::new(); + for t in tiers { + md += &format!("| {} | {} | {} |", t.tier, t.what.replace('|', "/"), t.cases.len()); + for k in &o.kernels { + let mut counts: BTreeMap = BTreeMap::new(); + let mut frontier = None; + for c in &t.cases { + let status = results.get(&(c.clone(), k.clone())).and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string(); + if frontier.is_none() && matches!(status.as_str(), "fail" | "crash" | "hang") { + frontier = Some(c.rsplit('/').next().unwrap_or(c).to_string()); + } + *totals.entry(k.clone()).or_default().entry(status.clone()).or_default() += 1; + *counts.entry(status).or_default() += 1; + } + let counts = counts.iter().map(|(s, n)| format!("{n} {s}")).collect::>().join(", "); + md += &format!(" {counts} | {} |", frontier.unwrap_or_else(|| "–".into())); + } + md += "\n"; + for c in &t.cases { + let want = read(&root().join("tests/kernel-cases").join(c).join("expected.json")); + rows += &format!("| {} | {} | {} |", c, want["what"].as_str().unwrap_or("").replace('|', "/"), want["source"].as_str().unwrap_or("?")); + for k in &o.kernels { + let r = results.get(&(c.clone(), k.clone())); + let g = golden["cases"][c].get(k); + let f = r.map(|r| flags(r, g)).unwrap_or_default(); + rows += &format!( + " {} | {} | {} | {} |", + cell(r), + r.map(|r| secs(&r["seconds"])).unwrap_or_default(), + g.map(|g| secs(&g["seconds"])).unwrap_or_else(|| "–".into()), + f.join("; ").replace('|', "/") + ); + } + rows += "\n"; + } + } + // Cases whose bodies SMLib's check gave no verdict on (it can't take the record yet). + let unchecked: Vec = o + .kernels + .iter() + .map(|k| { + let n = results + .iter() + .filter(|((_, kk), r)| { + kk == k && r["checks"].as_array().into_iter().flatten().any(|c| c["got"].as_str().is_some_and(|g| g.starts_with("no verdict"))) + }) + .count(); + format!("{k}: {n}") + }) + .collect(); + md += &format!( + "\nCases where SMLib's check gave no verdict on a body (SMLib can't take the record yet), so only the record's checks graded validity: {}.\n", + unchecked.join(", ") + ); + md += "\n## Cases, in ladder order\n\nTimes are the commands' own, in seconds, from the case's process; **golden** is the recorded time.\n\n| Case | What | Answer |"; + for k in &o.kernels { + md += &format!(" {k} | s | golden s | changes |"); + } + md += "\n|---|---|---|"; + md += &"---|---|---|---|".repeat(o.kernels.len()); + md += "\n"; + md += &rows; + std::fs::write(root().join("docs/kernel-ladder.md"), md).map_err(|e| format!("docs/kernel-ladder.md: {e}"))?; + let n: usize = tiers.iter().map(|t| t.cases.len()).sum(); + let summary: Vec = o + .kernels + .iter() + .map(|k| { + let pass = totals.get(k).and_then(|m| m.get("pass")).copied().unwrap_or(0); + format!("{k}: {pass}/{n} pass") + }) + .collect(); + Ok(format!("kernel ladder: {} → docs/kernel-ladder.md", summary.join(", "))) +} + +/// The golden from this run's results (keeping what it holds for kernels not run), and the +/// manifest with each tier sorted by the first kernel's time. +fn record( + o: &Options, + reason: &str, + manifest: &Value, + tiers: &[Tier], + results: &BTreeMap<(String, String), Value>, + golden: &Value, +) -> Result<(), String> { + let mut cases = golden["cases"].as_object().cloned().unwrap_or_default(); + for t in tiers { + for c in &t.cases { + let entry = cases.entry(c.clone()).or_insert_with(|| json!({})); + for k in &o.kernels { + if let Some(r) = results.get(&(c.clone(), k.clone())) { + entry[k.as_str()] = golden_entry(r); + } + } + } + } + let new_golden = json!({"reason": reason, "kernels": o.kernels, "cases": cases}); + let mut m = manifest.clone(); + let lead = o.kernels.first().cloned().unwrap_or_else(|| "default".into()); + if let Some(list) = m["tiers"].as_array_mut() { + for (t, tier) in list.iter_mut().zip(tiers) { + let mut sorted: Vec<(f64, usize, &String)> = tier + .cases + .iter() + .enumerate() + .map(|(i, c)| (results.get(&(c.clone(), lead.clone())).and_then(|r| r["seconds"].as_f64()).unwrap_or(f64::INFINITY), i, c)) + .collect(); + // Fastest first; equal times keep the manifest's order. + sorted.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); + t["cases"] = json!(sorted.iter().map(|(_, _, c)| c.as_str()).collect::>()); + } + } + let pretty = |v: &Value| serde_json::to_string_pretty(v).map(|s| s + "\n").map_err(|e| e.to_string()); + std::fs::write(golden_path(), pretty(&new_golden)?).map_err(|e| format!("golden.json: {e}"))?; + std::fs::write(manifest_path(), pretty(&m)?).map_err(|e| format!("ladder.json: {e}"))?; + println!("kernel ladder: golden recorded ({reason}); tiers re-sorted by {lead} time"); + Ok(()) +} + +pub fn run(rest: &[&str]) -> Result<(), String> { + let reason = rest.iter().position(|a| *a == "--record").map(|i| rest.get(i + 1).copied().filter(|r| !r.starts_with("--"))); + let reason = match reason { + None => None, + Some(None) => return Err("--record needs a reason: --record \"why the golden changes\"".into()), + Some(Some(r)) => Some(r.to_string()), + }; + let mut o = options(rest)?; + if reason.is_some() && !rest.contains(&"--jobs") { + o.jobs = 1; + } + let manifest = read(&manifest_path()); + if manifest.is_null() { + return Err(format!("{}: missing or not JSON", manifest_path().display())); + } + let tiers = tiers(&manifest)?; + check_listed(&tiers)?; + let base = root().join("tests/kernel-cases"); + let list: Vec = tiers.iter().flat_map(|t| t.cases.iter().map(|c| base.join(c))).collect(); + let results: BTreeMap<(String, String), Value> = + run_all(&o, &list).into_iter().map(|r| ((rel_of(&r), r["kernel"].as_str().unwrap_or("").to_string()), r)).collect(); + let golden = read(&golden_path()); + if let Some(reason) = &reason { + record(&o, reason, &manifest, &tiers, &results, &golden)?; + let tiers = self::tiers(&read(&manifest_path()))?; + println!("{}", write_report(&o, &tiers, &results, &read(&golden_path()))?); + } else { + println!("{}", write_report(&o, &tiers, &results, &golden)?); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flags_slowdowns_status_and_count_changes_only() { + let golden = json!({"status": "pass", "seconds": 1.0, "measured": {"faces": 6, "edges": 12}}); + let same = json!({"status": "pass", "seconds": 1.2, "measured": {"faces": 6, "edges": 12}}); + assert!(flags(&same, Some(&golden)).is_empty()); + // Twice as slow but within timer noise of a fast case. + let fast = json!({"status": "pass", "seconds": 0.03, "measured": {}}); + assert!(flags(&fast, Some(&json!({"status": "pass", "seconds": 0.01}))).is_empty()); + let worse = json!({"status": "fail", "seconds": 3.0, "measured": {"faces": 7, "edges": 12}}); + let f = flags(&worse, Some(&golden)); + assert_eq!(f.len(), 3, "{f:?}"); + assert_eq!(flags(&same, None), ["no golden"]); + } + + #[test] + fn tiers_need_paths() { + let m = json!({"tiers": [{"tier": "01-a", "what": "w", "cases": ["ladder/01-a/001-x"]}]}); + let t = tiers(&m).unwrap(); + assert_eq!((t[0].tier.as_str(), t[0].cases.len()), ("01-a", 1)); + assert!(tiers(&json!({"tiers": [{"tier": "01-a", "cases": [3]}]})).is_err()); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 43da91971..ee0c5926f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -11,6 +11,7 @@ mod features; mod i18n; mod icons; mod kernel_cases; +mod kernel_ladder; mod layers; mod licences; mod lists; @@ -51,6 +52,10 @@ commands: process each, killed after --timeout, default 180 s) with each kernel asked for, grade it against the case's own answer and write docs/kernel-cases.md (--results: every result as JSON too) + --ladder [--record REASON]: the complexity ladder instead + (tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared + with its timing golden, into docs/kernel-ladder.md; --record re-measures the + golden one case at a time and re-sorts each tier by time action-matrix [--bless] [--results FILE] [--from DIR] try every ordinary action on every face, edge, vertex and body of the design corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;