Files
solidcraft/xtask/src/kernel_ladder.rs
T
jyvikoandClaude Opus 5.5 b9e0543ef5 Wave 178/509: kernel-cases: the rig runs SMLib only and keeps each run's SMLib dumps
1 commits, squashed; the originals stay reachable from tag archive/merge-main-develop.

- 001c62ab1 jyviko: kernel-cases: the rig runs SMLib only and keeps each run's SMLib dumps

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ
2026-10-22 07:31:00 -04:00

275 lines
13 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! `cargo xtask kernel-cases --ladder [--record REASON]`: the kernel complexity ladder.
//!
//! `tests/kernel-cases/ladder/ladder.json` lists tiers in order of complexity and, per tier, its
//! cases in order (paths under `tests/kernel-cases/`, so a tier may also list rig cases from other
//! areas). Each case is a test like any rig case, graded against its own `expected.json`, on
//! SMLib. On top, `tests/kernel-cases/ladder/golden.json` records per case the status, the
//! commands' time and what was made (counts, volume, area): the run compares with it and flags
//! slowdowns and changes; cases run one at a time unless `--jobs` says otherwise, so the times are
//! not shared with other cases. `--record REASON` re-measures the golden and re-sorts each tier
//! by time, fastest first; both files are committed. The run's output folder (`--out`, default
//! `target/kernel-ladder`) holds `results.json` and the SMLib dumps, as the rig's does.
//!
//! The report, `docs/kernel-ladder.md`, gives per tier the counts per status and the robustness
//! frontier: the first case, in ladder order, that fails.
use std::collections::BTreeMap;
use std::path::PathBuf;
use serde_json::{Value, json};
use crate::kernel_cases::{cell, options, read, run_all, save_results};
use crate::root;
/// A case is flagged slower when it takes more than this many times its golden time…
const SLOWDOWN_FACTOR: f64 = 2.0;
/// …and at least this many seconds more (timer noise on fast cases is not a slowdown).
const SLOWDOWN_MIN_S: f64 = 0.25;
struct Tier {
tier: String,
what: String,
cases: Vec<String>,
}
fn manifest_path() -> PathBuf {
root().join("tests/kernel-cases/ladder/ladder.json")
}
fn golden_path() -> PathBuf {
root().join("tests/kernel-cases/ladder/golden.json")
}
fn tiers(manifest: &Value) -> Result<Vec<Tier>, String> {
let list = manifest["tiers"].as_array().ok_or("ladder.json: `tiers` must be an array")?;
let mut out = Vec::new();
for t in list {
let tier = t["tier"].as_str().ok_or("ladder.json: a tier without `tier`")?.to_string();
let cases = t["cases"]
.as_array()
.ok_or_else(|| format!("ladder.json: tier {tier} without `cases`"))?
.iter()
.map(|c| c.as_str().map(str::to_string).ok_or_else(|| format!("ladder.json: tier {tier}: a case that is not a path")))
.collect::<Result<Vec<_>, _>>()?;
out.push(Tier { tier, what: t["what"].as_str().unwrap_or("").to_string(), cases });
}
Ok(out)
}
/// Every case directory under `ladder/` is listed once in the manifest, and every listed case
/// exists: a case left out would never run.
fn check_listed(tiers: &[Tier]) -> Result<(), String> {
let base = root().join("tests/kernel-cases");
let mut listed: BTreeMap<&str, usize> = BTreeMap::new();
for t in tiers {
for c in &t.cases {
*listed.entry(c.as_str()).or_default() += 1;
let d = base.join(c);
if !(d.join("recipe.json").is_file() && d.join("expected.json").is_file()) {
return Err(format!("ladder.json: {c} has no recipe.json and expected.json"));
}
}
}
if let Some((c, _)) = listed.iter().find(|(_, n)| **n > 1) {
return Err(format!("ladder.json: {c} is listed more than once"));
}
let ladder = base.join("ladder");
for tier in std::fs::read_dir(&ladder).map_err(|e| format!("{}: {e}", ladder.display()))?.flatten() {
if !tier.path().is_dir() {
continue;
}
for case in std::fs::read_dir(tier.path()).map_err(|e| e.to_string())?.flatten() {
let rel = case.path().strip_prefix(&base).map(|p| p.to_string_lossy().replace('\\', "/")).unwrap_or_default();
if case.path().join("recipe.json").is_file() && !listed.contains_key(rel.as_str()) {
return Err(format!("ladder.json: {rel} is not listed in any tier"));
}
}
}
Ok(())
}
fn rel_of(r: &Value) -> String {
r["path"].as_str().unwrap_or("").to_string()
}
/// The golden entry for one result: status, the commands' time, what was made.
fn golden_entry(r: &Value) -> Value {
let mut g = json!({"status": r["status"], "seconds": r["seconds"]});
if let Some(m) = r.get("measured").filter(|m| m.is_object()) {
g["measured"] = m.clone();
}
g
}
/// What changed against the golden: a slowdown, a status change, other counts.
fn flags(r: &Value, g: Option<&Value>) -> Vec<String> {
let Some(g) = g else { return vec!["no golden".into()] };
let mut out = Vec::new();
if g["status"] != r["status"] {
out.push(format!("status was {}", g["status"].as_str().unwrap_or("?")));
}
if let (Some(now), Some(then)) = (r["seconds"].as_f64(), g["seconds"].as_f64())
&& now > then * SLOWDOWN_FACTOR
&& now - then > SLOWDOWN_MIN_S
{
out.push(format!("slower: {now:.2} s (golden {then:.2} s)"));
}
for k in ["body_count", "faces", "edges", "vertices"] {
let (now, then) = (&r["measured"][k], &g["measured"][k]);
if !now.is_null() && !then.is_null() && now != then {
out.push(format!("{k} {now} (golden {then})"));
}
}
out
}
fn secs(v: &Value) -> String {
v.as_f64().map(|s| format!("{s:.3}")).unwrap_or_else(|| "–".into())
}
fn write_report(tiers: &[Tier], results: &BTreeMap<String, Value>, golden: &Value) -> Result<String, String> {
let mut md = String::from(
"# Kernel complexity ladder\n\nGenerated by `cargo xtask kernel-cases --ladder`. The ladder in [`tests/kernel-cases/ladder/`](../tests/kernel-cases/ladder/) orders kernel operations by complexity: each tier builds on the shapes of the tiers before it, so the last case carries everything. Every case is a test with its own right answer (analysis or invariants, as in the [regression rig](kernel-cases.md)); the golden (`tests/kernel-cases/ladder/golden.json`) records each case's time and what it made, orders each tier by time and flags slowdowns. The cases run on SMLib.\n\nThe **frontier** is the first case of a tier, in ladder order, that fails (fail, crash or hang).\n\n## Tiers\n\n| Tier | What | Cases | Results | Frontier |\n|---|---|---|---|---|\n",
);
let mut rows = String::new();
let mut pass = 0;
for t in tiers {
let mut counts: BTreeMap<String, usize> = BTreeMap::new();
let mut frontier = None;
for c in &t.cases {
let status = results.get(c).and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
if frontier.is_none() && matches!(status.as_str(), "fail" | "crash" | "hang") {
frontier = Some(c.rsplit('/').next().unwrap_or(c).to_string());
}
pass += usize::from(status == "pass");
*counts.entry(status).or_default() += 1;
}
let counts = counts.iter().map(|(s, n)| format!("{n} {s}")).collect::<Vec<_>>().join(", ");
md += &format!("| {} | {} | {} | {counts} | {} |\n", t.tier, t.what.replace('|', "/"), t.cases.len(), frontier.unwrap_or_else(|| "–".into()));
for c in &t.cases {
let want = read(&root().join("tests/kernel-cases").join(c).join("expected.json"));
let r = results.get(c);
let g = golden["cases"].get(c);
let f = r.map(|r| flags(r, g)).unwrap_or_default();
rows += &format!(
"| {} | {} | {} | {} | {} | {} | {} |\n",
c,
want["what"].as_str().unwrap_or("").replace('|', "/"),
want["source"].as_str().unwrap_or("?"),
cell(r),
r.map(|r| secs(&r["seconds"])).unwrap_or_default(),
g.map(|g| secs(&g["seconds"])).unwrap_or_else(|| "–".into()),
f.join("; ").replace('|', "/")
);
}
}
// Cases whose bodies SMLib's check gave no verdict on (it can't take the record yet).
let unchecked = results
.values()
.filter(|r| r["checks"].as_array().into_iter().flatten().any(|c| c["got"].as_str().is_some_and(|g| g.starts_with("no verdict"))))
.count();
md += &format!(
"\nCases where SMLib's check gave no verdict on a body (SMLib can't take the record yet), so only the record's checks graded validity: {unchecked}.\n"
);
md += "\n## Cases, in ladder order\n\nTimes are the commands' own, in seconds, from the case's process; **golden** is the recorded time.\n\n| Case | What | Answer | Result | s | golden s | changes |\n|---|---|---|---|---|---|---|\n";
md += &rows;
std::fs::write(root().join("docs/kernel-ladder.md"), md).map_err(|e| format!("docs/kernel-ladder.md: {e}"))?;
let n: usize = tiers.iter().map(|t| t.cases.len()).sum();
Ok(format!("kernel ladder: {pass}/{n} pass → docs/kernel-ladder.md"))
}
/// The golden from this run's results (keeping what it holds for cases not run), and the
/// manifest with each tier sorted by time.
fn record(reason: &str, manifest: &Value, tiers: &[Tier], results: &BTreeMap<String, Value>, golden: &Value) -> Result<(), String> {
let mut cases = golden["cases"].as_object().cloned().unwrap_or_default();
for t in tiers {
for c in &t.cases {
if let Some(r) = results.get(c) {
cases.insert(c.clone(), golden_entry(r));
}
}
}
let new_golden = json!({"reason": reason, "cases": cases});
let mut m = manifest.clone();
if let Some(list) = m["tiers"].as_array_mut() {
for (t, tier) in list.iter_mut().zip(tiers) {
let mut sorted: Vec<(f64, usize, &String)> = tier
.cases
.iter()
.enumerate()
.map(|(i, c)| (results.get(c).and_then(|r| r["seconds"].as_f64()).unwrap_or(f64::INFINITY), i, c))
.collect();
// Fastest first; equal times keep the manifest's order.
sorted.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1)));
t["cases"] = json!(sorted.iter().map(|(_, _, c)| c.as_str()).collect::<Vec<_>>());
}
}
let pretty = |v: &Value| serde_json::to_string_pretty(v).map(|s| s + "\n").map_err(|e| e.to_string());
std::fs::write(golden_path(), pretty(&new_golden)?).map_err(|e| format!("golden.json: {e}"))?;
std::fs::write(manifest_path(), pretty(&m)?).map_err(|e| format!("ladder.json: {e}"))?;
println!("kernel ladder: golden recorded ({reason}); tiers re-sorted by time");
Ok(())
}
pub fn run(rest: &[&str]) -> Result<(), String> {
let reason = rest.iter().position(|a| *a == "--record").map(|i| rest.get(i + 1).copied().filter(|r| !r.starts_with("--")));
let reason = match reason {
None => None,
Some(None) => return Err("--record needs a reason: --record \"why the golden changes\"".into()),
Some(Some(r)) => Some(r.to_string()),
};
let mut o = options(rest, "target/kernel-ladder")?;
// Times are compared with the golden: cases run one at a time unless --jobs says otherwise,
// so they don't share the machine with each other.
if !rest.contains(&"--jobs") {
o.jobs = 1;
}
let manifest = read(&manifest_path());
if manifest.is_null() {
return Err(format!("{}: missing or not JSON", manifest_path().display()));
}
let tiers = tiers(&manifest)?;
check_listed(&tiers)?;
let base = root().join("tests/kernel-cases");
let list: Vec<PathBuf> = tiers.iter().flat_map(|t| t.cases.iter().map(|c| base.join(c))).collect();
let all = run_all(&o, &list);
save_results(&o, &all)?;
let results: BTreeMap<String, Value> = all.into_iter().map(|r| (rel_of(&r), r)).collect();
let golden = read(&golden_path());
if let Some(reason) = &reason {
record(reason, &manifest, &tiers, &results, &golden)?;
let tiers = self::tiers(&read(&manifest_path()))?;
println!("{}", write_report(&tiers, &results, &read(&golden_path()))?);
} else {
println!("{}", write_report(&tiers, &results, &golden)?);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn flags_slowdowns_status_and_count_changes_only() {
let golden = json!({"status": "pass", "seconds": 1.0, "measured": {"faces": 6, "edges": 12}});
let same = json!({"status": "pass", "seconds": 1.2, "measured": {"faces": 6, "edges": 12}});
assert!(flags(&same, Some(&golden)).is_empty());
// Twice as slow but within timer noise of a fast case.
let fast = json!({"status": "pass", "seconds": 0.03, "measured": {}});
assert!(flags(&fast, Some(&json!({"status": "pass", "seconds": 0.01}))).is_empty());
let worse = json!({"status": "fail", "seconds": 3.0, "measured": {"faces": 7, "edges": 12}});
let f = flags(&worse, Some(&golden));
assert_eq!(f.len(), 3, "{f:?}");
assert_eq!(flags(&same, None), ["no golden"]);
}
#[test]
fn tiers_need_paths() {
let m = json!({"tiers": [{"tier": "01-a", "what": "w", "cases": ["ladder/01-a/001-x"]}]});
let t = tiers(&m).unwrap();
assert_eq!((t[0].tier.as_str(), t[0].cases.len()), ("01-a", 1));
assert!(tiers(&json!({"tiers": [{"tier": "01-a", "cases": [3]}]})).is_err());
}
}