diff --git a/apps/solidcraft-cli/src/kernel_cases.rs b/apps/solidcraft-cli/src/kernel_cases.rs index d07d8ca17..4cf618080 100644 --- a/apps/solidcraft-cli/src/kernel_cases.rs +++ b/apps/solidcraft-cli/src/kernel_cases.rs @@ -633,7 +633,7 @@ fn flat_pattern_checks(c: &mut Checks, want: &Value, results: &[Value], rel: f64 } } -fn check_case(dir: &str, dumps: Option<&std::path::Path>) -> Value { +fn check_case(dir: &str, dumps: Option<&std::path::Path>, validation: solidcraft_engine::kernel::Validation) -> Value { let (area, name) = case_name(dir); let mut out = json!({"case": name, "area": area, "kernel": kernel()}); let want = match read_json(&format!("{dir}/expected.json")) { @@ -692,6 +692,7 @@ fn check_case(dir: &str, dumps: Option<&std::path::Path>) -> Value { } } let mut s = solidcraft_engine::Session::default(); + s.set_validation(validation); let clock = std::time::Instant::now(); let ran = s.run_script(&commands); // The commands' own time, without process start or grading: the ladder's timing golden. @@ -945,7 +946,14 @@ pub fn run(args: &[String]) -> Result<(), String> { if dumps.as_ref().is_some_and(|d| d.as_os_str().is_empty()) { return Err("kernel-cases: --dumps= needs a folder".into()); } - let results: Vec = dirs.iter().map(|d| check_case(d, dumps.as_deref())).collect(); + // `--validation=MODE`: the cases' features are checked under MODE (each_step, each_feature, + // off; default each_feature, the app's default). + let validation = match args.iter().find_map(|a| a.strip_prefix("--validation=")) { + Some(m) => solidcraft_engine::kernel::Validation::parse(m) + .ok_or_else(|| format!("kernel-cases: --validation= must be each_step, each_feature or off, not `{m}`"))?, + None => solidcraft_engine::kernel::Validation::default(), + }; + let results: Vec = dirs.iter().map(|d| check_case(d, dumps.as_deref(), validation)).collect(); if dumps.is_some() { solidcraft_engine::kernel::smlib_dump_dir(None).map_err(|e| format!("kernel-cases: --dumps: {e}"))?; } diff --git a/apps/solidcraft-cli/src/main.rs b/apps/solidcraft-cli/src/main.rs index d9cd5a7bb..190081347 100644 --- a/apps/solidcraft-cli/src/main.rs +++ b/apps/solidcraft-cli/src/main.rs @@ -1,7 +1,7 @@ //! SolidCraft headless command line. //! //! ```text -//! solidcraft-cli run [--in design|part.step|mesh.3mf] [--out FILE]... [--save FILE] [--quiet] [--results] [--timings] [--profile] [--json-out FILE] [--allow-errors] +//! solidcraft-cli run [--in design|part.step|mesh.3mf] [--out FILE]... [--save FILE] [--quiet] [--results] [--timings] [--profile] [--json-out FILE] [--allow-errors] [--validation MODE] //! solidcraft-cli eval [--in design.solidcraft|part.step] [--json-out FILE] (alias: inspect) measurements as JSON //! solidcraft-cli script [--out script.json] the design's history as a command script //! solidcraft-cli bench-open [--parse-only] [--save OUT.solidcraft] [--display LEVEL [--edit NAME EXPR]] [--ops script.json] loading timings as JSON lines @@ -15,7 +15,7 @@ //! solidcraft-cli commands the command registry as JSON //! solidcraft-cli recipe translate an oracle recipe to a script //! solidcraft-cli oracle ... replay recipes and compare with expected.json -//! solidcraft-cli kernel-cases [--json] [--dumps=DIR] ... the kernel regression rig (tests/kernel-cases) +//! solidcraft-cli kernel-cases [--json] [--dumps=DIR] [--validation=MODE] ... the kernel regression rig (tests/kernel-cases) //! solidcraft-cli mcp [--headless] [--in design|script] [--connect HOST:PORT [--token T]] [--allow-path DIR]... [--allow-any-path] //! [--param-check off|warn|strict] //! MCP server on stdio (docs/mcp.md) @@ -59,7 +59,7 @@ struct Command { } /// `run`'s lines in the usage. -const USAGE_RUN: &str = " solidcraft-cli run [--in design.solidcraft|part.step] [--out FILE]... [--save FILE] [--quiet] [--results] [--timings] [--profile] [--profile-out trace.json] [--json-out FILE] [--allow-errors]\n solidcraft-cli run --in [--out FILE]...\n (a timeline feature that fails fails the run and nothing is written; --allow-errors writes anyway)\n (--timings prints {timings: [{index, command, ms}], total_ms} for a script, with --results its results too;\n --profile adds each command's profile: {boolean_ms, tessellate_ms, other_ms}, the kernel's share)\n (--profile also records where the time goes, SMLib's calls, kernel operations, features, commands and import stages,\n and prints the table to stderr: count, total, self time and longest per span; --profile-out FILE writes the spans\n as a Chrome trace for chrome://tracing or ui.perfetto.dev; SOLIDCRAFT_PROFILE=1 does the same for any command)\n"; +const USAGE_RUN: &str = " solidcraft-cli run [--in design.solidcraft|part.step] [--out FILE]... [--save FILE] [--quiet] [--results] [--timings] [--profile] [--profile-out trace.json] [--json-out FILE] [--allow-errors] [--validation each_step|each_feature|off]\n solidcraft-cli run --in [--out FILE]...\n (a timeline feature that fails fails the run and nothing is written; --allow-errors writes anyway)\n (--validation: when SMLib's full check runs on the features computed (diagnostics.validation), default each_feature)\n (--timings prints {timings: [{index, command, ms}], total_ms} for a script, with --results its results too;\n --profile adds each command's profile: {boolean_ms, tessellate_ms, other_ms}, the kernel's share)\n (--profile also records where the time goes, SMLib's calls, kernel operations, features, commands and import stages,\n and prints the table to stderr: count, total, self time and longest per span; --profile-out FILE writes the spans\n as a Chrome trace for chrome://tracing or ui.perfetto.dev; SOLIDCRAFT_PROFILE=1 does the same for any command)\n"; /// `eval`'s lines in the usage. const USAGE_EVAL: &str = " solidcraft-cli eval [--in design.solidcraft|part.step] [--json-out FILE] [--profile] [--profile-out trace.json] (alias: inspect)\n solidcraft-cli eval --in [--json-out FILE]\n (with both inputs, runs the positional script on --in before measuring; does not automatically save the modified design)\n (--json-out FILE writes the JSON that run (with --results, else the measurement) and eval print to FILE instead of stdout,\n as plain UTF-8: a shell's > can re-encode it, e.g. Windows PowerShell adds a byte-order mark)\n (--profile and --profile-out as for run)\n"; /// `script`'s lines in the usage. @@ -83,7 +83,7 @@ const USAGE_RECIPE: &str = " solidcraft-cli recipe \n"; /// `oracle`'s lines in the usage. const USAGE_ORACLE: &str = " solidcraft-cli oracle ... [--json]\n"; /// `kernel-cases`'s lines in the usage. -const USAGE_KERNEL_CASES: &str = " solidcraft-cli kernel-cases ... [--json] [--dumps=DIR]\n (the kernel regression rig, tests/kernel-cases, on SMLib; --dumps: each case's\n failing SMLib calls leave their inputs for replay in DIR/-)\n"; +const USAGE_KERNEL_CASES: &str = " solidcraft-cli kernel-cases ... [--json] [--dumps=DIR] [--validation=each_step|each_feature|off]\n (the kernel regression rig, tests/kernel-cases, on SMLib; --dumps: each case's\n failing SMLib calls leave their inputs for replay in DIR/-; --validation: when SMLib's\n full check runs (diagnostics.validation), default each_feature)\n"; /// `fuzz`'s lines in the usage. const USAGE_FUZZ: &str = " solidcraft-cli fuzz [--from N] [--count N] [--jobs J] [--timeout S] [--steps N] [--minimise] [--out DIR] [--report FILE]\n"; @@ -278,8 +278,15 @@ pub fn load(path: &str) -> Result { /// `load`, with scripts and recipes run at the parameter-check level `check`, which the /// session keeps. fn load_checked(path: &str, check: solidcraft_engine::schema::ParamCheck) -> Result { + load_with(path, check, solidcraft_engine::kernel::Validation::default()) +} + +/// `load_checked`, the features computed under `validation` (`diagnostics.validation`), which +/// the session keeps. +fn load_with(path: &str, check: solidcraft_engine::schema::ParamCheck, validation: solidcraft_engine::kernel::Validation) -> Result { let mut s = Session::default(); s.param_check = check; + s.set_validation(validation); if solidcraft_engine::io::is_step_path(path) || solidcraft_engine::io::is_mesh_path(path) { s.execute("doc.open", &json!({"path": path})).map_err(|e| e.to_string())?; return Ok(s); @@ -287,7 +294,7 @@ fn load_checked(path: &str, check: solidcraft_engine::schema::ParamCheck) -> Res let bytes = solidcraft_engine::io::vfs::read(path, solidcraft_engine::io::MAX_DESIGN_BYTES).map_err(|e| format!("{path}: {e}"))?; if solidcraft_engine::io::is_design_container(&bytes) { let doc = solidcraft_engine::io::read_design(&bytes).map_err(|e| format!("{path}: {e}"))?; - let mut session = Session::opened_at(doc, None, Some(path)); + let mut session = Session::opened_at_validating(doc, None, Some(path), validation); session.param_check = check; return Ok(session); } @@ -377,10 +384,11 @@ fn cmd_run(args: &[String]) -> Result<(), String> { known_args( "run", args, - &["--in", "--out", "--save", "--json-out", "--profile-out"], + &["--in", "--out", "--save", "--json-out", "--profile-out", "--validation"], &["--quiet", "--results", "--timings", "--profile", "--allow-errors"], true, )?; + let validation = validation_flag("run", args)?; let base = flag(args, "--in"); let path = positional(args); if path.is_none() && base.is_none() { @@ -395,8 +403,12 @@ fn cmd_run(args: &[String]) -> Result<(), String> { let (mut s, results) = if want_results || timings { // Collect every command's result or time (scripts only), then process requested files too. let mut s = match base { - Some(b) => load(b)?, - None => Session::default(), + Some(b) => load_with(b, solidcraft_engine::schema::ParamCheck::Off, validation)?, + None => { + let mut s = Session::default(); + s.set_validation(validation); + s + } }; let needs = if want_results { "--results needs a script" } else { "--timings needs a script" }; let v = read_json(path.ok_or(needs)?)?; @@ -411,7 +423,7 @@ fn cmd_run(args: &[String]) -> Result<(), String> { }; (s, Some(out)) } else { - (open(base, path)?, None) + (open_validating(base, path, validation)?, None) }; // A feature that fails leaves the bodies as they were before it (a parameter change can // break a feature without failing the command that made it): writing them would hand out @@ -438,6 +450,12 @@ fn cmd_run(args: &[String]) -> Result<(), String> { /// The session `run`, `eval` and `snapshot` start from: `--in` (a design or STEP file) with the /// positional script run on top of it, or either one alone. fn open(base: Option<&str>, path: Option<&String>) -> Result { + open_validating(base, path, solidcraft_engine::kernel::Validation::default()) +} + +/// [`open`], the features computed under `validation`. +fn open_validating(base: Option<&str>, path: Option<&String>, validation: solidcraft_engine::kernel::Validation) -> Result { + let load = |p: &str| load_with(p, solidcraft_engine::schema::ParamCheck::Off, validation); match (base, path) { (Some(b), Some(p)) => { let mut s = load(b)?; @@ -450,6 +468,15 @@ fn open(base: Option<&str>, path: Option<&String>) -> Result { } } +/// `--validation MODE` (`diagnostics.validation`): the default, each_feature, when not given. +fn validation_flag(cmd: &str, args: &[String]) -> Result { + match flag(args, "--validation") { + Some(m) => solidcraft_engine::kernel::Validation::parse(m) + .ok_or_else(|| format!("{cmd}: --validation must be each_step, each_feature or off, not `{m}`")), + None => Ok(solidcraft_engine::kernel::Validation::default()), + } +} + /// Refuses an option `cmd` doesn't take, and one missing its value: a typo, or another /// command's option, would otherwise be ignored without a word. `positionals` says whether the /// command takes a word of its own (a file, a seed): `fuzz` takes none, and would start a run. diff --git a/tests/kernel-cases/README.md b/tests/kernel-cases/README.md index c1a6f146c..3267772e7 100644 --- a/tests/kernel-cases/README.md +++ b/tests/kernel-cases/README.md @@ -14,6 +14,10 @@ The CLI links SMLib by default (`scripts/fetch-smlib.sh` fetches the prebuilt li built with `--no-default-features`, every operation answers that SMLib is not built. The run's output folder (`--out`, default `target/kernel-cases`) gets `results.json`, every case's full result, and `dumps/-/`: the inputs of each SMLib call a case failed in, for replay (the CLI's `--dumps=DIR`, which the rig always passes). +`--validation MODE` runs the cases' features under that mode of SMLib's check +(`diagnostics.validation`): `each_feature` (the default: the bodies each feature keeps are checked +once), `each_step` (after every sweep, boolean and blend, as before) or `off`; the +`KNOWN_FAILING` lists are each_feature's. ## Rules diff --git a/xtask/src/kernel_cases.rs b/xtask/src/kernel_cases.rs index ce3d2e983..9e94eedec 100644 --- a/xtask/src/kernel_cases.rs +++ b/xtask/src/kernel_cases.rs @@ -32,6 +32,9 @@ pub(crate) struct Options { out: PathBuf, /// `--known-failing FILE`: run as a regression gate against that baseline (see [`gate`]). known_failing: Option, + /// `--validation MODE`: when SMLib's full check runs on the cases' features (each_step, + /// each_feature, off; the CLI's default, each_feature, when not given). + validation: Option, } /// The run's options; `out` is its output folder unless `--out` gives one. @@ -44,7 +47,11 @@ pub(crate) fn options(rest: &[&str], out: &str) -> Result { let p = PathBuf::from(p); if p.is_absolute() { p } else { root().join(p) } }); - Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing }) + let validation = arg("--validation"); + if let Some(v) = validation.as_deref().filter(|v| !["each_step", "each_feature", "off"].contains(v)) { + return Err(format!("--validation must be each_step, each_feature or off, not `{v}`")); + } + Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing, validation }) } /// Every case's full result as JSON, into the run's output folder. @@ -99,7 +106,11 @@ fn run_case(o: &Options, case: &Path) -> Value { }; let mut dumps = std::ffi::OsString::from("--dumps="); dumps.push(o.out.join("dumps")); - cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps).arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped()); + cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps); + if let Some(v) = &o.validation { + cmd.arg(format!("--validation={v}")); + } + cmd.arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped()); let base = |status: &str, error: String| { let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 3d13136d9..d4ae31264 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -47,7 +47,7 @@ commands: replay the reference tests (tests/oracle/*/recipe.json), compare with each case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli already built, instead of cargo run --release) - kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] [--coverage] + kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] [--validation MODE] [--coverage] the kernel regression rig: run every case in tests/kernel-cases/// on SMLib (one process each, killed after --timeout, default 180 s), grade it against the case's own answer and write docs/kernel-cases.md; DIR (default @@ -56,6 +56,9 @@ commands: --known-failing FILE: a regression gate instead of the report: fails on a case that fails and FILE does not list, and on a listed case that passes (tests/kernel-cases//KNOWN_FAILING, CI's Kernel rig workflow) + --validation MODE: when SMLib's full check runs on the cases' features: + each_step, each_feature (the default) or off (solidcraft-cli kernel-cases + --validation=MODE); with --ladder too --ladder [--record REASON | --gate]: the complexity ladder instead (tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared with its timing golden (one case at a time unless --jobs), into