//! `cargo xtask external-corpus [--bless "why"] [--only NAME] [--cli PATH]`: nightly import //! checks on real-world STEP files (#1397). //! //! `tests/external-corpus.toml` lists each source archive (URL, SHA-256, licence) and every //! file inside it (path, SHA-256, expected results). The run downloads each archive once into //! the gitignored `corpus/external/`, verifies every hash, imports each file headless with //! `solidcraft-cli` (`file.insert_step` then `inspect.measure`), and checks the body count, //! total volume and area within tolerances and the time and memory budgets. Import results //! should improve over time, so expectations are re-recorded with a reason: //! `--bless "why"` rewrites the expected values in the manifest. A file whose hash no longer //! matches upstream is reported as "source changed" and never counted as our failure. use std::collections::BTreeMap; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant, SystemTime}; use serde::Deserialize; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; const MANIFEST: &str = "tests/external-corpus.toml"; const CACHE_DIR: &str = "corpus/external"; /// Defaults for entries that do not set their own. const DEFAULT_VOLUME_REL_TOL: f64 = 0.001; // the issue: volume within ~0.1 % const DEFAULT_AREA_REL_TOL: f64 = 0.005; // area drifts more than volume as healing improves const DEFAULT_SECONDS_MAX: u64 = 300; const DEFAULT_MEMORY_MB_MAX: u64 = 8192; #[derive(Deserialize)] struct Source { name: String, url: String, /// Only "zip" is supported: an archive is unpacked into the cache under the source name. archive: String, archive_sha256: String, licence: String, #[allow(dead_code)] // shown for the reader of the manifest; the runner does not need it note: Option, } #[derive(Deserialize)] struct Entry { name: String, source: String, path: String, sha256: String, /// Expected results; absent until `--bless` records them. bodies: Option, volume_mm3: Option, area_mm2: Option, volume_rel_tol: Option, area_rel_tol: Option, seconds_max: Option, memory_mb_max: Option, #[allow(dead_code)] // written by --bless so a reviewer can see why the numbers moved blessed: Option, } #[derive(Deserialize)] struct Manifest { source: Vec, file: Vec, } /// What one file's import produced, for the report and for --bless. struct Measured { bodies: i64, volume_mm3: f64, area_mm2: f64, seconds: f64, memory_mb: Option, } pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> { let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied(); let bless_reason = match rest.iter().position(|a| *a == "--bless") { Some(_) => Some(arg("--bless").ok_or("--bless needs a reason: --bless \"why the numbers moved\"")?.to_string()), None => None, }; let only = arg("--only"); let cli: Vec = match arg("--cli") { Some(path) => vec![path.to_string()], None => { let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); vec![cargo, "run".into(), "-q".into(), "--profile".into(), "ci".into(), "-p".into(), "solidcraft-cli".into(), "--".into()] } }; let manifest_path = root.join(MANIFEST); let text = std::fs::read_to_string(&manifest_path).map_err(|e| format!("{}: {e}", manifest_path.display()))?; let manifest: Manifest = toml::from_str(&text).map_err(|e| format!("{}: {e}", manifest_path.display()))?; let sources: BTreeMap<&str, &Source> = manifest.source.iter().map(|s| (s.name.as_str(), s)).collect(); let cache = root.join(CACHE_DIR); std::fs::create_dir_all(&cache).map_err(|e| format!("{}: {e}", cache.display()))?; let mut pass = 0usize; let mut recorded = 0usize; let mut source_changed = Vec::new(); let mut failed: Vec = Vec::new(); let mut measured: BTreeMap = BTreeMap::new(); for entry in &manifest.file { if only.is_some_and(|o| o != entry.name) { continue; } let source = sources.get(&entry.source.as_str()).ok_or_else(|| format!("{}: unknown source '{}'", entry.name, entry.source))?; let unpacked = ensure_source(&cache, source)?; let file = unpacked.join(&entry.path); let actual = sha256_file(&file)?; if !actual.eq_ignore_ascii_case(&entry.sha256) { source_changed.push(format!("{} (upstream hash is now {actual})", entry.name)); continue; } match import_and_measure(&cli, &cache, entry, &file) { Ok(m) => { measured.insert(entry.name.clone(), m); let m = &measured[&entry.name]; if entry.volume_mm3.is_none() { recorded += 1; println!( " record {:<27} {:>14.3} mm3, {:>14.3} mm2, {} bodies, {:.1} s (no expected values yet)", entry.name, m.volume_mm3, m.area_mm2, m.bodies, m.seconds ); } else { let problems = check(entry, m); if problems.is_empty() { pass += 1; println!( " pass {:<27} {:>14.3} mm3, {:>14.3} mm2, {} bodies, {:.1} s{}", entry.name, m.volume_mm3, m.area_mm2, m.bodies, m.seconds, m.memory_mb.map_or(String::new(), |mb| format!(", {mb:.0} MB")) ); } else { for p in problems { failed.push(format!("{}: {p}", entry.name)); } } } } Err(e) => failed.push(format!("{}: {e}", entry.name)), } } if let Some(reason) = &bless_reason { let stamp = bless_stamp(reason); let rewritten = rewrite_expected(&text, &measured, &stamp); std::fs::write(&manifest_path, rewritten).map_err(|e| format!("{}: {e}", manifest_path.display()))?; println!("external-corpus: re-recorded {} entries ({})", measured.len(), reason); } println!("external-corpus: {pass} pass, {recorded} recorded, {} failed, {} source changed", failed.len(), source_changed.len()); for f in &failed { println!(" FAIL {f}"); } for s in &source_changed { println!(" SOURCE CHANGED {s} — re-download, re-hash and re-record the entry; this is not our failure"); } if !failed.is_empty() { return Err(format!("external-corpus: {} file(s) failed", failed.len())); } Ok(()) } /// Download and unpack one source into the cache unless it is already there at the pinned /// archive hash. Data only: nothing from the archive is ever executed. fn ensure_source(cache: &Path, source: &Source) -> Result { if source.archive != "zip" { return Err(format!("{}: archive '{}' is not supported (only \"zip\")", source.name, source.archive)); } let dir = cache.join(&source.name); let archive = cache.join(format!("{}.download", source.name)); let marker = dir.join(".fetched"); if marker.is_file() && std::fs::read_to_string(&marker).unwrap_or_default().trim() == source.archive_sha256 { return Ok(dir); } std::fs::create_dir_all(cache).map_err(|e| format!("{}: {e}", cache.display()))?; println!("external-corpus: downloading {} ({})", source.url, source.licence); let status = Command::new("curl").args(["-fsSL", "--retry", "3", "-o"]).arg(&archive).arg(&source.url).status().map_err(|e| format!("curl: {e}"))?; if !status.success() { return Err(format!("{}: download failed with {status}", source.name)); } let actual = sha256_file(&archive)?; if !actual.eq_ignore_ascii_case(&source.archive_sha256) { let _ = std::fs::remove_file(&archive); return Err(format!("{}: archive hash is {actual}, manifest says {}", source.name, source.archive_sha256)); } // Unpack fresh: a stale tree must not survive a re-pin. if dir.exists() { std::fs::remove_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; } std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; unpack_zip(&archive, &dir)?; let _ = std::fs::remove_file(&archive); std::fs::write(&marker, &source.archive_sha256).map_err(|e| format!("{}: {e}", marker.display()))?; Ok(dir) } /// Unpack a zip, keeping only regular files and refusing entries that would escape the cache /// (absolute paths or `..` components). fn unpack_zip(archive: &Path, dir: &Path) -> Result<(), String> { let f = std::fs::File::open(archive).map_err(|e| format!("{}: {e}", archive.display()))?; let mut zip = zip::ZipArchive::new(f).map_err(|e| format!("{}: {e}", archive.display()))?; for i in 0..zip.len() { let mut item = zip.by_index(i).map_err(|e| format!("entry {i}: {e}"))?; // zip 9 makes the raw name a Result (it may not decode) and `enclosed_name` is None // when the entry would escape the archive root. let raw = item.name().map(|n| n.to_string()).unwrap_or_else(|_| format!("entry {i}")); let Some(rel) = item.enclosed_name() else { return Err(format!("{}: entry {raw} escapes the archive root", archive.display())); }; if !item.is_file() { continue; } let out = dir.join(&rel); if let Some(parent) = out.parent() { std::fs::create_dir_all(parent).map_err(|e| format!("{}: {e}", parent.display()))?; } let mut dest = std::fs::File::create(&out).map_err(|e| format!("{}: {e}", out.display()))?; std::io::copy(&mut item, &mut dest).map_err(|e| format!("{}: {e}", out.display()))?; } Ok(()) } fn sha256_file(path: &Path) -> Result { let mut f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; let mut hasher = Sha256::new(); let mut buf = vec![0u8; 1 << 20]; loop { let n = f.read(&mut buf).map_err(|e| format!("{}: {e}", path.display()))?; if n == 0 { break; } hasher.update(&buf[..n]); } Ok(hasher.finalize().iter().map(|b| format!("{b:02x}")).collect()) } /// Import the file and measure the result in one headless CLI run; returns what was measured. /// Err means the import itself failed (a crash, a command error, unparsable output). fn import_and_measure(cli: &[String], cache: &Path, entry: &Entry, file: &Path) -> Result { let work = cache.join("_work"); std::fs::create_dir_all(&work).map_err(|e| format!("{}: {e}", work.display()))?; let script = work.join(format!("{}.json", entry.name)); let out = work.join(format!("{}.out.json", entry.name)); let steps = json!([ {"command": "file.insert_step", "params": {"path": file.display().to_string()}}, {"command": "inspect.measure", "params": {}} ]); std::fs::write(&script, serde_json::to_string(&steps).map_err(|e| e.to_string())?).map_err(|e| format!("{}: {e}", script.display()))?; // GNU time, when present, reports the child's peak resident set; without it the memory // budget is not measured (the time budget still is). let gnu_time = gnu_time(); let mem_file = work.join(format!("{}.mem", entry.name)); let mut cmd = match &gnu_time { Some(t) => { let mut c = Command::new(t); c.args(["-f", "%M", "-o"]).arg(&mem_file); c } None => Command::new(&cli[0]), }; if gnu_time.is_some() { cmd.args(&cli[1..]); } let started = Instant::now(); cmd.args(["run"]).arg(&script).args(["--results", "--json-out"]).arg(&out).stdout(Stdio::null()).stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(|e| format!("{}: failed to spawn: {e}", cli.join(" ")))?; let limit = Duration::from_secs(entry.seconds_max.unwrap_or(DEFAULT_SECONDS_MAX)); let status = loop { if let Some(status) = child.try_wait().map_err(|e| e.to_string())? { break status; } if started.elapsed() > limit { let _ = child.kill(); let _ = child.wait(); return Err(format!("import took over {} s (budget) — a hang or a blow-up", limit.as_secs())); } std::thread::sleep(Duration::from_millis(100)); }; let seconds = started.elapsed().as_secs_f64(); if !status.success() { let stderr = child.stderr.take().map(|mut e| { let mut t = String::new(); let _ = e.read_to_string(&mut t); t.trim().chars().take(400).collect::() }); return Err(format!( "import failed ({status}){}", stderr.map(|s| if s.is_empty() { String::new() } else { format!(": {s}") }).unwrap_or_default() )); } let raw = std::fs::read_to_string(&out).map_err(|e| format!("{}: {e}", out.display()))?; let results: Value = serde_json::from_str(&raw).map_err(|e| format!("{}: {e}", out.display()))?; let step = |i: usize| -> Result<&Value, String> { results.get(i).ok_or_else(|| format!("no result {i} in {}", out.display())) }; let inserted = step(0)?; if let Some(err) = inserted.get("error") { return Err(format!("file.insert_step: {err}")); } let measured = step(1)?; if let Some(err) = measured.get("error") { return Err(format!("inspect.measure: {err}")); } let bodies = measured["body_count"].as_i64().ok_or("inspect.measure: no body_count")?; if bodies == 0 { return Err("import produced no bodies".into()); } let volume_mm3 = measured["total"]["volume_mm3"].as_f64().ok_or("inspect.measure: no total volume")?; let area_mm2 = measured["total"]["area_mm2"].as_f64().ok_or("inspect.measure: no total area")?; // A solid measures with positive volume; a non-closed shell measures zero or negative. if volume_mm3 <= 0.0 { return Err(format!("not a closed solid: total volume {volume_mm3} mm3")); } let memory_mb = if gnu_time.is_some() { std::fs::read_to_string(&mem_file).ok().and_then(|t| t.trim().parse::().ok()).map(|kb| kb / 1024.0) } else { None }; Ok(Measured { bodies, volume_mm3, area_mm2, seconds, memory_mb }) } /// The checks against the manifest's expectations; empty means the file passed. fn check(entry: &Entry, m: &Measured) -> Vec { let mut problems = Vec::new(); let Some(want_volume) = entry.volume_mm3 else { return problems; // no expected values yet: recorded, nothing to check }; if let Some(want) = entry.bodies.filter(|want| m.bodies != *want) { problems.push(format!("body count {} is not the expected {want}", m.bodies)); } let rel = (m.volume_mm3 - want_volume).abs() / want_volume.abs(); if rel > entry.volume_rel_tol.unwrap_or(DEFAULT_VOLUME_REL_TOL) { problems.push(format!( "volume {:.3} mm3 is {:+.2} % off the expected {want_volume:.3} (tolerance {:.2} %)", m.volume_mm3, (m.volume_mm3 / want_volume - 1.0) * 100.0, entry.volume_rel_tol.unwrap_or(DEFAULT_VOLUME_REL_TOL) * 100.0 )); } if let Some(want_area) = entry.area_mm2 { let rel = (m.area_mm2 - want_area).abs() / want_area.abs(); if rel > entry.area_rel_tol.unwrap_or(DEFAULT_AREA_REL_TOL) { problems.push(format!( "area {:.3} mm2 is {:+.2} % off the expected {want_area:.3} (tolerance {:.2} %)", m.area_mm2, (m.area_mm2 / want_area - 1.0) * 100.0, entry.area_rel_tol.unwrap_or(DEFAULT_AREA_REL_TOL) * 100.0 )); } } if m.seconds > entry.seconds_max.unwrap_or(DEFAULT_SECONDS_MAX) as f64 { problems.push(format!("import took {:.1} s, over the budget of {} s", m.seconds, entry.seconds_max.unwrap_or(DEFAULT_SECONDS_MAX))); } if let Some(mb) = m.memory_mb { let max = entry.memory_mb_max.unwrap_or(DEFAULT_MEMORY_MB_MAX); if mb > max as f64 { problems.push(format!("peak memory {mb:.0} MB is over the budget of {max} MB")); } } problems } /// GNU time's path if `/usr/bin/time` is GNU time (its `%M` is the peak RSS in kB); None /// elsewhere, which skips the memory budget rather than guessing a platform's units. fn gnu_time() -> Option { let out = Command::new("/usr/bin/time").arg("--version").stderr(Stdio::null()).output().ok()?; let text = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); text.contains("GNU time").then(|| "/usr/bin/time".to_string()) } /// `--bless "why"`: a stamp recorded with each re-recorded entry, so the manifest says when /// and why the numbers last moved. fn bless_stamp(reason: &str) -> String { let days = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_secs() / 86_400).unwrap_or(0); let (y, m, d) = civil_from_days(days as i64); format!("{y:04}-{m:02}-{d:02}: {reason}") } /// Days since the Unix epoch to a civil date (Howard Hinnant's algorithm). fn civil_from_days(z: i64) -> (i64, u32, u32) { let z = z + 719_468; let era = if z >= 0 { z } else { z - 146_096 } / 146_097; let doe = (z - era * 146_097) as u64; let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365; let y = yoe as i64 + era * 400; let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); let mp = (5 * doy + 2) / 153; let d = (doy - (153 * mp + 2) / 5 + 1) as u32; let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; (if m <= 2 { y + 1 } else { y }, m, d) } /// Rewrite the expected-value lines of each blessed entry's `[[file]]` block in place, so /// comments and hand-written keys stay as they are. fn rewrite_expected(text: &str, measured: &BTreeMap, stamp: &str) -> String { let mut lines: Vec = text.lines().map(|l| l.to_string()).collect(); let mut i = 0; while i < lines.len() { if lines[i].trim() != "[[file]]" { i += 1; continue; } let start = i; let mut end = start + 1; while end < lines.len() && !lines[end].trim_start().starts_with("[[") { end += 1; } let name = lines[start + 1..end] .iter() .find_map(|l| l.trim().strip_prefix("name = \"")) .and_then(|r| r.strip_suffix('"')) .unwrap_or_default() .to_string(); let Some(m) = measured.get(&name) else { i = end; continue; }; let want: Vec<(&str, String)> = vec![ ("bodies", m.bodies.to_string()), ("volume_mm3", format!("{:.3}", m.volume_mm3)), ("area_mm2", format!("{:.3}", m.area_mm2)), ("blessed", format!("\"{stamp}\"")), ]; for (key, value) in want { let prefix = format!("{key} = "); match lines[start + 1..end].iter().position(|l| l.trim_start().starts_with(&prefix)) { Some(at) => lines[start + 1 + at] = format!("{key} = {value}"), None => { // Insert after the last expected-value line we own, or after `sha256`. let anchor = lines[start + 1..end] .iter() .rposition(|l| { let t = l.trim_start(); t.starts_with("sha256 = ") || t.starts_with("bodies = ") || t.starts_with("volume_mm3 = ") || t.starts_with("area_mm2 = ") || t.starts_with("blessed = ") }) .unwrap_or(0); lines.insert(start + 1 + anchor + 1, format!("{key} = {value}")); end += 1; } } } i = end; } let mut out = lines.join("\n"); if text.ends_with('\n') { out.push('\n'); } out } #[cfg(test)] mod tests { use super::civil_from_days; #[test] fn epoch_days_to_civil_dates() { assert_eq!(civil_from_days(0), (1970, 1, 1)); assert_eq!(civil_from_days(-1), (1969, 12, 31)); assert_eq!(civil_from_days(11_016), (2000, 2, 29)); // a leap day assert_eq!(civil_from_days(20_751), (2026, 10, 25)); // the day this corpus was first recorded } }