From b2525ed34ebdd790a091292ea4603fb00cea56bc Mon Sep 17 00:00:00 2001 From: Homero Thompson Date: Mon, 26 Oct 2026 11:14:11 +0000 Subject: [PATCH] Tests: the six timing assertions do not depend on the wall clock (#6917) (#6925) * claim: #6917 * Tests: the six timing assertions do not depend on the wall clock (#6917) Six tests asserted on the wall clock. Their result depended on how loaded the machine was and not only on the code under test: they passed on an idle machine and failed when the test binaries ran beside each other, which is how the suite runs (`cargo xtask test` starts every test binary at once). CONTRIBUTING.md:196 says it: "Tests run beside each other on a busy machine, so a test must not depend on wall-clock timing." The change is confined to the six test files the issue names: - geom: the tangent chain of a helix is a ratio (750 pieces against 3 000) instead of a limit of 200 ms. - io: the placement budget is spent inside the reader, so the clock is gone. - ui-egui: the classification of many faces is a ratio (900 against 3 600) instead of a limit of 10 s. - engine: the clearance measurement drops its 30 s limit; the boolean that gives up reports itself. - engine: the fillet beside a modelled thread is a ratio (a 1 mm thread against an 8 mm one) instead of a limit of 30 s. - ui-egui: the window frame budget goes from 250 ms to 20 s, the limit #6072 gave its own probe. The helper is the one #649 wrote, copied into the crates that need it, with the noise of a run as a parameter. --- crates/engine/src/cmd/clearance_tests.rs | 7 +- crates/engine/src/thread_tests.rs | 84 +++++++++++++++++++++--- crates/geom/src/tests.rs | 78 +++++++++++++++++++--- crates/io/src/threemf.rs | 7 +- crates/ui-egui/src/file_load_ui_tests.rs | 11 +++- crates/ui-egui/src/preview.rs | 76 ++++++++++++++++++--- 6 files changed, 230 insertions(+), 33 deletions(-) diff --git a/crates/engine/src/cmd/clearance_tests.rs b/crates/engine/src/cmd/clearance_tests.rs index 5cfa44b66..026b094f3 100644 --- a/crates/engine/src/cmd/clearance_tests.rs +++ b/crates/engine/src/cmd/clearance_tests.rs @@ -350,6 +350,11 @@ fn a_body_that_changed_shape_is_measured_again() { /// ring under it. With a fillet under the head against the bore's sharp edge, the fillet's ring /// is in the flange (about 0.49 mm³), next to walls that lie on one another: the boolean tried /// for minutes there and gave up; now it is reported as such within seconds. +/// +/// The clock is not read: what is under test is the result, and the boolean that gives up is +/// reported as such instead of hanging the test. The wall that lies on the other one is what +/// makes the pair unchecked, and that is what the assertions below check (#2999 dropped the clock +/// from the sibling tests the same way). #[test] fn a_pin_seated_line_to_line_in_its_bushing_is_measured() { let ring = std::f64::consts::PI * (7.0f64.powi(2) - 4.0f64.powi(2)) * 0.2; @@ -365,9 +370,7 @@ fn a_pin_seated_line_to_line_in_its_bushing_is_measured() { run(&mut s, "solid.fillet", json!({"edges": [[104, 0, z]], "radius": 0.3, "body": "Pin"})); run(&mut s, "solid.fillet", json!({"edges": [[107, 0, z + 1.5]], "radius": 0.5, "body": "Pin"})); } - let started = std::time::Instant::now(); let r = run(&mut s, "inspect.clearance", json!({"bodies": ["Pin"], "against": ["Bush"], "min_volume": 0.01, "clearance": 1e-6})); - assert!(started.elapsed().as_secs_f64() < 30.0, "sunk {sunk}: {:?}", started.elapsed()); if fillet { assert!(r["unchecked"][0]["reason"].as_str().is_some_and(|w| w.contains("pin on one cylinder")), "{r}"); continue; diff --git a/crates/engine/src/thread_tests.rs b/crates/engine/src/thread_tests.rs index 3bc55c4e0..87f587468 100644 --- a/crates/engine/src/thread_tests.rs +++ b/crates/engine/src/thread_tests.rs @@ -479,11 +479,11 @@ fn hex_nut(hole: f64) -> Session { s } -/// Issue 602: rounding the edges of a hex boss on a threaded shaft froze the app. The thread's -/// thousands of facets were healed pair by pair, once per feature after it, and the fillet's -/// preview then compared each of them with the whole old body. -#[test] -fn fillet_beside_a_modelled_thread_finishes() { +/// A screw with a modeled M5 thread of `length` millimetres: the number of faces the thread left +/// on it, and how long the fillet of the boss's top edges took. The fillet is the work under test: +/// it healed the thread's facets and compared each of them with the whole old body, which froze +/// the app for minutes (#602). +fn fillet_beside_a_thread(length: f64) -> (usize, std::time::Duration) { let mut s = Session::default(); for (id, p) in [ ("sketch.create", json!({"plane": "XY"})), @@ -494,12 +494,11 @@ fn fillet_beside_a_modelled_thread_finishes() { ("sketch.polygon.circumscribed", json!({"center": [0, 0], "radius": 2, "sides": 6})), ("sketch.finish", json!({})), ("solid.extrude", json!({"distance": 3, "profiles": [{"point": [0, 0]}], "operation": "join", "targets": ["Screw"]})), - ("solid.thread", json!({"face": [2.5, 0, 5], "designation": "M5", "length": 4, "modeled": true})), + ("solid.thread", json!({"face": [2.5, 0, 5], "designation": "M5", "length": length, "modeled": true})), ] { run(&mut s, id, p); } - let faces_before = s.model.state().body("Screw").map(|b| b.body.face_count()).unwrap_or(0); - assert!(faces_before > 1000, "the thread is faceted: {faces_before} faces"); + let faces = s.model.state().body("Screw").map(|b| b.body.face_count()).unwrap_or(0); let volume_before = volume(&mut s, "Screw"); let top: Vec = (0..6) .map(|i| { @@ -509,7 +508,74 @@ fn fillet_beside_a_modelled_thread_finishes() { .collect(); let started = std::time::Instant::now(); run(&mut s, "solid.fillet", json!({"edges": top, "radius": 0.3})); - assert!(started.elapsed().as_secs() < 30, "the fillet took {:?}", started.elapsed()); + let took = started.elapsed(); let volume_after = volume(&mut s, "Screw"); assert!(volume_after < volume_before && volume_after > volume_before * 0.99, "{volume_before} -> {volume_after}"); + (faces, took) +} + +/// Whether a job several times the size took at most about linear: `pair` times the small job, +/// then the big one. The quadratic that the bug cost grows with the square of the work, so eight +/// times the small time is the limit. The load of the other test binaries can rise between the two +/// runs, so a pair that misses is measured again, up to three times. Load only ever slows a run, +/// so each size keeps its fastest time and the ratio is taken of those: a retry cannot pass a +/// regression on a small run that a spike happened to slow, once an earlier small run was fast. +/// +/// `slack` is the room the noise of a busy machine gets, on top of the ratio. +fn scales_linearly(mut pair: impl FnMut() -> (std::time::Duration, std::time::Duration), slack: std::time::Duration) -> Result<(), String> { + let limit = |small: f64| 8.0 * small + slack.as_secs_f64(); + let (mut small, mut big) = (f64::INFINITY, f64::INFINITY); + let mut runs = Vec::new(); + for _ in 0..3 { + let (s, b) = pair(); + small = small.min(s.as_secs_f64()); + big = big.min(b.as_secs_f64()); + runs.push(format!("{s:?} then {b:?}")); + if big < limit(small) { + return Ok(()); + } + } + Err(format!("{}; fastest {small:.3} s and {big:.3} s, limit {:.3} s", runs.join(", "), limit(small))) +} + +#[test] +fn scales_linearly_keeps_each_sizes_fastest_run() { + let pairs = |p: Vec<(u64, u64)>| { + let mut p = p.into_iter().map(|(s, b)| (std::time::Duration::from_millis(s), std::time::Duration::from_millis(b))); + move || p.next().unwrap_or((std::time::Duration::from_secs(1), std::time::Duration::from_secs(1))) + }; + let slack = std::time::Duration::from_millis(0); + // Linear: four times the work, four times the time. + assert!(scales_linearly(pairs(vec![(5, 20)]), slack).is_ok()); + // A big run slowed by load, then a quiet pair. + assert!(scales_linearly(pairs(vec![(5, 80), (5, 20)]), slack).is_ok()); + // A quadratic cost (16x) whose later small runs a spike slowed: the fast first one still counts. + let r = scales_linearly(pairs(vec![(5, 80), (15, 80), (15, 80)]), slack); + assert!(r.as_ref().is_err_and(|e| e.contains("fastest 0.005 s and 0.080 s, limit 0.040 s")), "{r:?}"); +} + +/// Issue 602: rounding the edges of a hex boss on a threaded shaft froze the app. The thread's +/// thousands of facets were healed pair by pair, once per feature after it, and the fillet's +/// preview then compared each of them with the whole old body. +/// +/// A ratio and not a limit: a test must not depend on wall-clock timing (CONTRIBUTING.md), the +/// machine is busy. The longer thread leaves about five times the facets, and the fillet healed +/// them in a time that grew with their square: linear is about 5x, the quadratic 25x or more. +#[test] +fn fillet_beside_a_modelled_thread_finishes() { + /// The room the noise of a run on a busy machine gets, on top of the ratio. + const SLACK: std::time::Duration = std::time::Duration::from_millis(500); + let mut faces = 0; + if let Err(missed) = scales_linearly( + || { + let (_, small) = fillet_beside_a_thread(1.0); + let (facets, big) = fillet_beside_a_thread(8.0); + faces = facets; + (small, big) + }, + SLACK, + ) { + panic!("the fillet beside the thread does not scale linearly: {missed}"); + } + assert!(faces > 1000, "the thread is faceted: {faces} faces"); } diff --git a/crates/geom/src/tests.rs b/crates/geom/src/tests.rs index ff9b74368..ab614f0e4 100644 --- a/crates/geom/src/tests.rs +++ b/crates/geom/src/tests.rs @@ -168,13 +168,8 @@ fn tangent_chains_follow_smooth_edges() { assert_eq!(seam.tangent_chain(0, 5f64.to_radians()), vec![0]); } -/// #3745: the tangent chain of a long helix (a thread's crest, in many edge pieces) is found -/// without scanning every edge for every edge of it: hovering a threaded part's edge looked for -/// its rim this way, and took seconds inside the frame. Every piece joins, in edge order, and -/// a mesh of 3 000 pieces takes well under a frame. -#[test] -fn a_long_helical_tangent_chain_is_found_quickly() { - let pieces = 3_000usize; +/// A helix of `pieces` edge pieces (a thread's crest), each five points long. +fn helix(pieces: usize) -> Mesh { let at = |t: f64| Vec3::new(4.0 * t.cos(), 4.0 * t.sin(), 0.2 * t); let step = 0.05; let mut m = Mesh::default(); @@ -185,13 +180,78 @@ fn a_long_helical_tangent_chain_is_found_quickly() { } // The mesh's points (its size sets the join tolerance). m.positions = m.edges.iter().flatten().copied().collect(); + m +} + +/// The tangent chain of a helix of `pieces` pieces: every piece of it joins, and how long finding +/// the chain took. The mesh is built before the clock starts. +fn helix_chain(pieces: usize) -> std::time::Duration { + let m = helix(pieces); let began = std::time::Instant::now(); let chain = m.tangent_chain(pieces / 2, 2f64.to_radians()); - let ms = began.elapsed().as_secs_f64() * 1000.0; + let took = began.elapsed(); let mut sorted = chain.clone(); sorted.sort_unstable(); assert_eq!(sorted, (0..pieces).collect::>(), "every piece of the helix is in the chain"); - assert!(ms < 200.0, "{ms:.1} ms for {pieces} pieces"); + took +} + +/// Whether a job four times the size took at most about twice linear: `pair` times the small job, +/// then the big one. Linear is 4x, the quadratic that the bug cost 16x or more, so eight times the +/// small time is the limit. The load of the other test binaries can rise between the two runs, so a +/// pair that misses is measured again, up to three times. Load only ever slows a run, so each size +/// keeps its fastest time and the ratio is taken of those: a retry cannot pass a regression on a +/// small run that a spike happened to slow, once an earlier small run was fast. +/// +/// `slack` is the room the noise of a busy machine gets, on top of the ratio. +fn scales_linearly(mut pair: impl FnMut() -> (std::time::Duration, std::time::Duration), slack: std::time::Duration) -> Result<(), String> { + let limit = |small: f64| 8.0 * small + slack.as_secs_f64(); + let (mut small, mut big) = (f64::INFINITY, f64::INFINITY); + let mut runs = Vec::new(); + for _ in 0..3 { + let (s, b) = pair(); + small = small.min(s.as_secs_f64()); + big = big.min(b.as_secs_f64()); + runs.push(format!("{s:?} then {b:?}")); + if big < limit(small) { + return Ok(()); + } + } + Err(format!("{}; fastest {small:.3} s and {big:.3} s, limit {:.3} s", runs.join(", "), limit(small))) +} + +#[test] +fn scales_linearly_keeps_each_sizes_fastest_run() { + let pairs = |p: Vec<(u64, u64)>| { + let mut p = p.into_iter().map(|(s, b)| (std::time::Duration::from_millis(s), std::time::Duration::from_millis(b))); + move || p.next().unwrap_or((std::time::Duration::from_secs(1), std::time::Duration::from_secs(1))) + }; + let slack = std::time::Duration::from_millis(0); + // Linear: four times the work, four times the time. + assert!(scales_linearly(pairs(vec![(5, 20)]), slack).is_ok()); + // A big run slowed by load, then a quiet pair. + assert!(scales_linearly(pairs(vec![(5, 80), (5, 20)]), slack).is_ok()); + // A quadratic cost (16x) whose later small runs a spike slowed: the fast first one still counts. + let r = scales_linearly(pairs(vec![(5, 80), (15, 80), (15, 80)]), slack); + assert!(r.as_ref().is_err_and(|e| e.contains("fastest 0.005 s and 0.080 s, limit 0.040 s")), "{r:?}"); +} + +/// #3745: the tangent chain of a long helix (a thread's crest, in many edge pieces) is found +/// without scanning every edge for every edge of it: hovering a threaded part's edge looked for +/// its rim this way, and took seconds inside the frame. Every piece joins, in edge order. +/// +/// A ratio and not a limit: a test must not depend on wall-clock timing (CONTRIBUTING.md), the +/// machine is busy. Four times the pieces must not cost sixteen times the time. +#[test] +fn a_long_helical_tangent_chain_is_found_quickly() { + /// The room the noise of a run on a busy machine gets, on top of the ratio: the chain of 750 + /// pieces takes a fraction of a second, so this is the noise of one run and not a change of + /// scale. + const SLACK: std::time::Duration = std::time::Duration::from_millis(20); + let pieces = 3_000usize; + if let Err(missed) = scales_linearly(|| (helix_chain(pieces / 4), helix_chain(pieces)), SLACK) { + panic!("{pieces} pieces took {missed}"); + } } #[test] diff --git a/crates/io/src/threemf.rs b/crates/io/src/threemf.rs index ac4f47de2..65c703a2c 100644 --- a/crates/io/src/threemf.rs +++ b/crates/io/src/threemf.rs @@ -1331,9 +1331,13 @@ pub(crate) mod tests { /// Placements in every part draw on one budget (T4): three parts fanning out 300 ways each /// (27 million placements of an empty object) stop at it, and only the build item that went /// over fails. + /// + /// The clock is not read: the budget is spent inside the reader, so the reader stops on its + /// own and the test says what it wants to know, not how long it took. The one object that + /// survives and the warning that names the build item that went over are what show the budget + /// works (#2999 dropped the clock from the sibling tests the same way). #[test] fn parts_share_the_placement_budget() { - let t = std::time::Instant::now(); let fan = |id: u32, path: &str| assembly(id, &vec![(path, 1); 300]); let root = root_model(&fan(10, "/3D/Objects/a.model"), r#""#); let a = model_part("millimeter", &format!("{}{}", fan(1, "/3D/Objects/b.model"), tet_object(9, "Tet", 1.0))); @@ -1344,7 +1348,6 @@ pub(crate) mod tests { let (objs, warnings) = read_3mf(&package).unwrap(); assert_eq!(objs.len(), 1); assert!(warnings.len() == 1 && warnings[0].starts_with("build item 2") && warnings[0].contains("placements"), "{warnings:?}"); - assert!(t.elapsed() < std::time::Duration::from_secs(30), "took {:?}", t.elapsed()); } /// A loop through three parts with different object ids is found, and a chain of parts deeper diff --git a/crates/ui-egui/src/file_load_ui_tests.rs b/crates/ui-egui/src/file_load_ui_tests.rs index f7d4a2aca..c14b098e7 100644 --- a/crates/ui-egui/src/file_load_ui_tests.rs +++ b/crates/ui-egui/src/file_load_ui_tests.rs @@ -516,10 +516,17 @@ fn open_timed(path: &str) -> (Win, (f64, &'static str)) { /// The window stays responsive through every phase of opening a design: a STEP assembly imported, /// the design saved with its model and opened again, and saved without it and opened again. No -/// frame on the window thread takes longer than 250 ms (the test names the phase that did). +/// frame on the window thread takes longer than the budget (the test names the phase that did). #[test] fn opening_a_large_design_never_stalls_the_window() { - const BUDGET_MS: f64 = 250.0; + /// The frame budget: a frame that draws the design takes a fraction of this on an idle machine, + /// but a whole test run on a loaded one has every other binary of the suite asking for the CPU + /// at the same time, and the window thread is descheduled with the rest of them, so the same + /// frame then takes seconds. A window that works answers at once, so raising the budget does + /// not make the test take longer. The freeze this test watches for lasted minutes, not + /// seconds: a window stuck for a minute is still caught here. This is the limit #6072 gave the + /// agent probe, which the same contention kept from starting within three seconds. + const BUDGET_MS: f64 = 20_000.0; let dir = scratch("watchdog"); let step = step_assembly(&dir); let (mut w, (ms, phase)) = open_timed(&step); diff --git a/crates/ui-egui/src/preview.rs b/crates/ui-egui/src/preview.rs index b50020c74..c2b6ba059 100644 --- a/crates/ui-egui/src/preview.rs +++ b/crates/ui-egui/src/preview.rs @@ -1191,17 +1191,75 @@ mod tests { assert!(SurfaceIndex::new(&Mesh::default()).is_none()); } - /// Thousands of faces (a modeled thread's) are classified at once, not face by face against - /// every triangle: filleting beside a thread froze the app for minutes. - #[test] - fn many_faces_are_classified_quickly() { - let started = std::time::Instant::now(); - let n = 120; + /// How long the faces of a sheet of `n × n` squares take to classify against an old sheet of + /// the same size. The meshes are built before the clock starts and the faces are collected + /// before it stops: only the classification, the work that used to freeze the app, is measured. + fn classify_sheet(n: usize) -> std::time::Duration { let (old, same, lifted) = (sheet(n, 0.0), sheet(n, 0.0), sheet(n, 5.0)); let nf = n * n; - assert!(new_faces(&same, nf, &old).iter().all(|new| !new), "faces on the old surface are not new"); - assert!(new_faces(&lifted, nf, &old).iter().all(|new| *new), "faces off the old surface are new"); - assert!(started.elapsed().as_secs() < 10, "took {:?}", started.elapsed()); + let started = std::time::Instant::now(); + let on = new_faces(&same, nf, &old); + let off = new_faces(&lifted, nf, &old); + let took = started.elapsed(); + assert!(on.iter().all(|new| !new), "faces on the old surface are not new"); + assert!(off.iter().all(|new| *new), "faces off the old surface are new"); + took + } + + /// Whether a job four times the size took at most about twice linear: `pair` times the small job, + /// then the big one. Linear is 4x, the quadratic that the bug cost 16x or more, so eight times the + /// small time is the limit. The load of the other test binaries can rise between the two runs, so a + /// pair that misses is measured again, up to three times. Load only ever slows a run, so each size + /// keeps its fastest time and the ratio is taken of those: a retry cannot pass a regression on a + /// small run that a spike happened to slow, once an earlier small run was fast. + /// + /// `slack` is the room the noise of a busy machine gets, on top of the ratio. + fn scales_linearly(mut pair: impl FnMut() -> (std::time::Duration, std::time::Duration), slack: std::time::Duration) -> Result<(), String> { + let limit = |small: f64| 8.0 * small + slack.as_secs_f64(); + let (mut small, mut big) = (f64::INFINITY, f64::INFINITY); + let mut runs = Vec::new(); + for _ in 0..3 { + let (s, b) = pair(); + small = small.min(s.as_secs_f64()); + big = big.min(b.as_secs_f64()); + runs.push(format!("{s:?} then {b:?}")); + if big < limit(small) { + return Ok(()); + } + } + Err(format!("{}; fastest {small:.3} s and {big:.3} s, limit {:.3} s", runs.join(", "), limit(small))) + } + + #[test] + fn scales_linearly_keeps_each_sizes_fastest_run() { + let pairs = |p: Vec<(u64, u64)>| { + let mut p = p.into_iter().map(|(s, b)| (std::time::Duration::from_millis(s), std::time::Duration::from_millis(b))); + move || p.next().unwrap_or((std::time::Duration::from_secs(1), std::time::Duration::from_secs(1))) + }; + let slack = std::time::Duration::from_millis(0); + // Linear: four times the work, four times the time. + assert!(scales_linearly(pairs(vec![(5, 20)]), slack).is_ok()); + // A big run slowed by load, then a quiet pair. + assert!(scales_linearly(pairs(vec![(5, 80), (5, 20)]), slack).is_ok()); + // A quadratic cost (16x) whose later small runs a spike slowed: the fast first one still counts. + let r = scales_linearly(pairs(vec![(5, 80), (15, 80), (15, 80)]), slack); + assert!(r.as_ref().is_err_and(|e| e.contains("fastest 0.005 s and 0.080 s, limit 0.040 s")), "{r:?}"); + } + + /// Thousands of faces (a modeled thread's) are classified at once, not face by face against + /// every triangle: filleting beside a thread froze the app for minutes. + /// + /// A ratio and not a limit: a test must not depend on wall-clock timing (CONTRIBUTING.md), the + /// machine is busy. 900 faces are classified, then 3 600, four times as many: linear is 4x, the + /// face-against-triangle cost 16x or more. + #[test] + fn many_faces_are_classified_quickly() { + /// The room the noise of a run on a busy machine gets, on top of the ratio: the 900 faces + /// take a fraction of a second, so this is the noise of one run and not a change of scale. + const SLACK: std::time::Duration = std::time::Duration::from_millis(20); + if let Err(missed) = scales_linearly(|| (classify_sheet(30), classify_sheet(60)), SLACK) { + panic!("the faces do not scale linearly: {missed}"); + } } /// A fillet preview stands in for the body: the round face is new, the rest is kept, and the