diff --git a/crates/kernel/src/step_in/mod.rs b/crates/kernel/src/step_in/mod.rs index f40e44770..f95bee134 100644 --- a/crates/kernel/src/step_in/mod.rs +++ b/crates/kernel/src/step_in/mod.rs @@ -32,6 +32,8 @@ const MAX_BODIES: usize = 10_000; /// Most product occurrences and mapped representations visited (a shared sub-assembly used /// twice at every level would otherwise grow exponentially). const MAX_VISITS: usize = 100_000; +/// Most planned solids converted in one go (bounds the geometry held at once). +const MAX_WINDOW: usize = 64; /// Most representation entries walked while deciding which items are converted exactly once. A /// larger file keeps every conversion instead of paying for the walk. const MAX_CLOSURE_WORK: usize = 1 << 16; @@ -239,10 +241,6 @@ fn rigid_inverse(m: &mt::Matrix4) -> mt::Matrix4 { m.invert().unwrap_or_else(mt::Matrix4::identity) } -/// One converted representation item: the truck solid, whether its shells are closed, the -/// faces the file had, and the file face of each kernel face (band faces split into two). -type SolidEntry = (mt::Solid, bool, usize, Vec); - struct Reader { ex: Arc, units: HashMap, @@ -261,18 +259,18 @@ struct Reader { colors: HashMap, /// Opacity of styled items with a transparency (1 = opaque). opacity: HashMap, - /// Converted items (brep id → entry, or the error). Held behind `Arc` so the conversion is - /// handed on without copying wherever the file cannot ask for the item a second time. - solids: HashMap, String>>, - /// Items the file converts exactly once (see `index`): keeping their solids would hold a + /// Items that gave no part: reported once when the file could not be read, else (the + /// body could not be made) at every instance, as the message says. + unreadable: HashMap>, + /// Items the file converts exactly once (see `index`): keeping their parts would hold a /// second copy of the whole import for nothing. once: HashSet, /// An item placed more than once: its first instance's display, shared by the later ones /// (with the placement that first instance had), so the part is meshed once. shown: HashMap, mt::Matrix4)>, - /// An item placed more than once: its body oriented where the file defines it, so each + /// An item placed more than once: read and oriented where the file defines it, so each /// later instance is only placed (orienting meshes the whole part). - oriented: HashMap, + oriented: HashMap>, used_items: HashSet, /// Items already instanced for the product being read. node_items: HashSet, @@ -312,7 +310,7 @@ impl Reader { placements: HashMap::new(), colors: HashMap::new(), opacity: HashMap::new(), - solids: HashMap::new(), + unreadable: HashMap::new(), once: HashSet::new(), shown: HashMap::new(), oriented: HashMap::new(), @@ -598,38 +596,6 @@ impl Reader { mapped.map(|rep| self.rep_closure(&[rep])).unwrap_or_default() } - fn item_solid(&mut self, item: u64, u: Units) -> Option> { - if let Some(r) = self.solids.get(&item) { - return r.as_ref().ok().cloned(); - } - let cx = Ctx { ex: &self.ex, len: u.len, ang: u.ang, tol: u.tol }; - let r = match guard("step brep", || brep::read_item(&cx, item).map_err(KernelError::Failed)) { - Ok(out) => { - let label = self.label(item); - for f in &out.failed { - self.warn(format!("{label}: {f}")); - } - if !out.closed { - self.warn(format!("{label}: imported as an open body (the shell is not closed)")); - } - Ok((out.solid, out.closed, out.file_faces, out.face_origin)) - } - Err(e) => { - let label = self.label(item); - let msg = e.to_string().trim_start_matches("the operation failed: ").to_string(); - self.warn(format!("{label}: not imported: {msg}")); - Err(msg) - } - }; - let entry = r.map(Arc::new); - // An item the file converts exactly once hands its only reference on and keeps no copy; - // anything it may ask for again keeps its conversion. - if !self.once.contains(&item) { - self.solids.insert(item, entry.clone()); - } - entry.ok() - } - fn label(&self, item: u64) -> String { let name = self.ex.get(item).and_then(|e| e.params().first()).and_then(Param::as_str).unwrap_or("").trim().to_string(); if name.is_empty() { format!("solid #{item}") } else { format!("solid '{name}'") } @@ -726,43 +692,66 @@ impl Reader { return false; } let first = self.converted; - let id = self.plan[first].batch; + // The next solids, across instances and parts, converted together: parts still to + // orient are shared out between the workers. A part that comes back within the window + // waits for the next one, which places the orientation its first instance found. let mut last = first + 1; - while !single && last < self.plan.len() && self.plan[last].batch == id { - last += 1; + if !single { + let mut fresh: HashSet = self.plan.get(first).map(|p| p.item).into_iter().collect(); + while let Some(p) = self.plan.get(last) + && last - first < MAX_WINDOW + { + if !self.oriented.contains_key(&p.item) && !fresh.insert(p.item) { + break; + } + last += 1; + } } self.converted = last; let ex = self.ex.clone(); let mut sources = Vec::new(); - let mut solids = Vec::new(); + let mut jobs = Vec::new(); for i in first..last { - let (item, u) = (self.plan[i].item, self.plan[i].units); self.made.push(None); - let Some(entry) = self.item_solid(item, u) else { continue }; - // A one-use item is the only reference to its entry, so the solid moves - // out; an item whose conversion is cached for a later instance needs its - // own copy. - let (solid, closed, file_faces, face_origin) = match Arc::try_unwrap(entry) { - Ok(entry) => entry, - Err(shared) => (*shared).clone(), - }; - sources.push((i, item, closed, file_faces, face_origin)); + let Some(p) = self.plan.get(i) else { continue }; + let (item, units) = (p.item, p.units); + if let Some(again) = self.unreadable.get(&item) { + if let Some(msg) = again.clone() { + let label = self.label(item); + self.warn(format!("{label}: not imported: {msg}")); + } + continue; + } let source = match self.oriented.get(&item) { - Some(body) => parallel::Source::Oriented(body.clone()), - None => parallel::Source::Solid(solid), + Some(part) => parallel::Source::Oriented(part.clone()), + None => parallel::Source::Item { item, units }, }; - solids.push(parallel::Job { source, check_mesh: !self.unmeshed_checked.contains(&item) }); + let placement = self.batches.get(p.batch).map_or(mt::Matrix4::from_scale(1.0), |b| b.m); + sources.push((i, item)); + jobs.push(parallel::Job { source, placement, check_mesh: !self.unmeshed_checked.contains(&item) }); } - let Some(batch) = self.batches.get(id) else { return true }; - let (m, path, product, instance) = (batch.m, batch.path.clone(), batch.product.clone(), batch.instance.clone()); - let prepared = parallel::bodies(&solids, &m, self.workers); - for ((i, item, closed, file_faces, face_origin), body) in sources.into_iter().zip(prepared) { + // Reading and orienting new parts is shared out between the workers. + let prepared = parallel::bodies(&ex, &jobs, self.workers); + for ((i, item), made) in sources.into_iter().zip(prepared) { let Some(e) = ex.get(item) else { continue }; - match body { - Ok((body, missing, oriented)) => { - if !self.once.contains(&item) { - self.oriented.entry(item).or_insert(oriented); + let Some(batch) = self.plan.get(i).and_then(|p| self.batches.get(p.batch)) else { continue }; + let (m, path, product, instance) = (batch.m, batch.path.clone(), batch.product.clone(), batch.instance.clone()); + match made { + Ok(parallel::Made { body, missing, part, read }) => { + // What the read of the item found, reported once. + if let Some(failed) = read { + let label = self.label(item); + for f in &failed { + self.warn(format!("{label}: {f}")); + } + if !part.closed { + self.warn(format!("{label}: imported as an open body (the shell is not closed)")); + } } + if !self.once.contains(&item) { + self.oriented.entry(item).or_insert_with(|| part.clone()); + } + let (closed, file_faces, face_origin) = (part.closed, part.file_faces, part.face_origin.clone()); let own = e.params().first().and_then(Param::as_str).unwrap_or("").trim().to_string(); let name = if own.is_empty() { product.clone() } else { own }; // Faces the mesher gives up on (or that the budget cut short) are reported. @@ -811,9 +800,13 @@ impl Reader { face_origin, }); } - Err(err) => { + Err(parallel::Failure { read, msg }) => { let label = self.label(item); - self.warn(format!("{label}: not imported: {err}")); + self.warn(format!("{label}: not imported: {msg}")); + // Later instances don't read the item again. + if !self.oriented.contains_key(&item) { + self.unreadable.insert(item, (!read).then_some(msg)); + } } } } diff --git a/crates/kernel/src/step_in/parallel.rs b/crates/kernel/src/step_in/parallel.rs index 2d9854eb6..c8327ac98 100644 --- a/crates/kernel/src/step_in/parallel.rs +++ b/crates/kernel/src/step_in/parallel.rs @@ -1,21 +1,51 @@ //! Bounded desktop preparation of independent imported bodies, collected in file order. -use super::{Body, Result, guard, mt}; +use std::sync::Arc; + +use super::{Body, Ctx, Exchange, KernelError, Units, brep, guard, mt}; pub(super) struct Job { pub source: Source, + /// Where the body goes (its instance's placement in the assembly). + pub placement: mt::Matrix4, pub check_mesh: bool, } -/// What a job places: a solid read from the file (oriented here), or a body an earlier -/// instance of the same item already oriented. +/// What a job places: a solid item to read from the file (and orient), or a part an earlier +/// instance of the same item already read and oriented. pub(super) enum Source { - Solid(mt::Solid), - Oriented(Body), + Item { item: u64, units: Units }, + Oriented(Arc), } -/// The placed body, its faces the mesher missed, and the body oriented where the file defines -/// it (for later instances of the same item). -type Prepared = Result<(Body, usize, Body)>; +/// A solid item read and oriented where the file defines it, placed by each of its instances. +pub(super) struct Part { + pub body: Body, + /// The shells are closed. + pub closed: bool, + /// Faces the file had. + pub file_faces: usize, + /// The file face of each kernel face. + pub face_origin: Vec, +} + +/// A prepared instance. +pub(super) struct Made { + /// The part, placed. + pub body: Body, + /// Faces the mesher missed (counted when the job asked). + pub missing: usize, + pub part: Arc, + /// This job read the item: the faces the read left out, with the reason. + pub read: Option>, +} + +/// Why a job made no body: the item could not be read (`read`), or its body not made. +pub(super) struct Failure { + pub read: bool, + pub msg: String, +} + +type Prepared = std::result::Result; pub(super) fn workers() -> usize { #[cfg(not(target_arch = "wasm32"))] @@ -28,49 +58,40 @@ pub(super) fn workers() -> usize { } } -fn prepare(job: &Job, placement: &mt::Matrix4) -> Prepared { +fn prepare(ex: &Exchange, job: &Job) -> Prepared { + let (part, read) = match &job.source { + Source::Oriented(part) => (part.clone(), None), + Source::Item { item, units } => { + let cx = Ctx { ex, len: units.len, ang: units.ang, tol: units.tol }; + let out = guard("step brep", || brep::read_item(&cx, *item).map_err(KernelError::Failed)) + .map_err(|e| Failure { read: true, msg: e.to_string().trim_start_matches("the operation failed: ").to_string() })?; + // Orient once where the file defines the solid; each placement then takes its + // orientation from the matrix instead of meshing the whole part again. + let body = guard("STEP body preparation", || Body::new(out.solid)).map_err(|e| Failure { read: false, msg: e.to_string() })?; + (Arc::new(Part { body, closed: out.closed, file_faces: out.file_faces, face_origin: out.face_origin }), Some(out.failed)) + } + }; guard("STEP body preparation", || { - // Orient once where the file defines the solid; each placement then takes its - // orientation from the matrix instead of meshing the whole part again. - let oriented = match &job.source { - Source::Solid(solid) => Body::new(solid.clone())?, - Source::Oriented(body) => body.clone(), - }; - let body = oriented.placed(placement)?; + let body = part.body.placed(&job.placement)?; // Counted where the orientation mesh already counted them (a placement meshes the // same faces). - let missing = if job.check_mesh { oriented.unmeshed_faces() } else { 0 }; - Ok((body, missing, oriented)) + let missing = if job.check_mesh { part.body.unmeshed_faces() } else { 0 }; + Ok((body, missing)) }) + .map(|(body, missing)| Made { body, missing, part, read }) + .map_err(|e| Failure { read: false, msg: e.to_string() }) } -pub(super) fn bodies(solids: &[Job], placement: &mt::Matrix4, workers: usize) -> Vec { - #[cfg(not(target_arch = "wasm32"))] - if solids.len() >= 4 && workers > 1 { - return std::thread::scope(|scope| { - let size = solids.len().div_ceil(workers.min(8)); - let mut pending = Vec::new(); - for chunk in solids.chunks(size) { - let handle = std::thread::Builder::new() - .name("step-body".into()) - .spawn_scoped(scope, move || chunk.iter().map(|solid| prepare(solid, placement)).collect::>()) - .ok(); - pending.push((chunk, handle)); - } - let mut output = Vec::with_capacity(solids.len()); - for (chunk, handle) in pending { - match handle { - Some(handle) => match handle.join() { - Ok(results) => output.extend(results), - Err(_) => output.extend(chunk.iter().map(|_| Err(crate::KernelError::Internal("STEP body worker panicked".into())))), - }, - // Resource exhaustion must not lose an import: use the serial path. - None => output.extend(chunk.iter().map(|solid| prepare(solid, placement))), - } - } - output - }); +/// Prepare `jobs`, the results in the jobs' order. Solids still to read and orient (the slow +/// part) are shared out between up to `workers` threads, each taking the next job when it is +/// done with one, so one large part does not hold up the rest. +pub(super) fn bodies(ex: &Exchange, jobs: &[Job], workers: usize) -> Vec { + let to_orient = jobs.iter().filter(|j| matches!(j.source, Source::Item { .. })).count(); + if to_orient > 1 && workers > 1 { + return crate::par::map_on_threads(jobs, workers.min(8).min(to_orient), |job| prepare(ex, job)) + .into_iter() + .map(|r| r.unwrap_or_else(|| Err(Failure { read: false, msg: "STEP body worker panicked".into() }))) + .collect(); } - let _ = workers; - solids.iter().map(|solid| prepare(solid, placement)).collect() + jobs.iter().map(|job| prepare(ex, job)).collect() } diff --git a/crates/kernel/src/step_in/tests.rs b/crates/kernel/src/step_in/tests.rs index 2f7791915..bba7fbf4b 100644 --- a/crates/kernel/src/step_in/tests.rs +++ b/crates/kernel/src/step_in/tests.rs @@ -240,20 +240,30 @@ fn parallel_import_retains_units_order_sources_and_partial_failure_across_batche #[test] fn parallel_body_preparation_reports_errors_in_place_and_preserves_placement() { let body = cylinder(Vec3::new(2.0, 2.0, 0.0), Vec3::Z, 2.0, 4.0).unwrap(); - let solids = vec![body.solid.as_ref().clone(), mt::Solid::new(Vec::new()), body.solid.as_ref().clone(), body.solid.as_ref().clone()]; + let ex = p21::parse(&step_export(&[&body], "test").unwrap()).unwrap(); + let solid = *ex.entities.iter().find(|(_, e)| e.name() == "MANIFOLD_SOLID_BREP").map(|(id, _)| id).unwrap(); + let units = DEFAULT_UNITS; let placement = mt::Matrix4::from_translation(mt::Vector3::new(10.0, 20.0, 30.0)); - let jobs: Vec<_> = solids.into_iter().map(|solid| parallel::Job { source: parallel::Source::Solid(solid), check_mesh: true }).collect(); - let serial = parallel::bodies(&jobs, &placement, 1); - let threaded = parallel::bodies(&jobs, &placement, 4); + // The second item is not in the file. + let jobs: Vec<_> = [solid, 999_999_999, solid, solid] + .into_iter() + .map(|item| parallel::Job { source: parallel::Source::Item { item, units }, placement, check_mesh: true }) + .collect(); + let serial = parallel::bodies(&ex, &jobs, 1); + let threaded = parallel::bodies(&ex, &jobs, 4); assert_eq!(serial.len(), 4); assert_eq!(threaded.len(), 4); for (index, (a, b)) in serial.iter().zip(&threaded).enumerate() { if index == 1 { - assert_eq!(a.as_ref().unwrap_err(), b.as_ref().unwrap_err()); + let (Err(a), Err(b)) = (a, b) else { panic!("job 1 has no item") }; + assert!(a.read && b.read); + assert_eq!(a.msg, b.msg); } else { - assert_eq!(a.as_ref().unwrap().1, b.as_ref().unwrap().1); - let a = measure(&a.as_ref().unwrap().0).unwrap(); - let b = measure(&b.as_ref().unwrap().0).unwrap(); + let (Ok(a), Ok(b)) = (a, b) else { panic!("job {index} failed") }; + assert_eq!(a.missing, b.missing); + assert!(a.read.is_some() && b.read.is_some(), "each job read its item"); + let a = measure(&a.body).unwrap(); + let b = measure(&b.body).unwrap(); assert_same_measure(&a, &b); // Curved bounds come from the measurement tessellation, not analytic extrema. assert!(a.bbox.min.dist(Vec3::new(10.0, 20.0, 30.0)) < 1e-3); @@ -296,14 +306,14 @@ fn parallel_import_does_not_prepare_past_body_budget_with_duplicates_or_mapped_i reader.node_items.insert(2000); let mut out = Vec::new(); reader.rep_bodies(60, &mt::Matrix4::from_scale(1.0), &[], "Budget", None, &mut out, 0); - assert!(reader.solids.is_empty(), "the walk only plans the solids"); + assert!(reader.oriented.is_empty() && reader.unreadable.is_empty(), "the walk only plans the solids"); while reader.convert_next(workers == 1) {} let out = reader.remap(&out); assert_eq!(reader.bodies.len(), MAX_BODIES); assert_eq!(out, vec![MAX_BODIES - 2, MAX_BODIES - 1]); assert_eq!(reader.bodies.last().unwrap().name, "Second"); - assert_eq!(reader.solids.len(), 2); - assert!(!reader.solids.contains_key(&2003)); + assert_eq!(reader.oriented.len(), 2); + assert!(!reader.oriented.contains_key(&2003) && !reader.unreadable.contains_key(&2003)); assert!(!reader.used_items.contains(&3000)); assert_eq!(reader.warnings.len(), 1); assert!(reader.warnings.keys().next().unwrap().contains("more than")); @@ -656,29 +666,44 @@ fn hand_written_assembly() { /// The cube part (in cm) placed `n` times in a millimetre assembly, each instance moved along X /// and turned about Z. fn instanced_assembly(n: u64) -> String { + instanced_assembly_of(1, n) +} + +/// [`instanced_assembly`] with `parts` distinct parts (the cube solid under different items and +/// products), instance `k` placing part `k % parts`. +fn instanced_assembly_of(parts: u64, n: u64) -> String { let mut t = String::from(HEAD); t += "#8=PRODUCT_DEFINITION_CONTEXT('part definition',#9,'design');\n#9=APPLICATION_CONTEXT('');\n"; t += &cube_entities(); t += &context(20, "(LENGTH_UNIT()NAMED_UNIT(*)SI_UNIT(.CENTI.,.METRE.))"); t += &context(30, "(LENGTH_UNIT()NAMED_UNIT(*)SI_UNIT(.MILLI.,.METRE.))"); - t += &product(50, "Block"); t += &product(70, "Assembly"); - t += "#60=ADVANCED_BREP_SHAPE_REPRESENTATION('',(#101,#65),#20);\n#61=SHAPE_DEFINITION_REPRESENTATION(#54,#60);\n"; - t += "#65=AXIS2_PLACEMENT_3D('',#66,$,$);\n#66=CARTESIAN_POINT('',(0.,0.,0.));\n"; + for p in 0..parts { + let pd = 1000 + p * 100; + t += &product(pd, if p == 0 { "Block".to_string() } else { format!("Block{p}") }.as_str()); + let item = if p == 0 { 101 } else { pd + 20 }; + if p > 0 { + t += &format!("#{item}=MANIFOLD_SOLID_BREP('Cube{p}',#100);\n"); + } + t += &format!("#{}=ADVANCED_BREP_SHAPE_REPRESENTATION('',(#{item},#{}),#20);\n", pd + 10, pd + 15); + t += &format!("#{}=SHAPE_DEFINITION_REPRESENTATION(#{},#{});\n", pd + 11, pd + 4, pd + 10); + t += &format!("#{}=AXIS2_PLACEMENT_3D('',#{},$,$);\n#{}=CARTESIAN_POINT('',(0.,0.,0.));\n", pd + 15, pd + 16, pd + 16); + } let items: Vec = (0..n).map(|k| format!("#{}", 3000 + k * 10)).collect(); t += &format!("#80=SHAPE_REPRESENTATION('',({}),#30);\n#88=SHAPE_DEFINITION_REPRESENTATION(#74,#80);\n", items.join(",")); for k in 0..n { - let (a, turn) = (3000 + k * 10, k as f64 * 0.7); + let (a, turn, pd) = (3000 + k * 10, k as f64 * 0.7, 1000 + (k % parts) * 100); t += &format!("#{a}=AXIS2_PLACEMENT_3D('',#{},#{},#{});\n", a + 1, a + 2, a + 3); t += &format!("#{}=CARTESIAN_POINT('',({}.,0.,0.));\n", a + 1, 50 * k); t += &format!("#{}=DIRECTION('',(0.,0.,1.));\n#{}=DIRECTION('',({},{},0.));\n", a + 2, a + 3, turn.cos(), turn.sin()); let u = 6000 + k * 10; - t += &format!("#{u}=NEXT_ASSEMBLY_USAGE_OCCURRENCE('{k}','Block:{k}','',#70,#50,$);\n"); + t += &format!("#{u}=NEXT_ASSEMBLY_USAGE_OCCURRENCE('{k}','Block:{k}','',#70,#{pd},$);\n"); t += &format!("#{}=PRODUCT_DEFINITION_SHAPE('','',#{u});\n", u + 1); - t += &format!("#{}=ITEM_DEFINED_TRANSFORMATION('','',#65,#{a});\n", u + 2); + t += &format!("#{}=ITEM_DEFINED_TRANSFORMATION('','',#{},#{a});\n", u + 2, pd + 15); t += &format!( - "#{}=(REPRESENTATION_RELATIONSHIP('','',#60,#80)REPRESENTATION_RELATIONSHIP_WITH_TRANSFORMATION(#{})SHAPE_REPRESENTATION_RELATIONSHIP());\n", + "#{}=(REPRESENTATION_RELATIONSHIP('','',#{},#80)REPRESENTATION_RELATIONSHIP_WITH_TRANSFORMATION(#{})SHAPE_REPRESENTATION_RELATIONSHIP());\n", u + 3, + pd + 10, u + 2 ); t += &format!("#{}=CONTEXT_DEPENDENT_SHAPE_REPRESENTATION(#{},#{});\n", u + 4, u + 3, u + 1); @@ -710,6 +735,31 @@ fn instanced_parts_are_oriented_once() { assert_same_import(&imp, &step_import_with_workers(&t, 8).unwrap()); } +/// Distinct parts are oriented together, not one occurrence at a time: an assembly placing +/// two parts in turn converts in a few steps of many solids (the first finds both parts' +/// orientations side by side on the workers, the next places every other instance), where +/// converting instance by instance took one step each. +#[test] +fn distinct_parts_convert_together_across_instances() { + let n = 24; + let t = instanced_assembly_of(2, n); + let mut reading = StepReading::with_workers(&t, 8).unwrap(); + let mut steps = 0; + while reading.step() { + steps += 1; + } + assert_eq!(steps, 2, "both parts in the first step, every other instance in the second"); + let threaded = reading.finish().unwrap(); + assert_eq!(threaded.bodies.len(), n as usize); + assert!(threaded.warnings.is_empty(), "{:?}", threaded.warnings); + crate::body::ORIENTED.with(|c| c.set(0)); + let serial = step_import_with_workers(&t, 1).unwrap(); + assert_eq!(crate::body::ORIENTED.with(|c| c.get()), 2, "two parts"); + assert_same_import(&serial, &threaded); + assert_eq!(threaded.bodies[1].name, "Cube1"); + assert_eq!(threaded.bodies[1].path, vec!["Assembly".to_string(), "Block1".to_string()]); +} + /// A part an assembly places twice is meshed once: the second instance shows the first one's /// display mesh, placed (the same triangles and faces, at its own place), and keeps its own /// placed geometry for everything else.