diff --git a/apps/solidcraft/src/device_lost.rs b/apps/solidcraft/src/device_lost.rs index 2f598c69c..5e39d0d88 100644 --- a/apps/solidcraft/src/device_lost.rs +++ b/apps/solidcraft/src/device_lost.rs @@ -76,7 +76,8 @@ fn command( reason: &str, ) -> std::process::Command { let mut command = std::process::Command::new(exe); - command.args(args).env(LOST_ENV, lost_at.to_string()); + // The notice of a hang that restarted this launch isn't the next one's (`App::watch_for_a_hang`). + command.args(args).env(LOST_ENV, lost_at.to_string()).env_remove(startup::HUNG_ENV); match self::reason(Some(reason)) { Some(reason) => command.env(REASON_ENV, reason), // Not one inherited from an earlier loss. diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rs index cbead5a1d..574d2f2e9 100644 --- a/apps/solidcraft/src/main.rs +++ b/apps/solidcraft/src/main.rs @@ -93,6 +93,12 @@ struct App { come_forward: come_forward::ComeForward, /// The graphics device: lost, it can't draw again (`App::watch_device`, #4172). gpu: Option, + /// The window's backend, and whether it is the software renderer. + renderer: Option<(eframe::wgpu::Backend, bool)>, + /// A GPU that hangs after the window is up (`App::watch_for_a_hang`). + hang: startup::HangWatch, + /// The GPU hung: the next launch starts once this one has closed, on the backend saved then. + restart: bool, #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] space_mouse: space_mouse::SpaceMouse, #[cfg(target_os = "macos")] @@ -219,6 +225,7 @@ impl eframe::App for App { } } self.watch_device(); + self.watch_for_a_hang(ctx); // The design on screen is the one whose commands a crash report replays. self.solve.session.crash_journal = solidcraft_engine::crash::enabled(); // The OS closing the window (Alt+F4, Cmd+Q, the window manager): ask about unsaved @@ -319,11 +326,14 @@ impl Drop for App { // The window's GPU surface is still to be dropped, and on a lost device that may // never return (#9044). device_lost::end(); + } else if let Some((backend, _)) = self.renderer.filter(|_| self.restart) { + restart_after_hang(backend); } // An update asked to restart into the new version or to run its installer. if let Some((program, args)) = solidcraft_ui_egui::update::take_launch(&mut self.solve) { let mut command = std::process::Command::new(&program); - command.args(&args); + // This launch's notices aren't the new version's. + command.args(&args).env_remove(startup::HUNG_ENV); #[cfg(windows)] { use std::os::windows::process::CommandExt; @@ -364,6 +374,93 @@ impl App { gpu.reason = reason; self.solve.quit_requested = true; } + + /// The GPU hung (frames wait out their acquire, or the surface refuses its configuration, while + /// the user is there, #3321): close, keeping unsaved changes for recovery as a forced quit + /// does (the window, white, can't ask about them), and start again on another renderer once + /// closed ([`restart_after_hang`]). Windows only: elsewhere a hidden window's frames can wait + /// as long (Wayland, macOS). + fn watch_for_a_hang(&mut self, ctx: &egui::Context) { + if !cfg!(target_os = "windows") { + return; + } + let (now, input) = ctx.input(|i| { + let input = i.events.iter().any(|e| { + matches!( + e, + egui::Event::PointerMoved(_) | egui::Event::PointerButton { .. } | egui::Event::Key { .. } | egui::Event::MouseWheel { .. } + ) + }); + (i.time, input) + }); + let stalled = eframe::egui_wgpu::stalled_frames(); + let refused = eframe::egui_wgpu::refused_for().map(|d| d.as_secs_f64()); + if !self.hang.hung(now, input, stalled, refused) { + return; + } + let why = match refused { + Some(s) if s > 0.0 => format!("the surface refused its configuration for {s:.0} s"), + _ => format!("{stalled} frames in a row waited out their acquire"), + }; + let Some(backend) = self.renderer.and_then(|(backend, cpu)| startup::after_hang(backend, cpu)) else { + log::error!("graphics: {why}, on the software renderer"); + return; + }; + log::error!("graphics: {why}; the GPU hung, restarting on {backend}"); + eprintln!("SolidCraft graphics: the GPU stopped responding; restarting on {backend}"); + remember_backend(backend); + self.restart = true; + self.solve.quit_requested = true; + } +} + +/// Start SolidCraft again after the GPU hung on `hung`, with the same arguments, on the backend +/// [`remember_backend`] saved. `WGPU_BACKEND` and a start-up retry's backend are not passed on: +/// they would win over that backend and start the one that hung again. +fn restart_after_hang(hung: eframe::wgpu::Backend) { + let Ok(exe) = std::env::current_exe() else { return }; + let mut command = std::process::Command::new(exe); + command + .args(std::env::args_os().skip(1)) + .env(startup::HUNG_ENV, format!("{hung:?}")) + .env_remove(startup::WGPU_BACKEND_ENV) + .env_remove(startup::RETRY_BACKEND_ENV) + .env_remove(startup::FALLBACK_NOTE_ENV) + .env_remove(device_lost::LOST_ENV); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + // Not tied to this process: its own process group, no console. + command.creation_flags(0x0000_0200 | 0x0000_0008); + } + if let Err(e) = command.spawn() { + log::error!("graphics: starting again after the GPU hung: {e}"); + } +} + +/// Keep `backend` as `graphics.json`'s `backend` (its other settings stay). Written to a new file +/// that then replaces the old one: this runs as the GPU hangs, when the machine may reset next, +/// and a half-written file would lose the anti-aliasing setting too. +fn remember_backend(backend: &str) { + let Some(path) = graphics_path() else { return }; + let mut graphics = graphics().filter(serde_json::Value::is_object).unwrap_or_else(|| serde_json::json!({})); + graphics["backend"] = serde_json::json!(backend); + if let Err(e) = write_replacing(&path, graphics.to_string().as_bytes()) { + log::error!("graphics: keeping the {backend} backend in {}: {e}", path.display()); + } +} + +/// Write `bytes` to `path` all at once: to a file next to it, flushed to disk, then renamed over it. +fn write_replacing(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(".new"); + let new = path.with_file_name(name); + let mut file = std::fs::File::create(&new)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(&new, path) } /// Where the next file dialog opens (#7756): the folder of the file last chosen in one, or of the @@ -785,7 +882,8 @@ fn graphics() -> Option { graphics_path().and_then(|p| std::fs::read_to_string(p).ok()).and_then(|s| serde_json::from_str::(&s).ok()) } -/// The graphics backend preference (`graphics.json`'s `backend`: `auto`, `dx12`, `vulkan`, `gl`). +/// The graphics backend preference (`graphics.json`'s `backend`: `auto`, `dx12`, `vulkan`, `gl`, +/// or `cpu`, the software renderer). fn backend_preference() -> Option { graphics().and_then(|v| v.get("backend").and_then(serde_json::Value::as_str).map(str::to_string)) } @@ -970,7 +1068,11 @@ fn main() -> eframe::Result { safe_mode = true; notes.push(format!("The last {failed_starts} launches stopped before the window opened, so this one is in safe mode.")); } - let cpu_renderer = safe_mode || startup::cpu_renderer_env(std::env::var(startup::RENDERER_ENV).ok().as_deref()); + // The software renderer: safe mode, SOLIDCRAFT_RENDERER, or kept after the GPU hung on every + // API (`startup::after_hang`). + let cpu_renderer = safe_mode + || startup::cpu_renderer_env(std::env::var(startup::RENDERER_ENV).ok().as_deref()) + || startup::cpu_renderer_env(backend_preference().as_deref()); // The launch before this one closed because its graphics device was lost (`App::watch_device`). let after_loss = device_lost::recent(std::env::var(device_lost::LOST_ENV).ok().as_deref(), device_lost::now_secs()); if after_loss { @@ -980,6 +1082,10 @@ fn main() -> eframe::Result { } notes.push(device_lost::note(cpu_renderer, reason.as_deref())); } + // The launch before this one closed because its GPU hung (`App::watch_for_a_hang`). + if let Some(hung) = std::env::var(startup::HUNG_ENV).ok().filter(|v| !v.trim().is_empty()) { + notes.push(startup::hang_note(&hung, safe_mode, backend_preference().as_deref())); + } if safe_mode { notes.push("Safe mode: default settings (yours are kept for the next normal start), no recovery offer, software rendering.".into()); } @@ -1124,6 +1230,7 @@ fn main() -> eframe::Result { app.preferences.graphics_adapter = graphics().and_then(|v| v.get("adapter").and_then(serde_json::Value::as_str).map(str::to_string)).unwrap_or_default(); } + let mut renderer = None; if let Some(rs) = &cc.wgpu_render_state { app.services.graphics_adapters = rs .available_adapters @@ -1147,6 +1254,7 @@ fn main() -> eframe::Result { let scene_samples = scene_samples(info.backend, u32::from(samples), &supported); eprintln!("SolidCraft graphics: {:?}, MSAA {scene_samples}x (requested {}x)", rs.adapter.get_info().backend, samples); solidcraft_engine::crash::phase("viewport pipelines (shader compile)"); + renderer = Some((rs.adapter.get_info().backend, cpu_renderer)); let scene_samples = app.set_wgpu_split(rs, depth_bits, STENCIL_BITS, scene_samples); app.msaa_supported = supported; app.preferences.msaa = u32::from(samples); @@ -1252,6 +1360,9 @@ fn main() -> eframe::Result { take_focus_unwanted: control_port.is_some(), come_forward: come_forward::ComeForward::default(), gpu, + renderer, + hang: startup::HangWatch::default(), + restart: false, #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] space_mouse, #[cfg(target_os = "macos")] @@ -1279,8 +1390,8 @@ fn startup_step(step: &'static str, detail: &str) { } /// The renderer settings: with `cpu`, the software adapter (WARP, lavapipe, llvmpipe). The -/// backends come from `startup::choose_backends` (on Windows D3D12 first, and only the backend -/// in use is started); `preference` is `graphics.json`'s `backend`. +/// backends come from `startup::choose_backends` (on Windows D3D12 first, Vulkan on AMD +/// integrated GPUs, and only the backend in use is started); `preference` is `graphics.json`'s `backend`. fn wgpu_options(cpu: bool, preference: Option<&str>) -> eframe::egui_wgpu::WgpuConfiguration { let mut options = eframe::egui_wgpu::WgpuConfiguration::default(); if let eframe::egui_wgpu::WgpuSetup::CreateNew(setup) = &mut options.wgpu_setup { @@ -1288,7 +1399,7 @@ fn wgpu_options(cpu: bool, preference: Option<&str>) -> eframe::egui_wgpu::WgpuC // A retry applies only to this launch. WGPU_BACKEND always has priority. let retry = std::env::var(startup::RETRY_BACKEND_ENV).ok(); let preference = if cpu { preference } else { retry.as_deref().or(preference) }; - if let Some(backends) = startup::choose_backends(cfg!(target_os = "windows"), env_set, cpu, preference, startup::has_hardware_adapter) { + if let Some(backends) = startup::choose_backends(cfg!(target_os = "windows"), env_set, cpu, preference, startup::hardware_adapters) { eprintln!("SolidCraft graphics: starting {backends:?}"); setup.instance_descriptor.backends = backends; } @@ -1330,7 +1441,7 @@ fn no_window(error: eframe::Error, tried_cpu: bool, failed: eframe::wgpu::Backen log::warn!("{note}"); eprintln!("SolidCraft: {note}"); let mut command = std::process::Command::new(exe); - command.args(std::env::args_os().skip(1)).env(startup::FALLBACK_NOTE_ENV, note); + command.args(std::env::args_os().skip(1)).env(startup::FALLBACK_NOTE_ENV, note).env_remove(startup::HUNG_ENV); if next == "cpu" { command.env(startup::RENDERER_ENV, "cpu").env_remove(startup::RETRY_BACKEND_ENV); } else { diff --git a/apps/solidcraft/src/startup.rs b/apps/solidcraft/src/startup.rs index 683ded4ad..047d5d9a4 100644 --- a/apps/solidcraft/src/startup.rs +++ b/apps/solidcraft/src/startup.rs @@ -11,6 +11,10 @@ //! - **A renderer that cannot start**: on Windows, try the remaining graphics APIs in order, //! then the software renderer (`SOLIDCRAFT_RENDERER=cpu`). An explicit `WGPU_BACKEND` fails //! cleanly without switching. Exhausted retries show a message box with the error. +//! - **A GPU that hangs after the window is up** (Windows, [`HangWatch`]): frames that each wait +//! out their acquire while the user is there. The app closes, keeping unsaved changes for +//! recovery, and starts again on the next API (D3D12 → Vulkan → OpenGL → the software +//! renderer), kept in `graphics.json`. use std::path::{Path, PathBuf}; @@ -188,8 +192,14 @@ pub const WGPU_BACKEND_ENV: &str = "WGPU_BACKEND"; /// together and prefer Vulkan, and starting Vulkan loads every installed Vulkan driver: Intel's /// (`igvk64.dll`) crashed SolidCraft at startup that way, inside the driver, before anything /// could be reported. D3D12 is Windows' own graphics API, the one its drivers are tested with. -/// `has_hardware(backends)` says whether those backends have an adapter that isn't software -/// (it is only asked about the backends being tried, one at a time). +/// `hardware(backends)` lists those backends' adapters that aren't software (it is only asked +/// about the backends being tried, one at a time). +/// +/// One exception: when D3D12's only hardware adapters are AMD integrated GPUs and Vulkan has a +/// hardware adapter, Vulkan. On those (Ryzen 5400U, 5600U, 5700G) D3D12 hung the driver a few +/// seconds into the first frames and Windows reset it (a black screen, then a crash, #1789, +/// #2806); 0.11.0, which preferred Vulkan, worked there, and `WGPU_BACKEND=vulkan` runs it +/// smoothly (#3321). /// /// - `WGPU_BACKEND` set (`env_set`): `None`, whatever it says wins. /// - The software renderer (`cpu`, safe mode): D3D12 alone on Windows (WARP); `None` elsewhere. @@ -202,7 +212,7 @@ pub fn choose_backends( env_set: bool, cpu: bool, preference: Option<&str>, - mut has_hardware: impl FnMut(eframe::wgpu::Backends) -> bool, + mut hardware: impl FnMut(eframe::wgpu::Backends) -> Vec, ) -> Option { use eframe::wgpu::Backends; if env_set { @@ -221,7 +231,137 @@ pub fn choose_backends( if !windows { return None; } - Some(WINDOWS_ORDER.into_iter().find(|b| has_hardware(*b)).unwrap_or(Backends::DX12)) + for b in WINDOWS_ORDER { + let found = hardware(b); + if found.is_empty() { + continue; + } + if b == Backends::DX12 && found.iter().all(HardwareAdapter::hangs_on_d3d12) && !hardware(Backends::VULKAN).is_empty() { + log::warn!("graphics: Vulkan rather than D3D12 on an AMD integrated GPU (#3321)"); + return Some(Backends::VULKAN); + } + return Some(b); + } + Some(Backends::DX12) +} + +/// A hardware (not software) adapter a backend offers, as [`choose_backends`] needs it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HardwareAdapter { + /// The PCI vendor id. + pub vendor: u32, + pub device_type: eframe::wgpu::DeviceType, +} + +/// AMD's PCI vendor id. +const AMD: u32 = 0x1002; + +impl HardwareAdapter { + /// An AMD integrated GPU, whose D3D12 driver hung on SolidCraft's first frames (#3321). + pub fn hangs_on_d3d12(&self) -> bool { + self.vendor == AMD && self.device_type == eframe::wgpu::DeviceType::IntegratedGpu + } +} + +/// Frames in a row that stalled ([`eframe::egui_wgpu::stalled_frames`]: each waited out wgpu's +/// one-second acquire) after which the GPU is taken as hung. +pub const STALLED_FRAMES_FOR_RESTART: u32 = 5; +/// Seconds the window's surface may refuse its configuration ([`eframe::egui_wgpu::refused_for`]) +/// while the user is there before the GPU is taken as hung: after a reset, a surface that stays +/// unsupported leaves a blank window. Refusals that clear up (a resize, a display waking up) +/// take a moment, not this long. +pub const REFUSED_S_FOR_RESTART: f64 = 10.0; +/// Stalled frames count only this many seconds after the user's input: a hung GPU leaves a white +/// window the user clicks at, while a switched-off display can hold frames back with nobody there. +pub const STALL_INPUT_WINDOW_S: f64 = 10.0; +/// Set on the launch that replaces one whose GPU hung: the hung backend, for the notice. +pub const HUNG_ENV: &str = "SOLIDCRAFT_GPU_HUNG"; + +/// Watches for a hung GPU after the window is up (#3321). The failed-start count misses it: frames +/// still come, a second apart, so the start counts as good. +#[derive(Debug, Default)] +pub struct HangWatch { + last_input: Option, + /// When the user came back: the first input after none for [`STALL_INPUT_WINDOW_S`]. + arrived: f64, + /// The stalled streak when the user came back. Those frames stalled with nobody there (a + /// display switched off holds them back too), and the input that wakes the display comes + /// before the next frame shows whether the GPU is fine, so only stalls after it count. + stalled_before: u32, + hung: bool, +} + +impl HangWatch { + /// Per frame, at `now` seconds: whether the user gave input this frame, the frames in a row + /// that stalled, and the seconds the surface has refused its configuration. True once, the + /// frame the GPU is taken as hung. + pub fn hung(&mut self, now: f64, input: bool, stalled: u32, refused_s: Option) -> bool { + if stalled < self.stalled_before { + // The streak broke: every stall since came after the user was back. + self.stalled_before = 0; + } + if input { + if !self.user_there(now) { + self.arrived = now; + self.stalled_before = stalled; + } + self.last_input = Some(now); + } + if self.hung || !self.user_there(now) { + return false; + } + // A surface refused since before the user came back counts from then. + let refused = refused_s.map_or(0.0, |r| r.min(now - self.arrived)); + self.hung = stalled.saturating_sub(self.stalled_before) >= STALLED_FRAMES_FOR_RESTART || refused >= REFUSED_S_FOR_RESTART; + self.hung + } + + fn user_there(&self, now: f64) -> bool { + self.last_input.is_some_and(|t| (0.0..=STALL_INPUT_WINDOW_S).contains(&(now - t))) + } +} + +/// The backend a launch whose GPU hung on `backend` starts again on, kept in `graphics.json` for +/// later launches too (`cpu`: the software renderer, which has nothing to fall back to). Each +/// hang moves on to the next of [`WINDOWS_ORDER`], then to the software renderer, so a GPU hangs +/// at most once on each: a safe-mode restart would leave the backend that hung saved for the +/// next normal start. D3D12 goes to Vulkan, which runs the GPUs whose D3D12 driver hung (#1789); +/// it isn't probed here, as loading Vulkan's drivers into this process is what crashed once +/// (`WINDOWS_ORDER`): the new launch finds out, and falls back to software if it has no GPU. +pub fn after_hang(backend: eframe::wgpu::Backend, cpu: bool) -> Option<&'static str> { + match backend { + _ if cpu => None, + eframe::wgpu::Backend::Dx12 => Some("vulkan"), + eframe::wgpu::Backend::Vulkan => Some("gl"), + _ => Some(SOFTWARE_BACKEND), + } +} + +/// `graphics.json`'s `backend` for the software renderer, kept after the GPU hung on every API. +pub const SOFTWARE_BACKEND: &str = "cpu"; + +/// The notice on the launch after the GPU hung on `hung` (a backend's name): this one is in safe +/// mode, or on the `backend` preference. +pub fn hang_note(hung: &str, safe_mode: bool, backend: Option<&str>) -> String { + let now = match backend { + _ if safe_mode => "in safe mode".to_string(), + Some(b) => format!("on {}, which it keeps using (the backend in graphics.json)", api_name(b)), + None => "with another renderer".to_string(), + }; + let unsaved = if safe_mode { "the next normal start offers unsaved changes" } else { "unsaved changes are offered for recovery" }; + format!("The graphics driver stopped responding on {}, so SolidCraft restarted {now}; {unsaved}.", api_name(hung)) +} + +/// A backend's name (`Dx12`, `vulkan`, …) as people know the graphics API. +fn api_name(backend: &str) -> &str { + match backend.trim().to_ascii_lowercase().as_str() { + "dx12" => "Direct3D 12", + "vulkan" => "Vulkan", + "gl" => "OpenGL", + "metal" => "Metal", + "cpu" | "software" => "the software renderer", + _ => backend, + } } /// Internal, process-local retry choice; never written to the graphics preference. @@ -272,8 +412,8 @@ pub fn float_depth_everywhere(options: &eframe::egui_wgpu::WgpuConfiguration) -> /// The order backends are tried in on Windows. pub const WINDOWS_ORDER: [eframe::wgpu::Backends; 3] = [eframe::wgpu::Backends::DX12, eframe::wgpu::Backends::VULKAN, eframe::wgpu::Backends::GL]; -/// Whether `backends` (started on their own, nothing else) have a hardware adapter. -pub fn has_hardware_adapter(backends: eframe::wgpu::Backends) -> bool { +/// The hardware adapters of `backends` (started on their own, nothing else). +pub fn hardware_adapters(backends: eframe::wgpu::Backends) -> Vec { let mut desc = eframe::wgpu::InstanceDescriptor::new_without_display_handle(); desc.backends = backends; desc.flags = eframe::wgpu::InstanceFlags::from_build_config().with_env(); @@ -282,7 +422,12 @@ pub fn has_hardware_adapter(backends: eframe::wgpu::Backends) -> bool { let adapters = pollster::block_on(instance.enumerate_adapters(backends)); let found: Vec = adapters.iter().map(|a| format!("{} ({:?})", a.get_info().name, a.get_info().device_type)).collect(); log::info!("graphics: {backends:?} adapters: {found:?}"); - adapters.iter().any(|a| a.get_info().device_type != eframe::wgpu::DeviceType::Cpu) + adapters + .iter() + .map(|a| a.get_info()) + .filter(|i| i.device_type != eframe::wgpu::DeviceType::Cpu) + .map(|i| HardwareAdapter { vendor: i.vendor, device_type: i.device_type }) + .collect() } /// The environment variable that switches wgpu's indirect-call validation on or off. @@ -562,21 +707,32 @@ mod tests { let _ = std::fs::remove_dir_all(&d); } + /// A discrete NVIDIA GPU, and AMD's integrated one (#3321). + const NVIDIA: HardwareAdapter = HardwareAdapter { vendor: 0x10de, device_type: eframe::wgpu::DeviceType::DiscreteGpu }; + const AMD_IGPU: HardwareAdapter = HardwareAdapter { vendor: AMD, device_type: eframe::wgpu::DeviceType::IntegratedGpu }; + + /// A probe where the backends `with` have `gpus`, and the others none. + fn probe_with( + with: &'static [eframe::wgpu::Backends], + gpus: &'static [HardwareAdapter], + ) -> impl FnMut(eframe::wgpu::Backends) -> Vec { + use eframe::wgpu::Backends; + move |b: Backends| { + assert!(b == Backends::DX12 || b == Backends::VULKAN || b == Backends::GL, "one backend at a time: {b:?}"); + if with.contains(&b) { gpus.to_vec() } else { Vec::new() } + } + } + #[test] fn windows_starts_d3d12_first_and_vulkan_only_when_d3d12_has_no_gpu() { use eframe::wgpu::Backends; - let probe = |with: &'static [Backends]| { - move |b: Backends| { - assert!(b == Backends::DX12 || b == Backends::VULKAN || b == Backends::GL, "one backend at a time: {b:?}"); - with.contains(&b) - } - }; + let probe = |with: &'static [Backends]| probe_with(with, &[NVIDIA]); // D3D12 works: Vulkan and OpenGL are never even asked about (starting Vulkan loads // its drivers, which is what crashed). let mut asked = Vec::new(); let chosen = choose_backends(true, false, false, None, |b| { asked.push(b); - b == Backends::DX12 + if b == Backends::DX12 { vec![NVIDIA] } else { Vec::new() } }); assert_eq!(chosen, Some(Backends::DX12)); assert_eq!(asked, [Backends::DX12]); @@ -591,6 +747,77 @@ mod tests { assert_eq!(choose_backends(true, false, false, Some(""), probe(&[Backends::GL])), Some(Backends::GL)); } + /// AMD integrated GPUs hung under D3D12 at the first frames, the driver reset and the app + /// crashed (#1789, #2806); they run on Vulkan (#3321). + #[test] + fn an_amd_integrated_gpu_starts_vulkan_rather_than_d3d12() { + use eframe::wgpu::Backends; + let all = &[Backends::DX12, Backends::VULKAN, Backends::GL]; + assert_eq!(choose_backends(true, false, false, None, probe_with(all, &[AMD_IGPU])), Some(Backends::VULKAN)); + // Without a Vulkan GPU: D3D12 still, rather than OpenGL or software. + assert_eq!(choose_backends(true, false, false, None, probe_with(&[Backends::DX12], &[AMD_IGPU])), Some(Backends::DX12)); + // Another GPU beside it (a laptop's discrete one): D3D12, which can pick that one. + assert_eq!(choose_backends(true, false, false, None, probe_with(all, &[AMD_IGPU, NVIDIA])), Some(Backends::DX12)); + // A discrete AMD GPU keeps D3D12. + let discrete = &[HardwareAdapter { vendor: AMD, device_type: eframe::wgpu::DeviceType::DiscreteGpu }]; + assert_eq!(choose_backends(true, false, false, None, probe_with(all, discrete)), Some(Backends::DX12)); + // The preference and WGPU_BACKEND still win, and other systems are left to wgpu. + let never = |b: Backends| -> Vec { panic!("no probing expected, asked about {b:?}") }; + assert_eq!(choose_backends(true, false, false, Some("dx12"), never), Some(Backends::DX12)); + assert_eq!(choose_backends(true, true, false, None, never), None); + assert_eq!(choose_backends(false, false, false, None, never), None); + } + + /// After the window was up, D3D12 hung the GPU: frames came a second apart and never showed, + /// a white window with no crash (#1789, #3321). Five such frames while the user is there + /// restart the app, D3D12 on Vulkan and anything else in safe mode. + #[test] + fn a_gpu_that_hangs_after_the_window_is_up_restarts_on_another_renderer() { + use eframe::wgpu::Backend; + let stalled = STALLED_FRAMES_FOR_RESTART; + let mut watch = HangWatch::default(); + assert!(!watch.hung(0.0, true, 0, None)); + assert!(!watch.hung(4.0, false, stalled - 1, None), "not yet"); + assert!(watch.hung(5.0, false, stalled, None), "input 5 s ago"); + assert!(!watch.hung(6.0, true, stalled + 1, None), "once"); + // Nobody at the computer (a display switched off holds frames back too): no restart. + let mut away = HangWatch::default(); + assert!(!away.hung(100.0, false, stalled, None)); + assert!(!away.hung(200.0, false, 100, None)); + // The input that wakes the display comes before a frame shows the GPU is fine: the + // stalls from before don't count, and the frames on time that follow end the streak. + let mut woke = HangWatch::default(); + assert!(!woke.hung(0.0, true, 0, None)); + assert!(!woke.hung(STALL_INPUT_WINDOW_S + 1.0, false, 30, None)); + assert!(!woke.hung(STALL_INPUT_WINDOW_S + 2.0, true, 30, None), "back at a healthy app"); + assert!(!woke.hung(STALL_INPUT_WINDOW_S + 3.0, true, 0, None)); + let later: Vec = (1..=stalled).map(|n| woke.hung(STALL_INPUT_WINDOW_S + 3.0 + f64::from(n), true, n, None)).collect(); + assert_eq!(later.last(), Some(&true), "then it hangs: {later:?}"); + // Back at a GPU that hung while nobody was there: five more stalls. + let mut back = HangWatch::default(); + assert!(!back.hung(100.0, true, 30, None)); + assert!(!back.hung(103.0, true, 30 + stalled - 1, None)); + assert!(back.hung(104.0, true, 30 + stalled, None)); + // A surface refused for good after a reset never reaches the acquire: a blank window. + let mut refused = HangWatch::default(); + assert!(!refused.hung(0.0, true, 0, Some(60.0)), "refused since before the user came"); + assert!(!refused.hung(REFUSED_S_FOR_RESTART - 1.0, true, 0, Some(60.0 + REFUSED_S_FOR_RESTART - 1.0))); + assert!(refused.hung(REFUSED_S_FOR_RESTART, true, 0, Some(60.0 + REFUSED_S_FOR_RESTART))); + let mut resized = HangWatch::default(); + assert!(!resized.hung(0.0, true, 0, Some(0.5)), "a moment's refusal"); + + // Each API hangs at most once: D3D12, then Vulkan, then OpenGL, then software. + assert_eq!(after_hang(Backend::Dx12, false), Some("vulkan")); + assert_eq!(after_hang(Backend::Vulkan, false), Some("gl")); + assert_eq!(after_hang(Backend::Gl, false), Some(SOFTWARE_BACKEND)); + assert_eq!(after_hang(Backend::Dx12, true), None, "software: nothing to fall back to"); + assert!(cpu_renderer_env(Some(SOFTWARE_BACKEND)), "the saved software renderer starts as such"); + let note = hang_note("Dx12", false, Some("vulkan")); + assert!(note.contains("on Direct3D 12") && note.contains("on Vulkan") && note.contains("recovery"), "{note}"); + assert!(hang_note("Vulkan", true, Some("gl")).contains("in safe mode")); + assert!(hang_note("Gl", false, Some("cpu")).contains("on the software renderer, which it keeps using")); + } + #[test] fn failed_windows_backends_fall_back_in_order_and_stop_after_software() { use eframe::wgpu::Backends as B; @@ -617,7 +844,7 @@ mod tests { assert_eq!( choose_backends(windows, true, cpu, Some("vulkan"), |_| { probed = true; - false + Vec::new() }), None ); @@ -642,7 +869,7 @@ mod tests { #[test] fn wgpu_backend_the_preference_and_the_software_renderer_override_the_order() { use eframe::wgpu::Backends; - let never = |b: Backends| -> bool { panic!("no probing expected, asked about {b:?}") }; + let never = |b: Backends| -> Vec { panic!("no probing expected, asked about {b:?}") }; // WGPU_BACKEND: wgpu reads it itself, nothing is probed. assert_eq!(choose_backends(true, true, false, Some("gl"), never), None); assert_eq!(choose_backends(true, true, true, None, never), None); @@ -655,7 +882,7 @@ mod tests { assert_eq!(choose_backends(true, false, false, Some("dx12,vulkan"), never), Some(Backends::DX12 | Backends::VULKAN)); assert_eq!(choose_backends(false, false, false, Some("vulkan"), never), Some(Backends::VULKAN)); // An unknown preference is the default order. - assert_eq!(choose_backends(true, false, false, Some("glide"), |b| b == Backends::DX12), Some(Backends::DX12)); + assert_eq!(choose_backends(true, false, false, Some("glide"), probe_with(&[Backends::DX12], &[NVIDIA])), Some(Backends::DX12)); // Elsewhere: wgpu's own choice. assert_eq!(choose_backends(false, false, false, None, never), None); assert_eq!(choose_backends(false, false, false, Some("auto"), never), None); @@ -668,10 +895,10 @@ mod tests { // D3D12 never loads its OpenGL driver into the test process, and AMD's (`atio6axx.dll`) // aborted the process as it exited (#1156). if cfg!(target_os = "windows") { - let _ = choose_backends(true, false, false, None, has_hardware_adapter); + let _ = choose_backends(true, false, false, None, hardware_adapters); } else { for b in WINDOWS_ORDER { - let _ = has_hardware_adapter(b); + let _ = hardware_adapters(b); } } } diff --git a/docs/book/src/guide/troubleshooting.md b/docs/book/src/guide/troubleshooting.md index 685ee7ca2..5b9cfd3dd 100644 --- a/docs/book/src/guide/troubleshooting.md +++ b/docs/book/src/guide/troubleshooting.md @@ -134,7 +134,16 @@ old Intel Vulkan driver, `igvk64.dll`, did). Windows' Event Viewer names the dri From 0.11.1, SolidCraft on Windows starts Direct3D 12 first and loads no other graphics API when Direct3D 12 has a graphics card; Vulkan and then OpenGL are tried only on a computer without one. Earlier versions started Vulkan, Direct3D 12 and OpenGL together and preferred -Vulkan. To choose the graphics API yourself, in any version, open a Command Prompt and start +Vulkan. On a computer whose only graphics is AMD's integrated GPU (Ryzen laptops and desktops +without a graphics card), SolidCraft starts Vulkan instead: Direct3D 12 hung those drivers a +few seconds after the window opened, and the screen went black until Windows reset the driver. +If the graphics driver stops responding while SolidCraft is open (the window stays white or +black and every click takes a second), SolidCraft notices after a few seconds, closes, and +starts again: on Vulkan if it was using Direct3D 12 (it keeps using Vulkan, as `"backend": +"vulkan"` in `graphics.json`), otherwise in safe mode. Unsaved changes are offered for recovery. +The restarted SolidCraft ignores `WGPU_BACKEND` (below); remove it if you set it with `setx`, +or the next start uses that API again. +To choose the graphics API yourself, in any version, open a Command Prompt and start SolidCraft from it: ``` diff --git a/vendor/README.md b/vendor/README.md index 0b1aaa045..f4ee52ca2 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -96,6 +96,12 @@ SolidCraft changes: error scopes, awaiting every result before returning it. A refused UI pipeline (D3D12 `egui_pipeline` shader compilation, #6648) returns `WgpuError::CreateRendererError` through eframe so startup can retry another backend instead of invoking wgpu's panic handler. +- `src/lib.rs`, `src/winit.rs`: `stalled_frames` counts the frames in a row whose texture took + at least `STALLED_ACQUIRE` (950 ms) to acquire. wgpu waits at most a second for the next + image and D3D12 then goes on as if it had come, so a hung GPU shows only as frames that each + wait that second; the app restarts on another renderer after a few of them (#3321). Only the + main window counts. `refused_for` says how long the main window has skipped its frames on a + refused configure, which never reach the acquire. ## winit 0.30.13 (Apache-2.0, © The winit contributors, https://github.com/rust-windowing/winit) diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index 773207ea4..69b7e7472 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -70,6 +70,61 @@ pub(crate) fn startup_step(step: &'static str, detail: &str) { } } +/// SolidCraft: a frame whose texture took at least this long to acquire stalled. wgpu waits at +/// most a second for the swap chain's next image, and D3D12 then goes on as if it had come: a +/// hung GPU shows as frames that each wait that second and never reach the screen, a white +/// window with nothing reported (#3321). +pub const STALLED_ACQUIRE: std::time::Duration = std::time::Duration::from_millis(950); + +static STALLED_FRAMES: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + +/// SolidCraft: how many of the main window's frames in a row stalled acquiring their texture +/// ([`STALLED_ACQUIRE`]). Other viewports (the agent chat's window) don't count: a frame on time +/// there would start a stalled main window's count over. +pub fn stalled_frames() -> u32 { + STALLED_FRAMES.load(std::sync::atomic::Ordering::Relaxed) +} + +/// The frames in a row that stalled, after one whose texture took `elapsed`. Only the `winit` +/// painter acquires textures. +#[cfg_attr(not(feature = "winit"), allow(dead_code))] +fn count_stall(stalled: u32, elapsed: std::time::Duration) -> u32 { + if elapsed >= STALLED_ACQUIRE { + stalled.saturating_add(1) + } else { + 0 + } +} + +/// The main window's acquire took `elapsed`: its surface is configured again, if it wasn't. +#[cfg_attr(not(feature = "winit"), allow(dead_code))] +pub(crate) fn note_acquire(elapsed: std::time::Duration) { + let stalled = count_stall(stalled_frames(), elapsed); + STALLED_FRAMES.store(stalled, std::sync::atomic::Ordering::Relaxed); + if let Ok(mut since) = REFUSED_SINCE.lock() { + *since = None; + } +} + +static REFUSED_SINCE: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// SolidCraft: how long the main window has gone without acquiring a texture because its surface +/// refuses its configuration (or waits for a refused one's retry, #1935); `None` while it +/// acquires. Those frames are skipped before the acquire, so they never count as stalled: a +/// surface refused for good after a GPU reset leaves a blank window with nothing else to show +/// for it (#3321). +pub fn refused_for() -> Option { + REFUSED_SINCE.lock().ok().and_then(|since| *since).map(|since| since.elapsed()) +} + +/// The main window skipped a frame at `now` on a refused configure. +#[cfg_attr(not(feature = "winit"), allow(dead_code))] +pub(crate) fn note_refused(now: web_time::Instant) { + if let Ok(mut since) = REFUSED_SINCE.lock() { + since.get_or_insert(now); + } +} + /// An error produced by egui-wgpu. #[derive(thiserror::Error, Debug)] pub enum WgpuError { @@ -881,6 +936,20 @@ mod msaa_tests { assert_eq!(super::msaa_counts(F::MULTISAMPLE_X4, None), [1]); assert_eq!(super::clamp_msaa_samples(&[], 8), 1); } + + /// Frames that each waited out wgpu's one-second acquire count up; one on time starts over + /// (#3321). + #[test] + fn frames_that_wait_out_the_acquire_count_as_stalled() { + use std::time::Duration; + let second = Duration::from_millis(1000); + let stalled = (0..3).fold(0, |n, _| super::count_stall(n, second)); + assert_eq!(stalled, 3); + assert_eq!(super::count_stall(stalled, Duration::from_millis(16)), 0); + // A slow frame isn't a stall: the image came before the timeout. + assert_eq!(super::count_stall(0, Duration::from_millis(400)), 0); + assert_eq!(super::count_stall(u32::MAX, second), u32::MAX); + } } /// SolidCraft: run `f` with `device`'s errors captured, and the first error it caused. Uncaptured, diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index bc671a80d..f5c5d7507 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -687,6 +687,9 @@ impl Painter { && let Err(err) = self.recreate_surface(viewport_id, window) { log::error!("Failed to recreate surface for {viewport_id:?}: {err}"); + if viewport_id == ViewportId::ROOT { + crate::note_refused(now); + } return vsync_sec; } @@ -741,6 +744,10 @@ impl Painter { now, ) { + // SolidCraft: the main window's time without a texture (#3321, `refused_for`). + if viewport_id == ViewportId::ROOT { + crate::note_refused(now); + } return vsync_sec; } @@ -761,7 +768,13 @@ impl Painter { if let Some(error) = failed { log::warn!("The surface gave no frame: {error}"); } - vsync_sec += start.elapsed().as_secs_f32(); + let elapsed = start.elapsed(); + vsync_sec += elapsed.as_secs_f32(); + // SolidCraft: the main window's frames that wait out the acquire are a hung GPU + // (#3321). + if viewport_id == ViewportId::ROOT { + crate::note_acquire(elapsed); + } output_frame };