From a52ee70e4cd43ea2835fc3bc671547452bb2c790 Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Fri, 25 Jul 2025 14:04:46 +0200 Subject: [PATCH] Desktop: Directly upload frame buffer (#2930) * Upload frame buffer directly to gpu texture * Disable cef GPU acceleration to prevent crashes * Cleanup code * Address review comments --------- Co-authored-by: Timon Schelling --- desktop/src/app.rs | 38 ++-- desktop/src/cef.rs | 48 ++++- desktop/src/cef/internal/app.rs | 10 +- .../cef/internal/browser_process_handler.rs | 8 +- desktop/src/cef/internal/render_handler.rs | 4 +- desktop/src/main.rs | 9 +- desktop/src/render.rs | 191 +++++++----------- 7 files changed, 150 insertions(+), 158 deletions(-) diff --git a/desktop/src/app.rs b/desktop/src/app.rs index d929a71d..eced6556 100644 --- a/desktop/src/app.rs +++ b/desktop/src/app.rs @@ -1,7 +1,7 @@ use crate::CustomEvent; -use crate::FrameBuffer; use crate::WindowSize; use crate::render::GraphicsState; +use crate::render::WgpuContext; use std::sync::Arc; use std::sync::mpsc::Sender; use std::time::Duration; @@ -12,7 +12,6 @@ use winit::event::StartCause; use winit::event::WindowEvent; use winit::event_loop::ActiveEventLoop; use winit::event_loop::ControlFlow; -use winit::event_loop::EventLoopProxy; use winit::window::Window; use winit::window::WindowId; @@ -22,24 +21,24 @@ pub(crate) struct WinitApp { pub(crate) cef_context: cef::Context, pub(crate) window: Option>, cef_schedule: Option, - ui_frame_buffer: Option, + _ui_frame_buffer: Option, window_size_sender: Sender, - _viewport_frame_buffer: Option, + _viewport_frame_buffer: Option, graphics_state: Option, - event_loop_proxy: EventLoopProxy, + wgpu_context: WgpuContext, } impl WinitApp { - pub(crate) fn new(cef_context: cef::Context, window_size_sender: Sender, event_loop_proxy: EventLoopProxy) -> Self { + pub(crate) fn new(cef_context: cef::Context, window_size_sender: Sender, wgpu_context: WgpuContext) -> Self { Self { cef_context, window: None, cef_schedule: Some(Instant::now()), _viewport_frame_buffer: None, - ui_frame_buffer: None, + _ui_frame_buffer: None, graphics_state: None, window_size_sender, - event_loop_proxy, + wgpu_context, } } } @@ -47,8 +46,9 @@ impl WinitApp { impl ApplicationHandler for WinitApp { fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { // Set a timeout in case we miss any cef schedule requests - let timeout = Instant::now() + Duration::from_millis(100); + let timeout = Instant::now() + Duration::from_millis(10); let wait_until = timeout.min(self.cef_schedule.unwrap_or(timeout)); + self.cef_context.work(); event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until)); } @@ -71,7 +71,7 @@ impl ApplicationHandler for WinitApp { ) .unwrap(), ); - let graphics_state = futures::executor::block_on(GraphicsState::new(window.clone())); + let graphics_state = GraphicsState::new(window.clone(), self.wgpu_context.clone()); self.window = Some(window); self.graphics_state = Some(graphics_state); @@ -81,24 +81,21 @@ impl ApplicationHandler for WinitApp { fn user_event(&mut self, _: &ActiveEventLoop, event: CustomEvent) { match event { - CustomEvent::UiUpdate(frame_buffer) => { + CustomEvent::UiUpdate(texture) => { if let Some(graphics_state) = self.graphics_state.as_mut() { - graphics_state.update_texture(&frame_buffer); + graphics_state.bind_texture(&texture); + graphics_state.resize(texture.width(), texture.height()); } - self.ui_frame_buffer = Some(frame_buffer); if let Some(window) = &self.window { window.request_redraw(); } } CustomEvent::ScheduleBrowserWork(instant) => { - if let Some(graphics_state) = self.graphics_state.as_mut() - && let Some(frame_buffer) = &self.ui_frame_buffer - && graphics_state.ui_texture_outdated(frame_buffer) - { + if instant <= Instant::now() { self.cef_context.work(); - let _ = self.event_loop_proxy.send_event(CustomEvent::ScheduleBrowserWork(Instant::now() + Duration::from_millis(1))); + } else { + self.cef_schedule = Some(instant); } - self.cef_schedule = Some(instant); } } } @@ -113,9 +110,6 @@ impl ApplicationHandler for WinitApp { } WindowEvent::Resized(PhysicalSize { width, height }) => { let _ = self.window_size_sender.send(WindowSize::new(width as usize, height as usize)); - if let Some(ref mut graphics_state) = self.graphics_state { - graphics_state.resize(width, height); - } self.cef_context.notify_of_resize(); } diff --git a/desktop/src/cef.rs b/desktop/src/cef.rs index ef63672f..d84e9ad6 100644 --- a/desktop/src/cef.rs +++ b/desktop/src/cef.rs @@ -1,4 +1,4 @@ -use crate::{CustomEvent, FrameBuffer}; +use crate::{CustomEvent, WgpuContext, render::FrameBufferRef}; use std::{ sync::{Arc, Mutex, mpsc::Receiver}, time::Instant, @@ -15,7 +15,7 @@ use winit::event_loop::EventLoopProxy; pub(crate) trait CefEventHandler: Clone { fn window_size(&self) -> WindowSize; - fn draw(&self, frame_buffer: FrameBuffer); + fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>); /// Scheudule the main event loop to run the cef event loop after the timeout /// [`_cef_browser_process_handler_t::on_schedule_message_pump_work`] for more documentation. fn schedule_cef_message_loop_work(&self, scheduled_time: Instant); @@ -37,6 +37,7 @@ impl WindowSize { pub(crate) struct CefHandler { window_size_receiver: Arc>, event_loop_proxy: EventLoopProxy, + wgpu_context: WgpuContext, } struct WindowSizeReceiver { receiver: Receiver, @@ -51,10 +52,11 @@ impl WindowSizeReceiver { } } impl CefHandler { - pub(crate) fn new(window_size_receiver: Receiver, event_loop_proxy: EventLoopProxy) -> Self { + pub(crate) fn new(window_size_receiver: Receiver, event_loop_proxy: EventLoopProxy, wgpu_context: WgpuContext) -> Self { Self { window_size_receiver: Arc::new(Mutex::new(WindowSizeReceiver::new(window_size_receiver))), event_loop_proxy, + wgpu_context, } } } @@ -71,8 +73,44 @@ impl CefEventHandler for CefHandler { } *window_size } - fn draw(&self, frame_buffer: FrameBuffer) { - let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(frame_buffer)); + fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>) { + let width = frame_buffer.width() as u32; + let height = frame_buffer.height() as u32; + let texture = self.wgpu_context.device.create_texture(&wgpu::TextureDescriptor { + label: Some("CEF Texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Bgra8UnormSrgb, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.wgpu_context.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + frame_buffer.buffer(), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(4 * width), + rows_per_image: Some(height), + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(texture)); } fn schedule_cef_message_loop_work(&self, scheduled_time: std::time::Instant) { diff --git a/desktop/src/cef/internal/app.rs b/desktop/src/cef/internal/app.rs index a4d9d2e0..5da815d1 100644 --- a/desktop/src/cef/internal/app.rs +++ b/desktop/src/cef/internal/app.rs @@ -1,6 +1,6 @@ use cef::rc::{Rc, RcImpl}; use cef::sys::{_cef_app_t, cef_base_ref_counted_t}; -use cef::{BrowserProcessHandler, ImplApp, SchemeRegistrar, WrapApp}; +use cef::{BrowserProcessHandler, CefString, ImplApp, ImplCommandLine, SchemeRegistrar, WrapApp}; use crate::cef::CefEventHandler; use crate::cef::scheme_handler::GraphiteSchemeHandlerFactory; @@ -29,6 +29,14 @@ impl ImplApp for AppImpl { GraphiteSchemeHandlerFactory::register_schemes(registrar); } + fn on_before_command_line_processing(&self, _process_type: Option<&cef::CefString>, command_line: Option<&mut cef::CommandLine>) { + if let Some(cmd) = command_line { + // Disable GPU acceleration, because it is not supported for Offscreen Rendering and can cause crashes. + cmd.append_switch(Some(&CefString::from("disable-gpu"))); + cmd.append_switch(Some(&CefString::from("disable-gpu-compositing"))); + } + } + fn get_raw(&self) -> *mut _cef_app_t { self.object.cast() } diff --git a/desktop/src/cef/internal/browser_process_handler.rs b/desktop/src/cef/internal/browser_process_handler.rs index cd4cb175..5179dd3b 100644 --- a/desktop/src/cef/internal/browser_process_handler.rs +++ b/desktop/src/cef/internal/browser_process_handler.rs @@ -25,13 +25,13 @@ impl ImplBrowserProcessHandler for BrowserProcessHan cef::register_scheme_handler_factory(Some(&CefString::from(GRAPHITE_SCHEME)), None, Some(&mut SchemeHandlerFactory::new(GraphiteSchemeHandlerFactory::new()))); } - fn get_raw(&self) -> *mut _cef_browser_process_handler_t { - self.object.cast() - } - fn on_schedule_message_pump_work(&self, delay_ms: i64) { self.event_handler.schedule_cef_message_loop_work(Instant::now() + Duration::from_millis(delay_ms as u64)); } + + fn get_raw(&self) -> *mut _cef_browser_process_handler_t { + self.object.cast() + } } impl Clone for BrowserProcessHandlerImpl { diff --git a/desktop/src/cef/internal/render_handler.rs b/desktop/src/cef/internal/render_handler.rs index 9a0edacd..21d818a0 100644 --- a/desktop/src/cef/internal/render_handler.rs +++ b/desktop/src/cef/internal/render_handler.rs @@ -2,8 +2,8 @@ use cef::rc::{Rc, RcImpl}; use cef::sys::{_cef_render_handler_t, cef_base_ref_counted_t}; use cef::{Browser, ImplRenderHandler, PaintElementType, Rect, WrapRenderHandler}; -use crate::FrameBuffer; use crate::cef::CefEventHandler; +use crate::render::FrameBufferRef; pub(crate) struct RenderHandlerImpl { object: *mut RcImpl<_cef_render_handler_t, Self>, @@ -42,7 +42,7 @@ impl ImplRenderHandler for RenderHandlerImpl { ) { let buffer_size = (width * height * 4) as usize; let buffer_slice = unsafe { std::slice::from_raw_parts(buffer, buffer_size) }; - let frame_buffer = FrameBuffer::new(buffer_slice.to_vec(), width as usize, height as usize).expect("Failed to create frame buffer"); + let frame_buffer = FrameBufferRef::new(buffer_slice, width as usize, height as usize).expect("Failed to create frame buffer"); self.event_handler.draw(frame_buffer) } diff --git a/desktop/src/main.rs b/desktop/src/main.rs index f748b311..c53848f8 100644 --- a/desktop/src/main.rs +++ b/desktop/src/main.rs @@ -9,7 +9,7 @@ mod cef; use cef::{Setup, WindowSize}; mod render; -use render::FrameBuffer; +use render::WgpuContext; mod app; use app::WinitApp; @@ -18,7 +18,7 @@ mod dirs; #[derive(Debug)] pub(crate) enum CustomEvent { - UiUpdate(FrameBuffer), + UiUpdate(wgpu::Texture), ScheduleBrowserWork(Instant), } @@ -38,7 +38,8 @@ fn main() { let (window_size_sender, window_size_receiver) = std::sync::mpsc::channel(); - let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy())) { + let wgpu_context = futures::executor::block_on(WgpuContext::new()); + let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy(), wgpu_context.clone())) { Ok(c) => c, Err(cef::InitError::InitializationFailed) => { tracing::error!("Cef initialization failed"); @@ -48,7 +49,7 @@ fn main() { tracing::info!("Cef initialized successfully"); - let mut winit_app = WinitApp::new(cef_context, window_size_sender, event_loop.create_proxy()); + let mut winit_app = WinitApp::new(cef_context, window_size_sender, wgpu_context); event_loop.run_app(&mut winit_app).unwrap(); } diff --git a/desktop/src/render.rs b/desktop/src/render.rs index 6d3755a8..51415446 100644 --- a/desktop/src/render.rs +++ b/desktop/src/render.rs @@ -3,36 +3,19 @@ use std::sync::Arc; use thiserror::Error; use winit::window::Window; -pub(crate) struct FrameBuffer { - buffer: Vec, +pub(crate) struct FrameBufferRef<'a> { + buffer: &'a [u8], width: usize, height: usize, } -impl std::fmt::Debug for FrameBuffer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FrameBuffer") - .field("width", &self.width) - .field("height", &self.height) - .field("len", &self.buffer.len()) - .finish() - } -} - -#[derive(Error, Debug)] -pub(crate) enum FrameBufferError { - #[error("Invalid buffer size {buffer_size}, expected {expected_size} for width {width} multiplied with height {height} multiplied by 4 channels")] - InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize }, -} - -impl FrameBuffer { - pub(crate) fn new(buffer: Vec, width: usize, height: usize) -> Result { +impl<'a> FrameBufferRef<'a> { + pub(crate) fn new(buffer: &'a [u8], width: usize, height: usize) -> Result { let fb = Self { buffer, width, height }; fb.validate_size()?; Ok(fb) } - pub(crate) fn buffer(&self) -> &[u8] { - &self.buffer + self.buffer } pub(crate) fn width(&self) -> usize { @@ -56,34 +39,41 @@ impl FrameBuffer { } } } - -#[derive(Debug)] -pub(crate) struct GraphicsState { - surface: wgpu::Surface<'static>, - device: wgpu::Device, - queue: wgpu::Queue, - config: wgpu::SurfaceConfiguration, - texture: Option, - bind_group: Option, - render_pipeline: wgpu::RenderPipeline, - sampler: wgpu::Sampler, +impl<'a> std::fmt::Debug for FrameBufferRef<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FrameBuffer") + .field("width", &self.width) + .field("height", &self.height) + .field("len", &self.buffer.len()) + .finish() + } } -impl GraphicsState { - pub(crate) async fn new(window: Arc) -> Self { - let size = window.inner_size(); +#[derive(Error, Debug)] +pub(crate) enum FrameBufferError { + #[error("Invalid buffer size {buffer_size}, expected {expected_size} for width {width} multiplied with height {height} multiplied by 4 channels")] + InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize }, +} +#[derive(Debug, Clone)] +pub(crate) struct WgpuContext { + pub(crate) device: wgpu::Device, + pub(crate) queue: wgpu::Queue, + adapter: wgpu::Adapter, + instance: wgpu::Instance, +} + +impl WgpuContext { + pub(crate) async fn new() -> Self { let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor { backends: wgpu::Backends::PRIMARY, ..Default::default() }); - let surface = instance.create_surface(window).unwrap(); - let adapter = instance .request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::default(), - compatible_surface: Some(&surface), + compatible_surface: None, force_fallback_adapter: false, }) .await @@ -100,7 +90,28 @@ impl GraphicsState { .await .unwrap(); - let surface_caps = surface.get_capabilities(&adapter); + Self { device, queue, adapter, instance } + } +} + +#[derive(Debug)] +pub(crate) struct GraphicsState { + surface: wgpu::Surface<'static>, + context: WgpuContext, + config: wgpu::SurfaceConfiguration, + texture: Option, + bind_group: Option, + render_pipeline: wgpu::RenderPipeline, + sampler: wgpu::Sampler, +} + +impl GraphicsState { + pub(crate) fn new(window: Arc, context: WgpuContext) -> Self { + let size = window.inner_size(); + + let surface = context.instance.create_surface(window).unwrap(); + + let surface_caps = surface.get_capabilities(&context.adapter); let surface_format = surface_caps.formats.iter().find(|f| f.is_srgb()).copied().unwrap_or(surface_caps.formats[0]); let config = wgpu::SurfaceConfiguration { @@ -114,13 +125,13 @@ impl GraphicsState { desired_maximum_frame_latency: 2, }; - surface.configure(&device, &config); + surface.configure(&context.device, &config); // Create shader module - let shader = device.create_shader_module(wgpu::include_wgsl!("render/fullscreen_texture.wgsl")); + let shader = context.device.create_shader_module(wgpu::include_wgsl!("render/fullscreen_texture.wgsl")); // Create sampler - let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + let sampler = context.device.create_sampler(&wgpu::SamplerDescriptor { address_mode_u: wgpu::AddressMode::ClampToEdge, address_mode_v: wgpu::AddressMode::ClampToEdge, address_mode_w: wgpu::AddressMode::ClampToEdge, @@ -130,7 +141,7 @@ impl GraphicsState { ..Default::default() }); - let texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + let texture_bind_group_layout = context.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { entries: &[ wgpu::BindGroupLayoutEntry { binding: 0, @@ -152,13 +163,13 @@ impl GraphicsState { label: Some("texture_bind_group_layout"), }); - let render_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + let render_pipeline_layout = context.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Render Pipeline Layout"), bind_group_layouts: &[&texture_bind_group_layout], push_constant_ranges: &[], }); - let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + let render_pipeline = context.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Render Pipeline"), layout: Some(&render_pipeline_layout), vertex: wgpu::VertexState { @@ -196,94 +207,36 @@ impl GraphicsState { cache: None, }); - let mut graphics_state = Self { + Self { surface, - device, - queue, + context, config, texture: None, bind_group: None, render_pipeline, sampler, - }; - - // Initialize with a test pattern so we always have something to render - let width = 800; - let height = 600; - let initial_data = vec![34u8; width * height * 4]; // Gray texture #222222FF - - let fb = FrameBuffer::new(initial_data, width, height) - .map_err(|e| { - panic!("Failed to create initial FrameBuffer: {e}"); - }) - .unwrap(); - - graphics_state.update_texture(&fb); - - graphics_state + } } - pub(crate) fn ui_texture_outdated(&self, frame_buffer: &FrameBuffer) -> bool { - let width = frame_buffer.width() as u32; - let height = frame_buffer.height() as u32; - - self.config.width != width || self.config.height != height - } pub(crate) fn resize(&mut self, width: u32, height: u32) { if width > 0 && height > 0 && (self.config.width != width || self.config.height != height) { self.config.width = width; self.config.height = height; - self.surface.configure(&self.device, &self.config); - let texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("CEF Texture"), - size: wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Bgra8UnormSrgb, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - self.texture = Some(texture); + self.surface.configure(&self.context.device, &self.config); } } - pub(crate) fn update_texture(&mut self, frame_buffer: &FrameBuffer) { - let data = frame_buffer.buffer(); - let width = frame_buffer.width() as u32; - let height = frame_buffer.height() as u32; + pub(crate) fn bind_texture(&mut self, texture: &wgpu::Texture) { + let bind_group = self.create_bindgroup(texture); + self.texture = Some(texture.clone()); - self.resize(width, height); - - let Some(ref texture) = self.texture else { return }; - - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - data, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(4 * width), - rows_per_image: Some(height), - }, - wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - ); + self.bind_group = Some(bind_group); + } + fn create_bindgroup(&self, texture: &wgpu::Texture) -> wgpu::BindGroup { let texture_view = texture.create_view(&wgpu::TextureViewDescriptor::default()); - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + self.context.device.create_bind_group(&wgpu::BindGroupDescriptor { layout: &self.render_pipeline.get_bind_group_layout(0), entries: &[ wgpu::BindGroupEntry { @@ -296,16 +249,14 @@ impl GraphicsState { }, ], label: Some("texture_bind_group"), - }); - - self.bind_group = Some(bind_group); + }) } pub(crate) fn render(&mut self) -> Result<(), wgpu::SurfaceError> { let output = self.surface.get_current_texture()?; let view = output.texture.create_view(&wgpu::TextureViewDescriptor::default()); - let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("Render Encoder") }); + let mut encoder = self.context.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("Render Encoder") }); { let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -331,7 +282,7 @@ impl GraphicsState { tracing::warn!("No bind group available - showing clear color only"); } } - self.queue.submit(std::iter::once(encoder.finish())); + self.context.queue.submit(std::iter::once(encoder.finish())); output.present(); Ok(())