jess
/
Graphite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Desktop: Directly upload frame buffer (#2930) 

* Upload frame buffer directly to gpu texture

* Disable cef GPU acceleration to prevent crashes

* Cleanup code

* Address review comments

---------

Co-authored-by: Timon Schelling <me@timon.zip>
This commit is contained in:
Dennis Kobert 2025-07-25 14:04:46 +02:00 committed by GitHub
parent 735d58a647
commit a52ee70e4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 150 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
desktop/src/app.rs
View File

@ -1,7 +1,7 @@
use crate::CustomEvent;
use crate::FrameBuffer;
use crate::WindowSize;
use crate::render::GraphicsState;
use crate::render::WgpuContext;
use std::sync::Arc;
use std::sync::mpsc::Sender;
use std::time::Duration;
@ -12,7 +12,6 @@ use winit::event::StartCause;
use winit::event::WindowEvent;
use winit::event_loop::ActiveEventLoop;
use winit::event_loop::ControlFlow;
use winit::event_loop::EventLoopProxy;
use winit::window::Window;
use winit::window::WindowId;
@ -22,24 +21,24 @@ pub(crate) struct WinitApp {
pub(crate) cef_context: cef::Context<cef::Initialized>,
pub(crate) window: Option<Arc<Window>>,
cef_schedule: Option<Instant>,
ui_frame_buffer: Option<FrameBuffer>,
_ui_frame_buffer: Option<wgpu::Texture>,
window_size_sender: Sender<WindowSize>,
_viewport_frame_buffer: Option<FrameBuffer>,
_viewport_frame_buffer: Option<wgpu::Texture>,
graphics_state: Option<GraphicsState>,
event_loop_proxy: EventLoopProxy<CustomEvent>,
wgpu_context: WgpuContext,
}
impl WinitApp {
pub(crate) fn new(cef_context: cef::Context<cef::Initialized>, window_size_sender: Sender<WindowSize>, event_loop_proxy: EventLoopProxy<CustomEvent>) -> Self {
pub(crate) fn new(cef_context: cef::Context<cef::Initialized>, window_size_sender: Sender<WindowSize>, wgpu_context: WgpuContext) -> Self {
Self {
cef_context,
window: None,
cef_schedule: Some(Instant::now()),
_viewport_frame_buffer: None,
ui_frame_buffer: None,
_ui_frame_buffer: None,
graphics_state: None,
window_size_sender,
event_loop_proxy,
wgpu_context,
}
}
}
@ -47,8 +46,9 @@ impl WinitApp {
impl ApplicationHandler<CustomEvent> for WinitApp {
fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) {
// Set a timeout in case we miss any cef schedule requests
let timeout = Instant::now() + Duration::from_millis(100);
let timeout = Instant::now() + Duration::from_millis(10);
let wait_until = timeout.min(self.cef_schedule.unwrap_or(timeout));
self.cef_context.work();
event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until));
}
@ -71,7 +71,7 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
)
.unwrap(),
);
let graphics_state = futures::executor::block_on(GraphicsState::new(window.clone()));
let graphics_state = GraphicsState::new(window.clone(), self.wgpu_context.clone());
self.window = Some(window);
self.graphics_state = Some(graphics_state);
@ -81,27 +81,24 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
fn user_event(&mut self, _: &ActiveEventLoop, event: CustomEvent) {
match event {
CustomEvent::UiUpdate(frame_buffer) => {
CustomEvent::UiUpdate(texture) => {
if let Some(graphics_state) = self.graphics_state.as_mut() {
graphics_state.update_texture(&frame_buffer);
graphics_state.bind_texture(&texture);
graphics_state.resize(texture.width(), texture.height());
}
self.ui_frame_buffer = Some(frame_buffer);
if let Some(window) = &self.window {
window.request_redraw();
}
}
CustomEvent::ScheduleBrowserWork(instant) => {
if let Some(graphics_state) = self.graphics_state.as_mut()
&& let Some(frame_buffer) = &self.ui_frame_buffer
&& graphics_state.ui_texture_outdated(frame_buffer)
{
if instant <= Instant::now() {
self.cef_context.work();
let _ = self.event_loop_proxy.send_event(CustomEvent::ScheduleBrowserWork(Instant::now() + Duration::from_millis(1)));
}
} else {
self.cef_schedule = Some(instant);
}
}
}
}
fn window_event(&mut self, event_loop: &ActiveEventLoop, _window_id: WindowId, event: WindowEvent) {
let Some(event) = self.cef_context.handle_window_event(event) else { return };
@ -113,9 +110,6 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
}
WindowEvent::Resized(PhysicalSize { width, height }) => {
let _ = self.window_size_sender.send(WindowSize::new(width as usize, height as usize));
if let Some(ref mut graphics_state) = self.graphics_state {
graphics_state.resize(width, height);
}
self.cef_context.notify_of_resize();
}

48
desktop/src/cef.rs
View File

@ -1,4 +1,4 @@
use crate::{CustomEvent, FrameBuffer};
use crate::{CustomEvent, WgpuContext, render::FrameBufferRef};
use std::{
sync::{Arc, Mutex, mpsc::Receiver},
time::Instant,
@ -15,7 +15,7 @@ use winit::event_loop::EventLoopProxy;
pub(crate) trait CefEventHandler: Clone {
fn window_size(&self) -> WindowSize;
fn draw(&self, frame_buffer: FrameBuffer);
fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>);
/// Scheudule the main event loop to run the cef event loop after the timeout
/// [`_cef_browser_process_handler_t::on_schedule_message_pump_work`] for more documentation.
fn schedule_cef_message_loop_work(&self, scheduled_time: Instant);
@ -37,6 +37,7 @@ impl WindowSize {
pub(crate) struct CefHandler {
window_size_receiver: Arc<Mutex<WindowSizeReceiver>>,
event_loop_proxy: EventLoopProxy<CustomEvent>,
wgpu_context: WgpuContext,
}
struct WindowSizeReceiver {
receiver: Receiver<WindowSize>,
@ -51,10 +52,11 @@ impl WindowSizeReceiver {
}
}
impl CefHandler {
pub(crate) fn new(window_size_receiver: Receiver<WindowSize>, event_loop_proxy: EventLoopProxy<CustomEvent>) -> Self {
pub(crate) fn new(window_size_receiver: Receiver<WindowSize>, event_loop_proxy: EventLoopProxy<CustomEvent>, wgpu_context: WgpuContext) -> Self {
Self {
window_size_receiver: Arc::new(Mutex::new(WindowSizeReceiver::new(window_size_receiver))),
event_loop_proxy,
wgpu_context,
}
}
}
@ -71,8 +73,44 @@ impl CefEventHandler for CefHandler {
}
*window_size
}
fn draw(&self, frame_buffer: FrameBuffer) {
let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(frame_buffer));
fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>) {
let width = frame_buffer.width() as u32;
let height = frame_buffer.height() as u32;
let texture = self.wgpu_context.device.create_texture(&wgpu::TextureDescriptor {
label: Some("CEF Texture"),
size: wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Bgra8UnormSrgb,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
self.wgpu_context.queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
frame_buffer.buffer(),
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(4 * width),
rows_per_image: Some(height),
},
wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
);
let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(texture));
}
fn schedule_cef_message_loop_work(&self, scheduled_time: std::time::Instant) {

10
desktop/src/cef/internal/app.rs
View File

@ -1,6 +1,6 @@
use cef::rc::{Rc, RcImpl};
use cef::sys::{_cef_app_t, cef_base_ref_counted_t};
use cef::{BrowserProcessHandler, ImplApp, SchemeRegistrar, WrapApp};
use cef::{BrowserProcessHandler, CefString, ImplApp, ImplCommandLine, SchemeRegistrar, WrapApp};
use crate::cef::CefEventHandler;
use crate::cef::scheme_handler::GraphiteSchemeHandlerFactory;
@ -29,6 +29,14 @@ impl<H: CefEventHandler + Clone> ImplApp for AppImpl<H> {
GraphiteSchemeHandlerFactory::register_schemes(registrar);
}
fn on_before_command_line_processing(&self, _process_type: Option<&cef::CefString>, command_line: Option<&mut cef::CommandLine>) {
if let Some(cmd) = command_line {
// Disable GPU acceleration, because it is not supported for Offscreen Rendering and can cause crashes.
cmd.append_switch(Some(&CefString::from("disable-gpu")));
cmd.append_switch(Some(&CefString::from("disable-gpu-compositing")));
}
}
fn get_raw(&self) -> *mut _cef_app_t {
self.object.cast()
}

8
desktop/src/cef/internal/browser_process_handler.rs
View File

@ -25,13 +25,13 @@ impl<H: CefEventHandler + Clone> ImplBrowserProcessHandler for BrowserProcessHan
cef::register_scheme_handler_factory(Some(&CefString::from(GRAPHITE_SCHEME)), None, Some(&mut SchemeHandlerFactory::new(GraphiteSchemeHandlerFactory::new())));
}
fn get_raw(&self) -> *mut _cef_browser_process_handler_t {
self.object.cast()
}
fn on_schedule_message_pump_work(&self, delay_ms: i64) {
self.event_handler.schedule_cef_message_loop_work(Instant::now() + Duration::from_millis(delay_ms as u64));
}
fn get_raw(&self) -> *mut _cef_browser_process_handler_t {
self.object.cast()
}
}
impl<H: CefEventHandler + Clone> Clone for BrowserProcessHandlerImpl<H> {

4
desktop/src/cef/internal/render_handler.rs
View File

@ -2,8 +2,8 @@ use cef::rc::{Rc, RcImpl};
use cef::sys::{_cef_render_handler_t, cef_base_ref_counted_t};
use cef::{Browser, ImplRenderHandler, PaintElementType, Rect, WrapRenderHandler};
use crate::FrameBuffer;
use crate::cef::CefEventHandler;
use crate::render::FrameBufferRef;
pub(crate) struct RenderHandlerImpl<H: CefEventHandler> {
object: *mut RcImpl<_cef_render_handler_t, Self>,
@ -42,7 +42,7 @@ impl<H: CefEventHandler> ImplRenderHandler for RenderHandlerImpl<H> {
) {
let buffer_size = (width * height * 4) as usize;
let buffer_slice = unsafe { std::slice::from_raw_parts(buffer, buffer_size) };
let frame_buffer = FrameBuffer::new(buffer_slice.to_vec(), width as usize, height as usize).expect("Failed to create frame buffer");
let frame_buffer = FrameBufferRef::new(buffer_slice, width as usize, height as usize).expect("Failed to create frame buffer");
self.event_handler.draw(frame_buffer)
}

9
desktop/src/main.rs
View File

@ -9,7 +9,7 @@ mod cef;
use cef::{Setup, WindowSize};
mod render;
use render::FrameBuffer;
use render::WgpuContext;
mod app;
use app::WinitApp;
@ -18,7 +18,7 @@ mod dirs;
#[derive(Debug)]
pub(crate) enum CustomEvent {
UiUpdate(FrameBuffer),
UiUpdate(wgpu::Texture),
ScheduleBrowserWork(Instant),
}
@ -38,7 +38,8 @@ fn main() {
let (window_size_sender, window_size_receiver) = std::sync::mpsc::channel();
let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy())) {
let wgpu_context = futures::executor::block_on(WgpuContext::new());
let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy(), wgpu_context.clone())) {
Ok(c) => c,
Err(cef::InitError::InitializationFailed) => {
tracing::error!("Cef initialization failed");
@ -48,7 +49,7 @@ fn main() {
tracing::info!("Cef initialized successfully");
let mut winit_app = WinitApp::new(cef_context, window_size_sender, event_loop.create_proxy());
let mut winit_app = WinitApp::new(cef_context, window_size_sender, wgpu_context);
event_loop.run_app(&mut winit_app).unwrap();
}

191
desktop/src/render.rs
View File

@ -3,36 +3,19 @@ use std::sync::Arc;
use thiserror::Error;
use winit::window::Window;
pub(crate) struct FrameBuffer {
buffer: Vec<u8>,
pub(crate) struct FrameBufferRef<'a> {
buffer: &'a [u8],
width: usize,
height: usize,
}
impl std::fmt::Debug for FrameBuffer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FrameBuffer")
.field("width", &self.width)
.field("height", &self.height)
.field("len", &self.buffer.len())
.finish()
}
}
#[derive(Error, Debug)]
pub(crate) enum FrameBufferError {
#[error("Invalid buffer size {buffer_size}, expected {expected_size} for width {width} multiplied with height {height} multiplied by 4 channels")]
InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize },
}
impl FrameBuffer {
pub(crate) fn new(buffer: Vec<u8>, width: usize, height: usize) -> Result<Self, FrameBufferError> {
impl<'a> FrameBufferRef<'a> {
pub(crate) fn new(buffer: &'a [u8], width: usize, height: usize) -> Result<Self, FrameBufferError> {
let fb = Self { buffer, width, height };
fb.validate_size()?;
Ok(fb)
}
pub(crate) fn buffer(&self) -> &[u8] {
&self.buffer
self.buffer
}
pub(crate) fn width(&self) -> usize {
@ -56,34 +39,41 @@ impl FrameBuffer {
}
}
}
#[derive(Debug)]
pub(crate) struct GraphicsState {
surface: wgpu::Surface<'static>,
device: wgpu::Device,
queue: wgpu::Queue,
config: wgpu::SurfaceConfiguration,
texture: Option<wgpu::Texture>,
bind_group: Option<wgpu::BindGroup>,
render_pipeline: wgpu::RenderPipeline,
sampler: wgpu::Sampler,
impl<'a> std::fmt::Debug for FrameBufferRef<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FrameBuffer")
.field("width", &self.width)
.field("height", &self.height)
.field("len", &self.buffer.len())
.finish()
}
}
impl GraphicsState {
pub(crate) async fn new(window: Arc<Window>) -> Self {
let size = window.inner_size();
#[derive(Error, Debug)]
pub(crate) enum FrameBufferError {
#[error("Invalid buffer size {buffer_size}, expected {expected_size} for width {width} multiplied with height {height} multiplied by 4 channels")]
InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize },
}
#[derive(Debug, Clone)]
pub(crate) struct WgpuContext {
pub(crate) device: wgpu::Device,
pub(crate) queue: wgpu::Queue,
adapter: wgpu::Adapter,
instance: wgpu::Instance,
}
impl WgpuContext {
pub(crate) async fn new() -> Self {
let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor {
backends: wgpu::Backends::PRIMARY,
..Default::default()
});
let surface = instance.create_surface(window).unwrap();
let adapter = instance
.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::default(),
compatible_surface: Some(&surface),
compatible_surface: None,
force_fallback_adapter: false,
})
.await
@ -100,7 +90,28 @@ impl GraphicsState {
.await
.unwrap();
let surface_caps = surface.get_capabilities(&adapter);
Self { device, queue, adapter, instance }
}
}
#[derive(Debug)]
pub(crate) struct GraphicsState {
surface: wgpu::Surface<'static>,
context: WgpuContext,
config: wgpu::SurfaceConfiguration,
texture: Option<wgpu::Texture>,
bind_group: Option<wgpu::BindGroup>,
render_pipeline: wgpu::RenderPipeline,
sampler: wgpu::Sampler,
}
impl GraphicsState {
pub(crate) fn new(window: Arc<Window>, context: WgpuContext) -> Self {
let size = window.inner_size();
let surface = context.instance.create_surface(window).unwrap();
let surface_caps = surface.get_capabilities(&context.adapter);
let surface_format = surface_caps.formats.iter().find(|f| f.is_srgb()).copied().unwrap_or(surface_caps.formats[0]);
let config = wgpu::SurfaceConfiguration {
@ -114,13 +125,13 @@ impl GraphicsState {
desired_maximum_frame_latency: 2,
};
surface.configure(&device, &config);
surface.configure(&context.device, &config);
// Create shader module
let shader = device.create_shader_module(wgpu::include_wgsl!("render/fullscreen_texture.wgsl"));
let shader = context.device.create_shader_module(wgpu::include_wgsl!("render/fullscreen_texture.wgsl"));
// Create sampler
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
let sampler = context.device.create_sampler(&wgpu::SamplerDescriptor {
address_mode_u: wgpu::AddressMode::ClampToEdge,
address_mode_v: wgpu::AddressMode::ClampToEdge,
address_mode_w: wgpu::AddressMode::ClampToEdge,
@ -130,7 +141,7 @@ impl GraphicsState {
..Default::default()
});
let texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
let texture_bind_group_layout = context.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
@ -152,13 +163,13 @@ impl GraphicsState {
label: Some("texture_bind_group_layout"),
});
let render_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
let render_pipeline_layout = context.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Render Pipeline Layout"),
bind_group_layouts: &[&texture_bind_group_layout],
push_constant_ranges: &[],
});
let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
let render_pipeline = context.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("Render Pipeline"),
layout: Some(&render_pipeline_layout),
vertex: wgpu::VertexState {
@ -196,94 +207,36 @@ impl GraphicsState {
cache: None,
});
let mut graphics_state = Self {
Self {
surface,
device,
queue,
context,
config,
texture: None,
bind_group: None,
render_pipeline,
sampler,
};
// Initialize with a test pattern so we always have something to render
let width = 800;
let height = 600;
let initial_data = vec![34u8; width * height * 4]; // Gray texture #222222FF
let fb = FrameBuffer::new(initial_data, width, height)
.map_err(|e| {
panic!("Failed to create initial FrameBuffer: {e}");
})
.unwrap();
graphics_state.update_texture(&fb);
graphics_state
}
}
pub(crate) fn ui_texture_outdated(&self, frame_buffer: &FrameBuffer) -> bool {
let width = frame_buffer.width() as u32;
let height = frame_buffer.height() as u32;
self.config.width != width || self.config.height != height
}
pub(crate) fn resize(&mut self, width: u32, height: u32) {
if width > 0 && height > 0 && (self.config.width != width || self.config.height != height) {
self.config.width = width;
self.config.height = height;
self.surface.configure(&self.device, &self.config);
let texture = self.device.create_texture(&wgpu::TextureDescriptor {
label: Some("CEF Texture"),
size: wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Bgra8UnormSrgb,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
self.texture = Some(texture);
self.surface.configure(&self.context.device, &self.config);
}
}
pub(crate) fn update_texture(&mut self, frame_buffer: &FrameBuffer) {
let data = frame_buffer.buffer();
let width = frame_buffer.width() as u32;
let height = frame_buffer.height() as u32;
pub(crate) fn bind_texture(&mut self, texture: &wgpu::Texture) {
let bind_group = self.create_bindgroup(texture);
self.texture = Some(texture.clone());
self.resize(width, height);
let Some(ref texture) = self.texture else { return };
self.queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
data,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(4 * width),
rows_per_image: Some(height),
},
wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
);
self.bind_group = Some(bind_group);
}
fn create_bindgroup(&self, texture: &wgpu::Texture) -> wgpu::BindGroup {
let texture_view = texture.create_view(&wgpu::TextureViewDescriptor::default());
let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
self.context.device.create_bind_group(&wgpu::BindGroupDescriptor {
layout: &self.render_pipeline.get_bind_group_layout(0),
entries: &[
wgpu::BindGroupEntry {
@ -296,16 +249,14 @@ impl GraphicsState {
},
],
label: Some("texture_bind_group"),
});
self.bind_group = Some(bind_group);
})
}
pub(crate) fn render(&mut self) -> Result<(), wgpu::SurfaceError> {
let output = self.surface.get_current_texture()?;
let view = output.texture.create_view(&wgpu::TextureViewDescriptor::default());
let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("Render Encoder") });
let mut encoder = self.context.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("Render Encoder") });
{
let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
@ -331,7 +282,7 @@ impl GraphicsState {
tracing::warn!("No bind group available - showing clear color only");
}
}
self.queue.submit(std::iter::once(encoder.finish()));
self.context.queue.submit(std::iter::once(encoder.finish()));
output.present();
Ok(())