Layers/src/wayland_shell.rs

use crate::ffi::ViewportHandle;
use raw_window_handle::{
    RawDisplayHandle, RawWindowHandle, WaylandDisplayHandle, WaylandWindowHandle,
};
use smithay_client_toolkit::{
    compositor::{CompositorHandler, CompositorState},
    delegate_compositor, delegate_keyboard, delegate_layer, delegate_output, delegate_pointer,
    delegate_registry, delegate_seat,
    output::{OutputHandler, OutputState},
    registry::{ProvidesRegistryState, RegistryState},
    registry_handlers,
    seat::{
        keyboard::{KeyEvent, KeyboardHandler, Keysym, Modifiers, RawModifiers},
        pointer::{PointerEvent, PointerEventKind, PointerHandler},
        Capability, SeatHandler, SeatState,
    },
    shell::{
        wlr_layer::{
            Anchor, KeyboardInteractivity, Layer, LayerShell, LayerShellHandler, LayerSurface,
            LayerSurfaceConfigure,
        },
        WaylandSurface,
    },
};
use std::ptr::NonNull;
use wayland_client::{
    globals::registry_queue_init,
    protocol::{wl_keyboard, wl_output, wl_pointer, wl_seat, wl_surface},
    Connection, Proxy, QueueHandle,
};

const DEFAULT_LOGICAL_SIZE: (u32, u32) = (480, 640);
const MIN_LOGICAL_SIZE: (u32, u32) = (380, 220);
const ANCHOR_MARGIN: i32 = 24;

/// Try to run the Wayland (wlr-layer-shell) shell. Returns `true` if Wayland was used
/// — successfully started or not. Returns `false` only when no Wayland session was
/// available or the compositor doesn't advertise wlr-layer-shell, so the caller can
/// fall through to the winit/X11 path.
pub fn try_run() -> bool {
    let conn = match Connection::connect_to_env() {
        Ok(c) => c,
        Err(_) => {
            tracing::info!("wayland: no WAYLAND_DISPLAY, falling through to X11");
            return false;
        }
    };

    let (globals, mut event_queue) = match registry_queue_init::<State>(&conn) {
        Ok(pair) => pair,
        Err(e) => {
            tracing::warn!("wayland: registry init failed ({e}); falling through to X11");
            return false;
        }
    };
    let qh = event_queue.handle();

    let compositor = match CompositorState::bind(&globals, &qh) {
        Ok(c) => c,
        Err(_) => {
            tracing::warn!("wayland: wl_compositor missing; falling through to X11");
            return false;
        }
    };

    let layer_shell = match LayerShell::bind(&globals, &qh) {
        Ok(l) => l,
        Err(_) => {
            tracing::warn!(
                "wayland: zwlr_layer_shell_v1 not advertised (likely GNOME/mutter); falling through to X11"
            );
            return false;
        }
    };

    let surface = compositor.create_surface(&qh);
    let layer = layer_shell.create_layer_surface(
        &qh,
        surface,
        Layer::Top,
        Some("layers"),
        None,
    );
    layer.set_anchor(Anchor::TOP | Anchor::RIGHT);
    layer.set_size(DEFAULT_LOGICAL_SIZE.0, DEFAULT_LOGICAL_SIZE.1);
    layer.set_margin(ANCHOR_MARGIN, ANCHOR_MARGIN, 0, 0);
    layer.set_keyboard_interactivity(KeyboardInteractivity::OnDemand);
    layer.commit();

    let raw_display = RawDisplayHandle::Wayland(WaylandDisplayHandle::new(
        match NonNull::new(conn.backend().display_ptr() as *mut _) {
            Some(p) => p,
            None => {
                tracing::error!("wayland: null display ptr");
                return true;
            }
        },
    ));
    let raw_window = RawWindowHandle::Wayland(WaylandWindowHandle::new(
        match NonNull::new(layer.wl_surface().id().as_ptr() as *mut _) {
            Some(p) => p,
            None => {
                tracing::error!("wayland: null surface ptr");
                return true;
            }
        },
    ));

    let mut state = State {
        registry_state: RegistryState::new(&globals),
        seat_state: SeatState::new(&globals, &qh),
        output_state: OutputState::new(&globals, &qh),
        viewport: None,
        layer,
        keyboard: None,
        pointer: None,
        modifiers: Modifiers::default(),
        last_pointer: (0.0, 0.0),
        width: DEFAULT_LOGICAL_SIZE.0,
        height: DEFAULT_LOGICAL_SIZE.1,
        scale: 1,
        first_configure: true,
        raw_display,
        raw_window,
        exit: false,
    };

    tracing::info!("wayland: layer-shell up, entering dispatch loop");
    while !state.exit {
        if let Err(e) = event_queue.blocking_dispatch(&mut state) {
            tracing::error!("wayland dispatch: {e}");
            break;
        }
    }

    true
}

struct State {
    registry_state: RegistryState,
    seat_state: SeatState,
    output_state: OutputState,
    /// Held above `layer` so the wgpu surface drops before the wl_surface.
    viewport: Option<ViewportHandle>,
    layer: LayerSurface,
    keyboard: Option<wl_keyboard::WlKeyboard>,
    pointer: Option<wl_pointer::WlPointer>,
    modifiers: Modifiers,
    last_pointer: (f32, f32),
    width: u32,
    height: u32,
    scale: i32,
    first_configure: bool,
    raw_display: RawDisplayHandle,
    raw_window: RawWindowHandle,
    exit: bool,
}

impl State {
    fn ensure_viewport(&mut self) {
        if self.viewport.is_some() {
            return;
        }
        let scale = self.scale.max(1) as f32;
        let logical_w = self.width.max(MIN_LOGICAL_SIZE.0) as f32;
        let logical_h = self.height.max(MIN_LOGICAL_SIZE.1) as f32;
        match ViewportHandle::new_from_raw(self.raw_window, self.raw_display, logical_w, logical_h, scale) {
            Some(h) => {
                tracing::info!(
                    "wayland: viewport ready ({}x{} @ {})",
                    logical_w,
                    logical_h,
                    scale
                );
                self.viewport = Some(h);
            }
            None => tracing::error!("wayland: ViewportHandle::new_from_raw returned None"),
        }
    }

    fn render(&mut self, qh: &QueueHandle<Self>) {
        if let Some(vp) = self.viewport.as_mut() {
            vp.render_frame();
        }
        let surface = self.layer.wl_surface();
        surface.frame(qh, surface.clone());
        self.layer.commit();
    }
}

impl CompositorHandler for State {
    fn scale_factor_changed(
        &mut self,
        _conn: &Connection,
        _qh: &QueueHandle<Self>,
        _surface: &wl_surface::WlSurface,
        new_factor: i32,
    ) {
        if self.scale == new_factor {
            return;
        }
        self.scale = new_factor;
        self.layer.wl_surface().set_buffer_scale(new_factor);
        if let Some(vp) = self.viewport.as_mut() {
            vp.resize_px(self.width as f32, self.height as f32, new_factor as f32);
        }
    }

    fn transform_changed(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_surface::WlSurface,
        _: wl_output::Transform,
    ) {
    }

    fn frame(
        &mut self,
        _conn: &Connection,
        qh: &QueueHandle<Self>,
        _: &wl_surface::WlSurface,
        _time: u32,
    ) {
        self.render(qh);
    }

    fn surface_enter(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_surface::WlSurface,
        _: &wl_output::WlOutput,
    ) {
    }

    fn surface_leave(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_surface::WlSurface,
        _: &wl_output::WlOutput,
    ) {
    }
}

impl OutputHandler for State {
    fn output_state(&mut self) -> &mut OutputState {
        &mut self.output_state
    }
    fn new_output(&mut self, _: &Connection, _: &QueueHandle<Self>, _: wl_output::WlOutput) {}
    fn update_output(&mut self, _: &Connection, _: &QueueHandle<Self>, _: wl_output::WlOutput) {}
    fn output_destroyed(&mut self, _: &Connection, _: &QueueHandle<Self>, _: wl_output::WlOutput) {}
}

impl LayerShellHandler for State {
    fn closed(&mut self, _: &Connection, _: &QueueHandle<Self>, _: &LayerSurface) {
        self.exit = true;
    }

    fn configure(
        &mut self,
        _conn: &Connection,
        qh: &QueueHandle<Self>,
        _layer: &LayerSurface,
        configure: LayerSurfaceConfigure,
        _serial: u32,
    ) {
        let (w, h) = configure.new_size;
        let new_w = if w == 0 { self.width } else { w };
        let new_h = if h == 0 { self.height } else { h };
        let resized = new_w != self.width || new_h != self.height;
        self.width = new_w;
        self.height = new_h;

        self.ensure_viewport();
        if resized {
            if let Some(vp) = self.viewport.as_mut() {
                vp.resize_px(self.width as f32, self.height as f32, self.scale.max(1) as f32);
            }
        }

        if self.first_configure {
            self.first_configure = false;
            self.render(qh);
        }
    }
}

impl SeatHandler for State {
    fn seat_state(&mut self) -> &mut SeatState {
        &mut self.seat_state
    }
    fn new_seat(&mut self, _: &Connection, _: &QueueHandle<Self>, _: wl_seat::WlSeat) {}

    fn new_capability(
        &mut self,
        _conn: &Connection,
        qh: &QueueHandle<Self>,
        seat: wl_seat::WlSeat,
        capability: Capability,
    ) {
        if capability == Capability::Keyboard && self.keyboard.is_none() {
            if let Ok(kb) = self.seat_state.get_keyboard(qh, &seat, None) {
                self.keyboard = Some(kb);
            }
        }
        if capability == Capability::Pointer && self.pointer.is_none() {
            if let Ok(p) = self.seat_state.get_pointer(qh, &seat) {
                self.pointer = Some(p);
            }
        }
    }

    fn remove_capability(
        &mut self,
        _conn: &Connection,
        _: &QueueHandle<Self>,
        _: wl_seat::WlSeat,
        capability: Capability,
    ) {
        if capability == Capability::Keyboard {
            if let Some(k) = self.keyboard.take() {
                k.release();
            }
        }
        if capability == Capability::Pointer {
            if let Some(p) = self.pointer.take() {
                p.release();
            }
        }
    }

    fn remove_seat(&mut self, _: &Connection, _: &QueueHandle<Self>, _: wl_seat::WlSeat) {}
}

impl KeyboardHandler for State {
    fn enter(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_keyboard::WlKeyboard,
        _surface: &wl_surface::WlSurface,
        _: u32,
        _: &[u32],
        _keysyms: &[Keysym],
    ) {
    }

    fn leave(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_keyboard::WlKeyboard,
        _surface: &wl_surface::WlSurface,
        _: u32,
    ) {
    }

    fn press_key(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_keyboard::WlKeyboard,
        _: u32,
        event: KeyEvent,
    ) {
        if let Some(vp) = self.viewport.as_mut() {
            let (named, utf8) = map_key_event(&event);
            vp.push_key_event(named, utf8, encode_modifiers(&self.modifiers), true);
        }
    }

    fn release_key(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_keyboard::WlKeyboard,
        _: u32,
        event: KeyEvent,
    ) {
        if let Some(vp) = self.viewport.as_mut() {
            let (named, utf8) = map_key_event(&event);
            vp.push_key_event(named, utf8, encode_modifiers(&self.modifiers), false);
        }
    }

    fn repeat_key(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_keyboard::WlKeyboard,
        _: u32,
        _event: KeyEvent,
    ) {
    }

    fn update_modifiers(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_keyboard::WlKeyboard,
        _serial: u32,
        modifiers: Modifiers,
        _raw: RawModifiers,
        _layout: u32,
    ) {
        self.modifiers = modifiers;
    }
}

impl PointerHandler for State {
    fn pointer_frame(
        &mut self,
        _: &Connection,
        _: &QueueHandle<Self>,
        _: &wl_pointer::WlPointer,
        events: &[PointerEvent],
    ) {
        let Some(vp) = self.viewport.as_mut() else {
            return;
        };
        for event in events {
            if &event.surface != self.layer.wl_surface() {
                continue;
            }
            let (px, py) = event.position;
            let lx = px as f32;
            let ly = py as f32;
            self.last_pointer = (lx, ly);
            match event.kind {
                PointerEventKind::Enter { .. } => {
                    vp.push_mouse_move(lx, ly);
                }
                PointerEventKind::Leave { .. } => {
                    vp.push_mouse_left();
                }
                PointerEventKind::Motion { .. } => {
                    vp.push_mouse_move(lx, ly);
                }
                PointerEventKind::Press { button, .. } => {
                    if let Some(code) = map_button(button) {
                        vp.push_mouse_button(lx, ly, code, true);
                    }
                }
                PointerEventKind::Release { button, .. } => {
                    if let Some(code) = map_button(button) {
                        vp.push_mouse_button(lx, ly, code, false);
                    }
                }
                PointerEventKind::Axis {
                    horizontal,
                    vertical,
                    ..
                } => {
                    let dx = -(horizontal.absolute as f32);
                    let dy = -(vertical.absolute as f32);
                    vp.push_mouse_scroll(lx, ly, dx, dy);
                }
            }
        }
    }
}

fn map_button(button: u32) -> Option<u32> {
    match button {
        0x110 => Some(0), // BTN_LEFT
        0x111 => Some(1), // BTN_RIGHT
        0x112 => Some(2), // BTN_MIDDLE
        _ => None,
    }
}

fn map_key_event(event: &KeyEvent) -> (u32, Option<String>) {
    let named = match event.keysym {
        Keysym::Return => 1,
        Keysym::Escape => 2,
        Keysym::BackSpace => 3,
        Keysym::Tab => 4,
        Keysym::Left => 5,
        Keysym::Right => 6,
        Keysym::Up => 7,
        Keysym::Down => 8,
        Keysym::Delete => 9,
        Keysym::Home => 10,
        Keysym::End => 11,
        _ => 0,
    };
    let utf8 = if named == 0 { event.utf8.clone() } else { None };
    (named, utf8)
}

fn encode_modifiers(mods: &Modifiers) -> u32 {
    let mut bits = 0u32;
    if mods.shift {
        bits |= 1;
    }
    if mods.ctrl {
        bits |= 2;
    }
    if mods.alt {
        bits |= 4;
    }
    if mods.logo {
        bits |= 8;
    }
    bits
}

delegate_compositor!(State);
delegate_output!(State);
delegate_seat!(State);
delegate_keyboard!(State);
delegate_pointer!(State);
delegate_layer!(State);
delegate_registry!(State);

impl ProvidesRegistryState for State {
    fn registry(&mut self) -> &mut RegistryState {
        &mut self.registry_state
    }
    registry_handlers![OutputState, SeatState];
}