Desktop: Implement pointer lock for NumberInput (#3638)

* Desktop: Implement pointer lock for NumberInput

* add shift and ctrl modifiers

* fixup

desktop/src/app.rs

@@ -5,7 +5,7 @@ use std::sync::mpsc::{Receiver, Sender, SyncSender};
 use std::thread;
 use std::time::{Duration, Instant};
 use winit::application::ApplicationHandler;
-use winit::dpi::PhysicalSize;
+use winit::dpi::{PhysicalPosition, PhysicalSize};
 use winit::event::{ButtonSource, ElementState, MouseButton, WindowEvent};
 use winit::event_loop::{ActiveEventLoop, ControlFlow};
 use winit::window::WindowId;
@@ -27,6 +27,8 @@ pub(crate) struct App {
 	window_size: PhysicalSize<u32>,
 	window_maximized: bool,
 	window_fullscreen: bool,
+	pointer_position: PhysicalPosition<f64>,
+	pointer_lock_position: Option<PhysicalPosition<f64>>,
 	ui_scale: f64,
 	app_event_receiver: Receiver<AppEvent>,
 	app_event_scheduler: AppEventScheduler,
@@ -84,6 +86,8 @@ impl App {
 			window_size: PhysicalSize { width: 0, height: 0 },
 			window_maximized: false,
 			window_fullscreen: false,
+			pointer_position: Default::default(),
+			pointer_lock_position: Default::default(),
 			ui_scale: 1.,
 			app_event_receiver,
 			app_event_scheduler,
@@ -329,6 +333,12 @@ impl App {
 					window.clipboard_write(content);
 				}
 			}
+			DesktopFrontendMessage::PointerLock => {
+				self.pointer_lock_position = Some(self.pointer_position);
+				if let Some(window) = &self.window {
+					window.start_pointer_lock();
+				}
+			}
 			DesktopFrontendMessage::WindowClose => {
 				self.app_event_scheduler.schedule(AppEvent::CloseWindow);
 			}
@@ -480,6 +490,26 @@ impl ApplicationHandler for App {
 	}
 
 	fn window_event(&mut self, event_loop: &dyn ActiveEventLoop, _window_id: WindowId, event: WindowEvent) {
+		// Handle pointer lock release
+		if let Some(pointer_lock_position) = self.pointer_lock_position
+			&& let WindowEvent::PointerButton {
+				state: ElementState::Released,
+				button: ButtonSource::Mouse(MouseButton::Left),
+				..
+			} = event
+		{
+			self.pointer_lock_position = None;
+			if let Some(window) = &self.window {
+				window.end_pointer_lock();
+			}
+			self.cef_context.handle_window_event(&WindowEvent::PointerMoved {
+				device_id: None,
+				position: pointer_lock_position,
+				primary: true,
+				source: winit::event::PointerSource::Mouse,
+			});
+		}
+
 		self.cef_context.handle_window_event(&event);
 
 		match event {
@@ -556,6 +586,13 @@ impl ApplicationHandler for App {
 					self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
 				}
 			}
+
+			WindowEvent::PointerMoved { position, .. } | WindowEvent::PointerLeft { position: Some(position), .. } | WindowEvent::PointerEntered { position, .. }
+				if self.pointer_lock_position.is_none() =>
+			{
+				self.pointer_position = position;
+			}
+
 			_ => {}
 		}
 
@@ -563,6 +600,15 @@ impl ApplicationHandler for App {
 		self.cef_context.work();
 	}
 
+	fn device_event(&mut self, _event_loop: &dyn ActiveEventLoop, _device_id: Option<winit::event::DeviceId>, event: winit::event::DeviceEvent) {
+		if self.pointer_lock_position.is_some()
+			&& let winit::event::DeviceEvent::PointerMotion { delta: (x, y) } = event
+		{
+			let message = DesktopWrapperMessage::PointerLockMove { x, y };
+			self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
+		}
+	}
+
 	fn about_to_wait(&mut self, event_loop: &dyn ActiveEventLoop) {
 		// Set a timeout in case we miss any cef schedule requests
 		let timeout = Instant::now() + Duration::from_millis(10);

desktop/src/window.rs

@@ -159,6 +159,16 @@ impl Window {
 		self.winit_window.set_cursor(cursor);
 	}
 
+	pub(crate) fn start_pointer_lock(&self) {
+		let _ = self.winit_window.set_cursor_grab(winit::window::CursorGrabMode::Locked);
+		self.winit_window.set_cursor_visible(false);
+	}
+
+	pub(crate) fn end_pointer_lock(&self) {
+		let _ = self.winit_window.set_cursor_grab(winit::window::CursorGrabMode::None);
+		self.winit_window.set_cursor_visible(true);
+	}
+
 	pub(crate) fn update_menu(&self, entries: Vec<MenuItem>) {
 		self.native_handle.update_menu(entries);
 	}

desktop/wrapper/src/handle_desktop_wrapper_message.rs

@@ -172,5 +172,9 @@ pub(super) fn handle_desktop_wrapper_message(dispatcher: &mut DesktopWrapperMess
 				dispatcher.queue_editor_message(message);
 			}
 		}
+		DesktopWrapperMessage::PointerLockMove { x, y } => {
+			let message = AppWindowMessage::PointerLockMove { x, y };
+			dispatcher.queue_editor_message(message);
+		}
 	}
 }

desktop/wrapper/src/intercept_frontend_message.rs

@@ -136,6 +136,9 @@ pub(super) fn intercept_frontend_message(dispatcher: &mut DesktopWrapperMessageD
 		FrontendMessage::TriggerClipboardWrite { content } => {
 			dispatcher.respond(DesktopFrontendMessage::ClipboardWrite { content });
 		}
+		FrontendMessage::WindowPointerLock => {
+			dispatcher.respond(DesktopFrontendMessage::PointerLock);
+		}
 		FrontendMessage::WindowClose => {
 			dispatcher.respond(DesktopFrontendMessage::WindowClose);
 		}

desktop/wrapper/src/messages.rs

@@ -65,6 +65,7 @@ pub enum DesktopFrontendMessage {
 	ClipboardWrite {
 		content: String,
 	},
+	PointerLock,
 	WindowClose,
 	WindowMinimize,
 	WindowMaximize,
@@ -132,6 +133,10 @@ pub enum DesktopWrapperMessage {
 	ClipboardReadResult {
 		content: Option<String>,
 	},
+	PointerLockMove {
+		x: f64,
+		y: f64,
+	},
 }
 
 #[derive(Clone, serde::Serialize, serde::Deserialize, Debug)]

editor/src/messages/app_window/app_window_message.rs

@@ -6,6 +6,8 @@ use super::app_window_message_handler::AppWindowPlatform;
 #[derive(PartialEq, Clone, Debug, serde::Serialize, serde::Deserialize)]
 pub enum AppWindowMessage {
 	UpdatePlatform { platform: AppWindowPlatform },
+	PointerLock,
+	PointerLockMove { x: f64, y: f64 },
 	Close,
 	Minimize,
 	Maximize,

editor/src/messages/app_window/app_window_message_handler.rs

@@ -15,6 +15,12 @@ impl MessageHandler<AppWindowMessage, ()> for AppWindowMessageHandler {
 				self.platform = platform;
 				responses.add(FrontendMessage::UpdatePlatform { platform: self.platform });
 			}
+			AppWindowMessage::PointerLock => {
+				responses.add(FrontendMessage::WindowPointerLock);
+			}
+			AppWindowMessage::PointerLockMove { x, y } => {
+				responses.add(FrontendMessage::WindowPointerLockMove { x, y });
+			}
 			AppWindowMessage::Close => {
 				responses.add(FrontendMessage::WindowClose);
 			}

editor/src/messages/frontend/frontend_message.rs

@@ -364,6 +364,11 @@ pub enum FrontendMessage {
 	},
 
 	// Window prefix: cause the application window to do something
+	WindowPointerLock,
+	WindowPointerLockMove {
+		x: f64,
+		y: f64,
+	},
 	WindowClose,
 	WindowMinimize,
 	WindowMaximize,

frontend/src/components/widgets/inputs/NumberInput.svelte

@@ -1,7 +1,8 @@
 <script lang="ts">
-	import { createEventDispatcher, onMount, onDestroy } from "svelte";
+	import { createEventDispatcher, onMount, onDestroy, getContext } from "svelte";
 
 	import { evaluateMathExpression } from "@graphite/../wasm/pkg/graphite_wasm";
+	import type { Editor } from "@graphite/editor";
 	import { PRESS_REPEAT_DELAY_MS, PRESS_REPEAT_INTERVAL_MS } from "@graphite/io-managers/input";
 	import type { NumberInputMode, NumberInputIncrementBehavior, ActionShortcut } from "@graphite/messages";
 	import { browserVersion, isDesktop } from "@graphite/utility-functions/platform";
@@ -16,6 +17,8 @@
 
 	const dispatch = createEventDispatcher<{ value: number | undefined; startHistoryTransaction: undefined }>();
 
+	const editor = getContext<Editor>("editor");
+
 	// Content
 	/// When `value` is not provided (i.e. it's `undefined`), a dash is displayed.
 	export let value: number | undefined = undefined;
@@ -80,6 +83,8 @@
 	let initialValueBeforeDragging: number | undefined = undefined;
 	// Stores the total value change during the process of dragging the slider. Set to 0 when not dragging.
 	let cumulativeDragDelta = 0;
+	// Track whether the Shift key is currently held down.
+	let shiftKeyDown = false;
 	// Track whether the Ctrl key is currently held down.
 	let ctrlKeyDown = false;
 
@@ -91,17 +96,20 @@
 		...(mode === "Range" ? { "--progress-factor": Math.min(Math.max((rangeSliderValueAsRendered - rangeMin) / (rangeMax - rangeMin), 0), 1) } : {}),
 	};
 
-	// Keep track of the Ctrl key being held down.
+	// Keep track of the Shift and Ctrl key being held down.
-	const trackCtrl = (e: KeyboardEvent | MouseEvent) => (ctrlKeyDown = e.ctrlKey);
+	const trackShiftAndCtrl = (e: KeyboardEvent | MouseEvent) => {
+		shiftKeyDown = e.shiftKey;
+		ctrlKeyDown = e.ctrlKey;
+	};
 	onMount(() => {
-		addEventListener("keydown", trackCtrl);
+		addEventListener("keydown", trackShiftAndCtrl);
-		addEventListener("keyup", trackCtrl);
+		addEventListener("keyup", trackShiftAndCtrl);
-		addEventListener("mousemove", trackCtrl);
+		addEventListener("mousemove", trackShiftAndCtrl);
 	});
 	onDestroy(() => {
-		removeEventListener("keydown", trackCtrl);
+		removeEventListener("keydown", trackShiftAndCtrl);
-		removeEventListener("keyup", trackCtrl);
+		removeEventListener("keyup", trackShiftAndCtrl);
-		removeEventListener("mousemove", trackCtrl);
+		removeEventListener("mousemove", trackShiftAndCtrl);
 		clearTimeout(repeatTimeout);
 	});
 
@@ -369,6 +377,9 @@
 
 		// Enter dragging state
 		if (usePointerLock) target.requestPointerLock();
+		if (isDesktop()) {
+			editor.handle.appWindowPointerLock();
+		}
 		initialValueBeforeDragging = value;
 		cumulativeDragDelta = 0;
 
@@ -412,19 +423,16 @@
 
 			// Calculate and then update the dragged value offset, slowed down by 10x when Shift is held.
 			if (ignoredFirstMovement && initialValueBeforeDragging !== undefined) {
-				const CHANGE_PER_DRAG_PX = 0.1;
+				pointerLockMoveUpdate(e.movementX, e.shiftKey, e.ctrlKey, initialValueBeforeDragging);
-				const CHANGE_PER_DRAG_PX_SLOW = CHANGE_PER_DRAG_PX / 10;
+			}
-
+			ignoredFirstMovement = true;
-				const dragDelta = e.movementX * (e.shiftKey ? CHANGE_PER_DRAG_PX_SLOW : CHANGE_PER_DRAG_PX);
+		};
-				cumulativeDragDelta += dragDelta;
+		// On desktop we don't get `pointermove` events while in pointer lock (cef doesn't support pointer lock).
-
+		// We have to listen for our custom `pointerlockmove` events instead.
-				const combined = initialValueBeforeDragging + cumulativeDragDelta;
+		const pointerLockMove = (e: Event) => {
-				const combineSnapped = e.ctrlKey ? Math.round(combined) : combined;
+			if (ignoredFirstMovement && initialValueBeforeDragging !== undefined && e instanceof CustomEvent) {
-
+				const delta = (e.detail as { x: number }).x;
-				const newValue = updateValue(combineSnapped);
+				pointerLockMoveUpdate(delta, shiftKeyDown, ctrlKeyDown, initialValueBeforeDragging);
-
-				// If the value was altered within the `updateValue()` call, we need to rectify the cumulative drag delta to account for the change.
-				if (newValue !== undefined) cumulativeDragDelta -= combineSnapped - newValue;
 			}
 			ignoredFirstMovement = true;
 		};
@@ -443,14 +451,32 @@
 			// Clean up the event listeners.
 			removeEventListener("pointerup", pointerUp);
 			removeEventListener("pointermove", pointerMove);
+			removeEventListener("pointerlockmove", pointerLockMove);
 			if (usePointerLock) document.removeEventListener("pointerlockchange", pointerLockChange);
 		};
 
 		addEventListener("pointerup", pointerUp);
 		addEventListener("pointermove", pointerMove);
+		addEventListener("pointerlockmove", pointerLockMove);
 		if (usePointerLock) document.addEventListener("pointerlockchange", pointerLockChange);
 	}
 
+	function pointerLockMoveUpdate(delta: number, slow: boolean, snapping: boolean, initialValue: number) {
+		const CHANGE_PER_DRAG_PX = 0.1;
+		const CHANGE_PER_DRAG_PX_SLOW = CHANGE_PER_DRAG_PX / 10;
+
+		const dragDelta = delta * (slow ? CHANGE_PER_DRAG_PX_SLOW : CHANGE_PER_DRAG_PX);
+		cumulativeDragDelta += dragDelta;
+
+		const combined = initialValue + cumulativeDragDelta;
+		const combineSnapped = snapping ? Math.round(combined) : combined;
+
+		const newValue = updateValue(combineSnapped);
+
+		// If the value was altered within the `updateValue()` call, we need to rectify the cumulative drag delta to account for the change.
+		if (newValue !== undefined) cumulativeDragDelta -= combineSnapped - newValue;
+	}
+
 	// ===============================
 	// RANGE MODE: DRAGGING THE SLIDER
 	// ===============================

frontend/src/io-managers/input.ts

@@ -1,7 +1,7 @@
 import { get } from "svelte/store";
 
 import { type Editor } from "@graphite/editor";
-import { TriggerClipboardRead } from "@graphite/messages";
+import { TriggerClipboardRead, WindowPointerLockMove } from "@graphite/messages";
 import { type DialogState } from "@graphite/state-providers/dialog";
 import { type DocumentState } from "@graphite/state-providers/document";
 import { type FullscreenState } from "@graphite/state-providers/fullscreen";
@@ -500,6 +500,12 @@ export function createInputManager(editor: Editor, dialog: DialogState, portfoli
 		}
 	});
 
+	// Pointer lock movement events on desktop
+	editor.subscriptions.subscribeJsMessage(WindowPointerLockMove, (data) => {
+		const event = new CustomEvent("pointerlockmove", { detail: data });
+		window.dispatchEvent(event);
+	});
+
 	// Helper functions
 
 	function potentiallyRestoreCanvasFocus(e: Event) {

frontend/src/messages.ts

@@ -315,8 +315,6 @@ export class UpdateFullscreen extends JsMessage {
 	readonly fullscreen!: boolean;
 }
 
-export class CloseWindow extends JsMessage {}
-
 export class UpdateViewportHolePunch extends JsMessage {
 	readonly active!: boolean;
 }
@@ -332,6 +330,11 @@ export class UpdateUIScale extends JsMessage {
 	readonly scale!: number;
 }
 
+export class WindowPointerLockMove extends JsMessage {
+	readonly x!: number;
+	readonly y!: number;
+}
+
 // Rust enum `Key`
 export type KeyRaw = string;
 // Serde converts a Rust `Key` enum variant into this format with both the `Key` variant name (called `RawKey` in TS) and the localized `label` for the key
@@ -1728,6 +1731,7 @@ export const messageMakers: Record<string, MessageMaker> = {
 	UpdatePlatform,
 	UpdateMaximized,
 	UpdateFullscreen,
+	WindowPointerLockMove,
 	UpdatePropertiesPanelLayout,
 	UpdatePropertiesPanelState,
 	UpdateStatusBarHintsLayout,

frontend/wasm/src/editor_api.rs

@@ -274,6 +274,13 @@ impl EditorHandle {
 		self.dispatch(NodeGraphMessage::AddSecondaryExport);
 	}
 
+	/// Start Pointer Lock
+	#[wasm_bindgen(js_name = appWindowPointerLock)]
+	pub fn app_window_pointer_lock(&self) {
+		let message = AppWindowMessage::PointerLock;
+		self.dispatch(message);
+	}
+
 	/// Minimizes the application window to the taskbar or dock
 	#[wasm_bindgen(js_name = appWindowMinimize)]
 	pub fn app_window_minimize(&self) {