Improve Frontend -> Backend user input system (#348)

Includes refactor that sends coordinates of the document viewports to the backend so input is sent relative to the application window
Closes #124
Fixes #291

* Improve Frontend -> Backend user input system

* Code review changes

* More code review changes

* Fix TS error

editor/src/document/document_file.rs

@@ -284,7 +284,7 @@ impl MessageHandler<DocumentMessage, &InputPreprocessor> for DocumentMessageHand
 			}
 			CommitTransaction => self.document_backup = None,
 			ExportDocument => {
-				let bbox = self.document.visible_layers_bounding_box().unwrap_or([DVec2::ZERO, ipp.viewport_size]);
+				let bbox = self.document.visible_layers_bounding_box().unwrap_or([DVec2::ZERO, ipp.viewport_bounds.size()]);
 				let size = bbox[1] - bbox[0];
 				let name = match self.name.ends_with(FILE_SAVE_SUFFIX) {
 					true => self.name.clone().replace(FILE_SAVE_SUFFIX, FILE_EXPORT_SUFFIX),

editor/src/document/movement_handler.rs

@@ -5,7 +5,7 @@ use super::LayerData;
 use crate::message_prelude::*;
 use crate::{
 	consts::{VIEWPORT_SCROLL_RATE, VIEWPORT_ZOOM_LEVELS, VIEWPORT_ZOOM_MOUSE_RATE, VIEWPORT_ZOOM_SCALE_MAX, VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_WHEEL_RATE},
-	input::{mouse::ViewportPosition, InputPreprocessor},
+	input::{mouse::ViewportBounds, mouse::ViewportPosition, InputPreprocessor},
 };
 use glam::DVec2;
 use graphene::document::Document;
@@ -42,8 +42,8 @@ pub struct MovementMessageHandler {
 }
 
 impl MovementMessageHandler {
-	fn create_document_transform_from_layerdata(&self, layerdata: &LayerData, viewport_size: &ViewportPosition, responses: &mut VecDeque<Message>) {
+	fn create_document_transform_from_layerdata(&self, layerdata: &LayerData, viewport_bounds: &ViewportBounds, responses: &mut VecDeque<Message>) {
-		let half_viewport = *viewport_size / 2.;
+		let half_viewport = viewport_bounds.size() / 2.;
 		let scaled_half_viewport = half_viewport / layerdata.scale;
 		responses.push_back(
 			DocumentOperation::SetLayerTransform {
@@ -89,10 +89,10 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					let transformed_delta = document.root.transform.inverse().transform_vector2(delta);
 
 					layerdata.translation += transformed_delta;
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 				if self.rotating {
-					let half_viewport = ipp.viewport_size / 2.;
+					let half_viewport = ipp.viewport_bounds.size() / 2.;
 					let rotation = {
 						let start_vec = self.mouse_pos - half_viewport;
 						let end_vec = ipp.mouse.position - half_viewport;
@@ -109,7 +109,7 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 						}
 						.into(),
 					);
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 				if self.zooming {
 					let difference = self.mouse_pos.y as f64 - ipp.mouse.position.y as f64;
@@ -118,36 +118,36 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					let new = (layerdata.scale * amount).clamp(VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_SCALE_MAX);
 					layerdata.scale = new;
 					responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 				self.mouse_pos = ipp.mouse.position;
 			}
 			SetCanvasZoom(new) => {
 				layerdata.scale = new.clamp(VIEWPORT_ZOOM_SCALE_MIN, VIEWPORT_ZOOM_SCALE_MAX);
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			IncreaseCanvasZoom => {
 				layerdata.scale = *VIEWPORT_ZOOM_LEVELS.iter().find(|scale| **scale > layerdata.scale).unwrap_or(&layerdata.scale);
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			DecreaseCanvasZoom => {
 				layerdata.scale = *VIEWPORT_ZOOM_LEVELS.iter().rev().find(|scale| **scale < layerdata.scale).unwrap_or(&layerdata.scale);
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			WheelCanvasZoom => {
 				let scroll = ipp.mouse.scroll_delta.scroll_delta();
 				let mouse = ipp.mouse.position;
-				let viewport_size = ipp.viewport_size;
+				let viewport_bounds = ipp.viewport_bounds.size();
 				let mut zoom_factor = 1. + scroll.abs() * VIEWPORT_ZOOM_WHEEL_RATE;
 				if ipp.mouse.scroll_delta.y > 0 {
 					zoom_factor = 1. / zoom_factor
 				};
-				let new_viewport_size = viewport_size * (1. / zoom_factor);
+				let new_viewport_bounds = viewport_bounds * (1. / zoom_factor);
-				let delta_size = viewport_size - new_viewport_size;
+				let delta_size = viewport_bounds - new_viewport_bounds;
-				let mouse_percent = mouse / viewport_size;
+				let mouse_percent = mouse / viewport_bounds;
 				let delta = (delta_size * -2.) * (mouse_percent - DVec2::splat(0.5));
 
 				let transformed_delta = document.root.transform.inverse().transform_vector2(delta);
@@ -155,7 +155,7 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 				layerdata.scale = new;
 				layerdata.translation += transformed_delta;
 				responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			WheelCanvasTranslate { use_y_as_x } => {
 				let delta = match use_y_as_x {
@@ -164,11 +164,11 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 				} * VIEWPORT_SCROLL_RATE;
 				let transformed_delta = document.root.transform.inverse().transform_vector2(delta);
 				layerdata.translation += transformed_delta;
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 			}
 			SetCanvasRotation(new) => {
 				layerdata.rotation = new;
-				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+				self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				responses.push_back(FrontendMessage::SetCanvasRotation { new_radians: new }.into());
 			}
 			ZoomCanvasToFitAll => {
@@ -176,7 +176,7 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					let pos1 = document.root.transform.inverse().transform_point2(pos1);
 					let pos2 = document.root.transform.inverse().transform_point2(pos2);
 					let v1 = document.root.transform.inverse().transform_point2(DVec2::ZERO);
-					let v2 = document.root.transform.inverse().transform_point2(ipp.viewport_size);
+					let v2 = document.root.transform.inverse().transform_point2(ipp.viewport_bounds.size());
 
 					let center = v1.lerp(v2, 0.5) - pos1.lerp(pos2, 0.5);
 					let size = (pos2 - pos1) / (v2 - v1);
@@ -186,7 +186,7 @@ impl MessageHandler<MovementMessage, (&mut LayerData, &Document, &InputPreproces
 					layerdata.translation += center;
 					layerdata.scale *= new_scale;
 					responses.push_back(FrontendMessage::SetCanvasZoom { new_zoom: layerdata.scale }.into());
-					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_size, responses);
+					self.create_document_transform_from_layerdata(layerdata, &ipp.viewport_bounds, responses);
 				}
 			}
 		}

editor/src/input/input_preprocessor.rs

@@ -1,7 +1,7 @@
 use std::usize;
 
 use super::keyboard::{Key, KeyStates};
-use super::mouse::{MouseKeys, MouseState, ScrollDelta, ViewportPosition};
+use super::mouse::{EditorMouseState, MouseKeys, MouseState, ViewportBounds};
 use crate::message_prelude::*;
 use bitflags::bitflags;
 
@@ -11,13 +11,13 @@ pub use graphene::DocumentResponse;
 #[impl_message(Message, InputPreprocessor)]
 #[derive(PartialEq, Clone, Debug)]
 pub enum InputPreprocessorMessage {
-	MouseDown(MouseState, ModifierKeys),
+	MouseDown(EditorMouseState, ModifierKeys),
-	MouseUp(MouseState, ModifierKeys),
+	MouseUp(EditorMouseState, ModifierKeys),
-	MouseMove(ViewportPosition, ModifierKeys),
+	MouseMove(EditorMouseState, ModifierKeys),
-	MouseScroll(ScrollDelta, ModifierKeys),
+	MouseScroll(EditorMouseState, ModifierKeys),
 	KeyUp(Key, ModifierKeys),
 	KeyDown(Key, ModifierKeys),
-	ViewportResize(ViewportPosition),
+	BoundsOfViewports(Vec<ViewportBounds>),
 }
 
 bitflags! {
@@ -34,7 +34,7 @@ bitflags! {
 pub struct InputPreprocessor {
 	pub keyboard: KeyStates,
 	pub mouse: MouseState,
-	pub viewport_size: ViewportPosition,
+	pub viewport_bounds: ViewportBounds,
 }
 
 enum KeyPosition {
@@ -45,28 +45,43 @@ enum KeyPosition {
 impl MessageHandler<InputPreprocessorMessage, ()> for InputPreprocessor {
 	fn process_action(&mut self, message: InputPreprocessorMessage, _data: (), responses: &mut VecDeque<Message>) {
 		match message {
-			InputPreprocessorMessage::MouseMove(pos, modifier_keys) => {
+			InputPreprocessorMessage::MouseMove(editor_mouse_state, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
-				self.mouse.position = pos;
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+
 				responses.push_back(InputMapperMessage::PointerMove.into());
 			}
-			InputPreprocessorMessage::MouseScroll(delta, modifier_keys) => {
+			InputPreprocessorMessage::MouseDown(editor_mouse_state, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
-				self.mouse.scroll_delta = delta;
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+
+				if let Some(message) = self.translate_mouse_event(mouse_state, KeyPosition::Pressed) {
+					responses.push_back(message);
+				}
+			}
+			InputPreprocessorMessage::MouseUp(editor_mouse_state, modifier_keys) => {
+				self.handle_modifier_keys(modifier_keys, responses);
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+
+				if let Some(message) = self.translate_mouse_event(mouse_state, KeyPosition::Released) {
+					responses.push_back(message);
+				}
+			}
+			InputPreprocessorMessage::MouseScroll(editor_mouse_state, modifier_keys) => {
+				self.handle_modifier_keys(modifier_keys, responses);
+
+				let mouse_state = editor_mouse_state.to_mouse_state(&self.viewport_bounds);
+				self.mouse.position = mouse_state.position;
+				self.mouse.scroll_delta = mouse_state.scroll_delta;
+
 				responses.push_back(InputMapperMessage::MouseScroll.into());
 			}
-			InputPreprocessorMessage::MouseDown(state, modifier_keys) => {
-				self.handle_modifier_keys(modifier_keys, responses);
-				if let Some(message) = self.translate_mouse_event(state, KeyPosition::Pressed) {
-					responses.push_back(message);
-				}
-			}
-			InputPreprocessorMessage::MouseUp(state, modifier_keys) => {
-				self.handle_modifier_keys(modifier_keys, responses);
-				if let Some(message) = self.translate_mouse_event(state, KeyPosition::Released) {
-					responses.push_back(message);
-				}
-			}
 			InputPreprocessorMessage::KeyDown(key, modifier_keys) => {
 				self.handle_modifier_keys(modifier_keys, responses);
 				self.keyboard.set(key as usize);
@@ -77,15 +92,26 @@ impl MessageHandler<InputPreprocessorMessage, ()> for InputPreprocessor {
 				self.keyboard.unset(key as usize);
 				responses.push_back(InputMapperMessage::KeyUp(key).into());
 			}
-			InputPreprocessorMessage::ViewportResize(size) => {
+			InputPreprocessorMessage::BoundsOfViewports(bounds_of_viewports) => {
-				responses.push_back(
+				assert_eq!(bounds_of_viewports.len(), 1, "Only one viewport is currently supported");
-					graphene::Operation::TransformLayer {
+
-						path: vec![],
+				for bounds in bounds_of_viewports {
-						transform: glam::DAffine2::from_translation((size - self.viewport_size) / 2.).to_cols_array(),
+					let new_size = bounds.size();
-					}
+					let existing_size = self.viewport_bounds.size();
-					.into(),
+
-				);
+					let translation = (new_size - existing_size) / 2.;
-				self.viewport_size = size;
+
+					// TODO: Extend this to multiple viewports instead of setting it to the value of this last loop iteration
+					self.viewport_bounds = bounds;
+
+					responses.push_back(
+						graphene::Operation::TransformLayer {
+							path: vec![],
+							transform: glam::DAffine2::from_translation(translation).to_cols_array(),
+						}
+						.into(),
+					);
+				}
 			}
 		};
 	}
@@ -136,12 +162,16 @@ impl InputPreprocessor {
 
 #[cfg(test)]
 mod test {
+	use crate::input::mouse::ViewportPosition;
+
 	use super::*;
 
 	#[test]
 	fn process_action_mouse_move_handle_modifier_keys() {
 		let mut input_preprocessor = InputPreprocessor::default();
-		let message = InputPreprocessorMessage::MouseMove((4., 809.).into(), ModifierKeys::ALT);
+		let mut editor_mouse_state = EditorMouseState::new();
+		editor_mouse_state.editor_position = ViewportPosition::new(4., 809.);
+		let message = InputPreprocessorMessage::MouseMove(editor_mouse_state, ModifierKeys::ALT);
 		let mut responses = VecDeque::new();
 
 		input_preprocessor.process_action(message, (), &mut responses);
@@ -153,7 +183,7 @@ mod test {
 	#[test]
 	fn process_action_mouse_down_handle_modifier_keys() {
 		let mut input_preprocessor = InputPreprocessor::default();
-		let message = InputPreprocessorMessage::MouseDown(MouseState::new(), ModifierKeys::CONTROL);
+		let message = InputPreprocessorMessage::MouseDown(EditorMouseState::new(), ModifierKeys::CONTROL);
 		let mut responses = VecDeque::new();
 
 		input_preprocessor.process_action(message, (), &mut responses);
@@ -165,7 +195,7 @@ mod test {
 	#[test]
 	fn process_action_mouse_up_handle_modifier_keys() {
 		let mut input_preprocessor = InputPreprocessor::default();
-		let message = InputPreprocessorMessage::MouseUp(MouseState::new(), ModifierKeys::SHIFT);
+		let message = InputPreprocessorMessage::MouseUp(EditorMouseState::new(), ModifierKeys::SHIFT);
 		let mut responses = VecDeque::new();
 
 		input_preprocessor.process_action(message, (), &mut responses);

editor/src/input/mouse.rs

@@ -3,6 +3,26 @@ use glam::DVec2;
 
 // Origin is top left
 pub type ViewportPosition = DVec2;
+pub type EditorPosition = DVec2;
+
+#[derive(PartialEq, Clone, Debug, Default)]
+pub struct ViewportBounds {
+	pub top_left: DVec2,
+	pub bottom_right: DVec2,
+}
+
+impl ViewportBounds {
+	pub fn from_slice(slice: &[f64]) -> Self {
+		Self {
+			top_left: DVec2::from_slice(&slice[0..2]),
+			bottom_right: DVec2::from_slice(&slice[2..4]),
+		}
+	}
+
+	pub fn size(&self) -> DVec2 {
+		self.bottom_right - self.top_left
+	}
+}
 
 #[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Hash)]
 pub struct ScrollDelta {
@@ -35,7 +55,7 @@ impl MouseState {
 		Self::default()
 	}
 
-	pub fn from_pos(x: f64, y: f64) -> Self {
+	pub fn from_position(x: f64, y: f64) -> Self {
 		Self {
 			position: (x, y).into(),
 			mouse_keys: MouseKeys::default(),
@@ -43,7 +63,7 @@ impl MouseState {
 		}
 	}
 
-	pub fn from_u8_pos(keys: u8, position: ViewportPosition) -> Self {
+	pub fn from_keys_and_editor_position(keys: u8, position: ViewportPosition) -> Self {
 		let mouse_keys = MouseKeys::from_bits(keys).expect("invalid modifier keys");
 		Self {
 			position,
@@ -52,6 +72,45 @@ impl MouseState {
 		}
 	}
 }
+
+#[derive(Debug, Copy, Clone, Default, PartialEq)]
+pub struct EditorMouseState {
+	pub editor_position: EditorPosition,
+	pub mouse_keys: MouseKeys,
+	pub scroll_delta: ScrollDelta,
+}
+
+impl EditorMouseState {
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	pub fn from_editor_position(x: f64, y: f64) -> Self {
+		Self {
+			editor_position: (x, y).into(),
+			mouse_keys: MouseKeys::default(),
+			scroll_delta: ScrollDelta::default(),
+		}
+	}
+
+	pub fn from_keys_and_editor_position(keys: u8, editor_position: EditorPosition) -> Self {
+		let mouse_keys = MouseKeys::from_bits(keys).expect("invalid modifier keys");
+		Self {
+			editor_position,
+			mouse_keys,
+			scroll_delta: ScrollDelta::default(),
+		}
+	}
+
+	pub fn to_mouse_state(&self, active_viewport_bounds: &ViewportBounds) -> MouseState {
+		MouseState {
+			position: self.editor_position - active_viewport_bounds.top_left,
+			mouse_keys: self.mouse_keys,
+			scroll_delta: self.scroll_delta,
+		}
+	}
+}
+
 bitflags! {
 	#[derive(Default)]
 	#[repr(transparent)]

editor/src/misc/test_utils.rs

@@ -1,6 +1,6 @@
 use crate::{
 	input::{
-		mouse::{MouseKeys, MouseState, ScrollDelta},
+		mouse::{EditorMouseState, MouseKeys, ScrollDelta, ViewportPosition},
 		InputPreprocessorMessage, ModifierKeys,
 	},
 	message_prelude::{Message, ToolMessage},
@@ -18,8 +18,8 @@ pub trait EditorTestUtils {
 	/// Select given tool and drag it from (x1, y1) to (x2, y2)
 	fn drag_tool(&mut self, typ: ToolType, x1: f64, y1: f64, x2: f64, y2: f64);
 	fn move_mouse(&mut self, x: f64, y: f64);
-	fn mousedown(&mut self, state: MouseState);
+	fn mousedown(&mut self, state: EditorMouseState);
-	fn mouseup(&mut self, state: MouseState);
+	fn mouseup(&mut self, state: EditorMouseState);
 	fn lmb_mousedown(&mut self, x: f64, y: f64);
 	fn input(&mut self, message: InputPreprocessorMessage);
 	fn select_tool(&mut self, typ: ToolType);
@@ -44,28 +44,30 @@ impl EditorTestUtils for Editor {
 		self.move_mouse(x1, y1);
 		self.lmb_mousedown(x1, y1);
 		self.move_mouse(x2, y2);
-		self.mouseup(MouseState {
+		self.mouseup(EditorMouseState {
-			position: (x2, y2).into(),
+			editor_position: (x2, y2).into(),
 			mouse_keys: MouseKeys::empty(),
 			scroll_delta: ScrollDelta::default(),
 		});
 	}
 
 	fn move_mouse(&mut self, x: f64, y: f64) {
-		self.input(InputPreprocessorMessage::MouseMove((x, y).into(), ModifierKeys::default()));
+		let mut editor_mouse_state = EditorMouseState::new();
+		editor_mouse_state.editor_position = ViewportPosition::new(x, y);
+		self.input(InputPreprocessorMessage::MouseMove(editor_mouse_state, ModifierKeys::default()));
 	}
 
-	fn mousedown(&mut self, state: MouseState) {
+	fn mousedown(&mut self, state: EditorMouseState) {
 		self.input(InputPreprocessorMessage::MouseDown(state, ModifierKeys::default()));
 	}
 
-	fn mouseup(&mut self, state: MouseState) {
+	fn mouseup(&mut self, state: EditorMouseState) {
 		self.handle_message(InputPreprocessorMessage::MouseUp(state, ModifierKeys::default())).unwrap()
 	}
 
 	fn lmb_mousedown(&mut self, x: f64, y: f64) {
-		self.mousedown(MouseState {
+		self.mousedown(EditorMouseState {
-			position: (x, y).into(),
+			editor_position: (x, y).into(),
 			mouse_keys: MouseKeys::LEFT,
 			scroll_delta: ScrollDelta::default(),
 		})

frontend/src/components/panels/Document.vue

@@ -118,7 +118,7 @@
 						<CanvasRuler :origin="0" :majorMarkSpacing="100" :direction="RulerDirection.Vertical" />
 					</LayoutCol>
 					<LayoutCol :class="'canvas-area'">
-						<div class="canvas" @mousedown="canvasMouseDown" @mouseup="canvasMouseUp" @mousemove="canvasMouseMove" ref="canvas">
+						<div class="canvas" ref="canvas">
 							<svg v-html="viewportSvg" :style="{ width: canvasSvgWidth, height: canvasSvgHeight }"></svg>
 						</div>
 					</LayoutCol>
@@ -210,7 +210,6 @@
 <script lang="ts">
 import { defineComponent } from "vue";
 
-import { makeModifiersBitfield } from "@/utilities/input";
 import { ResponseType, registerResponseHandler, Response, UpdateCanvas, SetActiveTool, SetCanvasZoom, SetCanvasRotation } from "@/utilities/response-handler";
 import { SeparatorDirection, SeparatorType } from "@/components/widgets/widgets";
 import { comingSoon } from "@/utilities/errors";
@@ -259,25 +258,6 @@ export default defineComponent({
 
 			this.canvasSvgWidth = `${width}px`;
 			this.canvasSvgHeight = `${height}px`;
-
-			(await wasm).viewport_resize(width, height);
-		},
-		async canvasMouseDown(e: MouseEvent) {
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_down(e.offsetX, e.offsetY, e.buttons, modifiers);
-		},
-		async canvasMouseUp(e: MouseEvent) {
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_up(e.offsetX, e.offsetY, e.buttons, modifiers);
-		},
-		async canvasMouseMove(e: MouseEvent) {
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_move(e.offsetX, e.offsetY, modifiers);
-		},
-		async canvasMouseScroll(e: WheelEvent) {
-			e.preventDefault();
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_scroll(e.deltaX, e.deltaY, e.deltaZ, modifiers);
 		},
 		async setCanvasZoom(newZoom: number) {
 			(await wasm).set_canvas_zoom(newZoom / 100);
@@ -327,12 +307,8 @@ export default defineComponent({
 			}
 		});
 
-		// TODO: Move event listeners to `main.ts`
+		window.addEventListener("resize", this.viewportResize);
-		const canvas = this.$refs.canvas as HTMLElement;
+		window.addEventListener("DOMContentLoaded", this.viewportResize);
-		canvas.addEventListener("wheel", this.canvasMouseScroll, { passive: false });
-
-		window.addEventListener("resize", () => this.viewportResize());
-		window.addEventListener("DOMContentLoaded", () => this.viewportResize());
 	},
 	data() {
 		return {

frontend/src/main.ts

@@ -1,17 +1,26 @@
 import { createApp } from "vue";
 
 import { fullscreenModeChanged } from "@/utilities/fullscreen";
-import { handleKeyUp, handleKeyDown, handleMouseDown } from "@/utilities/input";
+import { onKeyUp, onKeyDown, onMouseMove, onMouseDown, onMouseUp, onMouseScroll, onWindowResize } from "@/utilities/input";
 import "@/utilities/errors";
 
 import App from "@/App.vue";
 
 // Bind global browser events
+window.addEventListener("resize", onWindowResize);
+window.addEventListener("DOMContentLoaded", onWindowResize);
+
 document.addEventListener("contextmenu", (e) => e.preventDefault());
 document.addEventListener("fullscreenchange", () => fullscreenModeChanged());
-window.addEventListener("keyup", (e: KeyboardEvent) => handleKeyUp(e));
+
-window.addEventListener("keydown", (e: KeyboardEvent) => handleKeyDown(e));
+window.addEventListener("keyup", onKeyUp);
-window.addEventListener("mousedown", (e: MouseEvent) => handleMouseDown(e));
+window.addEventListener("keydown", onKeyDown);
+
+window.addEventListener("mousemove", onMouseMove);
+window.addEventListener("mousedown", onMouseDown);
+window.addEventListener("mouseup", onMouseUp);
+
+window.addEventListener("wheel", onMouseScroll, { passive: false });
 
 // Initialize the Vue application
 createApp(App).mount("#app");

frontend/src/utilities/input.ts

@@ -3,7 +3,11 @@ import { dialogIsVisible, dismissDialog, submitDialog } from "@/utilities/dialog
 
 const wasm = import("@/../wasm/pkg");
 
-export function shouldRedirectKeyboardEventToBackend(e: KeyboardEvent): boolean {
+let viewportMouseInteractionOngoing = false;
+
+// Keyboard events
+
+function shouldRedirectKeyboardEventToBackend(e: KeyboardEvent): boolean {
 	// Don't redirect user input from text entry into HTML elements
 	const target = e.target as HTMLElement;
 	if (target.nodeName === "INPUT" || target.nodeName === "TEXTAREA" || target.isContentEditable) return false;
@@ -31,10 +35,10 @@ export function shouldRedirectKeyboardEventToBackend(e: KeyboardEvent): boolean
 	return true;
 }
 
-export async function handleKeyDown(e: KeyboardEvent) {
+export async function onKeyDown(e: KeyboardEvent) {
 	if (shouldRedirectKeyboardEventToBackend(e)) {
 		e.preventDefault();
-		const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
+		const modifiers = makeModifiersBitfield(e);
 		(await wasm).on_key_down(e.key, modifiers);
 		return;
 	}
@@ -48,27 +52,77 @@ export async function handleKeyDown(e: KeyboardEvent) {
 	}
 }
 
-export async function handleKeyUp(e: KeyboardEvent) {
+export async function onKeyUp(e: KeyboardEvent) {
 	if (shouldRedirectKeyboardEventToBackend(e)) {
 		e.preventDefault();
-		const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
+		const modifiers = makeModifiersBitfield(e);
 		(await wasm).on_key_up(e.key, modifiers);
 	}
 }
 
-export async function handleMouseDown(e: MouseEvent) {
+// Mouse events
+
+export async function onMouseMove(e: MouseEvent) {
+	if (!e.buttons) viewportMouseInteractionOngoing = false;
+
+	const modifiers = makeModifiersBitfield(e);
+	(await wasm).on_mouse_move(e.clientX, e.clientY, e.buttons, modifiers);
+}
+
+export async function onMouseDown(e: MouseEvent) {
 	const target = e.target && (e.target as HTMLElement);
-	const clickedInsideDialog = target && target.closest(".dialog-modal .floating-menu-content");
+	const inCanvas = target && target.closest(".canvas");
+	const inDialog = target && target.closest(".dialog-modal .floating-menu-content");
 
-	if (dialogIsVisible() && !clickedInsideDialog) {
+	// Block middle mouse button auto-scroll mode
+	if (e.button === 1) e.preventDefault();
+
+	if (dialogIsVisible() && !inDialog) {
 		dismissDialog();
-
 		e.preventDefault();
 		e.stopPropagation();
 	}
+
+	if (inCanvas) viewportMouseInteractionOngoing = true;
+
+	if (viewportMouseInteractionOngoing) {
+		const modifiers = makeModifiersBitfield(e);
+		(await wasm).on_mouse_down(e.clientX, e.clientY, e.buttons, modifiers);
+	}
 }
 
-export function makeModifiersBitfield(control: boolean, shift: boolean, alt: boolean): number {
+export async function onMouseUp(e: MouseEvent) {
-	// eslint-disable-next-line no-bitwise
+	if (!e.buttons) viewportMouseInteractionOngoing = false;
-	return Number(control) | (Number(shift) << 1) | (Number(alt) << 2);
+
+	const modifiers = makeModifiersBitfield(e);
+	(await wasm).on_mouse_up(e.clientX, e.clientY, e.buttons, modifiers);
+}
+
+export async function onMouseScroll(e: WheelEvent) {
+	const target = e.target && (e.target as HTMLElement);
+	const inCanvas = target && target.closest(".canvas");
+
+	if (inCanvas) {
+		e.preventDefault();
+		const modifiers = makeModifiersBitfield(e);
+		(await wasm).on_mouse_scroll(e.clientX, e.clientY, e.buttons, e.deltaX, e.deltaY, e.deltaZ, modifiers);
+	}
+}
+
+export async function onWindowResize() {
+	const viewports = Array.from(document.querySelectorAll(".canvas"));
+	const boundsOfViewports = viewports.map((canvas) => {
+		const bounds = canvas.getBoundingClientRect();
+		return [bounds.left, bounds.top, bounds.right, bounds.bottom];
+	});
+
+	const flattened = boundsOfViewports.flat();
+	const data = Float64Array.from(flattened);
+
+	if (boundsOfViewports.length > 0) (await wasm).bounds_of_viewports(data);
+}
+
+export function makeModifiersBitfield(e: MouseEvent | KeyboardEvent): number {
+	// eslint-disable-next-line no-bitwise
+	return Number(e.ctrlKey) | (Number(e.shiftKey) << 1) | (Number(e.altKey) << 2);
 }

frontend/wasm/src/document.rs

@@ -2,11 +2,11 @@ use crate::shims::Error;
 use crate::wrappers::{translate_key, translate_tool, Color};
 use crate::EDITOR_STATE;
 use editor::input::input_preprocessor::ModifierKeys;
-use editor::input::mouse::ScrollDelta;
+use editor::input::mouse::{EditorMouseState, ScrollDelta, ViewportBounds};
 use editor::message_prelude::*;
 use editor::misc::EditorError;
 use editor::tool::{tool_options::ToolOptions, tools, ToolType};
-use editor::{input::mouse::MouseState, LayerId};
+use editor::LayerId;
 use graphene::layers::BlendMode;
 use wasm_bindgen::prelude::*;
 
@@ -104,47 +104,57 @@ pub fn close_all_documents_with_confirmation() -> Result<(), JsValue> {
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(DocumentsMessage::CloseAllDocumentsWithConfirmation).map_err(convert_error))
 }
 
-// TODO: Call event when the panels are resized
+/// Send new bounds when document panel viewports get resized or moved within the editor
-/// Viewport resized
+/// [left, top, right, bottom]...
 #[wasm_bindgen]
-pub fn viewport_resize(new_width: f64, new_height: f64) -> Result<(), JsValue> {
+pub fn bounds_of_viewports(bounds_of_viewports: &[f64]) -> Result<(), JsValue> {
-	let ev = InputPreprocessorMessage::ViewportResize((new_width, new_height).into());
+	let chunked: Vec<_> = bounds_of_viewports.chunks(4).map(ViewportBounds::from_slice).collect();
+	let ev = InputPreprocessorMessage::BoundsOfViewports((chunked).into());
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
-// TODO: When a mouse button is down that started in the viewport, this should trigger even when the mouse is outside the viewport (or even the browser window if the browser supports it)
 /// Mouse movement within the screenspace bounds of the viewport
 #[wasm_bindgen]
-pub fn on_mouse_move(x: f64, y: f64, modifiers: u8) -> Result<(), JsValue> {
+pub fn on_mouse_move(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Result<(), JsValue> {
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+	let editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
 
-	// TODO: Convert these screenspace viewport coordinates to canvas coordinates based on the current zoom and pan
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
-	let ev = InputPreprocessorMessage::MouseMove((x, y).into(), mods);
+
+	let ev = InputPreprocessorMessage::MouseMove(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
 /// Mouse scrolling within the screenspace bounds of the viewport
 #[wasm_bindgen]
-pub fn on_mouse_scroll(delta_x: i32, delta_y: i32, delta_z: i32, modifiers: u8) -> Result<(), JsValue> {
+pub fn on_mouse_scroll(x: f64, y: f64, mouse_keys: u8, wheel_delta_x: i32, wheel_delta_y: i32, wheel_delta_z: i32, modifiers: u8) -> Result<(), JsValue> {
-	// TODO: Convert these screenspace viewport coordinates to canvas coordinates based on the current zoom and pan
+	let mut editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+	editor_mouse_state.scroll_delta = ScrollDelta::new(wheel_delta_x, wheel_delta_y, wheel_delta_z);
-	let ev = InputPreprocessorMessage::MouseScroll(ScrollDelta::new(delta_x, delta_y, delta_z), mods);
+
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseScroll(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
 /// A mouse button depressed within screenspace the bounds of the viewport
 #[wasm_bindgen]
 pub fn on_mouse_down(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Result<(), JsValue> {
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+	let editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
-	let ev = InputPreprocessorMessage::MouseDown(MouseState::from_u8_pos(mouse_keys, (x, y).into()), mods);
+
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseDown(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
 /// A mouse button released
 #[wasm_bindgen]
 pub fn on_mouse_up(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Result<(), JsValue> {
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+	let editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
-	let ev = InputPreprocessorMessage::MouseUp(MouseState::from_u8_pos(mouse_keys, (x, y).into()), mods);
+
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseUp(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }