Reduce history system's excessive memory usage from storing copies of images each step (#845)

* Limit saved undo history

* Do not commit path every mouse move event

* Reference count images in history system instead of cloning the image data

* Increase history storage step limit to 100

Co-authored-by: Keavon Chambers <keavon@keavon.com>

editor/src/consts.rs

@@ -77,5 +77,6 @@ pub const DEFAULT_FONT_STYLE: &str = "Normal (400)";
 pub const GRAPHITE_DOCUMENT_VERSION: &str = "0.0.15"; // Remember to save a simple document and replace the test file `graphite-test-document.graphite`
 pub const DEFAULT_DOCUMENT_NAME: &str = "Untitled Document";
 pub const FILE_SAVE_SUFFIX: &str = ".graphite";
+pub const MAX_UNDO_HISTORY_LEN: usize = 100; // TODO: Add this to user preferences
 
 pub const VIEWPORT_ZOOM_TO_FIT_PADDING_SCALE_FACTOR: f32 = 1.05;

editor/src/messages/frontend/utility_types.rs

@@ -15,8 +15,8 @@ pub struct FrontendDocumentDetails {
 pub struct FrontendImageData {
 	pub path: Vec<LayerId>,
 	pub mime: String,
-	#[serde(rename = "imageData")]
+	#[serde(skip)]
-	pub image_data: Vec<u8>,
+	pub image_data: std::rc::Rc<Vec<u8>>,
 }
 
 #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]

editor/src/messages/portfolio/document/document_message_handler.rs

@@ -49,9 +49,9 @@ pub struct DocumentMessageHandler {
 	pub overlays_visible: bool,
 
 	#[serde(skip)]
-	pub document_undo_history: Vec<DocumentSave>,
+	pub document_undo_history: VecDeque<DocumentSave>,
 	#[serde(skip)]
-	pub document_redo_history: Vec<DocumentSave>,
+	pub document_redo_history: VecDeque<DocumentSave>,
 
 	#[serde(with = "vectorize_layer_metadata")]
 	pub layer_metadata: HashMap<Vec<LayerId>, LayerMetadata>,
@@ -80,8 +80,8 @@ impl Default for DocumentMessageHandler {
 			snapping_enabled: true,
 			overlays_visible: true,
 
-			document_undo_history: Vec::new(),
+			document_undo_history: VecDeque::new(),
-			document_redo_history: Vec::new(),
+			document_redo_history: VecDeque::new(),
 
 			layer_metadata: vec![(vec![], LayerMetadata::new(true))].into_iter().collect(),
 			layer_range_selection_reference: Vec::new(),
@@ -508,7 +508,6 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 				);
 			}
 			MoveSelectedManipulatorPoints { layer_path, delta } => {
-				self.backup(responses);
 				if let Ok(_layer) = self.graphene_document.layer(&layer_path) {
 					responses.push_back(DocumentOperation::MoveSelectedManipulatorPoints { layer_path, delta }.into());
 				}
@@ -541,6 +540,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 					}
 					.into(),
 				);
+				let image_data = std::rc::Rc::new(image_data);
 				responses.push_back(
 					FrontendMessage::UpdateImageData {
 						document_id,
@@ -1322,7 +1322,10 @@ impl DocumentMessageHandler {
 
 	pub fn backup(&mut self, responses: &mut VecDeque<Message>) {
 		self.document_redo_history.clear();
-		self.document_undo_history.push((self.graphene_document.clone(), self.layer_metadata.clone()));
+		self.document_undo_history.push_back((self.graphene_document.clone(), self.layer_metadata.clone()));
+		if self.document_undo_history.len() > crate::consts::MAX_UNDO_HISTORY_LEN {
+			self.document_undo_history.pop_front();
+		}
 
 		// Push the UpdateOpenDocumentsList message to the bus in order to update the save status of the open documents
 		responses.push_back(PortfolioMessage::UpdateOpenDocumentsList.into());
@@ -1340,7 +1343,7 @@ impl DocumentMessageHandler {
 
 		let selected_paths: Vec<Vec<LayerId>> = self.selected_layers().map(|path| path.to_vec()).collect();
 
-		match self.document_undo_history.pop() {
+		match self.document_undo_history.pop_back() {
 			Some((document, layer_metadata)) => {
 				// Update the currently displayed layer on the Properties panel if the selection changes after an undo action
 				// Also appropriately update the Properties panel if an undo action results in a layer being deleted
@@ -1352,7 +1355,10 @@ impl DocumentMessageHandler {
 
 				let document = std::mem::replace(&mut self.graphene_document, document);
 				let layer_metadata = std::mem::replace(&mut self.layer_metadata, layer_metadata);
-				self.document_redo_history.push((document, layer_metadata));
+				self.document_redo_history.push_back((document, layer_metadata));
+				if self.document_redo_history.len() > crate::consts::MAX_UNDO_HISTORY_LEN {
+					self.document_redo_history.pop_front();
+				}
 
 				for layer in self.layer_metadata.keys() {
 					responses.push_back(DocumentMessage::LayerChanged { affected_layer_path: layer.clone() }.into())
@@ -1370,7 +1376,7 @@ impl DocumentMessageHandler {
 
 		let selected_paths: Vec<Vec<LayerId>> = self.selected_layers().map(|path| path.to_vec()).collect();
 
-		match self.document_redo_history.pop() {
+		match self.document_redo_history.pop_back() {
 			Some((document, layer_metadata)) => {
 				// Update currently displayed layer on property panel if selection changes after redo action
 				// Also appropriately update property panel if redo action results in a layer being added
@@ -1382,7 +1388,10 @@ impl DocumentMessageHandler {
 
 				let document = std::mem::replace(&mut self.graphene_document, document);
 				let layer_metadata = std::mem::replace(&mut self.layer_metadata, layer_metadata);
-				self.document_undo_history.push((document, layer_metadata));
+				self.document_undo_history.push_back((document, layer_metadata));
+				if self.document_undo_history.len() > crate::consts::MAX_UNDO_HISTORY_LEN {
+					self.document_undo_history.pop_front();
+				}
 
 				for layer in self.layer_metadata.keys() {
 					responses.push_back(DocumentMessage::LayerChanged { affected_layer_path: layer.clone() }.into())
@@ -1398,6 +1407,7 @@ impl DocumentMessageHandler {
 		// We can use the last state of the document to serve as the identifier to compare against
 		// This is useful since when the document is empty the identifier will be 0
 		self.document_undo_history
+			.iter()
 			.last()
 			.map(|(graphene_document, _)| graphene_document.current_state_identifier())
 			.unwrap_or(0)

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -476,6 +476,7 @@ impl MessageHandler<PortfolioMessage, (&InputPreprocessorMessageHandler, &Prefer
 							.into(),
 						);
 						let mime = "image/bmp".to_string();
+						let image_data = std::rc::Rc::new(image_data);
 						responses.push_back(
 							FrontendMessage::UpdateImageData {
 								document_id,

frontend/wasm/src/editor_api.rs

@@ -32,7 +32,7 @@ pub fn set_random_seed(seed: u64) {
 /// This avoids creating a json with a list millions of numbers long.
 #[wasm_bindgen(module = "@/wasm-communication/editor")]
 extern "C" {
-	fn updateImage(path: Vec<u64>, mime: String, imageData: Vec<u8>, document_id: u64);
+	fn updateImage(path: Vec<u64>, mime: String, imageData: &[u8], document_id: u64);
 }
 
 /// Provides a handle to access the raw WASM memory
@@ -100,7 +100,7 @@ impl JsEditorHandle {
 		// Special case for update image data to avoid serialization times.
 		if let FrontendMessage::UpdateImageData { document_id, image_data } = message {
 			for image in image_data {
-				updateImage(image.path, image.mime, image.image_data, document_id);
+				updateImage(image.path, image.mime, &image.image_data, document_id);
 			}
 			return;
 		}

graphene/src/document.rs

@@ -583,6 +583,7 @@ impl Document {
 				image_data,
 				mime,
 			} => {
+				let image_data = std::rc::Rc::new(image_data);
 				let layer = Layer::new(LayerDataType::Image(ImageLayer::new(mime, image_data)), transform);
 
 				self.set_layer(&path, layer, insert_index)?;
@@ -606,6 +607,7 @@ impl Document {
 			Operation::SetNodeGraphFrameImageData { layer_path, image_data } => {
 				let layer = self.layer_mut(&layer_path).expect("Setting NodeGraphFrame image data for invalid layer");
 				if let LayerDataType::NodeGraphFrame(node_graph_frame) = &mut layer.data {
+					let image_data = std::rc::Rc::new(image_data);
 					node_graph_frame.image_data = Some(crate::layers::nodegraph_layer::ImageData { image_data });
 				} else {
 					panic!("Incorrectly trying to set image data for a layer that is not an NodeGraphFrame layer type");
@@ -831,6 +833,7 @@ impl Document {
 			Operation::ImaginateSetImageData { layer_path, image_data } => {
 				let layer = self.layer_mut(&layer_path).expect("Setting Imaginate image data for invalid layer");
 				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
+					let image_data = std::rc::Rc::new(image_data);
 					imaginate.image_data = Some(ImaginateImageData { image_data });
 				} else {
 					panic!("Incorrectly trying to set image data for a layer that is not an Imaginate layer type");

graphene/src/layers/base64_serde.rs

@@ -2,20 +2,21 @@
 
 use serde::{Deserialize, Deserializer, Serializer};
 
-pub fn as_base64<T, S>(key: &T, serializer: S) -> Result<S::Ok, S::Error>
+pub fn as_base64<S>(key: &std::rc::Rc<Vec<u8>>, serializer: S) -> Result<S::Ok, S::Error>
 where
-	T: AsRef<[u8]>,
 	S: Serializer,
 {
-	serializer.serialize_str(&base64::encode(key.as_ref()))
+	serializer.serialize_str(&base64::encode(key.as_slice()))
 }
 
-pub fn from_base64<'a, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+pub fn from_base64<'a, D>(deserializer: D) -> Result<std::rc::Rc<Vec<u8>>, D::Error>
 where
 	D: Deserializer<'a>,
 {
 	use serde::de::Error;
+
 	String::deserialize(deserializer)
-		.and_then(|string| base64::decode(&string).map_err(|err| Error::custom(err.to_string())))
+		.and_then(|string| base64::decode(string).map_err(|err| Error::custom(err.to_string())))
+		.map(std::rc::Rc::new)
 		.map_err(serde::de::Error::custom)
 }

graphene/src/layers/image_layer.rs

@@ -14,7 +14,7 @@ use std::fmt::Write;
 pub struct ImageLayer {
 	pub mime: String,
 	#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
-	pub image_data: Vec<u8>,
+	pub image_data: std::rc::Rc<Vec<u8>>,
 	// TODO: Have the browser dispose of this blob URL when this is dropped (like when the layer is deleted)
 	#[serde(skip)]
 	pub blob_url: Option<String>,
@@ -75,7 +75,7 @@ impl LayerData for ImageLayer {
 }
 
 impl ImageLayer {
-	pub fn new(mime: String, image_data: Vec<u8>) -> Self {
+	pub fn new(mime: String, image_data: std::rc::Rc<Vec<u8>>) -> Self {
 		Self {
 			mime,
 			image_data,

graphene/src/layers/imaginate_layer.rs

@@ -53,7 +53,7 @@ pub enum ImaginateStatus {
 #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)]
 pub struct ImaginateImageData {
 	#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
-	pub image_data: Vec<u8>,
+	pub image_data: std::rc::Rc<Vec<u8>>,
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]

graphene/src/layers/nodegraph_layer.rs

@@ -29,7 +29,7 @@ pub struct NodeGraphFrameLayer {
 #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)]
 pub struct ImageData {
 	#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
-	pub image_data: Vec<u8>,
+	pub image_data: std::rc::Rc<Vec<u8>>,
 }
 
 impl LayerData for NodeGraphFrameLayer {