From 349ec5da72b8a6f6b12003eb63cf0272429e215a Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Sat, 3 Feb 2024 23:06:17 +0100 Subject: [PATCH] Add MemoizeImpure node and cache image base64 in graph (#1595) * Cache base64 representation of images when converting to graphic group * Fix build * Fix build again * Actually fix it this time --------- Co-authored-by: Keavon Chambers --- .../document_node_types.rs | 9 ++++ node-graph/gcore/src/graphic_element.rs | 14 +++++- .../gcore/src/graphic_element/renderer.rs | 25 +++++++--- node-graph/gcore/src/memo.rs | 49 ++++++++++++++++++- node-graph/gcore/src/raster.rs | 1 + node-graph/gcore/src/raster/image.rs | 18 ++++++- node-graph/gstd/src/gpu_nodes.rs | 3 ++ node-graph/gstd/src/image_color_palette.rs | 1 + node-graph/gstd/src/imaginate.rs | 9 +++- node-graph/gstd/src/raster.rs | 8 ++- node-graph/gstd/src/wasm_application_io.rs | 1 + .../interpreted-executor/src/node_registry.rs | 2 + 12 files changed, 127 insertions(+), 13 deletions(-) diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs index d67652a4..9d348583 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs @@ -1024,6 +1024,15 @@ fn static_nodes() -> Vec { manual_composition: Some(concrete!(())), ..Default::default() }, + DocumentNodeDefinition { + name: "MemoizeImpure", + category: "Structural", + implementation: NodeImplementation::proto("graphene_core::memo::ImpureMemoNode<_, _, _>"), + inputs: vec![DocumentInputType::value("Image", TaggedValue::ImageFrame(ImageFrame::empty()), true)], + outputs: vec![DocumentOutputType::new("Image", FrontendGraphDataType::Raster)], + manual_composition: Some(concrete!(Footprint)), + ..Default::default() + }, DocumentNodeDefinition { name: "Image", category: "Ignore", diff --git a/node-graph/gcore/src/graphic_element.rs b/node-graph/gcore/src/graphic_element.rs index 5c1de61c..91006b1a 100644 --- a/node-graph/gcore/src/graphic_element.rs +++ b/node-graph/gcore/src/graphic_element.rs @@ -159,7 +159,19 @@ async fn construct_artboard>( } impl From> for GraphicElement { - fn from(image_frame: ImageFrame) -> Self { + fn from(mut image_frame: ImageFrame) -> Self { + use base64::Engine; + + let image = &image_frame.image; + if !image.data.is_empty() { + let output = image.to_png(); + let preamble = "data:image/png;base64,"; + let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4); + base64_string.push_str(preamble); + base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string); + image_frame.image.base64_string = Some(base64_string); + } + GraphicElement::ImageFrame(image_frame) } } diff --git a/node-graph/gcore/src/graphic_element/renderer.rs b/node-graph/gcore/src/graphic_element/renderer.rs index a961f03b..93668955 100644 --- a/node-graph/gcore/src/graphic_element/renderer.rs +++ b/node-graph/gcore/src/graphic_element/renderer.rs @@ -1,6 +1,7 @@ mod quad; use crate::raster::{BlendMode, Image, ImageFrame}; +use crate::transform::Transform; use crate::uuid::{generate_uuid, ManipulatorGroupId}; use crate::{vector::VectorData, Artboard, Color, GraphicElement, GraphicGroup}; pub use quad::Quad; @@ -262,7 +263,16 @@ impl GraphicElementRendered for GraphicGroup { self.iter().filter_map(|element| element.bounding_box(transform * self.transform)).reduce(Quad::combine_bounds) } - fn add_click_targets(&self, _click_targets: &mut Vec) {} + fn add_click_targets(&self, click_targets: &mut Vec) { + for element in self.elements.iter().cloned() { + let mut new_click_targets = Vec::new(); + element.add_click_targets(&mut new_click_targets); + for click_target in new_click_targets.iter_mut() { + click_target.subpath.apply_transform(element.transform()) + } + click_targets.extend(new_click_targets); + } + } fn to_usvg_node(&self) -> usvg::Node { let root_node = usvg::Node::new(usvg::NodeKind::Group(usvg::Group::default())); @@ -453,12 +463,15 @@ impl GraphicElementRendered for ImageFrame { if image.data.is_empty() { return; } - let output = image.to_png(); - let preamble = "data:image/png;base64,"; - let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4); - base64_string.push_str(preamble); - base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string); + let base64_string = image.base64_string.clone().unwrap_or_else(|| { + let output = image.to_png(); + let preamble = "data:image/png;base64,"; + let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4); + base64_string.push_str(preamble); + base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string); + base64_string + }); render.leaf_tag("image", |attributes| { attributes.push("width", 1.to_string()); attributes.push("height", 1.to_string()); diff --git a/node-graph/gcore/src/memo.rs b/node-graph/gcore/src/memo.rs index 81d0c4a4..2d56899d 100644 --- a/node-graph/gcore/src/memo.rs +++ b/node-graph/gcore/src/memo.rs @@ -7,7 +7,7 @@ use core::cell::Cell; use core::marker::PhantomData; use core::pin::Pin; -// Caches the output of a given Node and acts as a proxy +/// Caches the output of a given Node and acts as a proxy #[derive(Default)] pub struct MemoNode { cache: Cell>, @@ -45,6 +45,53 @@ impl MemoNode { } } +/// Caches the output of a given Node and acts as a proxy. +/// In contrast to the relgular `MemoNode`. This node ignores all input. +/// Using this node might result in the document not updating properly, +/// use with caution. +#[derive(Default)] +pub struct ImpureMemoNode { + cache: Cell>, + node: CachedNode, + _phantom: std::marker::PhantomData, +} + +impl<'i, 'o: 'i, I: 'i, T: 'i + Clone + 'o, CachedNode: 'i> Node<'i, I> for ImpureMemoNode +where + CachedNode: for<'any_input> Node<'any_input, I>, + for<'a> >::Output: core::future::Future + 'a, +{ + // TODO: This should return a reference to the cached cached_value + // but that requires a lot of lifetime magic <- This was suggested by copilot but is pretty accurate xD + type Output = Pin + 'i>>; + fn eval(&'i self, input: I) -> Pin + 'i>> { + Box::pin(async move { + if let Some(cached_value) = self.cache.take() { + self.cache.set(Some(cached_value.clone())); + cached_value + } else { + let value = self.node.eval(input).await; + self.cache.set(Some(value.clone())); + value + } + }) + } + + fn reset(&self) { + self.cache.set(None); + } +} + +impl ImpureMemoNode { + pub const fn new(node: CachedNode) -> ImpureMemoNode { + ImpureMemoNode { + cache: Cell::new(None), + node, + _phantom: core::marker::PhantomData, + } + } +} + /// Stores both what a node was called with and what it returned. #[derive(Clone, Debug)] pub struct IORecord { diff --git a/node-graph/gcore/src/raster.rs b/node-graph/gcore/src/raster.rs index 765989f1..b1e617e0 100644 --- a/node-graph/gcore/src/raster.rs +++ b/node-graph/gcore/src/raster.rs @@ -645,6 +645,7 @@ mod test { width: 5, height: 5, data: vec![Color::from_rgbf32_unchecked(1., 0., 0.); 25], + base64_string: None, }); let image = image.then(ImageRefNode::new()); let window = WindowNode::new(radius, image); diff --git a/node-graph/gcore/src/raster/image.rs b/node-graph/gcore/src/raster/image.rs index 02f9aac1..0ea19c71 100644 --- a/node-graph/gcore/src/raster/image.rs +++ b/node-graph/gcore/src/raster/image.rs @@ -46,6 +46,10 @@ pub struct Image { pub height: u32, #[cfg_attr(feature = "serde", serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64"))] pub data: Vec

, + /// Optional: Stores a base64 string representation of the image which can be used to speed up the conversion + /// to an svg string. This is used as a cache in order to not have to encode the data on every graph evaluation. + #[cfg_attr(feature = "serde", serde(skip))] + pub base64_string: Option, } impl Debug for Image

{ @@ -108,6 +112,7 @@ impl Image

{ width: 0, height: 0, data: Vec::new(), + base64_string: None, } } @@ -116,6 +121,7 @@ impl Image

{ width, height, data: vec![color; (width * height) as usize], + base64_string: None, } } @@ -132,7 +138,12 @@ impl Image { /// Generate Image from some frontend image data (the canvas pixels as u8s in a flat array) pub fn from_image_data(image_data: &[u8], width: u32, height: u32) -> Self { let data = image_data.chunks_exact(4).map(|v| Color::from_rgba8_srgb(v[0], v[1], v[2], v[3])).collect(); - Image { width, height, data } + Image { + width, + height, + data, + base64_string: None, + } } pub fn to_png(&self) -> Vec { @@ -153,7 +164,7 @@ where { /// Flattens each channel cast to a u8 pub fn to_flat_u8(&self) -> (Vec, u32, u32) { - let Image { width, height, data } = self; + let Image { width, height, data, .. } = self; assert_eq!(data.len(), *width as usize * *height as usize); // Cache the last sRGB value we computed, speeds up fills. @@ -243,6 +254,7 @@ fn map_node(input: (u32, u32), data: Vec

) -> Image

{ width: input.0, height: input.1, data, + base64_string: None, } } @@ -379,6 +391,7 @@ impl From> for ImageFrame { data, width: image.image.width, height: image.image.height, + base64_string: None, }, transform: image.transform, alpha_blending: image.alpha_blending, @@ -394,6 +407,7 @@ impl From> for ImageFrame { data, width: image.image.width, height: image.image.height, + base64_string: None, }, transform: image.transform, alpha_blending: image.alpha_blending, diff --git a/node-graph/gstd/src/gpu_nodes.rs b/node-graph/gstd/src/gpu_nodes.rs index ef3e5466..92e5ded8 100644 --- a/node-graph/gstd/src/gpu_nodes.rs +++ b/node-graph/gstd/src/gpu_nodes.rs @@ -88,6 +88,7 @@ async fn map_gpu<'a: 'input>(image: ImageFrame, node: DocumentNode, edito data: image.image.data.iter().map(|c| quantization::quantize_color(*c, quantization)).collect(), width: image.image.width, height: image.image.height, + base64_string: None, }, transform: image.transform, alpha_blending: image.alpha_blending, @@ -140,6 +141,7 @@ async fn map_gpu<'a: 'input>(image: ImageFrame, node: DocumentNode, edito data: colors, width: image.image.width, height: image.image.height, + ..Default::default() }, transform: image.transform, alpha_blending: image.alpha_blending, @@ -586,6 +588,7 @@ async fn blend_gpu_image(foreground: ImageFrame, background: ImageFrame( fn image_to_base64(image: Image

) -> Result { use base64::prelude::*; - let Image { width, height, data } = image; + let Image { width, height, data, .. } = image; fn cast_with_f32>(data: Vec, width: u32, height: u32) -> Result where @@ -485,7 +485,12 @@ fn base64_to_image, P: Pixel>(base64_data: D) -> Result, _ => return Err(Error::UnsupportedPixelType(core::any::type_name::

())), }; - Ok(Image { data: result_data, width, height }) + Ok(Image { + data: result_data, + width, + height, + base64_string: None, + }) } pub fn pick_safe_imaginate_resolution((width, height): (f64, f64)) -> (u64, u64) { diff --git a/node-graph/gstd/src/raster.rs b/node-graph/gstd/src/raster.rs index dd1f1158..a66a10f1 100644 --- a/node-graph/gstd/src/raster.rs +++ b/node-graph/gstd/src/raster.rs @@ -111,6 +111,7 @@ fn sample(footprint: Footprint, image_frame: ImageFrame) -> ImageFrame ImageFrame { } } ImageFrame { - image: Image { width, height, data }, + image: Image { + width, + height, + data, + ..Default::default() + }, transform: DAffine2::from_translation(offset) * DAffine2::from_scale(size), ..Default::default() } diff --git a/node-graph/gstd/src/wasm_application_io.rs b/node-graph/gstd/src/wasm_application_io.rs index 0a6cd9cd..fc74935e 100644 --- a/node-graph/gstd/src/wasm_application_io.rs +++ b/node-graph/gstd/src/wasm_application_io.rs @@ -277,6 +277,7 @@ fn decode_image_node<'a: 'input>(data: Arc<[u8]>) -> ImageFrame { data: image.chunks(4).map(|pixel| Color::from_unassociated_alpha(pixel[0], pixel[1], pixel[2], pixel[3])).collect(), width: image.width(), height: image.height(), + ..Default::default() }, ..Default::default() }; diff --git a/node-graph/interpreted-executor/src/node_registry.rs b/node-graph/interpreted-executor/src/node_registry.rs index 01c08d8d..356c4888 100644 --- a/node-graph/interpreted-executor/src/node_registry.rs +++ b/node-graph/interpreted-executor/src/node_registry.rs @@ -652,6 +652,8 @@ fn node_registry() -> HashMap, input: (), output: wgpu_executor::WgpuSurface, params: [wgpu_executor::WgpuSurface]), async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: SurfaceFrame, params: [SurfaceFrame]), async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: RenderOutput, params: [RenderOutput]), + async_node!(graphene_core::memo::ImpureMemoNode<_, _, _>, input: Footprint, output: GraphicGroup, fn_params: [Footprint => GraphicGroup]), + async_node!(graphene_core::memo::ImpureMemoNode<_, _, _>, input: Footprint, output: VectorData, fn_params: [Footprint => VectorData]), register_node!(graphene_core::structural::ConsNode<_, _>, input: Image, params: [&str]), register_node!(graphene_std::raster::ImageFrameNode<_, _>, input: Image, params: [DAffine2]), register_node!(graphene_std::raster::NoisePatternNode<_, _, _, _, _, _, _, _, _, _, _, _, _, _, _>, input: (), params: [UVec2, u32, f32, NoiseType, DomainWarpType, f32, FractalType, u32, f32, f32, f32, f32, CellularDistanceFunction, CellularReturnType, f32]),