From 3adcc3031ae5de3662fa8d2690349f81a82d8515 Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Wed, 3 May 2023 13:14:41 +0200 Subject: [PATCH] Implement Infrastructure to reuse previous frames for brush drawing Implement Infrastructuro to reuse the previous evaluation of the node graph to blend the new stroke with instead of drawing the entire trace from scratch. This does not transition to a blending based approach because that still caused regressions but allows the brush node to work with input data natively. Test Plan: - Use the brush tool in the editor and check for regressions - Evaluate the performance Reviewers: Keavon Pull Request: https://github.com/GraphiteEditor/Graphite/pull/1190 --- editor/src/dispatcher.rs | 1 + .../document_node_types.rs | 13 ++ .../node_properties.rs | 8 +- .../portfolio/portfolio_message_handler.rs | 6 +- .../src/messages/tool/tool_message_handler.rs | 8 +- .../messages/tool/tool_messages/brush_tool.rs | 83 +++++++--- editor/src/messages/tool/utility_types.rs | 4 + editor/src/node_graph_executor.rs | 4 +- frontend/package.json | 2 +- frontend/wasm/src/editor_api.rs | 13 +- node-graph/gcore/src/lib.rs | 5 +- node-graph/gcore/src/raster.rs | 1 + node-graph/gcore/src/raster/color.rs | 2 + node-graph/gstd/src/any.rs | 48 ++++-- node-graph/gstd/src/brush.rs | 34 ----- node-graph/gstd/src/memo.rs | 32 ++-- node-graph/gstd/src/raster.rs | 144 +++++++++++------- .../interpreted-executor/src/executor.rs | 19 ++- .../interpreted-executor/src/node_registry.rs | 27 ++-- 19 files changed, 282 insertions(+), 172 deletions(-) diff --git a/editor/src/dispatcher.rs b/editor/src/dispatcher.rs index 3d7b6539..c744e365 100644 --- a/editor/src/dispatcher.rs +++ b/editor/src/dispatcher.rs @@ -170,6 +170,7 @@ impl Dispatcher { self.message_handlers.portfolio_message_handler.active_document_id().unwrap(), &self.message_handlers.input_preprocessor_message_handler, &self.message_handlers.portfolio_message_handler.persistent_data, + &self.message_handlers.portfolio_message_handler.executor, ), ); } else { diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs index a590e6ac..f9769ed7 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs @@ -106,6 +106,18 @@ fn static_nodes() -> Vec { outputs: vec![DocumentOutputType::new("Out", FrontendGraphDataType::General)], properties: |_document_node, _node_id, _context| node_properties::string_properties("The identity node simply returns the input"), }, + DocumentNodeType { + name: "Monitor", + category: "General", + identifier: NodeImplementation::proto("graphene_core::ops::IdNode"), + inputs: vec![DocumentInputType { + name: "In", + data_type: FrontendGraphDataType::General, + default: NodeInput::value(TaggedValue::None, true), + }], + outputs: vec![DocumentOutputType::new("Out", FrontendGraphDataType::General)], + properties: |_document_node, _node_id, _context| node_properties::string_properties("The Monitor node stores the value of its last evaluation"), + }, DocumentNodeType { name: "Downres", category: "Ignore", @@ -457,6 +469,7 @@ fn static_nodes() -> Vec { identifier: NodeImplementation::proto("graphene_std::brush::BrushNode"), inputs: vec![ DocumentInputType::value("None", TaggedValue::None, false), + DocumentInputType::value("Background", TaggedValue::ImageFrame(ImageFrame::empty()), true), DocumentInputType::value("Trace", TaggedValue::VecDVec2((0..2).map(|x| DVec2::new(x as f64 * 10., 0.)).collect()), true), DocumentInputType::value("Diameter", TaggedValue::F64(40.), false), DocumentInputType::value("Hardness", TaggedValue::F64(50.), false), diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs index 4d55515e..dbd1c9c8 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs @@ -588,11 +588,11 @@ pub fn blur_image_properties(document_node: &DocumentNode, node_id: NodeId, _con } pub fn brush_node_properties(document_node: &DocumentNode, node_id: NodeId, _context: &mut NodePropertiesContext) -> Vec { - let color = color_widget(document_node, node_id, 5, "Color", ColorInput::default(), true); + let color = color_widget(document_node, node_id, 6, "Color", ColorInput::default(), true); - let size = number_widget(document_node, node_id, 2, "Diameter", NumberInput::default().min(1.).max(100.).unit(" px"), true); - let hardness = number_widget(document_node, node_id, 3, "Hardness", NumberInput::default().min(0.).max(100.).unit("%"), true); - let flow = number_widget(document_node, node_id, 4, "Flow", NumberInput::default().min(1.).max(100.).unit("%"), true); + let size = number_widget(document_node, node_id, 3, "Diameter", NumberInput::default().min(1.).max(100.).unit(" px"), true); + let hardness = number_widget(document_node, node_id, 4, "Hardness", NumberInput::default().min(0.).max(100.).unit("%"), true); + let flow = number_widget(document_node, node_id, 5, "Flow", NumberInput::default().min(1.).max(100.).unit("%"), true); vec![color, LayoutGroup::Row { widgets: size }, LayoutGroup::Row { widgets: hardness }, LayoutGroup::Row { widgets: flow }] } diff --git a/editor/src/messages/portfolio/portfolio_message_handler.rs b/editor/src/messages/portfolio/portfolio_message_handler.rs index 0a41d44c..6da62867 100644 --- a/editor/src/messages/portfolio/portfolio_message_handler.rs +++ b/editor/src/messages/portfolio/portfolio_message_handler.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use super::utility_types::PersistentData; use crate::application::generate_uuid; use crate::consts::{DEFAULT_DOCUMENT_NAME, GRAPHITE_DOCUMENT_VERSION}; @@ -25,7 +27,7 @@ pub struct PortfolioMessageHandler { menu_bar_message_handler: MenuBarMessageHandler, documents: HashMap, document_ids: Vec, - executor: NodeGraphExecutor, + pub executor: NodeGraphExecutor, active_document_id: Option, copy_buffer: [Vec; INTERNAL_CLIPBOARD_COUNT as usize], pub persistent_data: PersistentData, @@ -564,7 +566,7 @@ impl MessageHandler Option { + pub fn introspect_node(&self, node_path: &[NodeId]) -> Option> { self.executor.introspect_node(node_path) } diff --git a/editor/src/messages/tool/tool_message_handler.rs b/editor/src/messages/tool/tool_message_handler.rs index 044503a8..600f3ecd 100644 --- a/editor/src/messages/tool/tool_message_handler.rs +++ b/editor/src/messages/tool/tool_message_handler.rs @@ -7,6 +7,7 @@ use crate::messages::layout::utility_types::misc::LayoutTarget; use crate::messages::portfolio::utility_types::PersistentData; use crate::messages::prelude::*; use crate::messages::tool::utility_types::ToolType; +use crate::node_graph_executor::NodeGraphExecutor; use document_legacy::layers::style::RenderData; use graphene_core::raster::color::Color; @@ -19,13 +20,13 @@ pub struct ToolMessageHandler { pub shape_editor: ShapeState, } -impl MessageHandler for ToolMessageHandler { +impl MessageHandler for ToolMessageHandler { #[remain::check] fn process_message( &mut self, message: ToolMessage, responses: &mut VecDeque, - (document, document_id, input, persistent_data): (&DocumentMessageHandler, u64, &InputPreprocessorMessageHandler, &PersistentData), + (document, document_id, input, persistent_data, node_graph): (&DocumentMessageHandler, u64, &InputPreprocessorMessageHandler, &PersistentData, &NodeGraphExecutor), ) { let render_data = RenderData::new(&persistent_data.font_cache, document.view_mode, None); @@ -94,6 +95,7 @@ impl MessageHandler, + points: Vec>, diameter: f64, hardness: f64, flow: f64, @@ -181,12 +185,28 @@ struct BrushToolData { impl BrushToolData { fn update_points(&self, responses: &mut VecDeque) { + if let Some(layer_path) = self.path.clone() { + let points = self.points.iter().flatten().cloned().collect(); + responses.add(NodeGraphMessage::SetQualifiedInputValue { + layer_path, + node_path: vec![0], + input_index: 2, + value: TaggedValue::VecDVec2(points), + }); + } + } + fn update_image(&self, node_graph: &NodeGraphExecutor, responses: &mut VecDeque) { + let Some(image) = node_graph.introspect_node(&[1]) else { return; }; + let image: &ImageFrame = image.downcast_ref().unwrap(); + self.set_image(image.clone(), responses) + } + fn set_image(&self, image_frame: ImageFrame, responses: &mut VecDeque) { if let Some(layer_path) = self.path.clone() { responses.add(NodeGraphMessage::SetQualifiedInputValue { layer_path, node_path: vec![0], input_index: 1, - value: TaggedValue::VecDVec2(self.points.clone()), + value: TaggedValue::ImageFrame(image_frame), }); } } @@ -201,7 +221,11 @@ impl Fsm for BrushToolFsmState { event: ToolMessage, tool_data: &mut Self::ToolData, ToolActionHandlerData { - document, global_tool_data, input, .. + document, + global_tool_data, + input, + node_graph, + .. }: &mut ToolActionHandlerData, tool_options: &Self::ToolOptions, responses: &mut VecDeque, @@ -217,9 +241,12 @@ impl Fsm for BrushToolFsmState { responses.add(DocumentMessage::StartTransaction); let existing_points = load_existing_points(document); let new_layer = existing_points.is_none(); - if let Some((layer_path, points)) = existing_points { + if let Some((layer_path, points, image)) = existing_points { tool_data.path = Some(layer_path); - tool_data.points = points; + //tool_data.set_image(image, responses); + if tool_data.points.is_empty() { + tool_data.points.push(points); + } } else { responses.add(DocumentMessage::DeselectAllLayers); tool_data.path = Some(document.get_path_for_new_layer()); @@ -227,7 +254,7 @@ impl Fsm for BrushToolFsmState { let pos = transform.inverse().transform_point2(input.mouse.position); - tool_data.points.push(pos); + tool_data.points.push(vec![pos]); tool_data.diameter = tool_options.diameter; tool_data.hardness = tool_options.hardness; @@ -236,6 +263,7 @@ impl Fsm for BrushToolFsmState { if new_layer { add_brush_render(tool_data, global_tool_data, responses); } else { + //tool_data.update_image(node_graph, responses); tool_data.update_points(responses); } @@ -244,15 +272,21 @@ impl Fsm for BrushToolFsmState { (Drawing, PointerMove) => { let pos = transform.inverse().transform_point2(input.mouse.position); - if tool_data.points.last() != Some(&pos) { + if tool_data.points.last().and_then(|x| x.last()) != Some(&pos) { // Linear interpolation for when the mouse has moved a lot between frames - if let Some(&last_point) = tool_data.points.last() { + if let Some(&last_point) = tool_data.points.last().and_then(|x| x.last()) { let distance = (last_point - pos).length(); let extra_points = (distance / (tool_data.diameter / 2.)).floor() as usize; - tool_data.points.extend((0..extra_points).map(|i| last_point.lerp(pos, (i as f64 + 1.) / (extra_points as f64 + 1.)))); + tool_data + .points + .last_mut() + .unwrap() + .extend((0..extra_points).map(|i| last_point.lerp(pos, (i as f64 + 1.) / (extra_points as f64 + 1.)))); } - tool_data.points.push(pos); + if let Some(x) = tool_data.points.last_mut() { + x.push(pos) + } } tool_data.update_points(responses); @@ -266,8 +300,8 @@ impl Fsm for BrushToolFsmState { responses.add(DocumentMessage::AbortTransaction); } - tool_data.path = None; tool_data.points.clear(); + tool_data.path = None; Ready } @@ -294,11 +328,13 @@ impl Fsm for BrushToolFsmState { fn add_brush_render(data: &BrushToolData, tool_data: &DocumentToolData, responses: &mut VecDeque) { let layer_path = data.path.clone().unwrap(); + let brush_node = DocumentNode { name: "Brush".to_string(), inputs: vec![ - NodeInput::ShortCircut(concrete!(())), - NodeInput::value(TaggedValue::VecDVec2(data.points.clone()), false), + NodeInput::value(TaggedValue::None, false), + NodeInput::value(TaggedValue::ImageFrame(ImageFrame::empty()), true), + NodeInput::value(TaggedValue::VecDVec2(data.points.last().cloned().unwrap_or_default()), false), // Diameter NodeInput::value(TaggedValue::F64(data.diameter), false), // Hardness @@ -312,12 +348,18 @@ fn add_brush_render(data: &BrushToolData, tool_data: &DocumentToolData, response metadata: graph_craft::document::DocumentNodeMetadata { position: (8, 4).into() }, ..Default::default() }; + let monitor_node = DocumentNode { + name: "Monitor".to_string(), + implementation: DocumentNodeImplementation::Unresolved("graphene_std::memo::MonitorNode<_>".into()), + ..Default::default() + }; let mut network = NodeNetwork::value_network(brush_node); + //network.push_node(monitor_node, true); network.push_output_node(); graph_modification_utils::new_custom_layer(network, layer_path, responses); } -fn load_existing_points(document: &DocumentMessageHandler) -> Option<(Vec, Vec)> { +fn load_existing_points(document: &DocumentMessageHandler) -> Option<(Vec, Vec, ImageFrame)> { if document.selected_layers().count() != 1 { return None; } @@ -327,11 +369,16 @@ fn load_existing_points(document: &DocumentMessageHandler) -> Option<(Vec { pub render_data: &'a RenderData<'a>, pub shape_overlay: &'a mut OverlayRenderer, pub shape_editor: &'a mut ShapeState, + pub node_graph: &'a NodeGraphExecutor, } impl<'a> ToolActionHandlerData<'a> { pub fn new( @@ -37,6 +39,7 @@ impl<'a> ToolActionHandlerData<'a> { render_data: &'a RenderData<'a>, shape_overlay: &'a mut OverlayRenderer, shape_editor: &'a mut ShapeState, + node_graph: &'a NodeGraphExecutor, ) -> Self { Self { document, @@ -46,6 +49,7 @@ impl<'a> ToolActionHandlerData<'a> { render_data, shape_overlay, shape_editor, + node_graph, } } } diff --git a/editor/src/node_graph_executor.rs b/editor/src/node_graph_executor.rs index aee73dd0..e7d8d0f4 100644 --- a/editor/src/node_graph_executor.rs +++ b/editor/src/node_graph_executor.rs @@ -17,6 +17,7 @@ use interpreted_executor::executor::DynamicExecutor; use glam::{DAffine2, DVec2}; use std::borrow::Cow; +use std::sync::Arc; #[derive(Debug, Clone, Default)] pub struct NodeGraphExecutor { @@ -37,6 +38,7 @@ impl NodeGraphExecutor { assert_eq!(scoped_network.outputs.len(), 1, "Graph with multiple outputs not yet handled"); let c = Compiler {}; let proto_network = c.compile_single(scoped_network, true)?; + assert_ne!(proto_network.nodes.len(), 0, "No protonodes exist?"); if let Err(e) = self.executor.update(proto_network) { error!("Failed to update executor:\n{}", e); @@ -53,7 +55,7 @@ impl NodeGraphExecutor { } } - pub fn introspect_node(&self, path: &[NodeId]) -> Option { + pub fn introspect_node(&self, path: &[NodeId]) -> Option> { self.executor.introspect(path).flatten() } diff --git a/frontend/package.json b/frontend/package.json index 489c707b..a9f8308a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,7 +19,7 @@ "build-wasm": "wasm-pack build ./wasm --dev --target=web", "build-wasm-prod": "wasm-pack build ./wasm --release --target=web", "tauri:build-wasm": "wasm-pack build ./wasm --release --target=web -- --features tauri", - "watch:wasm": "cargo watch --postpone --workdir wasm --shell \"wasm-pack build . --dev --target=web -- --color always\"", + "watch:wasm": "cargo watch --postpone --watch-when-idle --workdir wasm --shell \"wasm-pack build . --dev --target=web -- --color always\"", "--------------------": "", "print-building-help": "echo 'Graphite project failed to build. Did you remember to `npm install` the dependencies in `/frontend`?'", "print-linting-help": "echo 'Graphite project had lint errors, or may have otherwise failed. In the latter case, did you remember to `npm install` the dependencies in `/frontend`?'" diff --git a/frontend/wasm/src/editor_api.rs b/frontend/wasm/src/editor_api.rs index 3101243a..beb62e36 100644 --- a/frontend/wasm/src/editor_api.rs +++ b/frontend/wasm/src/editor_api.rs @@ -708,21 +708,26 @@ impl JsEditorHandle { /// Returns the string representation of the nodes contents #[wasm_bindgen(js_name = introspectNode)] - pub fn introspect_node(&self, node_path: Vec) -> Option { + pub fn introspect_node(&self, node_path: Vec) -> JsValue { let frontend_messages = EDITOR_INSTANCES.with(|instances| { // Mutably borrow the editors, and if successful, we can access them in the closure instances.try_borrow_mut().map(|mut editors| { // Get the editor instance for this editor ID, then dispatch the message to the backend, and return its response `FrontendMessage` queue - editors + let image = editors .get_mut(&self.editor_id) .expect("EDITOR_INSTANCES does not contain the current editor_id") .dispatcher .message_handlers .portfolio_message_handler - .introspect_node(&node_path) + .introspect_node(&node_path); + let image = image?; + let image = image.downcast_ref::>()?; + let serializer = serde_wasm_bindgen::Serializer::new().serialize_large_number_types_as_bigints(true); + let message_data = image.serialize(&serializer).expect("Failed to serialize FrontendMessage"); + Some(message_data) }) }); - frontend_messages.unwrap() + frontend_messages.unwrap().unwrap_or_default().into() } } diff --git a/node-graph/gcore/src/lib.rs b/node-graph/gcore/src/lib.rs index e366bb71..405150ad 100644 --- a/node-graph/gcore/src/lib.rs +++ b/node-graph/gcore/src/lib.rs @@ -39,8 +39,9 @@ pub trait Node<'i, Input: 'i>: 'i { type Output: 'i; fn eval(&'i self, input: Input) -> Self::Output; fn reset(self: Pin<&mut Self>) {} - #[cfg(feature = "alloc")] - fn serialize(&self) -> Option { + #[cfg(feature = "std")] + fn serialize(&self) -> Option> { + log::warn!("Node::serialize not implemented for {}", core::any::type_name::()); None } } diff --git a/node-graph/gcore/src/raster.rs b/node-graph/gcore/src/raster.rs index 1e0fd5dd..0f291357 100644 --- a/node-graph/gcore/src/raster.rs +++ b/node-graph/gcore/src/raster.rs @@ -139,6 +139,7 @@ pub trait UnassociatedAlpha: RGB + Alpha { pub trait Alpha { type AlphaChannel: Channel; + const TRANSPARENT: Self; fn alpha(&self) -> Self::AlphaChannel; fn a(&self) -> Self::AlphaChannel { self.alpha() diff --git a/node-graph/gcore/src/raster/color.rs b/node-graph/gcore/src/raster/color.rs index 223bb2cc..f52644df 100644 --- a/node-graph/gcore/src/raster/color.rs +++ b/node-graph/gcore/src/raster/color.rs @@ -96,6 +96,8 @@ impl Pixel for Color { impl Alpha for Color { type AlphaChannel = f32; + const TRANSPARENT: Self = Self::TRANSPARENT; + fn alpha(&self) -> f32 { self.alpha } diff --git a/node-graph/gstd/src/any.rs b/node-graph/gstd/src/any.rs index 107175cc..1c11e739 100644 --- a/node-graph/gstd/src/any.rs +++ b/node-graph/gstd/src/any.rs @@ -9,15 +9,45 @@ pub struct DynAnyNode { _i: PhantomData, _o: PhantomData, } -#[node_macro::node_fn(DynAnyNode<_I, _O>)] -fn any_node<_I: StaticType, _O: StaticType, N>(input: Any<'input>, node: &'any_input N) -> Any<'input> + +impl<'input, _I: 'input + StaticType, _O: 'input + StaticType, N: 'input, S0: 'input> Node<'input, Any<'input>> for DynAnyNode<_I, _O, S0> where N: for<'any_input> Node<'any_input, _I, Output = _O>, + S0: for<'any_input> Node<'any_input, (), Output = &'any_input N>, { - let node_name = core::any::type_name::(); - let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyNode Input, {e} in:\n{node_name}")); - Box::new(node.eval(*input)) + type Output = Any<'input>; + #[inline] + fn eval(&'input self, input: Any<'input>) -> Self::Output { + let node = self.node.eval(()); + { + let node_name = core::any::type_name::(); + let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyNode Input, {0} in:\n{1}", e, node_name)); + Box::new(node.eval(*input)) + } + } + + fn reset(self: std::pin::Pin<&mut Self>) { + let wrapped_node = unsafe { self.map_unchecked_mut(|e| &mut e.node) }; + Node::reset(wrapped_node); + } + + fn serialize(&self) -> Option> { + self.node.eval(()).serialize() + } } +impl<'input, _I: StaticType, _O: StaticType, N, S0: 'input> DynAnyNode<_I, _O, S0> +where + S0: for<'any_input> Node<'any_input, (), Output = &'any_input N>, +{ + pub const fn new(node: S0) -> Self { + Self { + node, + _i: core::marker::PhantomData, + _o: core::marker::PhantomData, + } + } +} + pub struct DynAnyRefNode { node: Node, _i: PhantomData<(I, O)>, @@ -28,11 +58,9 @@ where { type Output = Any<'input>; fn eval(&'input self, input: Any<'input>) -> Self::Output { - { - let node_name = core::any::type_name::(); - let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyRefNode Input, {e} in:\n{node_name}")); - Box::new(self.node.eval(*input)) - } + let node_name = core::any::type_name::(); + let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyRefNode Input, {e} in:\n{node_name}")); + Box::new(self.node.eval(*input)) } fn reset(self: std::pin::Pin<&mut Self>) { let wrapped_node = unsafe { self.map_unchecked_mut(|e| &mut e.node) }; diff --git a/node-graph/gstd/src/brush.rs b/node-graph/gstd/src/brush.rs index 2e4a5a33..7364f5a9 100644 --- a/node-graph/gstd/src/brush.rs +++ b/node-graph/gstd/src/brush.rs @@ -7,40 +7,6 @@ use graphene_core::vector::VectorData; use graphene_core::Node; use node_macro::node_fn; -// Spacing is a consistent 0.2 apart, even when tiled across pixels (from 0.9 to the neighboring 0.1), to avoid bias -const MULTISAMPLE_GRID: [(f64, f64); 25] = [ - // Row 1 - (0.1, 0.1), - (0.1, 0.3), - (0.1, 0.5), - (0.1, 0.7), - (0.1, 0.9), - // Row 2 - (0.3, 0.1), - (0.3, 0.3), - (0.3, 0.5), - (0.3, 0.7), - (0.3, 0.9), - // Row 3 - (0.5, 0.1), - (0.5, 0.3), - (0.5, 0.5), - (0.5, 0.7), - (0.5, 0.9), - // Row 4 - (0.7, 0.1), - (0.7, 0.3), - (0.7, 0.5), - (0.7, 0.7), - (0.7, 0.9), - // Row 5 - (0.9, 0.1), - (0.9, 0.3), - (0.9, 0.5), - (0.9, 0.7), - (0.9, 0.9), -]; - #[derive(Clone, Debug, PartialEq)] pub struct ReduceNode { pub initial: Initial, diff --git a/node-graph/gstd/src/memo.rs b/node-graph/gstd/src/memo.rs index 19ca6bab..19dce5cb 100644 --- a/node-graph/gstd/src/memo.rs +++ b/node-graph/gstd/src/memo.rs @@ -1,11 +1,12 @@ use graphene_core::Node; +#[cfg(feature = "serde")] use serde::Serialize; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::pin::Pin; use std::sync::atomic::AtomicBool; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use xxhash_rust::xxh3::Xxh3; /// Caches the output of a given Node and acts as a proxy @@ -53,32 +54,25 @@ impl CacheNode { /// Caches the output of the last graph evaluation for introspection #[derive(Default)] -pub struct MonitorNode { - output: Mutex>, - node: CachedNode, +pub struct MonitorNode { + output: Mutex>>, } -impl<'i, T: 'i + Serialize + Clone, I: 'i + Hash, CachedNode: 'i> Node<'i, I> for MonitorNode -where - CachedNode: for<'any_input> Node<'any_input, I, Output = T>, -{ +impl<'i, T: 'static + Clone> Node<'i, T> for MonitorNode { type Output = T; - fn eval(&'i self, input: I) -> Self::Output { - let output = self.node.eval(input); - *self.output.lock().unwrap() = Some(output.clone()); - output + fn eval(&'i self, input: T) -> Self::Output { + *self.output.lock().unwrap() = Some(Arc::new(input.clone())); + input } - fn serialize(&self) -> Option { + fn serialize(&self) -> Option> { let output = self.output.lock().unwrap(); - (*output).as_ref().and_then(|output| serde_json::to_string(output).ok()) + (*output).as_ref().and_then(|output| Some(output.clone() as Arc)) } } -impl std::marker::Unpin for MonitorNode {} - -impl MonitorNode { - pub const fn new(node: CachedNode) -> MonitorNode { - MonitorNode { output: Mutex::new(None), node } +impl MonitorNode { + pub const fn new() -> MonitorNode { + MonitorNode { output: Mutex::new(None) } } } diff --git a/node-graph/gstd/src/raster.rs b/node-graph/gstd/src/raster.rs index 41a889b3..fcbd0b19 100644 --- a/node-graph/gstd/src/raster.rs +++ b/node-graph/gstd/src/raster.rs @@ -1,9 +1,10 @@ use dyn_any::{DynAny, StaticType}; use glam::{DAffine2, DVec2}; -use graphene_core::raster::{Alpha, Channel, Image, ImageFrame, Luminance, Pixel, RasterMut, Sample}; +use graphene_core::raster::{Alpha, BlendMode, BlendNode, Channel, Image, ImageFrame, Luminance, Pixel, RasterMut, Sample}; use graphene_core::transform::Transform; -use graphene_core::Node; +use graphene_core::value::CopiedNode; +use graphene_core::{Color, Node}; use std::fmt::Debug; use std::marker::PhantomData; @@ -48,51 +49,6 @@ fn buffer_node(reader: R) -> Result, Error> { Ok(std::io::Read::bytes(reader).collect::, _>>()?) } -/* -pub fn file_node<'i, 's: 'i, P: AsRef + 'i>() -> impl Node<'i, 's, P, Output = Result, Error>> { - let fs = ValueNode(StdFs).then(CloneNode::new()); - let file = FileNode::new(fs); - - file.then(FlatMapResultNode::new(ValueNode::new(BufferNode))) -} - -pub fn image_node<'i, 's: 'i, P: AsRef + 'i>() -> impl Node<'i, 's, P, Output = Result> { - let file = file_node(); - let image_loader = FnNode::new(|data: Vec| image::load_from_memory(&data).map_err(Error::Image).map(|image| image.into_rgba32f())); - let image = file.then(FlatMapResultNode::new(ValueNode::new(image_loader))); - let convert_image = FnNode::new(|image: image::ImageBuffer<_, _>| { - let data = image - .enumerate_pixels() - .map(|(_, _, pixel): (_, _, &image::Rgba)| { - let c = pixel.channels(); - Color::from_rgbaf32(c[0], c[1], c[2], c[3]).unwrap() - }) - .collect(); - Image { - width: image.width(), - height: image.height(), - data, - } - }); - - image.then(MapResultNode::new(convert_image)) -} - -pub fn export_image_node<'i, 's: 'i>() -> impl Node<'i, 's, (Image, &'i str), Output = Result<(), Error>> { - FnNode::new(|input: (Image, &str)| { - let (image, path) = input; - let mut new_image = image::ImageBuffer::new(image.width, image.height); - for ((x, y, pixel), color) in new_image.enumerate_pixels_mut().zip(image.data.iter()) { - let color: Color = *color; - assert!(x < image.width); - assert!(y < image.height); - *pixel = image::Rgba(color.to_rgba8()) - } - new_image.save(path).map_err(Error::Image) - }) -} -*/ - pub struct DownresNode

{ _p: PhantomData

, } @@ -168,6 +124,17 @@ impl AxisAlignedBbox { end: DVec2::new(self.end.x.max(other.end.x), self.end.y.max(other.end.y)), } } + pub fn union_non_empty(&self, other: &AxisAlignedBbox) -> Option { + match (self.size() == DVec2::ZERO, other.size() == DVec2::ZERO) { + (true, true) => None, + (true, _) => Some(other.clone()), + (_, true) => Some(self.clone()), + _ => Some(AxisAlignedBbox { + start: DVec2::new(self.start.x.min(other.start.x), self.start.y.min(other.start.y)), + end: DVec2::new(self.end.x.max(other.end.x), self.end.y.max(other.end.y)), + }), + } + } } #[derive(Debug, Clone)] @@ -266,7 +233,7 @@ pub struct BlendImageTupleNode { } #[node_macro::node_fn(BlendImageTupleNode<_P, _Fg>)] -fn blend_image_tuple<_P: Pixel + Debug, MapFn, _Fg: Sample + Transform>(images: (ImageFrame<_P>, _Fg), map_fn: &'any_input MapFn) -> ImageFrame<_P> +fn blend_image_tuple<_P: Alpha + Pixel + Debug, MapFn, _Fg: Sample + Transform>(images: (ImageFrame<_P>, _Fg), map_fn: &'any_input MapFn) -> ImageFrame<_P> where MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P> + 'input + Clone, { @@ -282,25 +249,67 @@ pub struct BlendImageNode { _p: PhantomData

, } -// TODO: Implement proper blending #[node_macro::node_fn(BlendImageNode<_P>)] -fn blend_image_node<_P: Clone, MapFn, Frame: Sample + Transform, Background: RasterMut + Transform>( - foreground: Frame, - background: Background, - map_fn: &'any_input MapFn, -) -> Background +fn blend_image_node<_P: Alpha + Pixel + Debug, MapFn, Forground: Sample + Transform>(foreground: Forground, background: ImageFrame<_P>, map_fn: &'any_input MapFn) -> ImageFrame<_P> where MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P> + 'input, { - blend_image(foreground, background, map_fn) + blend_new_image(foreground, background, map_fn) } -fn blend_image<_P: Clone, MapFn, Frame: Sample + Transform, Background: RasterMut + Transform>(foreground: Frame, mut background: Background, map_fn: &MapFn) -> Background +#[derive(Debug, Clone, Copy)] +pub struct BlendReverseImageNode { + background: Background, + map_fn: MapFn, + _p: PhantomData

, +} + +#[node_macro::node_fn(BlendReverseImageNode<_P>)] +fn blend_image_node<_P: Alpha + Pixel + Debug, MapFn, Background: Transform + Sample>(foreground: ImageFrame<_P>, background: Background, map_fn: &'any_input MapFn) -> ImageFrame<_P> +where + MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P> + 'input, +{ + blend_new_image(background, foreground, map_fn) +} + +fn blend_new_image<_P: Alpha + Pixel + Debug, MapFn, Frame: Sample + Transform>(foreground: Frame, background: ImageFrame<_P>, map_fn: &MapFn) -> ImageFrame<_P> +where + MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P>, +{ + let foreground_aabb = compute_transformed_bounding_box(foreground.transform()).axis_aligned_bbox(); + let background_aabb = compute_transformed_bounding_box(background.transform()).axis_aligned_bbox(); + + let Some(aabb) = foreground_aabb.union_non_empty(&background_aabb) else {return ImageFrame::empty()}; + + if background_aabb.contains(foreground_aabb.start) && background_aabb.contains(foreground_aabb.end) { + return blend_image(foreground, background, map_fn); + } + + // Clamp the foreground image to the background image + let start = aabb.start.as_uvec2(); + let end = aabb.end.as_uvec2(); + + let new_background = Image::new(end.x - start.x, end.y - start.y, _P::TRANSPARENT); + let size = DVec2::new(new_background.width as f64, new_background.height as f64); + let transfrom = DAffine2::from_scale_angle_translation(size, 0., start.as_dvec2()); + let mut new_background = ImageFrame { + image: new_background, + transform: transfrom, + }; + + new_background = blend_image(background, new_background, map_fn); + blend_image(foreground, new_background, map_fn) +} + +fn blend_image<_P: Alpha + Pixel + Debug, MapFn, Frame: Sample + Transform, Background: RasterMut + Transform + Sample>( + foreground: Frame, + mut background: Background, + map_fn: &MapFn, +) -> Background where MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P>, { let background_size = DVec2::new(background.width() as f64, background.height() as f64); - // Transforms a point from the background image to the forground image let bg_to_fg = background.transform() * DAffine2::from_scale(1. / background_size); @@ -319,7 +328,7 @@ where if let Some(src_pixel) = foreground.sample(fg_point, area) { if let Some(dst_pixel) = background.get_pixel_mut(x, y) { - *dst_pixel = map_fn.eval((src_pixel, dst_pixel.clone())); + *dst_pixel = map_fn.eval((src_pixel, *dst_pixel)); } } } @@ -328,6 +337,23 @@ where background } +#[derive(Debug, Clone, Copy)] +pub struct ExtendImageNode { + background: Background, +} + +#[node_macro::node_fn(ExtendImageNode)] +fn extend_image_node(foreground: ImageFrame, background: ImageFrame) -> ImageFrame { + let foreground_aabb = compute_transformed_bounding_box(foreground.transform()).axis_aligned_bbox(); + let background_aabb = compute_transformed_bounding_box(background.transform()).axis_aligned_bbox(); + + if foreground_aabb.contains(background_aabb.start) && foreground_aabb.contains(background_aabb.end) { + return foreground; + } + + blend_image(foreground, background, &BlendNode::new(CopiedNode::new(BlendMode::Normal), CopiedNode::new(100.))) +} + #[derive(Clone, Debug, PartialEq)] pub struct MergeBoundingBoxNode { _data: PhantomData, @@ -340,7 +366,7 @@ fn merge_bounding_box_node<_Data: Transform>(input: (Option, _D let snd_aabb = compute_transformed_bounding_box(data.transform()).axis_aligned_bbox(); if let Some(fst_aabb) = initial_aabb { - Some(fst_aabb.union(&snd_aabb)) + fst_aabb.union_non_empty(&snd_aabb) } else { Some(snd_aabb) } diff --git a/node-graph/interpreted-executor/src/executor.rs b/node-graph/interpreted-executor/src/executor.rs index 659bf393..37d48472 100644 --- a/node-graph/interpreted-executor/src/executor.rs +++ b/node-graph/interpreted-executor/src/executor.rs @@ -17,6 +17,8 @@ pub struct DynamicExecutor { output: NodeId, tree: BorrowTree, typing_context: TypingContext, + // This allows us to keep the nodes around for one more frame which is used for introspection + orphaned_nodes: Vec, } impl Default for DynamicExecutor { @@ -25,6 +27,7 @@ impl Default for DynamicExecutor { output: Default::default(), tree: Default::default(), typing_context: TypingContext::new(&node_registry::NODE_REGISTRY), + orphaned_nodes: Vec::new(), } } } @@ -36,21 +39,27 @@ impl DynamicExecutor { let output = proto_network.output; let tree = BorrowTree::new(proto_network, &typing_context)?; - Ok(Self { tree, output, typing_context }) + Ok(Self { + tree, + output, + typing_context, + orphaned_nodes: Vec::new(), + }) } pub fn update(&mut self, proto_network: ProtoNetwork) -> Result<(), String> { self.output = proto_network.output; self.typing_context.update(&proto_network)?; trace!("setting output to {}", self.output); - let orphans = self.tree.update(proto_network, &self.typing_context)?; + let mut orphans = self.tree.update(proto_network, &self.typing_context)?; + core::mem::swap(&mut self.orphaned_nodes, &mut orphans); for node_id in orphans { self.tree.free_node(node_id) } Ok(()) } - pub fn introspect(&self, node_path: &[NodeId]) -> Option> { + pub fn introspect(&self, node_path: &[NodeId]) -> Option>> { self.tree.introspect(node_path) } @@ -128,7 +137,7 @@ impl BorrowTree { node.reset(); } old_nodes.remove(&id); - self.source_map.retain(|_, nid| *nid != id); + self.source_map.retain(|_, nid| !old_nodes.contains(nid)); } Ok(old_nodes.into_iter().collect()) } @@ -145,7 +154,7 @@ impl BorrowTree { node } - pub fn introspect(&self, node_path: &[NodeId]) -> Option> { + pub fn introspect(&self, node_path: &[NodeId]) -> Option>> { let id = self.source_map.get(node_path)?; let node = self.nodes.get(id)?; let reader = node.read().unwrap(); diff --git a/node-graph/interpreted-executor/src/node_registry.rs b/node-graph/interpreted-executor/src/node_registry.rs index 5feb68fa..5de2739f 100644 --- a/node-graph/interpreted-executor/src/node_registry.rs +++ b/node-graph/interpreted-executor/src/node_registry.rs @@ -154,7 +154,7 @@ fn node_registry() -> HashMap, input: ImageFrame, params: [ImageFrame]), register_node!(graphene_std::raster::MaskImageNode<_, _, _>, input: ImageFrame, params: [ImageFrame]), register_node!(graphene_std::raster::EmptyImageNode<_, _>, input: DAffine2, params: [Color]), - register_node!(graphene_std::memo::MonitorNode<_, _>, input: (), params: [ImageFrame]), + register_node!(graphene_std::memo::MonitorNode<_>, input: ImageFrame, params: []), #[cfg(feature = "gpu")] register_node!(graphene_std::executor::MapGpuSingleImageNode<_>, input: Image, params: [String]), vec![( @@ -175,14 +175,16 @@ fn node_registry() -> HashMap> = DowncastBothNode::new(args[0]); - let diameter: DowncastBothNode<(), f64> = DowncastBothNode::new(args[1]); - let hardness: DowncastBothNode<(), f64> = DowncastBothNode::new(args[2]); - let flow: DowncastBothNode<(), f64> = DowncastBothNode::new(args[3]); - let color: DowncastBothNode<(), Color> = DowncastBothNode::new(args[4]); + let image: DowncastBothNode<(), ImageFrame> = DowncastBothNode::new(args[0]); + let trace: DowncastBothNode<(), Vec> = DowncastBothNode::new(args[1]); + let diameter: DowncastBothNode<(), f64> = DowncastBothNode::new(args[2]); + let hardness: DowncastBothNode<(), f64> = DowncastBothNode::new(args[3]); + let flow: DowncastBothNode<(), f64> = DowncastBothNode::new(args[4]); + let color: DowncastBothNode<(), Color> = DowncastBothNode::new(args[5]); let stamp = BrushStampGeneratorNode::new(color, CopiedNode::new(hardness.eval(())), CopiedNode::new(flow.eval(()))); let stamp = stamp.eval(diameter.eval(())); @@ -191,16 +193,19 @@ fn node_registry() -> HashMap>(); - let background_bounds = ReduceNode::new(DebugClonedNode::new(None), ValueNode::new(MergeBoundingBoxNode::new())); + let background_bounds = ReduceNode::new(ClonedNode::new(None), ValueNode::new(MergeBoundingBoxNode::new())); let background_bounds = background_bounds.eval(frames.clone().into_iter()); - let background_bounds = DebugClonedNode::new(background_bounds.unwrap().to_transform()); + let background_bounds = MergeBoundingBoxNode::new().eval((background_bounds, image.eval(()))); + let background_bounds = ClonedNode::new(background_bounds.unwrap().to_transform()); let background_image = background_bounds.then(EmptyImageNode::new(CopiedNode::new(Color::TRANSPARENT))); - let blend_node = graphene_core::raster::BlendNode::new(CopiedNode::new(BlendMode::Normal), CopiedNode::new(100.)); + let background = ExtendImageNode::new(background_image); + let background_image = image.then(background); + let final_image = ReduceNode::new(background_image, ValueNode::new(BlendImageTupleNode::new(ValueNode::new(blend_node)))); - let final_image = DebugClonedNode::new(frames.into_iter()).then(final_image); + let final_image = ClonedNode::new(frames.into_iter()).then(final_image); let any: DynAnyNode<(), _, _> = graphene_std::any::DynAnyNode::new(ValueNode::new(final_image)); Box::pin(any) @@ -208,7 +213,7 @@ fn node_registry() -> HashMap), - vec![value_fn!(Vec), value_fn!(f64), value_fn!(f64), value_fn!(f64), value_fn!(Color)], + vec![value_fn!(ImageFrame), value_fn!(Vec), value_fn!(f64), value_fn!(f64), value_fn!(f64), value_fn!(Color)], ), )], vec![(