From 259dcdc628bafb184bf4fbec4731a11929bb81ce Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Tue, 30 May 2023 20:12:59 +0200 Subject: [PATCH] Use canvas as target for raster rendering (#1256) * Implement ApplicationIo * Simplify output duplication logic * Fix let node initialization for ExtractImageFrame * Async macros * Use manual node registry impl * Fix canvas insertion into the dom --- Cargo.lock | 3 + document-legacy/src/document.rs | 6 + document-legacy/src/layers/layer_layer.rs | 15 ++ document-legacy/src/operation.rs | 4 + .../document_node_types.rs | 149 ++++++++--- .../messages/tool/tool_messages/brush_tool.rs | 6 +- editor/src/node_graph_executor.rs | 53 +++- .../src/components/panels/Document.svelte | 16 ++ frontend/src/wasm-communication/editor.ts | 1 + frontend/wasm/Cargo.toml | 7 +- node-graph/gcore/Cargo.toml | 18 +- node-graph/gcore/src/application_io.rs | 159 ++++++++++++ .../src/application_io/wasm_application_io.rs | 131 ++++++++++ node-graph/gcore/src/lib.rs | 7 +- node-graph/gcore/src/ops.rs | 32 +-- node-graph/gcore/src/raster.rs | 2 +- node-graph/gcore/src/raster/adjustments.rs | 4 +- node-graph/gcore/src/raster/color.rs | 87 ++++++- node-graph/gcore/src/raster/image.rs | 69 +++--- node-graph/gcore/src/text.rs | 2 +- node-graph/graph-craft/src/document.rs | 232 +++++++----------- node-graph/graph-craft/src/document/value.rs | 10 +- node-graph/graph-craft/src/executor.rs | 2 - node-graph/gstd/src/any.rs | 1 + node-graph/gstd/src/memo.rs | 3 +- .../interpreted-executor/src/executor.rs | 1 - .../interpreted-executor/src/node_registry.rs | 49 +++- 27 files changed, 810 insertions(+), 259 deletions(-) create mode 100644 node-graph/gcore/src/application_io.rs create mode 100644 node-graph/gcore/src/application_io/wasm_application_io.rs diff --git a/Cargo.lock b/Cargo.lock index 61b613a8..f9aa7ce3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1684,6 +1684,7 @@ dependencies = [ "bytemuck", "dyn-any", "glam", + "js-sys", "kurbo", "log", "node-macro", @@ -1696,6 +1697,8 @@ dependencies = [ "specta", "spin 0.9.8", "spirv-std", + "wasm-bindgen", + "web-sys", ] [[package]] diff --git a/document-legacy/src/document.rs b/document-legacy/src/document.rs index a2264067..8a14936b 100644 --- a/document-legacy/src/document.rs +++ b/document-legacy/src/document.rs @@ -871,6 +871,12 @@ impl Document { } Some(Vec::new()) } + Operation::SetSurface { path, surface_id } => { + if let LayerDataType::Layer(layer) = &mut self.layer_mut(&path)?.data { + layer.cached_output_data = CachedOutputData::SurfaceId(surface_id); + } + Some(Vec::new()) + } Operation::TransformLayerInScope { path, transform, scope } => { let transform = DAffine2::from_cols_array(&transform); let scope = DAffine2::from_cols_array(&scope); diff --git a/document-legacy/src/layers/layer_layer.rs b/document-legacy/src/layers/layer_layer.rs index e56138b0..f8ea93a6 100644 --- a/document-legacy/src/layers/layer_layer.rs +++ b/document-legacy/src/layers/layer_layer.rs @@ -5,6 +5,7 @@ use crate::LayerId; use glam::{DAffine2, DMat2, DVec2}; use graphene_core::vector::VectorData; +use graphene_core::SurfaceId; use kurbo::{Affine, BezPath, Shape as KurboShape}; use serde::{Deserialize, Serialize}; use std::fmt::Write; @@ -15,6 +16,7 @@ pub enum CachedOutputData { None, BlobURL(String), VectorPath(Box), + SurfaceId(SurfaceId), } #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)] @@ -76,6 +78,19 @@ impl LayerData for LayerLayer { matrix ); } + CachedOutputData::SurfaceId(SurfaceId(id)) => { + // Render the image if it exists + let _ = write!( + svg, + r#" +
+ "#, + width.abs(), + height.abs(), + matrix, + id + ); + } _ => { // Render a dotted blue outline if there is no image or vector data let _ = write!( diff --git a/document-legacy/src/operation.rs b/document-legacy/src/operation.rs index c3ff39a5..9d9c589e 100644 --- a/document-legacy/src/operation.rs +++ b/document-legacy/src/operation.rs @@ -76,6 +76,10 @@ pub enum Operation { path: Vec, vector_data: graphene_core::vector::VectorData, }, + SetSurface { + path: Vec, + surface_id: graphene_core::SurfaceId, + }, TransformLayerInScope { path: Vec, transform: [f64; 6], diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs index 6f4f8ec1..8e6c206e 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs @@ -184,7 +184,7 @@ fn static_nodes() -> Vec { }, DocumentNodeType { name: "Downres", - category: "Ignore", + category: "Raster", identifier: NodeImplementation::DocumentNode(NodeNetwork { inputs: vec![0], outputs: vec![NodeOutput::new(1, 0)], @@ -234,15 +234,87 @@ fn static_nodes() -> Vec { DocumentNodeType { name: "Input Frame", category: "Ignore", + identifier: NodeImplementation::proto("graphene_core::ExtractImageFrame"), + inputs: vec![DocumentInputType { + name: "In", + data_type: FrontendGraphDataType::General, + default: NodeInput::Network(concrete!(EditorApi)), + }], + outputs: vec![DocumentOutputType { + name: "Image Frame", + data_type: FrontendGraphDataType::Raster, + }], + properties: node_properties::input_properties, + }, + DocumentNodeType { + name: "Create Canvas", + category: "Structural", identifier: NodeImplementation::DocumentNode(NodeNetwork { - inputs: vec![0, 1], - outputs: vec![NodeOutput::new(0, 0), NodeOutput::new(1, 0)], - nodes: [DocumentNode { - name: "Identity".to_string(), - inputs: vec![NodeInput::Network(concrete!(EditorApi))], - implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::ExtractImageFrame")), - ..Default::default() - }] + inputs: vec![0], + outputs: vec![NodeOutput::new(1, 0)], + nodes: [ + DocumentNode { + name: "Create Canvas".to_string(), + inputs: vec![NodeInput::Network(concrete!(EditorApi))], + implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::wasm_application_io::CreateSurfaceNode")), + ..Default::default() + }, + DocumentNode { + name: "Cache".to_string(), + inputs: vec![NodeInput::ShortCircut(concrete!(())), NodeInput::node(0, 0)], + implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_std::memo::CacheNode")), + ..Default::default() + }, + ] + .into_iter() + .enumerate() + .map(|(id, node)| (id as NodeId, node)) + .collect(), + ..Default::default() + }), + inputs: vec![DocumentInputType { + name: "In", + data_type: FrontendGraphDataType::General, + default: NodeInput::Network(concrete!(EditorApi)), + }], + outputs: vec![DocumentOutputType { + name: "Canvas", + data_type: FrontendGraphDataType::General, + }], + properties: node_properties::input_properties, + }, + DocumentNodeType { + name: "Draw Canvas", + category: "Structural", + identifier: NodeImplementation::DocumentNode(NodeNetwork { + inputs: vec![0, 2], + outputs: vec![NodeOutput::new(3, 0)], + nodes: [ + DocumentNode { + name: "Convert Image Frame".to_string(), + inputs: vec![NodeInput::Network(concrete!(ImageFrame))], + implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::ops::IntoNode<_, ImageFrame>")), + ..Default::default() + }, + DocumentNode { + name: "Create Canvas".to_string(), + inputs: vec![NodeInput::Network(concrete!(EditorApi))], + implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::wasm_application_io::CreateSurfaceNode")), + ..Default::default() + }, + DocumentNode { + name: "Cache".to_string(), + inputs: vec![NodeInput::ShortCircut(concrete!(())), NodeInput::node(1, 0)], + implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_std::memo::CacheNode")), + ..Default::default() + }, + DocumentNode { + name: "Draw Canvas".to_string(), + inputs: vec![NodeInput::node(0, 0), NodeInput::node(2, 0)], + implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::wasm_application_io::DrawImageFrameNode<_>")), + ..Default::default() + }, + ] .into_iter() .enumerate() .map(|(id, node)| (id as NodeId, node)) @@ -252,14 +324,18 @@ fn static_nodes() -> Vec { inputs: vec![ DocumentInputType { name: "In", - data_type: FrontendGraphDataType::General, - default: NodeInput::Network(concrete!(ImageFrame)), + data_type: FrontendGraphDataType::Raster, + default: NodeInput::value(TaggedValue::ImageFrame(ImageFrame::empty()), true), + }, + DocumentInputType { + name: "In", + data_type: FrontendGraphDataType::General, + default: NodeInput::Network(concrete!(EditorApi)), }, - DocumentInputType::value("Transform", TaggedValue::DAffine2(DAffine2::IDENTITY), false), ], outputs: vec![DocumentOutputType { - name: "Image Frame", - data_type: FrontendGraphDataType::Raster, + name: "Canvas", + data_type: FrontendGraphDataType::General, }], properties: node_properties::input_properties, }, @@ -267,12 +343,12 @@ fn static_nodes() -> Vec { name: "Begin Scope", category: "Ignore", identifier: NodeImplementation::DocumentNode(NodeNetwork { - inputs: vec![0, 2], + inputs: vec![0], outputs: vec![NodeOutput::new(1, 0), NodeOutput::new(2, 0)], nodes: [ DocumentNode { name: "SetNode".to_string(), - inputs: vec![NodeInput::Network(concrete!(EditorApi))], + inputs: vec![NodeInput::ShortCircut(concrete!(EditorApi))], implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::ops::SomeNode")), ..Default::default() }, @@ -284,7 +360,7 @@ fn static_nodes() -> Vec { }, DocumentNode { name: "RefNode".to_string(), - inputs: vec![NodeInput::Network(concrete!(())), NodeInput::lambda(1, 0)], + inputs: vec![NodeInput::ShortCircut(concrete!(())), NodeInput::lambda(1, 0)], implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_std::memo::RefNode<_, _>")), ..Default::default() }, @@ -299,7 +375,7 @@ fn static_nodes() -> Vec { inputs: vec![DocumentInputType { name: "In", data_type: FrontendGraphDataType::Raster, - default: NodeInput::value(TaggedValue::EditorApi(EditorApi::empty()), true), + default: NodeInput::Network(concrete!(EditorApi)), }], outputs: vec![ DocumentOutputType { @@ -1243,24 +1319,39 @@ impl DocumentNodeType { } } -pub fn wrap_network_in_scope(network: NodeNetwork) -> NodeNetwork { - // if the network has no inputs, it doesn't need to be wrapped in a scope - if network.inputs.is_empty() { - return network; +pub fn wrap_network_in_scope(mut network: NodeNetwork) -> NodeNetwork { + let node_ids = network.nodes.keys().copied().collect::>(); + + network.generate_node_paths(&[]); + for id in node_ids { + network.flatten(id); } - assert_eq!(network.inputs.len(), 1, "Networks wrapped in scope must have exactly one input"); - let input = network.nodes[&network.inputs[0]].inputs.iter().find(|&i| matches!(i, NodeInput::Network(_))).cloned(); + let mut network_inputs = Vec::new(); + let mut input_type = None; + for (id, node) in network.nodes.iter() { + for (index, input) in node.inputs.iter().enumerate() { + if let NodeInput::Network(_) = input { + if input_type.is_none() { + input_type = Some(input.clone()); + } + assert_eq!(input, input_type.as_ref().unwrap(), "Networks wrapped in scope must have the same input type"); + network_inputs.push(*id); + } + } + } + let len = network_inputs.len(); + network.inputs = network_inputs; - // if the network has no network inputs, it doesn't need to be wrapped in a scope either - let Some(input_type) = input else { + // if the network has no inputs, it doesn't need to be wrapped in a scope + if len == 0 { return network; - }; + } let inner_network = DocumentNode { name: "Scope".to_string(), implementation: DocumentNodeImplementation::Network(network), - inputs: vec![NodeInput::node(0, 1)], + inputs: core::iter::repeat(NodeInput::node(0, 1)).take(len).collect(), ..Default::default() }; @@ -1268,7 +1359,7 @@ pub fn wrap_network_in_scope(network: NodeNetwork) -> NodeNetwork { let nodes = vec![ resolve_document_node_type("Begin Scope") .expect("Begin Scope node type not found") - .to_document_node(vec![input_type], DocumentNodeMetadata::default()), + .to_document_node(vec![input_type.unwrap()], DocumentNodeMetadata::default()), inner_network, resolve_document_node_type("End Scope") .expect("End Scope node type not found") diff --git a/editor/src/messages/tool/tool_messages/brush_tool.rs b/editor/src/messages/tool/tool_messages/brush_tool.rs index 7012afba..24765552 100644 --- a/editor/src/messages/tool/tool_messages/brush_tool.rs +++ b/editor/src/messages/tool/tool_messages/brush_tool.rs @@ -41,9 +41,9 @@ impl Default for BrushOptions { fn default() -> Self { Self { diameter: 40., - hardness: 50., + hardness: 0., flow: 100., - spacing: 50., + spacing: 20., color: ToolColorOptions::default(), } } @@ -252,7 +252,7 @@ impl BrushToolData { self.transform = DAffine2::IDENTITY; - matches!(layer.cached_output_data, CachedOutputData::BlobURL(_)).then_some(&self.layer_path) + matches!(layer.cached_output_data, CachedOutputData::BlobURL(_) | CachedOutputData::SurfaceId(_)).then_some(&self.layer_path) } fn update_strokes(&self, brush_options: &BrushOptions, responses: &mut VecDeque) { diff --git a/editor/src/node_graph_executor.rs b/editor/src/node_graph_executor.rs index 0ee52477..639b9183 100644 --- a/editor/src/node_graph_executor.rs +++ b/editor/src/node_graph_executor.rs @@ -7,16 +7,19 @@ use crate::messages::prelude::*; use document_legacy::layers::layer_info::LayerDataType; use document_legacy::{LayerId, Operation}; +use dyn_any::DynAny; use graph_craft::document::value::TaggedValue; use graph_craft::document::{generate_uuid, DocumentNodeImplementation, NodeId, NodeNetwork}; use graph_craft::executor::Compiler; use graph_craft::{concrete, Type, TypeDescriptor}; +use graphene_core::application_io::ApplicationIo; use graphene_core::raster::{Image, ImageFrame}; use graphene_core::renderer::{SvgSegment, SvgSegmentList}; use graphene_core::text::FontCache; use graphene_core::vector::style::ViewMode; -use graphene_core::{Color, EditorApi}; +use graphene_core::wasm_application_io::{WasmApplicationIo, WasmSurfaceHandleFrame}; +use graphene_core::{Color, EditorApi, SurfaceFrame, SurfaceId}; use interpreted_executor::executor::DynamicExecutor; use glam::{DAffine2, DVec2}; @@ -31,7 +34,9 @@ pub struct NodeRuntime { font_cache: FontCache, receiver: Receiver, sender: Sender, + wasm_io: WasmApplicationIo, pub(crate) thumbnails: HashMap>, + canvas_cache: HashMap, SurfaceId>, } fn get_imaginate_index(name: &str) -> usize { @@ -70,6 +75,8 @@ impl NodeRuntime { sender, font_cache: FontCache::default(), thumbnails: Default::default(), + wasm_io: WasmApplicationIo::default(), + canvas_cache: Default::default(), } } pub async fn run(&mut self) { @@ -94,7 +101,7 @@ impl NodeRuntime { }) => { let (network, monitor_nodes) = Self::wrap_network(graph); - let result = self.execute_network(network, image_frame).await; + let result = self.execute_network(&path, network, image_frame).await; let mut responses = VecDeque::new(); self.update_thumbnails(&path, monitor_nodes, &mut responses); let response = GenerationResponse { @@ -113,22 +120,22 @@ impl NodeRuntime { fn wrap_network(network: NodeNetwork) -> (NodeNetwork, Vec>) { let mut scoped_network = wrap_network_in_scope(network); - scoped_network.generate_node_paths(&[]); + //scoped_network.generate_node_paths(&[]); let monitor_nodes = scoped_network .recursive_nodes() .filter(|(node, _, _)| node.implementation == DocumentNodeImplementation::proto("graphene_std::memo::MonitorNode<_>")) .map(|(_, _, path)| path) .collect(); - scoped_network.duplicate_outputs(&mut generate_uuid); - scoped_network.remove_dead_nodes(); + //scoped_network.remove_dead_nodes(); (scoped_network, monitor_nodes) } - async fn execute_network<'a>(&'a mut self, scoped_network: NodeNetwork, image_frame: Option>) -> Result { + async fn execute_network<'a>(&'a mut self, path: &[LayerId], scoped_network: NodeNetwork, image_frame: Option>) -> Result { let editor_api = EditorApi { - font_cache: Some(&self.font_cache), + font_cache: &self.font_cache, image_frame, + application_io: &self.wasm_io, }; // We assume only one output @@ -150,11 +157,30 @@ impl NodeRuntime { Some(t) if t == concrete!(()) => self.executor.execute(().into_dyn()).await.map_err(|e| e.to_string()), _ => Err("Invalid input type".to_string()), }; + match result { - Ok(result) => match TaggedValue::try_from_any(result) { - Some(x) => Ok(x), - None => Err("Invalid output type".to_string()), - }, + Ok(result) => { + if DynAny::type_id(result.as_ref()) == core::any::TypeId::of::() { + let Ok(value) = dyn_any::downcast::(result) else { unreachable!()}; + let new_id = value.surface_handle.surface_id; + let old_id = self.canvas_cache.insert(path.to_vec(), new_id); + if let Some(old_id) = old_id { + if old_id != new_id { + self.wasm_io.destroy_surface(old_id); + } + } + return Ok(TaggedValue::SurfaceFrame(SurfaceFrame { + surface_id: new_id, + transform: value.transform, + })); + } + + let type_name = DynAny::type_name(result.as_ref()); + match TaggedValue::try_from_any(result) { + Some(x) => Ok(x), + None => Err(format!("Invalid output type: {}", type_name)), + } + } Err(e) => Err(e), } } @@ -439,6 +465,11 @@ impl NodeGraphExecutor { responses.add(Operation::SetLayerTransform { path: layer_path.clone(), transform }); responses.add(Operation::SetVectorData { path: layer_path, vector_data }); } + TaggedValue::SurfaceFrame(SurfaceFrame { surface_id, transform }) => { + let transform = transform.to_cols_array(); + responses.add(Operation::SetLayerTransform { path: layer_path.clone(), transform }); + responses.add(Operation::SetSurface { path: layer_path, surface_id }); + } TaggedValue::ImageFrame(ImageFrame { image, transform }) => { // Don't update the frame's transform if the new transform is DAffine2::ZERO. let transform = (!transform.abs_diff_eq(DAffine2::ZERO, f64::EPSILON)).then_some(transform.to_cols_array()); diff --git a/frontend/src/components/panels/Document.svelte b/frontend/src/components/panels/Document.svelte index dbac9e5b..a75fc4d6 100644 --- a/frontend/src/components/panels/Document.svelte +++ b/frontend/src/components/panels/Document.svelte @@ -124,6 +124,17 @@ export async function updateDocumentArtwork(svg: string) { artworkSvg = svg; rasterizedCanvas = undefined; + + await tick(); + + const placeholders = window.document.querySelectorAll("[data-canvas] [data-canvas-placeholder]"); + // Replace the placeholders with the actual canvas elements + placeholders.forEach((placeholder) => { + const canvasName = placeholder.getAttribute("data-canvas-placeholder"); + // Get the canvas element from the global storage + const context = (window as any).imageCanvases[canvasName]; + placeholder.replaceWith(context.canvas); + }); } export function updateDocumentOverlays(svg: string) { @@ -578,6 +589,11 @@ // Allows dev tools to select the artwork without being blocked by the SVG containers pointer-events: none; + canvas { + width: 100%; + height: 100%; + } + // Prevent inheritance from reaching the child elements > * { pointer-events: auto; diff --git a/frontend/src/wasm-communication/editor.ts b/frontend/src/wasm-communication/editor.ts index edfe1ecd..6395bfa2 100644 --- a/frontend/src/wasm-communication/editor.ts +++ b/frontend/src/wasm-communication/editor.ts @@ -60,6 +60,7 @@ export async function initWasm(): Promise { // eslint-disable-next-line import/no-cycle await init(); wasmImport = await wasmMemory(); + window["imageCanvases"] = {}; // Provide a random starter seed which must occur after initializing the WASM module, since WASM can't generate its own random numbers diff --git a/frontend/wasm/Cargo.toml b/frontend/wasm/Cargo.toml index 38634b15..4ac4419b 100644 --- a/frontend/wasm/Cargo.toml +++ b/frontend/wasm/Cargo.toml @@ -38,7 +38,12 @@ bezier-rs = { path = "../../libraries/bezier-rs" } [dependencies.web-sys] version = "0.3.4" -features = ["Window"] +features = [ + "Window", + "CanvasRenderingContext2d", + "Document", + "HtmlCanvasElement", +] [dev-dependencies] wasm-bindgen-test = "0.3.22" diff --git a/node-graph/gcore/Cargo.toml b/node-graph/gcore/Cargo.toml index 41b38ec1..0df9208e 100644 --- a/node-graph/gcore/Cargo.toml +++ b/node-graph/gcore/Cargo.toml @@ -18,7 +18,7 @@ std = [ "num-traits/std", "rustybuzz", ] -default = ["async", "serde", "kurbo", "log", "std", "rand_chacha"] +default = ["async", "serde", "kurbo", "log", "std", "rand_chacha", "wasm"] log = ["dep:log"] serde = [ "dep:serde", @@ -32,6 +32,7 @@ async = ["async-trait", "alloc"] nightly = [] alloc = ["dyn-any", "bezier-rs", "once_cell"] type_id_logging = [] +wasm = ["wasm-bindgen", "web-sys", "js-sys", "std"] [dependencies] dyn-any = { path = "../../libraries/dyn-any", features = [ @@ -68,3 +69,18 @@ num-derive = { version = "0.3.3" } num-traits = { version = "0.2.15", default-features = false, features = [ "i128", ] } + + +wasm-bindgen = { version = "0.2.84", optional = true } +js-sys = { version = "0.3.55", optional = true } + +[dependencies.web-sys] +version = "0.3.4" +optional = true +features = [ + "Window", + "CanvasRenderingContext2d", + "ImageData", + "Document", + "HtmlCanvasElement", +] diff --git a/node-graph/gcore/src/application_io.rs b/node-graph/gcore/src/application_io.rs new file mode 100644 index 00000000..d06a8371 --- /dev/null +++ b/node-graph/gcore/src/application_io.rs @@ -0,0 +1,159 @@ +use crate::raster::ImageFrame; +use crate::transform::Transform; +use crate::transform::TransformMut; +use crate::Color; +use crate::Node; +use alloc::sync::Arc; +use dyn_any::StaticType; +use dyn_any::StaticTypeSized; +use glam::DAffine2; + +use core::hash::{Hash, Hasher}; + +use crate::text::FontCache; + +use core::fmt::Debug; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct SurfaceId(pub u64); + +#[derive(Debug, Clone, Copy, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct SurfaceFrame { + pub surface_id: SurfaceId, + pub transform: DAffine2, +} + +impl Hash for SurfaceFrame { + fn hash(&self, state: &mut H) { + self.surface_id.hash(state); + self.transform.to_cols_array().iter().for_each(|x| x.to_bits().hash(state)); + } +} + +unsafe impl StaticType for SurfaceFrame { + type Static = SurfaceFrame; +} + +#[derive(Clone)] +pub struct SurfaceHandle<'a, Surface> { + pub surface_id: SurfaceId, + pub surface: Surface, + application_io: &'a dyn ApplicationIo, +} + +unsafe impl StaticType for SurfaceHandle<'_, T> { + type Static = SurfaceHandle<'static, T>; +} + +#[derive(Clone)] +pub struct SurfaceHandleFrame<'a, Surface> { + pub surface_handle: Arc>, + pub transform: DAffine2, +} + +unsafe impl StaticType for SurfaceHandleFrame<'_, T> { + type Static = SurfaceHandleFrame<'static, T>; +} + +impl Transform for SurfaceHandleFrame<'_, T> { + fn transform(&self) -> DAffine2 { + self.transform + } +} + +impl TransformMut for SurfaceHandleFrame<'_, T> { + fn transform_mut(&mut self) -> &mut DAffine2 { + &mut self.transform + } +} + +// TODO: think about how to automatically clean up memory +/* +impl<'a, Surface> Drop for SurfaceHandle<'a, Surface> { + fn drop(&mut self) { + self.application_io.destroy_surface(self.surface_id) + } +}*/ + +pub trait ApplicationIo { + type Surface; + fn create_surface(&self) -> SurfaceHandle; + fn destroy_surface(&self, surface_id: SurfaceId); +} + +impl ApplicationIo for &T { + type Surface = T::Surface; + fn create_surface(&self) -> SurfaceHandle { + (**self).create_surface() + } + + fn destroy_surface(&self, surface_id: SurfaceId) { + (**self).destroy_surface(surface_id) + } +} + +pub struct EditorApi<'a, Io> { + pub image_frame: Option>, + pub font_cache: &'a FontCache, + pub application_io: &'a Io, +} + +impl<'a, Io> Clone for EditorApi<'a, Io> { + fn clone(&self) -> Self { + Self { + image_frame: self.image_frame.clone(), + font_cache: self.font_cache, + application_io: self.application_io, + } + } +} + +impl<'a, T> PartialEq for EditorApi<'a, T> { + fn eq(&self, other: &Self) -> bool { + self.image_frame == other.image_frame && self.font_cache == other.font_cache + } +} + +impl<'a, T> Hash for EditorApi<'a, T> { + fn hash(&self, state: &mut H) { + self.image_frame.hash(state); + self.font_cache.hash(state); + } +} + +impl<'a, T> Debug for EditorApi<'a, T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EditorApi").field("image_frame", &self.image_frame).field("font_cache", &self.font_cache).finish() + } +} + +unsafe impl StaticType for EditorApi<'_, T> { + type Static = EditorApi<'static, T::Static>; +} + +impl<'a, T> AsRef> for EditorApi<'a, T> { + fn as_ref(&self) -> &EditorApi<'a, T> { + self + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct ExtractImageFrame; + +impl<'a: 'input, 'input, T> Node<'input, &'a EditorApi<'a, T>> for ExtractImageFrame { + type Output = ImageFrame; + fn eval(&'input self, editor_api: &'a EditorApi<'a, T>) -> Self::Output { + editor_api.image_frame.clone().unwrap_or(ImageFrame::identity()) + } +} + +impl ExtractImageFrame { + pub fn new() -> Self { + Self + } +} + +#[cfg(feature = "wasm")] +pub mod wasm_application_io; diff --git a/node-graph/gcore/src/application_io/wasm_application_io.rs b/node-graph/gcore/src/application_io/wasm_application_io.rs new file mode 100644 index 00000000..a9a696dd --- /dev/null +++ b/node-graph/gcore/src/application_io/wasm_application_io.rs @@ -0,0 +1,131 @@ +use std::{cell::RefCell, collections::HashMap, sync::Mutex}; + +use super::{ApplicationIo, SurfaceHandle, SurfaceHandleFrame, SurfaceId}; +use crate::{ + raster::{color::SRGBA8, ImageFrame, Pixel}, + Node, +}; +use alloc::sync::Arc; +use dyn_any::StaticType; +use js_sys::{Object, Reflect}; +use wasm_bindgen::{Clamped, JsCast, JsValue}; +use web_sys::{window, CanvasRenderingContext2d, HtmlCanvasElement}; + +pub struct Canvas(CanvasRenderingContext2d); + +#[derive(Debug, Default)] +pub struct WasmApplicationIo { + ids: RefCell, + canvases: RefCell>, +} + +impl WasmApplicationIo { + pub fn new() -> Self { + Self::default() + } +} + +unsafe impl StaticType for WasmApplicationIo { + type Static = WasmApplicationIo; +} + +pub type WasmEditorApi<'a> = super::EditorApi<'a, WasmApplicationIo>; + +impl ApplicationIo for WasmApplicationIo { + type Surface = CanvasRenderingContext2d; + + fn create_surface(&self) -> SurfaceHandle { + let mut wrapper = || { + let document = window().expect("should have a window in this context").document().expect("window should have a document"); + + let canvas: HtmlCanvasElement = document.create_element("canvas")?.dyn_into::()?; + // TODO: replace "2d" with "bitmaprenderer" once we switch to ImageBitmap (lives on gpu) from ImageData (lives on cpu) + let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::().unwrap(); + let mut guard = self.ids.borrow_mut(); + let id = SurfaceId(*guard); + *guard += 1; + self.canvases.borrow_mut().insert(id, context.clone()); + // store the canvas in the global scope so it doesn't get garbage collected + + let window = window().expect("should have a window in this context"); + let window = Object::from(window); + + let image_canvases_key = JsValue::from_str("imageCanvases"); + + let mut canvases = Reflect::get(&window, &image_canvases_key); + if let Err(e) = canvases { + Reflect::set(&JsValue::from(web_sys::window().unwrap()), &image_canvases_key, &Object::new()).unwrap(); + canvases = Reflect::get(&window, &image_canvases_key); + } + + // Convert key and value to JsValue + let js_key = JsValue::from_str(format!("canvas{}", id.0).as_str()); + let js_value = JsValue::from(context.clone()); + + let canvases = Object::from(canvases.unwrap()); + + // Use Reflect API to set property + Reflect::set(&canvases, &js_key, &js_value)?; + Ok::<_, JsValue>(SurfaceHandle { + surface_id: id, + surface: context, + application_io: self, + }) + }; + + wrapper().expect("should be able to set canvas in global scope") + } + + fn destroy_surface(&self, surface_id: SurfaceId) { + self.canvases.borrow_mut().remove(&surface_id); + + let window = window().expect("should have a window in this context"); + let window = Object::from(window); + + let image_canvases_key = JsValue::from_str("imageCanvases"); + + let wrapper = || { + if let Ok(canvases) = Reflect::get(&window, &image_canvases_key) { + // Convert key and value to JsValue + let js_key = JsValue::from_str(format!("canvas{}", surface_id.0).as_str()); + + // Use Reflect API to set property + Reflect::delete_property(&canvases.into(), &js_key)?; + } + Ok::<_, JsValue>(()) + }; + + wrapper().expect("should be able to set canvas in global scope") + } +} + +pub type WasmSurfaceHandle<'a> = SurfaceHandle<'a, CanvasRenderingContext2d>; +pub type WasmSurfaceHandleFrame<'a> = SurfaceHandleFrame<'a, CanvasRenderingContext2d>; + +pub struct CreateSurfaceNode {} + +#[node_macro::node_fn(CreateSurfaceNode)] +fn create_surface_node<'a: 'input>(editor: &'a WasmEditorApi<'a>) -> Arc> { + editor.application_io.create_surface().into() +} + +pub struct DrawImageFrameNode { + surface_handle: Surface, +} + +#[node_macro::node_fn(DrawImageFrameNode)] +async fn draw_image_frame_node<'a: 'input>(image: ImageFrame, surface_handle: Arc>) -> SurfaceHandleFrame<'a, CanvasRenderingContext2d> { + let image_data = image.image.data; + let array: Clamped<&[u8]> = Clamped(bytemuck::cast_slice(image_data.as_slice())); + if image.image.width > 0 && image.image.height > 0 { + let canvas = surface_handle.surface.canvas().expect("Failed to get canvas"); + canvas.set_width(image.image.width); + canvas.set_height(image.image.height); + let image_data = web_sys::ImageData::new_with_u8_clamped_array_and_sh(array, image.image.width as u32, image.image.height as u32).expect("Failed to construct ImageData"); + surface_handle.surface.put_image_data(&image_data, 0.0, 0.0).unwrap(); + } + SurfaceHandleFrame { + surface_handle: surface_handle.into(), + transform: image.transform, + } +} diff --git a/node-graph/gcore/src/lib.rs b/node-graph/gcore/src/lib.rs index 520bf000..b1d90bad 100644 --- a/node-graph/gcore/src/lib.rs +++ b/node-graph/gcore/src/lib.rs @@ -33,6 +33,8 @@ pub use graphic_element::*; #[cfg(feature = "alloc")] pub mod vector; +pub mod application_io; + pub mod quantization; use core::any::TypeId; @@ -142,5 +144,6 @@ impl<'i, I: 'i, O: 'i> Node<'i, I> for Pin<&'i (dyn NodeIO<'i, I, Output = O> + } } -#[cfg(feature = "alloc")] -pub use crate::raster::image::{EditorApi, ExtractImageFrame}; +pub use crate::application_io::{ExtractImageFrame, SurfaceFrame, SurfaceId}; +#[cfg(feature = "wasm")] +pub use application_io::{wasm_application_io, wasm_application_io::WasmEditorApi as EditorApi}; diff --git a/node-graph/gcore/src/ops.rs b/node-graph/gcore/src/ops.rs index 57eb585a..c951a568 100644 --- a/node-graph/gcore/src/ops.rs +++ b/node-graph/gcore/src/ops.rs @@ -22,7 +22,7 @@ pub struct AddParameterNode { second: Second, } -#[node_macro::node_new(AddParameterNode)] +#[node_macro::node_fn(AddParameterNode)] fn add_parameter(first: U, second: T) -> >::Output where U: Add, @@ -30,24 +30,6 @@ where first + second } -#[automatically_derived] -impl<'input, U: 'input, T: 'input, S0: 'input> Node<'input, U> for AddParameterNode -where - U: Add, - S0: Node<'input, (), Output = T>, -{ - type Output = >::Output; - #[inline] - fn eval(&'input self, first: U) -> Self::Output { - let second = self.second.eval(()); - { - { - first + second - } - } - } -} - pub struct MulParameterNode { second: Second, } @@ -224,6 +206,18 @@ where } } +pub struct IntoNode { + _i: PhantomData, + _o: PhantomData, +} +#[node_macro::node_fn(IntoNode<_I, _O>)] +fn into<_I, _O>(input: _I) -> _O +where + _I: Into<_O>, +{ + input.into() +} + #[cfg(test)] mod test { use super::*; diff --git a/node-graph/gcore/src/raster.rs b/node-graph/gcore/src/raster.rs index 6d72bd88..13c649c5 100644 --- a/node-graph/gcore/src/raster.rs +++ b/node-graph/gcore/src/raster.rs @@ -5,7 +5,7 @@ use crate::Node; use bytemuck::{Pod, Zeroable}; use glam::DVec2; -pub use self::color::{Color, Luma}; +pub use self::color::{Color, Luma, SRGBA8}; pub mod adjustments; pub mod bbox; diff --git a/node-graph/gcore/src/raster/adjustments.rs b/node-graph/gcore/src/raster/adjustments.rs index ffe2cb7c..1d17115a 100644 --- a/node-graph/gcore/src/raster/adjustments.rs +++ b/node-graph/gcore/src/raster/adjustments.rs @@ -451,8 +451,8 @@ pub struct BlendNode { } #[node_macro::node_fn(BlendNode)] -fn blend_node(input: (Color, Color), blend_mode: BlendMode, opacity: f32) -> Color { - let opacity = opacity / 100.; +fn blend_node(input: (Color, Color), blend_mode: BlendMode, opacity: f64) -> Color { + let opacity = opacity as f32 / 100.; let (foreground, background) = input; diff --git a/node-graph/gcore/src/raster/color.rs b/node-graph/gcore/src/raster/color.rs index e7917cdf..2750289e 100644 --- a/node-graph/gcore/src/raster/color.rs +++ b/node-graph/gcore/src/raster/color.rs @@ -12,7 +12,92 @@ use spirv_std::num_traits::Euclid; use bytemuck::{Pod, Zeroable}; -use super::{Alpha, AssociatedAlpha, Luminance, Pixel, RGBMut, Rec709Primaries, RGB, SRGB}; +use super::{ + discrete_srgb::{float_to_srgb_u8, srgb_u8_to_float}, + Alpha, AssociatedAlpha, Luminance, Pixel, RGBMut, Rec709Primaries, RGB, SRGB, +}; + +#[repr(C)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "std", derive(specta::Type))] +#[derive(Debug, Default, Clone, Copy, PartialEq, DynAny, Pod, Zeroable)] +pub struct SRGBA8 { + red: u8, + green: u8, + blue: u8, + alpha: u8, +} + +impl From for SRGBA8 { + #[inline(always)] + fn from(c: Color) -> Self { + Self { + red: float_to_srgb_u8(c.r()), + green: float_to_srgb_u8(c.g()), + blue: float_to_srgb_u8(c.b()), + alpha: (c.a() * 255.0) as u8, + } + } +} + +impl From for Color { + #[inline(always)] + fn from(color: SRGBA8) -> Self { + Self { + red: srgb_u8_to_float(color.red), + green: srgb_u8_to_float(color.green), + blue: srgb_u8_to_float(color.blue), + alpha: color.alpha as f32 / 255.0, + } + } +} + +impl Luminance for SRGBA8 { + type LuminanceChannel = f32; + #[inline(always)] + fn luminance(&self) -> f32 { + // TODO: verify this is correct for sRGB + 0.2126 * self.red() + 0.7152 * self.green() + 0.0722 * self.blue() + } +} + +impl RGB for SRGBA8 { + type ColorChannel = f32; + #[inline(always)] + fn red(&self) -> f32 { + self.red as f32 / 255.0 + } + #[inline(always)] + fn green(&self) -> f32 { + self.green as f32 / 255.0 + } + #[inline(always)] + fn blue(&self) -> f32 { + self.blue as f32 / 255.0 + } +} + +impl Rec709Primaries for SRGBA8 {} +impl SRGB for SRGBA8 {} + +impl Alpha for SRGBA8 { + type AlphaChannel = f32; + #[inline(always)] + fn alpha(&self) -> f32 { + self.alpha as f32 / 255.0 + } + + const TRANSPARENT: Self = SRGBA8 { red: 0, green: 0, blue: 0, alpha: 0 }; + + fn multiplied_alpha(&self, alpha: Self::AlphaChannel) -> Self { + let alpha = alpha * 255.0; + let mut result = *self; + result.alpha = (alpha * self.alpha()) as u8; + result + } +} + +impl Pixel for SRGBA8 {} #[repr(C)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] diff --git a/node-graph/gcore/src/raster/image.rs b/node-graph/gcore/src/raster/image.rs index 6365a8cb..2392d06a 100644 --- a/node-graph/gcore/src/raster/image.rs +++ b/node-graph/gcore/src/raster/image.rs @@ -325,44 +325,43 @@ impl Hash for ImageFrame

{ } } -use crate::text::FontCache; -#[derive(Clone, Debug, Hash, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct EditorApi<'a> { - #[cfg_attr(feature = "serde", serde(skip))] - pub image_frame: Option>, - #[cfg_attr(feature = "serde", serde(skip))] - pub font_cache: Option<&'a FontCache>, -} +/* This does not work because of missing specialization + * so we have to manually implement this for now +impl + Pixel, P: Pixel> From> for Image

{ + fn from(image: Image) -> Self { + let data = image.data.into_iter().map(|x| x.into()).collect(); + Self { + data, + width: image.width, + height: image.height, + } + } +}*/ -unsafe impl StaticType for EditorApi<'_> { - type Static = EditorApi<'static>; -} - -impl EditorApi<'_> { - pub fn empty() -> Self { - Self { image_frame: None, font_cache: None } +impl From> for ImageFrame { + fn from(image: ImageFrame) -> Self { + let data = image.image.data.into_iter().map(|x| x.into()).collect(); + Self { + image: Image { + data, + width: image.image.width, + height: image.image.height, + }, + transform: image.transform, + } } } -impl<'a> AsRef> for EditorApi<'a> { - fn as_ref(&self) -> &EditorApi<'a> { - self - } -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct ExtractImageFrame; - -impl<'a: 'input, 'input> Node<'input, &'a EditorApi<'a>> for ExtractImageFrame { - type Output = ImageFrame; - fn eval(&'input self, editor_api: &'a EditorApi<'a>) -> Self::Output { - editor_api.image_frame.clone().unwrap_or(ImageFrame::identity()) - } -} - -impl ExtractImageFrame { - pub fn new() -> Self { - Self +impl From> for ImageFrame { + fn from(image: ImageFrame) -> Self { + let data = image.image.data.into_iter().map(|x| x.into()).collect(); + Self { + image: Image { + data, + width: image.image.width, + height: image.image.height, + }, + transform: image.transform, + } } } diff --git a/node-graph/gcore/src/text.rs b/node-graph/gcore/src/text.rs index 446e6afb..8b800464 100644 --- a/node-graph/gcore/src/text.rs +++ b/node-graph/gcore/src/text.rs @@ -16,6 +16,6 @@ pub struct TextGenerator { #[node_fn(TextGenerator)] fn generate_text<'a: 'input>(editor: &'a EditorApi<'a>, text: String, font_name: Font, font_size: f64) -> crate::vector::VectorData { - let buzz_face = editor.font_cache.and_then(|cache| cache.get(&font_name)).map(|data| load_face(data)); + let buzz_face = editor.font_cache.get(&font_name).map(|data| load_face(data)); crate::vector::VectorData::from_subpaths(to_path(&text, buzz_face, font_size, None)) } diff --git a/node-graph/graph-craft/src/document.rs b/node-graph/graph-craft/src/document.rs index ab7c647a..cb6e57f1 100644 --- a/node-graph/graph-craft/src/document.rs +++ b/node-graph/graph-craft/src/document.rs @@ -45,7 +45,7 @@ pub struct DocumentNode { impl DocumentNode { pub fn populate_first_network_input(&mut self, node_id: NodeId, output_index: usize, offset: usize, lambda: bool) { - let input = self + let (index, _) = self .inputs .iter() .enumerate() @@ -53,7 +53,6 @@ impl DocumentNode { .nth(offset) .unwrap_or_else(|| panic!("no network input found for {self:#?} and offset: {offset}")); - let index = input.0; self.inputs[index] = NodeInput::Node { node_id, output_index, lambda }; } @@ -67,7 +66,7 @@ impl DocumentNode { (ProtoNodeInput::None, ConstructionArgs::Value(tagged_value)) } NodeInput::Node { node_id, output_index, lambda } => { - assert_eq!(output_index, 0, "Outputs should be flattened before converting to protonode."); + assert_eq!(output_index, 0, "Outputs should be flattened before converting to protonode. {:#?}", self.name); (ProtoNodeInput::Node(node_id, lambda), ConstructionArgs::Nodes(vec![])) } NodeInput::Network(ty) => (ProtoNodeInput::Network(ty), ConstructionArgs::Nodes(vec![])), @@ -564,9 +563,8 @@ impl NodeNetwork { self.previous_outputs .iter_mut() .for_each(|nodes| nodes.iter_mut().for_each(|output| output.node_id = f(output.node_id))); - let mut empty = HashMap::new(); - std::mem::swap(&mut self.nodes, &mut empty); - self.nodes = empty + let nodes = std::mem::take(&mut self.nodes); + self.nodes = nodes .into_iter() .map(|(id, mut node)| { node.inputs.iter_mut().for_each(|input| input.map_ids(f)); @@ -596,70 +594,28 @@ impl NodeNetwork { network.generate_node_paths(new_path.as_slice()); } if node.path.is_some() { - log::warn!("Overwriting node path"); + log::warn!("Attempting to overwrite node path"); + } else { + node.path = Some(new_path); } - node.path = Some(new_path); } } - /// When a node has multiple outputs, we actually just duplicate the node and evaluate each output separately - pub fn duplicate_outputs(&mut self, mut gen_id: &mut impl FnMut() -> NodeId) { - let mut duplicating_nodes = HashMap::new(); - // Find the nodes where the inputs require duplicating - for node in &mut self.nodes.values_mut() { - // Recursively duplicate children - if let DocumentNodeImplementation::Network(network) = &mut node.implementation { - network.duplicate_outputs(gen_id); - } - - for input in &mut node.inputs { - let &mut NodeInput::Node { node_id, output_index, .. } = input else { - continue; - }; - // Use the initial node when getting the first output - if output_index == 0 { - continue; - } - // Get the existing duplicated node id (or create a new one) - let duplicated_node_id = *duplicating_nodes.entry((node_id, output_index)).or_insert_with(&mut gen_id); - // Use the first output from the duplicated node - *input = NodeInput::node(duplicated_node_id, 0); - } - } - // Find the network outputs that require duplicating - for network_output in &mut self.outputs { - // Use the initial node when getting the first output - if network_output.node_output_index == 0 { - continue; - } - // Get the existing duplicated node id (or create a new one) - let duplicated_node_id = *duplicating_nodes.entry((network_output.node_id, network_output.node_output_index)).or_insert_with(&mut gen_id); - // Use the first output from the duplicated node - *network_output = NodeOutput::new(duplicated_node_id, 0); - } - // Duplicate the nodes - for ((original_node_id, output_index), new_node_id) in duplicating_nodes { - let Some(original_node) = self.nodes.get(&original_node_id) else { - continue; - }; - let mut new_node = original_node.clone(); - // Update the required outputs from a nested network to be just the relevant output - if let DocumentNodeImplementation::Network(network) = &mut new_node.implementation { - if network.outputs.is_empty() { - continue; - } - network.outputs = vec![network.outputs[output_index]]; - } - self.nodes.insert(new_node_id, new_node); - } - - // Ensure all nodes only have one output + fn replace_node_inputs(&mut self, old_input: NodeInput, new_input: NodeInput) { for node in self.nodes.values_mut() { - if let DocumentNodeImplementation::Network(network) = &mut node.implementation { - if network.outputs.is_empty() { - continue; + let node_string = format!("{:?}", node); + node.inputs.iter_mut().for_each(|input| { + if *input == old_input { + *input = new_input.clone(); } - network.outputs = vec![network.outputs[0]]; + }); + } + } + + fn replace_network_outputs(&mut self, old_output: NodeOutput, new_output: NodeOutput) { + for output in self.outputs.iter_mut() { + if *output == old_output { + *output = new_output.clone(); } } } @@ -715,8 +671,14 @@ impl NodeNetwork { self.nodes.insert(id, node); return; } + // replace value inputs with value nodes for input in &mut node.inputs { + // Skip inputs that are already value nodes + if node.implementation == DocumentNodeImplementation::Unresolved("graphene_core::value::ValueNode".into()) { + break; + } + let mut dummy_input = NodeInput::ShortCircut(concrete!(())); std::mem::swap(&mut dummy_input, input); if let NodeInput::Value { tagged_value, exposed } = dummy_input { @@ -749,54 +711,61 @@ impl NodeNetwork { } } - match node.implementation { - DocumentNodeImplementation::Network(mut inner_network) => { - // Connect all network inputs to either the parent network nodes, or newly created value nodes. - inner_network.map_ids(|inner_id| map_ids(id, inner_id)); - let new_nodes = inner_network.nodes.keys().cloned().collect::>(); - // Copy nodes from the inner network into the parent network - self.nodes.extend(inner_network.nodes); - self.disabled.extend(inner_network.disabled); + if let DocumentNodeImplementation::Network(mut inner_network) = node.implementation { + // Connect all network inputs to either the parent network nodes, or newly created value nodes. + inner_network.map_ids(|inner_id| map_ids(id, inner_id)); + let new_nodes = inner_network.nodes.keys().cloned().collect::>(); + // Copy nodes from the inner network into the parent network + self.nodes.extend(inner_network.nodes); + self.disabled.extend(inner_network.disabled); - let mut network_offsets = HashMap::new(); - for (document_input, network_input) in node.inputs.into_iter().zip(inner_network.inputs.iter()) { - let offset = network_offsets.entry(network_input).or_insert(0); - match document_input { - NodeInput::Node { node_id, output_index, lambda } => { - let network_input = self.nodes.get_mut(network_input).unwrap(); - network_input.populate_first_network_input(node_id, output_index, *offset, lambda); - } - NodeInput::Network(_) => { - *network_offsets.get_mut(network_input).unwrap() += 1; - if let Some(index) = self.inputs.iter().position(|i| *i == id) { - self.inputs[index] = *network_input; - } - } - NodeInput::ShortCircut(_) => (), - NodeInput::Value { .. } => unreachable!("Value inputs should have been replaced with value nodes"), - NodeInput::Inline(_) => (), + let mut network_offsets = HashMap::new(); + assert_eq!( + node.inputs.len(), + inner_network.inputs.len(), + "The number of inputs to the node and the inner network must be the same {}", + node.name + ); + // Match the document node input and the inputs of the inner network + for (document_input, network_input) in node.inputs.into_iter().zip(inner_network.inputs.iter()) { + // Keep track of how many network inputs we have already connected for each node + let offset = network_offsets.entry(network_input).or_insert(0); + match document_input { + // If the input to self is a node, connect the corresponding output of the inner network to it + NodeInput::Node { node_id, output_index, lambda } => { + let network_input = self.nodes.get_mut(network_input).unwrap(); + network_input.populate_first_network_input(node_id, output_index, *offset, lambda); } - } - node.implementation = DocumentNodeImplementation::Unresolved("graphene_core::ops::IdNode".into()); - node.inputs = inner_network - .outputs - .iter() - .map(|&NodeOutput { node_id, node_output_index }| NodeInput::Node { - node_id, - output_index: node_output_index, - lambda: false, - }) - .collect(); - - for node_id in new_nodes { - self.flatten_with_fns(node_id, map_ids, gen_id); + NodeInput::Network(_) => { + *network_offsets.get_mut(network_input).unwrap() += 1; + if let Some(index) = self.inputs.iter().position(|i| *i == id) { + self.inputs[index] = *network_input; + } + } + NodeInput::ShortCircut(_) => (), + NodeInput::Value { .. } => unreachable!("Value inputs should have been replaced with value nodes"), + NodeInput::Inline(_) => (), } } - DocumentNodeImplementation::Unresolved(_) => (), - DocumentNodeImplementation::Extract => (), + + // Connect all nodes that were previously connected to this node to the nodes of the inner network + for (i, output) in inner_network.outputs.into_iter().enumerate() { + let node_input = |node_id, output_index, lambda| NodeInput::Node { node_id, output_index, lambda }; + + self.replace_node_inputs(node_input(id, i, false), node_input(output.node_id, output.node_output_index, false)); + self.replace_node_inputs(node_input(id, i, true), node_input(output.node_id, output.node_output_index, true)); + + self.replace_network_outputs(NodeOutput::new(id, i), output); + } + + for node_id in new_nodes { + self.flatten_with_fns(node_id, map_ids, gen_id); + } + } else { + // If the node is not a network, it is a primitive node and can be inserted into the network as is. + assert!(!self.nodes.contains_key(&id), "Trying to insert a node into the network caused an id conflict"); + self.nodes.insert(id, node); } - assert!(!self.nodes.contains_key(&id), "Trying to insert a node into the network caused an id conflict"); - self.nodes.insert(id, node); } fn remove_id_node(&mut self, id: NodeId) -> Result<(), String> { @@ -1090,17 +1059,8 @@ mod test { fn resolve_flatten_add_as_proto_network() { let construction_network = ProtoNetwork { inputs: vec![10], - output: 1, + output: 11, nodes: [ - ( - 1, - ProtoNode { - identifier: "graphene_core::ops::IdNode".into(), - input: ProtoNodeInput::Node(11, false), - construction_args: ConstructionArgs::Nodes(vec![]), - document_node_path: vec![1], - }, - ), ( 10, ProtoNode { @@ -1135,18 +1095,8 @@ mod test { fn flat_network() -> NodeNetwork { NodeNetwork { inputs: vec![10], - outputs: vec![NodeOutput::new(1, 0)], + outputs: vec![NodeOutput::new(11, 0)], nodes: [ - ( - 1, - DocumentNode { - name: "Inc".into(), - inputs: vec![NodeInput::node(11, 0)], - implementation: DocumentNodeImplementation::Unresolved("graphene_core::ops::IdNode".into()), - path: Some(vec![1]), - ..Default::default() - }, - ), ( 10, DocumentNode { @@ -1223,7 +1173,7 @@ mod test { outputs: network_outputs, nodes: [ ( - 10, + 1, DocumentNode { name: "Nested network".into(), inputs: vec![NodeInput::value(TaggedValue::F32(1.), false), NodeInput::value(TaggedValue::F32(2.), false)], @@ -1232,7 +1182,7 @@ mod test { }, ), ( - 11, + 2, DocumentNode { name: "Result".into(), inputs: vec![result_node_input], @@ -1246,26 +1196,25 @@ mod test { ..Default::default() }; let mut new_ids = 101..; - network.duplicate_outputs(&mut || new_ids.next().unwrap()); + network.flatten_with_fns(1, |self_id, inner_id| self_id * 10 + inner_id, || 10000); + network.flatten_with_fns(2, |self_id, inner_id| self_id * 10 + inner_id, || 10001); network.remove_dead_nodes(); network } #[test] fn simple_duplicate() { - let result = output_duplicate(vec![NodeOutput::new(10, 1)], NodeInput::node(10, 0)); + let result = output_duplicate(vec![NodeOutput::new(1, 0)], NodeInput::node(1, 0)); + println!("{:#?}", result); assert_eq!(result.outputs.len(), 1, "The number of outputs should remain as 1"); - assert_eq!(result.outputs[0], NodeOutput::new(101, 0), "The outer network output should be from a duplicated inner network"); - assert_eq!(result.nodes.keys().copied().collect::>(), vec![101], "Should just call nested network"); - let nested_network_node = result.nodes.get(&101).unwrap(); - assert_eq!(nested_network_node.name, "Nested network".to_string(), "Name should not change"); - assert_eq!(nested_network_node.inputs, vec![NodeInput::value(TaggedValue::F32(2.), false)], "Input should be 2"); - let inner_network = nested_network_node.implementation.get_network().expect("Implementation should be network"); - assert_eq!(inner_network.inputs, vec![2], "The input should be sent to the second node"); - assert_eq!(inner_network.outputs, vec![NodeOutput::new(2, 0)], "The output should be node id 2"); - assert_eq!(inner_network.nodes.get(&2).unwrap().name, "Identity 2", "The node should be identity 2"); + assert_eq!(result.outputs[0], NodeOutput::new(11, 0), "The outer network output should be from a duplicated inner network"); + let mut ids = result.nodes.keys().copied().collect::>(); + ids.sort(); + assert_eq!(ids, vec![11, 10010], "Should only contain identity and values"); } + // TODO: Write more tests + /* #[test] fn out_of_order_duplicate() { let result = output_duplicate(vec![NodeOutput::new(10, 1), NodeOutput::new(10, 0)], NodeInput::node(10, 0)); @@ -1303,4 +1252,5 @@ mod test { assert_eq!(inner_network.outputs, vec![NodeOutput::new(2, 0)], "The output should be node id 2"); assert_eq!(inner_network.nodes.get(&2).unwrap().name, "Identity 2", "The node should be identity 2"); } + */ } diff --git a/node-graph/graph-craft/src/document/value.rs b/node-graph/graph-craft/src/document/value.rs index 2a963861..492be729 100644 --- a/node-graph/graph-craft/src/document/value.rs +++ b/node-graph/graph-craft/src/document/value.rs @@ -56,11 +56,11 @@ pub enum TaggedValue { Font(graphene_core::text::Font), BrushStrokes(Vec), Segments(Vec>), - EditorApi(graphene_core::EditorApi<'static>), DocumentNode(DocumentNode), GraphicGroup(graphene_core::GraphicGroup), Artboard(graphene_core::Artboard), IVec2(glam::IVec2), + SurfaceFrame(graphene_core::SurfaceFrame), } #[allow(clippy::derived_hash_with_manual_eq)] @@ -121,11 +121,11 @@ impl Hash for TaggedValue { segment.hash(state) } } - Self::EditorApi(editor_api) => editor_api.hash(state), Self::DocumentNode(document_node) => document_node.hash(state), Self::GraphicGroup(graphic_group) => graphic_group.hash(state), Self::Artboard(artboard) => artboard.hash(state), Self::IVec2(v) => v.hash(state), + Self::SurfaceFrame(surface_id) => surface_id.hash(state), } } } @@ -173,11 +173,11 @@ impl<'a> TaggedValue { TaggedValue::Font(x) => Box::new(x), TaggedValue::BrushStrokes(x) => Box::new(x), TaggedValue::Segments(x) => Box::new(x), - TaggedValue::EditorApi(x) => Box::new(x), TaggedValue::DocumentNode(x) => Box::new(x), TaggedValue::GraphicGroup(x) => Box::new(x), TaggedValue::Artboard(x) => Box::new(x), TaggedValue::IVec2(x) => Box::new(x), + TaggedValue::SurfaceFrame(x) => Box::new(x), } } @@ -237,11 +237,11 @@ impl<'a> TaggedValue { TaggedValue::Font(_) => concrete!(graphene_core::text::Font), TaggedValue::BrushStrokes(_) => concrete!(Vec), TaggedValue::Segments(_) => concrete!(graphene_core::raster::IndexNode>>), - TaggedValue::EditorApi(_) => concrete!(graphene_core::EditorApi), TaggedValue::DocumentNode(_) => concrete!(crate::document::DocumentNode), TaggedValue::GraphicGroup(_) => concrete!(graphene_core::GraphicGroup), TaggedValue::Artboard(_) => concrete!(graphene_core::Artboard), TaggedValue::IVec2(_) => concrete!(glam::IVec2), + TaggedValue::SurfaceFrame(_) => concrete!(graphene_core::SurfaceFrame), } } @@ -289,11 +289,11 @@ impl<'a> TaggedValue { x if x == TypeId::of::() => Some(TaggedValue::Font(*downcast(input).unwrap())), x if x == TypeId::of::>() => Some(TaggedValue::BrushStrokes(*downcast(input).unwrap())), x if x == TypeId::of::>>>() => Some(TaggedValue::Segments(*downcast(input).unwrap())), - x if x == TypeId::of::() => Some(TaggedValue::EditorApi(*downcast(input).unwrap())), x if x == TypeId::of::() => Some(TaggedValue::DocumentNode(*downcast(input).unwrap())), x if x == TypeId::of::() => Some(TaggedValue::GraphicGroup(*downcast(input).unwrap())), x if x == TypeId::of::() => Some(TaggedValue::Artboard(*downcast(input).unwrap())), x if x == TypeId::of::() => Some(TaggedValue::IVec2(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::SurfaceFrame(*downcast(input).unwrap())), _ => None, } } diff --git a/node-graph/graph-craft/src/executor.rs b/node-graph/graph-craft/src/executor.rs index bbbcd9a2..4ed9a557 100644 --- a/node-graph/graph-craft/src/executor.rs +++ b/node-graph/graph-craft/src/executor.rs @@ -21,12 +21,10 @@ impl Compiler { proto_networks.map(move |mut proto_network| { if resolve_inputs { println!("resolving inputs"); - log::debug!("resolving inputs"); proto_network.resolve_inputs(); } proto_network.reorder_ids(); proto_network.generate_stable_node_ids(); - log::debug!("proto network: {:?}", proto_network); proto_network }) } diff --git a/node-graph/gstd/src/any.rs b/node-graph/gstd/src/any.rs index a908e467..567132a5 100644 --- a/node-graph/gstd/src/any.rs +++ b/node-graph/gstd/src/any.rs @@ -204,6 +204,7 @@ impl<'n: 'input, 'input, O: 'input + StaticType, I: 'input + StaticType> Node<'i #[inline] fn eval(&'input self, input: I) -> Self::Output { { + let node_name = self.node.node_name(); let input = Box::new(input); Box::pin(async move { let out: Box<&_> = dyn_any::downcast::<&O>(self.node.eval(input).await).unwrap_or_else(|e| panic!("DowncastBothRefNode Input {e}")); diff --git a/node-graph/gstd/src/memo.rs b/node-graph/gstd/src/memo.rs index 6c7fb092..45d00bc4 100644 --- a/node-graph/gstd/src/memo.rs +++ b/node-graph/gstd/src/memo.rs @@ -141,7 +141,8 @@ where { type Output = ::Output; fn eval(&'i self, _: &'i T) -> Self::Output { - self.input.eval(()) + let result = self.input.eval(()); + result } } diff --git a/node-graph/interpreted-executor/src/executor.rs b/node-graph/interpreted-executor/src/executor.rs index 6c5fb50a..376a1df6 100644 --- a/node-graph/interpreted-executor/src/executor.rs +++ b/node-graph/interpreted-executor/src/executor.rs @@ -50,7 +50,6 @@ impl DynamicExecutor { pub async fn update(&mut self, proto_network: ProtoNetwork) -> Result<(), String> { self.output = proto_network.output; self.typing_context.update(&proto_network)?; - trace!("setting output to {}", self.output); let mut orphans = self.tree.update(proto_network, &self.typing_context).await?; core::mem::swap(&mut self.orphaned_nodes, &mut orphans); for node_id in orphans { diff --git a/node-graph/interpreted-executor/src/node_registry.rs b/node-graph/interpreted-executor/src/node_registry.rs index 5b758e73..01539bef 100644 --- a/node-graph/interpreted-executor/src/node_registry.rs +++ b/node-graph/interpreted-executor/src/node_registry.rs @@ -8,6 +8,8 @@ use graphene_core::structural::Then; use graphene_core::value::{ClonedNode, CopiedNode, ValueNode}; use graphene_core::vector::brush_stroke::BrushStroke; use graphene_core::vector::VectorData; +use graphene_core::wasm_application_io::WasmSurfaceHandle; +use graphene_core::wasm_application_io::*; use graphene_core::{concrete, generic, value_fn}; use graphene_core::{fn_type, raster::*}; use graphene_core::{Cow, NodeIdentifier, Type, TypeDescriptor}; @@ -22,6 +24,7 @@ use dyn_any::StaticType; use glam::{DAffine2, DVec2}; use once_cell::sync::Lazy; use std::collections::HashMap; +use std::sync::Arc; macro_rules! construct_node { ($args: ident, $path:ty, [$($type:tt),*]) => { async move { @@ -41,7 +44,7 @@ macro_rules! construct_node { } macro_rules! register_node { - ($path:ty, input: $input:ty, params: [$($type:ty),*]) => { + ($path:ty, input: $input:ty, params: [ $($type:ty),*]) => { vec![ ( NodeIdentifier::new(stringify!($path)), @@ -158,6 +161,8 @@ fn node_registry() -> HashMap, input: f64, params: [&f64]), register_node!(graphene_core::ops::AddParameterNode<_>, input: &f64, params: [&f64]), register_node!(graphene_core::ops::SomeNode, input: graphene_core::EditorApi, params: []), + register_node!(graphene_core::ops::IntoNode<_, ImageFrame>, input: ImageFrame, params: []), + register_node!(graphene_core::ops::IntoNode<_, ImageFrame>, input: ImageFrame, params: []), register_node!(graphene_std::raster::DownresNode<_>, input: ImageFrame, params: []), register_node!(graphene_std::raster::MaskImageNode<_, _, _>, input: ImageFrame, params: [ImageFrame]), register_node!(graphene_std::raster::MaskImageNode<_, _, _>, input: ImageFrame, params: [ImageFrame]), @@ -209,6 +214,19 @@ fn node_registry() -> HashMap, input: DAffine2, params: [Color]), register_node!(graphene_std::memo::MonitorNode<_>, input: ImageFrame, params: []), register_node!(graphene_std::memo::MonitorNode<_>, input: graphene_core::GraphicGroup, params: []), + register_node!(graphene_core::wasm_application_io::CreateSurfaceNode, input: &graphene_core::EditorApi, params: []), + vec![( + NodeIdentifier::new("graphene_core::wasm_application_io::DrawImageFrameNode<_>"), + |args| { + Box::pin(async move { + let surface: DowncastBothNode<(), Arc> = DowncastBothNode::new(args[0]); + let node = graphene_core::wasm_application_io::DrawImageFrameNode::new(surface); + let any: DynAnyNode, _, _> = graphene_std::any::DynAnyNode::new(graphene_core::value::ValueNode::new(node)); + Box::pin(any) as TypeErasedPinned + }) + }, + NodeIOTypes::new(concrete!(ImageFrame), concrete!(WasmSurfaceHandleFrame), vec![value_fn!(Arc)]), + )], #[cfg(feature = "gpu")] vec![( NodeIdentifier::new("graphene_std::executor::MapGpuSingleImageNode<_>"), @@ -346,7 +364,7 @@ fn node_registry() -> HashMap> = DowncastBothNode::new(args[0]); let blend_mode: DowncastBothNode<(), BlendMode> = DowncastBothNode::new(args[1]); - let opacity: DowncastBothNode<(), f32> = DowncastBothNode::new(args[2]); + let opacity: DowncastBothNode<(), f64> = DowncastBothNode::new(args[2]); let blend_node = graphene_core::raster::BlendNode::new(CopiedNode::new(blend_mode.eval(()).await), CopiedNode::new(opacity.eval(()).await)); let node = graphene_std::raster::BlendImageNode::new(image, FutureWrapperNode::new(ValueNode::new(blend_node))); let any: DynAnyNode, _, _> = graphene_std::any::DynAnyNode::new(graphene_core::value::ValueNode::new(node)); @@ -356,7 +374,7 @@ fn node_registry() -> HashMap), concrete!(ImageFrame), - vec![value_fn!(ImageFrame), value_fn!(BlendMode), value_fn!(f32)], + vec![value_fn!(ImageFrame), value_fn!(BlendMode), value_fn!(f64)], ), )], raster_node!(graphene_core::raster::GrayscaleNode<_, _, _, _, _, _, _>, params: [Color, f64, f64, f64, f64, f64, f64]), @@ -475,6 +493,18 @@ fn node_registry() -> HashMap"), + |args| { + Box::pin(async move { + let input: DowncastBothNode<(), WasmSurfaceHandleFrame> = DowncastBothNode::new(args[0]); + let node = graphene_std::memo::EndLetNode::new(input); + let any: DynAnyInRefNode = graphene_std::any::DynAnyInRefNode::new(node); + Box::pin(any) as TypeErasedPinned + }) + }, + NodeIOTypes::new(generic!(T), concrete!(WasmSurfaceHandleFrame), vec![value_fn!(WasmSurfaceHandleFrame)]), + ), ( NodeIdentifier::new("graphene_std::memo::RefNode<_, _>"), |args| { @@ -633,6 +663,18 @@ fn node_registry() -> HashMap), vec![value_fn!(Vec)]), ), + ( + NodeIdentifier::new("graphene_std::memo::CacheNode"), + |args| { + Box::pin(async move { + let input: DowncastBothNode<(), Arc> = DowncastBothNode::new(args[0]); + let node: CacheNode, _> = graphene_std::memo::CacheNode::new(input); + let any = DynAnyNode::new(ValueNode::new(node)); + Box::pin(any) as TypeErasedPinned + }) + }, + NodeIOTypes::new(concrete!(()), concrete!(Arc), vec![value_fn!(Arc)]), + ), ], register_node!(graphene_core::structural::ConsNode<_, _>, input: Image, params: [&str]), register_node!(graphene_std::raster::ImageFrameNode<_, _>, input: Image, params: [DAffine2]), @@ -643,6 +685,7 @@ fn node_registry() -> HashMap, input: &QuantizationChannels, params: []), register_node!(graphene_core::transform::TransformNode<_, _, _, _, _>, input: VectorData, params: [DVec2, f64, DVec2, DVec2, DVec2]), register_node!(graphene_core::transform::TransformNode<_, _, _, _, _>, input: ImageFrame, params: [DVec2, f64, DVec2, DVec2, DVec2]), + register_node!(graphene_core::transform::TransformNode<_, _, _, _, _>, input: WasmSurfaceHandleFrame, params: [DVec2, f64, DVec2, DVec2, DVec2]), register_node!(graphene_core::transform::SetTransformNode<_>, input: VectorData, params: [VectorData]), register_node!(graphene_core::transform::SetTransformNode<_>, input: ImageFrame, params: [ImageFrame]), register_node!(graphene_core::transform::SetTransformNode<_>, input: VectorData, params: [DAffine2]),