diff --git a/Cargo.lock b/Cargo.lock index 7f9080a6..932f4478 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1773,6 +1773,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", + "web-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 574b8d34..ee4de000 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,9 +53,6 @@ opt-level = 3 [profile.dev.package.image] opt-level = 3 -[profile.dev.package.png] -opt-level = 3 - [profile.dev.package.xxhash-rust] opt-level = 3 diff --git a/editor/src/application.rs b/editor/src/application.rs index 434ebd41..1fe21fe3 100644 --- a/editor/src/application.rs +++ b/editor/src/application.rs @@ -23,6 +23,10 @@ impl Editor { responses } + + pub fn poll_node_graph_evaluation(&mut self, responses: &mut VecDeque) { + self.dispatcher.poll_node_graph_evaluation(responses); + } } impl Default for Editor { diff --git a/editor/src/dispatcher.rs b/editor/src/dispatcher.rs index c744e365..93c021ab 100644 --- a/editor/src/dispatcher.rs +++ b/editor/src/dispatcher.rs @@ -205,6 +205,10 @@ impl Dispatcher { list } + pub fn poll_node_graph_evaluation(&mut self, responses: &mut VecDeque) { + self.message_handlers.portfolio_message_handler.poll_node_graph_evaluation(responses); + } + /// Create the tree structure for logging the messages as a tree fn create_indents(queues: &[VecDeque]) -> String { String::from_iter(queues.iter().enumerate().skip(1).map(|(index, queue)| { diff --git a/editor/src/messages/portfolio/document/document_message_handler.rs b/editor/src/messages/portfolio/document/document_message_handler.rs index aae23128..958bd30f 100644 --- a/editor/src/messages/portfolio/document/document_message_handler.rs +++ b/editor/src/messages/portfolio/document/document_message_handler.rs @@ -137,8 +137,8 @@ impl MessageHandler responses.add(MoveSelectedLayersTo { folder_path: folder_path.clone(), - insert_index: insert_index.clone(), - reverse_index: reverse_index.clone(), + insert_index: *insert_index, + reverse_index: *reverse_index, }), DocumentResponse::CreatedLayer { path, is_selected } => { if self.layer_metadata.contains_key(path) { diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs index 91d5fcec..5049bb50 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs @@ -1,21 +1,19 @@ use crate::messages::layout::utility_types::widget_prelude::*; -use crate::messages::portfolio::utility_types::ImaginateServerStatus; + use crate::messages::prelude::*; -use document_legacy::layers::layer_info::LayerDataTypeDiscriminant; -use document_legacy::Operation; use glam::DVec2; +use graph_craft::concrete; use graph_craft::document::value::TaggedValue; use graph_craft::document::{DocumentNode, NodeId, NodeInput}; -use graph_craft::{concrete, imaginate_input::*}; use graphene_core::raster::{BlendMode, Color, ImageFrame, LuminanceCalculation, RedGreenBlue, RelativeAbsolute, SelectiveColorChoice}; use graphene_core::text::Font; use graphene_core::vector::style::{FillType, GradientType, LineCap, LineJoin}; -use graphene_core::EditorApi; + use graphene_core::{Cow, Type, TypeDescriptor}; use super::document_node_types::NodePropertiesContext; -use super::{FrontendGraphDataType, IMAGINATE_NODE}; +use super::FrontendGraphDataType; pub fn string_properties(text: impl Into) -> Vec { let widget = WidgetHolder::text_widget(text); @@ -731,7 +729,7 @@ pub fn adjust_selective_color_properties(document_node: &DocumentNode, node_id: .into_iter() .map(|section| { section - .into_iter() + .iter() .map(|choice| DropdownEntryData::new(choice.to_string()).on_update(update_value(move |_| TaggedValue::SelectiveColorChoice(*choice), node_id, colors_index))) .collect() }) @@ -956,7 +954,8 @@ pub fn node_section_font(document_node: &DocumentNode, node_id: NodeId, _context result } -pub fn imaginate_properties(document_node: &DocumentNode, node_id: NodeId, context: &mut NodePropertiesContext) -> Vec { +pub fn imaginate_properties(_document_node: &DocumentNode, _node_id: NodeId, _context: &mut NodePropertiesContext) -> Vec { + /* let imaginate_node = [context.nested_path, &[node_id]].concat(); let layer_path = context.layer_path.to_vec(); @@ -1513,6 +1512,8 @@ pub fn imaginate_properties(document_node: &DocumentNode, node_id: NodeId, conte layout.extend_from_slice(&[improve_faces, tiling]); layout + */ + todo!() } fn unknown_node_properties(document_node: &DocumentNode) -> Vec { diff --git a/editor/src/messages/portfolio/portfolio_message_handler.rs b/editor/src/messages/portfolio/portfolio_message_handler.rs index bcac1a19..55a9bdee 100644 --- a/editor/src/messages/portfolio/portfolio_message_handler.rs +++ b/editor/src/messages/portfolio/portfolio_message_handler.rs @@ -22,7 +22,7 @@ use graph_craft::document::{NodeId, NodeInput}; use graphene_core::raster::Image; use graphene_core::text::Font; -#[derive(Debug, Clone, Default)] +#[derive(Debug, Default)] pub struct PortfolioMessageHandler { menu_bar_message_handler: MenuBarMessageHandler, documents: HashMap, @@ -215,6 +215,7 @@ impl MessageHandler { self.persistent_data.imaginate_server_status = ImaginateServerStatus::Checking; @@ -456,7 +457,7 @@ impl MessageHandler { - let result = self.executor.evaluate_node_graph( + let result = self.executor.submit_node_graph_evaluation( (document_id, &mut self.documents), layer_path, (input_image_data, size), @@ -690,4 +691,10 @@ impl PortfolioMessageHandler { } } } + + pub fn poll_node_graph_evaluation(&mut self, responses: &mut VecDeque) { + self.executor.poll_node_graph_evaluation(responses).unwrap_or_else(|e| { + log::error!("Error while evaluating node graph: {}", e); + }); + } } diff --git a/editor/src/messages/tool/common_functionality/overlay_renderer.rs b/editor/src/messages/tool/common_functionality/overlay_renderer.rs index 06001ded..56403751 100644 --- a/editor/src/messages/tool/common_functionality/overlay_renderer.rs +++ b/editor/src/messages/tool/common_functionality/overlay_renderer.rs @@ -107,6 +107,7 @@ impl OverlayRenderer { // Eventually will get replaced with am immediate mode renderer for overlays } } + responses.add(OverlaysMessage::Rerender); } pub fn clear_subpath_overlays(&mut self, document: &Document, layer_path: Vec, responses: &mut VecDeque) { diff --git a/editor/src/messages/tool/common_functionality/shape_editor.rs b/editor/src/messages/tool/common_functionality/shape_editor.rs index 836772c5..faa54c84 100644 --- a/editor/src/messages/tool/common_functionality/shape_editor.rs +++ b/editor/src/messages/tool/common_functionality/shape_editor.rs @@ -219,7 +219,7 @@ impl ShapeState { let Ok(layer) = document.layer(layer_path) else { continue }; let Some(vector_data) = layer.as_vector_data() else { continue }; - let opposing_handle_lengths = opposing_handle_lengths.as_ref().map(|lengths| lengths.get(layer_path)).flatten(); + let opposing_handle_lengths = opposing_handle_lengths.as_ref().and_then(|lengths| lengths.get(layer_path)); let transform = document.multiply_transforms(layer_path).unwrap_or(glam::DAffine2::IDENTITY); diff --git a/editor/src/node_graph_executor.rs b/editor/src/node_graph_executor.rs index bf14c8b7..eaf01ea6 100644 --- a/editor/src/node_graph_executor.rs +++ b/editor/src/node_graph_executor.rs @@ -1,32 +1,36 @@ use crate::messages::frontend::utility_types::FrontendImageData; use crate::messages::portfolio::document::node_graph::wrap_network_in_scope; -use crate::messages::portfolio::document::utility_types::misc::DocumentRenderMode; + use crate::messages::portfolio::utility_types::PersistentData; use crate::messages::prelude::*; -use document_legacy::{document::pick_safe_imaginate_resolution, layers::layer_info::LayerDataType}; +use document_legacy::layers::layer_info::LayerDataType; use document_legacy::{LayerId, Operation}; -use dyn_any::DynAny; -use graph_craft::document::{generate_uuid, DocumentNodeImplementation, NodeId, NodeInput, NodeNetwork, NodeOutput}; + +use graph_craft::document::value::TaggedValue; +use graph_craft::document::{generate_uuid, DocumentNodeImplementation, NodeId, NodeNetwork}; use graph_craft::executor::Compiler; use graph_craft::imaginate_input::*; use graph_craft::{concrete, Type, TypeDescriptor}; use graphene_core::raster::{Image, ImageFrame}; use graphene_core::renderer::{SvgSegment, SvgSegmentList}; +use graphene_core::text::FontCache; use graphene_core::vector::style::ViewMode; -use graphene_core::vector::VectorData; + use graphene_core::{Color, EditorApi}; use interpreted_executor::executor::DynamicExecutor; use glam::{DAffine2, DVec2}; use std::borrow::Cow; +use std::cell::RefCell; +use std::sync::mpsc::{Receiver, Sender}; use std::sync::Arc; -#[derive(Debug, Clone, Default)] -pub struct NodeGraphExecutor { +pub struct NodeRuntime { pub(crate) executor: DynamicExecutor, - // TODO: This is a memory leak since layers are never removed - pub(crate) last_output_type: HashMap, Option>, + font_cache: FontCache, + receiver: Receiver, + sender: Sender, pub(crate) thumbnails: HashMap>, } @@ -35,7 +39,76 @@ fn get_imaginate_index(name: &str) -> usize { IMAGINATE_NODE.inputs.iter().position(|input| input.name == name).unwrap_or_else(|| panic!("Input {name} not found")) } -impl NodeGraphExecutor { +enum NodeRuntimeMessage { + GenerationRequest(GenerationRequest), + FontCacheUpdate(FontCache), +} + +pub(crate) struct GenerationRequest { + generation_id: u64, + graph: NodeNetwork, + path: Vec, + image_frame: Option>, +} +pub(crate) struct GenerationResponse { + generation_id: u64, + result: Result, + updates: VecDeque, + new_thumbnails: HashMap>, +} + +thread_local! { + static NODE_RUNTIME: RefCell> = RefCell::new(None); +} + +impl NodeRuntime { + fn new(receiver: Receiver, sender: Sender) -> Self { + let executor = DynamicExecutor::default(); + Self { + executor, + receiver, + sender, + font_cache: FontCache::default(), + thumbnails: Default::default(), + } + } + pub fn run(&mut self) { + let mut requests = self.receiver.try_iter().collect::>(); + // TODO: Currently we still render the document after we submit the node graph execution request. + // This should be avoided in the future. + requests.reverse(); + requests.dedup_by_key(|x| match x { + NodeRuntimeMessage::FontCacheUpdate(_) => None, + NodeRuntimeMessage::GenerationRequest(x) => Some(x.path.clone()), + }); + requests.reverse(); + for request in requests { + match request { + NodeRuntimeMessage::FontCacheUpdate(font_cache) => self.font_cache = font_cache, + NodeRuntimeMessage::GenerationRequest(GenerationRequest { + generation_id, + graph, + image_frame, + path, + .. + }) => { + let (network, monitor_nodes) = Self::wrap_network(graph); + + let result = self.execute_network(network, image_frame); + let mut responses = VecDeque::new(); + self.update_thumbnails(&path, monitor_nodes, &mut responses); + let response = GenerationResponse { + generation_id, + result, + updates: responses, + new_thumbnails: self.thumbnails.clone(), + }; + self.sender.send(response); + } + } + } + } + /// Wraps a network in a scope and returns the new network and the paths to the monitor nodes. fn wrap_network(network: NodeNetwork) -> (NodeNetwork, Vec>) { let mut scoped_network = wrap_network_in_scope(network); @@ -52,8 +125,12 @@ impl NodeGraphExecutor { (scoped_network, monitor_nodes) } - /// Executes the network by flattening it and creating a borrow stack. - fn execute_network<'a>(&'a mut self, scoped_network: NodeNetwork, editor_api: EditorApi<'a>) -> Result, String> { + fn execute_network<'a>(&'a mut self, scoped_network: NodeNetwork, image_frame: Option>) -> Result { + let editor_api = EditorApi { + font_cache: Some(&self.font_cache), + image_frame, + }; + // We assume only one output assert_eq!(scoped_network.outputs.len(), 1, "Graph with multiple outputs not yet handled"); let c = Compiler {}; @@ -68,316 +145,18 @@ impl NodeGraphExecutor { use dyn_any::IntoDynAny; use graph_craft::executor::Executor; - match self.executor.input_type() { + let result = match self.executor.input_type() { Some(t) if t == concrete!(EditorApi) => self.executor.execute(editor_api.into_dyn()).map_err(|e| e.to_string()), Some(t) if t == concrete!(()) => self.executor.execute(().into_dyn()).map_err(|e| e.to_string()), _ => Err("Invalid input type".to_string()), - } - } - - pub fn introspect_node(&self, path: &[NodeId]) -> Option> { - self.executor.introspect(path).flatten() - } - - pub fn previous_output_type(&self, path: &[LayerId]) -> Option { - self.last_output_type.get(path).cloned().flatten() - } - - /// Computes an input for a node in the graph - pub fn compute_input(&mut self, old_network: &NodeNetwork, node_path: &[NodeId], mut input_index: usize, editor_api: Cow>) -> Result { - let mut network = old_network.clone(); - // Adjust the output of the graph so we find the relevant output - 'outer: for end in (0..node_path.len()).rev() { - let mut inner_network = &mut network; - for &node_id in &node_path[..end] { - inner_network.outputs[0] = NodeOutput::new(node_id, 0); - - let Some(new_inner) = inner_network.nodes.get_mut(&node_id).and_then(|node| node.implementation.get_network_mut()) else { - return Err("Failed to find network".to_string()); - }; - inner_network = new_inner; - } - match &inner_network.nodes.get(&node_path[end]).unwrap().inputs[input_index] { - // If the input is from a parent network then adjust the input index and continue iteration - NodeInput::Network(_) => { - input_index = inner_network - .inputs - .iter() - .enumerate() - .filter(|&(_index, &id)| id == node_path[end]) - .nth(input_index) - .ok_or_else(|| "Invalid network input".to_string())? - .0; - } - // If the input is just a value, return that value - NodeInput::Value { tagged_value, .. } => return dyn_any::downcast::(tagged_value.clone().to_any()).map(|v| *v), - // If the input is from a node, set the node to be the output (so that is what is evaluated) - NodeInput::Node { node_id, output_index, .. } => { - inner_network.outputs[0] = NodeOutput::new(*node_id, *output_index); - break 'outer; - } - NodeInput::ShortCircut(_) => (), - } - } - - let (network, _) = Self::wrap_network(network); - let boxed = self.execute_network(network, editor_api.into_owned())?; - - dyn_any::downcast::(boxed).map(|v| *v) - } - - /// Encodes an image into a format using the image crate - fn encode_img(image: Image, resize: Option, format: image::ImageOutputFormat) -> Result<(Vec, (u32, u32)), String> { - use image::{ImageBuffer, Rgba}; - use std::io::Cursor; - - let (result_bytes, width, height) = image.into_flat_u8(); - - let mut output: ImageBuffer, _> = image::ImageBuffer::from_raw(width, height, result_bytes).ok_or_else(|| "Invalid image size".to_string())?; - if let Some(size) = resize { - let size = size.as_uvec2(); - if size.x > 0 && size.y > 0 { - output = image::imageops::resize(&output, size.x, size.y, image::imageops::Triangle); - } - } - let size = output.dimensions(); - let mut image_data: Vec = Vec::new(); - output.write_to(&mut Cursor::new(&mut image_data), format).map_err(|e| e.to_string())?; - Ok::<_, String>((image_data, size)) - } - - fn imaginate_parameters(&mut self, network: &NodeNetwork, node_path: &[LayerId], resolution: DVec2, editor_api: &EditorApi) -> Result { - let get = get_imaginate_index; - Ok(ImaginateGenerationParameters { - seed: self.compute_input::(network, node_path, get("Seed"), Cow::Borrowed(editor_api))? as u64, - resolution: resolution.as_uvec2().into(), - samples: self.compute_input::(network, node_path, get("Samples"), Cow::Borrowed(editor_api))? as u32, - sampling_method: self - .compute_input::(network, node_path, get("Sampling Method"), Cow::Borrowed(editor_api))? - .api_value() - .to_string(), - text_guidance: self.compute_input(network, node_path, get("Prompt Guidance"), Cow::Borrowed(editor_api))?, - text_prompt: self.compute_input(network, node_path, get("Prompt"), Cow::Borrowed(editor_api))?, - negative_prompt: self.compute_input(network, node_path, get("Negative Prompt"), Cow::Borrowed(editor_api))?, - image_creativity: Some(self.compute_input::(network, node_path, get("Image Creativity"), Cow::Borrowed(editor_api))? / 100.), - restore_faces: self.compute_input(network, node_path, get("Improve Faces"), Cow::Borrowed(editor_api))?, - tiling: self.compute_input(network, node_path, get("Tiling"), Cow::Borrowed(editor_api))?, - }) - } - - fn imaginate_base_image(&mut self, network: &NodeNetwork, imaginate_node_path: &[LayerId], resolution: DVec2, editor_api: &EditorApi) -> Result, String> { - let use_base_image = self.compute_input::(&network, &imaginate_node_path, get_imaginate_index("Adapt Input Image"), Cow::Borrowed(editor_api))?; - let input_image_frame: Option> = if use_base_image { - Some(self.compute_input::>(&network, &imaginate_node_path, get_imaginate_index("Input Image"), Cow::Borrowed(editor_api))?) - } else { - None }; - - let base_image = if let Some(ImageFrame { image, transform }) = input_image_frame { - // Only use if has size - if image.width > 0 && image.height > 0 { - let (image_data, size) = Self::encode_img(image, Some(resolution), image::ImageOutputFormat::Png)?; - let size = DVec2::new(size.0 as f64, size.1 as f64); - let mime = "image/png".to_string(); - Some((ImaginateBaseImage { image_data, size, mime }, transform)) - } else { - info!("Base image is input but has no size."); - None - } - } else { - None - }; - Ok(base_image) - } - - fn imaginate_mask_image( - &mut self, - network: &NodeNetwork, - node_path: &[LayerId], - editor_api: &EditorApi<'_>, - image_transform: Option, - document: &mut DocumentMessageHandler, - persistent_data: &PersistentData, - ) -> Result, String> { - if let Some(transform) = image_transform { - let mask_path: Option> = self.compute_input(&network, &node_path, get_imaginate_index("Masking Layer"), Cow::Borrowed(&editor_api))?; - - // Calculate the size of the frame - let size = DVec2::new(transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length()); - - // Render the masking layer within the frame - let old_transforms = document.remove_document_transform(); - let mask_is_some = mask_path.is_some(); - let mask_image = mask_path.filter(|mask_layer_path| document.document_legacy.layer(mask_layer_path).is_ok()).map(|mask_layer_path| { - let render_mode = DocumentRenderMode::LayerCutout(&mask_layer_path, graphene_core::raster::color::Color::WHITE); - let svg = document.render_document(size, transform.inverse(), persistent_data, render_mode); - - ImaginateMaskImage { svg, size } - }); - - if mask_is_some && mask_image.is_none() { - return Err( - "Imagination masking layer is missing.\nIt may have been deleted or moved. Please drag a new layer reference\ninto the 'Masking Layer' parameter input, then generate again." - .to_string(), - ); - } - - document.restore_document_transform(old_transforms); - Ok(mask_image) - } else { - Ok(None) - } - } - - fn generate_imaginate( - &mut self, - network: NodeNetwork, - imaginate_node_path: Vec, - (document, document_id): (&mut DocumentMessageHandler, u64), - layer_path: Vec, - mut editor_api: EditorApi<'_>, - (preferences, persistent_data): (&PreferencesMessageHandler, &PersistentData), - ) -> Result { - let image = editor_api.image_frame.take(); - - // Get the node graph layer - let layer = document.document_legacy.layer(&layer_path).map_err(|e| format!("No layer: {e:?}"))?; - let transform = layer.transform; - - let resolution: Option = self.compute_input(&network, &imaginate_node_path, get_imaginate_index("Resolution"), Cow::Borrowed(&editor_api))?; - let resolution = resolution.unwrap_or_else(|| { - let (x, y) = pick_safe_imaginate_resolution((transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length())); - DVec2::new(x as f64, y as f64) - }); - - let parameters = self.imaginate_parameters(&network, &imaginate_node_path, resolution, &editor_api)?; - - editor_api.image_frame = image; - let base = self.imaginate_base_image(&network, &imaginate_node_path, resolution, &editor_api)?; - let image_transform = base.as_ref().map(|base| base.1); - let base_image = base.map(|base| base.0); - - let mask_image = self.imaginate_mask_image(&network, &imaginate_node_path, &editor_api, image_transform, document, persistent_data)?; - - Ok(FrontendMessage::TriggerImaginateGenerate { - parameters: Box::new(parameters), - base_image: base_image.map(Box::new), - mask_image: mask_image.map(Box::new), - mask_paint_mode: if self.compute_input::(&network, &imaginate_node_path, get_imaginate_index("Inpaint"), Cow::Borrowed(&editor_api))? { - ImaginateMaskPaintMode::Inpaint - } else { - ImaginateMaskPaintMode::Outpaint + match result { + Ok(result) => match TaggedValue::try_from_any(result) { + Some(x) => Ok(x), + None => Err("Invalid output type".to_string()), }, - mask_blur_px: self.compute_input::(&network, &imaginate_node_path, get_imaginate_index("Mask Blur"), Cow::Borrowed(&editor_api))? as u32, - imaginate_mask_starting_fill: self.compute_input(&network, &imaginate_node_path, get_imaginate_index("Mask Starting Fill"), Cow::Borrowed(&editor_api))?, - hostname: preferences.imaginate_server_hostname.clone(), - refresh_frequency: preferences.imaginate_refresh_frequency, - document_id, - layer_path, - node_path: imaginate_node_path, + Err(e) => Err(e), } - .into()) - } - - /// Generate a new [`FrontendImageData`] from the [`Image`]. - fn to_frontend_image_data(image: Image, transform: Option<[f64; 6]>, layer_path: &[LayerId], node_id: Option, resize: Option) -> Result { - let (image_data, _size) = Self::encode_img(image, resize, image::ImageOutputFormat::Bmp)?; - - let mime = "image/bmp".to_string(); - let image_data = std::sync::Arc::new(image_data); - - Ok(FrontendImageData { - path: layer_path.to_vec(), - node_id, - image_data, - mime, - transform, - }) - } - - /// Evaluates a node graph, computing either the Imaginate node or the entire graph - pub fn evaluate_node_graph( - &mut self, - (document_id, documents): (u64, &mut HashMap), - layer_path: Vec, - (input_image_data, (width, height)): (Vec, (u32, u32)), - imaginate_node: Option>, - persistent_data: (&PreferencesMessageHandler, &PersistentData), - responses: &mut VecDeque, - ) -> Result<(), String> { - // Reformat the input image data into an RGBA f32 image - let image = graphene_core::raster::Image::from_image_data(&input_image_data, width, height); - - // Get the node graph layer - let document = documents.get_mut(&document_id).ok_or_else(|| "Invalid document".to_string())?; - let layer = document.document_legacy.layer(&layer_path).map_err(|e| format!("No layer: {e:?}"))?; - - // Construct the input image frame - let transform = DAffine2::IDENTITY; - let image_frame = ImageFrame { image, transform }; - let editor_api = EditorApi { - image_frame: Some(image_frame), - font_cache: Some(&persistent_data.1.font_cache), - }; - - let layer_layer = match &layer.data { - LayerDataType::Layer(layer) => Ok(layer), - _ => Err("Invalid layer type".to_string()), - }?; - let network = layer_layer.network.clone(); - - // Special execution path for generating Imaginate (as generation requires IO from outside node graph) - if let Some(imaginate_node) = imaginate_node { - responses.add(self.generate_imaginate(network, imaginate_node, (document, document_id), layer_path, editor_api, persistent_data)?); - return Ok(()); - } - // Execute the node graph - let (network, monitor_nodes) = Self::wrap_network(network); - let boxed_node_graph_output = self.execute_network(network, editor_api)?; - - // Check if the output is vector data - if core::any::TypeId::of::() == DynAny::type_id(boxed_node_graph_output.as_ref()) { - // Update the cached vector data on the layer - let vector_data: VectorData = dyn_any::downcast(boxed_node_graph_output).map(|v| *v)?; - let transform = vector_data.transform.to_cols_array(); - self.last_output_type.insert(layer_path.clone(), Some(concrete!(VectorData))); - responses.add(Operation::SetLayerTransform { path: layer_path.clone(), transform }); - responses.add(Operation::SetVectorData { path: layer_path, vector_data }); - } else if core::any::TypeId::of::>() == DynAny::type_id(boxed_node_graph_output.as_ref()) { - // Attempt to downcast to an image frame - let ImageFrame { image, transform } = dyn_any::downcast(boxed_node_graph_output).map(|image_frame| *image_frame)?; - self.last_output_type.insert(layer_path.clone(), Some(concrete!(ImageFrame))); - - // Don't update the frame's transform if the new transform is DAffine2::ZERO. - let transform = (!transform.abs_diff_eq(DAffine2::ZERO, f64::EPSILON)).then_some(transform.to_cols_array()); - - // If no image was generated, clear the frame - if image.width == 0 || image.height == 0 { - responses.add(DocumentMessage::FrameClear); - - // Update the transform based on the graph output - if let Some(transform) = transform { - responses.add(Operation::SetLayerTransform { path: layer_path.clone(), transform }); - } - } else { - let image_data = vec![Self::to_frontend_image_data(image, transform, &layer_path, None, None)?]; - responses.add(FrontendMessage::UpdateImageData { document_id, image_data }); - } - } else if core::any::TypeId::of::() == DynAny::type_id(boxed_node_graph_output.as_ref()) { - let artboard: graphene_core::Artboard = dyn_any::downcast(boxed_node_graph_output).map(|artboard| *artboard)?; - info!("{artboard:#?}"); - self.update_thumbnails(&layer_path, monitor_nodes, responses); - - return Err(format!("Artboard (see console)")); - } else if core::any::TypeId::of::() == DynAny::type_id(boxed_node_graph_output.as_ref()) { - let graphic_group: graphene_core::GraphicGroup = dyn_any::downcast(boxed_node_graph_output).map(|graphic| *graphic)?; - info!("{graphic_group:#?}"); - self.update_thumbnails(&layer_path, monitor_nodes, responses); - - return Err(format!("Graphic group (see console)")); - } - - Ok(()) } /// Recomputes the thumbnails for the layers in the graph, modifying the state and updating the UI. @@ -410,7 +189,7 @@ impl NodeGraphExecutor { } } let resize = Some(DVec2::splat(100.)); - let create_image_data = |(node_id, image)| Self::to_frontend_image_data(image, None, layer_path, Some(node_id), resize).ok(); + let create_image_data = |(node_id, image)| NodeGraphExecutor::to_frontend_image_data(image, None, layer_path, Some(node_id), resize).ok(); image_data.extend(render.image_data.into_iter().filter_map(create_image_data)) } if !image_data.is_empty() { @@ -419,6 +198,268 @@ impl NodeGraphExecutor { responses.add(NodeGraphMessage::SendGraph { should_rerender: false }); } } +} + +pub fn run_node_graph() { + NODE_RUNTIME.with(|runtime| { + let mut runtime = runtime.borrow_mut(); + if let Some(runtime) = runtime.as_mut() { + runtime.run(); + } + }); +} + +#[derive(Debug)] +pub struct NodeGraphExecutor { + pub(crate) executor: DynamicExecutor, + sender: Sender, + receiver: Receiver, + // TODO: This is a memory leak since layers are never removed + pub(crate) last_output_type: HashMap, Option>, + pub(crate) thumbnails: HashMap>, + futures: HashMap, +} + +#[derive(Debug, Clone)] +struct ExecutionContext { + layer_path: Vec, + document_id: u64, +} + +impl Default for NodeGraphExecutor { + fn default() -> Self { + let (request_sender, request_reciever) = std::sync::mpsc::channel(); + let (response_sender, response_reciever) = std::sync::mpsc::channel(); + NODE_RUNTIME.with(|runtime| { + let mut runtime = runtime.borrow_mut(); + *runtime = Some(NodeRuntime::new(request_reciever, response_sender)); + }); + + Self { + executor: Default::default(), + futures: Default::default(), + sender: request_sender, + receiver: response_reciever, + last_output_type: Default::default(), + thumbnails: Default::default(), + } + } +} + +impl NodeGraphExecutor { + /// Execute the network by flattening it and creating a borrow stack. + fn queue_execution(&self, network: NodeNetwork, image_frame: Option>, layer_path: Vec) -> u64 { + let generation_id = generate_uuid(); + let request = GenerationRequest { + path: layer_path, + graph: network, + image_frame, + generation_id, + }; + self.sender.send(NodeRuntimeMessage::GenerationRequest(request)); + + generation_id + } + + pub fn update_font_cache(&self, font_cache: FontCache) { + self.sender.send(NodeRuntimeMessage::FontCacheUpdate(font_cache)); + } + + pub fn introspect_node(&self, path: &[NodeId]) -> Option> { + self.executor.introspect(path).flatten() + } + + pub fn previous_output_type(&self, path: &[LayerId]) -> Option { + self.last_output_type.get(path).cloned().flatten() + } + + /// Computes an input for a node in the graph + pub fn compute_input(&mut self, _old_network: &NodeNetwork, _node_path: &[NodeId], _input_index: usize, _editor_api: Cow>) -> Result { + todo!() + /* + let mut network = old_network.clone(); + // Adjust the output of the graph so we find the relevant output + 'outer: for end in (0..node_path.len()).rev() { + let mut inner_network = &mut network; + for &node_id in &node_path[..end] { + inner_network.outputs[0] = NodeOutput::new(node_id, 0); + + let Some(new_inner) = inner_network.nodes.get_mut(&node_id).and_then(|node| node.implementation.get_network_mut()) else { + return Err("Failed to find network".to_string()); + }; + inner_network = new_inner; + } + match &inner_network.nodes.get(&node_path[end]).unwrap().inputs[input_index] { + // If the input is from a parent network then adjust the input index and continue iteration + NodeInput::Network(_) => { + input_index = inner_network + .inputs + .iter() + .enumerate() + .filter(|&(_index, &id)| id == node_path[end]) + .nth(input_index) + .ok_or_else(|| "Invalid network input".to_string())? + .0; + } + // If the input is just a value, return that value + NodeInput::Value { tagged_value, .. } => return Some(dyn_any::downcast::(tagged_value.clone().to_any()).map(|v| *v)), + // If the input is from a node, set the node to be the output (so that is what is evaluated) + NodeInput::Node { node_id, output_index, .. } => { + inner_network.outputs[0] = NodeOutput::new(*node_id, *output_index); + break 'outer; + } + NodeInput::ShortCircut(_) => (), + } + } + + self.queue_execution(network, editor_api.into_owned())? + */ + } + + /// Encodes an image into a format using the image crate + fn encode_img(image: Image, resize: Option, format: image::ImageOutputFormat) -> Result<(Vec, (u32, u32)), String> { + use image::{ImageBuffer, Rgba}; + use std::io::Cursor; + + let (result_bytes, width, height) = image.into_flat_u8(); + + let mut output: ImageBuffer, _> = image::ImageBuffer::from_raw(width, height, result_bytes).ok_or_else(|| "Invalid image size".to_string())?; + if let Some(size) = resize { + let size = size.as_uvec2(); + if size.x > 0 && size.y > 0 { + output = image::imageops::resize(&output, size.x, size.y, image::imageops::Triangle); + } + } + let size = output.dimensions(); + let mut image_data: Vec = Vec::new(); + output.write_to(&mut Cursor::new(&mut image_data), format).map_err(|e| e.to_string())?; + Ok::<_, String>((image_data, size)) + } + + /// Generate a new [`FrontendImageData`] from the [`Image`]. + fn to_frontend_image_data(image: Image, transform: Option<[f64; 6]>, layer_path: &[LayerId], node_id: Option, resize: Option) -> Result { + let (image_data, _size) = Self::encode_img(image, resize, image::ImageOutputFormat::Bmp)?; + + let mime = "image/bmp".to_string(); + let image_data = std::sync::Arc::new(image_data); + + Ok(FrontendImageData { + path: layer_path.to_vec(), + node_id, + image_data, + mime, + transform, + }) + } + + /// Evaluates a node graph, computing either the Imaginate node or the entire graph + pub fn submit_node_graph_evaluation( + &mut self, + (document_id, documents): (u64, &mut HashMap), + layer_path: Vec, + (input_image_data, (width, height)): (Vec, (u32, u32)), + _imaginate_node: Option>, + _persistent_data: (&PreferencesMessageHandler, &PersistentData), + _responses: &mut VecDeque, + ) -> Result<(), String> { + // Reformat the input image data into an RGBA f32 image + let image = graphene_core::raster::Image::from_image_data(&input_image_data, width, height); + + // Get the node graph layer + let document = documents.get_mut(&document_id).ok_or_else(|| "Invalid document".to_string())?; + let layer = document.document_legacy.layer(&layer_path).map_err(|e| format!("No layer: {e:?}"))?; + + // Construct the input image frame + let transform = DAffine2::IDENTITY; + let image_frame = ImageFrame { image, transform }; + + let layer_layer = match &layer.data { + LayerDataType::Layer(layer) => Ok(layer), + _ => Err("Invalid layer type".to_string()), + }?; + let network = layer_layer.network.clone(); + + // Special execution path for generating Imaginate (as generation requires IO from outside node graph) + /*if let Some(imaginate_node) = imaginate_node { + responses.add(self.generate_imaginate(network, imaginate_node, (document, document_id), layer_path, editor_api, persistent_data)?); + return Ok(()); + }*/ + // Execute the node graph + let generation_id = self.queue_execution(network, Some(image_frame), layer_path.clone()); + + self.futures.insert(generation_id, ExecutionContext { layer_path, document_id }); + + Ok(()) + } + + pub fn poll_node_graph_evaluation(&mut self, responses: &mut VecDeque) -> Result<(), String> { + let results = self.receiver.try_iter().collect::>(); + for response in results { + let GenerationResponse { + generation_id, + result, + updates, + new_thumbnails, + } = response; + self.thumbnails = new_thumbnails; + let node_graph_output = result.map_err(|e| format!("Node graph evaluation failed: {:?}", e))?; + let execution_context = self.futures.remove(&generation_id).ok_or_else(|| "Invalid generation ID".to_string())?; + responses.extend(updates); + self.process_node_graph_output(node_graph_output, execution_context.layer_path.clone(), responses, execution_context.document_id)?; + responses.add(DocumentMessage::LayerChanged { + affected_layer_path: execution_context.layer_path, + }); + responses.add(DocumentMessage::RenderDocument); + responses.add(ArtboardMessage::RenderArtboards); + responses.add(DocumentMessage::DocumentStructureChanged); + responses.add(BroadcastEvent::DocumentIsDirty); + responses.add(DocumentMessage::DirtyRenderDocument); + responses.add(DocumentMessage::Overlays(OverlaysMessage::Rerender)); + } + Ok(()) + } + + fn process_node_graph_output(&mut self, node_graph_output: TaggedValue, layer_path: Vec, responses: &mut VecDeque, document_id: u64) -> Result<(), String> { + self.last_output_type.insert(layer_path.clone(), Some(node_graph_output.ty())); + match node_graph_output { + TaggedValue::VectorData(vector_data) => { + // Update the cached vector data on the layer + let transform = vector_data.transform.to_cols_array(); + responses.add(Operation::SetLayerTransform { path: layer_path.clone(), transform }); + responses.add(Operation::SetVectorData { path: layer_path, vector_data }); + } + TaggedValue::ImageFrame(ImageFrame { image, transform }) => { + // Don't update the frame's transform if the new transform is DAffine2::ZERO. + let transform = (!transform.abs_diff_eq(DAffine2::ZERO, f64::EPSILON)).then_some(transform.to_cols_array()); + + // If no image was generated, clear the frame + if image.width == 0 || image.height == 0 { + responses.add(DocumentMessage::FrameClear); + + // Update the transform based on the graph output + if let Some(transform) = transform { + responses.add(Operation::SetLayerTransform { path: layer_path, transform }); + } + } else { + // Update the image data + let image_data = vec![Self::to_frontend_image_data(image, transform, &layer_path, None, None)?]; + responses.add(FrontendMessage::UpdateImageData { document_id, image_data }); + } + } + TaggedValue::Artboard(artboard) => { + info!("{artboard:#?}"); + return Err("Artboard (see console)".to_string()); + } + TaggedValue::GraphicGroup(graphic_group) => { + info!("{graphic_group:#?}"); + return Err("Graphic group (see console)".to_string()); + } + _ => { + return Err(format!("Invalid node graph output type: {:#?}", node_graph_output)); + } + }; + Ok(()) + } /// When a blob url for a thumbnail is loaded, update the state and the UI. pub fn insert_thumbnail_blob_url(&mut self, blob_url: String, layer_id: LayerId, node_id: NodeId, responses: &mut VecDeque) { @@ -426,7 +467,6 @@ impl NodeGraphExecutor { if let Some(segment) = layer.values_mut().flat_map(|segments| segments.iter_mut()).find(|segment| **segment == SvgSegment::BlobUrl(node_id)) { *segment = SvgSegment::String(blob_url); responses.add(NodeGraphMessage::SendGraph { should_rerender: false }); - return; } } } diff --git a/frontend/wasm/Cargo.toml b/frontend/wasm/Cargo.toml index e889ab46..da42b290 100644 --- a/frontend/wasm/Cargo.toml +++ b/frontend/wasm/Cargo.toml @@ -23,15 +23,23 @@ editor = { path = "../../editor", package = "graphite-editor" } document-legacy = { path = "../../document-legacy", package = "graphite-document-legacy" } graph-craft = { path = "../../node-graph/graph-craft" } log = "0.4" -graphene-core = { path = "../../node-graph/gcore", features = ["async", "std", "alloc"] } +graphene-core = { path = "../../node-graph/gcore", features = [ + "async", + "std", + "alloc", +] } serde = { version = "1.0", features = ["derive"] } wasm-bindgen = { version = "0.2.84" } serde-wasm-bindgen = "0.4.1" js-sys = "0.3.55" wasm-bindgen-futures = "0.4.33" -ron = {version = "0.8", optional = true} +ron = { version = "0.8", optional = true } bezier-rs = { path = "../../libraries/bezier-rs" } +[dependencies.web-sys] +version = "0.3.4" +features = ['Window'] + [dev-dependencies] wasm-bindgen-test = "0.3.22" diff --git a/frontend/wasm/src/editor_api.rs b/frontend/wasm/src/editor_api.rs index 841d564a..752bf692 100644 --- a/frontend/wasm/src/editor_api.rs +++ b/frontend/wasm/src/editor_api.rs @@ -18,6 +18,7 @@ use graphene_core::raster::color::Color; use serde::Serialize; use serde_wasm_bindgen::{self, from_value}; +use std::cell::RefCell; use std::sync::atomic::Ordering; use wasm_bindgen::prelude::*; @@ -53,6 +54,50 @@ pub struct JsEditorHandle { frontend_message_handler_callback: js_sys::Function, } +fn window() -> web_sys::Window { + web_sys::window().expect("no global `window` exists") +} + +fn request_animation_frame(f: &Closure) { + window().request_animation_frame(f.as_ref().unchecked_ref()).expect("should register `requestAnimationFrame` OK"); +} + +// Sends a message to the dispatcher in the Editor Backend +fn poll_node_graph_evaluation() { + // Process no further messages after a crash to avoid spamming the console + if EDITOR_HAS_CRASHED.load(Ordering::SeqCst) { + return; + } + editor::node_graph_executor::run_node_graph(); + + // Get the editor instances, dispatch the message, and store the `FrontendMessage` queue response + EDITOR_INSTANCES.with(|instances| { + JS_EDITOR_HANDLES.with(|handles| { + // Mutably borrow the editors, and if successful, we can access them in the closure + instances.try_borrow_mut().map(|mut editors| { + // Get the editor instance for this editor ID, then dispatch the message to the backend, and return its response `FrontendMessage` queue + for (id, editor) in editors.iter_mut() { + let handles = handles.borrow_mut(); + let handle = handles.get(id).unwrap(); + let mut messages = VecDeque::new(); + editor.poll_node_graph_evaluation(&mut messages); + // Send each `FrontendMessage` to the JavaScript frontend + + let mut responses = Vec::new(); + for message in messages.into_iter() { + responses.extend(editor.handle_message(message)); + } + + for response in responses.into_iter() { + handle.send_frontend_message_to_js(response); + } + // If the editor cannot be borrowed then it has encountered a panic - we should just ignore new dispatches + } + }) + }) + }); +} + #[wasm_bindgen] #[allow(clippy::too_many_arguments)] impl JsEditorHandle { @@ -168,6 +213,18 @@ impl JsEditorHandle { self.dispatch(GlobalsMessage::SetPlatform { platform }); self.dispatch(Message::Init); + + let f = std::rc::Rc::new(RefCell::new(None)); + let g = f.clone(); + + *g.borrow_mut() = Some(Closure::new(move || { + poll_node_graph_evaluation(); + + // Schedule ourself for another requestAnimationFrame callback. + request_animation_frame(f.borrow().as_ref().unwrap()); + })); + + request_animation_frame(g.borrow().as_ref().unwrap()); } #[wasm_bindgen(js_name = tauriResponse)] diff --git a/node-graph/graph-craft/src/document/value.rs b/node-graph/graph-craft/src/document/value.rs index bb85abc8..eaee966a 100644 --- a/node-graph/graph-craft/src/document/value.rs +++ b/node-graph/graph-craft/src/document/value.rs @@ -247,6 +247,59 @@ impl<'a> TaggedValue { TaggedValue::Optional2IVec2(_) => concrete!(Option<[glam::IVec2; 2]>), } } + + pub fn try_from_any(input: Box + 'a>) -> Option { + use dyn_any::downcast; + use std::any::TypeId; + + match DynAny::type_id(input.as_ref()) { + x if x == TypeId::of::<()>() => Some(TaggedValue::None), + x if x == TypeId::of::() => Some(TaggedValue::String(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::U32(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::F32(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::F64(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Bool(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::DVec2(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::OptionalDVec2(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::Image(*downcast(input).unwrap())), + x if x == TypeId::of::>>>() => Some(TaggedValue::RcImage(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::ImageFrame(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Color(*downcast(input).unwrap())), + x if x == TypeId::of::>>() => Some(TaggedValue::Subpaths(*downcast(input).unwrap())), + x if x == TypeId::of::>>() => Some(TaggedValue::RcSubpath(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::BlendMode(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::ImaginateSamplingMethod(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::ImaginateMaskStartingFill(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::ImaginateStatus(*downcast(input).unwrap())), + x if x == TypeId::of::>>() => Some(TaggedValue::LayerPath(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::DAffine2(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::LuminanceCalculation(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::VectorData(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Fill(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Stroke(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::VecF32(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::RedGreenBlue(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::RelativeAbsolute(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::SelectiveColorChoice(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::LineCap(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::LineJoin(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::FillType(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::GradientType(*downcast(input).unwrap())), + x if x == TypeId::of::)>>() => Some(TaggedValue::GradientPositions(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Quantization(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::OptionalColor(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::ManipulatorGroupIds(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Font(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::VecDVec2(*downcast(input).unwrap())), + x if x == TypeId::of::>>>() => Some(TaggedValue::Segments(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::EditorApi(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::DocumentNode(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::GraphicGroup(*downcast(input).unwrap())), + x if x == TypeId::of::() => Some(TaggedValue::Artboard(*downcast(input).unwrap())), + x if x == TypeId::of::>() => Some(TaggedValue::Optional2IVec2(*downcast(input).unwrap())), + _ => None, + } + } } pub struct UpcastNode { diff --git a/node-graph/gstd/src/memo.rs b/node-graph/gstd/src/memo.rs index 2633be34..e7458f2d 100644 --- a/node-graph/gstd/src/memo.rs +++ b/node-graph/gstd/src/memo.rs @@ -64,7 +64,7 @@ impl<'i, T: 'static + Clone> Node<'i, T> for MonitorNode { fn serialize(&self) -> Option> { let output = self.output.lock().unwrap(); - (*output).as_ref().and_then(|output| Some(output.clone() as Arc)) + (*output).as_ref().map(|output| output.clone() as Arc) } }