From 0a775fe9be8ab41f929e3928f53785186e854133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20Ic=C4=83?= Date: Wed, 15 Mar 2023 12:48:13 +0200 Subject: [PATCH] Take the transform of the ImageFrame into account when blending (#1072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Take the transform of the ImageFrame into account when blending The implementation computes the axis-aligned bounding box after we transform the corners of the source image, and then iterates through that box and computes the inverse of the affine transform of the source image. The samples are taken based on the u/v coordinates, so that the differences in size/aspect ratio between the images don't matter. This makes for a much simpler implementation, and gives us the flexibility to add different filtering methods in the future, for example. Signed-off-by: Ică Alexandru-Gabriel * Name the parameters for the blend node properly This avoids confusion between which one of the images is the `source` image and which one is the `destination`. Signed-off-by: Ică Alexandru-Gabriel * Remove rendundant computation for u/v coordinates Signed-off-by: Ică Alexandru-Gabriel * Rewrite the sampling/clamping logic * Add image frame transform node * Move transform node to transform module * Fix a few issues with our transformation logic * Fix math + do cleanup --------- Signed-off-by: Ică Alexandru-Gabriel Co-authored-by: Dennis Kobert --- .../document_node_types.rs | 2 +- node-graph/gcore/src/lib.rs | 2 + node-graph/gcore/src/raster.rs | 12 +++ node-graph/gcore/src/transform.rs | 53 ++++++++++++ node-graph/gcore/src/vector/vector_nodes.rs | 16 ---- node-graph/gstd/src/raster.rs | 84 +++++++++++++++++-- .../interpreted-executor/src/node_registry.rs | 3 +- 7 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 node-graph/gcore/src/transform.rs diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs index 68afd74f..d627f279 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs @@ -652,7 +652,7 @@ fn static_nodes() -> Vec { DocumentNodeType { name: "Transform", category: "Vector", - identifier: NodeImplementation::proto("graphene_core::vector::TransformNode<_, _, _, _>"), + identifier: NodeImplementation::proto("graphene_core::transform::TransformNode<_, _, _, _>"), inputs: vec![ DocumentInputType::value("Vector Data", TaggedValue::VectorData(graphene_core::vector::VectorData::empty()), true), DocumentInputType::value("Translation", TaggedValue::DVec2(DVec2::ZERO), false), diff --git a/node-graph/gcore/src/lib.rs b/node-graph/gcore/src/lib.rs index 49e9d84d..902556ba 100644 --- a/node-graph/gcore/src/lib.rs +++ b/node-graph/gcore/src/lib.rs @@ -19,6 +19,8 @@ pub mod value; pub mod gpu; pub mod raster; +#[cfg(feature = "alloc")] +pub mod transform; #[cfg(feature = "alloc")] pub mod vector; diff --git a/node-graph/gcore/src/raster.rs b/node-graph/gcore/src/raster.rs index 697e7830..456bd7cd 100644 --- a/node-graph/gcore/src/raster.rs +++ b/node-graph/gcore/src/raster.rs @@ -330,6 +330,7 @@ mod image { use core::hash::{Hash, Hasher}; use dyn_any::{DynAny, StaticType}; use glam::DAffine2; + use glam::DVec2; #[derive(Clone, Debug, PartialEq, DynAny, Default, specta::Type, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] @@ -425,6 +426,17 @@ mod image { transform: DAffine2::ZERO, } } + + pub fn get_mut(&mut self, x: usize, y: usize) -> &mut Color { + &mut self.image.data[y * (self.image.width as usize) + x] + } + + pub fn sample(&self, x: f64, y: f64) -> Color { + let x = x.clamp(0.0, self.image.width as f64 - 1.0) as usize; + let y = y.clamp(0.0, self.image.height as f64 - 1.0) as usize; + + self.image.data[y * (self.image.width as usize) + x] + } } impl Hash for ImageFrame { diff --git a/node-graph/gcore/src/transform.rs b/node-graph/gcore/src/transform.rs new file mode 100644 index 00000000..6602f5d7 --- /dev/null +++ b/node-graph/gcore/src/transform.rs @@ -0,0 +1,53 @@ +use glam::DAffine2; + +use glam::DVec2; + +use crate::raster::ImageFrame; +use crate::vector::VectorData; +use crate::Node; + +#[derive(Debug, Clone, Copy)] +pub struct TransformNode { + pub(crate) translate: Translation, + pub(crate) rotate: Rotation, + pub(crate) scale: Scale, + pub(crate) shear: Shear, +} + +#[node_macro::node_fn(TransformNode)] +pub(crate) fn transform_vector_data(mut vector_data: VectorData, translate: DVec2, rotate: f64, scale: DVec2, shear: DVec2) -> VectorData { + let transform = generate_transform(shear, &vector_data.transform, scale, rotate, translate); + vector_data.transform = transform * vector_data.transform; + vector_data +} + +impl<'input, Translation: 'input, Rotation: 'input, Scale: 'input, Shear: 'input> Node<'input, ImageFrame> for TransformNode +where + Translation: for<'any_input> Node<'any_input, (), Output = DVec2>, + Rotation: for<'any_input> Node<'any_input, (), Output = f64>, + Scale: for<'any_input> Node<'any_input, (), Output = DVec2>, + Shear: for<'any_input> Node<'any_input, (), Output = DVec2>, +{ + type Output = ImageFrame; + #[inline] + fn eval<'node: 'input>(&'node self, mut image_frame: ImageFrame) -> Self::Output { + let translate = self.translate.eval(()); + let rotate = self.rotate.eval(()); + let scale = self.scale.eval(()); + let shear = self.shear.eval(()); + + let transform = generate_transform(shear, &image_frame.transform, scale, rotate, translate); + image_frame.transform = transform * image_frame.transform; + image_frame + } +} + +// Generates a transform matrix that rotates around the center of the image +fn generate_transform(shear: DVec2, transform: &DAffine2, scale: DVec2, rotate: f64, translate: DVec2) -> DAffine2 { + let shear_matrix = DAffine2::from_cols_array(&[1., shear.y, shear.x, 1., 0., 0.]); + let pivot = transform.transform_point2(DVec2::splat(0.5)); + let translate_to_center = DAffine2::from_translation(-pivot); + + let transformation = translate_to_center.inverse() * DAffine2::from_scale_angle_translation(scale, rotate, translate) * shear_matrix * translate_to_center; + transformation +} diff --git a/node-graph/gcore/src/vector/vector_nodes.rs b/node-graph/gcore/src/vector/vector_nodes.rs index 30e2a983..774f665d 100644 --- a/node-graph/gcore/src/vector/vector_nodes.rs +++ b/node-graph/gcore/src/vector/vector_nodes.rs @@ -3,22 +3,6 @@ use super::VectorData; use crate::{Color, Node}; use glam::{DAffine2, DVec2}; -#[derive(Debug, Clone, Copy)] -pub struct TransformNode { - translate: Translation, - rotate: Rotation, - scale: Scale, - shear: Shear, -} - -#[node_macro::node_fn(TransformNode)] -fn transform_vector_data(mut vector_data: VectorData, translate: DVec2, rotate: f64, scale: DVec2, shear: DVec2) -> VectorData { - let (sin, cos) = rotate.sin_cos(); - - vector_data.transform = vector_data.transform * DAffine2::from_cols_array(&[scale.x + cos, shear.y + sin, shear.x - sin, scale.y + cos, translate.x, translate.y]); - vector_data -} - #[derive(Debug, Clone, Copy)] pub struct SetFillNode { fill_type: FillType, diff --git a/node-graph/gstd/src/raster.rs b/node-graph/gstd/src/raster.rs index d543a00d..1ef2c416 100644 --- a/node-graph/gstd/src/raster.rs +++ b/node-graph/gstd/src/raster.rs @@ -1,6 +1,6 @@ use dyn_any::{DynAny, StaticType}; -use glam::DAffine2; +use glam::{BVec2, DAffine2, DVec2}; use graphene_core::raster::{Color, Image, ImageFrame}; use graphene_core::Node; @@ -124,23 +124,91 @@ where image_frame } +#[derive(Debug, Clone)] +struct AxisAlignedBbox { + start: DVec2, + end: DVec2, +} + +#[derive(Debug, Clone)] +struct Bbox { + top_left: DVec2, + top_right: DVec2, + bottom_left: DVec2, + bottom_right: DVec2, +} + +impl Bbox { + fn axis_aligned_bbox(&self) -> AxisAlignedBbox { + let start_x = self.top_left.x.min(self.top_right.x).min(self.bottom_left.x).min(self.bottom_right.x); + let start_y = self.top_left.y.min(self.top_right.y).min(self.bottom_left.y).min(self.bottom_right.y); + let end_x = self.top_left.x.max(self.top_right.x).max(self.bottom_left.x).max(self.bottom_right.x); + let end_y = self.top_left.y.max(self.top_right.y).max(self.bottom_left.y).max(self.bottom_right.y); + + AxisAlignedBbox { + start: DVec2::new(start_x, start_y), + end: DVec2::new(end_x, end_y), + } + } +} + +fn compute_transformed_bounding_box(transform: DAffine2) -> Bbox { + let top_left = DVec2::new(0., 1.); + let top_right = DVec2::new(1., 1.); + let bottom_left = DVec2::new(0., 0.); + let bottom_right = DVec2::new(1., 0.); + let transform = |p| transform.transform_point2(p); + + Bbox { + top_left: transform(top_left), + top_right: transform(top_right), + bottom_left: transform(bottom_left), + bottom_right: transform(bottom_right), + } +} + #[derive(Debug, Clone, Copy)] -pub struct BlendImageNode { - second: Second, +pub struct BlendImageNode { + background: background, map_fn: MapFn, } // TODO: Implement proper blending #[node_macro::node_fn(BlendImageNode)] -fn blend_image(image: ImageFrame, second: ImageFrame, map_fn: &'any_input MapFn) -> ImageFrame +fn blend_image(foreground: ImageFrame, mut background: ImageFrame, map_fn: &'any_input MapFn) -> ImageFrame where MapFn: for<'any_input> Node<'any_input, (Color, Color), Output = Color> + 'input, { - let mut image = image; - for (pixel, sec_pixel) in &mut image.image.data.iter_mut().zip(second.image.data.iter()) { - *pixel = map_fn.eval((*pixel, *sec_pixel)); + let foreground_size = DVec2::new(foreground.image.width as f64, foreground.image.height as f64); + let background_size = DVec2::new(background.image.width as f64, background.image.height as f64); + + // Transforms a point from the background image to the forground image + let bg_to_fg = DAffine2::from_scale(foreground_size) * foreground.transform.inverse() * background.transform * DAffine2::from_scale(1. / background_size); + + // Footprint of the foreground image (0,0) (1, 1) in the background image space + let bg_aabb = compute_transformed_bounding_box(background.transform.inverse() * foreground.transform).axis_aligned_bbox(); + + // Clamp the foreground image to the background image + let start = (bg_aabb.start * background_size).max(DVec2::ZERO).as_uvec2(); + let end = (bg_aabb.end * background_size).min(background_size).as_uvec2(); + + for y in start.y..end.y { + for x in start.x..end.x { + let bg_point = DVec2::new(x as f64, y as f64); + let fg_point = bg_to_fg.transform_point2(bg_point); + if !((fg_point.cmpge(DVec2::ZERO) & fg_point.cmple(foreground_size)) == BVec2::new(true, true)) { + //log::debug!("Skipping pixel at {:?}", dest_point); + continue; + } + + let dst_pixel = background.get_mut(x as usize, y as usize); + let src_pixel = foreground.sample(fg_point.x, fg_point.y); + + *dst_pixel = map_fn.eval((src_pixel, *dst_pixel)); + } } - image + + background } #[derive(Debug, Clone, Copy)] diff --git a/node-graph/interpreted-executor/src/node_registry.rs b/node-graph/interpreted-executor/src/node_registry.rs index b51bf6e4..b3df0dbb 100644 --- a/node-graph/interpreted-executor/src/node_registry.rs +++ b/node-graph/interpreted-executor/src/node_registry.rs @@ -313,7 +313,8 @@ fn node_registry() -> HashMap, params: [QuantizationChannels]), raster_node!(graphene_core::quantization::DeQuantizeNode<_>, params: [QuantizationChannels]), register_node!(graphene_core::ops::CloneNode<_>, input: &QuantizationChannels, params: []), - register_node!(graphene_core::vector::TransformNode<_, _, _, _>, input: VectorData, params: [DVec2, f64, DVec2, DVec2]), + register_node!(graphene_core::transform::TransformNode<_, _, _, _>, input: VectorData, params: [DVec2, f64, DVec2, DVec2]), + register_node!(graphene_core::transform::TransformNode<_, _, _, _>, input: ImageFrame, params: [DVec2, f64, DVec2, DVec2]), register_node!(graphene_core::vector::SetFillNode<_, _, _, _, _, _, _>, input: VectorData, params: [ graphene_core::vector::style::FillType, graphene_core::Color, graphene_core::vector::style::GradientType, DVec2, DVec2, DAffine2, Vec<(f64, Option)>]), register_node!(graphene_core::vector::SetStrokeNode<_, _, _, _, _, _, _>, input: VectorData, params: [graphene_core::Color, f64, Vec, f64, graphene_core::vector::style::LineCap, graphene_core::vector::style::LineJoin, f64]), register_node!(graphene_core::vector::generator_nodes::UnitCircleGenerator, input: (), params: []),