Take the transform of the ImageFrame into account when blending (#1072)

* Take the transform of the ImageFrame into account when blending

The implementation computes the axis-aligned bounding box after we transform
the corners of the source image, and then iterates through that box
and computes the inverse of the affine transform of the source image.
The samples are taken based on the u/v coordinates, so that the differences
in size/aspect ratio between the images don't matter.

This makes for a much simpler implementation, and gives us the flexibility
to add different filtering methods in the future, for example.

Signed-off-by: Ică Alexandru-Gabriel <alexandru@seyhanlee.com>

* Name the parameters for the blend node properly

This avoids confusion between which one of the images is the `source`
image and which one is the `destination`.

Signed-off-by: Ică Alexandru-Gabriel <alexandru@seyhanlee.com>

* Remove rendundant computation for u/v coordinates

Signed-off-by: Ică Alexandru-Gabriel <alexandru@seyhanlee.com>

* Rewrite the sampling/clamping logic

* Add image frame transform node

* Move transform node to transform module

* Fix a few issues with our transformation logic

* Fix math + do cleanup

---------

Signed-off-by: Ică Alexandru-Gabriel <alexandru@seyhanlee.com>
Co-authored-by: Dennis Kobert <dennis@kobert.dev>

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs

@@ -652,7 +652,7 @@ fn static_nodes() -> Vec<DocumentNodeType> {
 		DocumentNodeType {
 			name: "Transform",
 			category: "Vector",
-			identifier: NodeImplementation::proto("graphene_core::vector::TransformNode<_, _, _, _>"),
+			identifier: NodeImplementation::proto("graphene_core::transform::TransformNode<_, _, _, _>"),
 			inputs: vec![
 				DocumentInputType::value("Vector Data", TaggedValue::VectorData(graphene_core::vector::VectorData::empty()), true),
 				DocumentInputType::value("Translation", TaggedValue::DVec2(DVec2::ZERO), false),

node-graph/gcore/src/lib.rs

@@ -19,6 +19,8 @@ pub mod value;
 pub mod gpu;
 
 pub mod raster;
+#[cfg(feature = "alloc")]
+pub mod transform;
 
 #[cfg(feature = "alloc")]
 pub mod vector;

node-graph/gcore/src/raster.rs

@@ -330,6 +330,7 @@ mod image {
 	use core::hash::{Hash, Hasher};
 	use dyn_any::{DynAny, StaticType};
 	use glam::DAffine2;
+	use glam::DVec2;
 
 	#[derive(Clone, Debug, PartialEq, DynAny, Default, specta::Type, Hash)]
 	#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
@@ -425,6 +426,17 @@ mod image {
 				transform: DAffine2::ZERO,
 			}
 		}
+
+		pub fn get_mut(&mut self, x: usize, y: usize) -> &mut Color {
+			&mut self.image.data[y * (self.image.width as usize) + x]
+		}
+
+		pub fn sample(&self, x: f64, y: f64) -> Color {
+			let x = x.clamp(0.0, self.image.width as f64 - 1.0) as usize;
+			let y = y.clamp(0.0, self.image.height as f64 - 1.0) as usize;
+
+			self.image.data[y * (self.image.width as usize) + x]
+		}
 	}
 
 	impl Hash for ImageFrame {

node-graph/gcore/src/transform.rs

@@ -0,0 +1,53 @@
+use glam::DAffine2;
+
+use glam::DVec2;
+
+use crate::raster::ImageFrame;
+use crate::vector::VectorData;
+use crate::Node;
+
+#[derive(Debug, Clone, Copy)]
+pub struct TransformNode<Translation, Rotation, Scale, Shear> {
+	pub(crate) translate: Translation,
+	pub(crate) rotate: Rotation,
+	pub(crate) scale: Scale,
+	pub(crate) shear: Shear,
+}
+
+#[node_macro::node_fn(TransformNode)]
+pub(crate) fn transform_vector_data(mut vector_data: VectorData, translate: DVec2, rotate: f64, scale: DVec2, shear: DVec2) -> VectorData {
+	let transform = generate_transform(shear, &vector_data.transform, scale, rotate, translate);
+	vector_data.transform = transform * vector_data.transform;
+	vector_data
+}
+
+impl<'input, Translation: 'input, Rotation: 'input, Scale: 'input, Shear: 'input> Node<'input, ImageFrame> for TransformNode<Translation, Rotation, Scale, Shear>
+where
+	Translation: for<'any_input> Node<'any_input, (), Output = DVec2>,
+	Rotation: for<'any_input> Node<'any_input, (), Output = f64>,
+	Scale: for<'any_input> Node<'any_input, (), Output = DVec2>,
+	Shear: for<'any_input> Node<'any_input, (), Output = DVec2>,
+{
+	type Output = ImageFrame;
+	#[inline]
+	fn eval<'node: 'input>(&'node self, mut image_frame: ImageFrame) -> Self::Output {
+		let translate = self.translate.eval(());
+		let rotate = self.rotate.eval(());
+		let scale = self.scale.eval(());
+		let shear = self.shear.eval(());
+
+		let transform = generate_transform(shear, &image_frame.transform, scale, rotate, translate);
+		image_frame.transform = transform * image_frame.transform;
+		image_frame
+	}
+}
+
+// Generates a transform matrix that rotates around the center of the image
+fn generate_transform(shear: DVec2, transform: &DAffine2, scale: DVec2, rotate: f64, translate: DVec2) -> DAffine2 {
+	let shear_matrix = DAffine2::from_cols_array(&[1., shear.y, shear.x, 1., 0., 0.]);
+	let pivot = transform.transform_point2(DVec2::splat(0.5));
+	let translate_to_center = DAffine2::from_translation(-pivot);
+
+	let transformation = translate_to_center.inverse() * DAffine2::from_scale_angle_translation(scale, rotate, translate) * shear_matrix * translate_to_center;
+	transformation
+}

node-graph/gcore/src/vector/vector_nodes.rs

@@ -3,22 +3,6 @@ use super::VectorData;
 use crate::{Color, Node};
 use glam::{DAffine2, DVec2};
 
-#[derive(Debug, Clone, Copy)]
-pub struct TransformNode<Translation, Rotation, Scale, Shear> {
-	translate: Translation,
-	rotate: Rotation,
-	scale: Scale,
-	shear: Shear,
-}
-
-#[node_macro::node_fn(TransformNode)]
-fn transform_vector_data(mut vector_data: VectorData, translate: DVec2, rotate: f64, scale: DVec2, shear: DVec2) -> VectorData {
-	let (sin, cos) = rotate.sin_cos();
-
-	vector_data.transform = vector_data.transform * DAffine2::from_cols_array(&[scale.x + cos, shear.y + sin, shear.x - sin, scale.y + cos, translate.x, translate.y]);
-	vector_data
-}
-
 #[derive(Debug, Clone, Copy)]
 pub struct SetFillNode<FillType, SolidColor, GradientType, Start, End, Transform, Positions> {
 	fill_type: FillType,

node-graph/gstd/src/raster.rs

@@ -1,6 +1,6 @@
 use dyn_any::{DynAny, StaticType};
 
-use glam::DAffine2;
+use glam::{BVec2, DAffine2, DVec2};
 use graphene_core::raster::{Color, Image, ImageFrame};
 use graphene_core::Node;
 
@@ -124,23 +124,91 @@ where
 	image_frame
 }
 
+#[derive(Debug, Clone)]
+struct AxisAlignedBbox {
+	start: DVec2,
+	end: DVec2,
+}
+
+#[derive(Debug, Clone)]
+struct Bbox {
+	top_left: DVec2,
+	top_right: DVec2,
+	bottom_left: DVec2,
+	bottom_right: DVec2,
+}
+
+impl Bbox {
+	fn axis_aligned_bbox(&self) -> AxisAlignedBbox {
+		let start_x = self.top_left.x.min(self.top_right.x).min(self.bottom_left.x).min(self.bottom_right.x);
+		let start_y = self.top_left.y.min(self.top_right.y).min(self.bottom_left.y).min(self.bottom_right.y);
+		let end_x = self.top_left.x.max(self.top_right.x).max(self.bottom_left.x).max(self.bottom_right.x);
+		let end_y = self.top_left.y.max(self.top_right.y).max(self.bottom_left.y).max(self.bottom_right.y);
+
+		AxisAlignedBbox {
+			start: DVec2::new(start_x, start_y),
+			end: DVec2::new(end_x, end_y),
+		}
+	}
+}
+
+fn compute_transformed_bounding_box(transform: DAffine2) -> Bbox {
+	let top_left = DVec2::new(0., 1.);
+	let top_right = DVec2::new(1., 1.);
+	let bottom_left = DVec2::new(0., 0.);
+	let bottom_right = DVec2::new(1., 0.);
+	let transform = |p| transform.transform_point2(p);
+
+	Bbox {
+		top_left: transform(top_left),
+		top_right: transform(top_right),
+		bottom_left: transform(bottom_left),
+		bottom_right: transform(bottom_right),
+	}
+}
+
 #[derive(Debug, Clone, Copy)]
-pub struct BlendImageNode<Second, MapFn> {
-	second: Second,
+pub struct BlendImageNode<background, MapFn> {
+	background: background,
 	map_fn: MapFn,
 }
 
 // TODO: Implement proper blending
 #[node_macro::node_fn(BlendImageNode)]
-fn blend_image<MapFn>(image: ImageFrame, second: ImageFrame, map_fn: &'any_input MapFn) -> ImageFrame
+fn blend_image<MapFn>(foreground: ImageFrame, mut background: ImageFrame, map_fn: &'any_input MapFn) -> ImageFrame
 where
 	MapFn: for<'any_input> Node<'any_input, (Color, Color), Output = Color> + 'input,
 {
-	let mut image = image;
-	for (pixel, sec_pixel) in &mut image.image.data.iter_mut().zip(second.image.data.iter()) {
-		*pixel = map_fn.eval((*pixel, *sec_pixel));
+	let foreground_size = DVec2::new(foreground.image.width as f64, foreground.image.height as f64);
+	let background_size = DVec2::new(background.image.width as f64, background.image.height as f64);
+
+	// Transforms a point from the background image to the forground image
+	let bg_to_fg = DAffine2::from_scale(foreground_size) * foreground.transform.inverse() * background.transform * DAffine2::from_scale(1. / background_size);
+
+	// Footprint of the foreground image (0,0) (1, 1) in the background image space
+	let bg_aabb = compute_transformed_bounding_box(background.transform.inverse() * foreground.transform).axis_aligned_bbox();
+
+	// Clamp the foreground image to the background image
+	let start = (bg_aabb.start * background_size).max(DVec2::ZERO).as_uvec2();
+	let end = (bg_aabb.end * background_size).min(background_size).as_uvec2();
+
+	for y in start.y..end.y {
+		for x in start.x..end.x {
+			let bg_point = DVec2::new(x as f64, y as f64);
+			let fg_point = bg_to_fg.transform_point2(bg_point);
+			if !((fg_point.cmpge(DVec2::ZERO) & fg_point.cmple(foreground_size)) == BVec2::new(true, true)) {
+				//log::debug!("Skipping pixel at {:?}", dest_point);
+				continue;
+			}
+
+			let dst_pixel = background.get_mut(x as usize, y as usize);
+			let src_pixel = foreground.sample(fg_point.x, fg_point.y);
+
+			*dst_pixel = map_fn.eval((src_pixel, *dst_pixel));
+		}
 	}
-	image
+
+	background
 }
 
 #[derive(Debug, Clone, Copy)]

node-graph/interpreted-executor/src/node_registry.rs

@@ -313,7 +313,8 @@ fn node_registry() -> HashMap<NodeIdentifier, HashMap<NodeIOTypes, NodeConstruct
 		raster_node!(graphene_core::quantization::QuantizeNode<_>, params: [QuantizationChannels]),
 		raster_node!(graphene_core::quantization::DeQuantizeNode<_>, params: [QuantizationChannels]),
 		register_node!(graphene_core::ops::CloneNode<_>, input: &QuantizationChannels, params: []),
-		register_node!(graphene_core::vector::TransformNode<_, _, _, _>, input: VectorData, params: [DVec2, f64, DVec2, DVec2]),
+		register_node!(graphene_core::transform::TransformNode<_, _, _, _>, input: VectorData, params: [DVec2, f64, DVec2, DVec2]),
+		register_node!(graphene_core::transform::TransformNode<_, _, _, _>, input: ImageFrame, params: [DVec2, f64, DVec2, DVec2]),
 		register_node!(graphene_core::vector::SetFillNode<_, _, _, _, _, _, _>, input: VectorData, params: [ graphene_core::vector::style::FillType, graphene_core::Color, graphene_core::vector::style::GradientType, DVec2, DVec2, DAffine2, Vec<(f64, Option<graphene_core::Color>)>]),
 		register_node!(graphene_core::vector::SetStrokeNode<_, _, _, _, _, _, _>, input: VectorData, params: [graphene_core::Color, f64, Vec<f32>, f64, graphene_core::vector::style::LineCap, graphene_core::vector::style::LineJoin, f64]),
 		register_node!(graphene_core::vector::generator_nodes::UnitCircleGenerator, input: (), params: []),