From 9b8935d2015698a51a7633ab5f83be85ab5060aa Mon Sep 17 00:00:00 2001
From: Dennis Kobert <dennis@kobert.dev>
Date: Thu, 7 Aug 2025 00:48:25 +0200
Subject: [PATCH] Add iai versions of the runtime benchmarks (#3016)

* Include graph runtime benchmarks in ci regression run

* Update benchmarking workflow

* Remove ci script changes
---
 Cargo.lock                                    |  1 +
 node-graph/interpreted-executor/Cargo.toml    | 13 +++++++++
 .../benches/run_cached_iai.rs                 | 27 +++++++++++++++++++
 .../benches/run_once_iai.rs                   | 21 +++++++++++++++
 .../benches/update_executor_iai.rs            | 23 ++++++++++++++++
 5 files changed, 85 insertions(+)
 create mode 100644 node-graph/interpreted-executor/benches/run_cached_iai.rs
 create mode 100644 node-graph/interpreted-executor/benches/run_once_iai.rs
 create mode 100644 node-graph/interpreted-executor/benches/update_executor_iai.rs

diff --git a/Cargo.lock b/Cargo.lock
index d5b04572..b9a131eb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2693,6 +2693,7 @@ dependencies = [
  "graphene-core",
  "graphene-path-bool",
  "graphene-std",
+ "iai-callgrind",
  "log",
  "once_cell",
  "serde",
diff --git a/node-graph/interpreted-executor/Cargo.toml b/node-graph/interpreted-executor/Cargo.toml
index 119b8aff..1a6185c8 100644
--- a/node-graph/interpreted-executor/Cargo.toml
+++ b/node-graph/interpreted-executor/Cargo.toml
@@ -28,6 +28,7 @@ serde = { workspace = true }
 # Workspace dependencies
 graph-craft = { workspace = true, features = ["loading"] }
 criterion = { workspace = true }
+iai-callgrind = { workspace = true }
 
 # Benchmarks
 [[bench]]
@@ -42,3 +43,15 @@ harness = false
 name = "run_cached"
 harness = false
 
+[[bench]]
+name = "update_executor_iai"
+harness = false
+
+[[bench]]
+name = "run_once_iai"
+harness = false
+
+[[bench]]
+name = "run_cached_iai"
+harness = false
+
diff --git a/node-graph/interpreted-executor/benches/run_cached_iai.rs b/node-graph/interpreted-executor/benches/run_cached_iai.rs
new file mode 100644
index 00000000..14aec4ae
--- /dev/null
+++ b/node-graph/interpreted-executor/benches/run_cached_iai.rs
@@ -0,0 +1,27 @@
+use graph_craft::util::*;
+use graphene_std::Context;
+use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+fn setup_run_cached(name: &str) -> DynamicExecutor {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let executor = futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap();
+
+	// Warm up the cache by running once
+	let context: Context = None;
+	let _ = futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), context.clone()));
+
+	executor
+}
+
+#[library_benchmark]
+#[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "parametric-dunescape", "red-dress", "valley-of-spires"], setup = setup_run_cached)]
+pub fn run_cached(executor: DynamicExecutor) {
+	let context: Context = None;
+	black_box(futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), black_box(context))).unwrap());
+}
+
+library_benchmark_group!(name = run_cached_group; benchmarks = run_cached);
+
+main!(library_benchmark_groups = run_cached_group);
diff --git a/node-graph/interpreted-executor/benches/run_once_iai.rs b/node-graph/interpreted-executor/benches/run_once_iai.rs
new file mode 100644
index 00000000..318c5338
--- /dev/null
+++ b/node-graph/interpreted-executor/benches/run_once_iai.rs
@@ -0,0 +1,21 @@
+use graph_craft::util::*;
+use graphene_std::Context;
+use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+fn setup_run_once(name: &str) -> DynamicExecutor {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap()
+}
+
+#[library_benchmark]
+#[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "parametric-dunescape", "red-dress", "valley-of-spires"], setup = setup_run_once)]
+pub fn run_once(executor: DynamicExecutor) {
+	let context: Context = None;
+	black_box(futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), black_box(context))).unwrap());
+}
+
+library_benchmark_group!(name = run_once_group; benchmarks = run_once);
+
+main!(library_benchmark_groups = run_once_group);
diff --git a/node-graph/interpreted-executor/benches/update_executor_iai.rs b/node-graph/interpreted-executor/benches/update_executor_iai.rs
new file mode 100644
index 00000000..949366a4
--- /dev/null
+++ b/node-graph/interpreted-executor/benches/update_executor_iai.rs
@@ -0,0 +1,23 @@
+use graph_craft::proto::ProtoNetwork;
+use graph_craft::util::*;
+use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+fn setup_update_executor(name: &str) -> (DynamicExecutor, ProtoNetwork) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let empty = ProtoNetwork::default();
+	let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+	(executor, proto_network)
+}
+
+#[library_benchmark]
+#[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "parametric-dunescape", "red-dress", "valley-of-spires"], setup = setup_update_executor)]
+pub fn update_executor(setup: (DynamicExecutor, ProtoNetwork)) {
+	let (mut executor, network) = setup;
+	let _ = black_box(futures::executor::block_on(executor.update(black_box(network))));
+}
+
+library_benchmark_group!(name = update_group; benchmarks = update_executor);
+
+main!(library_benchmark_groups = update_group);