From da13f21486ac9406680e692db72ddc19e365eaad Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Mon, 12 Aug 2024 12:07:36 +0200 Subject: [PATCH] Add profiling metrics for, and speed up, Graphene graph compilation (#1924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add compile time benchmark for the demo artwork * Implement node input replacement batching isometric-fountain time: [7.4307 ms 7.5072 ms 7.5974 ms] change: [-19.302% -18.136% -16.903%] (p = 0.00 < 0.05) Performance has improved. Found 10 outliers among 100 measurements (10.00%) 6 (6.00%) high mild 4 (4.00%) high severe painted-dreams time: [1.8108 ms 1.8223 ms 1.8350 ms] change: [-12.422% -11.524% -10.650%] (p = 0.00 < 0.05) Performance has improved. Found 13 outliers among 100 measurements (13.00%) 5 (5.00%) high mild 8 (8.00%) high severe procedural-string-lights time: [551.65 µs 560.58 µs 571.13 µs] change: [-5.7783% -2.5770% +1.3136%] (p = 0.20 > 0.05) No change in performance detected. Found 6 outliers among 100 measurements (6.00%) 1 (1.00%) high mild 5 (5.00%) high severe red-dress time: [9.7951 ms 9.9006 ms 10.016 ms] change: [-18.812% -17.558% -16.292%] (p = 0.00 < 0.05) Performance has improved. Found 5 outliers among 100 measurements (5.00%) 5 (5.00%) high mild valley-of-spires time: [4.7294 ms 4.7837 ms 4.8442 ms] change: [-16.889% -15.712% -14.615%] (p = 0.00 < 0.05) Performance has improved. Found 16 outliers among 100 measurements (16.00%) 11 (11.00%) high mild 5 (5.00%) high severe * Implement node dependency tracking isometric-fountain time: [6.5695 ms 6.6418 ms 6.7227 ms] change: [-13.426% -12.112% -10.681%] (p = 0.00 < 0.05) Performance has improved. Found 10 outliers among 100 measurements (10.00%) 5 (5.00%) high mild 5 (5.00%) high severe painted-dreams time: [1.7406 ms 1.7566 ms 1.7742 ms] change: [-4.7386% -3.5484% -2.3707%] (p = 0.00 < 0.05) Performance has improved. Found 11 outliers among 100 measurements (11.00%) 10 (10.00%) high mild 1 (1.00%) high severe procedural-string-lights time: [585.40 µs 590.98 µs 596.82 µs] change: [-3.7739% +0.7591% +4.6293%] (p = 0.76 > 0.05) No change in performance detected. Found 3 outliers among 100 measurements (3.00%) 3 (3.00%) high mild red-dress time: [8.4314 ms 8.5072 ms 8.5899 ms] change: [-26.797% -25.374% -23.958%] (p = 0.00 < 0.05) Performance has improved. Found 11 outliers among 100 measurements (11.00%) 5 (5.00%) high mild 6 (6.00%) high severe valley-of-spires time: [4.3700 ms 4.4118 ms 4.4579 ms] change: [-11.470% -10.198% -8.9826%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 7 (7.00%) high mild 5 (5.00%) high severe * Remove clone and use rustc hash for storing nodes isometric-fountain time: [5.9220 ms 5.9854 ms 6.0570 ms] change: [-11.974% -10.539% -9.2291%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 4 (4.00%) high mild 8 (8.00%) high severe painted-dreams time: [1.5337 ms 1.5470 ms 1.5618 ms] change: [-11.901% -10.907% -9.9075%] (p = 0.00 < 0.05) Performance has improved. Found 9 outliers among 100 measurements (9.00%) 5 (5.00%) high mild 4 (4.00%) high severe procedural-string-lights time: [496.44 µs 501.44 µs 506.33 µs] change: [-20.002% -15.633% -12.213%] (p = 0.00 < 0.05) Performance has improved. Found 4 outliers among 100 measurements (4.00%) 3 (3.00%) high mild 1 (1.00%) high severe red-dress time: [7.7037 ms 7.7871 ms 7.8774 ms] change: [-11.906% -10.576% -9.2560%] (p = 0.00 < 0.05) Performance has improved. Found 8 outliers among 100 measurements (8.00%) 7 (7.00%) high mild 1 (1.00%) high severe valley-of-spires time: [3.9182 ms 3.9501 ms 3.9851 ms] change: [-14.615% -13.075% -11.500%] (p = 0.00 < 0.05) Performance has improved. Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high severe * Fix test depending on stable node order * Simplify flattening * Remove unused dependant vec size safeguards * Improve topological sort and make assert debug only isometric-fountain time: [2.9515 ms 2.9971 ms 3.0459 ms] change: [-61.270% -60.533% -59.747%] (p = 0.00 < 0.05) Performance has improved. Found 11 outliers among 100 measurements (11.00%) 7 (7.00%) high mild 4 (4.00%) high severe Benchmarking painted-dreams: Warming up for 3.0000 s Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 8.1s, enable flat sampling, or reduce sample count to 50. painted-dreams time: [1.1134 ms 1.1322 ms 1.1494 ms] change: [-40.991% -37.660% -34.129%] (p = 0.00 < 0.05) Performance has improved. Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high severe procedural-string-lights time: [391.88 µs 397.80 µs 403.84 µs] change: [-19.953% -17.762% -14.930%] (p = 0.00 < 0.05) Performance has improved. Found 6 outliers among 100 measurements (6.00%) 3 (3.00%) high mild 3 (3.00%) high severe red-dress time: [3.0722 ms 3.1179 ms 3.1707 ms] change: [-59.878% -59.168% -58.422%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 5 (5.00%) high mild 7 (7.00%) high severe valley-of-spires time: [2.0234 ms 2.0470 ms 2.0737 ms] change: [-48.994% -47.910% -46.907%] (p = 0.00 < 0.05) Performance has improved. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe * Improve id remapping logic by reusing id mapping isometric-fountain time: [2.2464 ms 2.2841 ms 2.3260 ms] change: [-25.693% -23.790% -22.024%] (p = 0.00 < 0.05) Performance has improved. Found 6 outliers among 100 measurements (6.00%) 5 (5.00%) high mild 1 (1.00%) high severe Benchmarking painted-dreams: Warming up for 3.0000 s Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 7.3s, enable flat sampling, or reduce sample count to 50. painted-dreams time: [869.96 µs 886.99 µs 902.36 µs] change: [-26.928% -19.590% -8.0737%] (p = 0.00 < 0.05) Performance has improved. Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high severe procedural-string-lights time: [291.60 µs 296.60 µs 302.45 µs] change: [-28.175% -25.168% -22.078%] (p = 0.00 < 0.05) Performance has improved. Found 8 outliers among 100 measurements (8.00%) 2 (2.00%) high mild 6 (6.00%) high severe red-dress time: [2.7946 ms 2.8356 ms 2.8800 ms] change: [-10.991% -9.0546% -6.9757%] (p = 0.00 < 0.05) Performance has improved. Found 7 outliers among 100 measurements (7.00%) 7 (7.00%) high mild valley-of-spires time: [1.5583 ms 1.5801 ms 1.6039 ms] change: [-24.165% -22.811% -21.213%] (p = 0.00 < 0.05) Performance has improved. Found 7 outliers among 100 measurements (7.00%) 7 (7.00%) high mild * Remove flame graph file --- .gitignore | 3 + Cargo.lock | 300 +++++++++++- Cargo.toml | 10 +- .../node_graph/document_node_types.rs | 2 +- .../utility_types/network_interface.rs | 4 +- .../src/vector/vector_data/modification.rs | 18 +- node-graph/graph-craft/.cargo/config.toml | 3 + node-graph/graph-craft/Cargo.toml | 17 + .../graph-craft/benches/compile_demo_art.rs | 27 ++ node-graph/graph-craft/src/document.rs | 427 +++++++++--------- .../graph-craft/src/graphene_compiler.rs | 23 +- node-graph/graph-craft/src/proto.rs | 153 ++++--- node-graph/gstd/src/gpu_nodes.rs | 12 +- shell.nix | 4 + 14 files changed, 699 insertions(+), 304 deletions(-) create mode 100644 node-graph/graph-craft/.cargo/config.toml create mode 100644 node-graph/graph-craft/benches/compile_demo_art.rs diff --git a/.gitignore b/.gitignore index 42f08a0e..9f898935 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ target/ *.spv *.exrc +perf.data* +profile.json +flamegraph.svg diff --git a/Cargo.lock b/Cargo.lock index a6eed992..b00932b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -456,6 +456,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -844,6 +855,12 @@ dependencies = [ "toml 0.7.8", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.1.6" @@ -923,6 +940,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags 1.3.2", + "textwrap", + "unicode-width", +] + [[package]] name = "clipboard-win" version = "5.4.0" @@ -1143,6 +1171,15 @@ dependencies = [ "libm", ] +[[package]] +name = "cpp_demangle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -1161,6 +1198,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools 0.10.5", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -1238,6 +1311,27 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa 1.0.11", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.2.8" @@ -1306,6 +1400,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1657,6 +1760,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -2295,16 +2410,21 @@ dependencies = [ "base64 0.22.1", "bezier-rs", "bytemuck", + "criterion", "dyn-any", "futures", "glam", + "glob", + "graph-craft", "graphene-core", "js-sys", "log", "num-traits", + "pprof", "reqwest 0.12.5", "rustc-hash 2.0.0", "serde", + "serde_json", "specta", "tokio", "url", @@ -2350,7 +2470,7 @@ dependencies = [ "bytemuck", "dyn-any", "glam", - "half", + "half 2.4.1", "image 0.25.2", "js-sys", "kurbo", @@ -2604,6 +2724,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" + [[package]] name = "half" version = "2.4.1" @@ -2668,6 +2794,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -3017,7 +3152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96cd73af13ae2e7220a1c02fe7d6bb53be50612ba7fabbb5c88e7753645f1f3c" dependencies = [ "image 0.25.2", - "itertools", + "itertools 0.12.1", "rayon", "thiserror", ] @@ -3059,6 +3194,24 @@ dependencies = [ "cfb", ] +[[package]] +name = "inferno" +version = "0.11.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" +dependencies = [ + "ahash", + "indexmap 2.2.6", + "is-terminal", + "itoa 1.0.11", + "log", + "num-format", + "once_cell", + "quick-xml 0.26.0", + "rgb", + "str_stack", +] + [[package]] name = "instant" version = "0.1.13" @@ -3110,6 +3263,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -3817,6 +3979,16 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa 1.0.11", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -4088,6 +4260,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "open" version = "3.2.0" @@ -4515,6 +4693,34 @@ dependencies = [ "time", ] +[[package]] +name = "plotters" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" + +[[package]] +name = "plotters-svg" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +dependencies = [ + "plotters-backend", +] + [[package]] name = "png" version = "0.17.13" @@ -4549,6 +4755,27 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" +dependencies = [ + "backtrace", + "cfg-if", + "findshlibs", + "inferno", + "libc", + "log", + "nix 0.26.4", + "once_cell", + "parking_lot", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4631,6 +4858,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58" +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.31.0" @@ -5401,6 +5637,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half 1.8.3", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.204" @@ -5858,6 +6104,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "strict-num" version = "0.1.1" @@ -5921,6 +6173,29 @@ dependencies = [ "siphasher 1.0.1", ] +[[package]] +name = "symbolic-common" +version = "12.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16629323a4ec5268ad23a575110a724ad4544aae623451de600c747bf87b36cf" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c043a45f08f41187414592b3ceb53fb0687da57209cc77401767fb69d5b596" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -6358,6 +6633,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thin-slice" version = "0.1.1" @@ -6462,6 +6746,16 @@ dependencies = [ "strict-num", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -7419,7 +7713,7 @@ dependencies = [ "glam", "gpu-executor", "graphene-core", - "half", + "half 2.4.1", "log", "node-macro", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index bee98f51..5a24e70f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,13 +104,13 @@ graphite-editor = { opt-level = 1 } graphene-core = { opt-level = 1 } graphene-std = { opt-level = 1 } interpreted-executor = { opt-level = 1 } # This is a mitigation for https://github.com/rustwasm/wasm-pack/issues/981 which is needed because the node_registry function is too large -graphite-proc-macros = { opt-level = 3 } +graphite-proc-macros = { opt-level = 1 } autoquant = { opt-level = 3 } -image = { opt-level = 3 } +image = { opt-level = 2 } rustc-hash = { opt-level = 3 } -serde_derive = { opt-level = 3 } -specta-macros = { opt-level = 3 } -syn = { opt-level = 3 } +serde_derive = { opt-level = 1 } +specta-macros = { opt-level = 1 } +syn = { opt-level = 1 } [profile.release] lto = "thin" diff --git a/editor/src/messages/portfolio/document/node_graph/document_node_types.rs b/editor/src/messages/portfolio/document/node_graph/document_node_types.rs index 1e8d222a..343fb2f9 100644 --- a/editor/src/messages/portfolio/document/node_graph/document_node_types.rs +++ b/editor/src/messages/portfolio/document/node_graph/document_node_types.rs @@ -4406,7 +4406,7 @@ pub fn wrap_network_in_scope(mut network: NodeNetwork, editor_api: Arc Option { warn!("Failed to find width of {node_id:#?} in network_path {network_path:?} due to non-wasm arch"); - None + Some(0.) } #[cfg(target_arch = "wasm32")] @@ -1169,7 +1169,7 @@ impl NodeNetworkInterface { continue; }; nested_network.exports = old_network.exports; - nested_network.scope_injections = old_network.scope_injections; + nested_network.scope_injections = old_network.scope_injections.into_iter().collect(); let Some(nested_network_metadata) = network_metadata.nested_metadata_mut(&network_path) else { log::error!("Could not get nested network in from_old_network"); continue; diff --git a/node-graph/gcore/src/vector/vector_data/modification.rs b/node-graph/gcore/src/vector/vector_data/modification.rs index 5a819b76..fa2cf481 100644 --- a/node-graph/gcore/src/vector/vector_data/modification.rs +++ b/node-graph/gcore/src/vector/vector_data/modification.rs @@ -5,6 +5,7 @@ use crate::Node; use bezier_rs::BezierHandles; use dyn_any::{DynAny, StaticType}; +use core::hash::BuildHasher; use std::collections::{HashMap, HashSet}; /// Represents a procedural change to the [`PointDomain`] in [`VectorData`]. @@ -485,11 +486,12 @@ use serde::ser::SerializeSeq; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::fmt; use std::hash::Hash; -pub fn serialize_hashmap(hashmap: &HashMap, serializer: S) -> Result +pub fn serialize_hashmap(hashmap: &HashMap, serializer: S) -> Result where K: Serialize + Eq + Hash, V: Serialize, S: Serializer, + H: BuildHasher, { let mut seq = serializer.serialize_seq(Some(hashmap.len()))?; for (key, value) in hashmap { @@ -498,22 +500,24 @@ where seq.end() } -pub fn deserialize_hashmap<'de, K, V, D>(deserializer: D) -> Result, D::Error> +pub fn deserialize_hashmap<'de, K, V, D, H>(deserializer: D) -> Result, D::Error> where K: Deserialize<'de> + Eq + Hash, V: Deserialize<'de>, D: Deserializer<'de>, + H: BuildHasher + Default, { - struct HashMapVisitor { - marker: std::marker::PhantomData HashMap>, + struct HashMapVisitor { + marker: std::marker::PhantomData HashMap>, } - impl<'de, K, V> Visitor<'de> for HashMapVisitor + impl<'de, K, V, H> Visitor<'de> for HashMapVisitor where K: Deserialize<'de> + Eq + Hash, V: Deserialize<'de>, + H: BuildHasher + Default, { - type Value = HashMap; + type Value = HashMap; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a sequence of tuples") @@ -523,7 +527,7 @@ where where A: SeqAccess<'de>, { - let mut hashmap = HashMap::new(); + let mut hashmap = HashMap::default(); while let Some((key, value)) = seq.next_element()? { hashmap.insert(key, value); } diff --git a/node-graph/graph-craft/.cargo/config.toml b/node-graph/graph-craft/.cargo/config.toml new file mode 100644 index 00000000..a4ae00c0 --- /dev/null +++ b/node-graph/graph-craft/.cargo/config.toml @@ -0,0 +1,3 @@ +[profile.profiling] +inherits = "release" +debug = true diff --git a/node-graph/graph-craft/Cargo.toml b/node-graph/graph-craft/Cargo.toml index 3f3bf3da..a03d8536 100644 --- a/node-graph/graph-craft/Cargo.toml +++ b/node-graph/graph-craft/Cargo.toml @@ -49,3 +49,20 @@ wasm-bindgen-futures = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Workspace dependencies winit = { workspace = true } + + +[dev-dependencies] +criterion = { version = "0.3", features = ["html_reports"] } +glob = "0.3" +pprof = { version = "0.13", features = ["flamegraph"] } +serde_json = { workspace = true } +graph-craft = { workspace = true, features = ["serde"] } + + +[[bench]] +name = "compile_demo_art" +harness = false + +# [[bench]] +# name = "exec_demo_art" +# harness = false diff --git a/node-graph/graph-craft/benches/compile_demo_art.rs b/node-graph/graph-craft/benches/compile_demo_art.rs new file mode 100644 index 00000000..173fef26 --- /dev/null +++ b/node-graph/graph-craft/benches/compile_demo_art.rs @@ -0,0 +1,27 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use graph_craft::{document::NodeNetwork, graphene_compiler::Compiler, proto::ProtoNetwork}; + +pub fn compile_to_proto(c: &mut Criterion) { + let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); + for path in artworks { + let Ok(path) = path else { continue }; + let content = std::fs::read(&path).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + let name = path.file_stem().unwrap().to_str().unwrap(); + + c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); + } +} + +fn load_network(document_string: &str) -> NodeNetwork { + let document: serde_json::Value = serde_json::from_str(&document_string).expect("Failed to parse document"); + let network = serde_json::from_value::(document["network_interface"]["network"].clone()).expect("Failed to parse document"); + network +} +fn compile(network: NodeNetwork) -> ProtoNetwork { + let compiler = Compiler {}; + compiler.compile_single(network).unwrap() +} + +criterion_group!(benches, compile_to_proto); +criterion_main!(benches); diff --git a/node-graph/graph-craft/src/document.rs b/node-graph/graph-craft/src/document.rs index 67c6f7b9..208ffcc5 100644 --- a/node-graph/graph-craft/src/document.rs +++ b/node-graph/graph-craft/src/document.rs @@ -6,6 +6,7 @@ use glam::IVec2; use graphene_core::memo::MemoHashGuard; pub use graphene_core::uuid::generate_uuid; use graphene_core::{Cow, MemoHash, ProtoNodeIdentifier, Type}; +use rustc_hash::{FxHashMap, FxHashSet}; use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; @@ -222,11 +223,14 @@ pub struct Source { /// The path to this node and its inputs and outputs as of when [`NodeNetwork::generate_node_paths`] was called. #[derive(Clone, Debug, PartialEq, Eq, DynAny, Default)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[non_exhaustive] pub struct OriginalLocation { /// The original location to the document node - e.g. [grandparent_id, parent_id, node_id]. pub path: Option>, /// Each document input source maps to one proto node input (however one proto node input may come from several sources) pub inputs_source: HashMap, + /// List of nodes which depend on this node + pub dependants: Vec>, /// A list of flags indicating whether the input is exposed in the UI pub inputs_exposed: Vec, /// Skipping inputs is useful for the manual composition thing - whereby a hidden `Footprint` input is added as the first input. @@ -556,6 +560,13 @@ impl DocumentNodeImplementation { pub const fn proto(name: &'static str) -> Self { Self::ProtoNode(ProtoNodeIdentifier::new(name)) } + + pub fn output_count(&self) -> usize { + match self { + DocumentNodeImplementation::Network(network) => network.exports.len(), + _ => 1, + } + } } // TODO: Eventually remove this (probably starting late 2024) @@ -731,11 +742,11 @@ pub struct NodeNetwork { // pub import_types: Vec, /// The list of all nodes in this network. #[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")] - pub nodes: HashMap, + pub nodes: FxHashMap, /// A network may expose nodes as constants which can by used by other nodes using a `NodeInput::Scope(key)`. #[serde(default)] #[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")] - pub scope_injections: HashMap, + pub scope_injections: FxHashMap, } impl std::hash::Hash for NodeNetwork { @@ -772,49 +783,6 @@ impl NodeNetwork { } } - /// A graph with just an input node - // pub fn new_network() -> Self { - // Self { - // exports: vec![NodeInput::node(NodeId(0), 0)], - // nodes: [( - // NodeId(0), - // DocumentNode { - // name: "Input Frame".into(), - // manual_composition: Some(concrete!(u32)), - // implementation: DocumentNodeImplementation::ProtoNode("graphene_core::ops::IdentityNode".into()), - // metadata: DocumentNodeMetadata { position: (8, 4).into() }, - // ..Default::default() - // }, - // )] - // .into_iter() - // .collect(), - // ..Default::default() - // } - // } - - /// Appends a new node to the network after the output node and sets it as the new output - // pub fn push_node_to_document_network(&mut self, mut node: DocumentNode) -> NodeId { - // let id = NodeId(self.nodes.len().try_into().expect("Too many nodes in network")); - // // Set the correct position for the new node - // if node.metadata().position == IVec2::default() { - // if let Some(pos) = self.get_root_node().and_then(|root_node| self.nodes.get(&root_node.id)).map(|n| n.metadata().position) { - // node.metadata().position = pos + IVec2::new(8, 0); - // } - // } - // if !self.exports.is_empty() { - // let input = self.exports[0].clone(); - // if node.inputs.is_empty() { - // node.inputs.push(input); - // } else { - // node.inputs[0] = input; - // } - // } - // // Use node_graph.insert_node - // self.insert_node(id, node); - // self.exports = vec![NodeInput::node(id, 0)]; - // id - // } - /// Get the nested network given by the path of node ids pub fn nested_network(&self, nested_path: &[NodeId]) -> Option<&Self> { let mut network = Some(self); @@ -883,6 +851,7 @@ impl NodeNetwork { .into_iter() .map(|(id, mut node)| { node.inputs.iter_mut().for_each(|input| input.map_ids(f)); + node.original_location.dependants.iter_mut().for_each(|deps| deps.iter_mut().for_each(|id| *id = f(*id))); (f(id), node) }) .collect(); @@ -903,21 +872,49 @@ impl NodeNetwork { path: Some(new_path), inputs_exposed: node.inputs.iter().map(|input| input.is_exposed()).collect(), skip_inputs: if node.manual_composition.is_some() { 1 } else { 0 }, + dependants: (0..node.implementation.output_count()).map(|_| Vec::new()).collect(), ..Default::default() - } + }; } } } - /// Replace all references in any node of `old_input` with `new_input` - fn replace_node_inputs(&mut self, old_input: NodeInput, new_input: NodeInput) { - for node in self.nodes.values_mut() { - node.inputs.iter_mut().for_each(|input| { - if *input == old_input { - *input = new_input.clone(); + pub fn populate_dependants(&mut self) { + let mut dep_changes = Vec::new(); + for (node_id, node) in &mut self.nodes { + let len = node.original_location.dependants.len(); + node.original_location.dependants.extend(vec![vec![]; (node.implementation.output_count()).max(len) - len]); + for input in &node.inputs { + if let NodeInput::Node { node_id: dep_id, output_index, .. } = input { + dep_changes.push((*dep_id, *output_index, *node_id)); } - }); + } } + // println!("{:#?}", self.nodes.get(&NodeId(1))); + for (dep_id, output_index, node_id) in dep_changes { + let node = self.nodes.get_mut(&dep_id).expect("Encountered invalid node id"); + let len = node.original_location.dependants.len(); + node.original_location.dependants.extend(vec![vec![]; (output_index).max(len) - len]); + // println!("{node_id} {output_index} {}", node.implementation.output_count()); + node.original_location.dependants[output_index].push(node_id); + } + } + + /// Replace all references in any node of `old_input` with `new_input` + fn replace_node_inputs(&mut self, node_id: NodeId, old_input: (NodeId, usize), new_input: (NodeId, usize)) { + let Some(node) = self.nodes.get_mut(&node_id) else { return }; + node.inputs.iter_mut().for_each(|input| { + if let NodeInput::Node { + node_id: ref mut input_id, + ref mut output_index, + .. + } = input + { + if (*input_id, *output_index) == old_input { + (*input_id, *output_index) = new_input; + } + } + }); } /// Replace all references in any node of `old_output` with `new_output` @@ -1010,159 +1007,149 @@ impl NodeNetwork { return; } - // Skip nodes that are already value nodes - if node.implementation != DocumentNodeImplementation::ProtoNode("graphene_core::value::ClonedNode".into()) { - // Replace value exports with value nodes, added inside nested network - if let DocumentNodeImplementation::Network(nested_network) = &mut node.implementation { - for export in nested_network.exports.iter_mut() { - let previous_export = std::mem::replace(export, NodeInput::network(concrete!(()), 0)); - if let NodeInput::Value { tagged_value, exposed } = previous_export { - let value_node_id = gen_id(); - let merged_node_id = map_ids(id, value_node_id); - let mut original_location = node.original_location.clone(); - if let Some(path) = &mut original_location.path { - path.push(value_node_id); - } - nested_network.nodes.insert( - merged_node_id, - DocumentNode { - inputs: vec![NodeInput::Value { tagged_value, exposed }], - implementation: DocumentNodeImplementation::ProtoNode("graphene_core::value::ClonedNode".into()), - original_location, - ..Default::default() - }, - ); - *export = NodeInput::Node { - node_id: merged_node_id, - output_index: 0, - lambda: false, - }; - } else { - *export = previous_export; - } - } - } + let path = node.original_location.path.clone().unwrap_or_default(); - // Replace value inputs with value nodes, added to flattened network - for input in node.inputs.iter_mut() { - let previous_input = std::mem::replace(input, NodeInput::network(concrete!(()), 0)); - if let NodeInput::Value { tagged_value, exposed } = previous_input { - let value_node_id = gen_id(); - let merged_node_id = map_ids(id, value_node_id); - let mut original_location = node.original_location.clone(); - if let Some(path) = &mut original_location.path { - path.push(value_node_id); - } - self.nodes.insert( - merged_node_id, - DocumentNode { - inputs: vec![NodeInput::Value { tagged_value, exposed }], - implementation: DocumentNodeImplementation::ProtoNode("graphene_core::value::ClonedNode".into()), - original_location, - ..Default::default() - }, - ); - *input = NodeInput::Node { - node_id: merged_node_id, - output_index: 0, - lambda: false, - }; - } else { - *input = previous_input; + // Replace value inputs with dedicated value nodes + if node.implementation != DocumentNodeImplementation::ProtoNode("graphene_core::value::ClonedNode".into()) { + Self::replace_value_inputs_with_nodes(&mut node.inputs, &mut self.nodes, &path, gen_id, map_ids, id); + } + + let DocumentNodeImplementation::Network(mut inner_network) = node.implementation else { + // If the node is not a network, it is a primitive node and can be inserted into the network as is. + assert!(!self.nodes.contains_key(&id), "Trying to insert a node into the network caused an id conflict"); + + self.nodes.insert(id, node); + return; + }; + + // Replace value exports with value nodes, added inside nested network + Self::replace_value_inputs_with_nodes( + &mut inner_network.exports, + &mut inner_network.nodes, + &node.original_location.path.as_ref().unwrap_or(&vec![]), + gen_id, + map_ids, + id, + ); + + // Connect all network inputs to either the parent network nodes, or newly created value nodes for the parent node. + inner_network.map_ids(|inner_id| map_ids(id, inner_id)); + inner_network.populate_dependants(); + let new_nodes = inner_network.nodes.keys().cloned().collect::>(); + + for (key, value) in inner_network.scope_injections.into_iter() { + match self.scope_injections.entry(key) { + std::collections::hash_map::Entry::Occupied(o) => { + log::warn!("Found duplicate scope injection for key {}, ignoring", o.key()); + } + std::collections::hash_map::Entry::Vacant(v) => { + v.insert(value); } } } - if let DocumentNodeImplementation::Network(mut inner_network) = node.implementation { - // Connect all network inputs to either the parent network nodes, or newly created value nodes for the parent node. - inner_network.map_ids(|inner_id| map_ids(id, inner_id)); - let new_nodes = inner_network.nodes.keys().cloned().collect::>(); - - for (key, value) in inner_network.scope_injections.into_iter() { - match self.scope_injections.entry(key) { - std::collections::hash_map::Entry::Occupied(o) => { - log::warn!("Found duplicate scope injection for key {}, ignoring", o.key()); - } - std::collections::hash_map::Entry::Vacant(v) => { - v.insert(value); - } - } - } - - // Match the document node input and the inputs of the inner network - for (nested_node_id, mut nested_node) in inner_network.nodes.into_iter() { - for (nested_input_index, nested_input) in nested_node.clone().inputs.iter().enumerate() { - if let NodeInput::Network { import_index, .. } = nested_input { - let parent_input = node.inputs.get(*import_index).unwrap_or_else(|| panic!("Import index {} should always exist", import_index)); - match *parent_input { - // If the input to self is a node, connect the corresponding output of the inner network to it - NodeInput::Node { node_id, output_index, lambda } => { - let skip = node.original_location.skip_inputs; - nested_node.populate_first_network_input(node_id, output_index, nested_input_index, lambda, node.original_location.inputs(*import_index), skip); - } - NodeInput::Network { import_index, .. } => { - let parent_input_index = import_index; - let Some(NodeInput::Network { import_index, .. }) = nested_node.inputs.get_mut(nested_input_index) else { - log::error!("Nested node should have a network input"); - continue; - }; - *import_index = parent_input_index; - } - NodeInput::Value { .. } => unreachable!("Value inputs should have been replaced with value nodes"), - NodeInput::Inline(_) => (), - NodeInput::Scope(ref key) => { - let (import_id, _ty) = self.scope_injections.get(key.as_ref()).expect("Tried to import a non existent key from scope"); - // TODO use correct output index - nested_node.inputs[nested_input_index] = NodeInput::node(*import_id, 0); - } + // Match the document node input and the inputs of the inner network + for (nested_node_id, mut nested_node) in inner_network.nodes.into_iter() { + for (nested_input_index, nested_input) in nested_node.clone().inputs.iter().enumerate() { + if let NodeInput::Network { import_index, .. } = nested_input { + let parent_input = node.inputs.get(*import_index).unwrap_or_else(|| panic!("Import index {} should always exist", import_index)); + match *parent_input { + // If the input to self is a node, connect the corresponding output of the inner network to it + NodeInput::Node { node_id, output_index, lambda } => { + let skip = node.original_location.skip_inputs; + nested_node.populate_first_network_input(node_id, output_index, nested_input_index, lambda, node.original_location.inputs(*import_index), skip); + if let input_node = self.nodes.get_mut(&node_id).unwrap() { + input_node.original_location.dependants[output_index].push(nested_node_id); + }; + } + NodeInput::Network { import_index, .. } => { + let parent_input_index = import_index; + let Some(NodeInput::Network { import_index, .. }) = nested_node.inputs.get_mut(nested_input_index) else { + log::error!("Nested node should have a network input"); + continue; + }; + *import_index = parent_input_index; + } + NodeInput::Value { .. } => unreachable!("Value inputs should have been replaced with value nodes"), + NodeInput::Inline(_) => (), + NodeInput::Scope(ref key) => { + let (import_id, _ty) = self.scope_injections.get(key.as_ref()).expect("Tried to import a non existent key from scope"); + // TODO use correct output index + nested_node.inputs[nested_input_index] = NodeInput::node(*import_id, 0); } } } - self.nodes.insert(nested_node_id, nested_node); } - // TODO: Add support for flattening exports that are NodeInput::Network (https://github.com/GraphiteEditor/Graphite/issues/1762) - // Match the document node input and the exports of the inner network if the export is a NodeInput::Network - // for (i, export) in inner_network.exports.iter().enumerate() { - // if let NodeInput::Network { import_index, .. } = export { - // let parent_input = node.inputs.get(*import_index).expect(&format!("Import index {} should always exist", import_index)); - // match *parent_input { - // // If the input to self is a node, connect the corresponding output of the inner network to it - // NodeInput::Node { node_id, output_index, lambda } => { - // inner_network.populate_first_network_export(&mut node, node_id, output_index, lambda, i, node.original_location.outputs(i), 0); - // } - // NodeInput::Network { import_index, .. } => { - // let parent_input_index = import_index; - // let Some(NodeInput::Network { import_index, .. }) = inner_network.exports.get_mut(i) else { - // log::error!("Nested node should have a network input"); - // continue; - // }; - // *import_index = parent_input_index; - // } - // NodeInput::Value { .. } => unreachable!("Value inputs should have been replaced with value nodes"), - // NodeInput::Inline(_) => (), - // } - // } - // } + self.nodes.insert(nested_node_id, nested_node); + } + // TODO: Add support for flattening exports that are NodeInput::Network (https://github.com/GraphiteEditor/Graphite/issues/1762) - // Connect all nodes that were previously connected to this node to the nodes of the inner network - for (i, export) in inner_network.exports.into_iter().enumerate() { - let node_input = |node_id, output_index, lambda| NodeInput::Node { node_id, output_index, lambda }; - - if let NodeInput::Node { node_id, output_index, .. } = &export { - self.replace_node_inputs(node_input(id, i, false), node_input(*node_id, *output_index, false)); - self.replace_node_inputs(node_input(id, i, true), node_input(*node_id, *output_index, true)); + // Connect all nodes that were previously connected to this node to the nodes of the inner network + for (i, export) in inner_network.exports.into_iter().enumerate() { + if let NodeInput::Node { node_id, output_index, .. } = &export { + for deps in &node.original_location.dependants { + for dep in deps { + self.replace_node_inputs(*dep, (id, i), (*node_id, *output_index)); + } } - self.replace_network_outputs(NodeInput::node(id, i), export); + if let Some(new_output_node) = self.nodes.get_mut(node_id) { + for dep in &node.original_location.dependants[i] { + new_output_node.original_location.dependants[*output_index].push(*dep); + } + } } - for node_id in new_nodes { - self.flatten_with_fns(node_id, map_ids, gen_id); + self.replace_network_outputs(NodeInput::node(id, i), export); + } + + for node_id in new_nodes { + self.flatten_with_fns(node_id, map_ids, gen_id); + } + } + + #[inline(never)] + fn replace_value_inputs_with_nodes( + inputs: &mut [NodeInput], + collection: &mut FxHashMap, + path: &[NodeId], + gen_id: impl Fn() -> NodeId + Copy, + map_ids: impl Fn(NodeId, NodeId) -> NodeId + Copy, + id: NodeId, + ) { + // Replace value exports and imports with value nodes, added inside the nested network + for export in inputs { + let export: &mut NodeInput = export; + let previous_export = std::mem::replace(export, NodeInput::network(concrete!(()), 0)); + if let NodeInput::Value { tagged_value, exposed } = previous_export { + let value_node_id = gen_id(); + let merged_node_id = map_ids(id, value_node_id); + let mut original_location = OriginalLocation { + path: Some(path.to_vec()), + dependants: vec![vec![id]], + ..Default::default() + }; + + if let Some(path) = &mut original_location.path { + path.push(value_node_id); + } + collection.insert( + merged_node_id, + DocumentNode { + inputs: vec![NodeInput::Value { tagged_value, exposed }], + implementation: DocumentNodeImplementation::ProtoNode("graphene_core::value::ClonedNode".into()), + original_location, + ..Default::default() + }, + ); + *export = NodeInput::Node { + node_id: merged_node_id, + output_index: 0, + lambda: false, + }; + } else { + *export = previous_export; } - } else { - // If the node is not a network, it is a primitive node and can be inserted into the network as is. - assert!(!self.nodes.contains_key(&id), "Trying to insert a node into the network caused an id conflict"); - self.nodes.insert(id, node); } } @@ -1182,6 +1169,12 @@ impl NodeNetwork { assert_eq!(node.inputs.len(), 1, "Id node has more than one input"); if let NodeInput::Node { node_id, output_index, .. } = node.inputs[0] { let node_input_output_index = output_index; + // TODO fix + if let Some(input_node) = self.nodes.get_mut(&node_id) { + for &dep in &node.original_location.dependants[0] { + input_node.original_location.dependants[output_index].push(dep); + } + } let input_node_id = node_id; for output in self.nodes.values_mut() { @@ -1285,22 +1278,38 @@ impl NodeNetwork { pub fn into_proto_networks(self) -> impl Iterator { // let input_node = self.nodes.iter().find_map(|(node_id, node)| if node.name == "SetNode" { Some(node_id.clone()) } else { None }); let mut nodes: Vec<_> = self.nodes.into_iter().map(|(id, node)| (id, node.resolve_proto_node())).collect(); - nodes.sort_unstable_by_key(|(i, _)| *i); // Create a network to evaluate each output - self.exports.into_iter().filter_map(move |output| { - if let NodeInput::Node { node_id, .. } = output { - Some(ProtoNetwork { - inputs: Vec::new(), // Inputs field is not used. Should be deleted - // inputs: vec![input_node.expect("Set node should always exist")], - // inputs: self.imports.clone(), + if self.exports.len() == 1 { + if let NodeInput::Node { node_id, .. } = self.exports[0] { + return vec![ProtoNetwork { + inputs: Vec::new(), output: node_id, - nodes: nodes.clone(), - }) - } else { - None + nodes: nodes, + }] + .into_iter(); } - }) + } + + // Create a network to evaluate each output + let networks: Vec<_> = self + .exports + .into_iter() + .filter_map(move |output| { + if let NodeInput::Node { node_id, .. } = output { + Some(ProtoNetwork { + inputs: Vec::new(), // Inputs field is not used. Should be deleted + // inputs: vec![input_node.expect("Set node should always exist")], + // inputs: self.imports.clone(), + output: node_id, + nodes: nodes.clone(), + }) + } else { + None + } + }) + .collect(); + networks.into_iter() } /// Create a [`RecursiveNodeIter`] that iterates over all [`DocumentNode`]s, including ones that are deeply nested. @@ -1445,7 +1454,7 @@ mod test { .collect(), ..Default::default() }; - network.generate_node_paths(&[]); + network.populate_dependants(); network.flatten_with_fns(NodeId(1), |self_id, inner_id| NodeId(self_id.0 * 10 + inner_id.0), gen_node_id); let flat_network = flat_network(); println!("{flat_network:#?}"); @@ -1489,6 +1498,7 @@ mod test { inputs_source: [(Source { node: vec![NodeId(1)], index: 1 }, 1)].into(), inputs_exposed: vec![true, true], skip_inputs: 0, + ..Default::default() }, ..Default::default() @@ -1505,6 +1515,7 @@ mod test { inputs_source: HashMap::new(), inputs_exposed: vec![true], skip_inputs: 0, + ..Default::default() }, ..Default::default() }, @@ -1520,6 +1531,7 @@ mod test { inputs_source: HashMap::new(), inputs_exposed: vec![true, false], skip_inputs: 0, + ..Default::default() }, ..Default::default() }, @@ -1529,7 +1541,8 @@ mod test { .collect(), }; let network = flat_network(); - let resolved_network = network.into_proto_networks().collect::>(); + let mut resolved_network = network.into_proto_networks().collect::>(); + resolved_network[0].nodes.sort_unstable_by_key(|(id, _)| *id); println!("{:#?}", resolved_network[0]); println!("{construction_network:#?}"); @@ -1550,6 +1563,7 @@ mod test { inputs_source: [(Source { node: vec![NodeId(1)], index: 1 }, 1)].into(), inputs_exposed: vec![true, true], skip_inputs: 0, + ..Default::default() }, ..Default::default() }, @@ -1564,6 +1578,7 @@ mod test { inputs_source: HashMap::new(), inputs_exposed: vec![true, false], skip_inputs: 0, + ..Default::default() }, ..Default::default() }, @@ -1578,6 +1593,7 @@ mod test { inputs_source: HashMap::new(), inputs_exposed: vec![true], skip_inputs: 0, + ..Default::default() }, ..Default::default() }, @@ -1642,6 +1658,7 @@ mod test { ..Default::default() }; let _new_ids = 101..; + network.populate_dependants(); network.flatten_with_fns(NodeId(1), |self_id, inner_id| NodeId(self_id.0 * 10 + inner_id.0), || NodeId(10000)); network.flatten_with_fns(NodeId(2), |self_id, inner_id| NodeId(self_id.0 * 10 + inner_id.0), || NodeId(10001)); network.remove_dead_nodes(0); diff --git a/node-graph/graph-craft/src/graphene_compiler.rs b/node-graph/graph-craft/src/graphene_compiler.rs index 4a929b08..41a81820 100644 --- a/node-graph/graph-craft/src/graphene_compiler.rs +++ b/node-graph/graph-craft/src/graphene_compiler.rs @@ -6,32 +6,29 @@ use crate::proto::{LocalFuture, ProtoNetwork}; pub struct Compiler {} impl Compiler { - pub fn compile(&self, mut network: NodeNetwork) -> Result, String> { + pub fn compile(&self, mut network: NodeNetwork) -> impl Iterator> { let node_ids = network.nodes.keys().copied().collect::>(); + network.populate_dependants(); for id in node_ids { network.flatten(id); } network.resolve_scope_inputs(); network.remove_redundant_id_nodes(); - network.remove_dead_nodes(0); + // network.remove_dead_nodes(0); let proto_networks = network.into_proto_networks(); - let proto_networks_result: Vec = proto_networks - .map(move |mut proto_network| { - proto_network.resolve_inputs()?; - proto_network.generate_stable_node_ids(); - Ok(proto_network) - }) - .collect::, String>>()?; - - Ok(proto_networks_result.into_iter()) + proto_networks.map(move |mut proto_network| { + proto_network.resolve_inputs()?; + proto_network.generate_stable_node_ids(); + Ok(proto_network) + }) } pub fn compile_single(&self, network: NodeNetwork) -> Result { assert_eq!(network.exports.len(), 1, "Graph with multiple outputs not yet handled"); - let Some(proto_network) = self.compile(network)?.next() else { + let Some(proto_network) = self.compile(network).next() else { return Err("Failed to convert graph into proto graph".to_string()); }; - Ok(proto_network) + proto_network } } diff --git a/node-graph/graph-craft/src/proto.rs b/node-graph/graph-craft/src/proto.rs index c3c18732..faae28b9 100644 --- a/node-graph/graph-craft/src/proto.rs +++ b/node-graph/graph-craft/src/proto.rs @@ -3,6 +3,7 @@ use crate::document::{NodeId, OriginalLocation}; use dyn_any::DynAny; use graphene_core::*; +use rustc_hash::{FxHashMap, FxHashSet}; #[cfg(feature = "serde")] use std::borrow::Cow; @@ -341,9 +342,16 @@ impl ProtoNode { } } +#[derive(Clone, Copy, PartialEq)] +enum NodeState { + Unvisited, + Visiting, + Visited, +} + impl ProtoNetwork { fn check_ref(&self, ref_id: &NodeId, id: &NodeId) { - assert!( + debug_assert!( self.nodes.iter().any(|(check_id, _)| check_id == ref_id), "Node id:{id} has a reference which uses node id:{ref_id} which doesn't exist in network {self:#?}" ); @@ -398,6 +406,7 @@ impl ProtoNetwork { } } + // TODO: Remsove /// Create a hashmap with the list of nodes this proto network depends on/uses as inputs. pub fn collect_inwards_edges(&self) -> HashMap> { let mut edges: HashMap> = HashMap::new(); @@ -420,6 +429,35 @@ impl ProtoNetwork { edges } + fn collect_inwards_edges_with_mapping(&self) -> (Vec>, FxHashMap) { + let mut id_map = FxHashMap::with_capacity_and_hasher(self.nodes.len(), Default::default()); + + // Create dense mapping + id_map = self.nodes.iter().enumerate().map(|(idx, (id, _))| (*id, idx)).collect(); + + // Collect inwards edges using dense indices + let mut inwards_edges = vec![Vec::new(); self.nodes.len()]; + for (node_id, node) in &self.nodes { + let node_index = id_map[node_id]; + match &node.input { + ProtoNodeInput::Node(ref_id) | ProtoNodeInput::NodeLambda(ref_id) => { + self.check_ref(ref_id, &NodeId(node_index as u64)); + inwards_edges[node_index].push(id_map[ref_id]); + } + _ => {} + } + + if let ConstructionArgs::Nodes(ref_nodes) = &node.construction_args { + for (ref_id, _) in ref_nodes { + self.check_ref(ref_id, &NodeId(node_index as u64)); + inwards_edges[node_index].push(id_map[ref_id]); + } + } + } + + (inwards_edges, id_map) + } + /// Inserts a [`graphene_core::structural::ComposeNode`] for each node that has a [`ProtoNodeInput::Node`]. The compose node evaluates the first node, and then sends the result into the second node. pub fn resolve_inputs(&mut self) -> Result<(), String> { // Perform topological sort once @@ -487,35 +525,43 @@ impl ProtoNetwork { } }); } + // Based on https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search // This approach excludes nodes that are not connected - pub fn topological_sort(&self) -> Result, String> { - let mut sorted = Vec::new(); - let inwards_edges = self.collect_inwards_edges(); - fn visit(node_id: NodeId, temp_marks: &mut HashSet, sorted: &mut Vec, inwards_edges: &HashMap>, network: &ProtoNetwork) -> Result<(), String> { - if sorted.contains(&node_id) { - return Ok(()); - }; - if temp_marks.contains(&node_id) { - return Err(format!("Cycle detected {inwards_edges:#?}, {network:#?}")); - } + pub fn topological_sort(&self) -> Result<(Vec, FxHashMap), String> { + let (inwards_edges, id_map) = self.collect_inwards_edges_with_mapping(); + let mut sorted = Vec::with_capacity(self.nodes.len()); + let mut stack = vec![id_map[&self.output]]; + let mut state = vec![NodeState::Unvisited; self.nodes.len()]; - if let Some(dependencies) = inwards_edges.get(&node_id) { - temp_marks.insert(node_id); - for &dependant in dependencies { - visit(dependant, temp_marks, sorted, inwards_edges, network)?; + while let Some(&node_index) = stack.last() { + match state[node_index] { + NodeState::Unvisited => { + state[node_index] = NodeState::Visiting; + for &dep_index in inwards_edges[node_index].iter().rev() { + match state[dep_index] { + NodeState::Visiting => { + return Err(format!("Cycle detected involving node {}", self.nodes[dep_index].0)); + } + NodeState::Unvisited => { + stack.push(dep_index); + } + NodeState::Visited => {} + } + } + } + NodeState::Visiting => { + stack.pop(); + state[node_index] = NodeState::Visited; + sorted.push(NodeId(node_index as u64)); + } + NodeState::Visited => { + stack.pop(); } - temp_marks.remove(&node_id); } - sorted.push(node_id); - Ok(()) } - if !self.nodes.iter().any(|(id, _)| *id == self.output) { - return Err(format!("Output id {} does not exist", self.output)); - } - visit(self.output, &mut HashSet::new(), &mut sorted, &inwards_edges, self)?; - Ok(sorted) + Ok((sorted, id_map)) } fn is_topologically_sorted(&self) -> bool { @@ -536,57 +582,36 @@ impl ProtoNetwork { true } - /*// Based on https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm - pub fn topological_sort(&self) -> Vec { - let mut sorted = Vec::new(); - let outwards_edges = self.collect_outwards_edges(); - let mut inwards_edges = self.collect_inwards_edges(); - let mut no_incoming_edges: Vec<_> = self.nodes.iter().map(|entry| entry.0).filter(|id| !inwards_edges.contains_key(id)).collect(); - - assert_ne!(no_incoming_edges.len(), 0, "Acyclic graphs must have at least one node with no incoming edge"); - - while let Some(node_id) = no_incoming_edges.pop() { - sorted.push(node_id); - - if let Some(outwards_edges) = outwards_edges.get(&node_id) { - for &ref_id in outwards_edges { - let dependencies = inwards_edges.get_mut(&ref_id).unwrap(); - dependencies.retain(|&id| id != node_id); - if dependencies.is_empty() { - no_incoming_edges.push(ref_id) - } - } - } - } - sorted - }*/ - /// Sort the nodes vec so it is in a topological order. This ensures that no node takes an input from a node that is found later in the list. fn reorder_ids(&mut self) -> Result<(), String> { - let order = self.topological_sort()?; + let (order, id_map) = self.topological_sort()?; - // Map of node ids to their current index in the nodes vector - let current_positions: HashMap<_, _> = self.nodes.iter().enumerate().map(|(pos, (id, _))| (*id, pos)).collect(); + // // Map of node ids to their current index in the nodes vector + // let current_positions: FxHashMap<_, _> = self.nodes.iter().enumerate().map(|(pos, (id, _))| (*id, pos)).collect(); - // Map of node ids to their new index based on topological order - let new_positions: HashMap<_, _> = order.iter().enumerate().map(|(pos, id)| (*id, NodeId(pos as u64))).collect(); + // // Map of node ids to their new index based on topological order + let new_positions: FxHashMap<_, _> = order.iter().enumerate().map(|(pos, id)| (self.nodes[id.0 as usize].0, pos)).collect(); + // assert_eq!(id_map, current_positions); // Create a new nodes vector based on the topological order + let mut new_nodes = Vec::with_capacity(order.len()); for (index, &id) in order.iter().enumerate() { - let current_pos = *current_positions.get(&id).unwrap(); - new_nodes.push((NodeId(index as u64), self.nodes[current_pos].1.clone())); + let mut node = std::mem::take(&mut self.nodes[id.0 as usize].1); + // Update node references to reflect the new order + node.map_ids(|id| NodeId(*new_positions.get(&id).expect("node not found in lookup table") as u64), false); + new_nodes.push((NodeId(index as u64), node)); } // Update node references to reflect the new order - new_nodes.iter_mut().for_each(|(_, node)| { - node.map_ids(|id| *new_positions.get(&id).expect("node not found in lookup table"), false); - }); + // new_nodes.iter_mut().for_each(|(_, node)| { + // node.map_ids(|id| *new_positions.get(&id).expect("node not found in lookup table"), false); + // }); // Update the nodes vector and other references self.nodes = new_nodes; - self.inputs = self.inputs.iter().filter_map(|id| new_positions.get(id).copied()).collect(); - self.output = *new_positions.get(&self.output).unwrap(); + self.inputs = self.inputs.iter().filter_map(|id| new_positions.get(id).map(|x| NodeId(*x as u64))).collect(); + self.output = NodeId(*new_positions.get(&self.output).unwrap() as u64); assert_eq!(order.len(), self.nodes.len()); Ok(()) @@ -890,7 +915,8 @@ mod test { #[test] fn topological_sort() { let construction_network = test_network(); - let sorted = construction_network.topological_sort().expect("Error when calling 'topological_sort' on 'construction_network."); + let (sorted, _) = construction_network.topological_sort().expect("Error when calling 'topological_sort' on 'construction_network."); + let sorted: Vec<_> = sorted.iter().map(|x| construction_network.nodes[x.0 as usize].0).collect(); println!("{sorted:#?}"); assert_eq!(sorted, vec![NodeId(14), NodeId(10), NodeId(11), NodeId(1)]); } @@ -907,7 +933,8 @@ mod test { fn id_reordering() { let mut construction_network = test_network(); construction_network.reorder_ids().expect("Error when calling 'reorder_ids' on 'construction_network."); - let sorted = construction_network.topological_sort().expect("Error when calling 'topological_sort' on 'construction_network."); + let (sorted, _) = construction_network.topological_sort().expect("Error when calling 'topological_sort' on 'construction_network."); + let sorted: Vec<_> = sorted.iter().map(|x| construction_network.nodes[x.0 as usize].0).collect(); println!("nodes: {:#?}", construction_network.nodes); assert_eq!(sorted, vec![NodeId(0), NodeId(1), NodeId(2), NodeId(3)]); let ids: Vec<_> = construction_network.nodes.iter().map(|(id, _)| *id).collect(); @@ -922,7 +949,7 @@ mod test { let mut construction_network = test_network(); construction_network.reorder_ids().expect("Error when calling 'reorder_ids' on 'construction_network."); construction_network.reorder_ids().expect("Error when calling 'reorder_ids' on 'construction_network."); - let sorted = construction_network.topological_sort().expect("Error when calling 'topological_sort' on 'construction_network."); + let (sorted, _) = construction_network.topological_sort().expect("Error when calling 'topological_sort' on 'construction_network."); assert_eq!(sorted, vec![NodeId(0), NodeId(1), NodeId(2), NodeId(3)]); let ids: Vec<_> = construction_network.nodes.iter().map(|(id, _)| *id).collect(); println!("{ids:#?}"); diff --git a/node-graph/gstd/src/gpu_nodes.rs b/node-graph/gstd/src/gpu_nodes.rs index a4e7546e..c9d91dfd 100644 --- a/node-graph/gstd/src/gpu_nodes.rs +++ b/node-graph/gstd/src/gpu_nodes.rs @@ -31,7 +31,8 @@ async fn compile_gpu(node: &'input DocumentNode, typing_context: TypingContext, let mut typing_context = typing_context; let compiler = graph_craft::graphene_compiler::Compiler {}; let DocumentNodeImplementation::Network(ref network) = node.implementation else { panic!() }; - let proto_networks: Vec<_> = compiler.compile(network.clone())?.collect(); + let proto_networks: Result, _> = compiler.compile(network.clone()).collect(); + let proto_networks = proto_networks?; for network in proto_networks.iter() { typing_context.update(network).expect("Failed to type check network"); @@ -248,10 +249,10 @@ async fn create_compute_pass_descriptor( ..Default::default() }; log::debug!("compiling network"); - let proto_networks = compiler.compile(network.clone())?.collect(); + let proto_networks: Result, _> = compiler.compile(network.clone()).collect(); log::debug!("compiling shader"); let shader = compilation_client::compile( - proto_networks, + proto_networks?, vec![concrete!(u32), concrete!(Color)], vec![concrete!(Color)], ShaderIO { @@ -463,11 +464,12 @@ async fn blend_gpu_image(foreground: ImageFrame, background: ImageFrame, _> = compiler.compile(network.clone()).collect(); + let Ok(proto_networks_result) = proto_networks else { log::error!("Error compiling network in 'blend_gpu_image()"); return ImageFrame::empty(); }; - let proto_networks = proto_networks_result.collect(); + let proto_networks = proto_networks_result; log::debug!("compiling shader"); let shader = compilation_client::compile( diff --git a/shell.nix b/shell.nix index 5e77eb2f..5feb1012 100644 --- a/shell.nix +++ b/shell.nix @@ -54,6 +54,10 @@ in gcc-unwrapped.lib llvmPackages.libcxxStdenv pkg-config + # used for profiling + gnuplot + samply + cargo-flamegraph # For Tauri openssl