diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml new file mode 100644 index 00000000..b275118d --- /dev/null +++ b/.github/workflows/profiling.yaml @@ -0,0 +1,100 @@ +name: Profiling + +on: + pull_request: + branches: [ master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + profile: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - name: Install Valgrind + run: | + sudo apt-get update + sudo apt-get install -y valgrind + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install iai-callgrind + run: | + cargo install iai-callgrind-runner@0.12.3 + + - name: Checkout master branch + run: | + git fetch origin master:master + git checkout master + + - name: Run baseline benchmarks + run: | + cargo bench --bench compile_demo_art --features=iai -- --save-baseline=master + + - name: Checkout PR branch + run: | + git checkout ${{ github.event.pull_request.head.sha }} + + - name: Run PR benchmarks + id: benchmark + run: | + BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g') + echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT + echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Comment PR + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const benchmarkOutput = JSON.parse(`${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}`); + + let significantChanges = false; + let commentBody = "#### Performance Benchmark Results\n\n"; + + for (const benchmark of benchmarkOutput) { + if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) { + for (const summary of benchmark.callgrind_summary.summaries) { + for (const [eventKind, costsDiff] of Object.entries(summary.events)) { + if (costsDiff.diff_pct !== null && Math.abs(costsDiff.diff_pct) > 5) { + significantChanges = true; + const changeDirection = costsDiff.diff_pct > 0 ? "increase" : "decrease"; + const color = costsDiff.diff_pct > 0 ? "red" : "green"; + commentBody += `\`${benchmark.module_path}\` - ${eventKind}:\n`; + commentBody += `\\color{${color}}${changeDirection} of ${Math.abs(costsDiff.diff_pct).toFixed(2)}%\n`; + commentBody += `Old: ${costsDiff.old}, New: ${costsDiff.new}\n\n`; + } + } + } + } + } + + if (significantChanges) { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody + }); + } else { + console.log("No significant performance changes detected. Skipping comment."); + console.log(commentBody); + } diff --git a/Cargo.lock b/Cargo.lock index 580745f3..a434c8cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,15 +1207,6 @@ dependencies = [ "libm", ] -[[package]] -name = "cpp_demangle" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" -dependencies = [ - "cfg-if", -] - [[package]] name = "cpufeatures" version = "0.2.12" @@ -1415,15 +1406,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" -[[package]] -name = "debugid" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" -dependencies = [ - "uuid", -] - [[package]] name = "deranged" version = "0.3.11" @@ -1775,18 +1757,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "findshlibs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" -dependencies = [ - "cc", - "lazy_static", - "libc", - "winapi", -] - [[package]] name = "fixedbitset" version = "0.4.2" @@ -2432,10 +2402,10 @@ dependencies = [ "glob", "graph-craft", "graphene-core", + "iai-callgrind", "js-sys", "log", "num-traits", - "pprof", "reqwest 0.12.5", "rustc-hash 2.0.0", "serde", @@ -3042,6 +3012,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "iai-callgrind" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "283f598a969822c70af13aae1272ba09c97014c7344d3b24652e5b1d7b771c36" +dependencies = [ + "bincode", + "iai-callgrind-macros", + "iai-callgrind-runner", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e2ff2e86bdba764b66d94b65f2caa03da60d971a6930fdc2e67f12582c5bb8" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.72", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb92a65def0d3a0ef41029c411dc2ecdd3518708c062f8bd576fd4143be1c56b" +dependencies = [ + "serde", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -3194,24 +3198,6 @@ dependencies = [ "cfb", ] -[[package]] -name = "inferno" -version = "0.11.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" -dependencies = [ - "ahash", - "indexmap 2.2.6", - "is-terminal", - "itoa 1.0.11", - "log", - "num-format", - "once_cell", - "quick-xml 0.26.0", - "rgb", - "str_stack", -] - [[package]] name = "instant" version = "0.1.13" @@ -3979,16 +3965,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa 1.0.11", -] - [[package]] name = "num-integer" version = "0.1.46" @@ -4755,27 +4731,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "pprof" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" -dependencies = [ - "backtrace", - "cfg-if", - "findshlibs", - "inferno", - "libc", - "log", - "nix 0.26.4", - "once_cell", - "parking_lot", - "smallvec", - "symbolic-demangle", - "tempfile", - "thiserror", -] - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4858,15 +4813,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58" -[[package]] -name = "quick-xml" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.31.0" @@ -6094,12 +6040,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "str_stack" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" - [[package]] name = "strict-num" version = "0.1.1" @@ -6163,29 +6103,6 @@ dependencies = [ "siphasher 1.0.1", ] -[[package]] -name = "symbolic-common" -version = "12.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16629323a4ec5268ad23a575110a724ad4544aae623451de600c747bf87b36cf" -dependencies = [ - "debugid", - "memmap2", - "stable_deref_trait", - "uuid", -] - -[[package]] -name = "symbolic-demangle" -version = "12.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c043a45f08f41187414592b3ceb53fb0687da57209cc77401767fb69d5b596" -dependencies = [ - "cpp_demangle", - "rustc-demangle", - "symbolic-common", -] - [[package]] name = "syn" version = "1.0.109" diff --git a/node-graph/graph-craft/Cargo.toml b/node-graph/graph-craft/Cargo.toml index 4d885642..12781e6a 100644 --- a/node-graph/graph-craft/Cargo.toml +++ b/node-graph/graph-craft/Cargo.toml @@ -11,6 +11,8 @@ dealloc_nodes = [] wgpu = [] tokio = ["dep:tokio"] wayland = [] +criterion = [] +iai = [] [dependencies] # Local dependencies @@ -39,24 +41,27 @@ wgpu-executor = { workspace = true } serde = { workspace = true, optional = true } tokio = { workspace = true, optional = true } -[target.'cfg(target_arch = "wasm32")'.dependencies] # Workspace dependencies +[target.'cfg(target_arch = "wasm32")'.dependencies] web-sys = { workspace = true } js-sys = { workspace = true } wasm-bindgen = { workspace = true } wasm-bindgen-futures = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -# Workspace dependencies winit = { workspace = true } [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } -glob = "0.3" -pprof = { version = "0.13", features = ["flamegraph"] } +# Workspace dependencies serde_json = { workspace = true } graph-craft = { workspace = true, features = ["serde"] } +# Required dependencies +criterion = { version = "0.5", features = ["html_reports"]} +glob = "0.3" +iai-callgrind = { version = "0.12.3"} + +# Benchmarks [[bench]] name = "compile_demo_art" harness = false diff --git a/node-graph/graph-craft/benches/compile_demo_art.rs b/node-graph/graph-craft/benches/compile_demo_art.rs index 28c33683..49ebb7dd 100644 --- a/node-graph/graph-craft/benches/compile_demo_art.rs +++ b/node-graph/graph-craft/benches/compile_demo_art.rs @@ -1,28 +1,58 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; use graph_craft::document::NodeNetwork; use graph_craft::graphene_compiler::Compiler; use graph_craft::proto::ProtoNetwork; -pub fn compile_to_proto(c: &mut Criterion) { - let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); - for path in artworks { - let Ok(path) = path else { continue }; - let content = std::fs::read(&path).expect("failed to read file"); - let network = load_network(std::str::from_utf8(&content).unwrap()); - let name = path.file_stem().unwrap().to_str().unwrap(); +#[cfg(feature = "criterion")] +use criterion::{black_box, criterion_group, criterion_main, Criterion}; - c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); - } -} +#[cfg(all(not(feature = "criterion"), feature = "iai"))] +use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main}; fn load_network(document_string: &str) -> NodeNetwork { let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document"); serde_json::from_value::(document["network_interface"]["network"].clone()).expect("Failed to parse document") } + fn compile(network: NodeNetwork) -> ProtoNetwork { let compiler = Compiler {}; compiler.compile_single(network).unwrap() } +#[cfg(all(not(feature = "criterion"), feature = "iai"))] +fn load_from_name(name: &str) -> NodeNetwork { + let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + let content = std::str::from_utf8(&content).unwrap(); + black_box(compile(black_box(network))); + load_network(content) +} + +#[cfg(feature = "criterion")] +fn compile_to_proto(c: &mut Criterion) { + let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); + for path in artworks { + let Ok(path) = path else { continue }; + let name = path.file_stem().unwrap().to_str().unwrap(); + let content = std::fs::read(&path).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); + } +} + +#[cfg_attr(all(feature = "iai", not(feature = "criterion")), library_benchmark)] +#[cfg_attr(all(feature = "iai", not(feature="criterion")), benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))] +pub fn iai_compile_to_proto(input: NodeNetwork) { + black_box(compile(input)); +} + +#[cfg(feature = "criterion")] criterion_group!(benches, compile_to_proto); + +#[cfg(feature = "criterion")] criterion_main!(benches); + +#[cfg(all(not(feature = "criterion"), feature = "iai"))] +library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto); + +#[cfg(all(not(feature = "criterion"), feature = "iai"))] +main!(library_benchmark_groups = compile_group);