diff --git a/.config/mise/config.toml b/.config/mise/config.toml index e82e46d5b4f..8d21a77b320 100644 --- a/.config/mise/config.toml +++ b/.config/mise/config.toml @@ -18,6 +18,7 @@ protoc = "32.1" # CLI tools biome = "1.9.5-nightly.ff02a0b" +"cargo:cargo-codspeed" = "4.1.0" "cargo:cargo-hack" = "0.6.37" "cargo:cargo-insta" = "1.43.1" "cargo:cargo-llvm-cov" = "0.6.18" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..310d181ba92 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,80 @@ +name: CodSpeed Benchmarks + +on: + push: + branches: + - "main" + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +permissions: + contents: read + id-token: write + +jobs: + setup: + runs-on: ubuntu-24.04 + permissions: + id-token: write + outputs: + packages: ${{ steps.packages.outputs.packages }} + steps: + - name: Checkout source code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 2 + + - name: Install tools + uses: ./.github/actions/install-tools + with: + token: ${{ secrets.GITHUB_TOKEN }} + vault_address: ${{ secrets.VAULT_ADDR }} + rust: false + + - name: Determine changed packages that have codspeed + id: packages + run: | + PACKAGES_QUERY='query { affectedPackages(base: "HEAD^" filter: {has: {field: "TASK_NAME", value: "build:codspeed"}}) {items {name path}}}' + PACKAGES=$(turbo query "$PACKAGES_QUERY" \ + | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }') + + echo "packages=$PACKAGES" | tee -a $GITHUB_OUTPUT + + benchmarks: + name: Run benchmarks + needs: [setup] + runs-on: ubuntu-24.04 + strategy: + matrix: ${{ fromJSON(needs.setup.outputs.packages) }} + fail-fast: false + if: needs.setup.outputs.packages != '{"name":[],"include":[]}' + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 2 + + - name: Clean up disk + uses: ./.github/actions/clean-up-disk + + - name: Install tools + uses: ./.github/actions/install-tools + with: + token: ${{ secrets.GITHUB_TOKEN }} + vault_address: ${{ secrets.VAULT_ADDR }} + + - name: Prune repository + uses: ./.github/actions/prune-repository + with: + scope: ${{ matrix.name }} + + - name: Build the benchmark target + run: turbo run build:codspeed --filter=${{ matrix.name }} + + - name: Run the benchmark + uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1 + with: + mode: simulation + run: turbo run test:codspeed --filter=${{ matrix.name }} diff --git a/Cargo.lock b/Cargo.lock index f0b60e9120c..a7e35a7dbc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,15 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "ar_archive_writer" version = "0.2.0" @@ -1404,12 +1413,80 @@ dependencies = [ "cc", ] +[[package]] +name = "codspeed" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b847e05a34be5c38f3f2a5052178a3bd32e6b5702f3ea775efde95c483a539" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.2.16", + "glob", + "libc", + "nix 0.30.1", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a0e2a53beb18dec493ec133f226e0d35e8bb2fdc638ccf7351696eabee416c" +dependencies = [ + "clap", + "codspeed", + "codspeed-criterion-compat-walltime", + "colored", + "regex", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f652d6e6d40bba0f5c244744db94d92a26b7f083df18692df88fb0772f1c793" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "codspeed", + "criterion-plot 0.5.0", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + [[package]] name = "colorchoice" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "compact_str" version = "0.9.0" @@ -1596,7 +1673,7 @@ dependencies = [ "cast", "ciborium", "clap", - "criterion-plot", + "criterion-plot 0.8.1", "itertools 0.13.0", "num-traits", "oorandom", @@ -1621,6 +1698,16 @@ dependencies = [ "quote", ] +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "criterion-plot" version = "0.8.1" @@ -3730,8 +3817,7 @@ version = "0.0.0" dependencies = [ "anstyle-svg", "bstr", - "criterion", - "criterion-macro", + "codspeed-criterion-compat", "hashql-compiletest", "hashql-core", "hashql-diagnostics", @@ -4376,6 +4462,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "is_ci" version = "1.2.0" @@ -4397,6 +4494,15 @@ dependencies = [ "nom", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -8051,6 +8157,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "str_stack" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2361e04d9f9..91c4c0bc3a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ circular-buffer = { version = "1.1.0", default-features = fal clap = { version = "4.5.51", features = ["color", "error-context", "help", "std", "suggestions", "usage"] } clap_builder = { version = "4.5.51", default-features = false, features = ["std"] } clap_complete = { version = "4.5.60", default-features = false } +codspeed-criterion-compat = { version = "4.1.0" } console_error_panic_hook = { version = "0.1.7", default-features = false } convert_case = { version = "0.10.0", default-features = false } criterion = { version = "0.8.0" } diff --git a/libs/@local/hashql/core/src/heap/allocator.rs b/libs/@local/hashql/core/src/heap/allocator.rs index a293611ee49..bf3058dd066 100644 --- a/libs/@local/hashql/core/src/heap/allocator.rs +++ b/libs/@local/hashql/core/src/heap/allocator.rs @@ -24,6 +24,12 @@ impl Allocator { Self(Bump::with_capacity(capacity)) } + /// Sets the allocation limit for the allocator. + #[inline] + pub(crate) fn set_allocation_limit(&self, capacity: Option) { + self.0.set_allocation_limit(capacity); + } + /// Allocates a value using a closure to avoid moving before allocation. #[inline] pub(crate) fn alloc_with(&self, func: impl FnOnce() -> T) -> &mut T { diff --git a/libs/@local/hashql/core/src/heap/mod.rs b/libs/@local/hashql/core/src/heap/mod.rs index fb2763751fc..7c4a474f53f 100644 --- a/libs/@local/hashql/core/src/heap/mod.rs +++ b/libs/@local/hashql/core/src/heap/mod.rs @@ -241,6 +241,12 @@ impl Heap { } } + /// Sets the allocation limit for the heap. + #[inline] + pub fn set_allocation_limit(&self, capacity: Option) { + self.inner.set_allocation_limit(capacity); + } + /// Allocates a value in the arena, returning a mutable reference. /// /// Only accepts types that do **not** require [`Drop`]. Types requiring destructors diff --git a/libs/@local/hashql/core/src/heap/scratch.rs b/libs/@local/hashql/core/src/heap/scratch.rs index efdbe488e07..372198fcd53 100644 --- a/libs/@local/hashql/core/src/heap/scratch.rs +++ b/libs/@local/hashql/core/src/heap/scratch.rs @@ -30,11 +30,20 @@ pub struct Scratch { impl Scratch { /// Creates a new scratch allocator. #[must_use] + #[inline] pub fn new() -> Self { Self { inner: Allocator::new(), } } + + #[must_use] + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + Self { + inner: Allocator::with_capacity(capacity), + } + } } impl Default for Scratch { diff --git a/libs/@local/hashql/mir/Cargo.toml b/libs/@local/hashql/mir/Cargo.toml index 318a916cef5..2c48b2045f2 100644 --- a/libs/@local/hashql/mir/Cargo.toml +++ b/libs/@local/hashql/mir/Cargo.toml @@ -22,10 +22,9 @@ bstr = { workspace = true } simple-mermaid = { workspace = true } [dev-dependencies] -criterion = { workspace = true } -criterion-macro = { workspace = true } -hashql-compiletest = { workspace = true } -insta = { workspace = true } +codspeed-criterion-compat = { workspace = true } +hashql-compiletest = { workspace = true } +insta = { workspace = true } [lints] @@ -35,6 +34,10 @@ workspace = true name = "compiletest" harness = false +[[bench]] +name = "transform" +harness = false + [package.metadata.sync.turborepo] ignore-dev-dependencies = [ "@rust/hashql-compiletest", diff --git a/libs/@local/hashql/mir/benches/transform.rs b/libs/@local/hashql/mir/benches/transform.rs index 5918b2be04b..d3ea26c06aa 100644 --- a/libs/@local/hashql/mir/benches/transform.rs +++ b/libs/@local/hashql/mir/benches/transform.rs @@ -1,12 +1,12 @@ -#![feature(custom_test_frameworks)] -#![test_runner(criterion::runner)] -#![expect(clippy::min_ident_chars, clippy::many_single_char_names)] +#![expect( + clippy::min_ident_chars, + clippy::many_single_char_names, + clippy::significant_drop_tightening +)] -use core::{hint::black_box, time::Duration}; -use std::time::Instant; +use core::hint::black_box; -use criterion::Criterion; -use criterion_macro::criterion; +use codspeed_criterion_compat::{BatchSize, Bencher, Criterion, criterion_group, criterion_main}; use hashql_core::{ heap::{BumpAllocator as _, Heap, Scratch}, r#type::{TypeBuilder, environment::Environment}, @@ -242,133 +242,154 @@ fn create_complex_cfg<'heap>(env: &Environment<'heap>, interner: &Interner<'heap builder.finish(1, TypeBuilder::synthetic(env).integer()) } -fn run_fn( - iters: u64, +#[expect(unsafe_code)] +#[inline] +fn run_bencher( + bencher: &mut Bencher, body: for<'heap> fn(&Environment<'heap>, &Interner<'heap>) -> Body<'heap>, mut func: impl for<'env, 'heap> FnMut(&mut MirContext<'env, 'heap>, &mut Body<'heap>), -) -> Duration { +) { + // NOTE: `heap` must not be moved or reassigned; `heap_ptr` assumes its address is stable + // for the entire duration of this function. let mut heap = Heap::new(); - let mut total = Duration::ZERO; - - for _ in 0..iters { - heap.reset(); - let env = Environment::new(&heap); - let interner = Interner::new(&heap); - let mut body = black_box(body(&env, &interner)); - - let mut context = MirContext { - heap: &heap, - env: &env, - interner: &interner, - diagnostics: DiagnosticIssues::new(), - }; - - let start = Instant::now(); - func(&mut context, &mut body); - total += start.elapsed(); - - drop(black_box(body)); - } - - total + let heap_ptr = &raw mut heap; + + // Using `iter_custom` here would be better, but codspeed doesn't support it yet. + // + // IMPORTANT: `BatchSize::PerIteration` is critical for soundness. Do NOT change this to + // `SmallInput`, `LargeInput`, or any other batch size. Doing so will cause undefined + // behavior (use-after-free of arena allocations). + bencher.iter_batched_ref( + || { + // SAFETY: We create a `&mut Heap` from the raw pointer to call `reset()` and build + // the environment/interner/body. This is sound because: + // - `heap` outlives the entire `iter_batched` call (it's a local in the outer scope). + // - `BatchSize::PerIteration` ensures only one `(env, interner, body)` tuple exists at + // a time, and it is dropped before the next `setup()` call. + // - No other references to `heap` exist during this closure's execution. + // - This code runs single-threaded. + let heap = unsafe { &mut *heap_ptr }; + heap.reset(); + + let env = Environment::new(heap); + let interner = Interner::new(heap); + let body = body(&env, &interner); + + (env, interner, body) + }, + |(env, interner, body)| { + // SAFETY: We create a shared `&Heap` reference. This is sound because: + // - The `&mut Heap` from setup no longer exists (setup closure has returned) + // - The `env`, `interner`, and `body` already hold shared borrows of `heap` + // - Adding another `&Heap` is just shared-shared aliasing, which is allowed + let heap = unsafe { &*heap_ptr }; + + let mut context = MirContext { + heap, + env, + interner, + diagnostics: DiagnosticIssues::new(), + }; + + func(black_box(&mut context), black_box(body)); + context.diagnostics + }, + BatchSize::PerIteration, + ); } +#[inline] fn run( - iters: u64, + bencher: &mut Bencher, body: for<'heap> fn(&Environment<'heap>, &Interner<'heap>) -> Body<'heap>, mut pass: impl for<'env, 'heap> TransformPass<'env, 'heap>, -) -> Duration { - run_fn( - iters, +) { + run_bencher( + bencher, body, #[inline] |context, body| pass.run(context, body), - ) + ); } -#[criterion] fn cfg_simplify(criterion: &mut Criterion) { let mut group = criterion.benchmark_group("cfg_simplify"); group.bench_function("linear", |bencher| { - bencher.iter_custom(|iters| run(iters, create_linear_cfg, CfgSimplify::new())); + run(bencher, create_linear_cfg, CfgSimplify::new()); }); + group.bench_function("diamond", |bencher| { - bencher.iter_custom(|iters| run(iters, create_diamond_cfg, CfgSimplify::new())); + run(bencher, create_diamond_cfg, CfgSimplify::new()); }); + group.bench_function("complex", |bencher| { - bencher.iter_custom(|iters| run(iters, create_complex_cfg, CfgSimplify::new())); + run(bencher, create_complex_cfg, CfgSimplify::new()); }); } -#[criterion] fn sroa(criterion: &mut Criterion) { let mut group = criterion.benchmark_group("sroa"); group.bench_function("linear", |bencher| { - bencher.iter_custom(|iters| run(iters, create_linear_cfg, Sroa::new())); + run(bencher, create_linear_cfg, Sroa::new()); }); group.bench_function("diamond", |bencher| { - bencher.iter_custom(|iters| run(iters, create_diamond_cfg, Sroa::new())); + run(bencher, create_diamond_cfg, Sroa::new()); }); group.bench_function("complex", |bencher| { - bencher.iter_custom(|iters| run(iters, create_complex_cfg, Sroa::new())); + run(bencher, create_complex_cfg, Sroa::new()); }); } -#[criterion] fn dse(criterion: &mut Criterion) { let mut group = criterion.benchmark_group("dse"); group.bench_function("dead stores", |bencher| { - bencher.iter_custom(|iters| run(iters, create_dead_store_cfg, DeadStoreElimination::new())); + run(bencher, create_dead_store_cfg, DeadStoreElimination::new()); }); group.bench_function("linear", |bencher| { - bencher.iter_custom(|iters| run(iters, create_linear_cfg, DeadStoreElimination::new())); + run(bencher, create_linear_cfg, DeadStoreElimination::new()); }); group.bench_function("diamond", |bencher| { - bencher.iter_custom(|iters| run(iters, create_diamond_cfg, DeadStoreElimination::new())); + run(bencher, create_diamond_cfg, DeadStoreElimination::new()); }); group.bench_function("complex", |bencher| { - bencher.iter_custom(|iters| run(iters, create_complex_cfg, DeadStoreElimination::new())); + run(bencher, create_complex_cfg, DeadStoreElimination::new()); }); } -#[criterion] fn pipeline(criterion: &mut Criterion) { let mut group = criterion.benchmark_group("pipeline"); group.bench_function("linear", |bencher| { let mut scratch = Scratch::new(); - bencher.iter_custom(|iters| { - run_fn(iters, create_linear_cfg, |context, body| { - CfgSimplify::new_in(&mut scratch).run(context, body); - Sroa::new_in(&mut scratch).run(context, body); - DeadStoreElimination::new_in(&mut scratch).run(context, body); - }) + run_bencher(bencher, create_linear_cfg, |context, body| { + CfgSimplify::new_in(&mut scratch).run(context, body); + Sroa::new_in(&mut scratch).run(context, body); + DeadStoreElimination::new_in(&mut scratch).run(context, body); }); }); group.bench_function("diamond", |bencher| { let mut scratch = Scratch::new(); - bencher.iter_custom(|iters| { - run_fn(iters, create_diamond_cfg, |context, body| { - CfgSimplify::new_in(&mut scratch).run(context, body); - Sroa::new_in(&mut scratch).run(context, body); - DeadStoreElimination::new_in(&mut scratch).run(context, body); - }) + run_bencher(bencher, create_diamond_cfg, |context, body| { + CfgSimplify::new_in(&mut scratch).run(context, body); + Sroa::new_in(&mut scratch).run(context, body); + DeadStoreElimination::new_in(&mut scratch).run(context, body); }); }); group.bench_function("complex", |bencher| { let mut scratch = Scratch::new(); - bencher.iter_custom(|iters| { - run_fn(iters, create_complex_cfg, |context, body| { - CfgSimplify::new_in(&mut scratch).run(context, body); - Sroa::new_in(&mut scratch).run(context, body); - DeadStoreElimination::new_in(&mut scratch).run(context, body); - }) + run_bencher(bencher, create_complex_cfg, |context, body| { + CfgSimplify::new_in(&mut scratch).run(context, body); + Sroa::new_in(&mut scratch).run(context, body); + DeadStoreElimination::new_in(&mut scratch).run(context, body); }); }); } + +criterion_group!(benches, cfg_simplify, sroa, dse, pipeline); +criterion_main!(benches); diff --git a/libs/@local/hashql/mir/package.json b/libs/@local/hashql/mir/package.json index 7152978e9a8..1a0eee7fc9a 100644 --- a/libs/@local/hashql/mir/package.json +++ b/libs/@local/hashql/mir/package.json @@ -4,9 +4,11 @@ "private": true, "license": "AGPL-3", "scripts": { + "build:codspeed": "cargo codspeed build -p hashql-mir", "doc:dependency-diagram": "cargo run -p hash-repo-chores -- dependency-diagram --output docs/dependency-diagram.mmd --root hashql-mir --root-deps-and-dependents --link-mode non-roots --include-dev-deps --include-build-deps --logging-console-level info", "fix:clippy": "just clippy --fix", "lint:clippy": "just clippy", + "test:codspeed": "cargo codspeed run -p hashql-mir", "test:unit": "mise run test:unit @rust/hashql-mir" }, "dependencies": { diff --git a/libs/@local/hashql/mir/src/body/basic_blocks.rs b/libs/@local/hashql/mir/src/body/basic_blocks.rs index a37733a7784..683e09c6ff2 100644 --- a/libs/@local/hashql/mir/src/body/basic_blocks.rs +++ b/libs/@local/hashql/mir/src/body/basic_blocks.rs @@ -341,6 +341,7 @@ impl graph::Successors for BasicBlocks<'_> { where Self: 'this; + #[inline] fn successors(&self, node: Self::NodeId) -> Self::SuccIter<'_> { self.blocks[node].terminator.kind.successor_blocks() } diff --git a/libs/@local/hashql/mir/src/body/terminator/mod.rs b/libs/@local/hashql/mir/src/body/terminator/mod.rs index 85b453fdec5..1f861c46609 100644 --- a/libs/@local/hashql/mir/src/body/terminator/mod.rs +++ b/libs/@local/hashql/mir/src/body/terminator/mod.rs @@ -46,10 +46,12 @@ where { type Item = L::Item; + #[inline] fn next(&mut self) -> Option { for_both!(self; value => value.next()) } + #[inline] fn size_hint(&self) -> (usize, Option) { for_both!(self; value => value.size_hint()) } @@ -60,6 +62,7 @@ where L: DoubleEndedIterator, R: DoubleEndedIterator, { + #[inline] fn next_back(&mut self) -> Option { for_both!(self; value => value.next_back()) } @@ -70,6 +73,7 @@ where L: ExactSizeIterator, R: ExactSizeIterator, { + #[inline] fn len(&self) -> usize { for_both!(self; value => value.len()) } diff --git a/turbo.json b/turbo.json index 0b98a1e805d..c5f915ccada 100644 --- a/turbo.json +++ b/turbo.json @@ -28,6 +28,9 @@ "build:types": { "dependsOn": ["^build:types"] }, + "build:codspeed": { + "cache": false + }, "compile": { "cache": false }, @@ -70,6 +73,9 @@ "dependsOn": ["codegen", "^start:test:healthcheck"], "env": ["TEST_COVERAGE"] }, + "test:codspeed": { + "cache": false + }, "test:miri": {}, // Benchmarks "bench:unit": {