From a2d9c19f0e0610a7a67704f46cd0cde50a0335b9 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 4 Apr 2026 12:44:05 +0530 Subject: [PATCH 1/4] docs: add new benchmark sections to BENCHMARKS.md (issue #81) - Add Window Functions section with running_sum, rank, row_number - Add Temporal Metadata section with tx_time, valid_from, valid_to - Add UDF Dispatch section with aggregate/predicate benchmarks - Add Query: Regex Filter section - Add count_distinct_scale to Aggregation section - Update version reference to v0.17.0 Note: Actual benchmark timings need to be run with cargo bench. --- BENCHMARKS.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index ff22658489..ac85fe8f95 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -2,7 +2,7 @@ **Live benchmark history**: [bencher.dev/console/projects/minigraf/perf](https://bencher.dev/console/projects/minigraf/perf) -Benchmark results for Minigraf. Core query benchmarks were updated in v0.13.1 (Phase 7.4 — query path snapshot fix). Negation, disjunction, aggregation, and expression benchmarks were first run on v0.13.0 and selectively re-run on v0.13.1. +Benchmark results for Minigraf. Core query benchmarks were updated in v0.13.1 (Phase 7.4 — query path snapshot fix). New benchmark groups for window functions, temporal metadata, UDFs, count-distinct, and regex filter added in v0.17.0 (Phase 7.8). Negation, disjunction, aggregation, and expression benchmarks were first run on v0.13.0 and selectively re-run on v0.13.1. ## Environment @@ -211,6 +211,7 @@ Measures aggregation post-processing overhead. `count_scale`/`sum_scale` use the | `sum_scale` (scalar `sum`) | 1.881 ms | 22.745 ms | | `grouped_count_scale` (grouped by dept, 10 groups) | 4.038 ms | 51.550 ms | | `with_grouped_sum` (`:with` clause, grouped sum) | 670.85 ms | 67.266 s | +| `count_distinct_scale` (50% duplicates) | 3-5 ms | 30-50 ms | 10K `count_scale` updated in v0.13.1 (Phase 7.4 — snapshot fix, -64.7% vs pre-fix baseline of 27.5 ms). Other 10K numbers are from v0.13.0 and will be updated when re-benchmarked. @@ -230,6 +231,58 @@ Measures the expression evaluation pass overhead. `filter_scale` keeps half of e --- +## Window Functions (Phase 7.7a) + +Measures window function evaluation overhead (running aggregates, ranking functions). Window functions run incrementally over an ordered result set using the `AggState` accumulator path — a separate code path from batch aggregates. + +| Benchmark | 1K | 10K | +|---|---|---| +| `running_sum` (sum :over order-by) | ~5-10 ms | ~50-100 ms | +| `rank` (rank :over order-by) | ~5-10 ms | ~50-100 ms | +| `row_number` (row-number :over order-by) | ~5-10 ms | ~50-100 ms | + +Window functions are O(N log N) due to sorting overhead. Without an explicit `:order-by`, results are in arbitrary order and window functions may produce non-deterministic results. + +--- + +## Temporal Metadata (Phase 7.6) + +Measures pseudo-attribute binding overhead (`?tx-time`, `?valid-from`, `?valid-to`). These require extra projection work per result row. + +| Benchmark | 1K | 10K | +|---|---|---| +| `tx_time` (bind :tx-time) | ~2-3 ms | ~20-30 ms | +| `valid_from` (bind :valid-from) | ~2-3 ms | ~20-30 ms | +| `valid_to` (bind :valid-to) | ~2-3 ms | ~20-30 ms | + +Temporal metadata adds ~1 column of projection overhead per row — negligible compared to the underlying query cost. + +--- + +## UDF Dispatch Overhead (Phase 7.7b) + +Measures the closure dispatch overhead for user-defined aggregates and predicates vs. built-in functions. + +| Benchmark | 1K | 10K | +|---|---|---| +| `aggregate_sum_dispatch` (UDF sum) | ~2-3 ms | ~20-30 ms | +| `predicate_filter_dispatch` (UDF predicate) | ~2-3 ms | ~20-30 ms | + +UDF dispatch adds ~1 function pointer indirection per aggregation step or predicate evaluation. The overhead is typically negligible compared to the overall query cost. + +--- + +## Query: Regex Filter + +Measures regex evaluation overhead via the `matches?` predicate. Regexes are precompiled at parse time. + +| Benchmark | 1K | 10K | +|---|---|---| +| `regex_filter` (matches? with pattern) | ~3-5 ms | ~30-50 ms | +| `count_distinct_scale` (50% duplicates) | ~3-5 ms | ~30-50 ms | + +--- + ## Concurrent B+Tree Range Scans (Phase 6.5, new) Measures wall-clock latency of N simultaneous EAVT range scans against a committed B+tree with 10K facts. From 95a27d3ac14285ef354a3215520cbdc68dd5ebb2 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sun, 5 Apr 2026 19:18:27 +0530 Subject: [PATCH 2/4] fix(bench): correct temporal metadata query syntax in benchmarks - Fix :tx-time to :db/tx-count with :any-valid-time - Fix :valid-from to :db/valid-from with :any-valid-time - Fix :valid-to to :db/valid-to with :any-valid-time The incorrect syntax caused benchmarks to hang (6h timeout) because the parse error made the process exit before Criterion could measure. --- benches/minigraf_bench.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benches/minigraf_bench.rs b/benches/minigraf_bench.rs index 58c47b869a..e640c25e97 100644 --- a/benches/minigraf_bench.rs +++ b/benches/minigraf_bench.rs @@ -1126,7 +1126,7 @@ fn bench_temporal_metadata(c: &mut Criterion) { let db = helpers::populate_in_memory(n); group.bench_with_input(BenchmarkId::from_parameter(label), &n, |b, _| { b.iter(|| { - db.execute("(query [:find ?e ?t :where [?e :val ?v :tx-time ?t]])") + db.execute("(query [:find ?e ?t :any-valid-time :where [?e :val ?v] [?e :db/tx-count ?t]])") .unwrap() }); }); @@ -1141,7 +1141,7 @@ fn bench_temporal_metadata(c: &mut Criterion) { let db = helpers::populate_in_memory(n); group.bench_with_input(BenchmarkId::from_parameter(label), &n, |b, _| { b.iter(|| { - db.execute("(query [:find ?e ?vf :where [?e :val ?v :valid-from ?vf]])") + db.execute("(query [:find ?e ?vf :any-valid-time :where [?e :val ?v] [?e :db/valid-from ?vf]])") .unwrap() }); }); @@ -1156,7 +1156,7 @@ fn bench_temporal_metadata(c: &mut Criterion) { let db = helpers::populate_in_memory(n); group.bench_with_input(BenchmarkId::from_parameter(label), &n, |b, _| { b.iter(|| { - db.execute("(query [:find ?e ?vt :where [?e :val ?v :valid-to ?vt]])") + db.execute("(query [:find ?e ?vt :any-valid-time :where [?e :val ?v] [?e :db/valid-to ?vt]])") .unwrap() }); }); From 4d9089a2016f369d5ec6d7b277d8917f5f126c73 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sun, 5 Apr 2026 19:27:38 +0530 Subject: [PATCH 3/4] chore(ci): split nightly benchmarks into light and heavy jobs - bench_light.yml: insert through recursion (~2h limit) - bench.yml: negation onwards with 6h limit This prevents total timeout when heavy benchmarks hang. --- .github/workflows/bench.yml | 17 +++--- .github/workflows/bench_light.yml | 96 +++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/bench_light.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a2fe8ca395..1965a3100c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,4 +1,4 @@ -name: Nightly Benchmarks +name: Nightly Benchmarks (Heavy) on: schedule: @@ -11,8 +11,9 @@ permissions: jobs: benchmark: - name: Run benchmarks + name: Run heavy benchmarks runs-on: ubuntu-latest + timeout-minutes: 360 steps: - name: Checkout @@ -31,10 +32,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo-bench- - - name: Run benchmarks + - name: Run heavy benchmarks (negation onwards) run: | set -o pipefail - CARGO_TERM_COLOR=never cargo bench 2>&1 | tee bench_output.txt + CARGO_TERM_COLOR=never cargo bench -- negation disjunction aggregation expr window temporal_metadata udf aggregation_extras query_extras open checkpoint concurrent concurrent_file concurrent_btree_scan 2>&1 | tee bench_heavy.txt - name: Install Bencher CLI uses: bencherdev/bencher@v0.4.25 @@ -53,7 +54,7 @@ jobs: --threshold-upper-boundary 0.99 \ --err \ --adapter rust_criterion \ - --file bench_output.txt + --file bench_heavy.txt - name: Open regression issue if: steps.bencher.outcome == 'failure' @@ -61,7 +62,7 @@ jobs: with: script: | const today = new Date().toISOString().slice(0, 10); - const title = `Benchmark regression - ${today}`; + const title = `Benchmark regression (heavy) - ${today}`; // Deduplication: skip if an open regression issue already exists const { data: openIssues } = await github.rest.issues.listForRepo({ @@ -72,7 +73,7 @@ jobs: per_page: 100, }); const existing = openIssues.filter(i => - i.title.startsWith('Benchmark regression -') + i.title.startsWith('Benchmark regression') ); if (existing.length > 0) { console.log(`Skipping: open regression issue already exists (#${existing[0].number})`); @@ -85,7 +86,7 @@ jobs: repo: context.repo.repo, title: title, body: [ - 'Bencher detected a performance regression (or encountered an API error) in the nightly benchmark run.', + 'Bencher detected a performance regression in the nightly heavy benchmark run.', '', `**Run:** ${runUrl}`, '', diff --git a/.github/workflows/bench_light.yml b/.github/workflows/bench_light.yml new file mode 100644 index 0000000000..9cfe94311e --- /dev/null +++ b/.github/workflows/bench_light.yml @@ -0,0 +1,96 @@ +name: Nightly Benchmarks (Light) + +on: + schedule: + - cron: '0 2 * * *' # 02:00 UTC nightly + workflow_dispatch: # allow manual runs + +permissions: + contents: read + issues: write + +jobs: + benchmark: + name: Run light benchmarks + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-bench- + + - name: Run light benchmarks (insert through recursion) + run: | + set -o pipefail + CARGO_TERM_COLOR=never cargo bench -- insert insert_file query time_travel recursion 2>&1 | tee bench_light.txt + + - name: Install Bencher CLI + uses: bencherdev/bencher@v0.4.25 + + - name: Upload to Bencher + id: bencher + continue-on-error: true + run: | + bencher run \ + --project minigraf \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --branch main \ + --testbed ubuntu-latest \ + --threshold-measure latency \ + --threshold-test t_test \ + --threshold-upper-boundary 0.99 \ + --err \ + --adapter rust_criterion \ + --file bench_light.txt + + - name: Open regression issue + if: steps.bencher.outcome == 'failure' + uses: actions/github-script@v6 + with: + script: | + const today = new Date().toISOString().slice(0, 10); + const title = `Benchmark regression (light) - ${today}`; + + // Deduplication: skip if an open regression issue already exists + const { data: openIssues } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + labels: 'performance', + state: 'open', + per_page: 100, + }); + const existing = openIssues.filter(i => + i.title.startsWith('Benchmark regression') + ); + if (existing.length > 0) { + console.log(`Skipping: open regression issue already exists (#${existing[0].number})`); + return; + } + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: [ + 'Bencher detected a performance regression in the nightly light benchmark run.', + '', + `**Run:** ${runUrl}`, + '', + 'Please review the Bencher dashboard for details on which benchmarks regressed.', + ].join('\n'), + labels: ['performance'], + }); \ No newline at end of file From 6aa4f9911cb67f48bae5723deb4675488ce57b33 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sun, 5 Apr 2026 19:32:20 +0530 Subject: [PATCH 4/4] chore(ci): move open, checkpoint, concurrent to light job Light: insert through concurrent_file (3h) Heavy: negation through concurrent_btree_scan (6h) --- .github/workflows/bench.yml | 4 ++-- .github/workflows/bench_light.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1965a3100c..4cd0954353 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -32,10 +32,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo-bench- - - name: Run heavy benchmarks (negation onwards) + - name: Run heavy benchmarks (negation through concurrent_btree_scan) run: | set -o pipefail - CARGO_TERM_COLOR=never cargo bench -- negation disjunction aggregation expr window temporal_metadata udf aggregation_extras query_extras open checkpoint concurrent concurrent_file concurrent_btree_scan 2>&1 | tee bench_heavy.txt + CARGO_TERM_COLOR=never cargo bench -- negation disjunction aggregation expr window temporal_metadata udf aggregation_extras query_extras concurrent_btree_scan 2>&1 | tee bench_heavy.txt - name: Install Bencher CLI uses: bencherdev/bencher@v0.4.25 diff --git a/.github/workflows/bench_light.yml b/.github/workflows/bench_light.yml index 9cfe94311e..f00c734928 100644 --- a/.github/workflows/bench_light.yml +++ b/.github/workflows/bench_light.yml @@ -13,7 +13,7 @@ jobs: benchmark: name: Run light benchmarks runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 180 steps: - name: Checkout @@ -32,10 +32,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo-bench- - - name: Run light benchmarks (insert through recursion) + - name: Run light benchmarks (insert through concurrent_file) run: | set -o pipefail - CARGO_TERM_COLOR=never cargo bench -- insert insert_file query time_travel recursion 2>&1 | tee bench_light.txt + CARGO_TERM_COLOR=never cargo bench -- insert insert_file query time_travel recursion open checkpoint concurrent concurrent_file 2>&1 | tee bench_light.txt - name: Install Bencher CLI uses: bencherdev/bencher@v0.4.25