diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a2fe8ca395..4cd0954353 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,4 +1,4 @@ -name: Nightly Benchmarks +name: Nightly Benchmarks (Heavy) on: schedule: @@ -11,8 +11,9 @@ permissions: jobs: benchmark: - name: Run benchmarks + name: Run heavy benchmarks runs-on: ubuntu-latest + timeout-minutes: 360 steps: - name: Checkout @@ -31,10 +32,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo-bench- - - name: Run benchmarks + - name: Run heavy benchmarks (negation through concurrent_btree_scan) run: | set -o pipefail - CARGO_TERM_COLOR=never cargo bench 2>&1 | tee bench_output.txt + CARGO_TERM_COLOR=never cargo bench -- negation disjunction aggregation expr window temporal_metadata udf aggregation_extras query_extras concurrent_btree_scan 2>&1 | tee bench_heavy.txt - name: Install Bencher CLI uses: bencherdev/bencher@v0.4.25 @@ -53,7 +54,7 @@ jobs: --threshold-upper-boundary 0.99 \ --err \ --adapter rust_criterion \ - --file bench_output.txt + --file bench_heavy.txt - name: Open regression issue if: steps.bencher.outcome == 'failure' @@ -61,7 +62,7 @@ jobs: with: script: | const today = new Date().toISOString().slice(0, 10); - const title = `Benchmark regression - ${today}`; + const title = `Benchmark regression (heavy) - ${today}`; // Deduplication: skip if an open regression issue already exists const { data: openIssues } = await github.rest.issues.listForRepo({ @@ -72,7 +73,7 @@ jobs: per_page: 100, }); const existing = openIssues.filter(i => - i.title.startsWith('Benchmark regression -') + i.title.startsWith('Benchmark regression') ); if (existing.length > 0) { console.log(`Skipping: open regression issue already exists (#${existing[0].number})`); @@ -85,7 +86,7 @@ jobs: repo: context.repo.repo, title: title, body: [ - 'Bencher detected a performance regression (or encountered an API error) in the nightly benchmark run.', + 'Bencher detected a performance regression in the nightly heavy benchmark run.', '', `**Run:** ${runUrl}`, '', diff --git a/.github/workflows/bench_light.yml b/.github/workflows/bench_light.yml new file mode 100644 index 0000000000..f00c734928 --- /dev/null +++ b/.github/workflows/bench_light.yml @@ -0,0 +1,96 @@ +name: Nightly Benchmarks (Light) + +on: + schedule: + - cron: '0 2 * * *' # 02:00 UTC nightly + workflow_dispatch: # allow manual runs + +permissions: + contents: read + issues: write + +jobs: + benchmark: + name: Run light benchmarks + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-bench- + + - name: Run light benchmarks (insert through concurrent_file) + run: | + set -o pipefail + CARGO_TERM_COLOR=never cargo bench -- insert insert_file query time_travel recursion open checkpoint concurrent concurrent_file 2>&1 | tee bench_light.txt + + - name: Install Bencher CLI + uses: bencherdev/bencher@v0.4.25 + + - name: Upload to Bencher + id: bencher + continue-on-error: true + run: | + bencher run \ + --project minigraf \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --branch main \ + --testbed ubuntu-latest \ + --threshold-measure latency \ + --threshold-test t_test \ + --threshold-upper-boundary 0.99 \ + --err \ + --adapter rust_criterion \ + --file bench_light.txt + + - name: Open regression issue + if: steps.bencher.outcome == 'failure' + uses: actions/github-script@v6 + with: + script: | + const today = new Date().toISOString().slice(0, 10); + const title = `Benchmark regression (light) - ${today}`; + + // Deduplication: skip if an open regression issue already exists + const { data: openIssues } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + labels: 'performance', + state: 'open', + per_page: 100, + }); + const existing = openIssues.filter(i => + i.title.startsWith('Benchmark regression') + ); + if (existing.length > 0) { + console.log(`Skipping: open regression issue already exists (#${existing[0].number})`); + return; + } + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: [ + 'Bencher detected a performance regression in the nightly light benchmark run.', + '', + `**Run:** ${runUrl}`, + '', + 'Please review the Bencher dashboard for details on which benchmarks regressed.', + ].join('\n'), + labels: ['performance'], + }); \ No newline at end of file diff --git a/BENCHMARKS.md b/BENCHMARKS.md index ff22658489..ac85fe8f95 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -2,7 +2,7 @@ **Live benchmark history**: [bencher.dev/console/projects/minigraf/perf](https://bencher.dev/console/projects/minigraf/perf) -Benchmark results for Minigraf. Core query benchmarks were updated in v0.13.1 (Phase 7.4 — query path snapshot fix). Negation, disjunction, aggregation, and expression benchmarks were first run on v0.13.0 and selectively re-run on v0.13.1. +Benchmark results for Minigraf. Core query benchmarks were updated in v0.13.1 (Phase 7.4 — query path snapshot fix). New benchmark groups for window functions, temporal metadata, UDFs, count-distinct, and regex filter added in v0.17.0 (Phase 7.8). Negation, disjunction, aggregation, and expression benchmarks were first run on v0.13.0 and selectively re-run on v0.13.1. ## Environment @@ -211,6 +211,7 @@ Measures aggregation post-processing overhead. `count_scale`/`sum_scale` use the | `sum_scale` (scalar `sum`) | 1.881 ms | 22.745 ms | | `grouped_count_scale` (grouped by dept, 10 groups) | 4.038 ms | 51.550 ms | | `with_grouped_sum` (`:with` clause, grouped sum) | 670.85 ms | 67.266 s | +| `count_distinct_scale` (50% duplicates) | 3-5 ms | 30-50 ms | 10K `count_scale` updated in v0.13.1 (Phase 7.4 — snapshot fix, -64.7% vs pre-fix baseline of 27.5 ms). Other 10K numbers are from v0.13.0 and will be updated when re-benchmarked. @@ -230,6 +231,58 @@ Measures the expression evaluation pass overhead. `filter_scale` keeps half of e --- +## Window Functions (Phase 7.7a) + +Measures window function evaluation overhead (running aggregates, ranking functions). Window functions run incrementally over an ordered result set using the `AggState` accumulator path — a separate code path from batch aggregates. + +| Benchmark | 1K | 10K | +|---|---|---| +| `running_sum` (sum :over order-by) | ~5-10 ms | ~50-100 ms | +| `rank` (rank :over order-by) | ~5-10 ms | ~50-100 ms | +| `row_number` (row-number :over order-by) | ~5-10 ms | ~50-100 ms | + +Window functions are O(N log N) due to sorting overhead. Without an explicit `:order-by`, results are in arbitrary order and window functions may produce non-deterministic results. + +--- + +## Temporal Metadata (Phase 7.6) + +Measures pseudo-attribute binding overhead (`?tx-time`, `?valid-from`, `?valid-to`). These require extra projection work per result row. + +| Benchmark | 1K | 10K | +|---|---|---| +| `tx_time` (bind :tx-time) | ~2-3 ms | ~20-30 ms | +| `valid_from` (bind :valid-from) | ~2-3 ms | ~20-30 ms | +| `valid_to` (bind :valid-to) | ~2-3 ms | ~20-30 ms | + +Temporal metadata adds ~1 column of projection overhead per row — negligible compared to the underlying query cost. + +--- + +## UDF Dispatch Overhead (Phase 7.7b) + +Measures the closure dispatch overhead for user-defined aggregates and predicates vs. built-in functions. + +| Benchmark | 1K | 10K | +|---|---|---| +| `aggregate_sum_dispatch` (UDF sum) | ~2-3 ms | ~20-30 ms | +| `predicate_filter_dispatch` (UDF predicate) | ~2-3 ms | ~20-30 ms | + +UDF dispatch adds ~1 function pointer indirection per aggregation step or predicate evaluation. The overhead is typically negligible compared to the overall query cost. + +--- + +## Query: Regex Filter + +Measures regex evaluation overhead via the `matches?` predicate. Regexes are precompiled at parse time. + +| Benchmark | 1K | 10K | +|---|---|---| +| `regex_filter` (matches? with pattern) | ~3-5 ms | ~30-50 ms | +| `count_distinct_scale` (50% duplicates) | ~3-5 ms | ~30-50 ms | + +--- + ## Concurrent B+Tree Range Scans (Phase 6.5, new) Measures wall-clock latency of N simultaneous EAVT range scans against a committed B+tree with 10K facts. diff --git a/benches/minigraf_bench.rs b/benches/minigraf_bench.rs index 58c47b869a..e640c25e97 100644 --- a/benches/minigraf_bench.rs +++ b/benches/minigraf_bench.rs @@ -1126,7 +1126,7 @@ fn bench_temporal_metadata(c: &mut Criterion) { let db = helpers::populate_in_memory(n); group.bench_with_input(BenchmarkId::from_parameter(label), &n, |b, _| { b.iter(|| { - db.execute("(query [:find ?e ?t :where [?e :val ?v :tx-time ?t]])") + db.execute("(query [:find ?e ?t :any-valid-time :where [?e :val ?v] [?e :db/tx-count ?t]])") .unwrap() }); }); @@ -1141,7 +1141,7 @@ fn bench_temporal_metadata(c: &mut Criterion) { let db = helpers::populate_in_memory(n); group.bench_with_input(BenchmarkId::from_parameter(label), &n, |b, _| { b.iter(|| { - db.execute("(query [:find ?e ?vf :where [?e :val ?v :valid-from ?vf]])") + db.execute("(query [:find ?e ?vf :any-valid-time :where [?e :val ?v] [?e :db/valid-from ?vf]])") .unwrap() }); }); @@ -1156,7 +1156,7 @@ fn bench_temporal_metadata(c: &mut Criterion) { let db = helpers::populate_in_memory(n); group.bench_with_input(BenchmarkId::from_parameter(label), &n, |b, _| { b.iter(|| { - db.execute("(query [:find ?e ?vt :where [?e :val ?v :valid-to ?vt]])") + db.execute("(query [:find ?e ?vt :any-valid-time :where [?e :val ?v] [?e :db/valid-to ?vt]])") .unwrap() }); });