From 0346a4a1fa8284472e9cfc76ade07eeb33e184b7 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Fri, 10 Apr 2026 07:52:18 +0530 Subject: [PATCH] ci: raise Bencher threshold to 0.999 for concurrent benchmark groups Concurrent benchmarks measure mutex/thread scheduling latency, which is highly sensitive to shared CI runner load. At the standard 0.99 upper boundary the t-test fires on normal runner variance (as seen in issue #116 / run 24179274502) even when no code changed. Sequential benchmark groups keep the tighter 0.99 threshold. Applies to both bench_light.yml and bench.yml. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/bench.yml | 24 +++++++++++++++++++++++- .github/workflows/bench_light.yml | 15 ++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9f201500ab..bf8b304671 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -22,106 +22,128 @@ jobs: - name: insert filter: "^insert/" file: bench_insert.txt + threshold: "0.99" # ── Sequential: insert_file ───────────────────────────────────────── - name: insert_file filter: "^insert_file/" file: bench_insert_file.txt + threshold: "0.99" # ── Sequential: query (includes query_extras) ─────────────────────── - name: query filter: "^query/" file: bench_query.txt + threshold: "0.99" # ── Sequential: time_travel ───────────────────────────────────────── - name: time_travel filter: "^time_travel/" file: bench_time_travel.txt + threshold: "0.99" # ── Sequential: recursion ─────────────────────────────────────────── - name: recursion filter: "^recursion/" file: bench_recursion.txt + threshold: "0.99" # ── Sequential: open ──────────────────────────────────────────────── - name: open filter: "^open/" file: bench_open.txt + threshold: "0.99" # ── Sequential: checkpoint ────────────────────────────────────────── - name: checkpoint filter: "^checkpoint" file: bench_checkpoint.txt + threshold: "0.99" # ── Concurrent (in-memory): readers ──────────────────────────────── + # threshold=0.999: concurrent benchmarks are noisy on shared CI runners - name: concurrent/readers filter: "^concurrent/readers/" file: bench_concurrent_readers.txt + threshold: "0.999" # ── Concurrent (in-memory): readers_plus_writer ──────────────────── - name: concurrent/readers_plus_writer filter: "^concurrent/readers_plus_writer/" file: bench_concurrent_mixed.txt + threshold: "0.999" # ── Concurrent (in-memory): serialized_writers ───────────────────── - name: concurrent/serialized_writers filter: "^concurrent/serialized_writers/" file: bench_concurrent_writers.txt + threshold: "0.999" # ── Concurrent (file-backed): readers ────────────────────────────── - name: concurrent_file/readers filter: "^concurrent_file/readers/" file: bench_concurrent_file_readers.txt + threshold: "0.999" # ── Concurrent (file-backed): readers_plus_writer ────────────────── - name: concurrent_file/readers_plus_writer filter: "^concurrent_file/readers_plus_writer/" file: bench_concurrent_file_mixed.txt + threshold: "0.999" # ── Concurrent (file-backed): serialized_writers ─────────────────── - name: concurrent_file/serialized_writers filter: "^concurrent_file/serialized_writers/" file: bench_concurrent_file_writers.txt + threshold: "0.999" # ── Concurrent: B+tree scan ───────────────────────────────────────── - name: concurrent_btree_scan filter: "^concurrent_btree_scan" file: bench_concurrent_btree_scan.txt + threshold: "0.999" # ── Negation: not / not-join ──────────────────────────────────────── - name: negation filter: "^negation/" file: bench_negation.txt + threshold: "0.99" # ── Disjunction: or / or-join ─────────────────────────────────────── - name: disjunction filter: "^disjunction/" file: bench_disjunction.txt + threshold: "0.99" # ── Aggregation (includes aggregation_extras) ─────────────────────── - name: aggregation filter: "^aggregation/" file: bench_aggregation.txt + threshold: "0.99" # ── Expression filters / binding ──────────────────────────────────── - name: expr filter: "^expr/" file: bench_expr.txt + threshold: "0.99" # ── Window functions ──────────────────────────────────────────────── - name: window filter: "^window/" file: bench_window.txt + threshold: "0.99" # ── Temporal metadata ─────────────────────────────────────────────── - name: temporal_metadata filter: "^temporal_metadata/" file: bench_temporal_metadata.txt + threshold: "0.99" # ── User-defined functions ────────────────────────────────────────── - name: udf filter: "^udf/" file: bench_udf.txt + threshold: "0.99" steps: - name: Checkout @@ -160,7 +182,7 @@ jobs: --testbed ubuntu-latest \ --threshold-measure latency \ --threshold-test t_test \ - --threshold-upper-boundary 0.99 \ + --threshold-upper-boundary ${{ matrix.threshold }} \ --err \ --adapter rust_criterion \ --file ${{ matrix.file }} diff --git a/.github/workflows/bench_light.yml b/.github/workflows/bench_light.yml index 22a1522373..ae647493e8 100644 --- a/.github/workflows/bench_light.yml +++ b/.github/workflows/bench_light.yml @@ -22,63 +22,76 @@ jobs: - name: insert filter: "^insert" # matches insert/ and insert_file/ file: bench_insert.txt + threshold: "0.99" # ── Sequential: query ─────────────────────────────────────────────── - name: query filter: "^query/" file: bench_query.txt + threshold: "0.99" # ── Sequential: time_travel ───────────────────────────────────────── - name: time_travel filter: "^time_travel/" file: bench_time_travel.txt + threshold: "0.99" # ── Sequential: recursion ─────────────────────────────────────────── - name: recursion filter: "^recursion/" file: bench_recursion.txt + threshold: "0.99" # ── Sequential: open ──────────────────────────────────────────────── - name: open filter: "^open/" file: bench_open.txt + threshold: "0.99" # ── Sequential: checkpoint ────────────────────────────────────────── - name: checkpoint filter: "^checkpoint/" file: bench_checkpoint.txt + threshold: "0.99" # ── Concurrent (in-memory): readers ──────────────────────────────── # Note: "concurrent/readers/" (trailing slash) distinguishes this # group from concurrent/readers_plus_writer/. + # threshold=0.999: concurrent benchmarks are noisy on shared CI runners - name: concurrent/readers filter: "^concurrent/readers/" file: bench_concurrent_readers.txt + threshold: "0.999" # ── Concurrent (in-memory): readers_plus_writer ──────────────────── - name: concurrent/readers_plus_writer filter: "^concurrent/readers_plus_writer/" file: bench_concurrent_mixed.txt + threshold: "0.999" # ── Concurrent (in-memory): serialized_writers ───────────────────── - name: concurrent/serialized_writers filter: "^concurrent/serialized_writers/" file: bench_concurrent_writers.txt + threshold: "0.999" # ── Concurrent (file-backed): readers ────────────────────────────── - name: concurrent_file/readers filter: "^concurrent_file/readers/" file: bench_concurrent_file_readers.txt + threshold: "0.999" # ── Concurrent (file-backed): readers_plus_writer ────────────────── - name: concurrent_file/readers_plus_writer filter: "^concurrent_file/readers_plus_writer/" file: bench_concurrent_file_mixed.txt + threshold: "0.999" # ── Concurrent (file-backed): serialized_writers ─────────────────── - name: concurrent_file/serialized_writers filter: "^concurrent_file/serialized_writers/" file: bench_concurrent_file_writers.txt + threshold: "0.999" steps: - name: Checkout @@ -117,7 +130,7 @@ jobs: --testbed ubuntu-latest \ --threshold-measure latency \ --threshold-test t_test \ - --threshold-upper-boundary 0.99 \ + --threshold-upper-boundary ${{ matrix.threshold }} \ --err \ --adapter rust_criterion \ --file ${{ matrix.file }}