From 471f3ab75273419ace2c100d1b6353a9f28e6f08 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 13 Oct 2026 10:40:34 +0530 Subject: [PATCH] ci(bench): compare each run with the last 30 runs, not all history Bencher's t-test used every sample since May, so a correct step change kept alerting for weeks: query/predicate_pushdown/100k since #445 (#512, #495), and checkpoint/* and open/wal_replay/100k since #476's backport stopped the drop-save that had made those fixtures no-ops (#561, #562). Replaying main's history, a 30-run window still alerts on two runs in a row for a lasting change, then takes it as the baseline after about five runs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016QAt9R1aSPQG7rV5ouFDMU --- .github/workflows/bench.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6a46e9120..d3482f22e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -245,6 +245,12 @@ jobs: continue-on-error: true run: | set -o pipefail + # The t-test compares a run with the last 30 runs of its branch, not + # all history. Without a window, a correct step change (#445's + # predicate_pushdown, #476's checkpoint and wal_replay fixtures) kept + # alerting for weeks; with 30, a lasting change alerts on about five + # runs (enough for the two-in-a-row issue below), then becomes the + # baseline. # A branch other than main starts from main's history and thresholds. start_point=() if [ "$BENCH_REF" != main ]; then @@ -258,6 +264,7 @@ jobs: --testbed ubuntu-latest \ --threshold-measure latency \ --threshold-test t_test \ + --threshold-max-sample-size 30 \ --threshold-upper-boundary ${{ matrix.threshold }} \ --err \ --adapter rust_criterion \