From 387c83625d43934b331277fb769a67c47f23d65c Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 4 Nov 2026 12:17:42 +0530 Subject: [PATCH 1/2] ci: mirror isolated acceptance verification on v3 --- .github/workflows/smoke.yml | 44 +++++++++++++++++++------- docs/benchmarks/837-acceptance-gaps.md | 18 +++++++++-- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index e9ff6819f7..221bea3e4a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -109,13 +109,6 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ needs.source.outputs.sha }} - - name: Checkout the verifier from the workflow revision - uses: actions/checkout@v4 - with: - ref: ${{ github.workflow_sha }} - path: .acceptance-workflow - - name: Test the acceptance evidence verifier - run: python3 -m unittest discover -s .acceptance-workflow/.github/acceptance -v - name: Install Rust stable uses: dtolnay/rust-toolchain@stable - name: Run acceptance and retain source/test evidence @@ -130,10 +123,6 @@ jobs: echo "Command: $ACCEPTANCE_COMMAND" $ACCEPTANCE_COMMAND } 2>&1 | tee acceptance.txt - - name: Require the complete test inventory and measured RSS checks - env: - ACCEPTANCE_SUITE: ${{ matrix.name }} - run: python3 .acceptance-workflow/.github/acceptance/verify.py "$ACCEPTANCE_SUITE" acceptance.txt - name: Retain acceptance output including executed test names and RSS/state if: always() uses: actions/upload-artifact@v6 @@ -144,3 +133,36 @@ jobs: Cargo.lock if-no-files-found: error retention-days: 30 + + # Never execute the verifier in the selected source's workspace: that code + # can modify any file there. This job has a fresh runner, checks out only the + # workflow revision, and treats artifacts from this run as data, not code. + verify-acceptance: + name: Verify v3 acceptance evidence + needs: [source, acceptance] + if: ${{ !cancelled() && needs.source.outputs.acceptance == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require every acceptance job to succeed + env: + ACCEPTANCE_RESULT: ${{ needs.acceptance.result }} + run: test "$ACCEPTANCE_RESULT" = success + - name: Checkout the verifier from the workflow revision + uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + - name: Test the acceptance evidence verifier + run: python3 -I -m unittest discover -s .github/acceptance -v + - name: Download acceptance output from this run + uses: actions/download-artifact@v7 + with: + pattern: acceptance-* + path: acceptance-artifacts + merge-multiple: false + - name: Require the complete test inventory and measured RSS checks + run: | + for suite in recursive memory-limit limit large-transaction size-per-fact; do + python3 -I .github/acceptance/verify.py "$suite" "acceptance-artifacts/acceptance-$suite/acceptance.txt" + done diff --git a/docs/benchmarks/837-acceptance-gaps.md b/docs/benchmarks/837-acceptance-gaps.md index 00d881e470..4d34389f2e 100644 --- a/docs/benchmarks/837-acceptance-gaps.md +++ b/docs/benchmarks/837-acceptance-gaps.md @@ -37,8 +37,11 @@ No production behavior, dependency, file format, public API, or release changes. RSS records across five jobs, plus one fully passing harness summary per job. Zero tests, missing/renamed/duplicated tests, ignored tests, incomplete output and missing RSS evidence fail. Eight Python regressions cover these rejection paths. -The verifier comes from a separate checkout pinned to `github.workflow_sha`, -while the tests use the source SHA resolved by the source job. Recursive and +The verifier runs in a separate job on a fresh runner, checking out only +`github.workflow_sha`. It downloads this run's acceptance artifacts into a data +directory and reads their text; it never checks out or executes the measured +source. Verification also requires every acceptance job to have succeeded. +The tests use the source SHA resolved by the source job. Recursive and transaction tests now run serially; memory tests already did. Each artifact keeps `Cargo.lock` alongside complete output for dependency replay. @@ -114,3 +117,14 @@ does not imply zero allocations. These fixtures establish their stated bounds at the listed scales; they do not prove every shape, platform or asymptotic bound. Accounting remains enabled in both C24 latency controls. No Bencher memory metric or accounting-disabled measurement is claimed. + +## Verifier isolation follow-up + +The first dispatch above ran the pinned verifier in the test runner's workspace. +The follow-up moves it to a fresh job so code under test cannot overwrite the +verifier or its Python environment. `cache-mode: none` remains in force. The +job rejects any failed acceptance job, missing artifact or invalid test/RSS +inventory; artifacts from this run are data only. All eight verifier regressions +pass, and the actual job command passes the five retained outputs while rejecting +missing output and Python text posing as output. A fresh dispatch validates the +new job boundary; its result is recorded in the follow-up PR. From 8efc6bd7132908ed00e2982fdd2531b6fa3f34df Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 4 Nov 2026 12:23:51 +0530 Subject: [PATCH 2/2] docs: record isolated acceptance verifier validation --- docs/benchmarks/837-acceptance-gaps.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks/837-acceptance-gaps.md b/docs/benchmarks/837-acceptance-gaps.md index 4d34389f2e..19306ab399 100644 --- a/docs/benchmarks/837-acceptance-gaps.md +++ b/docs/benchmarks/837-acceptance-gaps.md @@ -39,8 +39,8 @@ Zero tests, missing/renamed/duplicated tests, ignored tests, incomplete output a missing RSS evidence fail. Eight Python regressions cover these rejection paths. The verifier runs in a separate job on a fresh runner, checking out only `github.workflow_sha`. It downloads this run's acceptance artifacts into a data -directory and reads their text; it never checks out or executes the measured -source. Verification also requires every acceptance job to have succeeded. +directory and reads their text; the selected source ref cannot select verifier +code. Verification also requires every acceptance job to have succeeded. The tests use the source SHA resolved by the source job. Recursive and transaction tests now run serially; memory tests already did. Each artifact keeps `Cargo.lock` alongside complete output for dependency replay. @@ -126,5 +126,15 @@ verifier or its Python environment. `cache-mode: none` remains in force. The job rejects any failed acceptance job, missing artifact or invalid test/RSS inventory; artifacts from this run are data only. All eight verifier regressions pass, and the actual job command passes the five retained outputs while rejecting -missing output and Python text posing as output. A fresh dispatch validates the -new job boundary; its result is recorded in the follow-up PR. +missing output and Python text posing as output. + +[Follow-up run 41788127874](https://github.com/project-minigraf/minigraf/actions/runs/41788127874) +passed all five acceptance suites and the long-haul smoke suite, using workflow +`66962f56ddc75303eebe247159d915afa4420a23` and source +`a2ac9545be7ee9bb62ca2d50ccd7b8bd6241ee3f`. The +[fresh verifier job](https://github.com/project-minigraf/minigraf/actions/runs/41788127874/job/126436026648) +passed its eight regressions and verified all five artifacts: ten passing tests +and nine measured RSS records. The workflow is byte-identical in the main and v3 +follow-up PRs (#849 and #850). CodeQL reported no new alerts on #849 at the +workflow revision above. These checks validate the new job boundary; the retained +raw measurements earlier in this report still describe the original dispatch.