From c1fafeeab07231f5112dce5d92021f885f1834af Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 4 Nov 2026 11:14:40 +0530 Subject: [PATCH 1/2] ci: require named v3 acceptance tests and RSS evidence (#837) --- .github/acceptance/.gitignore | 1 + .github/acceptance/test_verify.py | 51 +++++++++++++++++++++++++++++++ .github/acceptance/verify.py | 45 +++++++++++++++++++++++++++ .github/workflows/lint.yml | 3 ++ .github/workflows/smoke.yml | 21 ++++++++++--- 5 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 .github/acceptance/.gitignore create mode 100644 .github/acceptance/test_verify.py create mode 100644 .github/acceptance/verify.py diff --git a/.github/acceptance/.gitignore b/.github/acceptance/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/.github/acceptance/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/.github/acceptance/test_verify.py b/.github/acceptance/test_verify.py new file mode 100644 index 0000000000..063a7524dc --- /dev/null +++ b/.github/acceptance/test_verify.py @@ -0,0 +1,51 @@ +"""Regression checks for false-green acceptance output (no third-party modules).""" +import unittest + +from verify import verify + + +class AcceptanceOutputTests(unittest.TestCase): + def output(self, names, passed=None, ignored=0): + body = ''.join(f'test {name} ... measured output\nok\n' for name in names) + n = len(names) if passed is None else passed + return body + f'test result: ok. {n} passed; 0 failed; {ignored} ignored; 0 measured; 0 filtered out; finished in 1.00s\n' + + def test_all_named_tests_pass(self): + verify(self.output(['a', 'b']), ['a', 'b'], []) + + def test_zero_tests_rejected(self): + with self.assertRaises(ValueError): + verify(self.output([]), ['a'], []) + + def test_missing_or_renamed_test_rejected(self): + for names in (['a'], ['a', 'renamed'], ['a', 'a']): + with self.subTest(names=names), self.assertRaises(ValueError): + verify(self.output(names), ['a', 'b'], []) + + def test_ignored_or_failed_test_rejected(self): + for output in (self.output(['a'], passed=0, ignored=1), self.output(['a']).replace('ok. 1 passed; 0 failed', 'FAILED. 0 passed; 1 failed')): + with self.subTest(output=output), self.assertRaises(ValueError): + verify(output, ['a'], []) + + def test_truncated_or_multiple_summaries_rejected(self): + for output in ('test a ... ok\n', self.output(['a']) * 2): + with self.subTest(output=output), self.assertRaises(ValueError): + verify(output, ['a'], []) + + def test_required_rss_check_cannot_skip(self): + for marker in ('', 'RSS_CHECK probe skipped platform=macos\n'): + with self.subTest(marker=marker), self.assertRaises(ValueError): + verify(self.output(['a']) + marker, ['a'], ['probe']) + + def test_measured_rss_check_accepted(self): + verify(self.output(['a']) + 'RSS_CHECK probe measured added_bytes=32 bound_bytes=64\n', ['a'], ['probe']) + + def test_storage_test_full_name_is_required(self): + name = 'storage::persistent_facts::tests::size_per_fact_in_433_shape_1m' + verify(self.output([name]), [name], []) + with self.assertRaises(ValueError): + verify(self.output(['unrelated']), [name], []) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/acceptance/verify.py b/.github/acceptance/verify.py new file mode 100644 index 0000000000..a580044a72 --- /dev/null +++ b/.github/acceptance/verify.py @@ -0,0 +1,45 @@ +"""Reject empty/partial acceptance runs; invoked from the pinned workflow checkout.""" +import argparse +from pathlib import Path +import re + +SUITES = { + 'recursive': ( + ['reach_5m', 'nonlinear_closure_scaling', 'multistratum_recursion_scaling'], + ['reach_5m', 'nonlinear_256', 'nonlinear_512', 'multistratum_256', 'multistratum_512'], + ), + 'memory-limit': ( + ['cartesian_k13_streams', 'count_holds_no_rows', 'grouped_count_1m_groups'], + ['cartesian_k13_streams', 'count_holds_no_rows'], + ), + 'limit': (['limit_and_top_k_over_10m_facts'], ['limit_10m', 'top_k_10m']), + 'large-transaction': (['one_million_fact_transact_holds_one_chunk', 'custom_chunk_bounds'], []), + 'size-per-fact': (['storage::persistent_facts::tests::size_per_fact_in_433_shape_1m'], []), +} + + +def verify(output, expected_tests, expected_rss): + output = re.sub(r'\x1b\[[0-9;]*m', '', output) + names = re.findall(r'^test (\S+) \.\.\.', output, re.MULTILINE) + if not expected_tests or sorted(names) != sorted(expected_tests): + raise ValueError(f'executed tests {names}, expected {expected_tests}') + summaries = re.findall(r'^test result: (.+)$', output, re.MULTILINE) + prefix = f'ok. {len(expected_tests)} passed; 0 failed; 0 ignored;' + if len(summaries) != 1 or not summaries[0].startswith(prefix): + raise ValueError(f'expected one complete passing summary, got {summaries}') + # With --nocapture the first record can follow "test name ... " on a line. + measured = re.findall(r'RSS_CHECK (\S+) measured added_bytes=\d+ bound_bytes=\d+', output) + if sorted(measured) != sorted(expected_rss): + raise ValueError(f'measured RSS checks {measured}, expected {expected_rss}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('suite', choices=SUITES) + parser.add_argument('output', type=Path) + args = parser.parse_args() + try: + verify(args.output.read_text(), *SUITES[args.suite]) + except ValueError as error: + parser.exit(1, f'Acceptance evidence rejected: {error}\n') + print(f'Acceptance evidence verified: {args.suite}') diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 38efd76711..27662da543 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,6 +26,9 @@ jobs: with: components: rustfmt, clippy + - name: Test acceptance evidence verification + run: python3 -m unittest discover -s .github/acceptance -v + - name: Check formatting run: cargo fmt -- --check diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 94038d9af4..e9ff6819f7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -79,7 +79,7 @@ jobs: # Separate from Bencher latency: these existing tests assert fixed retained # state/RSS bounds (and QRY-012 exhaustion), rather than uploading RSS as time. - # Runtime within the existing 15/30/45-minute budgets remains unverified. + # Exact test inventories and RSS records are checked after each command. acceptance: name: v3 acceptance / ${{ matrix.name }} needs: source @@ -92,7 +92,7 @@ jobs: include: - name: recursive timeout: 45 - command: cargo test --profile bench --test recursive_acceptance_test -- --ignored --nocapture + command: cargo test --profile bench --test recursive_acceptance_test -- --ignored --nocapture --test-threads=1 - name: memory-limit timeout: 30 command: cargo test --profile bench --test memory_limit_acceptance_test -- --ignored --nocapture --test-threads=1 @@ -101,7 +101,7 @@ jobs: command: cargo test --profile bench --test limit_acceptance_test -- --ignored --nocapture - name: large-transaction timeout: 30 - command: cargo test --profile bench --features test-hooks --test large_transaction_acceptance_test -- --ignored --nocapture + command: cargo test --profile bench --features test-hooks --test large_transaction_acceptance_test -- --ignored --nocapture --test-threads=1 - name: size-per-fact timeout: 15 command: cargo test --lib size_per_fact_in_433_shape_1m -- --include-ignored --nocapture @@ -109,6 +109,13 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ needs.source.outputs.sha }} + - name: Checkout the verifier from the workflow revision + uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + path: .acceptance-workflow + - name: Test the acceptance evidence verifier + run: python3 -m unittest discover -s .acceptance-workflow/.github/acceptance -v - name: Install Rust stable uses: dtolnay/rust-toolchain@stable - name: Run acceptance and retain source/test evidence @@ -123,11 +130,17 @@ jobs: echo "Command: $ACCEPTANCE_COMMAND" $ACCEPTANCE_COMMAND } 2>&1 | tee acceptance.txt + - name: Require the complete test inventory and measured RSS checks + env: + ACCEPTANCE_SUITE: ${{ matrix.name }} + run: python3 .acceptance-workflow/.github/acceptance/verify.py "$ACCEPTANCE_SUITE" acceptance.txt - name: Retain acceptance output including executed test names and RSS/state if: always() uses: actions/upload-artifact@v6 with: name: acceptance-${{ matrix.name }} - path: acceptance.txt + path: | + acceptance.txt + Cargo.lock if-no-files-found: error retention-days: 30 From 66962f56ddc75303eebe247159d915afa4420a23 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 4 Nov 2026 12:17:32 +0530 Subject: [PATCH 2/2] ci: isolate acceptance verification from tested source --- .github/workflows/smoke.yml | 44 +++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index e9ff6819f7..221bea3e4a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -109,13 +109,6 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ needs.source.outputs.sha }} - - name: Checkout the verifier from the workflow revision - uses: actions/checkout@v4 - with: - ref: ${{ github.workflow_sha }} - path: .acceptance-workflow - - name: Test the acceptance evidence verifier - run: python3 -m unittest discover -s .acceptance-workflow/.github/acceptance -v - name: Install Rust stable uses: dtolnay/rust-toolchain@stable - name: Run acceptance and retain source/test evidence @@ -130,10 +123,6 @@ jobs: echo "Command: $ACCEPTANCE_COMMAND" $ACCEPTANCE_COMMAND } 2>&1 | tee acceptance.txt - - name: Require the complete test inventory and measured RSS checks - env: - ACCEPTANCE_SUITE: ${{ matrix.name }} - run: python3 .acceptance-workflow/.github/acceptance/verify.py "$ACCEPTANCE_SUITE" acceptance.txt - name: Retain acceptance output including executed test names and RSS/state if: always() uses: actions/upload-artifact@v6 @@ -144,3 +133,36 @@ jobs: Cargo.lock if-no-files-found: error retention-days: 30 + + # Never execute the verifier in the selected source's workspace: that code + # can modify any file there. This job has a fresh runner, checks out only the + # workflow revision, and treats artifacts from this run as data, not code. + verify-acceptance: + name: Verify v3 acceptance evidence + needs: [source, acceptance] + if: ${{ !cancelled() && needs.source.outputs.acceptance == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require every acceptance job to succeed + env: + ACCEPTANCE_RESULT: ${{ needs.acceptance.result }} + run: test "$ACCEPTANCE_RESULT" = success + - name: Checkout the verifier from the workflow revision + uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + - name: Test the acceptance evidence verifier + run: python3 -I -m unittest discover -s .github/acceptance -v + - name: Download acceptance output from this run + uses: actions/download-artifact@v7 + with: + pattern: acceptance-* + path: acceptance-artifacts + merge-multiple: false + - name: Require the complete test inventory and measured RSS checks + run: | + for suite in recursive memory-limit limit large-transaction size-per-fact; do + python3 -I .github/acceptance/verify.py "$suite" "acceptance-artifacts/acceptance-$suite/acceptance.txt" + done