Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/acceptance/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
__pycache__/
51 changes: 51 additions & 0 deletions .github/acceptance/test_verify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""Regression checks for false-green acceptance output (no third-party modules)."""
import unittest

from verify import verify


class AcceptanceOutputTests(unittest.TestCase):
def output(self, names, passed=None, ignored=0):
body = ''.join(f'test {name} ... measured output\nok\n' for name in names)
n = len(names) if passed is None else passed
return body + f'test result: ok. {n} passed; 0 failed; {ignored} ignored; 0 measured; 0 filtered out; finished in 1.00s\n'

def test_all_named_tests_pass(self):
verify(self.output(['a', 'b']), ['a', 'b'], [])

def test_zero_tests_rejected(self):
with self.assertRaises(ValueError):
verify(self.output([]), ['a'], [])

def test_missing_or_renamed_test_rejected(self):
for names in (['a'], ['a', 'renamed'], ['a', 'a']):
with self.subTest(names=names), self.assertRaises(ValueError):
verify(self.output(names), ['a', 'b'], [])

def test_ignored_or_failed_test_rejected(self):
for output in (self.output(['a'], passed=0, ignored=1), self.output(['a']).replace('ok. 1 passed; 0 failed', 'FAILED. 0 passed; 1 failed')):
with self.subTest(output=output), self.assertRaises(ValueError):
verify(output, ['a'], [])

def test_truncated_or_multiple_summaries_rejected(self):
for output in ('test a ... ok\n', self.output(['a']) * 2):
with self.subTest(output=output), self.assertRaises(ValueError):
verify(output, ['a'], [])

def test_required_rss_check_cannot_skip(self):
for marker in ('', 'RSS_CHECK probe skipped platform=macos\n'):
with self.subTest(marker=marker), self.assertRaises(ValueError):
verify(self.output(['a']) + marker, ['a'], ['probe'])

def test_measured_rss_check_accepted(self):
verify(self.output(['a']) + 'RSS_CHECK probe measured added_bytes=32 bound_bytes=64\n', ['a'], ['probe'])

def test_storage_test_full_name_is_required(self):
name = 'storage::persistent_facts::tests::size_per_fact_in_433_shape_1m'
verify(self.output([name]), [name], [])
with self.assertRaises(ValueError):
verify(self.output(['unrelated']), [name], [])


if __name__ == '__main__':
unittest.main()
45 changes: 45 additions & 0 deletions .github/acceptance/verify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
"""Reject empty/partial acceptance runs; invoked from the pinned workflow checkout."""
import argparse
from pathlib import Path
import re

SUITES = {
'recursive': (
['reach_5m', 'nonlinear_closure_scaling', 'multistratum_recursion_scaling'],
['reach_5m', 'nonlinear_256', 'nonlinear_512', 'multistratum_256', 'multistratum_512'],
),
'memory-limit': (
['cartesian_k13_streams', 'count_holds_no_rows', 'grouped_count_1m_groups'],
['cartesian_k13_streams', 'count_holds_no_rows'],
),
'limit': (['limit_and_top_k_over_10m_facts'], ['limit_10m', 'top_k_10m']),
'large-transaction': (['one_million_fact_transact_holds_one_chunk', 'custom_chunk_bounds'], []),
'size-per-fact': (['storage::persistent_facts::tests::size_per_fact_in_433_shape_1m'], []),
}


def verify(output, expected_tests, expected_rss):
output = re.sub(r'\x1b\[[0-9;]*m', '', output)
names = re.findall(r'^test (\S+) \.\.\.', output, re.MULTILINE)
if not expected_tests or sorted(names) != sorted(expected_tests):
raise ValueError(f'executed tests {names}, expected {expected_tests}')
summaries = re.findall(r'^test result: (.+)$', output, re.MULTILINE)
prefix = f'ok. {len(expected_tests)} passed; 0 failed; 0 ignored;'
if len(summaries) != 1 or not summaries[0].startswith(prefix):
raise ValueError(f'expected one complete passing summary, got {summaries}')
# With --nocapture the first record can follow "test name ... " on a line.
measured = re.findall(r'RSS_CHECK (\S+) measured added_bytes=\d+ bound_bytes=\d+', output)
if sorted(measured) != sorted(expected_rss):
raise ValueError(f'measured RSS checks {measured}, expected {expected_rss}')


if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('suite', choices=SUITES)
parser.add_argument('output', type=Path)
args = parser.parse_args()
try:
verify(args.output.read_text(), *SUITES[args.suite])
except ValueError as error:
parser.exit(1, f'Acceptance evidence rejected: {error}\n')
print(f'Acceptance evidence verified: {args.suite}')
3 changes: 3 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ jobs:
with:
components: rustfmt, clippy

- name: Test acceptance evidence verification
run: python3 -m unittest discover -s .github/acceptance -v

- name: Check formatting
run: cargo fmt -- --check

Expand Down
43 changes: 39 additions & 4 deletions .github/workflows/smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ jobs:

# Separate from Bencher latency: these existing tests assert fixed retained
# state/RSS bounds (and QRY-012 exhaustion), rather than uploading RSS as time.
# Runtime within the existing 15/30/45-minute budgets remains unverified.
# Exact test inventories and RSS records are checked after each command.
acceptance:
name: v3 acceptance / ${{ matrix.name }}
needs: source
Expand All @@ -92,7 +92,7 @@ jobs:
include:
- name: recursive
timeout: 45
command: cargo test --profile bench --test recursive_acceptance_test -- --ignored --nocapture
command: cargo test --profile bench --test recursive_acceptance_test -- --ignored --nocapture --test-threads=1
- name: memory-limit
timeout: 30
command: cargo test --profile bench --test memory_limit_acceptance_test -- --ignored --nocapture --test-threads=1
Expand All @@ -101,7 +101,7 @@ jobs:
command: cargo test --profile bench --test limit_acceptance_test -- --ignored --nocapture
- name: large-transaction
timeout: 30
command: cargo test --profile bench --features test-hooks --test large_transaction_acceptance_test -- --ignored --nocapture
command: cargo test --profile bench --features test-hooks --test large_transaction_acceptance_test -- --ignored --nocapture --test-threads=1
- name: size-per-fact
timeout: 15
command: cargo test --lib size_per_fact_in_433_shape_1m -- --include-ignored --nocapture
Expand All @@ -128,6 +128,41 @@ jobs:
uses: actions/upload-artifact@v6
with:
name: acceptance-${{ matrix.name }}
path: acceptance.txt
path: |
acceptance.txt
Cargo.lock
if-no-files-found: error
retention-days: 30

# Never execute the verifier in the selected source's workspace: that code
# can modify any file there. This job has a fresh runner, checks out only the
# workflow revision, and treats artifacts from this run as data, not code.
verify-acceptance:
name: Verify v3 acceptance evidence
needs: [source, acceptance]
if: ${{ !cancelled() && needs.source.outputs.acceptance == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Require every acceptance job to succeed
env:
ACCEPTANCE_RESULT: ${{ needs.acceptance.result }}
run: test "$ACCEPTANCE_RESULT" = success
- name: Checkout the verifier from the workflow revision
uses: actions/checkout@v4
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
- name: Test the acceptance evidence verifier
run: python3 -I -m unittest discover -s .github/acceptance -v
- name: Download acceptance output from this run
uses: actions/download-artifact@v7
with:
pattern: acceptance-*
path: acceptance-artifacts
merge-multiple: false
- name: Require the complete test inventory and measured RSS checks
run: |
for suite in recursive memory-limit limit large-transaction size-per-fact; do
python3 -I .github/acceptance/verify.py "$suite" "acceptance-artifacts/acceptance-$suite/acceptance.txt"
done
Loading