Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/acceptance/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
__pycache__/
51 changes: 51 additions & 0 deletions .github/acceptance/test_verify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""Regression checks for false-green acceptance output (no third-party modules)."""
import unittest

from verify import verify


class AcceptanceOutputTests(unittest.TestCase):
def output(self, names, passed=None, ignored=0):
body = ''.join(f'test {name} ... measured output\nok\n' for name in names)
n = len(names) if passed is None else passed
return body + f'test result: ok. {n} passed; 0 failed; {ignored} ignored; 0 measured; 0 filtered out; finished in 1.00s\n'

def test_all_named_tests_pass(self):
verify(self.output(['a', 'b']), ['a', 'b'], [])

def test_zero_tests_rejected(self):
with self.assertRaises(ValueError):
verify(self.output([]), ['a'], [])

def test_missing_or_renamed_test_rejected(self):
for names in (['a'], ['a', 'renamed'], ['a', 'a']):
with self.subTest(names=names), self.assertRaises(ValueError):
verify(self.output(names), ['a', 'b'], [])

def test_ignored_or_failed_test_rejected(self):
for output in (self.output(['a'], passed=0, ignored=1), self.output(['a']).replace('ok. 1 passed; 0 failed', 'FAILED. 0 passed; 1 failed')):
with self.subTest(output=output), self.assertRaises(ValueError):
verify(output, ['a'], [])

def test_truncated_or_multiple_summaries_rejected(self):
for output in ('test a ... ok\n', self.output(['a']) * 2):
with self.subTest(output=output), self.assertRaises(ValueError):
verify(output, ['a'], [])

def test_required_rss_check_cannot_skip(self):
for marker in ('', 'RSS_CHECK probe skipped platform=macos\n'):
with self.subTest(marker=marker), self.assertRaises(ValueError):
verify(self.output(['a']) + marker, ['a'], ['probe'])

def test_measured_rss_check_accepted(self):
verify(self.output(['a']) + 'RSS_CHECK probe measured added_bytes=32 bound_bytes=64\n', ['a'], ['probe'])

def test_storage_test_full_name_is_required(self):
name = 'storage::persistent_facts::tests::size_per_fact_in_433_shape_1m'
verify(self.output([name]), [name], [])
with self.assertRaises(ValueError):
verify(self.output(['unrelated']), [name], [])


if __name__ == '__main__':
unittest.main()
45 changes: 45 additions & 0 deletions .github/acceptance/verify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
"""Reject empty/partial acceptance runs; invoked from the pinned workflow checkout."""
import argparse
from pathlib import Path
import re

SUITES = {
'recursive': (
['reach_5m', 'nonlinear_closure_scaling', 'multistratum_recursion_scaling'],
['reach_5m', 'nonlinear_256', 'nonlinear_512', 'multistratum_256', 'multistratum_512'],
),
'memory-limit': (
['cartesian_k13_streams', 'count_holds_no_rows', 'grouped_count_1m_groups'],
['cartesian_k13_streams', 'count_holds_no_rows'],
),
'limit': (['limit_and_top_k_over_10m_facts'], ['limit_10m', 'top_k_10m']),
'large-transaction': (['one_million_fact_transact_holds_one_chunk', 'custom_chunk_bounds'], []),
'size-per-fact': (['storage::persistent_facts::tests::size_per_fact_in_433_shape_1m'], []),
}


def verify(output, expected_tests, expected_rss):
output = re.sub(r'\x1b\[[0-9;]*m', '', output)
names = re.findall(r'^test (\S+) \.\.\.', output, re.MULTILINE)
if not expected_tests or sorted(names) != sorted(expected_tests):
raise ValueError(f'executed tests {names}, expected {expected_tests}')
summaries = re.findall(r'^test result: (.+)$', output, re.MULTILINE)
prefix = f'ok. {len(expected_tests)} passed; 0 failed; 0 ignored;'
if len(summaries) != 1 or not summaries[0].startswith(prefix):
raise ValueError(f'expected one complete passing summary, got {summaries}')
# With --nocapture the first record can follow "test name ... " on a line.
measured = re.findall(r'RSS_CHECK (\S+) measured added_bytes=\d+ bound_bytes=\d+', output)
if sorted(measured) != sorted(expected_rss):
raise ValueError(f'measured RSS checks {measured}, expected {expected_rss}')


if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('suite', choices=SUITES)
parser.add_argument('output', type=Path)
args = parser.parse_args()
try:
verify(args.output.read_text(), *SUITES[args.suite])
except ValueError as error:
parser.exit(1, f'Acceptance evidence rejected: {error}\n')
print(f'Acceptance evidence verified: {args.suite}')
3 changes: 3 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ jobs:
with:
components: rustfmt, clippy

- name: Test acceptance evidence verification
run: python3 -m unittest discover -s .github/acceptance -v

- name: Check formatting
run: cargo fmt -- --check

Expand Down
21 changes: 17 additions & 4 deletions .github/workflows/smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ jobs:

# Separate from Bencher latency: these existing tests assert fixed retained
# state/RSS bounds (and QRY-012 exhaustion), rather than uploading RSS as time.
# Runtime within the existing 15/30/45-minute budgets remains unverified.
# Exact test inventories and RSS records are checked after each command.
acceptance:
name: v3 acceptance / ${{ matrix.name }}
needs: source
Expand All @@ -92,7 +92,7 @@ jobs:
include:
- name: recursive
timeout: 45
command: cargo test --profile bench --test recursive_acceptance_test -- --ignored --nocapture
command: cargo test --profile bench --test recursive_acceptance_test -- --ignored --nocapture --test-threads=1
- name: memory-limit
timeout: 30
command: cargo test --profile bench --test memory_limit_acceptance_test -- --ignored --nocapture --test-threads=1
Expand All @@ -101,14 +101,21 @@ jobs:
command: cargo test --profile bench --test limit_acceptance_test -- --ignored --nocapture
- name: large-transaction
timeout: 30
command: cargo test --profile bench --features test-hooks --test large_transaction_acceptance_test -- --ignored --nocapture
command: cargo test --profile bench --features test-hooks --test large_transaction_acceptance_test -- --ignored --nocapture --test-threads=1
- name: size-per-fact
timeout: 15
command: cargo test --lib size_per_fact_in_433_shape_1m -- --include-ignored --nocapture
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.source.outputs.sha }}
- name: Checkout the verifier from the workflow revision
uses: actions/checkout@v4
with:
ref: ${{ github.workflow_sha }}
path: .acceptance-workflow
- name: Test the acceptance evidence verifier
run: python3 -m unittest discover -s .acceptance-workflow/.github/acceptance -v
- name: Install Rust stable
uses: dtolnay/rust-toolchain@stable
- name: Run acceptance and retain source/test evidence
Expand All @@ -123,11 +130,17 @@ jobs:
echo "Command: $ACCEPTANCE_COMMAND"
$ACCEPTANCE_COMMAND
} 2>&1 | tee acceptance.txt
- name: Require the complete test inventory and measured RSS checks
env:
ACCEPTANCE_SUITE: ${{ matrix.name }}
run: python3 .acceptance-workflow/.github/acceptance/verify.py "$ACCEPTANCE_SUITE" acceptance.txt
- name: Retain acceptance output including executed test names and RSS/state
if: always()
uses: actions/upload-artifact@v6
with:
name: acceptance-${{ matrix.name }}
path: acceptance.txt
path: |
acceptance.txt
Cargo.lock
if-no-files-found: error
retention-days: 30
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
unchanged. INT-041/043/044 are deprecated for these built-ins. No v2 runtime
backport, window behavior, ordered-group eligibility, or storage change.

### Testing

- **Acceptance execution gaps** (#837): nonlinear and multistratum recursive
scaling, custom transaction chunk/clamp bounds, mandatory Linux RSS assertions,
and exact named-test/output verification in the nightly smoke route. The storage
bound now compares exact bytes rather than a truncated ratio. No engine, public
API or format change. See [bounds and validation](docs/benchmarks/837-acceptance-gaps.md).

### Documentation

- **Benchmark exclusions audit** (#836): [pinned enumeration and per-shape dispositions](docs/benchmarks/836-exclusions/README.md) reconcile 156 unselected native IDs, retain 110 manual row-frame cases, and distinguish linear full-prefix UDF accumulation from bounded replay. No new latency measurements or runtime changes.
Expand Down
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ Auto-migrates v7 → v8 on open (spec §9, with a backup meta page). v1–v6 are

## Test Coverage

**2058 tests** (2039 active, 19 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551).
**2062 tests** (2041 active, 21 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551).
See `docs/TEST_COVERAGE.md` for the full per-file breakdown.

The #760 deterministic storage suite adds three native unit tests in a separate `--cfg minigraf_shuttle` build, outside the default count. Run only `graph::storage::deterministic::` with that cfg; ordinary unit tests do not enter its scheduler. `deterministic-schedules.yml` runs relevant v3 PRs on Linux/macOS/Windows plus Linux MSRV, and larger nightly/manual searches with replay artifacts. The identical main workflow activates the v3 schedule after both PRs merge. See `docs/testing/deterministic-schedules.md` for scope, controls and findings.
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ For example, `(lag ?salary :over (:partition-by ?dept :order-by ?hire-date))` re

The unreleased v3.0.0 branch supports ordinary `(avg ?score)` (#777). It skips nil and nonnumeric values, counts only numbers, and returns a Float average or nil for an existing group without numbers. No matching rows produces no result row. It uses the same exact-total-then-divide arithmetic as window `avg`; a finite total can overflow before division. V2 requires `:over` for `avg`.

The unreleased v3.0.0 branch adds bounded sliding windows such as `(avg ?salary :over (:partition-by ?dept :order-by ?hire-date :rows 2 preceding))` (#183). See the [benchmark coverage and manual-case dispositions](docs/benchmarks/836-exclusions/README.md) for the limits of nightly coverage (#836).
The unreleased v3.0.0 branch adds bounded sliding windows such as `(avg ?salary :over (:partition-by ?dept :order-by ?hire-date :rows 2 preceding))` (#183). See the [benchmark coverage and manual-case dispositions](docs/benchmarks/836-exclusions/README.md) for the limits of nightly coverage (#836). Memory/scaling acceptance is checked separately from latency; see the [acceptance bounds and validation status](docs/benchmarks/837-acceptance-gaps.md) (#837).

On v3.0.0, an ascending value-only query such as `[:find ?v :where [?e :score ?v] :order-by [?v] :limit 10]` can read directly in index order and stop early when all committed `:score` history has one value kind (integer, float, or Boolean) and its source range has no pending writes. A query whose full order cannot be proved uses the existing sort (#641). Built-in `count`, `count-distinct`, `sum`, and `avg` can release completed groups when the pinned source proves their grouping order (#781). `min`/`max` also qualify when their arguments are proved Integer/Float/Boolean/nil; `sum-distinct` requires Integer/nil. A single Integer/Boolean/Float key (including mixed numeric groups) is supported. A longer effective tuple, including hidden `:with` keys, must reduce to one varying Integer/Boolean slot plus repeated occurrences and proved Integer/Boolean/nil constants. Any requested final Sort is retained for `:with`. Small grouped point queries pay proof startup cost; see the [measurements](docs/benchmarks/781-ordered-aggregate.md). Memory follows the active group and batches; distinct members within one group still grow. A limit can stop after enough emitted groups, but dropped min/max groups do not count and a retained Sort drains the input. Callbacks, `collect-set`, aggregate+window queries and unproved arguments keep full-input evaluation. When every built-in window uses the same proved nonconstant partition key, the engine can also finish and release one partition at a time. Its memory follows the largest partition plus bounded batches; one growing partition still grows. Unpartitioned windows, incompatible keys, aggregate queries, callbacks and `collect-set` keep full-input evaluation. A surrounding sort or distinct operation can still require more memory.

Expand Down
1 change: 1 addition & 0 deletions ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ Scope — release-gate testing and documentation:
- Fault injection across save/WAL/recovery (#390)
- Long-running soak test at scale (#392; done on `v3`: `tests/soak_test.rs`, 10M facts and a 5-hour churn weekly, alternating `main` and `v3`)
- Benchmarks at 100K–10M facts and a published performance envelope (#394; on `v3`: `examples/perf_envelope.rs`, results in `docs/BENCHMARKS.md`; follow-ups #528–#532, with #532 superseded by #551's removal of repeated committed-fact net-assert; [audit and measurements](docs/benchmarks/532-net-assert-audit.md))
- Acceptance evidence for memory/scaling (#837): named-test and Linux RSS gates, nonlinear/multistratum recursion and custom chunk bounds; [validation status](docs/benchmarks/837-acceptance-gaps.md)
- Expression keyword/ref equality (#774; on `v3`): resolve literal keywords once per operator, with memory bounded by the expression. [Measurements and regression coverage](docs/benchmarks/774-expression-keywords.md); performance-only, with no v2.x backport.
- Higher storage/WAL coverage gates and branch-coverage gates (#396; on `v3`: line and branch floors per file in `coverage-thresholds.toml`, checked on every PR)
- Durability, recovery and operations guide (#398; done on `v3`: `docs/DURABILITY.md`, also on the docs site; found #535)
Expand Down
Loading
Loading