diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml new file mode 100644 index 0000000000..fcfa7309e3 --- /dev/null +++ b/.github/workflows/soak.yml @@ -0,0 +1,125 @@ +name: Soak test (weekly) + +# Two branches alternate weekly: main on Saturday, v3 on Sunday, both 01:00 UTC. +# Scheduled runs always use this file from the default branch, so the branch +# under test is chosen from the cron string that fired (as in bench.yml and +# crash-kill.yml). Keep the cron strings here and in SOAK_REF/run-name +# identical. +run-name: >- + Soak test (${{ github.event_name == 'workflow_dispatch' && inputs.ref + || (github.event.schedule == '0 1 * * 0' && 'v3' || 'main') }}) + +# #392: grows a file-backed database to 10M+ facts, then churns it (retracts, +# re-asserts, long hot-entity histories) for the rest of the budget, with +# thousands of checkpoints and reopens every few minutes. Samples are checked +# against a reference throughout; the metrics (file size, open and checkpoint +# time, RSS, p50/p99 query latency) are uploaded for the performance envelope +# (#394). The test prints its seed; MINIGRAF_SOAK_SEED replays the writes. + +on: + schedule: + - cron: "0 1 * * 6" # main + - cron: "0 1 * * 0" # v3 + workflow_dispatch: + inputs: + ref: + description: Branch to test + required: true + default: main + minutes: + description: Workload budget in minutes + required: true + default: "300" + +permissions: + contents: read + issues: write + +env: + SOAK_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || (github.event.schedule == '0 1 * * 0' && 'v3' || 'main') }} + +jobs: + soak: + name: Soak at 10M facts + runs-on: ubuntu-latest + timeout-minutes: 360 + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.SOAK_REF }} + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + + # The database grows to several GB on main (format v7); /mnt has the + # runner's spare disk. + - name: Prepare the data directory + run: | + sudo mkdir -p /mnt/soak + sudo chown "$USER" /mnt/soak + df -h / /mnt + + - name: Build + run: cargo test --profile bench --test soak_test --no-run + + - name: Run the soak test + shell: bash + env: + MINIGRAF_SOAK_MINUTES: ${{ github.event_name == 'workflow_dispatch' && inputs.minutes || '300' }} + MINIGRAF_SOAK_TARGET_FACTS: "10000000" + MINIGRAF_SOAK_METRICS: ${{ github.workspace }}/soak-metrics.jsonl + MINIGRAF_SOAK_DIR: /mnt/soak + RUST_BACKTRACE: 1 + run: | + set -o pipefail + cargo test --profile bench --test soak_test soak -- --ignored --exact --nocapture 2>&1 | tee soak-output.txt + + - name: Upload metrics + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: soak-${{ env.SOAK_REF }}-${{ github.run_id }} + path: | + soak-metrics.jsonl + soak-output.txt + retention-days: 90 + if-no-files-found: warn + + - name: Record soak failure + if: ${{ failure() }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TITLE: "Weekly soak failure (${{ env.SOAK_REF }})" + run: | + report_file="soak-failure.md" + { + echo "The weekly soak test failed on ${{ env.SOAK_REF }}." + echo + echo "- Run: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + echo "- Commit: $(git rev-parse HEAD)" + echo "- Reproduce: cargo test --profile bench --test soak_test soak -- --ignored --exact --nocapture" + echo " (MINIGRAF_SOAK_SEED to the printed seed replays the writes; see tests/soak_test.rs for the other variables)" + echo + echo "## Test output" + echo + echo '```text' + tail -n 300 soak-output.txt 2>/dev/null || echo "(no output: the run failed before the test started)" + echo '```' + } > "$report_file" + + existing_issue="$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --search "$ISSUE_TITLE in:title" \ + --json number,title \ + --limit 100 \ + --jq ".[] | select(.title == \"$ISSUE_TITLE\") | .number" \ + | head -n 1)" + + if [ -n "$existing_issue" ]; then + gh issue comment "$existing_issue" --repo "$GITHUB_REPOSITORY" --body-file "$report_file" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$report_file" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 10b4955c16..5c8db5a603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **SIGKILL crash test checks committed data (#384).** `tests/crash_kill_test.rs` used to check only that the file reopened and one attribute query ran, so it missed #370 and could not reach #371. The child now runs a seeded workload (multi-valued batches, batched retracts, re-assertions) and logs each transaction after `execute` returns; it checkpoints after every transaction, every third, never (WAL replay), automatically every 2 writes, or every third with `SyncMode::Normal`. After a `SIGKILL` at a random point, the reopened file must hold exactly the model after the last logged transaction or the one in flight, through a full scan, attribute (AEVT), attribute+value (AVET) and entity-bound (EAVT) queries and `:as-of`, with `verify` clean. The checks run again after a second reopen and after a checkpoint plus reopen. Per-PR CI runs 5 rounds (one per mode); the nightly crash-kill workflow runs 100 per OS. `MINIGRAF_CRASH_KILL_SEED` replays a failure. - **Fuzzing reaches the v8 decoders, and the corpus is kept between runs (#387, #375).** On `v3`, the `btree_page` and `fact_page` targets still wrote the pre-release single-header layout and stopped at `STG-032` on every input, and `file_header` rarely got past the meta CRC. The page targets now start from a real v8 file built through the public API (rewritten with `LogWriter` at fixed tx times so its bytes are the same in every process). They overwrite one page with the input and recompute that page's checksum. `btree_page` fuzzes any leaf or internal node of the five trees, `file_header` fuzzes either meta slot (with or without a matching CRC, with or without the other slot), and `fact_page` fuzzes the fact page of a v7 golden file through migration and the read-only load. Each run executes one input-chosen probe query or `verify()`. The nightly fuzz workflow now runs every target in its own parallel job, on `main` and `v3` on alternating crons, and keeps each target's corpus in the Actions cache. A new weekly workflow (`fuzz-corpus.yml`) minimizes the cached corpus with `cargo fuzz cmin`, keeps every committed `seed_*` file, and proposes the result as a pull request. - **Two fuzz targets check results, not just crashes (#387).** `ops_sequence` decodes the input into up to 32 storage operations (transact, retract, a write transaction that commits or rolls back, checkpoint, reopen, and a crash that opens a copy of the file and WAL) and, after each one, compares the full rows now and at an input-chosen `:as-of`, plus one input-chosen query shape, with the reference model of the model-based test. It uses that test's driver and checks, now in `tests/reference_model`, so the proptest test and the fuzzer share one oracle. `corrupt_file` damages a file written over three checkpoints with up to 16 byte flips and an optional truncation, without fixing any checksum, and requires every query to fail or to return exactly the answers of the last checkpoint or the one before it, all from the same one. Its first run found that a file cut to under one page opens as a new, empty database (#506); the target skips those lengths until that is fixed. +- **Weekly soak test at scale (#392).** `tests/soak_test.rs` grows a file-backed database to 10M live facts in batches of 1–2,000 facts (multi-valued tags and shard values in one transaction, strings over 64 bytes, refs), then churns it for the rest of a 5-hour budget at 50 transactions a second: retracts and re-asserts of random facts, and long retract/assert histories on 32 hot entities. It checkpoints every 20–200 transactions and reopens every 2–10 minutes, half from a checkpointed file and half replaying the WAL. Every 10 minutes, after every reopen, and between churn writes it checks samples against a reference that derives each entity's facts from its number and keeps only what was retracted: EAVT per entity, entity and attribute on the tags, AVET by unique name, AEVT over a sparse attribute, `:as-of` on hot entities, and the transaction counter; a full scan runs in a child process at 2M facts (so its memory does not hide a leak), and `verify()` at the end. It fails if RSS after a reopen grows past 1.25× the baseline plus 256 MiB. Metrics (file size, open and checkpoint time, RSS and peak RSS, p50/p99 per query path) go to a JSON-lines file that `.github/workflows/soak.yml` uploads weekly, alternating `main` and `v3`. `soak_short` runs the same driver at 20K facts in the per-PR suite. Found #518: attribute-and-value patterns scan the whole attribute. ### Known issues diff --git a/CLAUDE.md b/CLAUDE.md index 49d1a97362..5c39448fad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -198,7 +198,7 @@ Auto-migrates v7 → v8 on open (spec §9, with a backup meta page). v1–v6 are ## Test Coverage -**1406 tests** (1397 passing, 9 ignored; unit + integration + doc). +**1409 tests** (1399 passing, 10 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 17828bf13b..266588374e 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1406 tests +cargo test # run 1409 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/ROADMAP.md b/ROADMAP.md index 3343805dd7..984dbf801a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -93,7 +93,7 @@ Scope — release-gate testing and documentation: - Wider property tests: file-backed, batched, retracts, temporal, joins (#386) - Persistent fuzz corpus and an operation-sequence target (#387; done on `v3`: corpus caching, weekly `cmin`, the v8 page targets, and the `ops_sequence` and `corrupt_file` targets) - Fault injection across save/WAL/recovery (#390) -- Long-running soak test at scale (#392) +- Long-running soak test at scale (#392; done on `v3`: `tests/soak_test.rs`, 10M facts and a 5-hour churn weekly, alternating `main` and `v3`) - Benchmarks at 100K–10M facts and a published performance envelope (#394) - Higher storage/WAL coverage gates and branch-coverage gates (#396) - Durability, recovery and operations guide (#398) diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 8df032a068..5528f48fdb 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,7 +4,7 @@ **Verified**: 2026-10-10 with `cargo test --quiet` -**Result**: 1,397 passing tests and 9 ignored tests (1,406 total) +**Result**: 1,399 passing tests and 10 ignored tests (1,409 total) This covers the unreleased file-format-v8 work (#287, #371): the index-key value-collision fix, the v7→v8 migration, and the drop of formats v1–v6. It otherwise still covers the public database API, the Datalog engine, storage and recovery paths, v2.0.0's locking and structured-error-code changes, and v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Merged from the v2.x line: `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a damaged slot directory is rejected rather than copied (#315). B+tree cursor tests check that every scan walks the tree, not the leaf `next_leaf` chain: full scans and start positions match the input, `seek` from the current position lands where a fresh descent from the root would (randomized, plus within a leaf, to a sibling, across subtrees, past the end, into the gap after a leaf, and backwards as a no-op), a seek to the sibling leaf reads exactly one page, reads are unchanged with every `next_leaf` zeroed, a child pointer into a non-B+tree page is STG-013, and a cycle in child pointers is an error, not a hang (v8 storage format PR 1, #374). Directory-sync unit tests check that creating the `.graph` file or the WAL, and deleting the WAL, fsync the parent directory after the file operation, and that reopening an existing file does not (#389). Disjunction tests check that `or-join` returns an empty result, not INT-031, when no rows reach it: on an empty database, after a pattern that matches nothing, after a `not` that removes every row, and in a rule body (#405). A stratification unit test builds the same two-rule negative cycle 64 times and checks that the INT-054 message names the same predicate pair every time (#410). Magic-sets tests check that a recursive rule with a literal start that recurses through another rule returns the same results as without the literal (#297). API tests check that `execute()` and `WriteTransaction::execute()` reject `$slot` bind tokens in every position with API-010 (#407). The REPL `run()` test now runs in a child process with null stdin, so it no longer hangs when the test runner's stdin is an open non-TTY stream (#415). v8 page-format tests (PR 2, #374, #388) check the 24-byte page header (a flipped bit, a wrong page id and a future generation each give their own STG code, legacy page types fail before the CRC, and a failed page is never cached), meta-page encoding, the free-list chain, WAL v2's base generation, and every row of the meta-selection table: both slots valid, a torn newest slot with the WAL's base generation (opens at the older one), a WAL newer than the valid meta, pages of the next generation on the free list or past `page_count` with no WAL (all STG-033, with the file unmodified), an unknown feature bit (STG-034), and a pre-release v8 single header (STG-032). Fault-injection tests crash `save()` at every page write (cleanly and with torn pages, including the meta write) and every sync, and crash a v7 migration at every write; each reopen finds exactly the old or the new checkpoint, and every reachable page plus the free list covers the file exactly. Twenty-five consecutive saves never write a page the previous meta references. Covering-key tests (PR 3, #433) check that byte order equals logical order for every key component and value type on random tuples (negative, zero and large integers, NaN and ±0.0, strings with 0x00, 64- and 65-byte strings, FOREVER, newest-transaction-first) and that every component round-trips; that prefix-compressed leaves and shortest-separator internal nodes round-trip, agree with a linear scan, keep `max(left) < s ≤ min(right)`, and reject an overflow; that the byte-key B+tree's builds, incremental rebuilds, cursor and `seek` match their expected sets; that dictionary ids are assigned once, never reused, shared between a keyword attribute and value, and deterministic; that a re-asserted or retracted long value writes no new value page and a forced hash collision still finds the right value; that one transaction with two timestamps is STG-037 and a missing dictionary entry STG-036; that committed reads by entity, entity and attribute, attribute, and in full equal an in-memory model across eight checkpoints and a reopen; that short-valued reads touch no value page; that a v7 file with every value type, long strings, refs and retractions migrates to exactly the same facts; that a pre-covering-key v8 file is STG-032; and that string values over 4,068 bytes and attributes or keywords over 1,024 bytes are WAL-003. A WAL test checks that reopening next to a fully checkpointed WAL does not reuse transaction numbers (#447). Copy-on-write checkpoint tests (PR 4, #434) check that lazy free-list pops and pushes conserve every page across page boundaries, partial head pages and chain exhaustion, never hand out a page freed in the same generation, and share unread chain pages unchanged; that a copy-on-write insert equals a sorted merge for random batches (including keys below the first and above the last, many into one leaf, root growth to depth 3, and 1 KB separators that split internal nodes), leaves the old tree readable and unchanged, frees exactly the replaced nodes, writes only new pages, writes exactly one leaf and two internal nodes for one entry in a depth-3 tree, and does not fragment leaves under single inserts; that a checkpoint after 1 or 100 right-edge facts writes at most the tree-depth difference more pages at 100,000 facts than at 10,000, and at most one path per new entry per tree for random keys; and that four reader threads querying throughout 40 checkpoints never fail and never see the fact count go backwards. Hardening tests (PR 5) damage an index leaf, an internal node, a dictionary leaf, a value page and a free-list page in turn. Every committed read or checkpoint that reaches the damaged page fails with STG-029, never with wrong or fewer rows. A query through `Minigraf` on a file with damaged leaves fails with STG-029, and the file still opens at its last checkpoint. A size test measures at most 200 bytes per fact (142 measured) on 144k facts in #433's shape; its 1.15M-fact variant is ignored locally and runs in the nightly smoke workflow. A shared dictionary-cache test checks that cached ident and timestamp lookups return the same facts and that the cache stays bounded. Merged from the v2.x line: storage and `edge_cases_test.rs` tests check that more than 65,535 uncheckpointed facts each resolve to their own fact through entity lookups and attribute scans, in memory and file-backed (#445). Merged from the v2.x line: `tests/golden_corpus_test.rs` opens a copy of each committed golden file in `tests/golden/` (v7 files written by minigraf 2.0.3: one checkpoint, several checkpoints, indexes rebuilt on open, a crash-left WAL, an already-checkpointed WAL, same-transaction multi-values) and checks its manifest's CRC32, `tx_count` and query results after open, after a checkpoint and reopen, and after one more write; a second test checks that every golden file has a manifest (#391). On `v3` the same test checks that each v7 file has a v8 meta page after its first open and gives the same results after reopening the migrated file, and adds seven v8 golden files (one checkpoint, seven copy-on-write checkpoints, a crash-left v2 WAL, an already-checkpointed v2 WAL, same-transaction multi-values, a v7 file migrated in place, and long values at the size limits); the #371 queries, skipped on v2.x, run and pass. `tests/model_based_test.rs` runs random sequences of transacts, retracts, write transactions, checkpoints, reopens and simulated crashes against a file and, after every step, compares the transaction counter, `verify`, full rows with windows and tx counts, and six query shapes under `:as-of N` and `:valid-at T` with a reference model (#385). `tests/property_test.rs` writes random histories (batched transacts, retracts, write transactions, checkpoints) in memory and to a file, and compares random queries (one to three joined patterns, `not` or `not-join`, a comparison predicate, `:as-of`, `:valid-at` or `:any-valid-time`) with a reference evaluator, on the open file and after a reopen; a query error fails the test (#386). `tests/crash_kill_test.rs` kills a child process running a seeded workload (multi-valued batches, batched retracts, five checkpoint modes) at a random point and checks that the reopened file holds exactly the transactions the child logged as committed, or those plus the one in flight, through full-scan, AEVT, AVET, EAVT and `:as-of` queries and `verify`, again after a second reopen and after a checkpoint plus reopen (#384). Integrity tests (#373) check that intact files verify with no problems and an exact fact and page count, and that `verify` reports each kind of damage with its code: an EAVT, AEVT, AVET or VAET leaf missing an entry (STG-038), keys out of order and a page reached from two trees (STG-039), a rotted index leaf (STG-029), a free page still in use and a leaked page (STG-035), a missing transaction entry (STG-036), and an entity id whose UUID maps to another id (STG-040). After each index or free-list damage, `rebuild_indexes` leaves a file that verifies clean, reads back every fact and accounts for every page. The source is an index that agrees with another one, or the only intact one; three intact indexes that all disagree, or a damaged dictionary, give STG-041 with the file unchanged. A rebuild commits pending facts, including new ids and long values. A crash at every write (clean and torn) and every sync of a rebuild reopens either at the old generation with only its old damage or at the new one with none, never losing a fact. Through `Minigraf` (`tests/integrity_test.rs`), a rebuild commits WAL entries and deletes the WAL, and flipping a byte in each leaf of a file in turn is always reported, then either repaired (index leaves) or refused with STG-041 and the file unchanged (dictionary leaves). Page-count tests (#497) check that a CRC-valid meta whose `page_count` is past the end of the file (one page past, 2^50, `u64::MAX`; with the other slot valid or damaged; with or without a WAL) and a file cut short by one page, three pages or down to three pages are `STG-044` at open, with the file unchanged and not grown, while a `page_count` exactly at the end opens; that a checkpoint whose pages did not all reach the backend fails with `INT-049` before writing its meta, and the file reopens at the previous generation; and that `verify` at a meta counting pages past the end reports `STG-044` without scanning to that count. Through `Minigraf` (`tests/integrity_test.rs`), a file truncated by one or three pages and a file whose meta was rewritten with `page_count` 2^50 and a valid CRC are `STG-044` read-write and read-only, with no WAL created and the file length unchanged. The `file_header` fuzz input from #497 is a corpus seed. A sub-page test (#506, `tests/integrity_test.rs`) checks that a file of 1 to 4,095 bytes is `STG-045` read-write and read-only, unchanged and with no WAL, when it is a database cut below one page or foreign bytes, and opens as a new database that takes writes when it is a prefix of the first meta page or all zeros; a file-backend unit test checks that `partial_page` returns the bytes past the last whole page. The `corrupt_file` fuzzer no longer skips files under one page: they must fail to open or open empty, and the #506 input is a corpus seed. Damaged v7 fact page tests (#496) check that a page in the counted fact range whose type byte is anything but the fact-page type (zero, `0x01`, the v2.x B+tree types, `0xFF`, a zeroed page; first and last page) is `STG-014` rather than skipped, while a range derived from the first index root skips only v2.x B+tree pages; that opening such a v7 file read-write or read-only fails with `STG-014` and writes nothing; and, in `tests/migration_matrix_test.rs`, that the golden `v7_basic.graph` with page 1's type byte changed or page 1 overwritten with noise fails both opens with the file unchanged and no WAL created. Both inputs are `fact_page` corpus seeds. Merged from the v2.x line: `tests/edge_cases_test.rs` checks the same refusal and acceptance through `Minigraf::open` on a database checkpointed twice (#506). Torn-WAL-tail tests (#513) check that opening a WAL writer cuts off a partial last entry (cut in the checksum, the header or the payload) or an entry with a bad checksum and everything after it, then appends where replay reaches, and leaves an intact WAL byte for byte; that an append that fails part-way makes every later append `WAL-007` while a rejected oversize fact (`WAL-003`, nothing written) does not; through `Minigraf`, that the failed transaction is not applied, later `transact` and `commit` are `WAL-007`, and a `checkpoint()` makes writes work again; and, in `tests/wal_test.rs`, that a commit made after reopening a crash-left WAL with each kind of torn tail survives a second crash. The fault matrix (#390, `src/fault_matrix.rs`) drives a test-only fault plan through the real file backend, WAL and directory fsync: an EIO, a short or torn write (1, 40 and 4,095 bytes), a full disk that keeps failing every later write, a failed fsync, and a failed fsync that also loses the writes since the last good one ("fsyncgate"), each at every write or sync call of a transact, a transact that creates the WAL, an automatic checkpoint, `checkpoint()`, the close-time checkpoint and `rebuild_indexes()`, on a database committed over two checkpoints with multi-valued, retracted and long values and entries pending in the WAL. After each fault, the handle holds the old facts or those plus the whole transaction, never part of one; after a failed checkpoint another is `STG-046`; a later write succeeds or is `WAL-007`/`STG-046`; a kill and reopen gives exactly the handle's facts, plus the failed transaction whole or not at all, and `verify` finds nothing; and a commit after recovery survives another kill. Two more tests fill the disk halfway through a checkpoint (`STG-046`, reads unchanged, reopen intact), and fail the fsync after a checkpoint's meta write and then retry the checkpoint, crashing at each of its writes: the retry is refused and every reopen is intact (without the refusal, a reopen hits `STG-029`). Unit tests check the fault plan's counting, and that a `rebuild_indexes` refused with `STG-041` leaves the handle usable. Live-read tests (#379) check that `triple_len` ends exactly where `tx↓` begins for every value type and index; that on random histories (several valid-time windows, retractions, re-assertions, several values per attribute, long values, 12 transactions per checkpoint) the committed live read for every scan kind and `:as-of` point is exactly the net-asserted set, that it stays equivalent with pending facts on top, and after reopen; and that a point read on a triple asserted and retracted 2,000 times reads under a third of the pages a history read does, with or without `:as-of`. `tests/live_reads_test.rs` runs the same history in a checkpointed file database and in memory and checks that point, attribute, full-scan, join, valid-time, `:as-of` and rule queries return the same rows, before and after reopen. One-window tests (#435) check that `net_asserted_facts` keeps each triple's newest transaction only (a later window replaces an earlier one, closing an open window, a retraction in the newest transaction hides it, records with two windows in one transaction keep both, once each) against a per-triple model on random histories; that the on-disk key walk agrees at every `:as-of` before and after reopen; and that `API-011` rejects two windows of one fact in one transaction without taking a `tx_count`. `tests/valid_time_window_test.rs` checks replacing, closing, extending, reopening and retract-then-reassert through entity-bound and attribute queries in memory, on a checkpointed file and after reopen, `API-011` from `transact` and from `WriteTransaction::execute`, and a later statement's window committed when two statements of a `WriteTransaction` give one fact different windows. Window-validation tests (#436) check that `check_valid_windows` rejects an empty window, an inverted one, and a `:valid-to` at or before the transaction time when `:valid-from` is left out, but not retractions; that `execute` rejects each such `transact` (both bounds, equal bounds, `:valid-to` only, a per-fact bound inverting the transaction's window) with `API-019` without writing or taking a `tx_count`, in memory, checkpointed and after reopen; that `tx.execute()` rejects them and the transaction still commits its other statements; that `commit()` rejects a default `valid_from` that has passed the `:valid-to` by commit time; and that `FactStorage::transact_batch` takes no `tx_count` on rejection. Statement-order tests (#477, `tests/statement_order_test.rs`) run every mix of two statements on one fact inside a `WriteTransaction` (retract then assert, assert then retract, two retracts, two windows, after a committed assertion and on an empty database) and check that reads inside the transaction and the committed rows agree, in memory, checkpointed and after reopen; that other facts of an overridden statement survive; that the commit is one transaction without the overridden records; and that two windows in one statement are `API-011` from `tx.execute()` while the rest commits. A unit test checks the commit collapse keeps each fact's newest statement in order. A log-writer test checks that an inverted window is `API-019`, a second window of a fact in one transaction `API-011`, and an assertion and a retraction of one fact in one transaction `API-020` in either order, each leaving the writer usable and the output exact, while duplicate retractions, identical repeats and the same fact in the next transaction are accepted. Cursor tests (#432, `tests/cursor_test.rs`) check that `query()` batches partition the `execute()` answer for several batch sizes (never empty, never over `max_rows`, `max_rows == 0` still progresses, `None` stays `None`), that the iterator and aggregates, rules, `:as-of` and `:any-valid-time` match `execute()`, that a cursor opened before a `transact`, a `WriteTransaction` commit, a retract or a checkpoint returns the pre-write answer to the end (in memory and file-backed), that `transact`/`retract`/`rule` are `API-012` and bind slots `API-010`, that a `WriteTransaction` thread gets `INT-001`, that rule limits give `INT-020` as from `execute()`, that `PreparedQuery::query` matches `PreparedQuery::execute`, and that a cursor is `Send` and outlives its database handle. Fact-log tests (#430, `tests/fact_log_test.rs`) check that `fact_log()` returns exactly the records written (assertions and retractions, with `tx_count` and `tx_id`) in memory, with everything pending, half checkpointed, fully checkpointed and after reopen, in both orders, with windows of 1 to 1,000 records and batch sizes of 1 to 1,000, in non-decreasing `tx_count` in Tx order; that every layout gives identical records, including valid-time windows and long string values; that a transaction larger than the window is split with no loss or duplicate; that attribute, attribute-prefix, entity and tx-range filters and their combinations match a model, and unknown names match nothing; that writes after open are not in the log; that `checkpoint()` and `rebuild_indexes()` are `API-013` while a log is open (a checkpoint with nothing to write is allowed), that automatic checkpoints from `execute()` and `WriteTransaction::commit` wait for the log, and that closing, dropping or finishing the log releases the database; that a `WriteTransaction` thread gets `INT-001`; and that a log is `Send`, outlives its database handle, and the deferred writes replay on the next open. Read-only open tests (#429, `tests/read_only_open_test.rs`) check that a read-only handle on a file with checkpointed facts and a pending WAL returns the same query results, `tx_count` and fact log as a read-write open; that queries, a cursor, a fact log, `verify`, `prepare`, rules and a UDF leave the `.graph` and WAL bytes and the `.graph` mtime unchanged, and no WAL is created; that `transact`, `retract`, `begin_write`, `checkpoint` and `rebuild_indexes` are `API-014` without taking a `tx_count`, also on a read-only in-memory database and through the builder path; that a missing file is `STG-042` and nothing is created; that an empty file is an empty database and stays 0 bytes; that read-only handles share a file while a read-write one is refused with `STG-025`, both ways; that a file without write permission opens read-only; and that small and whole-file page caches work. File-backend unit tests check that a shared-mode backend refuses page writes and syncs with `INT-056`, that the in-process registry counts readers and admits readers or one writer (also without kernel locks), and that a reader in another process shares the file with a reader here and refuses a writer here with `STG-026`. Storage unit tests check that a read-only open of a v7 file at every point of an interrupted migration (clean and torn writes) sees every fact and writes nothing, and that an empty backend stays empty. `tests/golden_corpus_test.rs` opens a copy of every golden file read-only first and checks its `tx_count` and queries, and that neither the file nor its WAL changes. Log-writer tests (#431, `tests/log_writer_test.rs`) copy a source's fact log (assertions, retractions, an explicit valid-time window, long strings, refs, keywords, checkpointed and pending facts) through `LogWriter` and check that the new file has no `.partial` or WAL, verifies clean, and gives the source's `tx_count`, fact log and query results (current, `:valid-at`, `:any-valid-time`, every `:as-of`, and entity-, attribute- and value-bound lookups) read-write, read-only and after reopen (#370); that it takes new writes at the source's next `tx_count`; that 90,000 facts over two batch commits verify clean and count correctly at `:as-of`; that a purged middle transaction is a hole whose `:as-of` equals the one before, and trailing holes keep the counter through `advance_tx_count`; that an empty log is an empty database with the counter; that `tx_count` 0, a lower or closed transaction and a backwards advance are `API-015`, a second `tx_id` `API-016`, a second window `API-011`, a retraction of an asserted fact in one transaction `API-020`, an empty window `API-019` and oversize values or idents `WAL-003`, each leaving the writer usable and the output exact, with repeats written once; that an existing target or WAL is `STG-043`, a second writer `STG-025` and `read_only` `API-014`; that a stale `.partial` is reused; that dropping an unfinished writer leaves nothing; that `finish` refuses a target created during the build and leaves it untouched; and that `LogWriter` is `Send`. Unit tests check that a batch commits only when a transaction opens past the batch size, never splitting one, and that `FileBackend::truncate` empties a file. A unit test in `error.rs` checks that `MinigrafError::invalid_argument` and `MinigrafError::closed`, the constructors for language bindings, give API-017 and API-018 with their text. Query-validation tests (#406, #414) check that a plain `:find` variable or a window function's argument, `:partition-by` or `:order-by` variable that no `:where` clause binds (including `?_` wildcards) is `PRS-080` from both `execute()` and `prepare()`, naming the variable, while variables bound by a pattern, an attribute position, a rule call, an expression binding, an `or-join`, every branch of an `or`, a pseudo-attribute or a window spec are accepted, and bind slots are not mistaken for `:find` variables; and that an unknown aggregate is `QRY-010` and an unknown window function `QRY-011` on an empty database, when no row matches, when rows match, through a rule query and through a prepared query, while a registered UDF in either position returns an empty result on an empty database. Three `BrowserDb` tests (wasm32, run with `wasm-bindgen-test` in a browser, not counted above) check that `query()` batches of 1, 7 and 1,000 give the rows of `execute()`, that a cursor keeps the answer from when it opened and returns the end after `close()`, and that a `transact` is API-012. The wasm32 targets run in CI (`.github/workflows/wasm.yml`). In a browser (`wasm-pack test --headless`, Firefox and Chrome): 32 `wasm_bindgen_test` tests: nineteen for `BrowserDb` (including IndexedDB persistence, the cursor, and the flush returning every dirty page and failing on one it cannot read, from #470), eleven for `BrowserBufferBackend` (which used `#[test]` and never ran before), and two checking that IndexedDB page keys accept only whole numbers from 0 to 2^53. Five of the `BrowserDb` tests (#411, #440) check that opening a v7 store written straight into IndexedDB stores a v8 meta page and only the live pages, with nothing written on the next open; that after each of 60 transacts memory, and at the end IndexedDB, hold exactly the pages not on the free list while the file grows by under one page per transact; that a commit applied in memory but never flushed reopens at the previous one and keeps working; that an export after churn is as long as the meta says and imports back with its free pages dropped; and that `importGraph()` deletes the replaced database's IndexedDB keys. Two buffer tests check that releasing a page drops its bytes and any unflushed write, and that export writes a released page as zeros. A native storage test checks that a file whose free pages are all zeros (as the browser exports them), both just migrated from v7 and after 20 saves, opens, verifies clean and keeps committing, and the free-list test checks that a checkpoint reports exactly the ids it newly lists. On `wasm32-wasip1` under Wasmtime: the unit and integration tests that do not use a file run, 935 passing; nine that spawn threads or a child process are ignored on WASI. These are not counted above. @@ -15,15 +15,16 @@ This covers the unreleased file-format-v8 work (#287, #371): the index-key value - **Storage correctness**: packed pages, B+tree indexes and range scans, cache behavior, file-header validation, migration (including v7→v8 index rebuild and v1–v6 rejection via `STG-028`), WAL replay, checksums, corruption handling, and fault injection. - **Reliability and concurrency**: concurrent reads and writes, rollback behavior, same-process handle exclusion, cross-process locking, PID namespaces, NFS and container lock behavior, and SIGKILL recovery. - **Public diagnostics**: structured parser, query, storage, WAL, API, and internal error codes; the error-code registry is checked against [`ERROR_REFERENCE.md`](ERROR_REFERENCE.md). -- **Compatibility and quality**: golden-file corpus of released file formats, cross-platform format compatibility, property-based tests, long-haul smoke coverage, XTDB/Datomic semantic compatibility, grammar conformance, and rustdoc examples. +- **Compatibility and quality**: golden-file corpus of released file formats, cross-platform format compatibility, property-based tests, long-haul smoke coverage, a weekly soak at 10M facts with a 5-hour churn (#392), XTDB/Datomic semantic compatibility, grammar conformance, and rustdoc examples. ### Ignored Tests -The nine ignored tests are intentionally excluded from a normal local run: +The ten ignored tests are intentionally excluded from a normal local run: - Six rustdoc examples that reference internal types and cannot compile as standalone examples. - One high-contention concurrency stress test intended for scheduled runs. - One long-haul smoke test intended for scheduled runs. +- One soak test at 10M facts (`soak`), run weekly by `.github/workflows/soak.yml` (#392); `soak_short` runs the same driver at 20K facts in the normal suite. - One size test at 1.15M facts (`size_per_fact_in_433_shape_1m`), run by the nightly smoke workflow. ### Reproducing diff --git a/docs/superpowers/plans/2026-10-10-soak-test.md b/docs/superpowers/plans/2026-10-10-soak-test.md new file mode 100644 index 0000000000..f469da326b --- /dev/null +++ b/docs/superpowers/plans/2026-10-10-soak-test.md @@ -0,0 +1,30 @@ +# Soak test at scale — Plan + +Spec: `docs/superpowers/specs/2026-10-10-soak-test-design.md` (#392). + +## v3 PR (`test/392-soak` → `v3`) + +1. `tests/soak_test.rs` + - `Config` (from env for `soak`, fixed for `soak_short`), `Every` schedule by + time or by step count so the short run is deterministic in a debug build. + - `Reference`: `content(n)` slots, `flipped` Vec + index map, hot history as + packed `u64`s, `tx`. + - `Driver`: op choice by phase weights, checkpoint and reopen schedule, + churn rate cap with checked reads in between. + - Checks: EAVT, AVET, AEVT shard, hot `:as-of`, tx counter; end: full-scan + count and `verify()`. + - `Metrics`: latency samples per path, checkpoint times, open times, RSS + from `/proc/self/status`; JSON lines via `serde_json`. + - Memory bound after the growth phase (spec §8). +2. Run `soak_short` in debug; run `soak` locally with a reduced target and a + few minutes; then a full 10M run under `--profile bench`. +3. `.github/workflows/soak.yml` (spec §9). +4. Docs: `docs/TEST_COVERAGE.md`, `CLAUDE.md` test count, `ROADMAP.md` if it + lists #392, `CHANGELOG.md` Unreleased. +5. PR to `v3` without a closing keyword for the tracker; `Closes #392` only + in the main PR description if the user wants it closed on merge. + +## main PR (`test/392-soak-v2` → `main`) + +Same test with `verify()` dropped and the v2 memory rule (spec §8); same +`soak.yml`. Run `soak_short` and a short `soak`. diff --git a/docs/superpowers/specs/2026-10-10-soak-test-design.md b/docs/superpowers/specs/2026-10-10-soak-test-design.md new file mode 100644 index 0000000000..e785887de1 --- /dev/null +++ b/docs/superpowers/specs/2026-10-10-soak-test-design.md @@ -0,0 +1,217 @@ +# Long-running soak test at scale — Design + +**Issue:** #392, tracker #383 +**Milestone:** v3.0.0. Test and workflow only; no library change. +**Branches:** a v3 PR first, then a `main` PR with the same harness adapted to +the v2 API (no `verify()`). The weekly workflow alternates between the two. + +## 1. Goal + +No test today runs for hours or at the size of a real deployment. The nightly +smoke suite has a 15-minute timeout and 5,000 facts. Bugs like #370 appeared +only after several checkpoints on a real workload. The soak test grows a +file-backed database to 10M+ facts with mixed batches, multi-valued +attributes, retracts and long histories on hot entities. It checkpoints and +reopens many times, checks answers against a reference throughout, and records +the numbers #394 needs. + +## 2. Measured feasibility (local, `--profile bench`) + +| | v3 (format v8) | main (format v7) | +|---|---|---| +| Insert rate (100-fact batches) | ~70K facts/s | ~70K facts/s | +| Checkpoint at 1M / 3.8M facts | 180 ms / 250 ms | 1.0 s / 2.6 s (linear) | +| File size per fact | ~110 B | ~430 B (≈4.3 GB at 10M) | + +A GitHub runner is about 2–3× slower. Both branches reach 10M facts well +inside the time budget, so the run is time-boxed and the extra time goes into +churn. + +## 3. Shape + +`tests/soak_test.rs`, `#![cfg(not(target_arch = "wasm32"))]`, two tests: + +- `soak` (`#[ignore]`): the full run, tuned by environment variables. +- `soak_full_scan_child`: a no-op unless the parent sets + `MINIGRAF_SOAK_SCAN_DB` (§6). +- `soak_short` (runs in the per-PR suite): the same driver with a tiny + configuration (about 20K facts, 3 reopens, 3 checks), a few seconds in a + debug build. It keeps the harness compiling and correct. + +| Variable | Default (`soak`) | Meaning | +|---|---|---| +| `MINIGRAF_SOAK_MINUTES` | 20 | Wall-clock budget for the workload | +| `MINIGRAF_SOAK_TARGET_FACTS` | 10,000,000 | Live facts the growth phase reaches | +| `MINIGRAF_SOAK_CHECK_MINUTES` | 10 | Interval between sampled checks | +| `MINIGRAF_SOAK_CHURN_TPS` | 50 | Write rate cap in the churn phase (§5) | +| `MINIGRAF_SOAK_SEED` | random, printed | Replays a run | +| `MINIGRAF_SOAK_METRICS` | `target/soak/metrics.jsonl` | Metrics output | +| `MINIGRAF_SOAK_FULL_SCAN_MAX` | 3,000,000 | Largest live count the final full scan runs at (§6) | +| `MINIGRAF_SOAK_DIR` | a temp dir | Where the `.graph` lives (the workflow points it at the runner's large disk) | + +It runs under `--profile bench` (optimized, `panic = "unwind"`). The release +profile aborts on panic and cannot run tests. + +## 4. Reference without storing 10M facts + +The process's own peak RSS is a measured number, so the reference must stay +small. Every ordinary entity's facts follow from its index, and only changes +are stored. + +**Ordinary entities** `:e{n}` (`n = 0, 1, 2, …`). `content(n)` is a fixed +function of `n` and the seed: + +- `:p/name "name-{n}"` (unique, for AVET lookups) +- `:p/score` integer `hash(n) % 1000` +- `:p/shard{n % 1024}`, two integers `n` and `-n - 1` in the same transaction + (a sparse, multi-valued attribute: about 1,700 entities each at 10M facts, so + an AEVT scan of one shard is cheap to check exactly) +- `:p/tag`, 1–4 keyword values in the same transaction (multi-valued, #371) +- every 50th entity: `:p/bio`, a string over 64 bytes (value pages) +- every 10th entity: `:p/ref`, a ref to `:e{hash(n) % n}` + +Each fact of `content(n)` has a slot number. An entity is created in one +transaction, and a batch holds several entities. The reference keeps: + +- `next_entity`: entities `0..next_entity` exist. +- `flipped`: slots `(entity, slot)` currently retracted, a `Vec` plus an + index map so a random one can be picked in O(1). + A retract op picks a random existing entity and slot. If the slot is live it + is retracted and inserted into `flipped`; if not it is re-asserted and + removed. +- `live_count`: total facts minus `flipped.len()`. + +The expected live facts of any entity, or of any shard, are computed from +`content` and `flipped`. Every entity can be sampled. + +**Hot entities** `:h0`–`:h31` carry long histories. Each churn step changes a +few of them: retract and re-assert `:h/state` (one of 8 values), and add or +retract `:h/tag` values (multi-valued). The reference keeps every hot change +as `(tx_count, triple, assert|retract)`, so `:as-of` at any past transaction +can be checked. A record packs into one `u64`; a 5-hour run keeps a few +million of them. + +**Transaction counter:** each `execute` increments the reference `tx`. After +every reopen `current_tx_count()` must equal it. + +All facts use the default valid time. Valid-time windows are covered by the +model-based and property tests (#385, #386). + +## 5. Workload + +One loop until the budget ends. Each step picks one op: + +| Op | Weight (growth / churn) | Detail | +|---|---|---| +| Grow | 70% / 10% | New entities; batch size log-uniform over 1–2,000 facts | +| Retract/re-assert | 15% / 50% | 1–50 random ordinary slots: live ones retracted in one `retract`, or retracted ones re-asserted in one `transact` | +| Hot churn | 15% / 40% | 1–8 changes on hot entities in one `transact` or `retract` | + +The growth phase runs at full speed until `live_count` reaches the target +(the run fails if the budget ends first). The churn phase uses the rest of the +budget at a capped write rate, `MINIGRAF_SOAK_CHURN_TPS` (default 50 +transactions per second): at full speed it would add tens of millions of fact +versions, more than main's in-memory store and disk can hold. Time between +writes goes to extra checked reads: an EAVT (98%) or AVET (2%) lookup of a random entity, +compared with the reference and timed like the scheduled checks. Re-asserts +are chosen once `flipped` holds 200K slots, so the live count stays roughly +flat while the history grows by a few million versions. + +Interleaved, by a seeded schedule: + +- `checkpoint()` every 20–200 transactions (v3: thousands of checkpoints per + run). Times are recorded. +- Every handle is opened with `wal_checkpoint_threshold(usize::MAX)`, so only + the schedule above checkpoints and dropping a handle does not. +- Reopen every 2–10 minutes: half the time right after `checkpoint()` (open + reads the file only), half with up to 200 transactions still in the WAL + (open replays them). Open time is recorded for each kind. + +## 6. Checks + +Every `MINIGRAF_SOAK_CHECK_MINUTES`, after every reopen, and at the end: + +- **EAVT:** `[:e{n} ?a ?v]` for 200 random entities and all 32 hot entities; + rendered rows equal the reference. +- **EA:** `[:e{n} :p/tag ?v]` for the same entities. With the multi-valued + shard attribute this is the shape #371 broke; on v2.0.4 the unmodified + workload fails these checks, so the test can see that bug. +- **AVET:** `[?e :p/name "name-{n}"]` for 5 random entities: present + exactly when that slot is live. This shape scans the whole attribute today + (#518: 0.5 s at 1M facts, 6 s at 10M), so it is sampled sparingly; raise + the sample once #518 is fixed. +- **AEVT:** `[?e :p/shard{k} ?v]` for 4 random shards; the result equals + every live shard slot of that shard. +- **`:as-of`:** for 4 hot entities and 3 random past transactions each, the + rows equal the history replayed to that transaction. +- **Counter:** `current_tx_count()` equals the reference. + +- **Full scan:** the row count of `[?e ?a ?v]` equals `live_count` plus the + hot entities' live facts. A query builds its whole answer when it opens + (#432), so a full scan costs memory per row: about 300 B at 1M simple + facts, about 1.1 KB here (2.2 GB at 2M); a `count` aggregate costs the same. + The allocator keeps that memory after the query, so in this process it would + hide a leak of that size from §8. The scan therefore runs in a child process + (the test binary re-run as `soak_full_scan_child`, as `crash_kill_test` + does) after a checkpoint, with the parent's handle closed: once, when the + live count first reaches 2M, and at the end only up to + `MINIGRAF_SOAK_FULL_SCAN_MAX` (default 3M) live facts. The child reports the + row count, the time and its peak RSS. + +At the end, after a final reopen: + +- **v3 only:** `verify()` reports no problems. + +Rendered rows never carry a `Uuid` (CLAUDE.md test conventions); failures name +the seed, the step and the check. + +## 7. Metrics + +The test appends one JSON line per check to the metrics file: + +```json +{"t_s": 3600, "phase": "churn", "tx": 41230, "live_facts": 10012345, + "versions": 12500000, "file_bytes": 1234567890, "open_ms_file": 812, "open_ms_wal": 940, + "reopens": 40, "checkpoints": 2100, "checkpoint_ms_p50": 210, "checkpoint_ms_max": 900, + "rss_bytes": 512000000, "peak_rss_bytes": 700000000, + "query_us": {"eavt": [p50, p99], "avet": [...], "aevt": [...], "as_of": [...]}} +``` + +Lines written after a full scan also carry `full_scan_ms` and +`full_scan_peak_rss_bytes` (the child's). RSS +and peak RSS come from `/proc/self/status` (`VmRSS`, `VmHWM`); elsewhere they +are `null`. + +## 8. Memory bound + +After the growth phase the test records a baseline: RSS right after a reopen. +At each later post-reopen check: + +- **v3:** RSS must stay below `1.25 × baseline + 256 MiB`. The page cache is + bounded and the reference grows only slowly, so the steady state is flat. +- **main:** v2 keeps facts in memory, so RSS rightly grows with history. RSS + per fact version must stay below `1.25 ×` the baseline ratio, plus 256 MiB. + +## 9. Workflow `soak.yml` + +Weekly, alternating like `bench.yml` and `crash-kill.yml`: main on Saturday +01:00 UTC, v3 on Sunday 01:00 UTC, chosen from the cron string; also +`workflow_dispatch` with a `ref` input. The same file is committed to both +branches. + +- `ubuntu-latest`, `timeout-minutes: 360`. +- `MINIGRAF_SOAK_DIR=/mnt/soak`: the runner's spare disk holds main's + multi-GB file. +- `cargo test --profile bench --test soak_test soak -- --ignored --nocapture` + with `MINIGRAF_SOAK_MINUTES=300`. +- Always upload `metrics.jsonl` and the test log as artifacts (90-day + retention), which feed #394. +- On failure, open or comment on a "Weekly soak failure ()" issue with the + seed and the log tail, as `crash-kill.yml` does. + +## 10. Out of scope + +Kills mid-write (#384), fault injection (#390), valid-time windows (#385, +#386), multi-process access, and other OSes (RSS is read from `/proc`). +Publishing the performance envelope is #394; this test only produces the +numbers. diff --git a/tests/soak_test.rs b/tests/soak_test.rs new file mode 100644 index 0000000000..25209f3a5d --- /dev/null +++ b/tests/soak_test.rs @@ -0,0 +1,1135 @@ +//! Long-running soak test at scale (#392). +//! +//! Grows a file-backed database to millions of facts in batches of mixed +//! sizes, with multi-valued attributes, retracts, re-asserts and long +//! histories on a few hot entities. It checkpoints on a schedule, reopens at +//! random, and checks samples against a reference through EAVT, AVET, AEVT, +//! `:as-of` and, at the end, a full scan. Metrics go to a JSON-lines file. +//! +//! `soak_short` runs a tiny configuration in the per-PR suite. The full run +//! is ignored; `.github/workflows/soak.yml` runs it weekly: +//! +//! ```text +//! MINIGRAF_SOAK_MINUTES=20 MINIGRAF_SOAK_TARGET_FACTS=10000000 \ +//! cargo test --profile bench --test soak_test soak -- --ignored --nocapture +//! ``` +//! +//! Other variables: `MINIGRAF_SOAK_SEED` (replays the write sequence; the +//! time-based schedule still varies), `MINIGRAF_SOAK_CHECK_MINUTES`, +//! `MINIGRAF_SOAK_CHURN_TPS`, `MINIGRAF_SOAK_METRICS`, `MINIGRAF_SOAK_DIR`. +//! Design: `docs/superpowers/specs/2026-10-10-soak-test-design.md`. +#![cfg(not(target_arch = "wasm32"))] + +use minigraf::db::{Minigraf, OpenOptions}; +use minigraf::{QueryResult, Value}; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Instant; +use uuid::Uuid; + +// ── Random numbers ──────────────────────────────────────────────────────────── + +/// SplitMix64 finalizer: a fixed hash, independent of any crate version. +fn mix(mut z: u64) -> u64 { + z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + mix(self.0) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n.max(1) + } + + /// Uniform in `lo..=hi`. + fn range(&mut self, lo: u64, hi: u64) -> u64 { + lo + self.below(hi - lo + 1) + } + + fn unit(&mut self) -> f64 { + (self.next() >> 11) as f64 / (1u64 << 53) as f64 + } + + fn chance(&mut self, p: f64) -> bool { + self.unit() < p + } + + /// Log-uniform in `lo..=hi`: small batches are common, large ones occur. + fn log_uniform(&mut self, lo: u64, hi: u64) -> u64 { + let (l, h) = ((lo as f64).ln(), ((hi + 1) as f64).ln()); + ((l + (h - l) * self.unit()).exp() as u64).clamp(lo, hi) + } +} + +// ── Facts ───────────────────────────────────────────────────────────────────── + +const SHARDS: u64 = 1024; +const HOT: usize = 32; +const HOT_STATES: usize = 8; +const HOT_TAGS: usize = 16; +const HOT_NOTES: usize = 3; +const HOT_SLOTS: usize = HOT_STATES + HOT_TAGS + HOT_NOTES; +/// Re-asserts are chosen once this many ordinary slots are retracted. +const FLIP_CAP: usize = 200_000; + +#[derive(Clone)] +enum Val { + Int(i64), + Kw(String), + Str(String), + /// A ref to ordinary entity `n`. + Ref(u64), +} + +impl Val { + fn edn(&self) -> String { + match self { + Val::Int(n) => n.to_string(), + Val::Kw(k) => k.clone(), + Val::Str(s) => format!("\"{s}\""), + Val::Ref(n) => format!("#uuid \"{}\"", entity_uuid(&format!(":e{n}"))), + } + } + + fn render(&self) -> String { + match self { + Val::Int(n) => format!("i{n}"), + Val::Kw(k) => format!("k{k}"), + Val::Str(s) => render_str(s), + Val::Ref(n) => format!("r:e{n}"), + } + } +} + +fn render_str(s: &str) -> String { + if s.len() <= 24 { + format!("s\"{s}\"") + } else { + format!("s\"{}..{}\"#{}", &s[..12], &s[s.len() - 8..], s.len()) + } +} + +/// The UUID a keyword entity such as `:e5` maps to. +fn entity_uuid(keyword: &str) -> Uuid { + Uuid::new_v5(&Uuid::NAMESPACE_OID, keyword.as_bytes()) +} + +/// Renders a query value; refs and entities appear by name, never as a `Uuid`. +fn render_value(v: &Value, names: &HashMap) -> String { + match v { + Value::Integer(n) => format!("i{n}"), + Value::Keyword(k) => format!("k{k}"), + Value::String(s) => render_str(s), + Value::Ref(u) => match names.get(u) { + Some(name) => format!("r{name}"), + None => "r?".to_string(), + }, + Value::Boolean(b) => format!("b{b}"), + Value::Float(f) => format!("f{f}"), + Value::Null => "null".to_string(), + } +} + +/// The facts ordinary entity `n` is created with, in slot order: a unique +/// name, a score, two values of a sparse shard attribute, 1–4 tags (all in the +/// same transaction, #371), +/// a long string on every 50th entity, a ref on every 10th. +fn content(seed: u64, n: u64) -> Vec<(String, Val)> { + let h = mix(seed ^ n.wrapping_mul(0x2545_F491_4F6C_DD1D)); + let mut facts = vec![ + (":p/name".to_string(), Val::Str(format!("name-{n}"))), + (":p/score".to_string(), Val::Int((h % 1000) as i64)), + (format!(":p/shard{}", n % SHARDS), Val::Int(n as i64)), + (format!(":p/shard{}", n % SHARDS), Val::Int(-(n as i64) - 1)), + ]; + let start = (h >> 10) % 16; + for i in 0..1 + (h >> 20) % 4 { + // 5 is coprime with 16, so the tags are distinct. + facts.push(( + ":p/tag".to_string(), + Val::Kw(format!(":t{}", (start + i * 5) % 16)), + )); + } + if n % 50 == 0 { + facts.push(( + ":p/bio".to_string(), + Val::Str(format!("bio-{n}-{}", "b".repeat(80))), + )); + } + if n % 10 == 0 && n > 0 { + facts.push((":p/ref".to_string(), Val::Ref((h >> 30) % n))); + } + facts +} + +fn hot_fact(slot: usize) -> (&'static str, Val) { + if slot < HOT_STATES { + (":h/state", Val::Kw(format!(":s{slot}"))) + } else if slot < HOT_STATES + HOT_TAGS { + (":h/tag", Val::Kw(format!(":g{}", slot - HOT_STATES))) + } else { + let i = slot - HOT_STATES - HOT_TAGS; + (":h/note", Val::Str(format!("note-{i}-{}", "n".repeat(90)))) + } +} + +// ── Reference ───────────────────────────────────────────────────────────────── + +/// What the database must hold, without storing every fact: ordinary +/// entities follow from `content`, and only retracted slots are kept. +struct Reference { + seed: u64, + /// Ordinary entities `0..next_entity` exist. + next_entity: u64, + /// Facts the ordinary entities were created with. + created: u64, + flipped: Vec<(u64, u8)>, + flipped_at: HashMap<(u64, u8), usize>, + hot_live: [[bool; HOT_SLOTS]; HOT], + /// `tx << 16 | hot << 8 | slot << 1 | asserted`, in commit order. + hot_history: Vec, + /// Transactions committed (`current_tx_count`). + tx: u64, + /// Fact versions written: every assert and every retract. + versions: u64, +} + +impl Reference { + fn new(seed: u64) -> Self { + Reference { + seed, + next_entity: 0, + created: 0, + flipped: Vec::new(), + flipped_at: HashMap::new(), + hot_live: [[false; HOT_SLOTS]; HOT], + hot_history: Vec::new(), + tx: 0, + versions: 0, + } + } + + fn live_ordinary(&self) -> u64 { + self.created - self.flipped.len() as u64 + } + + fn live_hot(&self) -> u64 { + self.hot_live.iter().flatten().filter(|l| **l).count() as u64 + } + + fn is_flipped(&self, n: u64, slot: u8) -> bool { + self.flipped_at.contains_key(&(n, slot)) + } + + fn flip(&mut self, n: u64, slot: u8) { + self.flipped_at.insert((n, slot), self.flipped.len()); + self.flipped.push((n, slot)); + } + + fn unflip(&mut self, n: u64, slot: u8) { + let i = self.flipped_at.remove(&(n, slot)).expect("slot is flipped"); + self.flipped.swap_remove(i); + if let Some(moved) = self.flipped.get(i) { + self.flipped_at.insert(*moved, i); + } + } + + fn record_hot(&mut self, hot: usize, slot: usize, asserted: bool) { + self.hot_live[hot][slot] = asserted; + self.hot_history + .push(self.tx << 16 | (hot as u64) << 8 | (slot as u64) << 1 | asserted as u64); + } + + /// `[attr value]` rows of ordinary entity `n`, sorted. + fn entity_rows(&self, n: u64) -> Vec { + let mut rows: Vec = content(self.seed, n) + .into_iter() + .enumerate() + .filter(|(slot, _)| !self.is_flipped(n, *slot as u8)) + .map(|(_, (a, v))| format!("k{a} {}", v.render())) + .collect(); + rows.sort(); + rows + } + + /// `[attr value]` rows of hot entity `h`, now or as of transaction `t`. + fn hot_rows(&self, h: usize, as_of: Option) -> Vec { + let live = match as_of { + None => self.hot_live[h], + Some(t) => { + let mut live = [false; HOT_SLOTS]; + for r in &self.hot_history { + if r >> 16 > t { + break; + } + if (r >> 8 & 0xFF) as usize == h { + live[(r >> 1 & 0x7F) as usize] = r & 1 == 1; + } + } + live + } + }; + let mut rows: Vec = (0..HOT_SLOTS) + .filter(|s| live[*s]) + .map(|s| { + let (a, v) = hot_fact(s); + format!("k{a} {}", v.render()) + }) + .collect(); + rows.sort(); + rows + } + + /// `[entity value]` rows of attribute `:p/shard{k}`, sorted. + fn shard_rows(&self, k: u64) -> Vec { + let mut rows: Vec = (k..self.next_entity) + .step_by(SHARDS as usize) + .flat_map(|n| { + [(2, n as i64), (3, -(n as i64) - 1)] + .into_iter() + .filter(move |(slot, _)| !self.is_flipped(n, *slot)) + .map(move |(_, v)| format!(":e{n} i{v}")) + }) + .collect(); + rows.sort(); + rows + } +} + +// ── Configuration ───────────────────────────────────────────────────────────── + +struct Config { + seed: u64, + target_facts: u64, + /// Schedules count steps instead of milliseconds (`soak_short`). + step_clock: bool, + /// Long run: wall-clock budget. Short run: churn steps after growth. + budget: u64, + check_every: u64, + reopen_every: (u64, u64), + checkpoint_every: (u64, u64), + churn_tps: Option, + sample_entities: u64, + sample_names: u64, + sample_shards: u64, + sample_as_of: usize, + /// A full scan builds every row in memory, so it runs once when the live + /// count first reaches `full_scan_at`, and at the end only up to + /// `full_scan_max` live facts. + full_scan_at: u64, + full_scan_max: u64, + /// Fail on unbounded memory growth (spec §8). + memory_bound: bool, + metrics: Option, + dir: Option, +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .map(|v| { + v.parse() + .unwrap_or_else(|_| panic!("{name} must be an integer")) + }) + .unwrap_or(default) +} + +impl Config { + fn short(seed: u64) -> Self { + Config { + seed, + target_facts: 20_000, + step_clock: true, + budget: 400, + check_every: 150, + reopen_every: (60, 160), + checkpoint_every: (3, 12), + churn_tps: None, + sample_entities: 20, + sample_names: 3, + sample_shards: 2, + sample_as_of: 2, + full_scan_at: 10_000, + full_scan_max: u64::MAX, + memory_bound: false, + metrics: None, + dir: None, + } + } + + fn from_env() -> Self { + let seed = env_u64( + "MINIGRAF_SOAK_SEED", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos() as u64), + ); + let minute = 60_000; + Config { + seed, + target_facts: env_u64("MINIGRAF_SOAK_TARGET_FACTS", 10_000_000), + step_clock: false, + budget: env_u64("MINIGRAF_SOAK_MINUTES", 20) * minute, + check_every: env_u64("MINIGRAF_SOAK_CHECK_MINUTES", 10) * minute, + reopen_every: (2 * minute, 10 * minute), + checkpoint_every: (20, 200), + churn_tps: Some(env_u64("MINIGRAF_SOAK_CHURN_TPS", 50) as f64), + sample_entities: 200, + // #518: an AVET lookup scans the whole attribute (~6 s at 10M). + sample_names: 5, + sample_shards: 4, + sample_as_of: 4, + full_scan_at: 2_000_000, + full_scan_max: env_u64("MINIGRAF_SOAK_FULL_SCAN_MAX", 3_000_000), + memory_bound: true, + metrics: Some( + std::env::var_os("MINIGRAF_SOAK_METRICS") + .map_or_else(|| PathBuf::from("target/soak/metrics.jsonl"), PathBuf::from), + ), + dir: std::env::var_os("MINIGRAF_SOAK_DIR").map(PathBuf::from), + } + } +} + +// ── Metrics ─────────────────────────────────────────────────────────────────── + +const PATHS: [&str; 6] = ["eavt", "ea", "avet", "aevt", "as_of", "hot"]; + +#[derive(Default)] +struct Metrics { + /// Query latencies in µs since the last emitted line, per path. + latency: HashMap<&'static str, Vec>, + checkpoint_ms: Vec, + checkpoints: u64, + reopens: u64, + open_ms_file: Option, + open_ms_wal: Option, + /// Peak RSS of the last full-scan child process. + full_scan_rss: Option, + out: Option, +} + +fn percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + sorted[((sorted.len() - 1) as f64 * p).round() as usize] +} + +/// `(VmRSS, VmHWM)` in bytes; `None` where `/proc` is not available. +fn rss() -> Option<(u64, u64)> { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let field = |name: &str| -> Option { + let line = status.lines().find(|l| l.starts_with(name))?; + let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb * 1024) + }; + Some((field("VmRSS:")?, field("VmHWM:")?)) +} + +fn file_bytes(path: &Path) -> u64 { + let mut wal = path.as_os_str().to_os_string(); + wal.push(".wal"); + [path, Path::new(&wal)] + .iter() + .filter_map(|p| std::fs::metadata(p).ok()) + .map(|m| m.len()) + .sum() +} + +// ── Driver ──────────────────────────────────────────────────────────────────── + +#[derive(Clone, Copy, PartialEq)] +enum Phase { + Growth, + Churn, +} + +struct Soak { + cfg: Config, + path: PathBuf, + db: Option, + /// Drives every write, so a seed replays the same write sequence. + rng: Rng, + /// Drives sampling, reads and the schedule. + read_rng: Rng, + rf: Reference, + metrics: Metrics, + phase: Phase, + start: Instant, + steps: u64, + churn_start: Option<(u64, Instant)>, + churn_writes: u64, + since_checkpoint: u64, + checkpoint_due: u64, + next_check: u64, + next_reopen: u64, + /// RSS right after the first reopen-after-checkpoint of the churn phase. + rss_baseline: Option, + full_scanned: bool, +} + +fn open(path: &Path) -> Minigraf { + OpenOptions::new() + .wal_checkpoint_threshold(usize::MAX) + .path(path) + .open() + .unwrap_or_else(|e| panic!("open failed: {}", e.code())) +} + +impl Soak { + fn new(cfg: Config, path: PathBuf) -> Self { + let out = cfg.metrics.as_ref().map(|p| { + if let Some(dir) = p.parent() { + std::fs::create_dir_all(dir).expect("metrics directory"); + } + File::create(p).expect("metrics file") + }); + let mut read_rng = Rng(cfg.seed ^ 0x5EED); + let checkpoint_due = read_rng.range(cfg.checkpoint_every.0, cfg.checkpoint_every.1); + let next_reopen = read_rng.range(cfg.reopen_every.0, cfg.reopen_every.1); + Soak { + db: Some(open(&path)), + path, + rng: Rng(cfg.seed), + read_rng, + rf: Reference::new(cfg.seed), + metrics: Metrics { + out, + ..Metrics::default() + }, + phase: Phase::Growth, + start: Instant::now(), + steps: 0, + churn_start: None, + churn_writes: 0, + since_checkpoint: 0, + checkpoint_due, + next_check: cfg.check_every, + next_reopen, + rss_baseline: None, + full_scanned: false, + cfg, + } + } + + fn db(&self) -> &Minigraf { + self.db.as_ref().expect("database is open") + } + + fn clock(&self) -> u64 { + if self.cfg.step_clock { + self.steps + } else { + self.start.elapsed().as_millis() as u64 + } + } + + fn context(&self) -> String { + format!( + "seed {} step {} tx {} entities {}", + self.cfg.seed, self.steps, self.rf.tx, self.rf.next_entity + ) + } + + fn done(&self) -> bool { + if self.cfg.step_clock { + return self + .churn_start + .is_some_and(|(s, _)| self.steps - s >= self.cfg.budget); + } + if self.clock() < self.cfg.budget { + return false; + } + assert!( + self.phase == Phase::Churn, + "budget ended before {} live facts ({} reached); {}", + self.cfg.target_facts, + self.rf.live_ordinary(), + self.context() + ); + true + } + + fn run(&mut self) { + eprintln!( + "soak: seed {} target {} facts, budget {} {}", + self.cfg.seed, + self.cfg.target_facts, + self.cfg.budget, + if self.cfg.step_clock { + "churn steps" + } else { + "ms" + } + ); + while !self.done() { + self.steps += 1; + if !self.full_scanned && self.rf.live_ordinary() >= self.cfg.full_scan_at { + self.full_scanned = true; + let ms = self.full_scan(); + self.emit(Some(ms)); + } + if self.phase == Phase::Growth && self.rf.live_ordinary() >= self.cfg.target_facts { + self.enter_churn(); + } + if self.write_allowed() { + self.write_step(); + } else { + self.read_step(); + } + let now = self.clock(); + if now >= self.next_reopen { + let after_checkpoint = self.read_rng.chance(0.5); + self.reopen(after_checkpoint); + self.next_reopen = self.clock() + + self + .read_rng + .range(self.cfg.reopen_every.0, self.cfg.reopen_every.1); + } + if now >= self.next_check { + self.check_all(); + self.emit(None); + self.next_check = self.clock() + self.cfg.check_every; + } + } + self.finish(); + } + + fn enter_churn(&mut self) { + eprintln!( + "soak: growth done at {} live facts after {:?}; {}", + self.rf.live_ordinary(), + self.start.elapsed(), + self.context() + ); + self.phase = Phase::Churn; + self.churn_start = Some((self.steps, Instant::now())); + // The memory baseline: a fresh handle on a checkpointed file. + self.reopen(true); + self.check_all(); + self.emit(None); + } + + fn write_allowed(&self) -> bool { + match (self.phase, self.cfg.churn_tps, self.churn_start) { + (Phase::Churn, Some(tps), Some((_, at))) => { + (self.churn_writes as f64) < tps * at.elapsed().as_secs_f64() + } + _ => true, + } + } + + // ── Writes ──────────────────────────────────────────────────────────────── + + fn execute_write(&mut self, cmd: &str, facts: u64) { + if let Err(e) = self.db().execute(cmd) { + panic!("write failed ({}); {}", e.code(), self.context()); + } + self.rf.tx += 1; + self.rf.versions += facts; + if self.phase == Phase::Churn { + self.churn_writes += 1; + } + self.since_checkpoint += 1; + if self.since_checkpoint >= self.checkpoint_due { + self.checkpoint(); + } + } + + fn checkpoint(&mut self) { + let t = Instant::now(); + if let Err(e) = self.db().checkpoint() { + panic!("checkpoint failed ({}); {}", e.code(), self.context()); + } + self.metrics + .checkpoint_ms + .push(t.elapsed().as_millis() as u64); + self.metrics.checkpoints += 1; + self.since_checkpoint = 0; + self.checkpoint_due = self + .read_rng + .range(self.cfg.checkpoint_every.0, self.cfg.checkpoint_every.1); + } + + fn write_step(&mut self) { + let roll = self.rng.below(100); + let (grow, flip) = match self.phase { + Phase::Growth => (70, 85), + Phase::Churn => (10, 60), + }; + if roll < grow || self.rf.next_entity == 0 { + self.grow(); + } else if roll < flip { + self.flip_ordinary(); + } else { + self.churn_hot(); + } + } + + fn grow(&mut self) { + let want = self.rng.log_uniform(1, 2_000); + let mut cmd = String::from("(transact ["); + let mut facts = 0u64; + let first = self.rf.next_entity; + let mut n = first; + while facts < want || n == first { + for (a, v) in content(self.cfg.seed, n) { + cmd.push_str(&format!("[:e{n} {a} {}]", v.edn())); + facts += 1; + } + n += 1; + } + cmd.push_str("])"); + self.execute_write(&cmd, facts); + self.rf.next_entity = n; + self.rf.created += facts; + } + + fn flip_ordinary(&mut self) { + let k = self.rng.log_uniform(1, 50) as usize; + let reassert = !self.rf.flipped.is_empty() + && (self.rf.flipped.len() >= FLIP_CAP || self.rng.chance(0.5)); + let mut chosen: HashSet<(u64, u8)> = HashSet::new(); + if reassert { + for _ in 0..k { + let i = self.rng.below(self.rf.flipped.len() as u64) as usize; + chosen.insert(self.rf.flipped[i]); + } + } else { + for _ in 0..k * 2 { + let n = self.rng.below(self.rf.next_entity); + let slot = self.rng.below(content(self.cfg.seed, n).len() as u64) as u8; + if !self.rf.is_flipped(n, slot) { + chosen.insert((n, slot)); + } + if chosen.len() == k { + break; + } + } + } + if chosen.is_empty() { + return; + } + // Sorted so the command, and so a replayed run, is deterministic. + let mut chosen: Vec<(u64, u8)> = chosen.into_iter().collect(); + chosen.sort_unstable(); + let mut cmd = String::from(if reassert { + "(transact [" + } else { + "(retract [" + }); + for (n, slot) in &chosen { + let (a, v) = &content(self.cfg.seed, *n)[*slot as usize]; + cmd.push_str(&format!("[:e{n} {a} {}]", v.edn())); + } + cmd.push_str("])"); + self.execute_write(&cmd, chosen.len() as u64); + for (n, slot) in chosen { + if reassert { + self.rf.unflip(n, slot); + } else { + self.rf.flip(n, slot); + } + } + } + + fn churn_hot(&mut self) { + let live = self.rf.live_hot(); + let assert = live == 0 || (live < (HOT * HOT_SLOTS) as u64 && self.rng.chance(0.5)); + let k = self.rng.range(1, 8) as usize; + let mut chosen: Vec<(usize, usize)> = Vec::new(); + for _ in 0..k * 4 { + let h = self.rng.below(HOT as u64) as usize; + let s = self.rng.below(HOT_SLOTS as u64) as usize; + if self.rf.hot_live[h][s] != assert && !chosen.contains(&(h, s)) { + chosen.push((h, s)); + } + if chosen.len() == k { + break; + } + } + if chosen.is_empty() { + return; + } + let mut cmd = String::from(if assert { "(transact [" } else { "(retract [" }); + for (h, s) in &chosen { + let (a, v) = hot_fact(*s); + cmd.push_str(&format!("[:h{h} {a} {}]", v.edn())); + } + cmd.push_str("])"); + self.execute_write(&cmd, chosen.len() as u64); + for (h, s) in chosen { + self.rf.record_hot(h, s, assert); + } + } + + // ── Reads and checks ────────────────────────────────────────────────────── + + /// Runs `query`, times it under `path`, and renders each row. + fn rows( + &mut self, + path: &'static str, + query: &str, + names: &HashMap, + ) -> Vec { + let t = Instant::now(); + let result = self.db().execute(query); + let us = t.elapsed().as_micros() as u64; + self.metrics.latency.entry(path).or_default().push(us); + match result { + Ok(QueryResult::QueryResults { results, .. }) => { + let mut rows: Vec = results + .iter() + .map(|row| { + row.iter() + .map(|v| render_value(v, names)) + .collect::>() + .join(" ") + }) + .collect(); + rows.sort(); + rows + } + Ok(_) => panic!("not a query result: {query}; {}", self.context()), + Err(e) => panic!("query failed ({}): {query}; {}", e.code(), self.context()), + } + } + + fn expect_rows(&self, what: &str, got: Vec, want: Vec) { + if got != want { + let missing: Vec<&String> = want.iter().filter(|r| !got.contains(r)).take(10).collect(); + let extra: Vec<&String> = got.iter().filter(|r| !want.contains(r)).take(10).collect(); + panic!( + "{what}: {} rows, expected {}; missing {missing:?}; unexpected {extra:?}; {}", + got.len(), + want.len(), + self.context() + ); + } + } + + /// Names for the referents of entity `n`'s ref slot. + fn ref_names(&self, n: u64) -> HashMap { + content(self.cfg.seed, n) + .into_iter() + .filter_map(|(_, v)| match v { + Val::Ref(m) => Some((entity_uuid(&format!(":e{m}")), format!(":e{m}"))), + _ => None, + }) + .collect() + } + + fn check_entity(&mut self, n: u64) { + let names = self.ref_names(n); + let got = self.rows( + "eavt", + &format!("(query [:find ?a ?v :where [:e{n} ?a ?v]])"), + &names, + ); + self.expect_rows(&format!("EAVT :e{n}"), got, self.rf.entity_rows(n)); + // Entity and attribute bound, on the multi-valued tags (#371). + let got = self.rows( + "ea", + &format!("(query [:find ?v :where [:e{n} :p/tag ?v]])"), + &HashMap::new(), + ); + let want: Vec = self + .rf + .entity_rows(n) + .into_iter() + .filter_map(|r| r.strip_prefix("k:p/tag ").map(str::to_string)) + .collect(); + self.expect_rows(&format!("EA :e{n} :p/tag"), got, want); + } + + fn check_name(&mut self, n: u64) { + let names = HashMap::from([(entity_uuid(&format!(":e{n}")), format!(":e{n}"))]); + let got = self.rows( + "avet", + &format!("(query [:find ?e :where [?e :p/name \"name-{n}\"]])"), + &names, + ); + let want = if self.rf.is_flipped(n, 0) { + vec![] + } else { + vec![format!("r:e{n}")] + }; + self.expect_rows(&format!("AVET name-{n}"), got, want); + } + + fn check_shard(&mut self, k: u64) { + let names: HashMap = (k..self.rf.next_entity) + .step_by(SHARDS as usize) + .map(|n| (entity_uuid(&format!(":e{n}")), format!(":e{n}"))) + .collect(); + let got = self.rows( + "aevt", + &format!("(query [:find ?e ?v :where [?e :p/shard{k} ?v]])"), + &names, + ); + // `?e` renders as a ref; the reference writes the bare name. + let got = got + .into_iter() + .map(|r| r.trim_start_matches('r').to_string()) + .collect(); + self.expect_rows(&format!("AEVT :p/shard{k}"), got, self.rf.shard_rows(k)); + } + + fn check_hot(&mut self, h: usize, as_of: Option) { + let (path, clause) = match as_of { + Some(t) => ("as_of", format!(":as-of {t} ")), + None => ("hot", String::new()), + }; + let got = self.rows( + path, + &format!("(query [:find ?a ?v {clause}:where [:h{h} ?a ?v]])"), + &HashMap::new(), + ); + let label = match as_of { + Some(t) => format!("hot :h{h} as of {t}"), + None => format!("hot :h{h}"), + }; + self.expect_rows(&label, got, self.rf.hot_rows(h, as_of)); + } + + /// One checked read between capped churn writes. + fn read_step(&mut self) { + let n = self.read_rng.below(self.rf.next_entity); + if self.read_rng.chance(0.98) { + self.check_entity(n); + } else { + self.check_name(n); + } + } + + fn check_all(&mut self) { + let tx = self.db().current_tx_count(); + assert_eq!(tx, self.rf.tx, "tx counter; {}", self.context()); + if self.rf.next_entity > 0 { + for _ in 0..self.cfg.sample_entities { + let n = self.read_rng.below(self.rf.next_entity); + self.check_entity(n); + } + for _ in 0..self.cfg.sample_names { + let n = self.read_rng.below(self.rf.next_entity); + self.check_name(n); + } + for _ in 0..self.cfg.sample_shards { + let k = self.read_rng.below(SHARDS); + self.check_shard(k); + } + } + for h in 0..HOT { + self.check_hot(h, None); + } + for _ in 0..self.cfg.sample_as_of { + let h = self.read_rng.below(HOT as u64) as usize; + for _ in 0..3 { + let t = self.read_rng.range(1, self.rf.tx.max(1)); + self.check_hot(h, Some(t)); + } + } + } + + fn reopen(&mut self, after_checkpoint: bool) { + if after_checkpoint { + self.checkpoint(); + } + drop(self.db.take()); + let t = Instant::now(); + self.db = Some(open(&self.path)); + let ms = t.elapsed().as_millis() as u64; + self.metrics.reopens += 1; + if after_checkpoint { + self.metrics.open_ms_file = Some(ms); + } else { + self.metrics.open_ms_wal = Some(ms); + } + self.check_all(); + if after_checkpoint && self.phase == Phase::Churn { + self.check_memory(); + } + } + + /// Spec §8: a fresh handle on a checkpointed file stays near the + /// baseline taken when the churn phase began. + fn check_memory(&mut self) { + let Some((now, _)) = rss() else { return }; + let Some(base) = self.rss_baseline else { + self.rss_baseline = Some(now); + eprintln!("soak: memory baseline {} MiB", now >> 20); + return; + }; + let limit = base + base / 4 + (256 << 20); + if self.cfg.memory_bound && now > limit { + panic!( + "RSS {} MiB above the bound {} MiB (baseline {} MiB); {}", + now >> 20, + limit >> 20, + base >> 20, + self.context() + ); + } + } + + fn emit(&mut self, full_scan_ms: Option) { + let mut query_us = serde_json::Map::new(); + for path in PATHS { + let mut v = self.metrics.latency.remove(path).unwrap_or_default(); + v.sort_unstable(); + query_us.insert( + path.to_string(), + serde_json::json!({"n": v.len(), "p50": percentile(&v, 0.5), "p99": percentile(&v, 0.99)}), + ); + } + let mut ck = std::mem::take(&mut self.metrics.checkpoint_ms); + ck.sort_unstable(); + let (rss_now, rss_peak) = rss().map_or((None, None), |(a, b)| (Some(a), Some(b))); + let line = serde_json::json!({ + "t_s": self.start.elapsed().as_secs(), + "phase": if self.phase == Phase::Growth { "growth" } else { "churn" }, + "tx": self.rf.tx, + "live_facts": self.rf.live_ordinary() + self.rf.live_hot(), + "versions": self.rf.versions, + "file_bytes": file_bytes(&self.path), + "open_ms_file": self.metrics.open_ms_file, + "open_ms_wal": self.metrics.open_ms_wal, + "reopens": self.metrics.reopens, + "checkpoints": self.metrics.checkpoints, + "checkpoint_ms_p50": percentile(&ck, 0.5), + "checkpoint_ms_max": ck.last().copied().unwrap_or(0), + "rss_bytes": rss_now, + "peak_rss_bytes": rss_peak, + "query_us": query_us, + "full_scan_ms": full_scan_ms, + "full_scan_peak_rss_bytes": self.metrics.full_scan_rss, + }); + eprintln!("soak: {line}"); + if let Some(out) = self.metrics.out.as_mut() { + writeln!(out, "{line}").expect("write metrics"); + } + } + + /// Checks the row count of `[?e ?a ?v]` in a child process, which + /// prints the count, the time and its own peak RSS. A query builds its + /// whole answer when it opens (#432), about 1 KB per row here, and the + /// allocator keeps that memory after it is freed: run in this process, it + /// would hide a leak of that size from the memory bound (spec §8). + fn full_scan(&mut self) -> u64 { + self.checkpoint(); + drop(self.db.take()); + let out = Command::new(std::env::current_exe().expect("test binary path")) + .args(["soak_full_scan_child", "--exact", "--nocapture"]) + .env("MINIGRAF_SOAK_SCAN_DB", &self.path) + .output() + .expect("spawn full-scan child"); + let stdout = String::from_utf8_lossy(&out.stdout); + let Some(line) = stdout.lines().find_map(|l| l.strip_prefix("soak-scan: ")) else { + let stderr = String::from_utf8_lossy(&out.stderr); + let tail: Vec<&str> = stderr.lines().rev().take(20).collect(); + panic!("full-scan child failed: {tail:?}; {}", self.context()); + }; + let field = |name: &str| -> Option { + let mut words = line.split_whitespace(); + words.find(|w| *w == name)?; + words.next()?.parse().ok() + }; + let rows = field("rows").expect("child row count"); + let ms = field("ms").expect("child time"); + self.metrics.full_scan_rss = field("peak"); + self.db = Some(open(&self.path)); + assert_eq!( + rows, + self.rf.live_ordinary() + self.rf.live_hot(), + "full scan row count; {}", + self.context() + ); + ms + } + + fn finish(&mut self) { + self.reopen(true); + let full_scan_ms = if self.rf.live_ordinary() <= self.cfg.full_scan_max { + Some(self.full_scan()) + } else { + eprintln!( + "soak: final full scan skipped above {} live facts", + self.cfg.full_scan_max + ); + None + }; + let report = self.db().verify().expect("verify runs"); + assert!( + report.is_ok(), + "verify found {} problems; {}", + report.problems.len(), + self.context() + ); + self.emit(full_scan_ms); + eprintln!( + "soak: done: {} live facts, {} versions, {} tx, {} checkpoints, {} reopens in {:?}", + self.rf.live_ordinary() + self.rf.live_hot(), + self.rf.versions, + self.rf.tx, + self.metrics.checkpoints, + self.metrics.reopens, + self.start.elapsed() + ); + } +} + +fn run(cfg: Config) { + let tmp = match &cfg.dir { + Some(dir) => { + std::fs::create_dir_all(dir).expect("soak directory"); + tempfile::tempdir_in(dir).expect("temp dir") + } + None => tempfile::tempdir().expect("temp dir"), + }; + let path = tmp.path().join("soak.graph"); + Soak::new(cfg, path).run(); +} + +/// The driver at a size a debug build runs in seconds. +#[test] +fn soak_short() { + run(Config::short(392)); +} + +/// The full soak: see the module docs for its variables. +#[test] +#[ignore] +fn soak() { + run(Config::from_env()); +} + +/// Child half of `Soak::full_scan`. A no-op when `MINIGRAF_SOAK_SCAN_DB` is +/// absent, as in `crash_kill_child_entrypoint`. +#[test] +fn soak_full_scan_child() { + let Some(path) = std::env::var_os("MINIGRAF_SOAK_SCAN_DB") else { + return; + }; + let db = open(Path::new(&path)); + let t = Instant::now(); + let rows = match db.execute("(query [:find ?e ?a ?v :where [?e ?a ?v]])") { + Ok(QueryResult::QueryResults { results, .. }) => results.len(), + Ok(_) => panic!("full scan: not a query result"), + Err(e) => panic!("full scan failed ({})", e.code()), + }; + let ms = t.elapsed().as_millis(); + let peak = rss().map_or(0, |(_, peak)| peak); + println!("soak-scan: rows {rows} ms {ms} peak {peak}"); +}