From 58d4c7a7336b05e7ef6e44d92b90f810718da94a Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 10 Oct 2026 17:50:58 +0530 Subject: [PATCH] test: weekly soak test at 10M facts with checked samples and metrics (#392, v2.x) The v2.x copy of the v3 soak test (PR #519): no transaction writes or retracts two values of one entity and attribute (#371; the v3 workload fails on v2.0.4), no verify(), a memory bound scaled by the fact versions held, and fewer checkpoints and :as-of samples since both cost time in the database size on v2.x. Adds .github/workflows/soak.yml, which alternates main (Saturday) and v3 (Sunday) weekly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012wXxZguv7PgKQjMMH4yJ5h --- .github/workflows/soak.yml | 125 ++ CHANGELOG.md | 4 + CLAUDE.md | 2 +- README.md | 2 +- ROADMAP.md | 2 +- docs/TEST_COVERAGE.md | 7 +- .../superpowers/plans/2026-10-10-soak-test.md | 30 + .../specs/2026-10-10-soak-test-design.md | 217 +++ tests/soak_test.rs | 1168 +++++++++++++++++ 9 files changed, 1551 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/soak.yml create mode 100644 docs/superpowers/plans/2026-10-10-soak-test.md create mode 100644 docs/superpowers/specs/2026-10-10-soak-test-design.md create mode 100644 tests/soak_test.rs diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml new file mode 100644 index 0000000000..fcfa7309e3 --- /dev/null +++ b/.github/workflows/soak.yml @@ -0,0 +1,125 @@ +name: Soak test (weekly) + +# Two branches alternate weekly: main on Saturday, v3 on Sunday, both 01:00 UTC. +# Scheduled runs always use this file from the default branch, so the branch +# under test is chosen from the cron string that fired (as in bench.yml and +# crash-kill.yml). Keep the cron strings here and in SOAK_REF/run-name +# identical. +run-name: >- + Soak test (${{ github.event_name == 'workflow_dispatch' && inputs.ref + || (github.event.schedule == '0 1 * * 0' && 'v3' || 'main') }}) + +# #392: grows a file-backed database to 10M+ facts, then churns it (retracts, +# re-asserts, long hot-entity histories) for the rest of the budget, with +# thousands of checkpoints and reopens every few minutes. Samples are checked +# against a reference throughout; the metrics (file size, open and checkpoint +# time, RSS, p50/p99 query latency) are uploaded for the performance envelope +# (#394). The test prints its seed; MINIGRAF_SOAK_SEED replays the writes. + +on: + schedule: + - cron: "0 1 * * 6" # main + - cron: "0 1 * * 0" # v3 + workflow_dispatch: + inputs: + ref: + description: Branch to test + required: true + default: main + minutes: + description: Workload budget in minutes + required: true + default: "300" + +permissions: + contents: read + issues: write + +env: + SOAK_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || (github.event.schedule == '0 1 * * 0' && 'v3' || 'main') }} + +jobs: + soak: + name: Soak at 10M facts + runs-on: ubuntu-latest + timeout-minutes: 360 + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.SOAK_REF }} + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + + # The database grows to several GB on main (format v7); /mnt has the + # runner's spare disk. + - name: Prepare the data directory + run: | + sudo mkdir -p /mnt/soak + sudo chown "$USER" /mnt/soak + df -h / /mnt + + - name: Build + run: cargo test --profile bench --test soak_test --no-run + + - name: Run the soak test + shell: bash + env: + MINIGRAF_SOAK_MINUTES: ${{ github.event_name == 'workflow_dispatch' && inputs.minutes || '300' }} + MINIGRAF_SOAK_TARGET_FACTS: "10000000" + MINIGRAF_SOAK_METRICS: ${{ github.workspace }}/soak-metrics.jsonl + MINIGRAF_SOAK_DIR: /mnt/soak + RUST_BACKTRACE: 1 + run: | + set -o pipefail + cargo test --profile bench --test soak_test soak -- --ignored --exact --nocapture 2>&1 | tee soak-output.txt + + - name: Upload metrics + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: soak-${{ env.SOAK_REF }}-${{ github.run_id }} + path: | + soak-metrics.jsonl + soak-output.txt + retention-days: 90 + if-no-files-found: warn + + - name: Record soak failure + if: ${{ failure() }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TITLE: "Weekly soak failure (${{ env.SOAK_REF }})" + run: | + report_file="soak-failure.md" + { + echo "The weekly soak test failed on ${{ env.SOAK_REF }}." + echo + echo "- Run: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + echo "- Commit: $(git rev-parse HEAD)" + echo "- Reproduce: cargo test --profile bench --test soak_test soak -- --ignored --exact --nocapture" + echo " (MINIGRAF_SOAK_SEED to the printed seed replays the writes; see tests/soak_test.rs for the other variables)" + echo + echo "## Test output" + echo + echo '```text' + tail -n 300 soak-output.txt 2>/dev/null || echo "(no output: the run failed before the test started)" + echo '```' + } > "$report_file" + + existing_issue="$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --search "$ISSUE_TITLE in:title" \ + --json number,title \ + --limit 100 \ + --jq ".[] | select(.title == \"$ISSUE_TITLE\") | .number" \ + | head -n 1)" + + if [ -n "$existing_issue" ]; then + gh issue comment "$existing_issue" --repo "$GITHUB_REPOSITORY" --body-file "$report_file" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$report_file" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index b5e01d1214..0161f4b603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A file shorter than one page is refused at open with `STG-045` unless it is a torn first write (#506).** Open treated any file under 4 KiB as a new database and wrote a new header over it, so a database cut below one page by an interrupted copy, or an unrelated file at the path, was silently replaced. Such a file now opens as a new database only when its bytes are zeros or a prefix of the header a new database writes (what a crash during creation can leave); anything else is `STG-045` and the file is not modified. A zero-length file is still a new database. New code `STG-045`, numbered as in v3.0.0, which also fixes this. - **A commit made after a torn WAL entry is no longer lost on recovery (#513).** Recovery replays the WAL up to its first partial or damaged entry, but the writer appended after the end of the file. After a crash during an append, the next session's commits went in after the partial entry, so a second crash before any checkpoint lost them, although each had been fsynced and reported as committed. A failed append (short write, full disk, I/O error) did the same to the next commit through that handle. Opening a database for writing now cuts the WAL back to the end of its last valid entry, and syncs it, before anything is appended. After a failed append, every later write through that handle fails with the new `WAL-007` until the database is reopened or a `checkpoint()` succeeds. The transaction whose append failed is not applied; after a reopen it is there in full or not at all. Affected every release since v0.5.0. Also fixed in v3.0.0. +### Tests + +- **Weekly soak test at scale (#392).** `tests/soak_test.rs` grows a file-backed database to 10M live facts in batches of 1–2,000 facts, then churns it for the rest of a 5-hour budget at 50 transactions a second (retracts, re-asserts, long histories on 32 hot entities), with a checkpoint every 20–200 transactions and a reopen every 2–10 minutes. Samples are checked against a reference through EAVT, entity and attribute, AVET, AEVT, `:as-of` and the transaction counter, with a full scan in a child process at 2M facts. RSS after a reopen is bounded relative to the fact versions held. Metrics go to a JSON-lines file that `.github/workflows/soak.yml` uploads weekly, alternating `main` (Saturday) and `v3` (Sunday). On v2.x no transaction writes two values of one entity and attribute, because of the known issue #371; the v3 workload fails here. It also checkpoints every 100–1,000 transactions and samples `:as-of` less, since both cost time in the database size on v2.x. `soak_short` runs the driver at 20K facts in the normal suite. + ## v2.0.4 — 2026-10-08 Patch release on the v2.x line under the [support policy](PHILOSOPHY.md#support-policy): one `BrowserDb` durability hardening fix, plus new compatibility and crash tests. File format is unchanged (v7). No API changes. Native and WASI builds behave exactly as in v2.0.3. diff --git a/CLAUDE.md b/CLAUDE.md index 6309efe1d3..9c0d7e9e91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -174,7 +174,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1231 tests** (1223 passing, 8 ignored; unit + integration + doc). +**1234 tests** (1225 passing, 9 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 79e6dc60aa..2b2160260a 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1231 tests +cargo test # run 1234 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/ROADMAP.md b/ROADMAP.md index 846b5e0b03..4bad7607eb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -79,7 +79,7 @@ Scope — release-gate testing and documentation: - Wider property tests: file-backed, batched, retracts, temporal, joins (#386) - Persistent fuzz corpus and an operation-sequence target (#387) - Fault injection across save/WAL/recovery (#390) -- Long-running soak test at scale (#392) +- Long-running soak test at scale (#392; `tests/soak_test.rs`, 10M facts and a 5-hour churn weekly, alternating `main` and `v3`) - Benchmarks at 100K–10M facts and a published performance envelope (#394) - Higher storage/WAL coverage gates and branch-coverage gates (#396) - Durability, recovery and operations guide (#398) diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index d3b24084df..7c5bdab061 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,7 +4,7 @@ **Verified**: 2026-10-10 with `cargo test` -**Result**: 1,223 passing tests and 8 ignored tests (1,231 total), including the golden-file corpus (#391) +**Result**: 1,225 passing tests and 9 ignored tests (1,234 total), including the golden-file corpus (#391) Since v2.0.4, `tests/edge_cases_test.rs` checks that a file of 1 to 4,095 bytes is `STG-045`, unchanged and with no WAL, when it is a database cut below one page or foreign bytes, and opens as a new database that takes a write and reads it back after reopen when it is a prefix of the initial header or all zeros (#506). @@ -24,15 +24,16 @@ The v2.0.4 suite covers the public database API, the Datalog engine, storage and - **Storage correctness**: packed pages, B+tree indexes and range scans, cache behavior, file-header validation, migration, WAL replay, checksums, corruption handling, and fault injection. - **Reliability and concurrency**: concurrent reads and writes, rollback behavior, same-process handle exclusion, cross-process locking, PID namespaces, NFS and container lock behavior, and SIGKILL recovery. - **Public diagnostics**: structured parser, query, storage, WAL, API, and internal error codes; the error-code registry is checked against [`ERROR_REFERENCE.md`](ERROR_REFERENCE.md). -- **Compatibility and quality**: golden-file corpus of released file formats, cross-platform format compatibility, property-based tests, long-haul smoke coverage, XTDB/Datomic semantic compatibility, grammar conformance, and rustdoc examples. +- **Compatibility and quality**: golden-file corpus of released file formats, cross-platform format compatibility, property-based tests, long-haul smoke coverage, a weekly soak at 10M facts with a 5-hour churn (#392), XTDB/Datomic semantic compatibility, grammar conformance, and rustdoc examples. ### Ignored Tests -The eight ignored tests are intentionally excluded from a normal local run: +The nine ignored tests are intentionally excluded from a normal local run: - Six rustdoc examples that reference internal types and cannot compile as standalone examples. - One high-contention concurrency stress test intended for scheduled runs. - One long-haul smoke test intended for scheduled runs. +- One soak test at 10M facts (`soak`), run weekly by `.github/workflows/soak.yml` (#392); `soak_short` runs the same driver at 20K facts in the normal suite. ### Reproducing diff --git a/docs/superpowers/plans/2026-10-10-soak-test.md b/docs/superpowers/plans/2026-10-10-soak-test.md new file mode 100644 index 0000000000..f469da326b --- /dev/null +++ b/docs/superpowers/plans/2026-10-10-soak-test.md @@ -0,0 +1,30 @@ +# Soak test at scale — Plan + +Spec: `docs/superpowers/specs/2026-10-10-soak-test-design.md` (#392). + +## v3 PR (`test/392-soak` → `v3`) + +1. `tests/soak_test.rs` + - `Config` (from env for `soak`, fixed for `soak_short`), `Every` schedule by + time or by step count so the short run is deterministic in a debug build. + - `Reference`: `content(n)` slots, `flipped` Vec + index map, hot history as + packed `u64`s, `tx`. + - `Driver`: op choice by phase weights, checkpoint and reopen schedule, + churn rate cap with checked reads in between. + - Checks: EAVT, AVET, AEVT shard, hot `:as-of`, tx counter; end: full-scan + count and `verify()`. + - `Metrics`: latency samples per path, checkpoint times, open times, RSS + from `/proc/self/status`; JSON lines via `serde_json`. + - Memory bound after the growth phase (spec §8). +2. Run `soak_short` in debug; run `soak` locally with a reduced target and a + few minutes; then a full 10M run under `--profile bench`. +3. `.github/workflows/soak.yml` (spec §9). +4. Docs: `docs/TEST_COVERAGE.md`, `CLAUDE.md` test count, `ROADMAP.md` if it + lists #392, `CHANGELOG.md` Unreleased. +5. PR to `v3` without a closing keyword for the tracker; `Closes #392` only + in the main PR description if the user wants it closed on merge. + +## main PR (`test/392-soak-v2` → `main`) + +Same test with `verify()` dropped and the v2 memory rule (spec §8); same +`soak.yml`. Run `soak_short` and a short `soak`. diff --git a/docs/superpowers/specs/2026-10-10-soak-test-design.md b/docs/superpowers/specs/2026-10-10-soak-test-design.md new file mode 100644 index 0000000000..e785887de1 --- /dev/null +++ b/docs/superpowers/specs/2026-10-10-soak-test-design.md @@ -0,0 +1,217 @@ +# Long-running soak test at scale — Design + +**Issue:** #392, tracker #383 +**Milestone:** v3.0.0. Test and workflow only; no library change. +**Branches:** a v3 PR first, then a `main` PR with the same harness adapted to +the v2 API (no `verify()`). The weekly workflow alternates between the two. + +## 1. Goal + +No test today runs for hours or at the size of a real deployment. The nightly +smoke suite has a 15-minute timeout and 5,000 facts. Bugs like #370 appeared +only after several checkpoints on a real workload. The soak test grows a +file-backed database to 10M+ facts with mixed batches, multi-valued +attributes, retracts and long histories on hot entities. It checkpoints and +reopens many times, checks answers against a reference throughout, and records +the numbers #394 needs. + +## 2. Measured feasibility (local, `--profile bench`) + +| | v3 (format v8) | main (format v7) | +|---|---|---| +| Insert rate (100-fact batches) | ~70K facts/s | ~70K facts/s | +| Checkpoint at 1M / 3.8M facts | 180 ms / 250 ms | 1.0 s / 2.6 s (linear) | +| File size per fact | ~110 B | ~430 B (≈4.3 GB at 10M) | + +A GitHub runner is about 2–3× slower. Both branches reach 10M facts well +inside the time budget, so the run is time-boxed and the extra time goes into +churn. + +## 3. Shape + +`tests/soak_test.rs`, `#![cfg(not(target_arch = "wasm32"))]`, two tests: + +- `soak` (`#[ignore]`): the full run, tuned by environment variables. +- `soak_full_scan_child`: a no-op unless the parent sets + `MINIGRAF_SOAK_SCAN_DB` (§6). +- `soak_short` (runs in the per-PR suite): the same driver with a tiny + configuration (about 20K facts, 3 reopens, 3 checks), a few seconds in a + debug build. It keeps the harness compiling and correct. + +| Variable | Default (`soak`) | Meaning | +|---|---|---| +| `MINIGRAF_SOAK_MINUTES` | 20 | Wall-clock budget for the workload | +| `MINIGRAF_SOAK_TARGET_FACTS` | 10,000,000 | Live facts the growth phase reaches | +| `MINIGRAF_SOAK_CHECK_MINUTES` | 10 | Interval between sampled checks | +| `MINIGRAF_SOAK_CHURN_TPS` | 50 | Write rate cap in the churn phase (§5) | +| `MINIGRAF_SOAK_SEED` | random, printed | Replays a run | +| `MINIGRAF_SOAK_METRICS` | `target/soak/metrics.jsonl` | Metrics output | +| `MINIGRAF_SOAK_FULL_SCAN_MAX` | 3,000,000 | Largest live count the final full scan runs at (§6) | +| `MINIGRAF_SOAK_DIR` | a temp dir | Where the `.graph` lives (the workflow points it at the runner's large disk) | + +It runs under `--profile bench` (optimized, `panic = "unwind"`). The release +profile aborts on panic and cannot run tests. + +## 4. Reference without storing 10M facts + +The process's own peak RSS is a measured number, so the reference must stay +small. Every ordinary entity's facts follow from its index, and only changes +are stored. + +**Ordinary entities** `:e{n}` (`n = 0, 1, 2, …`). `content(n)` is a fixed +function of `n` and the seed: + +- `:p/name "name-{n}"` (unique, for AVET lookups) +- `:p/score` integer `hash(n) % 1000` +- `:p/shard{n % 1024}`, two integers `n` and `-n - 1` in the same transaction + (a sparse, multi-valued attribute: about 1,700 entities each at 10M facts, so + an AEVT scan of one shard is cheap to check exactly) +- `:p/tag`, 1–4 keyword values in the same transaction (multi-valued, #371) +- every 50th entity: `:p/bio`, a string over 64 bytes (value pages) +- every 10th entity: `:p/ref`, a ref to `:e{hash(n) % n}` + +Each fact of `content(n)` has a slot number. An entity is created in one +transaction, and a batch holds several entities. The reference keeps: + +- `next_entity`: entities `0..next_entity` exist. +- `flipped`: slots `(entity, slot)` currently retracted, a `Vec` plus an + index map so a random one can be picked in O(1). + A retract op picks a random existing entity and slot. If the slot is live it + is retracted and inserted into `flipped`; if not it is re-asserted and + removed. +- `live_count`: total facts minus `flipped.len()`. + +The expected live facts of any entity, or of any shard, are computed from +`content` and `flipped`. Every entity can be sampled. + +**Hot entities** `:h0`–`:h31` carry long histories. Each churn step changes a +few of them: retract and re-assert `:h/state` (one of 8 values), and add or +retract `:h/tag` values (multi-valued). The reference keeps every hot change +as `(tx_count, triple, assert|retract)`, so `:as-of` at any past transaction +can be checked. A record packs into one `u64`; a 5-hour run keeps a few +million of them. + +**Transaction counter:** each `execute` increments the reference `tx`. After +every reopen `current_tx_count()` must equal it. + +All facts use the default valid time. Valid-time windows are covered by the +model-based and property tests (#385, #386). + +## 5. Workload + +One loop until the budget ends. Each step picks one op: + +| Op | Weight (growth / churn) | Detail | +|---|---|---| +| Grow | 70% / 10% | New entities; batch size log-uniform over 1–2,000 facts | +| Retract/re-assert | 15% / 50% | 1–50 random ordinary slots: live ones retracted in one `retract`, or retracted ones re-asserted in one `transact` | +| Hot churn | 15% / 40% | 1–8 changes on hot entities in one `transact` or `retract` | + +The growth phase runs at full speed until `live_count` reaches the target +(the run fails if the budget ends first). The churn phase uses the rest of the +budget at a capped write rate, `MINIGRAF_SOAK_CHURN_TPS` (default 50 +transactions per second): at full speed it would add tens of millions of fact +versions, more than main's in-memory store and disk can hold. Time between +writes goes to extra checked reads: an EAVT (98%) or AVET (2%) lookup of a random entity, +compared with the reference and timed like the scheduled checks. Re-asserts +are chosen once `flipped` holds 200K slots, so the live count stays roughly +flat while the history grows by a few million versions. + +Interleaved, by a seeded schedule: + +- `checkpoint()` every 20–200 transactions (v3: thousands of checkpoints per + run). Times are recorded. +- Every handle is opened with `wal_checkpoint_threshold(usize::MAX)`, so only + the schedule above checkpoints and dropping a handle does not. +- Reopen every 2–10 minutes: half the time right after `checkpoint()` (open + reads the file only), half with up to 200 transactions still in the WAL + (open replays them). Open time is recorded for each kind. + +## 6. Checks + +Every `MINIGRAF_SOAK_CHECK_MINUTES`, after every reopen, and at the end: + +- **EAVT:** `[:e{n} ?a ?v]` for 200 random entities and all 32 hot entities; + rendered rows equal the reference. +- **EA:** `[:e{n} :p/tag ?v]` for the same entities. With the multi-valued + shard attribute this is the shape #371 broke; on v2.0.4 the unmodified + workload fails these checks, so the test can see that bug. +- **AVET:** `[?e :p/name "name-{n}"]` for 5 random entities: present + exactly when that slot is live. This shape scans the whole attribute today + (#518: 0.5 s at 1M facts, 6 s at 10M), so it is sampled sparingly; raise + the sample once #518 is fixed. +- **AEVT:** `[?e :p/shard{k} ?v]` for 4 random shards; the result equals + every live shard slot of that shard. +- **`:as-of`:** for 4 hot entities and 3 random past transactions each, the + rows equal the history replayed to that transaction. +- **Counter:** `current_tx_count()` equals the reference. + +- **Full scan:** the row count of `[?e ?a ?v]` equals `live_count` plus the + hot entities' live facts. A query builds its whole answer when it opens + (#432), so a full scan costs memory per row: about 300 B at 1M simple + facts, about 1.1 KB here (2.2 GB at 2M); a `count` aggregate costs the same. + The allocator keeps that memory after the query, so in this process it would + hide a leak of that size from §8. The scan therefore runs in a child process + (the test binary re-run as `soak_full_scan_child`, as `crash_kill_test` + does) after a checkpoint, with the parent's handle closed: once, when the + live count first reaches 2M, and at the end only up to + `MINIGRAF_SOAK_FULL_SCAN_MAX` (default 3M) live facts. The child reports the + row count, the time and its peak RSS. + +At the end, after a final reopen: + +- **v3 only:** `verify()` reports no problems. + +Rendered rows never carry a `Uuid` (CLAUDE.md test conventions); failures name +the seed, the step and the check. + +## 7. Metrics + +The test appends one JSON line per check to the metrics file: + +```json +{"t_s": 3600, "phase": "churn", "tx": 41230, "live_facts": 10012345, + "versions": 12500000, "file_bytes": 1234567890, "open_ms_file": 812, "open_ms_wal": 940, + "reopens": 40, "checkpoints": 2100, "checkpoint_ms_p50": 210, "checkpoint_ms_max": 900, + "rss_bytes": 512000000, "peak_rss_bytes": 700000000, + "query_us": {"eavt": [p50, p99], "avet": [...], "aevt": [...], "as_of": [...]}} +``` + +Lines written after a full scan also carry `full_scan_ms` and +`full_scan_peak_rss_bytes` (the child's). RSS +and peak RSS come from `/proc/self/status` (`VmRSS`, `VmHWM`); elsewhere they +are `null`. + +## 8. Memory bound + +After the growth phase the test records a baseline: RSS right after a reopen. +At each later post-reopen check: + +- **v3:** RSS must stay below `1.25 × baseline + 256 MiB`. The page cache is + bounded and the reference grows only slowly, so the steady state is flat. +- **main:** v2 keeps facts in memory, so RSS rightly grows with history. RSS + per fact version must stay below `1.25 ×` the baseline ratio, plus 256 MiB. + +## 9. Workflow `soak.yml` + +Weekly, alternating like `bench.yml` and `crash-kill.yml`: main on Saturday +01:00 UTC, v3 on Sunday 01:00 UTC, chosen from the cron string; also +`workflow_dispatch` with a `ref` input. The same file is committed to both +branches. + +- `ubuntu-latest`, `timeout-minutes: 360`. +- `MINIGRAF_SOAK_DIR=/mnt/soak`: the runner's spare disk holds main's + multi-GB file. +- `cargo test --profile bench --test soak_test soak -- --ignored --nocapture` + with `MINIGRAF_SOAK_MINUTES=300`. +- Always upload `metrics.jsonl` and the test log as artifacts (90-day + retention), which feed #394. +- On failure, open or comment on a "Weekly soak failure ()" issue with the + seed and the log tail, as `crash-kill.yml` does. + +## 10. Out of scope + +Kills mid-write (#384), fault injection (#390), valid-time windows (#385, +#386), multi-process access, and other OSes (RSS is read from `/proc`). +Publishing the performance envelope is #394; this test only produces the +numbers. diff --git a/tests/soak_test.rs b/tests/soak_test.rs new file mode 100644 index 0000000000..f398a0c6e1 --- /dev/null +++ b/tests/soak_test.rs @@ -0,0 +1,1168 @@ +//! Long-running soak test at scale (#392). +//! +//! Grows a file-backed database to millions of facts in batches of mixed +//! sizes, with multi-valued attributes, retracts, re-asserts and long +//! histories on a few hot entities. It checkpoints on a schedule, reopens at +//! random, and checks samples against a reference through EAVT, AVET, AEVT, +//! `:as-of` and, at the end, a full scan. Metrics go to a JSON-lines file. +//! +//! `soak_short` runs a tiny configuration in the per-PR suite. The full run +//! is ignored; `.github/workflows/soak.yml` runs it weekly: +//! +//! ```text +//! MINIGRAF_SOAK_MINUTES=20 MINIGRAF_SOAK_TARGET_FACTS=10000000 \ +//! cargo test --profile bench --test soak_test soak -- --ignored --nocapture +//! ``` +//! +//! Other variables: `MINIGRAF_SOAK_SEED` (replays the write sequence; the +//! time-based schedule still varies), `MINIGRAF_SOAK_CHECK_MINUTES`, +//! `MINIGRAF_SOAK_CHURN_TPS`, `MINIGRAF_SOAK_METRICS`, `MINIGRAF_SOAK_DIR`. +//! Design: `docs/superpowers/specs/2026-10-10-soak-test-design.md`. +//! +//! This is the v2.x (format v7) copy of the v3 test. It differs in three +//! places: no transaction writes or retracts two values of one entity and +//! attribute (#371, a known v2.x issue fixed in v3.0.0; the v3 workload +//! fails here), so repeated attributes go in follow-up transactions; there is +//! no `verify()`; and the memory bound scales with the fact versions held, +//! since v2.x keeps every fact in memory. The full run also checkpoints and +//! samples `:as-of` less often, because both cost time in the size of the +//! database on v2.x. +#![cfg(not(target_arch = "wasm32"))] + +use minigraf::db::{Minigraf, OpenOptions}; +use minigraf::{QueryResult, Value}; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Instant; +use uuid::Uuid; + +// ── Random numbers ──────────────────────────────────────────────────────────── + +/// SplitMix64 finalizer: a fixed hash, independent of any crate version. +fn mix(mut z: u64) -> u64 { + z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + mix(self.0) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n.max(1) + } + + /// Uniform in `lo..=hi`. + fn range(&mut self, lo: u64, hi: u64) -> u64 { + lo + self.below(hi - lo + 1) + } + + fn unit(&mut self) -> f64 { + (self.next() >> 11) as f64 / (1u64 << 53) as f64 + } + + fn chance(&mut self, p: f64) -> bool { + self.unit() < p + } + + /// Log-uniform in `lo..=hi`: small batches are common, large ones occur. + fn log_uniform(&mut self, lo: u64, hi: u64) -> u64 { + let (l, h) = ((lo as f64).ln(), ((hi + 1) as f64).ln()); + ((l + (h - l) * self.unit()).exp() as u64).clamp(lo, hi) + } +} + +// ── Facts ───────────────────────────────────────────────────────────────────── + +const SHARDS: u64 = 1024; +const HOT: usize = 32; +const HOT_STATES: usize = 8; +const HOT_TAGS: usize = 16; +const HOT_NOTES: usize = 3; +const HOT_SLOTS: usize = HOT_STATES + HOT_TAGS + HOT_NOTES; +/// Re-asserts are chosen once this many ordinary slots are retracted. +const FLIP_CAP: usize = 200_000; + +#[derive(Clone)] +enum Val { + Int(i64), + Kw(String), + Str(String), + /// A ref to ordinary entity `n`. + Ref(u64), +} + +impl Val { + fn edn(&self) -> String { + match self { + Val::Int(n) => n.to_string(), + Val::Kw(k) => k.clone(), + Val::Str(s) => format!("\"{s}\""), + Val::Ref(n) => format!("#uuid \"{}\"", entity_uuid(&format!(":e{n}"))), + } + } + + fn render(&self) -> String { + match self { + Val::Int(n) => format!("i{n}"), + Val::Kw(k) => format!("k{k}"), + Val::Str(s) => render_str(s), + Val::Ref(n) => format!("r:e{n}"), + } + } +} + +fn render_str(s: &str) -> String { + if s.len() <= 24 { + format!("s\"{s}\"") + } else { + format!("s\"{}..{}\"#{}", &s[..12], &s[s.len() - 8..], s.len()) + } +} + +/// The UUID a keyword entity such as `:e5` maps to. +fn entity_uuid(keyword: &str) -> Uuid { + Uuid::new_v5(&Uuid::NAMESPACE_OID, keyword.as_bytes()) +} + +/// Renders a query value; refs and entities appear by name, never as a `Uuid`. +fn render_value(v: &Value, names: &HashMap) -> String { + match v { + Value::Integer(n) => format!("i{n}"), + Value::Keyword(k) => format!("k{k}"), + Value::String(s) => render_str(s), + Value::Ref(u) => match names.get(u) { + Some(name) => format!("r{name}"), + None => "r?".to_string(), + }, + Value::Boolean(b) => format!("b{b}"), + Value::Float(f) => format!("f{f}"), + Value::Null => "null".to_string(), + } +} + +/// The facts ordinary entity `n` is created with, in slot order: a unique +/// name, a score, two values of a sparse shard attribute, 1–4 tags (all in the +/// same transaction, #371), +/// a long string on every 50th entity, a ref on every 10th. +fn content(seed: u64, n: u64) -> Vec<(String, Val)> { + let h = mix(seed ^ n.wrapping_mul(0x2545_F491_4F6C_DD1D)); + let mut facts = vec![ + (":p/name".to_string(), Val::Str(format!("name-{n}"))), + (":p/score".to_string(), Val::Int((h % 1000) as i64)), + (format!(":p/shard{}", n % SHARDS), Val::Int(n as i64)), + (format!(":p/shard{}", n % SHARDS), Val::Int(-(n as i64) - 1)), + ]; + let start = (h >> 10) % 16; + for i in 0..1 + (h >> 20) % 4 { + // 5 is coprime with 16, so the tags are distinct. + facts.push(( + ":p/tag".to_string(), + Val::Kw(format!(":t{}", (start + i * 5) % 16)), + )); + } + if n % 50 == 0 { + facts.push(( + ":p/bio".to_string(), + Val::Str(format!("bio-{n}-{}", "b".repeat(80))), + )); + } + if n % 10 == 0 && n > 0 { + facts.push((":p/ref".to_string(), Val::Ref((h >> 30) % n))); + } + facts +} + +fn hot_fact(slot: usize) -> (&'static str, Val) { + if slot < HOT_STATES { + (":h/state", Val::Kw(format!(":s{slot}"))) + } else if slot < HOT_STATES + HOT_TAGS { + (":h/tag", Val::Kw(format!(":g{}", slot - HOT_STATES))) + } else { + let i = slot - HOT_STATES - HOT_TAGS; + (":h/note", Val::Str(format!("note-{i}-{}", "n".repeat(90)))) + } +} + +// ── Reference ───────────────────────────────────────────────────────────────── + +/// What the database must hold, without storing every fact: ordinary +/// entities follow from `content`, and only retracted slots are kept. +struct Reference { + seed: u64, + /// Ordinary entities `0..next_entity` exist. + next_entity: u64, + /// Facts the ordinary entities were created with. + created: u64, + flipped: Vec<(u64, u8)>, + flipped_at: HashMap<(u64, u8), usize>, + hot_live: [[bool; HOT_SLOTS]; HOT], + /// `tx << 16 | hot << 8 | slot << 1 | asserted`, in commit order. + hot_history: Vec, + /// Transactions committed (`current_tx_count`). + tx: u64, + /// Fact versions written: every assert and every retract. + versions: u64, +} + +impl Reference { + fn new(seed: u64) -> Self { + Reference { + seed, + next_entity: 0, + created: 0, + flipped: Vec::new(), + flipped_at: HashMap::new(), + hot_live: [[false; HOT_SLOTS]; HOT], + hot_history: Vec::new(), + tx: 0, + versions: 0, + } + } + + fn live_ordinary(&self) -> u64 { + self.created - self.flipped.len() as u64 + } + + fn live_hot(&self) -> u64 { + self.hot_live.iter().flatten().filter(|l| **l).count() as u64 + } + + fn is_flipped(&self, n: u64, slot: u8) -> bool { + self.flipped_at.contains_key(&(n, slot)) + } + + fn flip(&mut self, n: u64, slot: u8) { + self.flipped_at.insert((n, slot), self.flipped.len()); + self.flipped.push((n, slot)); + } + + fn unflip(&mut self, n: u64, slot: u8) { + let i = self.flipped_at.remove(&(n, slot)).expect("slot is flipped"); + self.flipped.swap_remove(i); + if let Some(moved) = self.flipped.get(i) { + self.flipped_at.insert(*moved, i); + } + } + + fn record_hot(&mut self, hot: usize, slot: usize, asserted: bool) { + self.hot_live[hot][slot] = asserted; + self.hot_history + .push(self.tx << 16 | (hot as u64) << 8 | (slot as u64) << 1 | asserted as u64); + } + + /// `[attr value]` rows of ordinary entity `n`, sorted. + fn entity_rows(&self, n: u64) -> Vec { + let mut rows: Vec = content(self.seed, n) + .into_iter() + .enumerate() + .filter(|(slot, _)| !self.is_flipped(n, *slot as u8)) + .map(|(_, (a, v))| format!("k{a} {}", v.render())) + .collect(); + rows.sort(); + rows + } + + /// `[attr value]` rows of hot entity `h`, now or as of transaction `t`. + fn hot_rows(&self, h: usize, as_of: Option) -> Vec { + let live = match as_of { + None => self.hot_live[h], + Some(t) => { + let mut live = [false; HOT_SLOTS]; + for r in &self.hot_history { + if r >> 16 > t { + break; + } + if (r >> 8 & 0xFF) as usize == h { + live[(r >> 1 & 0x7F) as usize] = r & 1 == 1; + } + } + live + } + }; + let mut rows: Vec = (0..HOT_SLOTS) + .filter(|s| live[*s]) + .map(|s| { + let (a, v) = hot_fact(s); + format!("k{a} {}", v.render()) + }) + .collect(); + rows.sort(); + rows + } + + /// `[entity value]` rows of attribute `:p/shard{k}`, sorted. + fn shard_rows(&self, k: u64) -> Vec { + let mut rows: Vec = (k..self.next_entity) + .step_by(SHARDS as usize) + .flat_map(|n| { + [(2, n as i64), (3, -(n as i64) - 1)] + .into_iter() + .filter(move |(slot, _)| !self.is_flipped(n, *slot)) + .map(move |(_, v)| format!(":e{n} i{v}")) + }) + .collect(); + rows.sort(); + rows + } +} + +// ── Configuration ───────────────────────────────────────────────────────────── + +struct Config { + seed: u64, + target_facts: u64, + /// Schedules count steps instead of milliseconds (`soak_short`). + step_clock: bool, + /// Long run: wall-clock budget. Short run: churn steps after growth. + budget: u64, + check_every: u64, + reopen_every: (u64, u64), + checkpoint_every: (u64, u64), + churn_tps: Option, + sample_entities: u64, + sample_names: u64, + sample_shards: u64, + sample_as_of: usize, + /// A full scan builds every row in memory, so it runs once when the live + /// count first reaches `full_scan_at`, and at the end only up to + /// `full_scan_max` live facts. + full_scan_at: u64, + full_scan_max: u64, + /// Fail on unbounded memory growth (spec §8). + memory_bound: bool, + metrics: Option, + dir: Option, +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .map(|v| { + v.parse() + .unwrap_or_else(|_| panic!("{name} must be an integer")) + }) + .unwrap_or(default) +} + +impl Config { + fn short(seed: u64) -> Self { + Config { + seed, + target_facts: 20_000, + step_clock: true, + budget: 400, + check_every: 150, + reopen_every: (60, 160), + checkpoint_every: (3, 12), + churn_tps: None, + sample_entities: 20, + sample_names: 3, + sample_shards: 2, + sample_as_of: 2, + full_scan_at: 10_000, + full_scan_max: u64::MAX, + memory_bound: false, + metrics: None, + dir: None, + } + } + + fn from_env() -> Self { + let seed = env_u64( + "MINIGRAF_SOAK_SEED", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos() as u64), + ); + let minute = 60_000; + Config { + seed, + target_facts: env_u64("MINIGRAF_SOAK_TARGET_FACTS", 10_000_000), + step_clock: false, + budget: env_u64("MINIGRAF_SOAK_MINUTES", 20) * minute, + check_every: env_u64("MINIGRAF_SOAK_CHECK_MINUTES", 10) * minute, + reopen_every: (2 * minute, 10 * minute), + // A v2.x checkpoint rewrites the whole file (1.2 s at 2M facts), so + // it runs less often than on v3; still thousands in a 5-hour run. + checkpoint_every: (100, 1_000), + churn_tps: Some(env_u64("MINIGRAF_SOAK_CHURN_TPS", 50) as f64), + sample_entities: 200, + // #518: an AVET lookup scans the whole attribute (~6 s at 10M). + sample_names: 5, + sample_shards: 4, + // `:as-of` scans all history on v2.x (2.2 s at 2M facts). + sample_as_of: 1, + full_scan_at: 2_000_000, + full_scan_max: env_u64("MINIGRAF_SOAK_FULL_SCAN_MAX", 3_000_000), + memory_bound: true, + metrics: Some( + std::env::var_os("MINIGRAF_SOAK_METRICS") + .map_or_else(|| PathBuf::from("target/soak/metrics.jsonl"), PathBuf::from), + ), + dir: std::env::var_os("MINIGRAF_SOAK_DIR").map(PathBuf::from), + } + } +} + +// ── Metrics ─────────────────────────────────────────────────────────────────── + +const PATHS: [&str; 6] = ["eavt", "ea", "avet", "aevt", "as_of", "hot"]; + +#[derive(Default)] +struct Metrics { + /// Query latencies in µs since the last emitted line, per path. + latency: HashMap<&'static str, Vec>, + checkpoint_ms: Vec, + checkpoints: u64, + reopens: u64, + open_ms_file: Option, + open_ms_wal: Option, + /// Peak RSS of the last full-scan child process. + full_scan_rss: Option, + out: Option, +} + +fn percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + sorted[((sorted.len() - 1) as f64 * p).round() as usize] +} + +/// `(VmRSS, VmHWM)` in bytes; `None` where `/proc` is not available. +fn rss() -> Option<(u64, u64)> { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let field = |name: &str| -> Option { + let line = status.lines().find(|l| l.starts_with(name))?; + let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb * 1024) + }; + Some((field("VmRSS:")?, field("VmHWM:")?)) +} + +fn file_bytes(path: &Path) -> u64 { + let mut wal = path.as_os_str().to_os_string(); + wal.push(".wal"); + [path, Path::new(&wal)] + .iter() + .filter_map(|p| std::fs::metadata(p).ok()) + .map(|m| m.len()) + .sum() +} + +// ── Driver ──────────────────────────────────────────────────────────────────── + +#[derive(Clone, Copy, PartialEq)] +enum Phase { + Growth, + Churn, +} + +struct Soak { + cfg: Config, + path: PathBuf, + db: Option, + /// Drives every write, so a seed replays the same write sequence. + rng: Rng, + /// Drives sampling, reads and the schedule. + read_rng: Rng, + rf: Reference, + metrics: Metrics, + phase: Phase, + start: Instant, + steps: u64, + churn_start: Option<(u64, Instant)>, + churn_writes: u64, + since_checkpoint: u64, + checkpoint_due: u64, + next_check: u64, + next_reopen: u64, + /// RSS right after the first reopen-after-checkpoint of the churn phase. + rss_baseline: Option<(u64, u64)>, + full_scanned: bool, +} + +fn open(path: &Path) -> Minigraf { + OpenOptions::new() + .wal_checkpoint_threshold(usize::MAX) + .path(path) + .open() + .unwrap_or_else(|e| panic!("open failed: {}", e.code())) +} + +impl Soak { + fn new(cfg: Config, path: PathBuf) -> Self { + let out = cfg.metrics.as_ref().map(|p| { + if let Some(dir) = p.parent() { + std::fs::create_dir_all(dir).expect("metrics directory"); + } + File::create(p).expect("metrics file") + }); + let mut read_rng = Rng(cfg.seed ^ 0x5EED); + let checkpoint_due = read_rng.range(cfg.checkpoint_every.0, cfg.checkpoint_every.1); + let next_reopen = read_rng.range(cfg.reopen_every.0, cfg.reopen_every.1); + Soak { + db: Some(open(&path)), + path, + rng: Rng(cfg.seed), + read_rng, + rf: Reference::new(cfg.seed), + metrics: Metrics { + out, + ..Metrics::default() + }, + phase: Phase::Growth, + start: Instant::now(), + steps: 0, + churn_start: None, + churn_writes: 0, + since_checkpoint: 0, + checkpoint_due, + next_check: cfg.check_every, + next_reopen, + rss_baseline: None, + full_scanned: false, + cfg, + } + } + + fn db(&self) -> &Minigraf { + self.db.as_ref().expect("database is open") + } + + fn clock(&self) -> u64 { + if self.cfg.step_clock { + self.steps + } else { + self.start.elapsed().as_millis() as u64 + } + } + + fn context(&self) -> String { + format!( + "seed {} step {} tx {} entities {}", + self.cfg.seed, self.steps, self.rf.tx, self.rf.next_entity + ) + } + + fn done(&self) -> bool { + if self.cfg.step_clock { + return self + .churn_start + .is_some_and(|(s, _)| self.steps - s >= self.cfg.budget); + } + if self.clock() < self.cfg.budget { + return false; + } + assert!( + self.phase == Phase::Churn, + "budget ended before {} live facts ({} reached); {}", + self.cfg.target_facts, + self.rf.live_ordinary(), + self.context() + ); + true + } + + fn run(&mut self) { + eprintln!( + "soak: seed {} target {} facts, budget {} {}", + self.cfg.seed, + self.cfg.target_facts, + self.cfg.budget, + if self.cfg.step_clock { + "churn steps" + } else { + "ms" + } + ); + while !self.done() { + self.steps += 1; + if !self.full_scanned && self.rf.live_ordinary() >= self.cfg.full_scan_at { + self.full_scanned = true; + let ms = self.full_scan(); + self.emit(Some(ms)); + } + if self.phase == Phase::Growth && self.rf.live_ordinary() >= self.cfg.target_facts { + self.enter_churn(); + } + if self.write_allowed() { + self.write_step(); + } else { + self.read_step(); + } + let now = self.clock(); + if now >= self.next_reopen { + let after_checkpoint = self.read_rng.chance(0.5); + self.reopen(after_checkpoint); + self.next_reopen = self.clock() + + self + .read_rng + .range(self.cfg.reopen_every.0, self.cfg.reopen_every.1); + } + if now >= self.next_check { + self.check_all(); + self.emit(None); + self.next_check = self.clock() + self.cfg.check_every; + } + } + self.finish(); + } + + fn enter_churn(&mut self) { + eprintln!( + "soak: growth done at {} live facts after {:?}; {}", + self.rf.live_ordinary(), + self.start.elapsed(), + self.context() + ); + self.phase = Phase::Churn; + self.churn_start = Some((self.steps, Instant::now())); + // The memory baseline: a fresh handle on a checkpointed file. + self.reopen(true); + self.check_all(); + self.emit(None); + } + + fn write_allowed(&self) -> bool { + match (self.phase, self.cfg.churn_tps, self.churn_start) { + (Phase::Churn, Some(tps), Some((_, at))) => { + (self.churn_writes as f64) < tps * at.elapsed().as_secs_f64() + } + _ => true, + } + } + + // ── Writes ──────────────────────────────────────────────────────────────── + + fn execute_write(&mut self, cmd: &str, facts: u64) { + if let Err(e) = self.db().execute(cmd) { + panic!("write failed ({}); {}", e.code(), self.context()); + } + self.rf.tx += 1; + self.rf.versions += facts; + if self.phase == Phase::Churn { + self.churn_writes += 1; + } + self.since_checkpoint += 1; + if self.since_checkpoint >= self.checkpoint_due { + self.checkpoint(); + } + } + + fn checkpoint(&mut self) { + let t = Instant::now(); + if let Err(e) = self.db().checkpoint() { + panic!("checkpoint failed ({}); {}", e.code(), self.context()); + } + self.metrics + .checkpoint_ms + .push(t.elapsed().as_millis() as u64); + self.metrics.checkpoints += 1; + self.since_checkpoint = 0; + self.checkpoint_due = self + .read_rng + .range(self.cfg.checkpoint_every.0, self.cfg.checkpoint_every.1); + } + + fn write_step(&mut self) { + let roll = self.rng.below(100); + let (grow, flip) = match self.phase { + Phase::Growth => (70, 85), + Phase::Churn => (10, 60), + }; + if roll < grow || self.rf.next_entity == 0 { + self.grow(); + } else if roll < flip { + self.flip_ordinary(); + } else { + self.churn_hot(); + } + } + + /// New entities. The k-th value of an attribute goes in a k-th + /// transaction, so no transaction writes two values of one entity and + /// attribute (#371). + fn grow(&mut self) { + let want = self.rng.log_uniform(1, 2_000); + let mut rounds: Vec<(String, u64)> = Vec::new(); + let mut facts = 0u64; + let first = self.rf.next_entity; + let mut n = first; + while facts < want || n == first { + let mut seen: HashMap = HashMap::new(); + for (a, v) in content(self.cfg.seed, n) { + let count = seen.entry(a.clone()).or_default(); + let round = *count; + *count += 1; + if rounds.len() <= round { + rounds.push((String::new(), 0)); + } + rounds[round] + .0 + .push_str(&format!("[:e{n} {a} {}]", v.edn())); + rounds[round].1 += 1; + facts += 1; + } + n += 1; + } + for (body, count) in rounds { + self.execute_write(&format!("(transact [{body}])"), count); + } + self.rf.next_entity = n; + self.rf.created += facts; + } + + fn flip_ordinary(&mut self) { + let k = self.rng.log_uniform(1, 50) as usize; + let reassert = !self.rf.flipped.is_empty() + && (self.rf.flipped.len() >= FLIP_CAP || self.rng.chance(0.5)); + let mut chosen: HashSet<(u64, u8)> = HashSet::new(); + if reassert { + for _ in 0..k { + let i = self.rng.below(self.rf.flipped.len() as u64) as usize; + chosen.insert(self.rf.flipped[i]); + } + } else { + for _ in 0..k * 2 { + let n = self.rng.below(self.rf.next_entity); + let slot = self.rng.below(content(self.cfg.seed, n).len() as u64) as u8; + if !self.rf.is_flipped(n, slot) { + chosen.insert((n, slot)); + } + if chosen.len() == k { + break; + } + } + } + if chosen.is_empty() { + return; + } + // Sorted so the command, and so a replayed run, is deterministic. + let mut chosen: Vec<(u64, u8)> = chosen.into_iter().collect(); + chosen.sort_unstable(); + // One value per entity and attribute in a transaction (#371). + let mut seen: HashSet<(u64, String)> = HashSet::new(); + chosen.retain(|(n, slot)| { + seen.insert((*n, content(self.cfg.seed, *n)[*slot as usize].0.clone())) + }); + let mut cmd = String::from(if reassert { + "(transact [" + } else { + "(retract [" + }); + for (n, slot) in &chosen { + let (a, v) = &content(self.cfg.seed, *n)[*slot as usize]; + cmd.push_str(&format!("[:e{n} {a} {}]", v.edn())); + } + cmd.push_str("])"); + self.execute_write(&cmd, chosen.len() as u64); + for (n, slot) in chosen { + if reassert { + self.rf.unflip(n, slot); + } else { + self.rf.flip(n, slot); + } + } + } + + fn churn_hot(&mut self) { + let live = self.rf.live_hot(); + let assert = live == 0 || (live < (HOT * HOT_SLOTS) as u64 && self.rng.chance(0.5)); + let k = self.rng.range(1, 8) as usize; + let mut chosen: Vec<(usize, usize)> = Vec::new(); + for _ in 0..k * 4 { + let h = self.rng.below(HOT as u64) as usize; + let s = self.rng.below(HOT_SLOTS as u64) as usize; + // One value per entity and attribute in a transaction (#371). + if self.rf.hot_live[h][s] != assert + && !chosen + .iter() + .any(|(ch, cs)| *ch == h && hot_fact(*cs).0 == hot_fact(s).0) + { + chosen.push((h, s)); + } + if chosen.len() == k { + break; + } + } + if chosen.is_empty() { + return; + } + let mut cmd = String::from(if assert { "(transact [" } else { "(retract [" }); + for (h, s) in &chosen { + let (a, v) = hot_fact(*s); + cmd.push_str(&format!("[:h{h} {a} {}]", v.edn())); + } + cmd.push_str("])"); + self.execute_write(&cmd, chosen.len() as u64); + for (h, s) in chosen { + self.rf.record_hot(h, s, assert); + } + } + + // ── Reads and checks ────────────────────────────────────────────────────── + + /// Runs `query`, times it under `path`, and renders each row. + fn rows( + &mut self, + path: &'static str, + query: &str, + names: &HashMap, + ) -> Vec { + let t = Instant::now(); + let result = self.db().execute(query); + let us = t.elapsed().as_micros() as u64; + self.metrics.latency.entry(path).or_default().push(us); + match result { + Ok(QueryResult::QueryResults { results, .. }) => { + let mut rows: Vec = results + .iter() + .map(|row| { + row.iter() + .map(|v| render_value(v, names)) + .collect::>() + .join(" ") + }) + .collect(); + rows.sort(); + rows + } + Ok(_) => panic!("not a query result: {query}; {}", self.context()), + Err(e) => panic!("query failed ({}): {query}; {}", e.code(), self.context()), + } + } + + fn expect_rows(&self, what: &str, got: Vec, want: Vec) { + if got != want { + let missing: Vec<&String> = want.iter().filter(|r| !got.contains(r)).take(10).collect(); + let extra: Vec<&String> = got.iter().filter(|r| !want.contains(r)).take(10).collect(); + panic!( + "{what}: {} rows, expected {}; missing {missing:?}; unexpected {extra:?}; {}", + got.len(), + want.len(), + self.context() + ); + } + } + + /// Names for the referents of entity `n`'s ref slot. + fn ref_names(&self, n: u64) -> HashMap { + content(self.cfg.seed, n) + .into_iter() + .filter_map(|(_, v)| match v { + Val::Ref(m) => Some((entity_uuid(&format!(":e{m}")), format!(":e{m}"))), + _ => None, + }) + .collect() + } + + fn check_entity(&mut self, n: u64) { + let names = self.ref_names(n); + let got = self.rows( + "eavt", + &format!("(query [:find ?a ?v :where [:e{n} ?a ?v]])"), + &names, + ); + self.expect_rows(&format!("EAVT :e{n}"), got, self.rf.entity_rows(n)); + // Entity and attribute bound, on the multi-valued tags (#371). + let got = self.rows( + "ea", + &format!("(query [:find ?v :where [:e{n} :p/tag ?v]])"), + &HashMap::new(), + ); + let want: Vec = self + .rf + .entity_rows(n) + .into_iter() + .filter_map(|r| r.strip_prefix("k:p/tag ").map(str::to_string)) + .collect(); + self.expect_rows(&format!("EA :e{n} :p/tag"), got, want); + } + + fn check_name(&mut self, n: u64) { + let names = HashMap::from([(entity_uuid(&format!(":e{n}")), format!(":e{n}"))]); + let got = self.rows( + "avet", + &format!("(query [:find ?e :where [?e :p/name \"name-{n}\"]])"), + &names, + ); + let want = if self.rf.is_flipped(n, 0) { + vec![] + } else { + vec![format!("r:e{n}")] + }; + self.expect_rows(&format!("AVET name-{n}"), got, want); + } + + fn check_shard(&mut self, k: u64) { + let names: HashMap = (k..self.rf.next_entity) + .step_by(SHARDS as usize) + .map(|n| (entity_uuid(&format!(":e{n}")), format!(":e{n}"))) + .collect(); + let got = self.rows( + "aevt", + &format!("(query [:find ?e ?v :where [?e :p/shard{k} ?v]])"), + &names, + ); + // `?e` renders as a ref; the reference writes the bare name. + let got = got + .into_iter() + .map(|r| r.trim_start_matches('r').to_string()) + .collect(); + self.expect_rows(&format!("AEVT :p/shard{k}"), got, self.rf.shard_rows(k)); + } + + fn check_hot(&mut self, h: usize, as_of: Option) { + let (path, clause) = match as_of { + Some(t) => ("as_of", format!(":as-of {t} ")), + None => ("hot", String::new()), + }; + let got = self.rows( + path, + &format!("(query [:find ?a ?v {clause}:where [:h{h} ?a ?v]])"), + &HashMap::new(), + ); + let label = match as_of { + Some(t) => format!("hot :h{h} as of {t}"), + None => format!("hot :h{h}"), + }; + self.expect_rows(&label, got, self.rf.hot_rows(h, as_of)); + } + + /// One checked read between capped churn writes. + fn read_step(&mut self) { + let n = self.read_rng.below(self.rf.next_entity); + if self.read_rng.chance(0.98) { + self.check_entity(n); + } else { + self.check_name(n); + } + } + + fn check_all(&mut self) { + let tx = self.db().current_tx_count(); + assert_eq!(tx, self.rf.tx, "tx counter; {}", self.context()); + if self.rf.next_entity > 0 { + for _ in 0..self.cfg.sample_entities { + let n = self.read_rng.below(self.rf.next_entity); + self.check_entity(n); + } + for _ in 0..self.cfg.sample_names { + let n = self.read_rng.below(self.rf.next_entity); + self.check_name(n); + } + for _ in 0..self.cfg.sample_shards { + let k = self.read_rng.below(SHARDS); + self.check_shard(k); + } + } + for h in 0..HOT { + self.check_hot(h, None); + } + for _ in 0..self.cfg.sample_as_of { + let h = self.read_rng.below(HOT as u64) as usize; + for _ in 0..3 { + let t = self.read_rng.range(1, self.rf.tx.max(1)); + self.check_hot(h, Some(t)); + } + } + } + + fn reopen(&mut self, after_checkpoint: bool) { + if after_checkpoint { + self.checkpoint(); + } + drop(self.db.take()); + let t = Instant::now(); + self.db = Some(open(&self.path)); + let ms = t.elapsed().as_millis() as u64; + self.metrics.reopens += 1; + if after_checkpoint { + self.metrics.open_ms_file = Some(ms); + } else { + self.metrics.open_ms_wal = Some(ms); + } + self.check_all(); + if after_checkpoint && self.phase == Phase::Churn { + self.check_memory(); + } + } + + /// Spec §8: a fresh handle on a checkpointed file stays near the + /// baseline taken when the churn phase began. + fn check_memory(&mut self) { + let Some((now, _)) = rss() else { return }; + let versions = self.rf.versions.max(1); + let Some((base, base_versions)) = self.rss_baseline else { + self.rss_baseline = Some((now, versions)); + eprintln!("soak: memory baseline {} MiB", now >> 20); + return; + }; + // v2.x keeps every fact version in memory, so the bound grows with them. + let scaled = (base as u128 * versions as u128 / base_versions as u128) as u64; + let limit = scaled + scaled / 4 + (256 << 20); + if self.cfg.memory_bound && now > limit { + panic!( + "RSS {} MiB above the bound {} MiB (baseline {} MiB at {} versions); {}", + now >> 20, + limit >> 20, + base >> 20, + base_versions, + self.context() + ); + } + } + + fn emit(&mut self, full_scan_ms: Option) { + let mut query_us = serde_json::Map::new(); + for path in PATHS { + let mut v = self.metrics.latency.remove(path).unwrap_or_default(); + v.sort_unstable(); + query_us.insert( + path.to_string(), + serde_json::json!({"n": v.len(), "p50": percentile(&v, 0.5), "p99": percentile(&v, 0.99)}), + ); + } + let mut ck = std::mem::take(&mut self.metrics.checkpoint_ms); + ck.sort_unstable(); + let (rss_now, rss_peak) = rss().map_or((None, None), |(a, b)| (Some(a), Some(b))); + let line = serde_json::json!({ + "t_s": self.start.elapsed().as_secs(), + "phase": if self.phase == Phase::Growth { "growth" } else { "churn" }, + "tx": self.rf.tx, + "live_facts": self.rf.live_ordinary() + self.rf.live_hot(), + "versions": self.rf.versions, + "file_bytes": file_bytes(&self.path), + "open_ms_file": self.metrics.open_ms_file, + "open_ms_wal": self.metrics.open_ms_wal, + "reopens": self.metrics.reopens, + "checkpoints": self.metrics.checkpoints, + "checkpoint_ms_p50": percentile(&ck, 0.5), + "checkpoint_ms_max": ck.last().copied().unwrap_or(0), + "rss_bytes": rss_now, + "peak_rss_bytes": rss_peak, + "query_us": query_us, + "full_scan_ms": full_scan_ms, + "full_scan_peak_rss_bytes": self.metrics.full_scan_rss, + }); + eprintln!("soak: {line}"); + if let Some(out) = self.metrics.out.as_mut() { + writeln!(out, "{line}").expect("write metrics"); + } + } + + /// Checks the row count of `[?e ?a ?v]` in a child process, which + /// prints the count, the time and its own peak RSS. A query builds its + /// whole answer when it opens (#432), about 1 KB per row here, and the + /// allocator keeps that memory after it is freed: run in this process, it + /// would hide a leak of that size from the memory bound (spec §8). + fn full_scan(&mut self) -> u64 { + self.checkpoint(); + drop(self.db.take()); + let out = Command::new(std::env::current_exe().expect("test binary path")) + .args(["soak_full_scan_child", "--exact", "--nocapture"]) + .env("MINIGRAF_SOAK_SCAN_DB", &self.path) + .output() + .expect("spawn full-scan child"); + let stdout = String::from_utf8_lossy(&out.stdout); + let Some(line) = stdout.lines().find_map(|l| l.strip_prefix("soak-scan: ")) else { + let stderr = String::from_utf8_lossy(&out.stderr); + let tail: Vec<&str> = stderr.lines().rev().take(20).collect(); + panic!("full-scan child failed: {tail:?}; {}", self.context()); + }; + let field = |name: &str| -> Option { + let mut words = line.split_whitespace(); + words.find(|w| *w == name)?; + words.next()?.parse().ok() + }; + let rows = field("rows").expect("child row count"); + let ms = field("ms").expect("child time"); + self.metrics.full_scan_rss = field("peak"); + self.db = Some(open(&self.path)); + assert_eq!( + rows, + self.rf.live_ordinary() + self.rf.live_hot(), + "full scan row count; {}", + self.context() + ); + ms + } + + fn finish(&mut self) { + self.reopen(true); + let full_scan_ms = if self.rf.live_ordinary() <= self.cfg.full_scan_max { + Some(self.full_scan()) + } else { + eprintln!( + "soak: final full scan skipped above {} live facts", + self.cfg.full_scan_max + ); + None + }; + self.emit(full_scan_ms); + eprintln!( + "soak: done: {} live facts, {} versions, {} tx, {} checkpoints, {} reopens in {:?}", + self.rf.live_ordinary() + self.rf.live_hot(), + self.rf.versions, + self.rf.tx, + self.metrics.checkpoints, + self.metrics.reopens, + self.start.elapsed() + ); + } +} + +fn run(cfg: Config) { + let tmp = match &cfg.dir { + Some(dir) => { + std::fs::create_dir_all(dir).expect("soak directory"); + tempfile::tempdir_in(dir).expect("temp dir") + } + None => tempfile::tempdir().expect("temp dir"), + }; + let path = tmp.path().join("soak.graph"); + Soak::new(cfg, path).run(); +} + +/// The driver at a size a debug build runs in seconds. +#[test] +fn soak_short() { + run(Config::short(392)); +} + +/// The full soak: see the module docs for its variables. +#[test] +#[ignore] +fn soak() { + run(Config::from_env()); +} + +/// Child half of `Soak::full_scan`. A no-op when `MINIGRAF_SOAK_SCAN_DB` is +/// absent, as in `crash_kill_child_entrypoint`. +#[test] +fn soak_full_scan_child() { + let Some(path) = std::env::var_os("MINIGRAF_SOAK_SCAN_DB") else { + return; + }; + let db = open(Path::new(&path)); + let t = Instant::now(); + let rows = match db.execute("(query [:find ?e ?a ?v :where [?e ?a ?v]])") { + Ok(QueryResult::QueryResults { results, .. }) => results.len(), + Ok(_) => panic!("full scan: not a query result"), + Err(e) => panic!("full scan failed ({})", e.code()), + }; + let ms = t.elapsed().as_millis(); + let peak = rss().map_or(0, |(_, peak)| peak); + println!("soak-scan: rows {rows} ms {ms} peak {peak}"); +}