diff --git a/.github/workflows/envelope.yml b/.github/workflows/envelope.yml new file mode 100644 index 0000000000..f147f9aa12 --- /dev/null +++ b/.github/workflows/envelope.yml @@ -0,0 +1,65 @@ +name: Performance envelope + +# #394: builds 100K and 1M fact fixtures and measures open and checkpoint time, +# RSS, and p50/p99 latency per query shape (examples/perf_envelope.rs). The +# published envelope in docs/BENCHMARKS.md also has 10M facts, measured +# locally. Runs on demand; on pull requests that touch the harness it runs a +# 10K smoke pass so the harness keeps working. + +on: + workflow_dispatch: + inputs: + ref: + description: Branch to measure + required: true + default: v3 + scales: + description: Comma-separated fact counts + required: true + default: "100k,1m" + pull_request: + paths: + - examples/perf_envelope.rs + - .github/workflows/envelope.yml + +permissions: + contents: read + +jobs: + envelope: + name: Envelope (${{ github.event_name == 'pull_request' && '10k smoke' || inputs.scales }}) + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || '' }} + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + + - name: Build + run: cargo build --profile envelope --example perf_envelope + + - name: Run + shell: bash + env: + SCALES: ${{ github.event_name == 'pull_request' && '10k' || inputs.scales }} + BUDGET: ${{ github.event_name == 'pull_request' && '1' || '15' }} + run: | + set -o pipefail + ./target/envelope/examples/perf_envelope --scales "$SCALES" \ + --budget-secs "$BUDGET" --json envelope.json | tee envelope.md + cat envelope.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload results + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: envelope-${{ github.run_id }} + path: | + envelope.json + envelope.md + retention-days: 90 + if-no-files-found: warn diff --git a/CHANGELOG.md b/CHANGELOG.md index cf3a5282f9..bda52a1493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Two fuzz targets check results, not just crashes (#387).** `ops_sequence` decodes the input into up to 32 storage operations (transact, retract, a write transaction that commits or rolls back, checkpoint, reopen, and a crash that opens a copy of the file and WAL) and, after each one, compares the full rows now and at an input-chosen `:as-of`, plus one input-chosen query shape, with the reference model of the model-based test. It uses that test's driver and checks, now in `tests/reference_model`, so the proptest test and the fuzzer share one oracle. `corrupt_file` damages a file written over three checkpoints with up to 16 byte flips and an optional truncation, without fixing any checksum, and requires every query to fail or to return exactly the answers of the last checkpoint or the one before it, all from the same one. Its first run found that a file cut to under one page opens as a new, empty database (#506); the target skips those lengths until that is fixed. - **Weekly soak test at scale (#392).** `tests/soak_test.rs` grows a file-backed database to 10M live facts in batches of 1–2,000 facts (multi-valued tags and shard values in one transaction, strings over 64 bytes, refs), then churns it for the rest of a 5-hour budget at 50 transactions a second: retracts and re-asserts of random facts, and long retract/assert histories on 32 hot entities. It checkpoints every 20–200 transactions and reopens every 2–10 minutes, half from a checkpointed file and half replaying the WAL. Every 10 minutes, after every reopen, and between churn writes it checks samples against a reference that derives each entity's facts from its number and keeps only what was retracted: EAVT per entity, entity and attribute on the tags, AVET by unique name, AEVT over a sparse attribute, `:as-of` on hot entities, and the transaction counter; a full scan runs in a child process at 2M facts (so its memory does not hide a leak), and `verify()` at the end. It fails if RSS after a reopen grows past 1.25× the baseline plus 256 MiB. Metrics (file size, open and checkpoint time, RSS and peak RSS, p50/p99 per query path) go to a JSON-lines file that `.github/workflows/soak.yml` uploads weekly, alternating `main` and `v3`. `soak_short` runs the same driver at 20K facts in the per-PR suite. Found #518: attribute-and-value patterns scan the whole attribute. - **Coverage gates that gate (#396).** The per-module gate in `coverage-gates.yml` had never checked anything: the instrumented test run failed (the lock tests could not find `pid_ns_helper`), the report it grepped was empty, and every module was skipped with a warning, so the job passed. It also read `cargo llvm-cov --text`, which has no per-file percentages, and named `btree_v6`, which v3 no longer has. The job now runs the whole suite with branch coverage on a pinned nightly, fails if any test fails, and checks each file's line and branch percentage against `coverage-thresholds.toml` with `scripts/coverage_gates.py`; a file missing from the report, or without branch data, fails. The thresholds file covers the WAL, `db.rs`, the fact log and log writer, every storage file, and the evaluator, executor and stratification, with floors at the measured values less one point. It also lists the paths left uncovered on purpose (invariant checks, browser-only and Windows-only paths). New tests cover the WAL's fact-size and fact-count limits (the old fact-size test changed bytes inside the fact and never reached the limit), a WAL opened in a missing directory, a failed cut of a torn WAL tail, short v7 pages and slots past the count, backup meta pages of another generation or length, a torn first meta with data in page 1, `save` with nothing pending, `into_backend` with pending facts, `rebuild_indexes` with nothing checkpointed, unsorted or repeated B+tree input, a stored key inserted with another value, a tree pointer to a page of another type, the read-only `MutexStorageBackend`, and two threads missing the same page in the page cache (before, only a lucky race reached that path). Branch coverage of `wal.rs` went from 83% to 95%, `packed_pages.rs` from 86% to 100%, `persistent_facts.rs` from 82% to 88%, and `btree.rs` from 85% to 85.5%. +- **Performance envelope at 100K–10M facts (#394).** `examples/perf_envelope.rs` builds and caches fixtures of 100K, 1M and 10M facts (six attributes per entity, a ref tree, two valid-time windows, an entity with 1,000 versions). It reports open and checkpoint times, RSS, and p50/p99 latency for twelve query shapes, each shape in its own process. It builds with a new `envelope` Cargo profile (`release` at opt-level 3, as an application compiles the library; `release` itself is opt-level "z"). `docs/BENCHMARKS.md` and the README publish the results: point lookups and single-entity time travel stay flat (12–14 µs at 10M facts). Scans, aggregations and joins grow linearly with the attributes they touch. Rule queries are supported to 100K facts only. A Python FFI round trip (`bench/envelope.py` in minigraf-python) adds about 70 µs per query. `.github/workflows/envelope.yml` runs 100K and 1M on demand, and a 10K smoke pass on PRs that change the harness. The nightly Criterion `query/*` groups gain 100K and 1M scales. Profiling at 1M facts found #528, #529, #530, #531 and #532. ### Known issues diff --git a/CLAUDE.md b/CLAUDE.md index 3aa954a4c5..f4eaf586d4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,6 +88,9 @@ cargo test --test concurrency_test cargo run --example embedded cargo run --example file_storage +# Performance envelope (#394): latency/RSS at 100K–10M facts; opt-level 3 profile +cargo run --profile envelope --example perf_envelope -- --scales 100k,1m + # Run demo scripts cargo run < demos/demo_commands.txt cargo run < demos/demo_recursive.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fb383d262..7cef6bf95b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,6 +131,7 @@ Releases are managed by the project maintainer. The process is documented in iss 5. Tag pushed — CI publishes to crates.io, PyPI, npm, and Maven Central automatically 6. Tier 1 bindings (Python) released alongside the core release; Tier 2 bindings follow on a best-effort schedule (see [support tiers](PHILOSOPHY.md#support-tiers)) 7. Known-issues issue updated, and `data-integrity`/`corruption`/`durability` issues fixed by this release closed with a link to it (see above) +8. For a minor or major release: the performance envelope re-run and the snapshot in [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) and the README refreshed (`cargo run --profile envelope --example perf_envelope`; see [Reproducing](docs/BENCHMARKS.md#reproducing)) ## Code of Conduct diff --git a/Cargo.toml b/Cargo.toml index 6182085c30..3449c5f9f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,6 +100,13 @@ opt-level = "z" # uniffi-bindgen reads UNIFFI_META symbols from .symtab (not .dynsym). +# examples/perf_envelope.rs (#394): opt-level 3, as an application's release +# build compiles the library; `release` is tuned for the REPL binary's size. +[profile.envelope] +inherits = "release" +opt-level = 3 +strip = "none" + [profile.bench] strip = "none" debug = true diff --git a/README.md b/README.md index fa590cc8ac..d59e0e58da 100644 --- a/README.md +++ b/README.md @@ -247,12 +247,20 @@ See [ROADMAP.md](ROADMAP.md) for planned work and current direction. ## Performance -See [BENCHMARKS.md](docs/BENCHMARKS.md) for the current reproducible local snapshot and benchmark commands. -| Point query at 1M facts | 4.3–4.5 s (selective B+tree lookup for bounded patterns; O(N) for full-attribute scans) | -| Open time at 1M facts | 1.31 s (2.4× faster than v5 — indexes no longer loaded into RAM) | -| Peak heap at 1M facts | 1.05 GB (~21% less than v5 — indexes paged in on demand) | +Performance envelope on the `v3` branch (format v8), measured up to 10M facts on a laptop (Intel i7-1065G7, SSD, page cache warm; see [BENCHMARKS.md](docs/BENCHMARKS.md#performance-envelope-v300-394) for every shape, RSS and the method): -File-backed databases enforce a maximum fact size of **4 080 serialised bytes** per fact. In-memory databases have no limit. +| Workload | 100K facts | 1M facts | 10M facts | +|---|---:|---:|---:| +| Point lookup (bound entity + attribute, incl. `:as-of` / `:valid-at`), p50 / p99 | 13 µs / 16 µs | 12 µs / 48 µs | 12 µs / 46 µs | +| Entity with 1,000 versions, p50 / p99 | 0.26 / 0.30 ms | 0.26 / 0.29 ms | 0.27 / 0.30 ms | +| Open a file (RSS after open: 3 MiB) | 0.12 ms | 0.14 ms | 0.12 ms | +| Checkpoint after 1 new fact, p50 / p99 | 3.1 / 5.1 ms | 2.9 / 4.3 ms | 3.8 / 21 ms | +| Attribute scan (N/6 rows), p50 | 43 ms | 0.46 s | 4.8 s | +| Selective 2-clause join (1 row), p50 | 54 ms | 0.61 s | 6.5 s | + +Point lookups and time travel on one entity stay flat at every tested size. Scans, aggregations and joins grow linearly with the facts of the attributes they touch, because a query materializes them before joining (#473). They are interactive up to about 100K facts. Not supported yet: rule queries on databases with 1M or more facts (#528), and graphs over 10M facts (not tested). + +File-backed databases limit string values to 4,068 bytes (`WAL-003`). In-memory databases have no limit. **Durability tuning:** every write is `fsync`'d immediately by default (`SyncMode::Full`). Bulk loaders/migrations that can safely re-run from a checkpoint watermark on failure can trade that for throughput with `OpenOptions::new().synchronous(SyncMode::Normal)` — `checkpoint()` still fsyncs unconditionally in both modes. See the [Performance Tuning](https://project-minigraf.github.io/minigraf-docs/latest/performance-tuning/#configuration-knobs) for the full tradeoff and the write-batching pattern that pairs with it. diff --git a/ROADMAP.md b/ROADMAP.md index 5741d6d49d..294131ba69 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -94,7 +94,7 @@ Scope — release-gate testing and documentation: - Persistent fuzz corpus and an operation-sequence target (#387; done on `v3`: corpus caching, weekly `cmin`, the v8 page targets, and the `ops_sequence` and `corrupt_file` targets) - Fault injection across save/WAL/recovery (#390) - Long-running soak test at scale (#392; done on `v3`: `tests/soak_test.rs`, 10M facts and a 5-hour churn weekly, alternating `main` and `v3`) -- Benchmarks at 100K–10M facts and a published performance envelope (#394) +- Benchmarks at 100K–10M facts and a published performance envelope (#394; on `v3`: `examples/perf_envelope.rs`, results in `docs/BENCHMARKS.md`; follow-ups #528–#532) - Higher storage/WAL coverage gates and branch-coverage gates (#396; on `v3`: line and branch floors per file in `coverage-thresholds.toml`, checked on every PR) - Durability, recovery and operations guide (#398) diff --git a/benches/minigraf_bench.rs b/benches/minigraf_bench.rs index 4220bfb774..0e0436bc9b 100644 --- a/benches/minigraf_bench.rs +++ b/benches/minigraf_bench.rs @@ -175,7 +175,13 @@ fn bench_insert_file(c: &mut Criterion) { // ── Task 5: query/ ──────────────────────────────────────────────────────────── fn bench_query(c: &mut Criterion) { - const SCALES: &[(&str, usize)] = &[("1k", 1_000), ("10k", 10_000)]; + // 100k and 1m track the performance envelope (#394) in the nightly run. + const SCALES: &[(&str, usize)] = &[ + ("1k", 1_000), + ("10k", 10_000), + ("100k", 100_000), + ("1m", 1_000_000), + ]; // point_entity: EAVT range scan on a known entity { diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index ca094c867a..5bdeeeb8bc 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -2,6 +2,105 @@ This document records reproducible local benchmark snapshots. Completed release-performance history belongs in the [CHANGELOG](../CHANGELOG.md); continuous CI history is available on [Bencher](https://bencher.dev/perf/minigraf/plots). +## Performance Envelope (v3.0.0, #394) + +**Version**: `v3` branch (format v8), unreleased · **Date**: 2026-10-11 · **Command**: `cargo run --profile envelope --example perf_envelope -- --scales 100k,1m,10m` · Intel Core i7-1065G7 (4 cores / 8 threads, 16 GB RAM), Rust 1.94.0, Linux 6.12.108, on AC power and otherwise idle; the `.graph` file on an SSD with the OS page cache warm. + +This section lists the graph sizes and query shapes that are tested, with their p50 and p99 latency, and the ones that are not supported yet. Re-run it for each minor release (CONTRIBUTING.md). + +### What is supported + +| Workload | Tested up to | p50 / p99 at 10M facts | Notes | +|---|---:|---:|---| +| Point lookup: bound entity and attribute, including `:as-of` and `:valid-at` | 10M facts | 12–14 µs / 20–48 µs | Flat from 100K to 10M | +| Entity with a long history (1,000 versions), current and `:as-of` | 10M facts | 0.26 ms / 0.30 ms | Grows with that entity's history, not the graph | +| Open a checkpointed file | 10M facts | 0.12 ms, 3 MiB RSS | Indexes are paged in on demand | +| Checkpoint after 1 new fact | 10M facts | 3.8 ms / 21 ms | Copy-on-write: cost follows the change | +| Attribute scan, attribute + value, grouped aggregation, joins | 1M facts | see below | Linear in the facts of the attributes they touch: about 0.1–0.5 s at 1M facts, seconds at 10M | +| Rule queries (recursion) | 100K facts | 2.7 s at 100K | Read the whole database (#529) | + +**Not supported yet:** + +- Rule queries on a database with 1,000,000 or more live facts fail with `INT-020`, whatever the size of the answer (#528). With the limit raised (`:max-results 3000000`), `recursive_ancestors` returns its 6 rows at 1M facts in 34 s. +- Graphs over 10M facts are not tested. +- Interactive latency for scans and joins at 10M facts. They take 3–21 s and materialize every fact of the attributes they touch, 0.7–3.6 GiB of RSS. + +### Storage + +| Metric | 100K | 1M | 10M | +|---|---:|---:|---:| +| File size | 10 MiB | 120 MiB | 1,202 MiB | +| Build time (200-entity transactions, checkpoint every 200K facts) | 3.2 s | 12.7 s | 195 s | +| Build peak RSS | 185 MiB | 403 MiB | 421 MiB | +| Open | 120 µs | 136 µs | 124 µs | +| RSS after open | 3 MiB | 3 MiB | 3 MiB | +| Checkpoint, 1 new fact (p50 / p99) | 3.1 ms / 5.1 ms | 2.9 ms / 4.3 ms | 3.8 ms / 20.5 ms | +| Checkpoint, 10K new facts | 41 ms | 83 ms | 722 ms | +| Peak RSS (open + 21 checkpoints) | 20 MiB | 21 MiB | 21 MiB | + +### Query latency (p50 / p99) + +| Shape | Query | 100K | 1M | 10M | +|---|---|---:|---:|---:| +| `point_entity` | `[:p{m} :age ?a]` | 13 µs / 16 µs | 12 µs / 48 µs | 12 µs / 46 µs | +| `as_of` | `:as-of {tx}`, `[:p{m/4} :age ?a]` | 14 µs / 36 µs | 14 µs / 20 µs | 14 µs / 30 µs | +| `valid_at` | `:valid-at "2021-06-01…"`, `[:p{m} :status ?s]` | 15 µs / 49 µs | 14 µs / 20 µs | 14 µs / 23 µs | +| `hot_current` | `[:hot :counter ?v]`, 1,000 versions | 263 µs / 301 µs | 256 µs / 293 µs | 268 µs / 300 µs | +| `hot_as_of` | same, `:as-of` halfway | 257 µs / 284 µs | 262 µs / 293 µs | 264 µs / 304 µs | +| `attribute_value` | `[?e :name "p{m}"]` (1 row) | 24.5 ms / 25.6 ms | 288 ms / 297 ms | 3.16 s / 3.16 s | +| `attribute_scan` | `[?e :age ?a]` (N/6 rows) | 43.4 ms / 49.4 ms | 462 ms / 484 ms | 4.83 s / 5.18 s | +| `count_grouped` | `[:find ?g (count ?e) :where [?e :group ?g]]` | 46.3 ms / 48.6 ms | 521 ms / 524 ms | 5.66 s / 6.20 s | +| `join_2` | `[:p{m} :follows ?f] [?f :name ?n]` (1 row) | 54.0 ms / 63.0 ms | 606 ms / 610 ms | 6.50 s / 7.02 s | +| `join_3_selective` | `[:p{m} :manager ?b] [?s :manager ?b] [?s :name ?n]` (10 rows) | 117 ms / 125 ms | 1.24 s / 1.28 s | 13.5 s / 13.8 s | +| `join_3_scan` | `[?e :follows ?f] [?f :group 7] [?f :name ?n]` | 150 ms / 155 ms | 1.92 s / 1.96 s | 20.9 s / 21.2 s | +| `recursive_ancestors` | `(boss :p{leaf} ?b)` over `:manager` (5–7 rows) | 2.71 s / 2.72 s | fails: `INT-020` | > 60 s (abandoned) | + +Below 100 runs, p99 is the slowest run. Shapes slower than about 75 ms ran fewer than 200 times; the full JSON report has run counts. + +Peak RSS per shape, each in its own process (baseline 4 MiB): + +| Shape | 100K | 1M | 10M | +|---|---:|---:|---:| +| point lookups, `:as-of`, `:valid-at`, hot entity | 4 MiB | 4 MiB | 4 MiB | +| `attribute_value` | 22 MiB | 137 MiB | 729 MiB | +| `attribute_scan` | 22 MiB | 169 MiB | 1,665 MiB | +| `count_grouped` | 24 MiB | 198 MiB | 1,783 MiB | +| `join_2` | 33 MiB | 307 MiB | 2,342 MiB | +| `join_3_selective` | 45 MiB | 376 MiB | 2,856 MiB | +| `join_3_scan` | 59 MiB | 456 MiB | 3,610 MiB | +| `recursive_ancestors` | 375 MiB | 2,479 MiB | 4,832 MiB (at abandonment) | + +### Fixture + +`N` facts over `N/6` entities `:p{i}`, six facts each: `:name "p{i}"`, `:age i%90`, `:group i%100`, `:follows :p{(7919·i+1) mod E}`, `:manager :p{i/10}` (a tree with fan-out 10), and `:status` in two valid-time windows (`"a"` for 2020–2023, `"b"` from 2023). Plus `:hot :counter`, retracted and reasserted 1,000 times. `m = E/2`. Every query drains its cursor with `next_batch(1000)`. Shapes run in their own process on the fixture opened read-only. The storage rows use a copy. + +The harness builds with `--profile envelope`: `release` at opt-level 3, which is how an application's release build compiles the library. `release` and `bench` (which inherits it) are opt-level "z" for the REPL binary's size, and run these queries about 1.5× slower (`attribute_scan` at 1M facts: 703 ms against 462 ms). The Criterion numbers elsewhere in this file are opt-level "z". + +### Where the time goes (1M facts) + +`perf` with frame pointers on `attribute_scan` and `join_3_selective`: + +- **Up-front materialization (50–60%).** Before matching, the executor turns every live fact of each pattern's attribute into a `Fact` (`selective_fact_fetch` → `live_scan`). A pattern without a bound entity is read in full, so a selective join costs as much as scanning its widest attribute. The streaming engine (#473: `Scan` and `IndexNestedLoopJoin` operators) replaces this. +- **DICT lookup per row (26–35%).** `uuid_of` does a B+tree `get` for every row's entity id (#530). +- **Bindings as `HashMap` (about 25%).** SipHash insert, lookup and drop per row; #473's columnar batches replace them. +- **`net_asserted_facts` over committed facts that `live_facts` already resolved (about 10%)** (#532). +- **Keyword constants hashed to a UUID per fact (about 8% of the join)** (#531). +- `attribute_value` reads the whole attribute instead of seeking AVET (#518). + +### Python FFI round trip + +**Command**: `python bench/envelope.py --report envelope.json --scale ` in [minigraf-python](https://github.com/project-minigraf/minigraf-python) (`v3`), on the same fixtures and report. The binding is built with `maturin develop --release` against this commit. It runs the same queries through `db.query(...)`, drains them with `next_batch(1000)`, and decodes each JSON batch with `json.loads`. Python 3.14.7. + +| Shape | Rust p50 (100K / 1M / 10M) | Python p50 (100K / 1M / 10M) | Python peak RSS at 10M | +|---|---:|---:|---:| +| `point_entity` | 13 µs / 12 µs / 12 µs | 77 µs / 84 µs / 88 µs | 21 MiB | +| `hot_current` | 263 µs / 256 µs / 268 µs | 330 µs / 326 µs / 328 µs | 21 MiB | +| `attribute_scan` | 43.4 ms / 462 ms / 4.83 s | 49.4 ms / 537 ms / 5.73 s | 1,601 MiB | +| `count_grouped` | 46.3 ms / 521 ms / 5.66 s | 46.0 ms / 506 ms / 5.45 s | 1,736 MiB | +| `join_3_scan` | 150 ms / 1.92 s / 20.9 s | 150 ms / 1.92 s / 21.3 s | 3,372 MiB | + +The binding adds about 70 µs per query, plus about 0.5 µs per returned row for the JSON batches (0.9 s for 1.67M rows). The interpreter adds about 16 MiB of RSS. Peak RSS is otherwise the cursor's: the answer is computed when the cursor opens. #473 re-measures this table as an acceptance test for the streaming engine. + ## Current Local Snapshot **Version**: v2.0.0 @@ -92,6 +191,10 @@ For changes that may affect performance, record the exact command, version, tool ## Reproducing ```bash +# The performance envelope (#394): builds and caches fixtures in target/perf-envelope +# (10M facts: about 1.2 GiB on disk, 3 minutes to build, up to 5 GiB RSS while measuring). +cargo run --profile envelope --example perf_envelope -- --scales 100k,1m,10m --json envelope.json + # Run the current documented query snapshot. cargo bench -- 'query/(point_entity|point_attribute|join_3pattern)' @@ -109,4 +212,4 @@ Criterion writes HTML reports to `target/criterion/`. A benchmark result should - Results are measurements, not service-level guarantees. - The full suite intentionally includes expensive large-fixture and quadratic workloads; run focused groups during development and the complete suite for a release-level performance review. -- Heap and RSS measurements require a separate profiler run and are not included in this snapshot. +- The Criterion snapshots do not include heap or RSS; the performance envelope reports RSS from `/proc` (Linux only). The weekly soak test (#392) also records RSS, file size, and open and checkpoint time at 10M facts. diff --git a/docs/superpowers/specs/2026-10-11-perf-envelope-design.md b/docs/superpowers/specs/2026-10-11-perf-envelope-design.md new file mode 100644 index 0000000000..7ee013ce40 --- /dev/null +++ b/docs/superpowers/specs/2026-10-11-perf-envelope-design.md @@ -0,0 +1,86 @@ +# Performance Envelope (#394) — Design + +**Date**: 2026-10-11 · **Issue**: #394 · **Branch**: `v3` (format v8) + +## Goal + +Publish the graph sizes and query shapes Minigraf supports and tests, with p50/p99 +latency, open time, checkpoint time and RSS at 100K, 1M and 10M facts. State the +sizes that are not supported yet. Find the top costs of the attribute scan and the +3-clause join at 1M facts and file follow-up issues. + +## Decisions + +1. **Envelope binary plus Criterion.** `examples/perf_envelope.rs` builds and caches + the fixtures, times every query shape, reads RSS from `/proc/self/status`, and prints + a markdown table and JSON. Criterion reports a mean with a confidence interval but no + p99 and no RSS, and 10M facts would not fit the nightly time budget. The nightly + `query/*` Criterion groups gain 100K and 1M scales for regression tracking. No new + dependencies (`serde_json` and `tempfile` are already dev-dependencies). +2. **Where it runs.** 10M numbers come from local runs. `.github/workflows/envelope.yml` + runs 100K and 1M on `workflow_dispatch` and uploads the table; on PRs that touch the + harness it runs a 10K smoke pass. Nothing is scheduled. +3. **Python FFI round trip** (carried over from #462): `bench/envelope.py` in + `minigraf-python` (`v3`), run against the fixture file the core harness builds, + drains `db.query(...)` with `next_batch(1000)` and reports wall time and RSS. The + numbers go into `docs/BENCHMARKS.md` here. Separate PR to the binding repo. #473 + re-measures it as an acceptance test for the streaming engine. + +## Fixture + +`N` facts over `E = N / 6` entities `:p{i}`, six facts each, built in transactions of +200 entities with an explicit checkpoint every 200K facts so RSS stays bounded: + +| Attribute | Value | Purpose | +|---|---|---| +| `:name` | `"p{i}"` | string; attribute-value lookup | +| `:age` | `i % 90` | integer; point entity, attribute scan | +| `:group` | `i % 100` | integer; grouped aggregation | +| `:follows` | `:p{(i·7919 + 1) mod E}` | ref; 2-clause join | +| `:manager` | `:p{i / 10}` | ref tree (fan-out 10); 3-clause join, recursion | +| `:status` | `"a"` valid 2020–2023, `"b"` valid from 2023 | two valid-time windows | + +Plus one hot entity `:hot` whose `:counter` is retracted and reasserted 1,000 times +(#323, #379), so its history is the same at every scale. + +The fixture is cached at `target/perf-envelope/fixture-v{version}-{scale}.graph`, with +its build metadata (entity count, the `:as-of` counters, build time and peak RSS) in a +`.json` next to it. Bump the version when the fixture changes. + +## Measurements + +The harness builds with `--profile envelope`: `release` at opt-level 3, which is how +an application's release build compiles the library. `release` (and `bench`, which +inherits it) is opt-level "z" for the REPL binary's size and runs these queries about +1.5× slower. + +The build, the storage measurements and every query shape run in their own child +process, so each peak RSS (VmHWM) is that step's alone. Opening costs about 0.1 ms, +so a process per shape is cheap. + +- **Open**: `Minigraf::open` on a copy of the fixture (OS page cache warm; a cold open + needs root to drop caches), RSS after open. +- **Checkpoint**: after 1 new fact (20 runs, p50/p99) and after 10K new facts, on the + same copy. +- **Queries**: the fixture opened read-only. The first run checks the row count, and is + abandoned after `--cap-secs` (default 60): the shape is reported as capped. A query + error (such as INT-020) is reported as the shape's result. Then two more warm-up runs + if a run takes under a second, then runs until 200 or a 15 s budget, at least 5. + Every run drains the cursor with `next_batch(1000)`. Reported: p50, p99 (nearest + rank; the maximum below 100 runs), max, run count, rows, peak RSS. + +Query shapes: point entity; attribute + value; attribute scan; selective 2- and +3-clause joins; whole-graph 3-clause join; recursive rule from a leaf (demand-driven); +`:as-of`; `:valid-at`; grouped count; hot entity current value and `:as-of`. + +## Envelope + +`docs/BENCHMARKS.md` gets the full table per scale; the README gets a short summary: +which shapes are supported at which sizes, p50/p99, and that sizes over 10M facts are +not tested and not supported yet. The release checklist (CONTRIBUTING.md) re-runs the +envelope for each minor release. + +## Profiling + +`perf record` on the attribute scan and the 3-clause join at 1M facts; one follow-up +issue per top cost, linked from #394. diff --git a/examples/perf_envelope.rs b/examples/perf_envelope.rs new file mode 100644 index 0000000000..f95600b8f3 --- /dev/null +++ b/examples/perf_envelope.rs @@ -0,0 +1,714 @@ +//! Performance envelope (#394): query latency (p50/p99), open time, checkpoint +//! time and RSS at 100K, 1M and 10M facts. +//! +//! Usage: +//! +//! ```text +//! cargo run --profile envelope --example perf_envelope -- [--scales 100k,1m,10m] +//! [--dir target/perf-envelope] [--json out.json] [--budget-secs 15] +//! [--cap-secs 60] [--rebuild] +//! ``` +//! +//! The `envelope` profile is `release` at opt-level 3, which is how an +//! application's release build compiles the library; `release` itself (and +//! `bench`, which inherits it) is tuned for the REPL binary's size. +//! +//! Each fixture is built once and cached in `--dir`. The build, the storage +//! measurements and every query shape run in their own child process, so each +//! peak RSS is that step's alone. Storage measurements run on a copy, because +//! checkpoints write to it; queries open the fixture read-only. The OS page +//! cache is warm: a cold open needs root to drop caches. +//! +//! Each shape runs until 200 runs or `--budget-secs`, at least 5 runs; with +//! fewer than 100 runs p99 is the maximum. A shape whose first run is still +//! running after `--cap-secs` is reported as capped. +//! +//! RSS comes from `/proc/self/status` and is reported as 0 off Linux. +//! Design: `docs/superpowers/specs/2026-10-11-perf-envelope-design.md`. + +// A measurement tool, not library code: a malformed argument or report is a bug +// in the tool and may panic. +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::cast_possible_truncation, + clippy::cast_sign_loss +)] + +#[cfg(not(target_arch = "wasm32"))] +use minigraf::{Minigraf, OpenOptions}; +#[cfg(not(target_arch = "wasm32"))] +use serde_json::{Value as Json, json}; +#[cfg(not(target_arch = "wasm32"))] +use std::path::{Path, PathBuf}; +#[cfg(not(target_arch = "wasm32"))] +use std::time::{Duration, Instant}; + +/// Bump when the fixture's contents change, so stale cached files are not reused. +#[cfg(not(target_arch = "wasm32"))] +const FIXTURE_VERSION: u32 = 1; +/// Facts per entity (see `build`). +#[cfg(not(target_arch = "wasm32"))] +const FACTS_PER_ENTITY: usize = 6; +/// Entities per transaction. +#[cfg(not(target_arch = "wasm32"))] +const BATCH_ENTITIES: usize = 200; +/// Explicit checkpoint interval during the build, in facts; keeps RSS bounded. +#[cfg(not(target_arch = "wasm32"))] +const BUILD_CHECKPOINT_FACTS: usize = 200_000; +/// Retract/reassert rounds of `:hot :counter`. +#[cfg(not(target_arch = "wasm32"))] +const HOT_HISTORY: usize = 1_000; + +#[cfg(not(target_arch = "wasm32"))] +fn main() -> anyhow::Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + match args.first().map(String::as_str) { + Some("__build") => { + let out = build(Path::new(&args[1]), args[2].parse()?)?; + println!("{out}"); + Ok(()) + } + Some("__storage") => { + let out = measure_storage(Path::new(&args[1]), Path::new(&args[2]))?; + println!("{out}"); + Ok(()) + } + Some("__query") => { + let out = measure_query( + Path::new(&args[1]), + args[2].parse()?, + args[3].parse()?, + args[4].parse()?, + )?; + println!("{out}"); + Ok(()) + } + _ => run(&args), + } +} + +// ── Parent ──────────────────────────────────────────────────────────────────── + +#[cfg(not(target_arch = "wasm32"))] +fn run(args: &[String]) -> anyhow::Result<()> { + let mut scales = vec!["100k".to_string(), "1m".to_string(), "10m".to_string()]; + let mut dir = PathBuf::from("target/perf-envelope"); + let mut json_out: Option = None; + let mut budget_secs: u64 = 15; + let mut cap_secs: u64 = 60; + let mut rebuild = false; + let mut it = args.iter(); + while let Some(a) = it.next() { + let mut val = || { + it.next() + .cloned() + .ok_or_else(|| anyhow::anyhow!("{a} needs a value")) + }; + match a.as_str() { + "--scales" => scales = val()?.split(',').map(str::to_string).collect(), + "--dir" => dir = PathBuf::from(val()?), + "--json" => json_out = Some(PathBuf::from(val()?)), + "--budget-secs" => budget_secs = val()?.parse()?, + "--cap-secs" => cap_secs = val()?.parse()?, + "--rebuild" => rebuild = true, + other => anyhow::bail!("unknown argument {other}"), + } + } + std::fs::create_dir_all(&dir)?; + let exe = std::env::current_exe()?; + + let mut results = Vec::new(); + for label in &scales { + let n = parse_scale(label)?; + let fixture = dir.join(format!("fixture-v{FIXTURE_VERSION}-{label}.graph")); + let meta = fixture.with_extension("json"); + if rebuild || !fixture.exists() || !meta.exists() { + let _ = std::fs::remove_file(&fixture); + let _ = std::fs::remove_file(fixture.with_extension("graph.wal")); + eprintln!("perf_envelope: building {label} fixture ({n} facts)"); + let out = child(&exe, &["__build", path_str(&fixture), &n.to_string()])?; + std::fs::write(&meta, out.to_string())?; + } + eprintln!("perf_envelope: measuring {label}"); + let copy = dir.join(format!("measure-{label}.graph")); + let storage = child(&exe, &["__storage", path_str(&fixture), path_str(©)]); + let _ = std::fs::remove_file(©); + let _ = std::fs::remove_file(copy.with_extension("graph.wal")); + let mut out = storage?; + let meta = read_meta(&fixture)?; + let mut queries = Vec::new(); + for (i, shape) in fixture_shapes(&meta).iter().enumerate() { + let q = child( + &exe, + &[ + "__query", + path_str(&fixture), + &i.to_string(), + &budget_secs.to_string(), + &cap_secs.to_string(), + ], + )?; + match q.get("p50_ms") { + Some(_) => eprintln!( + " {:<20} p50 {:>10.3} ms p99 {:>10.3} ms ({} runs, {} rows)", + shape.name, + q["p50_ms"].as_f64().unwrap(), + q["p99_ms"].as_f64().unwrap(), + q["runs"], + q["rows"] + ), + None if q.get("error").is_some() => { + eprintln!(" {:<20} fails: {}", shape.name, q["error"]) + } + None => eprintln!(" {:<20} capped at {} ms", shape.name, q["capped_ms"]), + } + queries.push(q); + } + out["queries"] = json!(queries); + out["label"] = json!(label); + out["build"] = meta; + results.push(out); + } + + let report = json!({ + "minigraf": env!("CARGO_PKG_VERSION"), + "fixture_version": FIXTURE_VERSION, + "budget_secs": budget_secs, + "cap_secs": cap_secs, + "scales": results, + }); + if let Some(p) = json_out { + std::fs::write(p, serde_json::to_string_pretty(&report)?)?; + } + print_markdown(&report); + Ok(()) +} + +#[cfg(not(target_arch = "wasm32"))] +fn child(exe: &Path, args: &[&str]) -> anyhow::Result { + let out = std::process::Command::new(exe) + .args(args) + .stderr(std::process::Stdio::inherit()) + .output()?; + anyhow::ensure!(out.status.success(), "child {} failed", args[0]); + let stdout = String::from_utf8(out.stdout)?; + let last = stdout.lines().last().unwrap_or_default(); + Ok(serde_json::from_str(last)?) +} + +#[cfg(not(target_arch = "wasm32"))] +fn parse_scale(label: &str) -> anyhow::Result { + let l = label.to_ascii_lowercase(); + let (num, mul) = if let Some(n) = l.strip_suffix('m') { + (n, 1_000_000) + } else if let Some(n) = l.strip_suffix('k') { + (n, 1_000) + } else { + (l.as_str(), 1) + }; + let n = num.parse::()? * mul; + anyhow::ensure!( + n >= FACTS_PER_ENTITY * BATCH_ENTITIES, + "scale {label} is below {} facts", + FACTS_PER_ENTITY * BATCH_ENTITIES + ); + Ok(n) +} + +#[cfg(not(target_arch = "wasm32"))] +fn path_str(p: &Path) -> &str { + p.to_str().expect("utf-8 path") +} + +#[cfg(not(target_arch = "wasm32"))] +fn print_markdown(report: &Json) { + let scales = report["scales"].as_array().unwrap(); + let labels: Vec<&str> = scales + .iter() + .map(|s| s["label"].as_str().unwrap()) + .collect(); + let header = |first: &str| { + let mut h = format!("| {first} |"); + let mut sep = String::from("|---|"); + for l in &labels { + h.push_str(&format!(" {l} |")); + sep.push_str("---:|"); + } + println!("{h}\n{sep}"); + }; + let row = |name: &str, f: &dyn Fn(&Json) -> String| { + let mut r = format!("| {name} |"); + for s in scales { + r.push_str(&format!(" {} |", f(s))); + } + println!("{r}"); + }; + + println!("### Storage\n"); + header("Metric"); + row("Facts", &|s| s["build"]["facts"].to_string()); + row("File size", &|s| { + mib(s["build"]["file_bytes"].as_u64().unwrap()) + }); + row("Build time", &|s| { + secs(s["build"]["build_ms"].as_f64().unwrap()) + }); + row("Build peak RSS", &|s| { + kib(s["build"]["peak_rss_kb"].as_u64().unwrap()) + }); + row("Open", &|s| ms(s["open_ms"].as_f64().unwrap())); + row("RSS after open", &|s| { + kib(s["rss_after_open_kb"].as_u64().unwrap()) + }); + row("Checkpoint, 1 new fact (p50 / p99)", &|s| { + let c = &s["checkpoint_1_fact"]; + format!( + "{} / {}", + ms(c["p50_ms"].as_f64().unwrap()), + ms(c["p99_ms"].as_f64().unwrap()) + ) + }); + row("Checkpoint, 10K new facts", &|s| { + ms(s["checkpoint_10k_ms"].as_f64().unwrap()) + }); + row("Peak RSS (open + checkpoints)", &|s| { + kib(s["peak_rss_kb"].as_u64().unwrap()) + }); + + println!("\n### Query latency (p50 / p99)\n"); + header("Shape"); + let names: Vec = scales[0]["queries"] + .as_array() + .unwrap() + .iter() + .map(|q| q["name"].as_str().unwrap().to_string()) + .collect(); + for name in &names { + row(&format!("`{name}`"), &|s| { + let q = query(s, name); + match (q["p50_ms"].as_f64(), q["error"].as_str()) { + (Some(p50), _) => format!("{} / {}", ms(p50), ms(q["p99_ms"].as_f64().unwrap())), + (None, Some(e)) => format!("fails: {}", error_code(e)), + (None, None) => format!("> {} s", report["cap_secs"]), + } + }); + } + + println!("\n### Rows, runs and peak RSS per shape\n"); + header("Shape"); + for name in &names { + row(&format!("`{name}`"), &|s| { + let q = query(s, name); + format!( + "{} rows, {} runs, {}", + q.get("rows").map_or("?".into(), Json::to_string), + q.get("runs").map_or("0".into(), Json::to_string), + kib(q["peak_rss_kb"].as_u64().unwrap()) + ) + }); + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn query<'a>(scale: &'a Json, name: &str) -> &'a Json { + scale["queries"] + .as_array() + .unwrap() + .iter() + .find(|q| q["name"] == name) + .unwrap() +} + +/// The `[XXX-NNN]` code at the start of a `MinigrafError` message, or the message. +#[cfg(not(target_arch = "wasm32"))] +fn error_code(msg: &str) -> &str { + msg.strip_prefix('[') + .and_then(|m| m.split_once(']')) + .map_or(msg, |(code, _)| code) +} + +#[cfg(not(target_arch = "wasm32"))] +fn ms(v: f64) -> String { + if v < 1.0 { + format!("{:.0} µs", v * 1000.0) + } else if v < 1000.0 { + format!("{v:.2} ms") + } else { + secs(v) + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn secs(v: f64) -> String { + format!("{:.2} s", v / 1000.0) +} + +#[cfg(not(target_arch = "wasm32"))] +fn kib(v: u64) -> String { + mib(v * 1024) +} + +#[cfg(not(target_arch = "wasm32"))] +fn mib(v: u64) -> String { + format!("{:.0} MiB", v as f64 / (1024.0 * 1024.0)) +} + +// ── Build ───────────────────────────────────────────────────────────────────── + +/// Builds `n` facts over `n / 6` entities `:p{i}` (six facts each) plus a hot +/// entity with a long history, fully checkpointed. Returns the build metadata +/// the measurements need. +#[cfg(not(target_arch = "wasm32"))] +fn build(path: &Path, n: usize) -> anyhow::Result { + let start = Instant::now(); + let e = n / FACTS_PER_ENTITY; + let db = OpenOptions::new() + .wal_checkpoint_threshold(usize::MAX) + .path(path) + .open()?; + let mut since_checkpoint = 0; + let mut tx_half = 0; + for lo in (0..e).step_by(BATCH_ENTITIES) { + let hi = (lo + BATCH_ENTITIES).min(e); + let mut cmd = String::from("(transact ["); + for i in lo..hi { + let follows = (i * 7919 + 1) % e; + cmd.push_str(&format!( + "[:p{i} :name \"p{i}\"][:p{i} :age {}][:p{i} :group {}][:p{i} :follows :p{follows}]", + i % 90, + i % 100 + )); + // :p0 is the root of the :manager tree. + if i > 0 { + cmd.push_str(&format!("[:p{i} :manager :p{}]", i / 10)); + } + } + cmd.push_str("])"); + db.execute(&cmd)?; + let mut a = String::from( + "(transact {:valid-from \"2020-01-01T00:00:00Z\" :valid-to \"2023-01-01T00:00:00Z\"} [", + ); + let mut b = String::from("(transact {:valid-from \"2023-01-01T00:00:00Z\"} ["); + for i in lo..hi { + a.push_str(&format!("[:p{i} :status \"a\"]")); + b.push_str(&format!("[:p{i} :status \"b\"]")); + } + a.push_str("])"); + b.push_str("])"); + db.execute(&a)?; + db.execute(&b)?; + if lo < e / 2 && hi >= e / 2 { + tx_half = db.current_tx_count(); + } + since_checkpoint += (hi - lo) * FACTS_PER_ENTITY; + if since_checkpoint >= BUILD_CHECKPOINT_FACTS { + db.checkpoint()?; + since_checkpoint = 0; + } + } + db.execute("(transact [[:hot :counter 0]])")?; + let mut hot_tx_half = 0; + for i in 1..HOT_HISTORY { + db.execute(&format!("(retract [[:hot :counter {}]])", i - 1))?; + db.execute(&format!("(transact [[:hot :counter {i}]])"))?; + if i == HOT_HISTORY / 2 { + hot_tx_half = db.current_tx_count(); + } + } + db.checkpoint()?; + drop(db); + let build_ms = start.elapsed().as_secs_f64() * 1000.0; + Ok(json!({ + "facts": n, + "entities": e, + "tx_half": tx_half, + "hot_tx_half": hot_tx_half, + "build_ms": build_ms, + "peak_rss_kb": proc_status_kb("VmHWM"), + "file_bytes": std::fs::metadata(path)?.len(), + })) +} + +// ── Measure ─────────────────────────────────────────────────────────────────── + +#[cfg(not(target_arch = "wasm32"))] +struct Shape { + name: &'static str, + query: String, + /// Expected row count, checked on every run. + rows: usize, +} + +#[cfg(not(target_arch = "wasm32"))] +fn shapes(e: usize, tx_half: u64, hot_tx_half: u64) -> Vec { + let m = e / 2; + let leaf = e - 1; + // :manager depth from the leaf to :p0. + let mut depth = 0; + let mut i = leaf; + while i > 0 { + i /= 10; + depth += 1; + } + // Entities whose manager is :p{m / 10}: 10 siblings, all below `e` for m = e/2. + let siblings = (10 * (m / 10)..10 * (m / 10) + 10) + .filter(|&s| s > 0) + .count(); + vec![ + Shape { + name: "point_entity", + query: format!("[:find ?a :where [:p{m} :age ?a]]"), + rows: 1, + }, + Shape { + name: "attribute_value", + query: format!("[:find ?e :where [?e :name \"p{m}\"]]"), + rows: 1, + }, + Shape { + name: "attribute_scan", + query: "[:find ?e ?a :where [?e :age ?a]]".into(), + rows: e, + }, + Shape { + name: "join_2", + query: format!("[:find ?n :where [:p{m} :follows ?f] [?f :name ?n]]"), + rows: 1, + }, + Shape { + name: "join_3_selective", + query: format!("[:find ?n :where [:p{m} :manager ?b] [?s :manager ?b] [?s :name ?n]]"), + rows: siblings, + }, + Shape { + name: "join_3_scan", + query: "[:find ?e ?n :where [?e :follows ?f] [?f :group 7] [?f :name ?n]]".into(), + // Not checked: depends on how :follows maps onto group 7. + rows: usize::MAX, + }, + Shape { + name: "recursive_ancestors", + query: format!("[:find ?b :where (boss :p{leaf} ?b)]"), + rows: depth, + }, + Shape { + name: "as_of", + query: format!("[:find ?a :as-of {tx_half} :where [:p{} :age ?a]]", m / 4), + rows: 1, + }, + Shape { + name: "valid_at", + query: format!( + "[:find ?s :valid-at \"2021-06-01T00:00:00Z\" :where [:p{m} :status ?s]]" + ), + rows: 1, + }, + Shape { + name: "count_grouped", + query: "[:find ?g (count ?e) :where [?e :group ?g]]".into(), + rows: 100.min(e), + }, + Shape { + name: "hot_current", + query: "[:find ?v :where [:hot :counter ?v]]".into(), + rows: 1, + }, + Shape { + name: "hot_as_of", + query: format!( + "[:find ?v :as-of {hot_tx_half} :valid-at :any-valid-time :where [:hot :counter ?v]]" + ), + rows: 1, + }, + ] +} + +#[cfg(not(target_arch = "wasm32"))] +fn read_meta(fixture: &Path) -> anyhow::Result { + Ok(serde_json::from_str(&std::fs::read_to_string( + fixture.with_extension("json"), + )?)?) +} + +#[cfg(not(target_arch = "wasm32"))] +fn fixture_shapes(meta: &Json) -> Vec { + shapes( + meta["entities"].as_u64().unwrap() as usize, + meta["tx_half"].as_u64().unwrap(), + meta["hot_tx_half"].as_u64().unwrap(), + ) +} + +/// Open time and RSS after open, then checkpoint times, on a copy of the fixture. +#[cfg(not(target_arch = "wasm32"))] +fn measure_storage(fixture: &Path, copy: &Path) -> anyhow::Result { + std::fs::copy(fixture, copy)?; + std::fs::File::open(copy)?.sync_all()?; + + let t = Instant::now(); + let db = OpenOptions::new() + .wal_checkpoint_threshold(usize::MAX) + .path(copy) + .open()?; + let open_ms = t.elapsed().as_secs_f64() * 1000.0; + let rss_after_open_kb = proc_status_kb("VmRSS"); + + let mut samples = Vec::new(); + for i in 0..20 { + db.execute(&format!("(transact [[:extra/{i} :x {i}]])"))?; + let t = Instant::now(); + db.checkpoint()?; + samples.push(t.elapsed()); + } + let ck1 = summarize(&mut samples); + + for lo in (0..10_000).step_by(1_000) { + let mut cmd = String::from("(transact ["); + for i in lo..lo + 1_000 { + cmd.push_str(&format!("[:bulk/{i} :x {i}]")); + } + cmd.push_str("])"); + db.execute(&cmd)?; + } + let t = Instant::now(); + db.checkpoint()?; + let checkpoint_10k_ms = t.elapsed().as_secs_f64() * 1000.0; + drop(db); + + Ok(json!({ + "open_ms": open_ms, + "rss_after_open_kb": rss_after_open_kb, + "checkpoint_1_fact": {"p50_ms": ck1.p50_ms, "p99_ms": ck1.p99_ms, "runs": ck1.runs}, + "checkpoint_10k_ms": checkpoint_10k_ms, + "peak_rss_kb": proc_status_kb("VmHWM"), + })) +} + +/// Times one query shape on the fixture, opened read-only. Runs in its own +/// process, so its peak RSS is this shape's alone (plus the open baseline). +#[cfg(not(target_arch = "wasm32"))] +fn measure_query( + fixture: &Path, + index: usize, + budget_secs: u64, + cap_secs: u64, +) -> anyhow::Result { + let shape = fixture_shapes(&read_meta(fixture)?).swap_remove(index); + let db = OpenOptions::new().read_only(true).path(fixture).open()?; + db.execute("(rule [(boss ?x ?y) [?x :manager ?y]])")?; + db.execute("(rule [(boss ?x ?z) [?x :manager ?y] (boss ?y ?z)])")?; + let q = format!("(query {})", shape.query); + + // The first run doubles as the cap check: if it is still running after + // `cap_secs`, report the shape as capped and exit without waiting for it. + let db = std::sync::Arc::new(db); + let (tx, rx) = std::sync::mpsc::channel(); + { + let (db, q) = (db.clone(), q.clone()); + std::thread::spawn(move || { + let t = Instant::now(); + let r = drain(&db, &q).map(|rows| (rows, t.elapsed())); + let _ = tx.send(r); + }); + } + let mut out = json!({"name": shape.name, "query": shape.query}); + let (rows, first) = match rx.recv_timeout(Duration::from_secs(cap_secs)) { + Ok(Ok(r)) => r, + // A query error (such as a result limit) is a result, not a harness failure. + Ok(Err(e)) => { + out["error"] = json!(e.to_string().lines().next().unwrap_or_default()); + out["peak_rss_kb"] = json!(proc_status_kb("VmHWM")); + return Ok(out); + } + Err(_) => { + out["capped_ms"] = json!(cap_secs * 1000); + out["peak_rss_kb"] = json!(proc_status_kb("VmHWM")); + println!("{out}"); + std::process::exit(0); + } + }; + if shape.rows != usize::MAX { + anyhow::ensure!( + rows == shape.rows, + "{}: expected {} rows, got {rows}", + shape.name, + shape.rows + ); + } + out["rows"] = json!(rows); + // Warm-up: the first run plus two more, unless a run takes over a second. + if first < Duration::from_secs(1) { + drain(&db, &q)?; + drain(&db, &q)?; + } + + let budget = Duration::from_secs(budget_secs); + let mut samples = Vec::new(); + let start = Instant::now(); + while samples.len() < 200 && (samples.len() < 5 || start.elapsed() < budget) { + let t = Instant::now(); + let r = drain(&db, &q)?; + samples.push(t.elapsed()); + anyhow::ensure!(r == rows, "{}: row count changed", shape.name); + } + let s = summarize(&mut samples); + out["runs"] = json!(s.runs); + out["p50_ms"] = json!(s.p50_ms); + out["p99_ms"] = json!(s.p99_ms); + out["max_ms"] = json!(s.max_ms); + out["peak_rss_kb"] = json!(proc_status_kb("VmHWM")); + Ok(out) +} + +/// Runs `q` and drains its cursor in batches of 1,000 rows; returns the row count. +#[cfg(not(target_arch = "wasm32"))] +fn drain(db: &Minigraf, q: &str) -> anyhow::Result { + let mut cursor = db.query(q)?; + let mut rows = 0; + while let Some(batch) = cursor.next_batch(1_000)? { + rows += batch.len(); + } + Ok(rows) +} + +#[cfg(not(target_arch = "wasm32"))] +struct Summary { + runs: usize, + p50_ms: f64, + p99_ms: f64, + max_ms: f64, +} + +/// Nearest-rank percentiles; with fewer than 100 runs p99 is the maximum. +#[cfg(not(target_arch = "wasm32"))] +fn summarize(samples: &mut [Duration]) -> Summary { + samples.sort(); + let at = |p: f64| { + let rank = ((p * samples.len() as f64).ceil() as usize).clamp(1, samples.len()); + samples[rank - 1].as_secs_f64() * 1000.0 + }; + Summary { + runs: samples.len(), + p50_ms: at(0.50), + p99_ms: at(0.99), + max_ms: at(1.0), + } +} + +/// A `kB` field of `/proc/self/status`; 0 where it does not exist. +#[cfg(not(target_arch = "wasm32"))] +fn proc_status_kb(field: &str) -> u64 { + std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines() + .find_map(|l| l.strip_prefix(field)?.strip_prefix(':').map(str::to_owned)) + }) + .and_then(|v| v.trim().trim_end_matches("kB").trim().parse().ok()) + .unwrap_or(0) +} + +#[cfg(target_arch = "wasm32")] +fn main() {}