From 15aaf224e2061627567f22e2112346ad6c6fa161 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Fri, 25 Sep 2026 10:41:45 +0530 Subject: [PATCH 01/10] revert: move file format v8 (#377) from main to the v3 branch (#378) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main must stay releasable as v2.x so v2.1–v2.3 (and #370's unreleased fix) can ship without a forced one-way format migration. The v8 work (#371, #287) is preserved unchanged on the long-lived `v3` branch (at b8a9762), which will merge main regularly and becomes main when v3.0.0 is cut. This reverts commit b8a9762fe7607bb79ae084f5f0c2d4bffb26008a. Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 12 - CLAUDE.md | 11 +- README.md | 2 +- ROADMAP.md | 4 +- docs/ERROR_REFERENCE.md | 34 +- docs/TEST_COVERAGE.md | 15 +- .../2026-09-24-index-key-value-collision.md | 1120 ----------------- ...-09-21-index-key-value-collision-design.md | 171 --- examples/generate_compat_fixture.rs | 18 +- fuzz/fuzz_targets/btree_page.rs | 24 +- fuzz/fuzz_targets/fact_page.rs | 14 +- src/browser/mod.rs | 8 - src/error.rs | 18 +- src/graph/storage.rs | 33 +- src/query/datalog/executor.rs | 27 +- src/storage/btree.rs | 547 ++++++++ src/storage/btree_v6.rs | 24 - src/storage/index.rs | 397 ++---- src/storage/mod.rs | 212 +++- src/storage/persistent_facts.rs | 884 ++++++++++--- tests/btree_v6_test.rs | 56 +- tests/cross_platform_compat_test.rs | 22 +- tests/error_codes_storage_test.rs | 17 - tests/fixtures/v7_multivalue.graph | Bin 49152 -> 0 bytes tests/migration_matrix_test.rs | 52 +- tests/multi_value_test.rs | 551 -------- tests/wal_test.rs | 54 +- 27 files changed, 1735 insertions(+), 2592 deletions(-) delete mode 100644 docs/superpowers/plans/2026-09-24-index-key-value-collision.md delete mode 100644 docs/superpowers/specs/2026-09-21-index-key-value-collision-design.md create mode 100644 src/storage/btree.rs delete mode 100644 tests/fixtures/v7_multivalue.graph delete mode 100644 tests/multi_value_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 557cbc003a..7484edbc60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,6 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - -### Breaking changes - -- **File format v8** (#287, #371). Index keys now include the encoded value and the assert/retract flag. A v7 file (written by v2.x) is re-indexed and upgraded to v8 the first time it is opened; the upgrade is one-way and v2.x cannot open a v8 file (`STG-006`). Bindings users get this upgrade silently on open. -- **Formats v1–v6 are no longer readable.** Opening one fails with the new `STG-028`; open it once with Minigraf v2.x to upgrade it to v7 first. `STG-003` and `STG-004` are deprecated (no longer emitted, still registered, never reused). `STG-005` and `STG-006` message text changed. - -### Fixed - -- Two values of the same attribute written in one transaction are both returned by every query path. Previously only one came back, and entity-bound and attribute-bound queries could return different ones (#371, #287). -- A retract that removes several values of one attribute in one call now hides all of them (#371). - ## v2.0.0 — 2026-08-26 This was originally slated as v1.3.0, matching the GitHub milestone name under diff --git a/CLAUDE.md b/CLAUDE.md index d54e87d9e1..2b83e26fa9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,14 +104,15 @@ cargo run < demos/demo_negation.txt - `storage.rs`: `FactStorage` — in-memory store, `transact_batch`, `retract`, `get_facts_as_of`, `get_facts_valid_at`, `net_asserted_facts` 2. **`src/storage/`** — Persistence layer - - `mod.rs`: `StorageBackend` trait, `FileHeader` v8 (84 bytes), `CommittedFactReader` / `CommittedIndexReader` traits + - `mod.rs`: `StorageBackend` trait, `FileHeader` v7 (84 bytes), `CommittedFactReader` / `CommittedIndexReader` traits - `backend/file.rs`: Single `.graph` file backend (4KB pages, cross-platform) - `backend/memory.rs`: In-memory backend for testing - `index.rs`: EAVT / AEVT / AVET / VAET key types, `FactRef`, `encode_value` - `btree_v6.rs`: On-disk B+tree (`build_btree`, `OnDiskIndexReader`, `MutexStorageBackend`) + - `btree.rs`: Legacy v5 B+tree (migration only) - `cache.rs`: LRU page cache (`PageCache`, default 256 pages) - `packed_pages.rs`: Packed fact pages (~25 facts/4KB page), `MAX_FACT_BYTES` - - `persistent_facts.rs`: `PersistentFactStorage` — v8 save/load, auto-migration v7→v8 + - `persistent_facts.rs`: `PersistentFactStorage` — v7 save/load, auto-migration v1–v6→v7 3. **`src/query/datalog/`** — Datalog engine - `parser.rs`: EDN/Datalog parser — `transact`, `retract`, `query`, `rule`, `:as-of`, `:valid-at`, `not`, `not-join` @@ -152,7 +153,7 @@ enum Value { String(String), Integer(i64), Float(f64), Boolean(bool), **Important**: `tx_count` (sequential 1, 2, 3…) is what `:as-of N` compares against. The REPL displays `tx_id` (Unix ms). A single `(transact [...])` command increments `tx_count` once regardless of how many facts it contains (`transact_batch`). -### File Format (v8) +### File Format (v7) ``` Page 0: Header (84 bytes) — magic "MGRF", version, page/fact counts, @@ -162,11 +163,11 @@ After: On-disk B+tree index pages (one node per 4KB page) Sidecar: .wal — CRC32-protected WAL entries; replayed on open; deleted on checkpoint ``` -Auto-migrates v7 → v8 on open (index rebuild). v1–v6 are rejected (STG-028). +Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1157 tests passing** (1149 passing, 8 ignored; unit + integration + doc). +**1153 tests passing** (1145 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 721a0fa11a..82414ce6dd 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1157 tests +cargo test # run 1153 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/ROADMAP.md b/ROADMAP.md index 969c4c6b16..1876689185 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,13 +10,13 @@ Completed releases and their implementation details live in the [CHANGELOG](CHAN ## v3.0.0 — File Format Policy -**#287/#371 have landed in-tree (unreleased): file format v8, replacing v7.** Fixing #287's `EavtKey`/`AevtKey` missing-value-bytes data-loss bug bumped the format from v7 to v8; index keys now include the encoded value and the assert/retract flag. Support for v1–v6 is dropped at the same time — opening one fails with `STG-028`. A v7 file is auto-migrated (re-indexed) to v8 the first time it is opened. v3.0.0 itself has not been released yet; this work will ship as part of it. Any database opened at least once under a v1.x release will already be on v7; there are no known users on older formats. +**v3.0.0 supports file format v8 only.** Fixing #287's `EavtKey`/`AevtKey` missing-value-bytes data-loss bug bumps the format from v7 to v8. Support for v1–v6 is dropped at the same time. The v1–v6 → v7 auto-migration code in `persistent_facts.rs` can be removed when cutting this release; a v7→v8 migration replaces it. Any database opened at least once under a v1.x release will already be on v7; there are no known users on older formats. This was the GitHub milestone named “2.0” before v2.0.0 used that version number for the kernel-locking and structured-error-code breaking changes; it was consequently renumbered to v3.0.0. Scope: -- ✅ `EavtKey`/`AevtKey` value-bytes fix, format v8, v7→v8 migration, v1–v6 dropped (#287, #371) +- `EavtKey`/`AevtKey` missing-value-bytes fix and file-format v7→v8 migration (#287) - `lag`/`lead` window functions (#182) - Sliding row frames — `:rows N preceding` (#183; builds on #182) - `PreparedQuery` over UniFFI (#181) diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index e14e299159..d4b1fe9dfc 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -111,9 +111,9 @@ with no `CodedError` anywhere in its chain. | QRY-009 | Rules lock poisoned | Query Execution | | STG-001 | Invalid header: too short | Storage | | STG-002 | Invalid magic number: not a .graph file | Storage | -| STG-003 | Invalid v4/v5/v6 header too short (deprecated) | Storage | -| STG-004 | Invalid v6 header too short (deprecated) | Storage | -| STG-005 | Invalid header too short | Storage | +| STG-003 | Invalid v4/v5/v6 header too short | Storage | +| STG-004 | Invalid v6 header too short | Storage | +| STG-005 | Invalid v7 header too short | Storage | | STG-006 | Unsupported format version | Storage | | STG-007 | page_count must be greater than 0 | Storage | | STG-008 | eavt_root_page must be less than page_count | Storage | @@ -136,7 +136,6 @@ with no `CodedError` anywhere in its chain. | STG-025 | Database already open in this process | Storage | | STG-026 | Database locked by another process | Storage | | STG-027 | Filesystem does not support file locking | Storage | -| STG-028 | Format version no longer supported | Storage | | WAL-001 | Invalid WAL magic number | WAL | | WAL-002 | Unsupported WAL version | WAL | | WAL-003 | Fact serialised size exceeds maximum | WAL | @@ -209,8 +208,6 @@ with no `CodedError` anywhere in its chain. | INT-054 | Unstratifiable negative recursion cycle | Internal | | INT-055 | Rule predicate disappeared during rollback | Internal | -Deprecated codes: STG-003 and STG-004 (pre-v7 header sizes) are deprecated as of v3.0.0 and are no longer emitted, because formats v1–v6 are rejected with STG-028 before any header-size check. They stay registered and documented, and error codes are never removed or reused. - --- ## PRS — Parser Errors @@ -1701,8 +1698,6 @@ See the [file format section in README](../README.md#file-format) for version hi ### STG-003 Invalid v4/v5/v6 header too short -> **Deprecated in v3.0.0.** No longer emitted: files in formats v1–v6 are rejected with [STG-028](#stg-028-format-version-no-longer-supported) before any header-size check. Kept for reference; this code will not be reused. - **Error text**: `Invalid v4/v5/v6 header: expected at least 72 bytes, got {}` **Cause**: A file identified as format version 4, 5, or 6 is too short to hold a valid header of that version. The file is truncated at the header. @@ -1714,8 +1709,6 @@ See the [file format section in README](../README.md#file-format) for version hi ### STG-004 Invalid v6 header too short -> **Deprecated in v3.0.0.** No longer emitted: files in formats v1–v6 are rejected with [STG-028](#stg-028-format-version-no-longer-supported) before any header-size check. Kept for reference; this code will not be reused. - **Error text**: `Invalid v6 header: expected 80 bytes, got {}` **Cause**: A file identified as format version 6 is shorter than the required 80-byte v6 header. The file is truncated. @@ -1725,18 +1718,20 @@ See the [file format section in README](../README.md#file-format) for version hi **Scenario**: A v6-format `.graph` file was written by an older pre-release version and its header is incomplete. -### STG-005 Invalid header too short +### STG-005 Invalid v7 header too short -**Error text**: `Invalid header: expected 84 bytes, got {}` +**Error text**: `Invalid v7 header: expected 84 bytes, got {}` -**Cause**: The file declares a supported format version (7 or later) but is shorter than the 84-byte header. The file is truncated. +**Cause**: A file identified as format version 7 (current format) is shorter than the required 84-byte header. The file is truncated. **Resolution**: - Restore from backup. If the file is newly created, delete it. See the [file format section in README](../README.md#file-format). +**Scenario**: A v7-format `.graph` file was partially written by a crash during header initialisation. + ### STG-006 Unsupported format version -**Error text**: `Unsupported format version: {} (supported: 7-{})` +**Error text**: `Unsupported format version: {} (supported: 1-{})` **Cause**: The file was written by a newer version of Minigraf than is currently installed. The format version number in the header is outside the range this library can read. @@ -1989,17 +1984,6 @@ See the [file format section in README](../README.md#file-format) for version hi **NFSv3 `nolock` is not covered by this error — it is a silent, undetectable gap (#334).** An export explicitly mounted with `-o nolock` behaves differently from plain "no `lockd`": the Linux NFS client serves `flock`/OFD locks out of its own local, in-kernel lock table instead of going over NLM to the server. `try_lock` sees an ordinary local success, `classify` takes the same branch it would on a filesystem where locking genuinely works, and this error is never raised — `allow_unlocked` is never consulted because the code never learns the mount can't really lock. Two separate client hosts writing the same `nolock` export each get a false `Ok(())` from their own kernel with zero cross-host coordination, and can silently corrupt the file. There is no way to detect this from inside `try_lock`'s result, so `nolock` NFSv3 exports are an unsupported deployment for multi-writer use, the same way running mixed Minigraf versions against one file is unsupported (see the kernel-locking CHANGELOG entry) — avoid `nolock` exports rather than relying on this check to catch them. -### STG-028 Format version no longer supported - -**Error text**: `Format version {} is no longer supported (oldest supported: {}). Open the file once with Minigraf v2.x to upgrade it to format v7, then open it with this version.` - -**Cause**: The file was last written by Minigraf v1.x or earlier in a format older than v7. Minigraf v3.0.0 and later read only format v7 (written by v2.x) and v8. - -**Resolution**: -- Open the file once with Minigraf v2.x, which upgrades it to format v7, then open it with this version. The file is not modified by the failed open. - -**Scenario**: A `.graph` file created with Minigraf v1.x and never opened with v2.x. - --- ## WAL — Write-Ahead Log Errors diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index eecbbd86d3..2ccaaadcec 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -1,22 +1,17 @@ # Minigraf Test Coverage -## Coverage Summary +## v2.0.0 Coverage Summary -**Verified**: 2026-09-25 with `cargo test --quiet` +**Verified**: 2026-08-26 with `cargo test --quiet` -**Result**: 1,149 passing tests and 8 ignored tests (1,157 total) +**Result**: 1,145 passing tests and 8 ignored tests (1,153 total) -This supersedes the v2.0.0-era count (1,145 passing, 8 ignored) with the unreleased -file-format-v8 work (#287, #371): the index-key value-collision fix, the v7→v8 -migration, and the drop of formats v1–v6. It otherwise still covers the public -database API, the Datalog engine, storage and recovery paths, and v2.0.0's locking -and structured-error-code changes. +The v2.0.0 suite covers the public database API, the Datalog engine, storage and recovery paths, and the release’s locking and structured-error-code changes. ### Covered Areas - **Core database behavior**: in-memory and file-backed operation, transactions, checkpoints, retractions, bi-temporal queries, recursive rules, prepared statements, aggregation, expressions, disjunction, window functions, UDFs, and magic-sets evaluation. -- **Multi-value and index-key correctness** (`tests/multi_value_test.rs`, 10 tests): same-transaction multi-valued facts surviving query-time dedup, retracting several values of one attribute in one call, valid-time stints (facts differing only in valid time), maximum-value-size boundaries, and format v8 index keys (value bytes + assert/retract flag) (#287, #371). -- **Storage correctness**: packed pages, B+tree indexes and range scans, cache behavior, file-header validation, migration (including v7→v8 index rebuild and v1–v6 rejection via `STG-028`), WAL replay, checksums, corruption handling, and fault injection. +- **Storage correctness**: packed pages, B+tree indexes and range scans, cache behavior, file-header validation, migration, WAL replay, checksums, corruption handling, and fault injection. - **Reliability and concurrency**: concurrent reads and writes, rollback behavior, same-process handle exclusion, cross-process locking, PID namespaces, NFS and container lock behavior, and SIGKILL recovery. - **Public diagnostics**: structured parser, query, storage, WAL, API, and internal error codes; the error-code registry is checked against [`ERROR_REFERENCE.md`](ERROR_REFERENCE.md). - **Compatibility and quality**: cross-platform format compatibility, property-based tests, long-haul smoke coverage, XTDB/Datomic semantic compatibility, grammar conformance, and rustdoc examples. diff --git a/docs/superpowers/plans/2026-09-24-index-key-value-collision.md b/docs/superpowers/plans/2026-09-24-index-key-value-collision.md deleted file mode 100644 index 273dddda89..0000000000 --- a/docs/superpowers/plans/2026-09-24-index-key-value-collision.md +++ /dev/null @@ -1,1120 +0,0 @@ -# Index Key Value Collision (#371, #287) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Two values of one attribute written in one transaction are both stored, indexed and returned by every query path; file format moves to v8 with v7 auto-migration; v1–v6 are no longer readable. - -**Architecture:** Append `value_bytes` + `asserted` to `EavtKey`/`AevtKey` and `asserted` to `AvetKey`/`VaetKey` (sort prefix unchanged). All key construction goes through per-key constructors. A v7 file is always sent down the existing index-rebuild path on open, which writes v8 B+trees and a v8 header. The query-time dedup key in `selective_fact_fetch` gains the encoded value. v1–v6 headers are rejected with a new `STG-028`. - -**Tech Stack:** Rust (MSRV 1.89), postcard, cargo test / clippy / fmt, cargo bench (criterion). - -**Spec:** `docs/superpowers/specs/2026-09-21-index-key-value-collision-design.md` - -## Global Constraints - -- Work only in worktree `.worktrees/fix-371-index-value-collision`, branch `fix/371-index-value-collision`. -- Never use `{:?}` of `Result`, `Fact`, `Value`, `EdnValue` or anything holding a `Uuid` in `assert!`/`assert_eq!` messages (CodeQL `rust/cleartext-logging`). Use plain strings, `unwrap`/`expect`, or count/bool asserts. -- Do not run `cargo test --release` (release profile is `panic = "abort"`); use `cargo bench` for performance. -- Every commit message ends with: - ``` - Co-Authored-By: Claude Opus 5.5 - Claude-Session: https://claude.ai/code/session_01PuKbXn6NaHBMDVkScdBNQZ - ``` -- Commits and the PR reference `#371` and `#287`; only the PR body uses closing keywords (`Closes #371`, `Closes #287`). -- No version bump, no tag, no `release.yml` edits (cargo-dist owns it). -- New error code: `STG-028`. `STG-003`, `STG-004` become deprecated (kept registered and documented, never reused). **Amended after review:** the plan originally deleted them as "retired"; error codes are never removed. -- `FORMAT_VERSION = 8`, new `MIN_FORMAT_VERSION = 7`. -- The pre-push hook runs fmt + clippy + test; CI's clippy may be newer than local — run `cargo clippy --all-targets --all-features -- -D warnings` before pushing. - -## Key finding from planning (read before Task 3/4) - -On-disk B+trees are built from sorted `Vec`s (`build_sorted_index_entries` + `merge_sorted_vecs`), not `BTreeMap`s, so after a checkpoint **both** colliding entries are already on disk today. The value loss after checkpoint comes from the `selective_fact_fetch` dedup (Task 4); the EAVT/AEVT disagreement comes from `sort_unstable_by` putting the two equal-keyed entries in different orders, so "first seen" differs. The pending in-memory `BTreeMap`s (before checkpoint) lose a value through key overwrite (Task 3). Tests in each task target the state that task fixes. - -## File map - -| File | Change | -|------|--------| -| `tests/fixtures/v7_multivalue.graph` | New: v7 file with colliding values, generated by current (v2.0.0-format) code in Task 1 | -| `src/storage/mod.rs` | `FORMAT_VERSION` 8, `MIN_FORMAT_VERSION` 7, header parse/validate for v7+ only, drop `FACT_PAGE_FORMAT_ONE_PER_PAGE`, `pub mod btree` removed | -| `src/storage/btree.rs` | Deleted | -| `src/storage/persistent_facts.rs` | Remove v1/v5/one-per-page migrations and their tests; forced rebuild for `version < FORMAT_VERSION`; constructors in `build_sorted_index_entries` | -| `src/error.rs`, `docs/ERROR_REFERENCE.md` | Add `STG-028`, retire `STG-003`/`STG-004`, reword `STG-005`/`STG-006` | -| `src/storage/index.rs` | New key fields, constructors, `take_while` lookups, `max_uuid`/`lookup_eavt_entity_attr` removed | -| `src/graph/storage.rs` | Range bounds via constructors | -| `src/storage/btree_v6.rs` | Test key literals via helper | -| `src/query/datalog/executor.rs` | `selective_fact_fetch` dedup key includes value bytes | -| `tests/multi_value_test.rs` | New: issue regression (3 states), batched retract, max-size value, fixture migration | -| `tests/migration_matrix_test.rs`, `tests/btree_v6_test.rs`, `tests/wal_test.rs`, `tests/error_codes_storage_test.rs`, `tests/cross_platform_compat_test.rs`, `src/browser/mod.rs` | Legacy-version tests become rejection tests; v7 fixture tests assert v8 after open | -| `examples/generate_compat_fixture.rs` | Doc: `compat.graph` is the frozen v7 fixture; do not regenerate | -| `CHANGELOG.md`, `ROADMAP.md`, `CLAUDE.md`, `README.md`, `docs/TEST_COVERAGE.md`, `.wiki/Architecture.md` | Doc sync (Task 6) | - ---- - -### Task 1: Generate the v7 multi-value fixture (before any code change) - -The fixture must be produced by today's v7 writer, so this runs first on unmodified code. - -**Files:** -- Create: `tests/fixtures/v7_multivalue.graph` -- Scratch (not committed): `examples/gen_v7_multivalue.rs` - -**Interfaces:** -- Produces: `tests/fixtures/v7_multivalue.graph` — v7 file, one checkpoint, no WAL. Contents: `[:t/x :kind :k/a] [:t/x :kind :k/b] [:t/x :note "two"]` plus 30 filler entities `:t/f-` each with `:kind :k/c` and `:note "f"`, all in **one** transact. Then a second transact `[:t/y :tag :g/a] [:t/y :tag :g/b]` followed by a batched retract `[[:t/y :tag :g/a] [:t/y :tag :g/b]]`, then checkpoint. - -- [ ] **Step 1: Confirm the code is unmodified v7** - -Run: `git diff main --stat -- src` → Expected: no output. `grep -n "FORMAT_VERSION: u32" src/storage/mod.rs` → `= 7`. - -- [ ] **Step 2: Write the scratch generator** - -`examples/gen_v7_multivalue.rs`: -```rust -fn main() -> anyhow::Result<()> { - use std::path::PathBuf; - let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); - let tmp = dir.join("v7_multivalue.graph.tmp"); - let _ = std::fs::remove_file(&tmp); - let _ = std::fs::remove_file(dir.join("v7_multivalue.graph.tmp.wal")); - - let db = minigraf::Minigraf::open(&tmp)?; - let mut facts = vec![ - "[:t/x :kind :k/a]".to_string(), - "[:t/x :kind :k/b]".to_string(), - r#"[:t/x :note "two"]"#.to_string(), - ]; - for i in 0..30 { - facts.push(format!("[:t/f-{i} :kind :k/c]")); - facts.push(format!(r#"[:t/f-{i} :note "f"]"#)); - } - db.execute(&format!("(transact [{}])", facts.join(" ")))?; - db.execute("(transact [[:t/y :tag :g/a] [:t/y :tag :g/b]])")?; - db.execute("(retract [[:t/y :tag :g/a] [:t/y :tag :g/b]])")?; - db.checkpoint()?; - drop(db); - let _ = std::fs::remove_file(dir.join("v7_multivalue.graph.tmp.wal")); - std::fs::rename(&tmp, dir.join("v7_multivalue.graph"))?; - Ok(()) -} -``` - -- [ ] **Step 3: Generate and verify it is v7** - -Run: `cargo run --example gen_v7_multivalue && xxd -l 8 tests/fixtures/v7_multivalue.graph` -Expected: `4d47 5246 0700 0000` (magic `MGRF`, version 7). No `.wal` sidecar left in `tests/fixtures/`. - -- [ ] **Step 4: Delete the scratch generator, commit the fixture** - -```bash -rm examples/gen_v7_multivalue.rs -git add tests/fixtures/v7_multivalue.graph -git commit -m "test: add v7 fixture with same-transaction multi-valued facts (#371, #287) - -Generated by the v7 writer at 84bf375 (v2.0.0 format). Generator source is -recorded in tests/multi_value_test.rs. - -" -``` -(The generator source from Step 2 is pasted verbatim into the doc comment of the fixture test in Task 5.) - ---- - -### Task 2: Drop support for file formats v1–v6 - -Still `FORMAT_VERSION = 7` after this task. All tests pass at the end. - -**Files:** -- Modify: `src/storage/mod.rs` (constants ~48–52, `from_bytes` ~180–285, `validate` ~288, header tests ~370–631) -- Delete: `src/storage/btree.rs`; remove `pub mod btree;` from `src/storage/mod.rs:9` -- Modify: `src/storage/persistent_facts.rs` (imports lines 9–15; `load` 207–447; delete `load_one_per_page_legacy` 449–492, `migrate_v1_to_v2` 494–568, `migrate_v5_to_v6` 570–~820; tests) -- Modify: `src/error.rs` (enum ~142–166, `REGISTRY` ~790–940, exhaustive match ~1720–1750, `ALL` list ~1910–1940) -- Modify: `docs/ERROR_REFERENCE.md` (table ~112–138, sections ~1677–1745, add STG-028 section after STG-027 ~1984) -- Modify: `tests/migration_matrix_test.rs`, `tests/btree_v6_test.rs:110-157`, `tests/wal_test.rs:~730-790`, `tests/error_codes_storage_test.rs` - -**Interfaces:** -- Produces: `pub const MIN_FORMAT_VERSION: u32 = 7;` in `crate::storage`; `ErrorCode::Stg028` with args `(found_version, MIN_FORMAT_VERSION)`. - -- [ ] **Step 1: Write failing rejection tests** - -Replace `v3_empty_migrates_without_error` in `tests/migration_matrix_test.rs` with: -```rust -/// Formats v1–v6 are no longer readable (v3.0.0 file-format policy). Opening -/// one must fail with STG-028 and must not modify the file. -#[test] -fn pre_v7_versions_are_rejected_with_stg_028() { - for version in 1u32..=6 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("old.graph"); - let mut page = vec![0u8; PAGE_SIZE]; - page[0..4].copy_from_slice(&MAGIC_NUMBER); - page[4..8].copy_from_slice(&version.to_le_bytes()); - page[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 - std::fs::write(&path, &page).unwrap(); - - let err = match Minigraf::open(&path) { - Ok(_) => panic!("pre-v7 file must not open"), - Err(e) => e, - }; - assert_eq!(err.code(), "STG-028", "pre-v7 file must fail with STG-028"); - assert!( - err.to_string().contains("v2.x"), - "STG-028 message must tell the user how to upgrade" - ); - assert_eq!( - std::fs::read(&path).unwrap(), - page, - "rejected file must be left unmodified" - ); - } -} -``` -Add to `unsupported_version_fails_loudly` after the `is_err` assert: -```rust - assert_eq!(result.err().unwrap().code(), "STG-006"); -``` - -- [ ] **Step 2: Run to confirm failure** - -Run: `cargo test --test migration_matrix_test` -Expected: `pre_v7_versions_are_rejected_with_stg_028` FAILS (v1–v6 currently migrate); compile error for `Stg028` is NOT expected since the test only uses the code string. - -- [ ] **Step 3: Add STG-028, retire STG-003/004, reword STG-005/006 in `src/error.rs`** - -- Enum: remove `Stg003`, `Stg004`; add `Stg028` after `Stg027`. -- `REGISTRY`: delete the `Stg003` and `Stg004` rows; change the `Stg005` and `Stg006` rows and add `Stg028`: -```rust - ( - ErrorCode::Stg005, - "STG-005", - "Invalid header: expected 84 bytes, got {}", - ErrorCategory::Storage, - ), - ( - ErrorCode::Stg006, - "STG-006", - "Unsupported format version: {} (supported: 7-{})", - ErrorCategory::Storage, - ), -``` -```rust - ( - ErrorCode::Stg028, - "STG-028", - "Format version {} is no longer supported (oldest supported: {}). Open the file once with Minigraf v2.x to upgrade it to format v7, then open it with this version.", - ErrorCategory::Storage, - ), -``` -- Exhaustive `match` in `every_error_code_variant_has_a_registry_entry` and the `ALL` array: remove `Stg003`/`Stg004`, add `Stg028` after `Stg027`. - -- [ ] **Step 4: Update `docs/ERROR_REFERENCE.md`** - -- Table: delete the `STG-003` and `STG-004` rows; change `STG-005` row to `| STG-005 | Invalid header too short | Storage |`; add `| STG-028 | Format version no longer supported | Storage |` after `STG-027`. -- Directly under the table add the paragraph: - `Retired codes: STG-003 and STG-004 (pre-v7 header sizes) were retired in v3.0.0, when formats v1–v6 stopped being readable. Retired codes are never reused.` -- Delete the `### STG-003 …` and `### STG-004 …` sections. -- Replace the STG-005 section with: -```markdown -### STG-005 Invalid header too short - -**Error text**: `Invalid header: expected 84 bytes, got {}` - -**Cause**: The file declares a supported format version (7 or later) but is shorter than the 84-byte header. The file is truncated. - -**Resolution**: -- Restore from backup. If the file is newly created, delete it. See the [file format section in README](../README.md#file-format). -``` -- STG-006 section: change the Error text line to `` **Error text**: `Unsupported format version: {} (supported: 7-{})` ``. -- Add after the STG-027 section: -```markdown -### STG-028 Format version no longer supported - -**Error text**: `Format version {} is no longer supported (oldest supported: {}). Open the file once with Minigraf v2.x to upgrade it to format v7, then open it with this version.` - -**Cause**: The file was last written by Minigraf v1.x or earlier in a format older than v7. Minigraf v3.0.0 and later read only format v7 (written by v2.x) and v8. - -**Resolution**: -- Open the file once with Minigraf v2.x, which upgrades it to format v7, then open it with this version. The file is not modified by the failed open. - -**Scenario**: A `.graph` file created with Minigraf v1.x and never opened with v2.x. -``` - -- [ ] **Step 5: Header parsing and validation in `src/storage/mod.rs`** - -After `pub const FORMAT_VERSION: u32 = 7;` add: -```rust -/// Oldest file format version this library reads. Older files must first be -/// opened with Minigraf v2.x, which upgrades them to v7 (see STG-028). -pub const MIN_FORMAT_VERSION: u32 = 7; -``` -Delete `FACT_PAGE_FORMAT_ONE_PER_PAGE` (and its doc comment) and `pub mod btree;`. - -Replace `from_bytes` body from `let version = read_u32_le(bytes, 4)?;` to the end of the `header_checksum` computation with: -```rust - let version = read_u32_le(bytes, 4)?; - if version < MIN_FORMAT_VERSION { - bail_coded!(ErrorCode::Stg028, version, MIN_FORMAT_VERSION); - } - if bytes.len() < 84 { - bail_coded!(ErrorCode::Stg005, bytes.len()); - } - let page_count = read_u64_le(bytes, 8)?; - let node_count = read_u64_le(bytes, 16)?; - let last_checkpointed_tx_count = read_u64_le(bytes, 24)?; - let fact_page_count = read_u64_le(bytes, 72)?; - let header_checksum = read_u32_le(bytes, 80)?; -``` -(Keep the final `Ok(FileHeader { … })` as is.) Update the doc comment to: `Accepts only v7+ (84-byte) headers; older versions fail with STG-028.` Keep the `bytes.len() < 64` → `Stg001` check before reading the version. - -In `validate`: -```rust - if self.version < MIN_FORMAT_VERSION { - bail_coded!(ErrorCode::Stg028, self.version, MIN_FORMAT_VERSION); - } - if self.version > FORMAT_VERSION { - bail_coded!(ErrorCode::Stg006, self.version, FORMAT_VERSION); - } -``` -Update field doc comments that say "zero-initialised when reading v5/v6 or older headers" to drop that clause. - -Header unit tests in `src/storage/mod.rs`: delete `test_file_header_from_bytes_v3_accepted`, `test_file_header_v6_reads_header_checksum_zero`, `test_validate_accepts_versions_1_to_7`, `test_file_header_from_bytes_truncated_v4_rejected`, `test_file_header_v5_fact_page_format_roundtrip`, `test_v4_header_reads_fact_page_format_zero`, `test_validate_accepts_version_5`, `from_bytes_v4_too_short_returns_stg_003`, `from_bytes_v6_too_short_returns_stg_004`. Add: -```rust - #[test] - fn from_bytes_pre_v7_returns_stg_028() { - for version in 1u32..=6 { - let mut bytes = FileHeader::new().to_bytes(); - bytes[4..8].copy_from_slice(&version.to_le_bytes()); - let err = FileHeader::from_bytes(&bytes).expect_err("pre-v7 header must be rejected"); - assert_eq!(crate::error::code_of(&err), "STG-028"); - } - } - - #[test] - fn validate_pre_v7_returns_stg_028() { - let mut h = FileHeader::new(); - h.version = 6; - let err = h.validate().expect_err("v6 must be rejected"); - assert_eq!(crate::error::code_of(&err), "STG-028"); - } -``` -Before writing these, check how the existing `from_bytes_v7_too_short_returns_stg_005` test extracts the code (around line 623) and use the same helper instead of `crate::error::code_of` if it differs. - -- [ ] **Step 6: Remove legacy load paths in `src/storage/persistent_facts.rs`** - -- Imports: delete line 9 (`use crate::storage::btree::{…}`). -- In `load`: delete the `version < 2` and `version == 5` branches and the `fact_page_format == 0 || … ONE_PER_PAGE` block (`load_one_per_page_legacy` call through `return Ok(())`). Delete the now-unused `let fact_page_format = header.fact_page_format;`. Change `if header.version >= 7 && header.header_checksum != 0` (two places) to `if header.header_checksum != 0`. Change `let num_fact_pages = if header.version >= 6 && header.fact_page_count > 0` to `if header.fact_page_count > 0`. -- Delete `load_one_per_page_legacy`, `migrate_v1_to_v2`, `migrate_v5_to_v6` entirely. -- Tests: delete `make_v1_backend`, `test_migrate_v1_to_v2_assigns_defaults`, `test_migrate_v1_tx_counter_set_correctly`, `test_v4_database_migrates_to_v5_on_open`, `test_v6_migration_from_v5_unit`, `test_v6_migration_crash_safe_unit`. Replace `test_v5_migration_large_page_count_does_not_hang` (keep its fuzz-artifact doc paragraph, add one line: "Since v3.0.0 a v5 header is rejected before any page is touched.") with the same header bytes and: -```rust - let err = match PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256) { - Ok(_) => panic!("v5 header must be rejected"), - Err(e) => e, - }; - assert!(err.to_string().contains("no longer supported"), "must fail with STG-028"); -``` -- If `read_all_from_pages` in `packed_pages.rs` is now used only by `CommittedFactLoaderImpl::stream_all` (line ~89), leave it. - -- [ ] **Step 7: Rework legacy integration tests** - -- `tests/btree_v6_test.rs` (the v5 test around lines 110–157): replace with a test that writes the same v5 header, asserts `OpenOptions::new().path(&path).open()` fails with code `STG-028`, and that the file bytes are unchanged. Rename to `test_v5_file_is_rejected`. -- `tests/wal_test.rs` (v2 file test around 730–790): replace the body after building the v2 page with an assert that `Minigraf::open(&db_path)` fails with code `STG-028`. Rename the test by replacing `v2` upgrade wording with `test_v2_file_is_rejected`. -- `tests/error_codes_storage_test.rs`: add -```rust -#[test] -fn open_pre_v7_version_returns_stg_028() { - let dir = scratch_dir("pre_v7"); - let path = dir.join("db.graph"); - write_file(&path, &header_page(|b| { - b[4..8].copy_from_slice(&6u32.to_le_bytes()); // version = 6 - })); - let err = match Minigraf::open(&path) { - Ok(_) => panic!("opening a v6 file should fail"), - Err(e) => e, - }; - assert_eq!(err.code(), "STG-028"); -} -``` -(Match the imports/helpers already at the top of that file.) - -- [ ] **Step 8: Run the full suite** - -Run: `cargo test 2>&1 | tail -30` -Expected: all pass, including `registry_matches_error_reference_doc_bidirectionally`. Also run `cargo build --features browser --target wasm32-unknown-unknown` only if the target is installed (`rustup target list --installed | grep wasm32`); otherwise note it for CI. - -Also: `grep -rn "btree::\|Stg003\|Stg004\|ONE_PER_PAGE\|migrate_v" src tests fuzz benches examples` → Expected: no hits. - -- [ ] **Step 9: Commit** - -```bash -git add -A src tests docs/ERROR_REFERENCE.md -git commit -m "feat!: drop support for file formats v1–v6 (#287) - -Formats older than v7 now fail to open with STG-028, which tells the user -to open the file once with v2.x. Removes the v1→v2, one-per-page and -v5→v6 migrations and the legacy v5 B+tree module. STG-003/STG-004 are -retired. - -" -``` - ---- - -### Task 3: v8 index keys, constructors, forced v7 rebuild - -**Files:** -- Modify: `src/storage/index.rs` (keys 99–144, `Indexes::insert` 169–217, lookups 219–320, tests) -- Modify: `src/storage/persistent_facts.rs` (`build_sorted_index_entries` 1173–1260, `load` `needs_rebuild`, tests asserting version 7) -- Modify: `src/graph/storage.rs:615-760` (`get_facts_by_entity`, `get_facts_by_attribute` bounds) -- Modify: `src/storage/btree_v6.rs` tests (key literals at ~815, 1084–1330) -- Modify: `src/storage/mod.rs` (`FORMAT_VERSION = 8`, tests `test_format_version_is_7`, `test_validate_accepts_version_7`, `test_new_header_has_version_7`, `*_v7*` names/asserts) -- Modify: `tests/cross_platform_compat_test.rs`, `examples/generate_compat_fixture.rs` (doc), `src/browser/mod.rs` test - -**Interfaces:** -- Consumes: `MIN_FORMAT_VERSION` (Task 2). -- Produces (in `crate::storage::index`): - - `EavtKey { entity, attribute, valid_from, valid_to, tx_count, value_bytes: Vec, asserted: bool }` - - `AevtKey { attribute, entity, valid_from, valid_to, tx_count, value_bytes: Vec, asserted: bool }` - - `AvetKey { …existing…, asserted: bool }` (last field) - - `VaetKey { …existing…, asserted: bool }` (last field) - - `EavtKey::from_fact(&Fact) -> EavtKey`, `EavtKey::entity_start(EntityId) -> EavtKey` - - `AevtKey::from_fact(&Fact) -> AevtKey`, `AevtKey::attribute_start(&str) -> AevtKey` - - `AvetKey::from_fact(&Fact) -> AvetKey` - - `VaetKey::from_fact(&Fact) -> Option` (`None` unless value is `Value::Ref`) - -- [ ] **Step 1: Write failing unit tests in `src/storage/index.rs` tests module** - -```rust - fn fact_at(entity: Uuid, attr: &str, value: Value, asserted: bool) -> Fact { - let mut f = Fact::with_valid_time(entity, attr.to_string(), value, 100, 7, 100, VALID_TIME_FOREVER); - f.asserted = asserted; - f - } - - /// #371: two values of one attribute in one transaction must both be indexed. - #[test] - fn same_tx_multi_value_keeps_both_entries_in_all_indexes() { - let e = Uuid::from_u128(1); - let t1 = Uuid::from_u128(10); - let t2 = Uuid::from_u128(11); - let mut idx = Indexes::new(); - idx.insert(&fact_at(e, ":kind", Value::Ref(t1), true), FactRef { page_id: 0, slot_index: 0 }); - idx.insert(&fact_at(e, ":kind", Value::Ref(t2), true), FactRef { page_id: 0, slot_index: 1 }); - assert_eq!(idx.lookup_eavt_entity(e).len(), 2, "EAVT must keep both values"); - assert_eq!(idx.lookup_aevt_attr(":kind").len(), 2, "AEVT must keep both values"); - assert_eq!(idx.eavt.len(), 2); - assert_eq!(idx.aevt.len(), 2); - assert_eq!(idx.avet.len(), 2); - assert_eq!(idx.vaet.len(), 2); - } - - /// An assertion and a retraction of the same value at the same tx_count - /// must not collide in any index. - #[test] - fn same_tx_assert_and_retract_keep_both_entries_in_all_indexes() { - let e = Uuid::from_u128(1); - let t = Uuid::from_u128(10); - let mut idx = Indexes::new(); - idx.insert(&fact_at(e, ":kind", Value::Ref(t), true), FactRef { page_id: 0, slot_index: 0 }); - idx.insert(&fact_at(e, ":kind", Value::Ref(t), false), FactRef { page_id: 0, slot_index: 1 }); - assert_eq!(idx.eavt.len(), 2, "EAVT must keep assert and retract"); - assert_eq!(idx.aevt.len(), 2, "AEVT must keep assert and retract"); - assert_eq!(idx.avet.len(), 2, "AVET must keep assert and retract"); - assert_eq!(idx.vaet.len(), 2, "VAET must keep assert and retract"); - } - - #[test] - fn lookups_do_not_leak_neighbouring_entities_or_attributes() { - let (a, b) = (Uuid::from_u128(1), Uuid::from_u128(2)); - let mut idx = Indexes::new(); - idx.insert(&fact_at(a, ":x", Value::Integer(1), true), FactRef { page_id: 0, slot_index: 0 }); - idx.insert(&fact_at(b, ":x", Value::Integer(1), true), FactRef { page_id: 0, slot_index: 1 }); - idx.insert(&fact_at(a, ":xy", Value::Integer(1), true), FactRef { page_id: 0, slot_index: 2 }); - assert_eq!(idx.lookup_eavt_entity(a).len(), 2); - assert_eq!(idx.lookup_aevt_attr(":x").len(), 2); - assert_eq!(idx.lookup_avet_attr_value(":x", &Value::Integer(1)).len(), 2); - } - - #[test] - fn entity_and_attribute_start_keys_sort_before_every_matching_key() { - let e = Uuid::from_u128(5); - let f = fact_at(e, ":a", Value::Null, false); - assert!(EavtKey::entity_start(e) <= EavtKey::from_fact(&f)); - assert!(AevtKey::attribute_start(":a") <= AevtKey::from_fact(&f)); - assert!(EavtKey::entity_start(Uuid::from_u128(6)) > EavtKey::from_fact(&f)); - } -``` -Also add to `src/storage/persistent_facts.rs` tests (pending state, fails today via `BTreeMap` overwrite): -```rust - #[test] - fn same_tx_multi_value_visible_through_entity_and_attribute_lookups_before_save() { - let pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); - let e = Uuid::from_u128(42); - pfs.storage() - .transact( - vec![ - (e, ":kind".to_string(), Value::Keyword(":k/a".to_string())), - (e, ":kind".to_string(), Value::Keyword(":k/b".to_string())), - ], - None, - ) - .unwrap(); - assert_eq!(pfs.storage().get_facts_by_entity(&e).unwrap().len(), 2); - assert_eq!(pfs.storage().get_facts_by_attribute(&":kind".to_string()).unwrap().len(), 2); - } -``` - -- [ ] **Step 2: Run to confirm failure** - -Run: `cargo test --lib storage::index 2>&1 | tail -20` → Expected: compile errors for `entity_start`/`attribute_start`/`from_fact`. Temporarily comment out `entity_and_attribute_start_keys_sort_before_every_matching_key` and re-run: the two collision tests FAIL (len 1 ≠ 2). Run `cargo test --lib same_tx_multi_value_visible_through` → FAILS. Uncomment afterwards. - -- [ ] **Step 3: New key fields and constructors in `src/storage/index.rs`** - -Replace the four key structs' doc comments and add fields at the end: -```rust -/// EAVT: sort by (Entity, Attribute, ValidFrom, ValidTo, TxCount, ValueBytes, Asserted) -/// -/// `value_bytes` and `asserted` come last so entity/attribute range scans keep -/// their order; they keep facts that differ only in value or assert/retract -/// from colliding (#371, #287). -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct EavtKey { - pub entity: EntityId, - pub attribute: Attribute, - pub valid_from: i64, - pub valid_to: i64, - pub tx_count: u64, - pub value_bytes: Vec, - pub asserted: bool, -} -``` -Same pattern for `AevtKey` (doc: `(Attribute, Entity, ValidFrom, ValidTo, TxCount, ValueBytes, Asserted)`), and append `pub asserted: bool,` to `AvetKey` and `VaetKey` with `, Asserted` appended in their sort-order doc lines. - -Add constructors: -```rust -impl EavtKey { - pub fn from_fact(f: &Fact) -> Self { - EavtKey { - entity: f.entity, - attribute: f.attribute.clone(), - valid_from: f.valid_from, - valid_to: f.valid_to, - tx_count: f.tx_count, - value_bytes: encode_value(&f.value), - asserted: f.asserted, - } - } - - /// Smallest possible key for `entity`: every EAVT key of that entity sorts at or after it. - pub fn entity_start(entity: EntityId) -> Self { - EavtKey { - entity, - attribute: String::new(), - valid_from: i64::MIN, - valid_to: i64::MIN, - tx_count: 0, - value_bytes: Vec::new(), - asserted: false, - } - } -} - -impl AevtKey { - pub fn from_fact(f: &Fact) -> Self { - AevtKey { - attribute: f.attribute.clone(), - entity: f.entity, - valid_from: f.valid_from, - valid_to: f.valid_to, - tx_count: f.tx_count, - value_bytes: encode_value(&f.value), - asserted: f.asserted, - } - } - - /// Smallest possible key for `attribute`. - pub fn attribute_start(attribute: &str) -> Self { - AevtKey { - attribute: attribute.to_string(), - entity: EntityId::nil(), - valid_from: i64::MIN, - valid_to: i64::MIN, - tx_count: 0, - value_bytes: Vec::new(), - asserted: false, - } - } -} - -impl AvetKey { - pub fn from_fact(f: &Fact) -> Self { - AvetKey { - attribute: f.attribute.clone(), - value_bytes: encode_value(&f.value), - valid_from: f.valid_from, - valid_to: f.valid_to, - entity: f.entity, - tx_count: f.tx_count, - asserted: f.asserted, - } - } -} - -impl VaetKey { - /// `None` unless the fact's value is a `Value::Ref` (only refs are VAET-indexed). - pub fn from_fact(f: &Fact) -> Option { - match &f.value { - Value::Ref(target) => Some(VaetKey { - ref_target: *target, - attribute: f.attribute.clone(), - valid_from: f.valid_from, - valid_to: f.valid_to, - source_entity: f.entity, - tx_count: f.tx_count, - asserted: f.asserted, - }), - _ => None, - } - } -} -``` -Replace `Indexes::insert` body with: -```rust - self.eavt.insert(EavtKey::from_fact(fact), fact_ref); - self.aevt.insert(AevtKey::from_fact(fact), fact_ref); - self.avet.insert(AvetKey::from_fact(fact), fact_ref); - if let Some(k) = VaetKey::from_fact(fact) { - self.vaet.insert(k, fact_ref); - } -``` -Replace the lookups (delete `lookup_eavt_entity_attr` — it has no callers — and `max_uuid`): -```rust - /// Query EAVT index for a specific entity (returns all facts for that entity). - pub fn lookup_eavt_entity(&self, entity: EntityId) -> Vec { - self.eavt - .range(EavtKey::entity_start(entity)..) - .take_while(|(k, _)| k.entity == entity) - .map(|(_, v)| *v) - .collect() - } - - /// Query AEVT index for a specific attribute (returns all facts with that attribute). - pub fn lookup_aevt_attr(&self, attribute: &str) -> Vec { - self.aevt - .range(AevtKey::attribute_start(attribute)..) - .take_while(|(k, _)| k.attribute == attribute) - .map(|(_, v)| *v) - .collect() - } - - /// Query AVET index for attribute + value. - pub fn lookup_avet_attr_value(&self, attribute: &str, value: &Value) -> Vec { - let value_bytes = encode_value(value); - let start = AvetKey { - attribute: attribute.to_string(), - value_bytes: value_bytes.clone(), - valid_from: i64::MIN, - valid_to: i64::MIN, - entity: EntityId::nil(), - tx_count: 0, - asserted: false, - }; - self.avet - .range(start..) - .take_while(|(k, _)| k.attribute == attribute && k.value_bytes == value_bytes) - .map(|(_, v)| *v) - .collect() - } - - /// Query VAET index for ref target (reverse references). - pub fn lookup_vaet_ref(&self, target: EntityId) -> Vec { - let start = VaetKey { - ref_target: target, - attribute: String::new(), - valid_from: i64::MIN, - valid_to: i64::MIN, - source_entity: EntityId::nil(), - tx_count: 0, - asserted: false, - }; - self.vaet - .range(start..) - .take_while(|(k, _)| k.ref_target == target) - .map(|(_, v)| *v) - .collect() - } -``` -Update the existing tests in this module that build key literals (`test_eavt_key_ordering_by_entity`, `test_avet_key_orders_by_value_bytes`) by adding `value_bytes: Vec::new(), asserted: true,` / `asserted: true,`. - -- [ ] **Step 4: Use constructors everywhere else** - -- `src/storage/persistent_facts.rs` `build_sorted_index_entries`: replace each struct literal with `EavtKey::from_fact(f)`, `AevtKey::from_fact(f)`, `AvetKey::from_fact(f)`, and the VAET `filter_map` with `VaetKey::from_fact(f).map(|k| (k, fr))`. Remove `encode_value` from the import on line 15 if unused. -- `src/graph/storage.rs` `get_facts_by_entity`: `let start = EavtKey::entity_start(*entity_id);` and `let end = EavtKey::entity_start(next_entity);`. `get_facts_by_attribute`: `let start = AevtKey::attribute_start(attribute);` and `let end_opt = next_string_prefix(attribute).map(|a| AevtKey::attribute_start(&a));`. -- `src/storage/btree_v6.rs` tests: in `make_eavt` and every `EavtKey { … }` literal in the tests module, add `value_bytes: Vec::new(), asserted: true,`. -- Run `grep -rn "EavtKey {\|AevtKey {\|AvetKey {\|VaetKey {" src tests fuzz benches` → only `index.rs` (constructors, lookups, tests) and `btree_v6.rs` tests remain. - -- [ ] **Step 5: Bump to v8 and force the rebuild for v7 in `load`** - -`src/storage/mod.rs`: `pub const FORMAT_VERSION: u32 = 8;` and update the header doc comment "84 bytes in v7" → "84 bytes (v7 and v8 share the layout)". Rename/update tests: `test_format_version_is_7` → `test_format_version_is_8` asserting 8; `test_validate_accepts_version_7` keeps asserting v7 validates (v7 is still readable) and add `h.version = 8` validates; `test_new_header_has_version_7` → `_8`. - -`src/storage/persistent_facts.rs` `load`: replace the `needs_rebuild` computation with: -```rust - // v7 files carry index keys without value bytes / asserted flags; they - // cannot be decoded as v8 keys, so rebuild every index from the fact - // pages (#371, #287). The header is written last, so a crash mid-rebuild - // leaves a v7 header and the next open rebuilds again. - let needs_rebuild = if header.version < FORMAT_VERSION { - true - } else if num_fact_pages == 0 || header.eavt_root_page == 0 { - num_fact_pages > 0 // rebuild if facts exist but no index root - } else { - let backend = self - .backend - .lock() - .map_err(|_| err_coded!(ErrorCode::Stg016))?; - let total_data_pages = header.page_count.saturating_sub(1); - compute_page_checksum(&*backend, 1, total_data_pages)? != header.index_checksum - }; -``` -(This also drops the fact-pages-only checksum fallback: only v7 files could carry it, and they now always rebuild.) Add `use crate::storage::FORMAT_VERSION;` if not already imported. - -Add to `persistent_facts.rs` tests: -```rust - #[test] - fn v7_header_forces_rebuild_and_upgrades_to_v8() { - use crate::storage::FileHeader; - let e = Uuid::from_u128(7); - let mut backend = { - let mut s = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); - s.storage() - .transact(vec![(e, ":n".to_string(), Value::Integer(1))], None) - .unwrap(); - s.mark_dirty(); - s.save().unwrap(); - s.into_backend().unwrap() - }; - // Downgrade the header to v7 (keys on disk are now v8-shaped, which is - // fine: the rebuild never reads them). - let mut h = FileHeader::from_bytes(&backend.read_page(0).unwrap()).unwrap(); - h.version = 7; - h.header_checksum = 0; - h.header_checksum = compute_header_checksum(&h); - let mut page = h.to_bytes(); - page.resize(PAGE_SIZE, 0); - backend.write_page(0, &page).unwrap(); - - let s = PersistentFactStorage::new(backend, 256).unwrap(); - assert_eq!(s.storage().get_facts_by_entity(&e).unwrap().len(), 1); - let b = s.into_backend().unwrap(); - let h2 = FileHeader::from_bytes(&b.read_page(0).unwrap()).unwrap(); - assert_eq!(h2.version, 8, "v7 file must be upgraded to v8 on open"); - } -``` -Check `compute_header_checksum`'s contract (line ~1119) — if it already zeroes the checksum field itself, drop the `h.header_checksum = 0;` line. - -Update remaining assertions of header version 7 in `persistent_facts.rs` tests (`test_save_writes_v4_header` and the `save() must write v7 header` assert near line 1932) to `FORMAT_VERSION`/8. - -- [ ] **Step 6: v7 compat fixture now migrates** - -- `tests/cross_platform_compat_test.rs` `fixture_readable_by_native`: after the existing asserts, add: -```rust - drop(db); - let raw = std::fs::read(&path).expect("read migrated fixture"); - let version = u32::from_le_bytes(raw[4..8].try_into().unwrap()); - assert_eq!(version, 8, "v7 fixture must be upgraded to v8 on open"); -``` - Update its doc comment: the fixture is a frozen v7 file written by v2.0.0; opening it exercises v7→v8 migration. (If `db` is used after, move the drop below the last use; if the file is only rewritten on checkpoint/close, the `drop` makes the check valid.) -- `examples/generate_compat_fixture.rs` doc: replace the "Do not regenerate it unless the v7 file format itself has changed" paragraph with: "`compat.graph` is the frozen v7 fixture written by Minigraf v2.0.0; the cross-platform tests use it to check v7→v8 migration. Do not regenerate it — running this example now writes a v8 file." Change line 1 to "Generates a `.graph` file with two known facts (the committed `compat.graph` is the frozen v7 output of this example from v2.0.0)." -- `src/browser/mod.rs` `native_fixture_readable_by_browser_db`: add after the queries: -```rust - let exported = db.export_graph().expect("export after import"); - let bytes = exported.to_vec(); - assert_eq!( - u32::from_le_bytes(bytes[4..8].try_into().unwrap()), - 8, - "v7 fixture must be upgraded to v8 on import" - ); -``` - -- [ ] **Step 7: Run tests** - -Run: `cargo test 2>&1 | tail -30` -Expected: all pass, including the four new `index.rs` tests, `same_tx_multi_value_visible_through_entity_and_attribute_lookups_before_save`, `v7_header_forces_rebuild_and_upgrades_to_v8`, and `fixture_readable_by_native`. - -- [ ] **Step 8: Commit** - -```bash -git add -A src tests examples -git commit -m "feat!: file format v8 — value bytes and asserted flag in index keys (#371, #287) - -EavtKey/AevtKey gain value_bytes and asserted; AvetKey/VaetKey gain -asserted. Fields are appended so range-scan order is unchanged. All key -construction goes through per-key constructors. A v7 file is always -re-indexed from its fact pages on open and rewritten with a v8 header. - -" -``` - ---- - -### Task 4: Query-time dedup includes the value; end-to-end regression tests - -**Files:** -- Create: `tests/multi_value_test.rs` -- Modify: `src/query/datalog/executor.rs:370-450` (`selective_fact_fetch`) - -**Interfaces:** -- Consumes: `crate::storage::index::encode_value(&Value) -> Vec` (existing). - -- [ ] **Step 1: Write the end-to-end tests** - -`tests/multi_value_test.rs`: -```rust -//! Same-transaction multi-valued facts (#371, #287): every query path must -//! return every value, before checkpoint, after WAL replay, and after -//! checkpoint + reopen. -#![cfg(not(target_arch = "wasm32"))] - -use minigraf::{Minigraf, QueryResult, Value}; -use std::collections::BTreeSet; - -mod common; -use common::{CrashTx, run_crashing_child}; - -fn rows(r: QueryResult) -> Vec> { - match r { - QueryResult::QueryResults { results, .. } => results, - _ => panic!("expected QueryResults"), - } -} - -fn kw(v: &Value) -> String { - match v { - Value::Keyword(k) => k.clone(), - _ => panic!("expected keyword value"), - } -} - -/// Values of `attr` on the entity `ent` through EAVT (entity bound), AEVT -/// (attribute bound, joined to a marker attribute) and a full scan. -fn three_paths(db: &Minigraf, ent: &str, attr: &str, marker: &str) -> [BTreeSet; 3] { - let eavt = rows(db.execute(&format!("(query [:find ?v :where [{ent} {attr} ?v]])")).unwrap()) - .iter() - .map(|r| kw(&r[0])) - .collect(); - let aevt = rows( - db.execute(&format!( - r#"(query [:find ?v :where [?e {attr} ?v] [?e :note "{marker}"]])"# - )) - .unwrap(), - ) - .iter() - .map(|r| kw(&r[0])) - .collect(); - let scan = rows( - db.execute(&format!( - r#"(query [:find ?a ?v :where [?e ?a ?v] [?e :note "{marker}"]])"# - )) - .unwrap(), - ) - .iter() - .filter(|r| matches!(&r[0], Value::Keyword(a) if a == attr)) - .map(|r| kw(&r[1])) - .collect(); - [eavt, aevt, scan] -} - -fn setup_statements(i: usize) -> Vec { - let mut facts = vec![ - format!("[:t/x{i} :kind :k/a]"), - format!("[:t/x{i} :kind :k/b]"), - format!(r#"[:t/x{i} :note "two{i}"]"#), - ]; - for f in 0..30 { - facts.push(format!("[:t/f{i}-{f} :kind :k/c]")); - facts.push(format!(r#"[:t/f{i}-{f} :note "f"]"#)); - } - vec![format!("(transact [{}])", facts.join(" "))] -} - -fn both() -> BTreeSet { - [":k/a", ":k/b"].iter().map(|s| s.to_string()).collect() -} - -#[test] -fn multi_value_visible_on_every_path_before_checkpoint() { - let db = Minigraf::in_memory().unwrap(); - for s in setup_statements(0) { - db.execute(&s).unwrap(); - } - for (name, got) in ["eavt", "aevt", "scan"].iter().zip(three_paths(&db, ":t/x0", ":kind", "two0")) { - assert_eq!(got, both(), "{name} path must return both values before checkpoint"); - } -} - -#[test] -fn multi_value_visible_on_every_path_after_checkpoint_and_reopen() { - // Several graph shapes: the issue saw EAVT and AEVT disagree depending on content. - for i in 0..8 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - { - let db = Minigraf::open(&path).unwrap(); - for s in setup_statements(i) { - db.execute(&s).unwrap(); - } - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let x = format!(":t/x{i}"); - let marker = format!("two{i}"); - for (name, got) in ["eavt", "aevt", "scan"].iter().zip(three_paths(&db, &x, ":kind", &marker)) { - assert_eq!(got, both(), "{name} path must return both values after reopen"); - } - } -} - -#[test] -fn multi_value_visible_on_every_path_after_wal_replay() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - let stmts = setup_statements(0); - let refs: Vec<&str> = stmts.iter().map(String::as_str).collect(); - run_crashing_child(&path, 1_000_000, &refs, CrashTx::Implicit); - let db = Minigraf::open(&path).unwrap(); - for (name, got) in ["eavt", "aevt", "scan"].iter().zip(three_paths(&db, ":t/x0", ":kind", "two0")) { - assert_eq!(got, both(), "{name} path must return both values after WAL replay"); - } -} - -#[test] -fn batched_retract_of_both_values_hides_both_on_every_path() { - for i in 0..8 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - { - let db = Minigraf::open(&path).unwrap(); - for s in setup_statements(i) { - db.execute(&s).unwrap(); - } - db.checkpoint().unwrap(); - } - { - let db = Minigraf::open(&path).unwrap(); - db.execute(&format!("(retract [[:t/x{i} :kind :k/a] [:t/x{i} :kind :k/b]])")) - .unwrap(); - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let x = format!(":t/x{i}"); - let marker = format!("two{i}"); - for (name, got) in ["eavt", "aevt", "scan"].iter().zip(three_paths(&db, &x, ":kind", &marker)) { - assert!(got.is_empty(), "{name} path must hide both retracted values"); - } - } -} -``` -Note: `tests/common/mod.rs` dispatches the crash child via a test named `common::crash_child_entrypoint`; check how `wal_test.rs` includes `common` and mirror it exactly (some binaries need an extra `#[test]` shim — copy whatever `wal_test.rs` does). - -- [ ] **Step 2: Run to confirm failure** - -Run: `cargo test --test multi_value_test 2>&1 | tail -30` -Expected: `…_after_checkpoint_and_reopen` and `batched_retract_…` FAIL (dedup drops a value / leaves a retracted value visible); `…_before_checkpoint` and `…_after_wal_replay` may already PASS after Task 3 — record which. - -- [ ] **Step 3: Fix the dedup key in `selective_fact_fetch`** - -In `src/query/datalog/executor.rs`, update the doc line `deduplicated by (entity, attribute, tx_count, asserted)` to `deduplicated by (entity, attribute, tx_count, asserted, value)`, then replace the `seen` declaration comment and type, and both key constructions: -```rust - // Dedup key: (entity, attribute, tx_count, asserted, encoded value). - // `asserted` keeps an assertion and a retraction from the same - // WriteTransaction apart; the encoded value keeps two values of one - // attribute written in the same transaction apart (#371). Uses - // `encode_value` bytes rather than `Value` to avoid Float hashing. - let mut seen: HashSet<(uuid::Uuid, String, u64, bool, Vec)> = HashSet::new(); -``` -```rust - let key = ( - fact.entity, - fact.attribute.clone(), - fact.tx_count, - fact.asserted, - crate::storage::index::encode_value(&fact.value), - ); -``` -(both loops). - -Add a unit test next to existing `selective_fact_fetch` tests (search `selective_fact_fetch` in the executor tests module; if none exist, the integration tests above are the coverage and no unit test is added). - -- [ ] **Step 4: Run tests** - -Run: `cargo test --test multi_value_test && cargo test 2>&1 | tail -15` -Expected: all pass. - -- [ ] **Step 5: Commit** - -```bash -git add tests/multi_value_test.rs src/query/datalog/executor.rs -git commit -m "fix: keep same-transaction multi-valued facts through query-time dedup (#371) - -selective_fact_fetch deduplicated on (entity, attribute, tx_count, -asserted), which collapsed two values written in one transaction and left -one half of a batched retract un-cancelled. The key now includes the -encoded value. - -" -``` - ---- - -### Task 5: v7 multi-value fixture migration and maximum-size values - -**Files:** -- Modify: `tests/multi_value_test.rs` - -**Interfaces:** -- Consumes: `tests/fixtures/v7_multivalue.graph` (Task 1), migration (Task 3), dedup fix (Task 4). - -- [ ] **Step 1: Add the tests** - -Append to `tests/multi_value_test.rs`: -```rust -/// `tests/fixtures/v7_multivalue.graph` was written by the v7 writer at -/// 84bf375 (v2.0.0 format) with this generator: -/// -/// ```ignore -/// -/// ``` -/// -/// Opening it must migrate to v8 and return both values on every path, and -/// the batched retract of `:t/y`'s two `:tag` values must hide both. -#[test] -fn v7_multivalue_fixture_migrates_and_reads_correctly() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("v7.graph"); - std::fs::write(&path, include_bytes!("fixtures/v7_multivalue.graph")).unwrap(); - - { - let db = Minigraf::open(&path).unwrap(); - for (name, got) in ["eavt", "aevt", "scan"].iter().zip(three_paths(&db, ":t/x", ":kind", "two")) { - assert_eq!(got, both(), "{name} path must return both values after migration"); - } - let tags = rows(db.execute("(query [:find ?v :where [:t/y :tag ?v]])").unwrap()); - assert!(tags.is_empty(), "batched retract in the v7 file must hide both values"); - } - - let raw = std::fs::read(&path).unwrap(); - assert_eq!(u32::from_le_bytes(raw[4..8].try_into().unwrap()), 8, "file must be v8 after open"); - - // Second open is a plain v8 open (no rebuild) and reads the same. - let db = Minigraf::open(&path).unwrap(); - for (name, got) in ["eavt", "aevt", "scan"].iter().zip(three_paths(&db, ":t/x", ":kind", "two")) { - assert_eq!(got, both(), "{name} path must return both values on second open"); - } -} - -/// Index entries now carry the value bytes; a near-maximum string value must -/// still checkpoint and read back through every index. -#[test] -fn near_max_size_values_checkpoint_and_read_back() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("big.graph"); - let big_a = "a".repeat(3_800); - let big_b = "b".repeat(3_800); - { - let db = Minigraf::open(&path).unwrap(); - db.execute(&format!( - r#"(transact [[:t/big :blob "{big_a}"] [:t/big :blob "{big_b}"] [:t/big :note "big"]])"# - )) - .unwrap(); - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let by_entity = rows(db.execute("(query [:find ?v :where [:t/big :blob ?v]])").unwrap()); - assert_eq!(by_entity.len(), 2, "EAVT path must return both large values"); - let by_attr = rows(db.execute(r#"(query [:find ?v :where [?e :blob ?v] [?e :note "big"]])"#).unwrap()); - assert_eq!(by_attr.len(), 2, "AEVT path must return both large values"); - let by_value = rows(db.execute(&format!(r#"(query [:find ?e :where [?e :blob "{big_a}"]])"#)).unwrap()); - assert_eq!(by_value.len(), 1, "AVET path must find the large value"); -} -``` -Before settling on 3 800, check `MAX_FACT_BYTES` (`src/storage/packed_pages.rs:48`) and the transact path's size check: pick the largest string length that still transacts successfully today (try 3 800; if the transact fails with a size error, lower it in steps of 100 and note the chosen value in a comment). The point is a value whose fact is within ~300 bytes of `MAX_FACT_BYTES`. - -- [ ] **Step 2: Run** - -Run: `cargo test --test multi_value_test 2>&1 | tail -20` -Expected: both new tests pass. If `near_max_size_values_checkpoint_and_read_back` fails inside `build_btree`/`write_leaf_page` (entry larger than a page), STOP and report — the spec's oversized-entry risk has materialised and needs a design decision. - -- [ ] **Step 3: Commit** - -```bash -git add tests/multi_value_test.rs -git commit -m "test: v7 multi-value fixture migration and near-max-size values (#371, #287) - -" -``` - ---- - -### Task 6: Benchmarks and documentation sync - -**Files:** -- Modify: `CHANGELOG.md`, `ROADMAP.md`, `CLAUDE.md`, `README.md`, `docs/TEST_COVERAGE.md`, `docs/BENCHMARKS.md` (only if the numbers are recorded there) -- Modify (separate repo): `.wiki/Architecture.md` in the **main checkout** (`/home/aditya/Work/AMC/Minigraf/minigraf/.wiki`) — it is not inside the worktree - -- [ ] **Step 1: Benchmark comparison** - -```bash -cargo bench --bench minigraf_bench -- --save-baseline v8 2>&1 | tail -40 -``` -Then in a throwaway worktree of `main` (`git worktree add /tmp/claude-1000/…/scratchpad/main-bench main`), run `cargo bench --bench minigraf_bench -- --save-baseline v7` and compare with `critcmp` if installed, or record both criterion summaries. Also record file size for 10 000 facts on both (write a tiny scratch example that transacts 10 000 facts, checkpoints, prints `std::fs::metadata(path).len()`; do not commit it). Remove the throwaway worktree afterwards. Put the numbers in the PR description. - -- [ ] **Step 2: CHANGELOG.md** — under an `## [Unreleased]` section (create it at the top if missing, following the file's existing heading style): -```markdown -### Breaking changes -- **File format v8** (#287, #371). Index keys now include the encoded value and the assert/retract flag. A v7 file (written by v2.x) is re-indexed and upgraded to v8 the first time it is opened; the upgrade is one-way and v2.x cannot open a v8 file (`STG-006`). Bindings users get this upgrade silently on open. -- **Formats v1–v6 are no longer readable.** Opening one fails with the new `STG-028`; open it once with Minigraf v2.x to upgrade it to v7 first. `STG-003` and `STG-004` are retired. `STG-005` and `STG-006` message text changed. - -### Fixed -- Two values of the same attribute written in one transaction are both returned by every query path. Previously only one came back, and entity-bound and attribute-bound queries could return different ones (#371, #287). -- A retract that removes several values of one attribute in one call now hides all of them (#371). -``` -- [ ] **Step 3: ROADMAP.md** — in "v3.0.0 — File Format Policy": replace the first paragraph's future tense with a statement that #287/#371 landed: format v8, v7 migration, v1–v6 dropped; remove "The v1–v6 → v7 auto-migration code … can be removed when cutting this release; a v7→v8 migration replaces it." In Scope, mark the `#287` line as done (follow the file's existing done-marker convention; if none, change to "- ✅ `EavtKey`/`AevtKey` value-bytes fix, format v8, v7→v8 migration, v1–v6 dropped (#287, #371)"). -- [ ] **Step 4: CLAUDE.md** — Module Structure: `mod.rs` "`FileHeader` v8 (84 bytes)"; remove the `btree.rs` line; `persistent_facts.rs` "v8 save/load, auto-migration v7→v8"; "### File Format (v8)" and replace "Auto-migrates v1/v2/…/v6 → v7 on open/checkpoint." with "Auto-migrates v7 → v8 on open (index rebuild). v1–v6 are rejected (STG-028)."; update the test count line with the exact numbers from `cargo test 2>&1 | grep -E "^test result"` summed (passed, ignored). -- [ ] **Step 5: README.md** — `grep -n "v7\|format version\|migrat" README.md`; update any file-format section to v8 and the supported-versions statement. -- [ ] **Step 6: docs/TEST_COVERAGE.md** — add `tests/multi_value_test.rs` (6 tests) and update per-file counts for the files changed in Tasks 2–5 and the total, matching `CLAUDE.md`. -- [ ] **Step 7: Cross-check** — `grep -rn "v7\b\|1-7\|v1–v6\|FORMAT_VERSION" CLAUDE.md ROADMAP.md README.md CHANGELOG.md docs/TEST_COVERAGE.md` and confirm no doc contradicts another. -- [ ] **Step 8: Commit docs** - -```bash -git add CHANGELOG.md ROADMAP.md CLAUDE.md README.md docs/TEST_COVERAGE.md -git commit -m "docs: file format v8, v1–v6 dropped, multi-value fix (#371, #287) - -" -``` -- [ ] **Step 9: Wiki** — in `/home/aditya/Work/AMC/Minigraf/minigraf/.wiki`: update `Architecture.md` (key layouts table with the new trailing fields, format v8, migration policy). Do **not** commit/push the wiki until the PR is approved for merge; note it in the PR description as a pending follow-up. - ---- - -### Task 7: Verify, push, PR - -- [ ] **Step 1: Full verification** - -```bash -cargo fmt --all -- --check -cargo clippy --all-targets --all-features -- -D warnings -cargo test 2>&1 | grep -E "^test result|FAILED|panicked" -cargo doc --no-deps 2>&1 | grep -i warning -``` -Expected: fmt clean, clippy clean, every `test result: ok`, no doc warnings. Also `cd fuzz && cargo check` if the fuzz crate builds locally (it references storage types via the public API only — confirm it still compiles). - -- [ ] **Step 2: Push and open the PR** - -```bash -git push -u origin fix/371-index-value-collision -gh pr create --title "fix!: file format v8 — keep same-transaction multi-valued facts (#371, #287)" --body-file /pr-body.md -``` -PR body: summary of root causes (pending `BTreeMap` overwrite, query-time dedup without value, unstable sort order), what changed (key fields, v8 + v7 rebuild, v1–v6 dropped, STG-028, retired STG-003/004), test list, benchmark and file-size numbers from Task 6, "Wiki `Architecture.md` update pending merge", `Closes #371`, `Closes #287`, and end with: -``` -🤖 Generated with [Claude Code](https://claude.com/claude-code) - -https://claude.ai/code/session_01PuKbXn6NaHBMDVkScdBNQZ -``` - -- [ ] **Step 3: Own CI until green** — watch `gh pr checks --watch`; fix every failure (clippy on CI's newer stable, wasm/browser build, CodeQL). Do not merge without explicit user confirmation. Keep the worktree until the PR is merged. diff --git a/docs/superpowers/specs/2026-09-21-index-key-value-collision-design.md b/docs/superpowers/specs/2026-09-21-index-key-value-collision-design.md deleted file mode 100644 index cde3736a7c..0000000000 --- a/docs/superpowers/specs/2026-09-21-index-key-value-collision-design.md +++ /dev/null @@ -1,171 +0,0 @@ -# Index key value collision (#371, #287) — Design - -**Date**: 2026-09-21 -**Issues**: #371 (same-transaction multi-valued facts read back as one value), #287 (`EavtKey`/`AevtKey` carry no value bytes) -**Target**: file format v8, v3.0.0 milestone - -## Problem - -When one `transact` writes two values of the same attribute for one entity, both facts share -`(entity, attribute, valid_from, valid_to, tx_count)`. Two layers then discard one of them: - -1. **Index keys.** `EavtKey` and `AevtKey` have no value bytes. Colliding entries overwrite each - other in the pending `BTreeMap`s. At checkpoint, `build_sorted_index_entries` sorts them with - `sort_unstable_by`, so which colliding entry survives can differ between EAVT and AEVT. - Reads through the two paths can return different values for the same entity. -2. **Query-time dedup.** `selective_fact_fetch` (`src/query/datalog/executor.rs`) dedups fetched - facts on `(entity, attribute, tx_count, asserted)`. That key has no value either, so even with - correct indexes it keeps only the first of the colliding facts. For a batched retract, the - second retraction never reaches `net_asserted_facts`, so its assertion is never cancelled. - -A full scan (no bound entity or attribute) reads the fact pages directly and returns both values, -which is why the paths disagree. - -## Scope - -In scope: - -- Add value bytes and `asserted` to `EavtKey` and `AevtKey`. -- Add `asserted` to `AvetKey` and `VaetKey`. Both already distinguish different values (AVET has - `value_bytes`, VAET has the ref target), but an assertion and a retraction of the same value at - the same `tx_count` share a key. This is inferred from the key definitions; a test written first - must demonstrate it before the change is applied. -- File format v7 → v8, with migration. v7 (the format written by v2.0.0) is the only older - format still supported. -- Dropping support for formats v1–v6, as the v3.0.0 file-format policy in `ROADMAP.md` requires. - Doing it here avoids adapting v5 migration code to the new key types only to delete it. -- Add value bytes to the `seen` key in `selective_fact_fetch`. - -Out of scope: - -- Version bump, tag, and release. -- The `seen` set in `executor.rs` `Or`-branch evaluation. It keys on bound values and is not - affected. - -## Design - -### Key layout - -The new fields are appended to the end of each key, so the existing sort prefix and range-scan -order are unchanged. - -| Key | Order (v8) | -|------|-----------------------------------------------------------------------------| -| EAVT | entity, attribute, valid_from, valid_to, tx_count, **value_bytes, asserted** | -| AEVT | attribute, entity, valid_from, valid_to, tx_count, **value_bytes, asserted** | -| AVET | attribute, value_bytes, valid_from, valid_to, entity, tx_count, **asserted** | -| VAET | ref_target, attribute, valid_from, valid_to, source_entity, tx_count, **asserted** | - -`value_bytes` is `encode_value(&fact.value)`, as AVET already uses. - -Range bounds (`get_facts_by_entity`, `get_facts_by_attribute`, `lookup_eavt_*`, `lookup_aevt_*`, -and B+tree range tests) use `value_bytes: vec![]` and `asserted: false` as the minimum. Existing -upper bounds are exclusive next-entity or next-attribute keys, so they need only the same minimum -values. `lookup_eavt_entity`, `lookup_eavt_entity_attr` and similar `Indexes` helpers build -upper bounds from `u64::MAX` sentinels (and one from a `"zzz…"` attribute). With `value_bytes` -appended, an inclusive `tx_count: u64::MAX` bound no longer covers every key, so these helpers -switch to the exclusive next-entity / next-attribute bound that `graph/storage.rs` already uses. -Several are `#[allow(dead_code)]`; if unused outside tests they are removed instead. - -Alternative rejected: placing `value_bytes` directly after the attribute, as AVET does. It changes -the sort order and hurts temporal range scans on entity and attribute. - -### Query-time dedup - -`selective_fact_fetch` `seen` key becomes -`(entity, attribute, tx_count, asserted, encode_value(&value))`. Both the entity-driven and -attribute-driven loops use it. - -### Format v8 and migration - -- `FORMAT_VERSION` becomes 8. The 84-byte header layout is unchanged; only `version` differs. -- Keys are postcard-encoded in B+tree pages. Postcard is not self-describing, so a v7 key - cannot be decoded as a v8 key. Every code path that decodes on-disk keys (`OnDiskIndexReader` - range scans, `stream_all_entries` in `save()`) must therefore never see a pre-v8 tree. -- On open, a v7 header forces the existing `needs_rebuild` path in `PersistentFactStorage::load`, - before the index reader is wired and before any `save()`. That path re-reads packed fact pages - with real `FactRef`s (`read_all_with_refs`), rebuilds all four B+trees with v8 keys, and writes - a v8 header (`FileHeader::new()` already uses `FORMAT_VERSION`). Fact pages are not modified. -- Crash safety: the rebuild overwrites index pages first and writes the header last. A crash in - between leaves a v7 header, so the next open rebuilds again. This rests on the - version check, not on the index checksum. -- `FileHeader::validate` still rejects versions above `FORMAT_VERSION` (`STG-006`), so v2.x opening - a v8 file fails cleanly. The upgrade is one-way and happens on first open; the CHANGELOG must say - so, since bindings users (e.g. temporal_reasoning) get the upgrade silently on open. -- WAL entries carry facts, not index keys, so a sidecar WAL on a v7 file replays unchanged after - the rebuild. -- The browser backend (`src/browser/mod.rs`, `import_graph`) uses the same - `PersistentFactStorage::load`, so it gets the migration with no separate code. - -### Dropping v1–v6 - -- `FileHeader::validate` rejects versions below 7 with a new error code (next free `STG-0xx`), - whose text names the version found and says the file must first be opened with Minigraf v2.x to - upgrade it to v7. Versions above 8 keep `STG-006`, whose text changes from - `supported: 1-{}` to `supported: 7-{}`. Both are added to `docs/ERROR_REFERENCE.md`. -- Removed: `src/storage/btree.rs` (v5 paged-blob indexes), `migrate_v1_to_v2`, - `migrate_v5_to_v6`, `load_one_per_page_legacy`, the `fact_page_format` one-per-page branch in - `load`, and the pre-v7 branches of header parsing and checksum handling (the `version >= 6`, - `version >= 7` and fact-pages-only checksum fallbacks). Their tests go with them. -- Kept: every v7 read path, since v7 is still opened (and migrated). -- All hand-built key literals (`build_sorted_index_entries`, `Indexes::insert`, - `graph/storage.rs` range bounds, tests) go through one constructor per key type - (`EavtKey::from_fact` etc.) so a future field change cannot miss a site. -- Planning must list every removed function and test, and confirm nothing else reaches the removed - code (fuzz targets, examples, benches, bindings). - -## Testing - -Regression tests are written first and shown failing before any fix: - -1. Issue reproduction: two values of one attribute in one transact; read through EAVT - (`[:e :attr ?v]`), AEVT (`[?e :attr ?v]`) and full scan (`[?e ?a ?v]`), in three states: - before checkpoint (pending `BTreeMap`s), after reopen without checkpoint (WAL replay), and after - checkpoint and reopen (on-disk B+tree). Run over several graph shapes, including filler entities - with two attributes each, which is what made EAVT and AEVT disagree in the issue. All paths - return both values in all states. -2. Batched retract of both values, then checkpoint and reopen: no path returns either value. -3. Same value asserted and retracted in one `WriteTransaction` with the same valid window: all - four indexes keep both facts. This test must fail on the current code first; if it cannot be - made to fail, `asserted` is not added to AVET/VAET and the spec is revised. -4. v7 → v8 migration, from real v7 files written by v2.0.0 code: - - The existing `tests/fixtures/compat.graph` is v7. Its native test - (`tests/cross_platform_compat_test.rs`) and browser test (`src/browser/mod.rs`) now cover - the migration path; add assertions that the header is v8 after open and that a second open - does not rebuild. - - Add a second fixture with colliding values, generated on `main` (v2.0.0) with the existing - `generate_compat_fixture` example pattern. After migration, both values read back through - all paths. The generation steps are recorded in the fixture's test doc comment. -5. Files with versions 1–6 fail to open with the new error code; files with version 9 fail with - `STG-006`. Neither is modified on disk. -6. Maximum-size value: a fact whose value is near `MAX_FACT_BYTES` checkpoints and reads back - through all four indexes. AVET already stores full value bytes today, so this is not a new - risk class, but EAVT/AEVT separators in internal nodes now carry values too. -7. Unit tests: key ordering, range-bound minima, per-key constructors, and `selective_fact_fetch` - dedup with two facts that differ only in value. -8. Update existing assertions on `FORMAT_VERSION == 7` and header version 7. - -Tests follow the project convention: no `{:?}` of `Result`/`Fact`/`Value` in assert messages. - -## Documentation and process - -- Update `CHANGELOG.md`, `ROADMAP.md` (move #287 work out of planned scope), `README.md` if it - cites the format version, `docs/TEST_COVERAGE.md`, `CLAUDE.md` (test count, format version), and - wiki `Architecture.md` (key layout, format v8). Wiki is committed and pushed separately. -- PR closes #371 and #287 only. No version bump or tag in this PR. -- Work in worktree `.worktrees/fix-371-index-value-collision`, branch - `fix/371-index-value-collision`. Own the PR until CI is green; do not merge without explicit - confirmation. - -## Risks - -- Larger EAVT/AEVT keys lower B+tree fanout and grow the file. Measure file size and run the - benchmark suite (`cargo bench`, not `cargo test --release`) against `main` before claiming no - regression, and record the numbers in the PR. -- Oversized entries: `build_btree` writes one entry per page when an entry exceeds the fill - threshold, but an entry larger than a page would overflow `write_leaf_page`. Facts are capped at - `MAX_FACT_BYTES` and AVET entries already carry full value bytes, so EAVT/AEVT entries have the - same bound as AVET today. Test 6 confirms this rather than assuming it. -- `encode_value` in the `selective_fact_fetch` dedup key allocates per fetched fact. Acceptable - for correctness; note it in the benchmark comparison. -- v8 files are unreadable by v2.x. This is the intended breaking change for v3.0.0. diff --git a/examples/generate_compat_fixture.rs b/examples/generate_compat_fixture.rs index bf1760e752..b8b0b098cc 100644 --- a/examples/generate_compat_fixture.rs +++ b/examples/generate_compat_fixture.rs @@ -1,14 +1,12 @@ -//! Generates a `.graph` file with two known facts, for inspecting what the -//! current storage layer writes. +//! Generates `tests/fixtures/compat.graph` — a minimal v7 `.graph` file containing +//! two known facts used by the cross-platform compatibility tests. //! +//! Run once when the file format changes or the fixture needs regenerating: //! cargo run --example generate_compat_fixture //! -//! `tests/fixtures/compat.graph` is the frozen v7 fixture written by Minigraf -//! v2.0.0; the cross-platform tests use it to check v7→v8 migration on open. -//! This example must never overwrite it — running it now would write a v8 -//! file, silently invalidating that migration coverage. It writes to -//! `target/generate_compat_fixture/compat_generated.graph` instead, which is -//! not read by any test. +//! The fixture is committed to the repository. Do not regenerate it unless the +//! v7 file format itself has changed — regenerating changes the binary and +//! every cross-platform test that embeds it via `include_bytes!`. // wasm-pack compiles examples for the browser target; provide a no-op entry // point so the example compiles cleanly. The actual generator only makes sense @@ -21,9 +19,7 @@ fn main() -> anyhow::Result<()> { use std::path::PathBuf; let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let out_dir = PathBuf::from(manifest_dir).join("target/generate_compat_fixture"); - std::fs::create_dir_all(&out_dir)?; - let fixture_path = out_dir.join("compat_generated.graph"); + let fixture_path = PathBuf::from(manifest_dir).join("tests/fixtures/compat.graph"); let tmp_path = fixture_path.with_extension("graph.tmp"); // Remove any leftover from a previous run. diff --git a/fuzz/fuzz_targets/btree_page.rs b/fuzz/fuzz_targets/btree_page.rs index 5e9fe30279..c93a26b03a 100644 --- a/fuzz/fuzz_targets/btree_page.rs +++ b/fuzz/fuzz_targets/btree_page.rs @@ -9,31 +9,11 @@ fuzz_target!(|data: &[u8]| { let path = dir.path().join("fuzz.graph"); let mut content = vec![0u8; 4096 * 3]; content[0..4].copy_from_slice(b"MGRF"); - // v8 header (v1-v6 are rejected; v7 would always force a full rebuild - // from fact pages and never touch the root page below, which is a - // different code path than the one this target means to fuzz). - content[4..8].copy_from_slice(&8u32.to_le_bytes()); - // page_count = 3 (header page 0, unused page 1, fuzzed root page 2). - content[8..16].copy_from_slice(&3u64.to_le_bytes()); - // eavt_root_page = 2: points `load()`'s index wiring, and the query - // below, at the fuzzed page as the EAVT B+tree root. fact_page_count is - // left at 0, so `load()` takes the "no facts, trust the root" branch - // and does not force a rebuild that would ignore this page. - content[32..40].copy_from_slice(&2u64.to_le_bytes()); - // header_checksum (bytes 80-83) and index_checksum (bytes 64-67) are left - // zero: zero is the "unset" sentinel that skips checksum verification. + content[4..8].copy_from_slice(&7u32.to_le_bytes()); let copy_len = data.len().min(4096); content[4096 * 2..4096 * 2 + copy_len].copy_from_slice(&data[..copy_len]); if std::fs::write(&path, &content).is_err() { return; } - let Ok(db) = minigraf::db::Minigraf::open(&path) else { - return; - }; - // `open()` alone only wires up `OnDiskIndexReader` against the root page — - // it never reads it. Run an entity-bound query so the EAVT range scan - // actually decodes the fuzzed page as a v8 B+tree leaf/internal node. - let _ = db.execute( - r#"(query [:find ?a ?v :where [#uuid "00000000-0000-0000-0000-000000000000" ?a ?v]])"#, - ); + let _ = minigraf::db::Minigraf::open(&path); }); diff --git a/fuzz/fuzz_targets/fact_page.rs b/fuzz/fuzz_targets/fact_page.rs index ee38ecfc02..c63d38219e 100644 --- a/fuzz/fuzz_targets/fact_page.rs +++ b/fuzz/fuzz_targets/fact_page.rs @@ -9,19 +9,7 @@ fuzz_target!(|data: &[u8]| { let path = dir.path().join("fuzz.graph"); let mut content = vec![0u8; 4096 * 2]; content[0..4].copy_from_slice(b"MGRF"); - // v8 header (v1-v6 are rejected; v7 would always force a full rebuild - // regardless of these fields, which is a different code path than the - // one this target means to fuzz). - content[4..8].copy_from_slice(&8u32.to_le_bytes()); - // page_count = 2 (header page 0 + one fact page at page 1). - content[8..16].copy_from_slice(&2u64.to_le_bytes()); - // fact_page_count = 1, eavt_root_page left at 0: with no index root, - // `load()` always rebuilds from the fact pages, so opening alone decodes - // page 1 as a packed fact page (postcard `Fact` deserialization) without - // needing a follow-up query. - content[72..80].copy_from_slice(&1u64.to_le_bytes()); - // header_checksum (bytes 80-83) and index_checksum (bytes 64-67) are left - // zero: zero is the "unset" sentinel that skips checksum verification. + content[4..8].copy_from_slice(&7u32.to_le_bytes()); let copy_len = data.len().min(4096); content[4096..4096 + copy_len].copy_from_slice(&data[..copy_len]); if std::fs::write(&path, &content).is_err() { diff --git a/src/browser/mod.rs b/src/browser/mod.rs index e8a065f423..75daf39f01 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -483,14 +483,6 @@ mod tests { "expected 1 age result from native fixture" ); assert_eq!(results2[0][0], serde_json::Value::Number(30.into())); - - let exported = db.export_graph().expect("export after import"); - let bytes = exported.to_vec(); - assert_eq!( - u32::from_le_bytes(bytes[4..8].try_into().unwrap()), - 8, - "v7 fixture must be upgraded to v8 on import" - ); } /// #275: `BrowserBufferBackend` is already an in-memory page store, so the diff --git a/src/error.rs b/src/error.rs index bf4fed8ab7..6c781ace89 100644 --- a/src/error.rs +++ b/src/error.rs @@ -138,12 +138,7 @@ pub(crate) enum ErrorCode { Int000, Stg001, Stg002, - // STG-003/STG-004 are deprecated since v3.0.0: they reported truncated - // v4–v6 headers, and formats older than v7 are now rejected with STG-028 - // before any size check. Kept so the codes are never reused. - #[allow(dead_code)] Stg003, - #[allow(dead_code)] Stg004, Stg005, Stg006, @@ -168,7 +163,6 @@ pub(crate) enum ErrorCode { Stg025, Stg026, Stg027, - Stg028, Wal001, Wal002, Wal003, @@ -807,13 +801,13 @@ pub(crate) const REGISTRY: &[(ErrorCode, &str, &str, ErrorCategory)] = &[ ( ErrorCode::Stg005, "STG-005", - "Invalid header: expected 84 bytes, got {}", + "Invalid v7 header: expected 84 bytes, got {}", ErrorCategory::Storage, ), ( ErrorCode::Stg006, "STG-006", - "Unsupported format version: {} (supported: 7-{})", + "Unsupported format version: {} (supported: 1-{})", ErrorCategory::Storage, ), ( @@ -942,12 +936,6 @@ pub(crate) const REGISTRY: &[(ErrorCode, &str, &str, ErrorCategory)] = &[ "Failed to lock database at {}: {}. This filesystem does not support file locking (common on NFSv3 without lockd, and on some FUSE mounts). Set `allow_unlocked` in `OpenOptions` to open anyway — that accepts the risk that concurrent writers corrupt the file.", ErrorCategory::Storage, ), - ( - ErrorCode::Stg028, - "STG-028", - "Format version {} is no longer supported (oldest supported: {}). Open the file once with Minigraf v2.x to upgrade it to format v7, then open it with this version.", - ErrorCategory::Storage, - ), ( ErrorCode::Wal001, "WAL-001", @@ -1756,7 +1744,6 @@ mod tests { | ErrorCode::Stg025 | ErrorCode::Stg026 | ErrorCode::Stg027 - | ErrorCode::Stg028 | ErrorCode::Wal001 | ErrorCode::Wal002 | ErrorCode::Wal003 @@ -1947,7 +1934,6 @@ mod tests { ErrorCode::Stg025, ErrorCode::Stg026, ErrorCode::Stg027, - ErrorCode::Stg028, ErrorCode::Wal001, ErrorCode::Wal002, ErrorCode::Wal003, diff --git a/src/graph/storage.rs b/src/graph/storage.rs index 6935515976..74c9405d78 100644 --- a/src/graph/storage.rs +++ b/src/graph/storage.rs @@ -621,9 +621,21 @@ impl FactStorage { use crate::storage::index::EavtKey; let d = self.data.read().unwrap_or_else(|e| e.into_inner()); - let start = EavtKey::entity_start(*entity_id); + let start = EavtKey { + entity: *entity_id, + attribute: String::new(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; let next_entity = uuid::Uuid::from_u128(entity_id.as_u128().wrapping_add(1)); - let end = EavtKey::entity_start(next_entity); + let end = EavtKey { + entity: next_entity, + attribute: String::new(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; // Fallback: no indexes built yet if d.pending_indexes.eavt.is_empty() && d.committed_index_reader.is_none() { @@ -687,9 +699,20 @@ impl FactStorage { .collect()); } - let start = AevtKey::attribute_start(attribute); - let end_opt: Option = - next_string_prefix(attribute).map(|next_attr| AevtKey::attribute_start(&next_attr)); + let start = AevtKey { + attribute: attribute.clone(), + entity: uuid::Uuid::nil(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + let end_opt: Option = next_string_prefix(attribute).map(|next_attr| AevtKey { + attribute: next_attr, + entity: uuid::Uuid::nil(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }); let mut facts = Vec::new(); diff --git a/src/query/datalog/executor.rs b/src/query/datalog/executor.rs index 6aa3d8b1a4..dfaa16aacc 100644 --- a/src/query/datalog/executor.rs +++ b/src/query/datalog/executor.rs @@ -373,23 +373,10 @@ impl DatalogExecutor { /// selective, but multi-pattern joins still need attribute candidates for patterns that do not /// bind an entity. If any pattern has neither a bound entity nor a bound attribute, or if the /// distinct lookup count exceeds `threshold`, returns `None` to use a full scan. Otherwise - /// returns `Some(facts)`, deduplicated by `(entity, attribute, tx_count, asserted, value)`. + /// returns `Some(facts)`, deduplicated by `(entity, attribute, tx_count, asserted)`. fn selective_fact_fetch(&self, patterns: &[Pattern], threshold: usize) -> Option> { use std::collections::HashSet; - // Dedup key: (entity, attribute, tx_count, asserted, encoded value, - // valid_from, valid_to). `asserted` keeps an assertion and a - // retraction from the same WriteTransaction apart; the encoded value - // keeps two values of one attribute written in the same transaction - // apart (#371). Uses `encode_value` bytes rather than `Value` to - // avoid Float hashing. `valid_from`/`valid_to` keep two distinct - // valid-time stints of the same (entity, attribute, value, tx_count, - // asserted) tuple apart — without them, two per-fact valid-time - // windows on the same value in one transaction collapse into a - // single arbitrary survivor and the other window's row is silently - // dropped from `:valid-at` and `:any-valid-time` results (#371). - type FactDedupKey = (uuid::Uuid, String, u64, bool, Vec, i64, i64); - let mut entity_ids: HashSet = HashSet::new(); let mut attributes: HashSet = HashSet::new(); @@ -417,7 +404,11 @@ impl DatalogExecutor { return None; } - let mut seen: HashSet = HashSet::new(); + // Dedup key: (entity uuid, attribute string, tx_count, asserted) — avoids Value debug + // formatting. Including `asserted` ensures that a retraction and an assertion committed in + // the same WriteTransaction (same tx_count) are both retained, since they differ only by + // the `asserted` flag. + let mut seen: HashSet<(uuid::Uuid, String, u64, bool)> = HashSet::new(); let mut all_facts: Vec = Vec::new(); for uid in &entity_ids { @@ -429,9 +420,6 @@ impl DatalogExecutor { fact.attribute.clone(), fact.tx_count, fact.asserted, - crate::storage::index::encode_value(&fact.value), - fact.valid_from, - fact.valid_to, ); if seen.insert(key) { all_facts.push(fact); @@ -451,9 +439,6 @@ impl DatalogExecutor { fact.attribute.clone(), fact.tx_count, fact.asserted, - crate::storage::index::encode_value(&fact.value), - fact.valid_from, - fact.valid_to, ); if seen.insert(key) { all_facts.push(fact); diff --git a/src/storage/btree.rs b/src/storage/btree.rs new file mode 100644 index 0000000000..9e4db54e1e --- /dev/null +++ b/src/storage/btree.rs @@ -0,0 +1,547 @@ +//! Legacy v5 B+tree serialisation — kept for v5→v7 file format migration only. +//! The write_* functions are no longer called in production; they exist +//! only to support tests that verify round-trip serialisation. +// Suppress dead-code warnings: this entire module is legacy v5 code; the write_* +// functions are only invoked by the unit tests in this file. +#![allow(dead_code)] +//! B+tree page serialisation for covering index persistence. +//! +//! `write_*_index` writes a `BTreeMap` to B+tree leaf pages starting +//! at `start_page_id` on the backend. Returns the root page ID. +//! `read_*_index` is the inverse — reads all entries back into a BTreeMap. +//! +//! # Page layout +//! +//! Because `StorageBackend::write_page` / `read_page` require exactly +//! `PAGE_SIZE` (4096) bytes, this module uses a simple paged-blob strategy: +//! +//! * Each index is serialised to a single postcard byte-string. +//! * That byte-string is split into chunks of `DATA_BYTES_PER_PAGE` bytes. +//! * Each chunk is stored in one backend page with the following header: +//! +//! ```text +//! Offset Size Field +//! 0 1 page_type (0x11 = index data page) +//! 1 4 total_byte_len (u32 LE, only meaningful in the first page) +//! 5 4 chunk_byte_len (u32 LE, number of valid bytes in this page) +//! 9 1 is_last_page (0x00 = more pages follow, 0x01 = last page) +//! 10 4082 data bytes (padded with zeros if chunk_byte_len < 4082) +//! ``` +//! +//! `total_byte_len` in page 0 tells the reader how many bytes to expect in +//! total so it can pre-allocate and know when it's done. +//! +//! Phase 6.1 uses these only for save/load. In-memory BTreeMaps handle queries. + +use crate::error::{ErrorCode, bail_coded, err_coded}; +use anyhow::Result; +use std::collections::BTreeMap; + +use crate::storage::index::{AevtKey, AvetKey, EavtKey, FactRef, VaetKey}; +use crate::storage::{PAGE_SIZE, StorageBackend}; + +// ─── Constants ─────────────────────────────────────────────────────────────── + +/// Maximum entries per leaf page (reserved for Phase 6.2 true B+tree layout). +#[allow(dead_code)] +pub const LEAF_CAPACITY: usize = 50; +/// Maximum children per internal page (reserved for Phase 6.2 true B+tree layout). +#[allow(dead_code)] +pub const INTERNAL_CAPACITY: usize = 50; + +/// Page type byte: index data page. +pub const PAGE_TYPE_INDEX: u8 = 0x11; + +/// Header bytes per page: type(1) + total_len(4) + chunk_len(4) + is_last(1) = 10. +const PAGE_HEADER_SIZE: usize = 10; + +/// Number of data bytes available per page. +const DATA_BYTES_PER_PAGE: usize = PAGE_SIZE - PAGE_HEADER_SIZE; + +// ─── Low-level page I/O ─────────────────────────────────────────────────────── + +fn write_blob(blob: &[u8], backend: &mut dyn StorageBackend, start_page_id: u64) -> Result { + if blob.len() > u32::MAX as usize { + bail_coded!( + ErrorCode::Int048, + format!( + "index blob too large to serialize: {} bytes (max {})", + blob.len(), + u32::MAX + ) + ); + } + // Safe: guarded by the > u32::MAX check above. + let total_len = u32::try_from(blob.len()).map_err(|_| { + err_coded!( + ErrorCode::Int048, + format!("blob.len() overflows u32 (len={})", blob.len()) + ) + })?; + let num_pages = if blob.is_empty() { + 1usize // always write at least one page + } else { + blob.len().div_ceil(DATA_BYTES_PER_PAGE) + }; + + for i in 0..num_pages { + let offset = i.checked_mul(DATA_BYTES_PER_PAGE).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + format!("btree write_blob: page offset overflow at i={i}") + ) + })?; + let chunk = if offset < blob.len() { + let end = blob + .len() + .min(offset.checked_add(DATA_BYTES_PER_PAGE).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + "btree write_blob: end offset overflow".to_string() + ) + })?); + blob.get(offset..end).ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("btree write_blob: slice {offset}..{end} out of bounds") + ) + })? + } else { + &[] + }; + let chunk_len = u32::try_from(chunk.len()).map_err(|_| { + err_coded!( + ErrorCode::Int048, + format!("chunk.len() overflows u32 (len={})", chunk.len()) + ) + })?; + // num_pages >= 1 is guaranteed: write_blob always writes at least one page. + let is_last: u8 = if i == num_pages - 1 { 0x01 } else { 0x00 }; + + let mut page = vec![0u8; PAGE_SIZE]; + *page.get_mut(0).ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "btree: page index 0 out of bounds".to_string() + ) + })? = PAGE_TYPE_INDEX; + page.get_mut(1..5) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "btree: page slice 1..5 out of bounds".to_string() + ) + })? + .copy_from_slice(&total_len.to_le_bytes()); + page.get_mut(5..9) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "btree: page slice 5..9 out of bounds".to_string() + ) + })? + .copy_from_slice(&chunk_len.to_le_bytes()); + *page.get_mut(9).ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "btree: page index 9 out of bounds".to_string() + ) + })? = is_last; + if !chunk.is_empty() { + let end = PAGE_HEADER_SIZE.checked_add(chunk.len()).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + "btree write_blob: data end overflow".to_string() + ) + })?; + page.get_mut(PAGE_HEADER_SIZE..end) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("btree: page slice {PAGE_HEADER_SIZE}..{end} out of bounds") + ) + })? + .copy_from_slice(chunk); + } + + let page_offset = u64::try_from(i).map_err(|_| { + err_coded!( + ErrorCode::Int048, + format!("btree write_blob: page index {i} overflows u64") + ) + })?; + backend.write_page( + start_page_id.checked_add(page_offset).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + "btree write_blob: page_id overflow".to_string() + ) + })?, + &page, + )?; + } + + // Return the page ID immediately after the last page written. + let num_pages_u64 = u64::try_from(num_pages).map_err(|_| { + err_coded!( + ErrorCode::Int048, + format!("btree write_blob: num_pages {num_pages} overflows u64") + ) + })?; + start_page_id.checked_add(num_pages_u64).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + "btree write_blob: final page_id overflow".to_string() + ) + }) +} + +fn read_blob(backend: &dyn StorageBackend, start_page_id: u64) -> Result> { + // Read the first page to get total_len. + let first_page = backend.read_page(start_page_id)?; + let first_byte = *first_page.first().ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "btree read_blob: first_page index 0 out of bounds".to_string() + ) + })?; + if first_byte != PAGE_TYPE_INDEX { + bail_coded!( + ErrorCode::Int049, + format!( + "Expected index page type 0x{:02x}, got 0x{:02x}", + PAGE_TYPE_INDEX, first_byte + ) + ); + } + + let total_len = u32::from_le_bytes( + first_page + .get(1..5) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "btree read_blob: first_page slice 1..5 out of bounds".to_string() + ) + })? + .try_into() + .map_err(|_| { + err_coded!( + ErrorCode::Int049, + "btree read_blob: slice 1..5 not exactly 4 bytes".to_string() + ) + })?, + ) as usize; + let mut blob = Vec::with_capacity(total_len); + + let mut page_id = start_page_id; + loop { + let page = backend.read_page(page_id)?; + let page_byte = *page.first().ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("btree read_blob: page index 0 out of bounds (page {page_id})") + ) + })?; + if page_byte != PAGE_TYPE_INDEX { + bail_coded!(ErrorCode::Stg012, page_id); + } + let chunk_len = u32::from_le_bytes( + page.get(5..9) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("btree read_blob: page slice 5..9 out of bounds (page {page_id})") + ) + })? + .try_into() + .map_err(|_| { + err_coded!( + ErrorCode::Int049, + "btree read_blob: slice 5..9 not exactly 4 bytes".to_string() + ) + })?, + ) as usize; + let is_last = *page.get(9).ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("btree read_blob: page index 9 out of bounds (page {page_id})") + ) + })? == 0x01; + + if chunk_len > 0 { + let end = PAGE_HEADER_SIZE.checked_add(chunk_len).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + format!("btree read_blob: data end overflow (chunk_len={chunk_len})") + ) + })?; + blob.extend_from_slice(page.get(PAGE_HEADER_SIZE..end).ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("btree read_blob: page slice {PAGE_HEADER_SIZE}..{end} out of bounds") + ) + })?); + } + + if is_last { + break; + } + page_id = page_id.checked_add(1).ok_or_else(|| { + err_coded!( + ErrorCode::Int048, + "btree read_blob: page_id overflow".to_string() + ) + })?; + } + + Ok(blob) +} + +// ─── Generic index write/read ───────────────────────────────────────────────── + +fn write_index_generic( + map: &BTreeMap, + backend: &mut dyn StorageBackend, + start_page_id: u64, +) -> Result +where + K: serde::Serialize + for<'de> serde::Deserialize<'de> + Ord + Clone, +{ + let entries: Vec<(&K, &FactRef)> = map.iter().collect(); + let blob = postcard::to_allocvec(&entries)?; + write_blob(&blob, backend, start_page_id) +} + +fn read_index_generic( + root_page_id: u64, + backend: &dyn StorageBackend, +) -> Result> +where + K: serde::Serialize + for<'de> serde::Deserialize<'de> + Ord + Clone, +{ + let blob = read_blob(backend, root_page_id)?; + let entries: Vec<(K, FactRef)> = postcard::from_bytes(&blob)?; + Ok(entries.into_iter().collect()) +} + +// ─── Public API ─────────────────────────────────────────────────────────────── + +/// Write the EAVT index to pages starting at `start_page_id`. +/// +/// Returns the next free page ID (i.e., `start_page_id + pages_written`). +pub fn write_eavt_index( + map: &BTreeMap, + backend: &mut dyn StorageBackend, + start_page_id: u64, +) -> Result { + write_index_generic(map, backend, start_page_id) +} + +/// Read the EAVT index from pages starting at `root_page_id`. +pub fn read_eavt_index( + root_page_id: u64, + backend: &dyn StorageBackend, +) -> Result> { + read_index_generic(root_page_id, backend) +} + +/// Write the AEVT index. Returns next free page ID. +pub fn write_aevt_index( + map: &BTreeMap, + backend: &mut dyn StorageBackend, + start_page_id: u64, +) -> Result { + write_index_generic(map, backend, start_page_id) +} + +/// Read the AEVT index. +pub fn read_aevt_index( + root_page_id: u64, + backend: &dyn StorageBackend, +) -> Result> { + read_index_generic(root_page_id, backend) +} + +/// Write the AVET index. Returns next free page ID. +pub fn write_avet_index( + map: &BTreeMap, + backend: &mut dyn StorageBackend, + start_page_id: u64, +) -> Result { + write_index_generic(map, backend, start_page_id) +} + +/// Read the AVET index. +pub fn read_avet_index( + root_page_id: u64, + backend: &dyn StorageBackend, +) -> Result> { + read_index_generic(root_page_id, backend) +} + +/// Write the VAET index. Returns next free page ID. +pub fn write_vaet_index( + map: &BTreeMap, + backend: &mut dyn StorageBackend, + start_page_id: u64, +) -> Result { + write_index_generic(map, backend, start_page_id) +} + +/// Read the VAET index. +pub fn read_vaet_index( + root_page_id: u64, + backend: &dyn StorageBackend, +) -> Result> { + read_index_generic(root_page_id, backend) +} + +/// Write all four indexes to pages starting at `start_page_id`. +/// +/// Returns `(eavt_root, aevt_root, avet_root, vaet_root)` — the start page ID +/// of each index blob. Each `*_root` value is the first page of that index's +/// blob; subsequent pages follow contiguously. +pub fn write_all_indexes( + eavt: &std::collections::BTreeMap, + aevt: &std::collections::BTreeMap, + avet: &std::collections::BTreeMap, + vaet: &std::collections::BTreeMap, + backend: &mut dyn StorageBackend, + start_page_id: u64, +) -> Result<(u64, u64, u64, u64)> { + let eavt_root = start_page_id; + let after_eavt = write_eavt_index(eavt, backend, eavt_root)?; + + let aevt_root = after_eavt; + let after_aevt = write_aevt_index(aevt, backend, aevt_root)?; + + let avet_root = after_aevt; + let after_avet = write_avet_index(avet, backend, avet_root)?; + + let vaet_root = after_avet; + write_vaet_index(vaet, backend, vaet_root)?; + + Ok((eavt_root, aevt_root, avet_root, vaet_root)) +} + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::backend::memory::MemoryBackend; + use crate::storage::index::{EavtKey, FactRef}; + use std::collections::BTreeMap; + use uuid::Uuid; + + fn eavt_entry(entity_lo: u128, attr: &str, tx: u64) -> (EavtKey, FactRef) { + ( + EavtKey { + entity: Uuid::from_u128(entity_lo), + attribute: attr.to_string(), + valid_from: 0, + valid_to: i64::MAX, + tx_count: tx, + }, + FactRef { + page_id: tx, + slot_index: 0, + }, + ) + } + + #[test] + fn test_empty_eavt_roundtrip() { + let mut backend = MemoryBackend::new(); + let map: BTreeMap = BTreeMap::new(); + let next = write_eavt_index(&map, &mut backend, 1).unwrap(); + // Empty serialization should still write exactly one page. + assert_eq!(next, 2, "empty index should consume exactly 1 page"); + let recovered = read_eavt_index(1, &backend).unwrap(); + assert_eq!(recovered.len(), 0); + } + + #[test] + fn test_small_eavt_roundtrip() { + let mut backend = MemoryBackend::new(); + let mut map = BTreeMap::new(); + for i in 0u128..10 { + let (k, v) = eavt_entry(i, ":name", i as u64 + 1); + map.insert(k, v); + } + let next = write_eavt_index(&map, &mut backend, 1).unwrap(); + assert!(next >= 2, "should write at least one page"); + let recovered = read_eavt_index(1, &backend).unwrap(); + assert_eq!(recovered.len(), 10); + for (k, v) in &map { + assert_eq!(recovered.get(k), Some(v), "missing key {:?}", k); + } + } + + #[test] + fn test_large_eavt_roundtrip_multi_leaf() { + // Force multi-page: insert more entries than LEAF_CAPACITY (50). + // With 150 entries × ~50 bytes each ≈ 7500 bytes > DATA_BYTES_PER_PAGE (4086), + // so this will span at least 2 pages. + let mut backend = MemoryBackend::new(); + let mut map = BTreeMap::new(); + for i in 0u128..150 { + let (k, v) = eavt_entry(i, ":attr", i as u64 + 1); + map.insert(k, v); + } + let next = write_eavt_index(&map, &mut backend, 1).unwrap(); + assert!(next > 2, "150 entries must span multiple pages"); + let recovered = read_eavt_index(1, &backend).unwrap(); + assert_eq!(recovered.len(), 150); + for (k, v) in &map { + assert_eq!(recovered.get(k), Some(v)); + } + } + + #[test] + fn test_eavt_preserves_sort_order() { + let mut backend = MemoryBackend::new(); + let mut map = BTreeMap::new(); + for i in 0u128..100 { + let (k, v) = eavt_entry(i, ":x", i as u64 + 1); + map.insert(k, v); + } + let root = write_eavt_index(&map, &mut backend, 1).unwrap(); + let recovered = read_eavt_index(1, &backend).unwrap(); + let orig_keys: Vec<_> = map.keys().collect(); + let rec_keys: Vec<_> = recovered.keys().collect(); + assert_eq!(orig_keys, rec_keys, "Sort order must be preserved"); + // Ensure the return value is sane. + assert!(root >= 2); + } + + // ══ #359: STG-012 regression test ═════════════════════════════════════ + + /// A non-first page of a multi-page index blob whose type byte has been + /// corrupted (page found is not `PAGE_TYPE_INDEX`) must surface as the + /// coded STG-012, not a generic error. + #[test] + fn read_blob_corrupted_non_first_page_returns_stg_012() { + let mut backend = MemoryBackend::new(); + let mut map = BTreeMap::new(); + // Enough entries to span at least 2 pages (see + // test_large_eavt_roundtrip_multi_leaf above). + for i in 0u128..150 { + let (k, v) = eavt_entry(i, ":attr", i as u64 + 1); + map.insert(k, v); + } + let next = write_eavt_index(&map, &mut backend, 1).unwrap(); + assert!(next > 2, "150 entries must span multiple pages"); + + // Corrupt page 2's type byte (the second page of the blob starting + // at page 1) so it no longer reads as PAGE_TYPE_INDEX. + let mut corrupt_page = backend.read_page(2).unwrap(); + corrupt_page[0] = 0x00; + backend.write_page(2, &corrupt_page).unwrap(); + + let err = read_eavt_index(1, &backend).expect_err("corrupted page must fail to read"); + let coded: crate::error::MinigrafError = err.into(); + assert_eq!(coded.code(), "STG-012"); + } +} diff --git a/src/storage/btree_v6.rs b/src/storage/btree_v6.rs index e9a6020b34..5c5a9b9e14 100644 --- a/src/storage/btree_v6.rs +++ b/src/storage/btree_v6.rs @@ -818,8 +818,6 @@ mod tests { valid_from: 0, valid_to: i64::MAX, tx_count: tx, - value_bytes: Vec::new(), - asserted: true, }, FactRef { page_id: tx + 1, @@ -1089,8 +1087,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let next_entity = Uuid::from_u128(43); let end = EavtKey { @@ -1099,8 +1095,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let refs = range_scan(root, &start, Some(&end), &backend, &cache).unwrap(); @@ -1131,8 +1125,6 @@ mod tests { valid_from: 0, valid_to: 0, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let refs = range_scan::(root, &start, None, &backend, &cache).unwrap(); assert_eq!(refs.len(), 0); @@ -1154,8 +1146,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let refs = range_scan::(root, &start, None, &backend, &cache).unwrap(); assert_eq!(refs.len(), 5, "entities 5..9 = 5 entries"); @@ -1177,8 +1167,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let end = EavtKey { entity: Uuid::from_u128(200), @@ -1186,8 +1174,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let refs = range_scan(root, &start, Some(&end), &backend, &cache).unwrap(); // NOTE: The end key has attribute="" which sorts BEFORE ":a". So entity 200's @@ -1257,8 +1243,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let err = range_scan::(root, &start, None, &backend, &cache) .expect_err("following a next_leaf pointer into a non-leaf page must fail"); @@ -1288,8 +1272,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let end = EavtKey { entity: Uuid::from_u128(10), @@ -1297,8 +1279,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let refs = reader.range_scan_eavt(&start, Some(&end)).unwrap(); // Same exclusion logic: entity 10's entry {10, ":x", ...} > end {10, "", ...} @@ -1340,8 +1320,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let end = EavtKey { entity: Uuid::from_u128(20), @@ -1349,8 +1327,6 @@ mod tests { valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - value_bytes: Vec::new(), - asserted: true, }; let barrier = Arc::new(Barrier::new(8)); diff --git a/src/storage/index.rs b/src/storage/index.rs index 9099c46605..8eb59ea89b 100644 --- a/src/storage/index.rs +++ b/src/storage/index.rs @@ -7,6 +7,19 @@ use crate::graph::types::{Attribute, EntityId, Fact, Value}; use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Generate the maximum UUID value (all bits set to 1). +/// Used as an upper bound for range queries on UUID fields. +fn max_uuid() -> Uuid { + // This is safe because MAX_UUID is a valid hardcoded UUID string literal. + // It will never fail at runtime. + const MAX_UUID_BYTES: [u8; 16] = [ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, + ]; + Uuid::from_bytes(MAX_UUID_BYTES) +} // ─── FactRef ──────────────────────────────────────────────────────────────── @@ -84,11 +97,7 @@ pub fn encode_value(v: &Value) -> Vec { // ─── Index Key Types ───────────────────────────────────────────────────────── -/// EAVT: sort by (Entity, Attribute, ValidFrom, ValidTo, TxCount, ValueBytes, Asserted) -/// -/// `value_bytes` and `asserted` come last so entity/attribute range scans keep -/// their order; they keep facts that differ only in value or assert/retract -/// from colliding (#371, #287). +/// EAVT: sort by (Entity, Attribute, ValidFrom, ValidTo, TxCount) #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub struct EavtKey { pub entity: EntityId, @@ -96,15 +105,9 @@ pub struct EavtKey { pub valid_from: i64, pub valid_to: i64, pub tx_count: u64, - pub value_bytes: Vec, - pub asserted: bool, } -/// AEVT: sort by (Attribute, Entity, ValidFrom, ValidTo, TxCount, ValueBytes, Asserted) -/// -/// `value_bytes` and `asserted` come last so entity/attribute range scans keep -/// their order; they keep facts that differ only in value or assert/retract -/// from colliding (#371, #287). +/// AEVT: sort by (Attribute, Entity, ValidFrom, ValidTo, TxCount) #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub struct AevtKey { pub attribute: Attribute, @@ -112,11 +115,9 @@ pub struct AevtKey { pub valid_from: i64, pub valid_to: i64, pub tx_count: u64, - pub value_bytes: Vec, - pub asserted: bool, } -/// AVET: sort by (Attribute, ValueBytes, ValidFrom, ValidTo, Entity, TxCount, Asserted) +/// AVET: sort by (Attribute, ValueBytes, ValidFrom, ValidTo, Entity, TxCount) /// /// `value_bytes` is the canonical encoding from `encode_value`. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -127,10 +128,9 @@ pub struct AvetKey { pub valid_to: i64, pub entity: EntityId, pub tx_count: u64, - pub asserted: bool, } -/// VAET: sort by (RefTarget, Attribute, ValidFrom, ValidTo, SourceEntity, TxCount, Asserted) +/// VAET: sort by (RefTarget, Attribute, ValidFrom, ValidTo, SourceEntity, TxCount) /// /// Only facts with `Value::Ref` are indexed here. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -141,93 +141,6 @@ pub struct VaetKey { pub valid_to: i64, pub source_entity: EntityId, pub tx_count: u64, - pub asserted: bool, -} - -impl EavtKey { - pub fn from_fact(f: &Fact) -> Self { - EavtKey { - entity: f.entity, - attribute: f.attribute.clone(), - valid_from: f.valid_from, - valid_to: f.valid_to, - tx_count: f.tx_count, - value_bytes: encode_value(&f.value), - asserted: f.asserted, - } - } - - /// Smallest possible key for `entity`: every EAVT key of that entity sorts at or after it. - pub fn entity_start(entity: EntityId) -> Self { - EavtKey { - entity, - attribute: String::new(), - valid_from: i64::MIN, - valid_to: i64::MIN, - tx_count: 0, - value_bytes: Vec::new(), - asserted: false, - } - } -} - -impl AevtKey { - pub fn from_fact(f: &Fact) -> Self { - AevtKey { - attribute: f.attribute.clone(), - entity: f.entity, - valid_from: f.valid_from, - valid_to: f.valid_to, - tx_count: f.tx_count, - value_bytes: encode_value(&f.value), - asserted: f.asserted, - } - } - - /// Smallest possible key for `attribute`. - pub fn attribute_start(attribute: &str) -> Self { - AevtKey { - attribute: attribute.to_string(), - entity: EntityId::nil(), - valid_from: i64::MIN, - valid_to: i64::MIN, - tx_count: 0, - value_bytes: Vec::new(), - asserted: false, - } - } -} - -impl AvetKey { - pub fn from_fact(f: &Fact) -> Self { - AvetKey { - attribute: f.attribute.clone(), - value_bytes: encode_value(&f.value), - valid_from: f.valid_from, - valid_to: f.valid_to, - entity: f.entity, - tx_count: f.tx_count, - asserted: f.asserted, - } - } -} - -impl VaetKey { - /// `None` unless the fact's value is a `Value::Ref` (only refs are VAET-indexed). - pub fn from_fact(f: &Fact) -> Option { - match &f.value { - Value::Ref(target) => Some(VaetKey { - ref_target: *target, - attribute: f.attribute.clone(), - valid_from: f.valid_from, - valid_to: f.valid_to, - source_entity: f.entity, - tx_count: f.tx_count, - asserted: f.asserted, - }), - _ => None, - } - } } // ─── Indexes ───────────────────────────────────────────────────────────────── @@ -254,67 +167,159 @@ impl Indexes { /// `FactRef { page_id: 0, slot_index: 0 }` as a placeholder; real /// page IDs are assigned by `save()` and updated via `reindex_from_facts`. pub fn insert(&mut self, fact: &Fact, fact_ref: FactRef) { - self.eavt.insert(EavtKey::from_fact(fact), fact_ref); - self.aevt.insert(AevtKey::from_fact(fact), fact_ref); - self.avet.insert(AvetKey::from_fact(fact), fact_ref); - if let Some(k) = VaetKey::from_fact(fact) { - self.vaet.insert(k, fact_ref); + self.eavt.insert( + EavtKey { + entity: fact.entity, + attribute: fact.attribute.clone(), + valid_from: fact.valid_from, + valid_to: fact.valid_to, + tx_count: fact.tx_count, + }, + fact_ref, + ); + + self.aevt.insert( + AevtKey { + attribute: fact.attribute.clone(), + entity: fact.entity, + valid_from: fact.valid_from, + valid_to: fact.valid_to, + tx_count: fact.tx_count, + }, + fact_ref, + ); + + self.avet.insert( + AvetKey { + attribute: fact.attribute.clone(), + value_bytes: encode_value(&fact.value), + valid_from: fact.valid_from, + valid_to: fact.valid_to, + entity: fact.entity, + tx_count: fact.tx_count, + }, + fact_ref, + ); + + if let Value::Ref(target) = &fact.value { + self.vaet.insert( + VaetKey { + ref_target: *target, + attribute: fact.attribute.clone(), + valid_from: fact.valid_from, + valid_to: fact.valid_to, + source_entity: fact.entity, + tx_count: fact.tx_count, + }, + fact_ref, + ); } } /// Query EAVT index for a specific entity (returns all facts for that entity). pub fn lookup_eavt_entity(&self, entity: EntityId) -> Vec { - self.eavt - .range(EavtKey::entity_start(entity)..) - .take_while(|(k, _)| k.entity == entity) - .map(|(_, v)| *v) - .collect() + let start = EavtKey { + entity, + attribute: String::new(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + // Use exclusive range with a very high attribute string + let end = EavtKey { + entity, + attribute: String::from("zzzzzzzzzzzzzzzzzz"), + valid_from: i64::MAX, + valid_to: i64::MAX, + tx_count: u64::MAX, + }; + self.eavt.range(start..end).map(|(_, v)| *v).collect() + } + + /// Query EAVT index for entity + attribute. + #[allow(dead_code)] + pub fn lookup_eavt_entity_attr(&self, entity: EntityId, attribute: &str) -> Vec { + let start = EavtKey { + entity, + attribute: attribute.to_string(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + let end = EavtKey { + entity, + attribute: attribute.to_string(), + valid_from: i64::MAX, + valid_to: i64::MAX, + tx_count: u64::MAX, + }; + self.eavt.range(start..=end).map(|(_, v)| *v).collect() } /// Query AEVT index for a specific attribute (returns all facts with that attribute). pub fn lookup_aevt_attr(&self, attribute: &str) -> Vec { - self.aevt - .range(AevtKey::attribute_start(attribute)..) - .take_while(|(k, _)| k.attribute == attribute) - .map(|(_, v)| *v) - .collect() + let max_uuid = max_uuid(); + let start = AevtKey { + attribute: attribute.to_string(), + entity: EntityId::default(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + let end = AevtKey { + attribute: attribute.to_string(), + entity: max_uuid, + valid_from: i64::MAX, + valid_to: i64::MAX, + tx_count: u64::MAX, + }; + self.aevt.range(start..=end).map(|(_, v)| *v).collect() } /// Query AVET index for attribute + value. pub fn lookup_avet_attr_value(&self, attribute: &str, value: &Value) -> Vec { + let max_uuid = max_uuid(); let value_bytes = encode_value(value); let start = AvetKey { attribute: attribute.to_string(), value_bytes: value_bytes.clone(), valid_from: i64::MIN, valid_to: i64::MIN, - entity: EntityId::nil(), + entity: EntityId::default(), tx_count: 0, - asserted: false, }; - self.avet - .range(start..) - .take_while(|(k, _)| k.attribute == attribute && k.value_bytes == value_bytes) - .map(|(_, v)| *v) - .collect() + let end = AvetKey { + attribute: attribute.to_string(), + value_bytes, + valid_from: i64::MAX, + valid_to: i64::MAX, + entity: max_uuid, + tx_count: u64::MAX, + }; + self.avet.range(start..=end).map(|(_, v)| *v).collect() } /// Query VAET index for ref target (reverse references). pub fn lookup_vaet_ref(&self, target: EntityId) -> Vec { + let max_uuid = max_uuid(); let start = VaetKey { ref_target: target, attribute: String::new(), valid_from: i64::MIN, valid_to: i64::MIN, - source_entity: EntityId::nil(), + source_entity: EntityId::default(), tx_count: 0, - asserted: false, }; - self.vaet - .range(start..) - .take_while(|(k, _)| k.ref_target == target) - .map(|(_, v)| *v) - .collect() + let end = VaetKey { + ref_target: target, + // Use max char to include all attributes + attribute: char::MAX.to_string(), + valid_from: i64::MAX, + valid_to: i64::MAX, + source_entity: max_uuid, + tx_count: u64::MAX, + }; + self.vaet.range(start..=end).map(|(_, v)| *v).collect() } } @@ -324,126 +329,6 @@ mod tests { use crate::graph::types::{Fact, VALID_TIME_FOREVER, Value}; use uuid::Uuid; - fn fact_at(entity: Uuid, attr: &str, value: Value, asserted: bool) -> Fact { - let mut f = Fact::with_valid_time( - entity, - attr.to_string(), - value, - 100, - 7, - 100, - VALID_TIME_FOREVER, - ); - f.asserted = asserted; - f - } - - /// #371: two values of one attribute in one transaction must both be indexed. - #[test] - fn same_tx_multi_value_keeps_both_entries_in_all_indexes() { - let e = Uuid::from_u128(1); - let t1 = Uuid::from_u128(10); - let t2 = Uuid::from_u128(11); - let mut idx = Indexes::new(); - idx.insert( - &fact_at(e, ":kind", Value::Ref(t1), true), - FactRef { - page_id: 0, - slot_index: 0, - }, - ); - idx.insert( - &fact_at(e, ":kind", Value::Ref(t2), true), - FactRef { - page_id: 0, - slot_index: 1, - }, - ); - assert_eq!( - idx.lookup_eavt_entity(e).len(), - 2, - "EAVT must keep both values" - ); - assert_eq!( - idx.lookup_aevt_attr(":kind").len(), - 2, - "AEVT must keep both values" - ); - assert_eq!(idx.eavt.len(), 2); - assert_eq!(idx.aevt.len(), 2); - assert_eq!(idx.avet.len(), 2); - assert_eq!(idx.vaet.len(), 2); - } - - /// An assertion and a retraction of the same value at the same tx_count - /// must not collide in any index. - #[test] - fn same_tx_assert_and_retract_keep_both_entries_in_all_indexes() { - let e = Uuid::from_u128(1); - let t = Uuid::from_u128(10); - let mut idx = Indexes::new(); - idx.insert( - &fact_at(e, ":kind", Value::Ref(t), true), - FactRef { - page_id: 0, - slot_index: 0, - }, - ); - idx.insert( - &fact_at(e, ":kind", Value::Ref(t), false), - FactRef { - page_id: 0, - slot_index: 1, - }, - ); - assert_eq!(idx.eavt.len(), 2, "EAVT must keep assert and retract"); - assert_eq!(idx.aevt.len(), 2, "AEVT must keep assert and retract"); - assert_eq!(idx.avet.len(), 2, "AVET must keep assert and retract"); - assert_eq!(idx.vaet.len(), 2, "VAET must keep assert and retract"); - } - - #[test] - fn lookups_do_not_leak_neighbouring_entities_or_attributes() { - let (a, b) = (Uuid::from_u128(1), Uuid::from_u128(2)); - let mut idx = Indexes::new(); - idx.insert( - &fact_at(a, ":x", Value::Integer(1), true), - FactRef { - page_id: 0, - slot_index: 0, - }, - ); - idx.insert( - &fact_at(b, ":x", Value::Integer(1), true), - FactRef { - page_id: 0, - slot_index: 1, - }, - ); - idx.insert( - &fact_at(a, ":xy", Value::Integer(1), true), - FactRef { - page_id: 0, - slot_index: 2, - }, - ); - assert_eq!(idx.lookup_eavt_entity(a).len(), 2); - assert_eq!(idx.lookup_aevt_attr(":x").len(), 2); - assert_eq!( - idx.lookup_avet_attr_value(":x", &Value::Integer(1)).len(), - 2 - ); - } - - #[test] - fn entity_and_attribute_start_keys_sort_before_every_matching_key() { - let e = Uuid::from_u128(5); - let f = fact_at(e, ":a", Value::Null, false); - assert!(EavtKey::entity_start(e) <= EavtKey::from_fact(&f)); - assert!(AevtKey::attribute_start(":a") <= AevtKey::from_fact(&f)); - assert!(EavtKey::entity_start(Uuid::from_u128(6)) > EavtKey::from_fact(&f)); - } - #[test] fn test_fact_ref_fields() { let r = FactRef { @@ -497,8 +382,6 @@ mod tests { valid_from: 0, valid_to: i64::MAX, tx_count: 1, - value_bytes: Vec::new(), - asserted: true, }; let k2 = EavtKey { entity: e2, @@ -506,8 +389,6 @@ mod tests { valid_from: 0, valid_to: i64::MAX, tx_count: 1, - value_bytes: Vec::new(), - asserted: true, }; assert!(k1 < k2); } @@ -522,7 +403,6 @@ mod tests { valid_to: i64::MAX, entity: e, tx_count: 1, - asserted: true, }; let k2 = AvetKey { attribute: ":score".to_string(), @@ -531,7 +411,6 @@ mod tests { valid_to: i64::MAX, entity: e, tx_count: 2, - asserted: true, }; assert!(k1 < k2); } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 14a28b8de1..3145504265 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -6,6 +6,7 @@ /// /// Inspired by SQLite's VFS (Virtual File System) architecture. pub mod backend; +pub mod btree; pub mod btree_v6; pub mod cache; pub mod index; @@ -44,12 +45,10 @@ pub const PAGE_SIZE: usize = 4096; pub const MAGIC_NUMBER: [u8; 4] = *b"MGRF"; /// Current file format version -pub const FORMAT_VERSION: u32 = 8; - -/// Oldest file format version this library reads. Older files must first be -/// opened with Minigraf v2.x, which upgrades them to v7 (see STG-028). -pub const MIN_FORMAT_VERSION: u32 = 7; +pub const FORMAT_VERSION: u32 = 7; +/// fact_page_format: legacy one-per-page (v4 and earlier, or unset byte = 0x00). +pub const FACT_PAGE_FORMAT_ONE_PER_PAGE: u8 = 0x01; /// fact_page_format: packed pages (v5+). pub const FACT_PAGE_FORMAT_PACKED: u8 = 0x02; @@ -96,7 +95,7 @@ pub trait StorageBackend: Send + Sync { fn is_new(&self) -> bool; } -/// File header for .graph files — 84 bytes (v7 and v8 share the layout). +/// File header for .graph files — 84 bytes in v7. /// /// Layout (all fields little-endian): /// 0..4 magic ("MGRF") @@ -125,14 +124,14 @@ pub struct FileHeader { pub avet_root_page: u64, pub vaet_root_page: u64, pub index_checksum: u32, - /// fact_page_format: only 0x02 (packed, `FACT_PAGE_FORMAT_PACKED`) is - /// written or read; v1-v6 files (which could carry 0x00 unset/legacy or - /// 0x01 one-per-page) are rejected outright (STG-028), not migrated. + /// fact_page_format (v5+): 0x00 = unset/legacy, 0x01 = one-per-page, 0x02 = packed. pub fact_page_format: u8, pub(crate) _padding: [u8; 3], /// Number of pages (starting at page 1) holding committed fact data. + /// New in v6; zero-initialised when reading v5 or older headers. pub fact_page_count: u64, /// CRC32 checksum of the first 80 bytes of the header (excluding this field). + /// New in v7; zero-initialised when reading v6 or older headers. pub header_checksum: u32, } @@ -179,8 +178,14 @@ impl FileHeader { /// Deserialize the header from bytes. /// - /// Accepts only v7+ (84-byte) headers; older versions fail with STG-028. + /// Accepts v3 (64-byte), v4/v5 (72-byte), and v6 (80-byte) headers. + /// v3 headers are returned with zero-filled index fields; the + /// v3→v4 migration in persistent_facts.rs upgrades them on next save. + /// v4 headers have fact_page_format = 0x00 (legacy/unset). + /// v5 headers are returned with fact_page_count = 0. pub fn from_bytes(bytes: &[u8]) -> Result { + // Both v3 (64 bytes) and v4/v5/v6 (72+ bytes) must pass at least 64-byte + // validation before the version field is read. if bytes.len() < 64 { bail_coded!(ErrorCode::Stg001, bytes.len()); } @@ -197,17 +202,53 @@ impl FileHeader { } let version = read_u32_le(bytes, 4)?; - if version < MIN_FORMAT_VERSION { - bail_coded!(ErrorCode::Stg028, version, MIN_FORMAT_VERSION); - } - if bytes.len() < 84 { - bail_coded!(ErrorCode::Stg005, bytes.len()); - } let page_count = read_u64_le(bytes, 8)?; let node_count = read_u64_le(bytes, 16)?; let last_checkpointed_tx_count = read_u64_le(bytes, 24)?; - let fact_page_count = read_u64_le(bytes, 72)?; - let header_checksum = read_u32_le(bytes, 80)?; + + // v3 and earlier: no index fields — return with zero-filled index fields. + // The v3→v4 migration in persistent_facts.rs will upgrade on next save. + if version <= 3 { + return Ok(FileHeader { + magic, + version, + page_count, + node_count, + last_checkpointed_tx_count, + eavt_root_page: 0, + aevt_root_page: 0, + avet_root_page: 0, + vaet_root_page: 0, + index_checksum: 0, + fact_page_format: 0, + _padding: [0; 3], + fact_page_count: 0, + header_checksum: 0, + }); + } + + // v4, v5, v6: need at least 72 bytes + if bytes.len() < 72 { + bail_coded!(ErrorCode::Stg003, bytes.len()); + } + + let fact_page_count = if version >= 6 { + if bytes.len() < 80 { + bail_coded!(ErrorCode::Stg004, bytes.len()); + } + read_u64_le(bytes, 72)? + } else { + 0 + }; + + let header_checksum = if version >= 7 { + if bytes.len() < 84 { + bail_coded!(ErrorCode::Stg005, bytes.len()); + } + read_u32_le(bytes, 80)? + } else { + 0 + }; Ok(FileHeader { magic, @@ -244,10 +285,7 @@ impl FileHeader { if self.magic != MAGIC_NUMBER { bail_coded!(ErrorCode::Int039); } - if self.version < MIN_FORMAT_VERSION { - bail_coded!(ErrorCode::Stg028, self.version, MIN_FORMAT_VERSION); - } - if self.version > FORMAT_VERSION { + if self.version < 1 || self.version > FORMAT_VERSION { bail_coded!(ErrorCode::Stg006, self.version, FORMAT_VERSION); } // Validate logical relationships @@ -332,6 +370,20 @@ pub trait CommittedIndexReader: Send + Sync { mod tests { use super::*; + #[test] + fn test_file_header_from_bytes_v3_accepted() { + // v3 files have a 64-byte header. from_bytes must accept them and + // return zeroed index fields. + let mut bytes = vec![0u8; 64]; + bytes[0..4].copy_from_slice(b"MGRF"); + bytes[4..8].copy_from_slice(&3u32.to_le_bytes()); // version = 3 + bytes[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 + let header = FileHeader::from_bytes(&bytes).unwrap(); + assert_eq!(header.version, 3); + assert_eq!(header.eavt_root_page, 0); + assert_eq!(header.index_checksum, 0); + } + #[test] fn test_file_header_validation() { let header = FileHeader::new(); @@ -343,8 +395,8 @@ mod tests { } #[test] - fn test_format_version_is_8() { - assert_eq!(FORMAT_VERSION, 8); + fn test_format_version_is_7() { + assert_eq!(FORMAT_VERSION, 7); } #[test] @@ -354,13 +406,6 @@ mod tests { assert!(h.validate().is_ok()); } - #[test] - fn test_validate_accepts_version_8() { - let mut h = FileHeader::new(); - h.version = 8; - assert!(h.validate().is_ok()); - } - #[test] fn test_validate_page_count_must_be_positive() { let mut h = FileHeader::new(); @@ -403,10 +448,10 @@ mod tests { } #[test] - fn test_new_header_has_version_8() { + fn test_new_header_has_version_7() { let header = FileHeader::new(); assert_eq!(header.version, FORMAT_VERSION); - assert_eq!(header.version, 8); + assert_eq!(header.version, 7); } #[test] @@ -445,7 +490,7 @@ mod tests { assert_eq!(b.len(), 84, "v7 header must be exactly 84 bytes"); assert_eq!(&b[0..4], b"MGRF"); - assert_eq!(&b[4..8], &FORMAT_VERSION.to_le_bytes()); + assert_eq!(&b[4..8], &7u32.to_le_bytes()); assert_eq!(&b[8..16], &0x0102_0304_0506_0708_u64.to_le_bytes()); assert_eq!(&b[16..24], &0x1112_1314_1516_1718_u64.to_le_bytes()); assert_eq!(&b[24..32], &0x2122_2324_2526_2728_u64.to_le_bytes()); @@ -460,6 +505,17 @@ mod tests { assert_eq!(&b[80..84], &0xC1C2_C3C4_u32.to_le_bytes()); } + #[test] + fn test_file_header_v6_reads_header_checksum_zero() { + let mut bytes = vec![0u8; 80]; + bytes[0..4].copy_from_slice(b"MGRF"); + bytes[4..8].copy_from_slice(&6u32.to_le_bytes()); + bytes[8..16].copy_from_slice(&2u64.to_le_bytes()); + let h = FileHeader::from_bytes(&bytes).unwrap(); + assert_eq!(h.version, 6); + assert_eq!(h.header_checksum, 0); + } + #[test] fn test_file_header_v7_truncated_rejected() { let mut bytes = vec![0u8; 80]; @@ -468,6 +524,15 @@ mod tests { assert!(FileHeader::from_bytes(&bytes).is_err()); } + #[test] + fn test_validate_accepts_versions_1_to_7() { + let mut h = FileHeader::new(); + for v in 1u32..=7 { + h.version = v; + assert!(h.validate().is_ok(), "version {} should be accepted", v); + } + } + #[test] fn test_file_header_v7_header_checksum_roundtrip() { let mut h = FileHeader::new(); @@ -478,11 +543,48 @@ mod tests { assert_eq!(parsed.header_checksum, 42); } + #[test] + fn test_file_header_from_bytes_truncated_v4_rejected() { + // A header that claims version=4 but has fewer than 72 bytes must be rejected. + let mut bytes = vec![0u8; 68]; // only 68 bytes, not 72 + bytes[0..4].copy_from_slice(b"MGRF"); + bytes[4..8].copy_from_slice(&4u32.to_le_bytes()); // version = 4 + let result = FileHeader::from_bytes(&bytes); + assert!(result.is_err(), "truncated v4 header must be rejected"); + } + + #[test] + fn test_file_header_v5_fact_page_format_roundtrip() { + let mut h = FileHeader::new(); + h.fact_page_format = FACT_PAGE_FORMAT_PACKED; + let bytes = h.to_bytes(); + let parsed = FileHeader::from_bytes(&bytes).unwrap(); + assert_eq!(parsed.fact_page_format, FACT_PAGE_FORMAT_PACKED); + } + + #[test] + fn test_v4_header_reads_fact_page_format_zero() { + // v4 header has _padding = 0, so fact_page_format must come back as 0 + let mut bytes = vec![0u8; 72]; + bytes[0..4].copy_from_slice(b"MGRF"); + bytes[4..8].copy_from_slice(&4u32.to_le_bytes()); // version = 4 + bytes[8..16].copy_from_slice(&2u64.to_le_bytes()); // page_count = 2 + let h = FileHeader::from_bytes(&bytes).unwrap(); + assert_eq!(h.fact_page_format, 0); + } + + #[test] + fn test_validate_accepts_version_5() { + let mut h = FileHeader::new(); + h.version = 5; + assert!(h.validate().is_ok()); + } + // ══ #359: STG-0xx regression tests ═══════════════════════════════════ // - // STG-001/005 guard byte lengths (<64/<84) that the production - // `FileBackend` path never actually produces -- it always hands - // `FileHeader::from_bytes` a full `PAGE_SIZE` (4096-byte) buffer, + // STG-001/003/004/005 guard byte lengths (<64/<72/<80/<84) that the + // production `FileBackend` path never actually produces -- it always + // hands `FileHeader::from_bytes` a full `PAGE_SIZE` (4096-byte) buffer, // whatever the real file's length beyond the minimum one page. These // codes are only reachable by calling `FileHeader::from_bytes` directly // with a short slice, which `storage` being crate-private makes @@ -498,32 +600,32 @@ mod tests { } #[test] - fn from_bytes_v7_too_short_returns_stg_005() { - let mut bytes = vec![0u8; 82]; // >= 80, < 84 + fn from_bytes_v4_too_short_returns_stg_003() { + let mut bytes = vec![0u8; 70]; // >= 64, < 72 bytes[0..4].copy_from_slice(b"MGRF"); - bytes[4..8].copy_from_slice(&7u32.to_le_bytes()); // version = 7 - let err = FileHeader::from_bytes(&bytes).expect_err("82 bytes is too short for v7"); + bytes[4..8].copy_from_slice(&4u32.to_le_bytes()); // version = 4 + let err = FileHeader::from_bytes(&bytes).expect_err("70 bytes is too short for v4"); let coded: crate::error::MinigrafError = err.into(); - assert_eq!(coded.code(), "STG-005"); + assert_eq!(coded.code(), "STG-003"); } #[test] - fn from_bytes_pre_v7_returns_stg_028() { - for version in 1u32..=6 { - let mut bytes = FileHeader::new().to_bytes(); - bytes[4..8].copy_from_slice(&version.to_le_bytes()); - let err = FileHeader::from_bytes(&bytes).expect_err("pre-v7 header must be rejected"); - let coded: crate::error::MinigrafError = err.into(); - assert_eq!(coded.code(), "STG-028"); - } + fn from_bytes_v6_too_short_returns_stg_004() { + let mut bytes = vec![0u8; 75]; // >= 72, < 80 + bytes[0..4].copy_from_slice(b"MGRF"); + bytes[4..8].copy_from_slice(&6u32.to_le_bytes()); // version = 6 + let err = FileHeader::from_bytes(&bytes).expect_err("75 bytes is too short for v6"); + let coded: crate::error::MinigrafError = err.into(); + assert_eq!(coded.code(), "STG-004"); } #[test] - fn validate_pre_v7_returns_stg_028() { - let mut h = FileHeader::new(); - h.version = 6; - let err = h.validate().expect_err("v6 must be rejected"); + fn from_bytes_v7_too_short_returns_stg_005() { + let mut bytes = vec![0u8; 82]; // >= 80, < 84 + bytes[0..4].copy_from_slice(b"MGRF"); + bytes[4..8].copy_from_slice(&7u32.to_le_bytes()); // version = 7 + let err = FileHeader::from_bytes(&bytes).expect_err("82 bytes is too short for v7"); let coded: crate::error::MinigrafError = err.into(); - assert_eq!(coded.code(), "STG-028"); + assert_eq!(coded.code(), "STG-005"); } } diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index 8afcbe53a0..2a057513f4 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -6,14 +6,15 @@ use crate::graph::FactStorage; /// low-level page-based storage backends. use crate::graph::types::Fact; use crate::storage::FACT_PAGE_FORMAT_PACKED; +use crate::storage::btree::{read_aevt_index, read_avet_index, read_eavt_index, read_vaet_index}; use crate::storage::btree_v6::{ MutexStorageBackend, OnDiskIndexReader, btree_entries, build_btree, merge_sorted_vecs, stream_all_entries, }; use crate::storage::cache::PageCache; -use crate::storage::index::{AevtKey, AvetKey, EavtKey, FactRef, VaetKey}; +use crate::storage::index::{AevtKey, AvetKey, EavtKey, FactRef, VaetKey, encode_value}; use crate::storage::packed_pages::pack_facts; -use crate::storage::{FORMAT_VERSION, FileHeader, PAGE_SIZE, StorageBackend}; +use crate::storage::{FileHeader, PAGE_SIZE, StorageBackend}; use anyhow::Result; use crc32fast::Hasher; use std::sync::atomic::{AtomicU64, Ordering}; @@ -93,6 +94,18 @@ impl crate::storage::CommittedFactReader } } +/// V1 fact format (Phase 3, before bi-temporal fields were added). +/// +/// Used only during migration from v1 → v2 file format. +#[derive(Debug, serde::Deserialize)] +struct FactV1 { + entity: crate::graph::types::EntityId, + attribute: crate::graph::types::Attribute, + value: crate::graph::types::Value, + tx_id: crate::graph::types::TxId, + asserted: bool, +} + /// Persistent fact storage with serialization support. /// /// Architecture: @@ -164,8 +177,7 @@ impl PersistentFactStorage { // The load condition combines two checks: // - `!is_new`: FileBackend reports false when the file existed on disk // (even with only a header page, page_count == 1). This ensures - // `load()` runs (and its version check rejects pre-v7 files) even - // for a file that has no fact pages. + // migrate_v5_to_v6 runs for a v5 file that has no fact pages. // - `page_count > 1`: catches MemoryBackend, which always reports // is_new == true; page count > 1 means facts were previously saved. let (is_new_backend, page_count) = { @@ -204,8 +216,19 @@ impl PersistentFactStorage { (h, header_page) }; + // Migrate v1 → v2 if needed + if header.version < 2 { + return self.migrate_v1_to_v2(); + } + + // Migrate v5 → v6 (paged-blob indexes → on-disk B+tree) + if header.version == 5 { + return self.migrate_v5_to_v6(&header); + } + // For v7+ files, validate header checksum using raw bytes from disk - if header.header_checksum != 0 { + // For older versions (v6 and earlier), header_checksum is 0 so validation is skipped + if header.version >= 7 && header.header_checksum != 0 { let computed = compute_header_checksum_from_bytes(&raw_header_bytes); if header.header_checksum != computed { bail_coded!(ErrorCode::Int053); @@ -218,8 +241,21 @@ impl PersistentFactStorage { // Clear existing storage self.storage.clear()?; + let fact_page_format = header.fact_page_format; + + if fact_page_format == 0 + || fact_page_format == crate::storage::FACT_PAGE_FORMAT_ONE_PER_PAGE + { + // Legacy one-per-page format (v4 or earlier): load all facts, then migrate to v5. + self.load_one_per_page_legacy(&header)?; + self.storage.restore_tx_counter()?; + self.dirty = true; + self.save()?; + return Ok(()); + } + // v6 packed format - let num_fact_pages = if header.fact_page_count > 0 { + let num_fact_pages = if header.version >= 6 && header.fact_page_count > 0 { header.fact_page_count } else { let first_index_page = [ @@ -238,21 +274,32 @@ impl PersistentFactStorage { self.committed_fact_pages .store(num_fact_pages, Ordering::SeqCst); - // v7 files carry index keys without value bytes / asserted flags; they - // cannot be decoded as v8 keys, so rebuild every index from the fact - // pages (#371, #287). The header is written last, so a crash mid-rebuild - // leaves a v7 header and the next open rebuilds again. - let needs_rebuild = if header.version < FORMAT_VERSION { - true - } else if num_fact_pages == 0 || header.eavt_root_page == 0 { + // Compute page-based checksum to verify data integrity. + // New files (post-fix): checksum covers ALL pages (facts + indexes). + // Old files (pre-fix): checksum covers only fact pages. + // Try full checksum first; fall back to fact-only for backwards compat. + let needs_rebuild = if num_fact_pages == 0 || header.eavt_root_page == 0 { num_fact_pages > 0 // rebuild if facts exist but no index root } else { let backend = self .backend .lock() .map_err(|_| err_coded!(ErrorCode::Stg016))?; + let stored = header.index_checksum; + // Total data pages: pages 1 through page_count-1 (everything except header) let total_data_pages = header.page_count.saturating_sub(1); - compute_page_checksum(&*backend, 1, total_data_pages)? != header.index_checksum + let full_checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; + if full_checksum == stored { + false // new-style checksum matches: facts + indexes verified + } else { + // Fall back: old files stored checksum over fact pages only + let fact_checksum = compute_page_checksum(&*backend, 1, num_fact_pages)?; + if fact_checksum == stored { + false // old-style checksum matches: facts verified, indexes unprotected + } else { + true // neither matches: corruption detected, rebuild + } + } }; // Register CommittedFactReader on FactStorage (before WAL replay) @@ -287,15 +334,8 @@ impl PersistentFactStorage { let (eavt_entries, aevt_entries, avet_entries, vaet_entries) = build_sorted_index_entries(&all_facts, &real_refs); - // Fix up tx_counter from actual facts. Empty transactions - // (`(transact [])`, `(retract [])`) still allocate a tx_count - // without producing any facts, so the highest fact tx_count can - // undercount the counter the file was actually at; floor it at - // the old header's last_checkpointed_tx_count so migration never - // rewinds the counter and lets new transactions reuse tx_counts - // that `:as-of N` already served facts for (#371, #287). - let max_fact_tx = all_facts.iter().map(|f| f.tx_count).max().unwrap_or(0); - let max_tx = max_fact_tx.max(header.last_checkpointed_tx_count); + // Fix up tx_counter from actual facts + let max_tx = all_facts.iter().map(|f| f.tx_count).max().unwrap_or(0); self.storage.restore_tx_counter_from(max_tx); // Build v6 B+tree indexes directly @@ -331,12 +371,6 @@ impl PersistentFactStorage { next3, )?; - // Sync index pages to disk before writing the header. - // The header update is the atomic commit point: once it's durable, - // recovery uses the new root pages. All data those roots reference - // must already be on stable storage. - backend.sync()?; - // Write header with full-coverage checksum (facts + indexes) let total_data_pages = next4.saturating_sub(1); let full_checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; @@ -378,7 +412,7 @@ impl PersistentFactStorage { } else { // No rebuild needed - validate header checksum for v7+ files // Re-read header from disk to get any updates from rebuild path - if header.header_checksum != 0 { + if header.version >= 7 && header.header_checksum != 0 { let backend = self .backend .lock() @@ -411,6 +445,376 @@ impl PersistentFactStorage { Ok(()) } + /// Load facts from legacy one-per-page format (v4 and earlier). + fn load_one_per_page_legacy(&mut self, header: &FileHeader) -> Result { + let page_count = header.page_count; + let backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + let mut loaded: usize = 0; + let mut skipped: usize = 0; + for page_id in 1..page_count { + let page = backend.read_page(page_id)?; + // Try to deserialize a fact from this page (legacy format: raw postcard bytes) + match postcard::from_bytes::(&page) { + Ok(fact) => { + self.storage.load_fact(fact)?; + loaded = loaded.saturating_add(1); + } + Err(e) => { + skipped = skipped.saturating_add(1); + eprintln!( + "Warning: failed to deserialize fact at page {}: {}. Skipping.", + page_id, e + ); + } + } + } + if skipped > 0 { + eprintln!( + "Warning: {} facts failed to deserialize during legacy load (version {})", + skipped, header.version + ); + } + Ok(loaded) + } + + /// Migrate a v1 file (Phase 3 format, no bi-temporal fields) to v2. + /// + /// V1 facts only have (entity, attribute, value, tx_id, asserted). + /// V2 facts add tx_count, valid_from, valid_to. + /// + /// Migration strategy: + /// - Sort v1 facts by tx_id ascending + /// - Group facts with the same tx_id into the same tx_count (monotonic counter) + /// - Set valid_from = tx_id as i64 (wall-clock approximation) + /// - Set valid_to = VALID_TIME_FOREVER (open-ended) + /// - Write the migrated data back in v2 format + fn migrate_v1_to_v2(&mut self) -> Result<()> { + use crate::graph::types::VALID_TIME_FOREVER; + + let backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + let header_page = backend.read_page(0)?; + let header = FileHeader::from_bytes(&header_page)?; + let page_count = header.page_count; + + // Read all v1 facts (track deserialization failures) + let mut v1_facts: Vec = Vec::new(); + let mut skipped: usize = 0; + for page_id in 1..page_count { + let page = backend.read_page(page_id)?; + match postcard::from_bytes::(&page) { + Ok(fact) => v1_facts.push(fact), + Err(e) => { + skipped = skipped.saturating_add(1); + eprintln!( + "Warning: failed to deserialize v1 fact at page {}: {}. Skipping.", + page_id, e + ); + } + } + } + if skipped > 0 { + eprintln!( + "Warning: {} v1 facts failed to deserialize during migration", + skipped + ); + } + drop(backend); + + // Sort by tx_id ascending so we can group them + v1_facts.sort_by_key(|f| f.tx_id); + + // Assign tx_count, grouping facts with the same tx_id into the same tx_count + let mut tx_count: u64 = 0; + let mut prev_tx_id: Option = None; + let mut migrated: Vec = Vec::new(); + + for v1 in v1_facts { + if prev_tx_id != Some(v1.tx_id) { + tx_count = tx_count.saturating_add(1); + prev_tx_id = Some(v1.tx_id); + } + let valid_from = v1.tx_id.cast_signed(); + let mut fact = Fact::with_valid_time( + v1.entity, + v1.attribute, + v1.value, + v1.tx_id, + tx_count, + valid_from, + VALID_TIME_FOREVER, + ); + // Preserve the asserted flag (with_valid_time sets asserted=true by default) + fact.asserted = v1.asserted; + migrated.push(fact); + } + + self.storage.clear()?; + for fact in migrated { + self.storage.load_fact(fact)?; + } + self.storage.restore_tx_counter()?; + + // Persist in v2 format immediately + self.dirty = true; + self.save()?; + Ok(()) + } + + /// Migrate a v5 file (paged-blob indexes) to v6 (on-disk B+tree indexes). + fn migrate_v5_to_v6(&mut self, header: &FileHeader) -> Result<()> { + let num_fact_pages = { + let first_index_page = [ + header.eavt_root_page, + header.aevt_root_page, + header.avet_root_page, + header.vaet_root_page, + ] + .iter() + .filter(|&&p| p > 0) + .copied() + .min() + .unwrap_or(header.page_count); + first_index_page.saturating_sub(1) + }; + + // Validate the calculated range contains valid pages. + // If the file was in an inconsistent state (partial checkpoint), + // the calculated range might be incorrect. Do a quick validation + // by checking that the first fact page can be read (doesn't need to + // be a packed page - the index checksum will catch any real corruption). + let validated_num_fact_pages = if num_fact_pages > 0 { + let backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + // Just verify we can read the page - actual validation happens via checksum + if backend.read_page(1).is_ok() { + num_fact_pages + } else { + eprintln!( + "Warning: cannot read first fact page (page 1). Header claims {}. Using 0.", + num_fact_pages + ); + 0 + } + } else { + num_fact_pages + }; + + self.committed_fact_pages + .store(validated_num_fact_pages, Ordering::SeqCst); + + // Verify index integrity via checksum before trusting the old indexes. + // Try full checksum (facts + indexes) first, then fact-only for old files. + let use_old_indexes = validated_num_fact_pages > 0 && header.index_checksum > 0 && { + let backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + let total_data_pages = header.page_count.saturating_sub(1); + let full = compute_page_checksum(&*backend, 1, total_data_pages)?; + if full == header.index_checksum { + true + } else { + let fact_only = compute_page_checksum(&*backend, 1, validated_num_fact_pages)?; + fact_only == header.index_checksum + } + }; + + let (eavt, aevt, avet, vaet) = if use_old_indexes { + // Read and trust the old v5 indexes + let backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + let e = if header.eavt_root_page > 0 { + read_eavt_index(header.eavt_root_page, &*backend)? + } else { + std::collections::BTreeMap::new() + }; + let a = if header.aevt_root_page > 0 { + read_aevt_index(header.aevt_root_page, &*backend)? + } else { + std::collections::BTreeMap::new() + }; + let av = if header.avet_root_page > 0 { + read_avet_index(header.avet_root_page, &*backend)? + } else { + std::collections::BTreeMap::new() + }; + let v = if header.vaet_root_page > 0 { + read_vaet_index(header.vaet_root_page, &*backend)? + } else { + std::collections::BTreeMap::new() + }; + (e, a, av, v) + } else { + // Checksum mismatch or missing - rebuild indexes from facts + let all_facts = { + let backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + crate::storage::packed_pages::read_all_from_pages( + &*backend, + 1, + validated_num_fact_pages, + )? + }; + // Build indexes from fact data + let mut eavt_map = std::collections::BTreeMap::new(); + let mut aevt_map = std::collections::BTreeMap::new(); + let mut avet_map = std::collections::BTreeMap::new(); + let mut vaet_map = std::collections::BTreeMap::new(); + for (i, fact) in all_facts.iter().enumerate() { + let slot_index = u16::try_from(i).map_err(|_| err_coded!(ErrorCode::Stg020, i))?; + let fr = FactRef { + page_id: 1, + slot_index, + }; + eavt_map.insert( + EavtKey { + entity: fact.entity, + attribute: fact.attribute.clone(), + valid_from: fact.valid_from, + valid_to: fact.valid_to, + tx_count: fact.tx_count, + }, + fr, + ); + aevt_map.insert( + AevtKey { + attribute: fact.attribute.clone(), + entity: fact.entity, + valid_from: fact.valid_from, + valid_to: fact.valid_to, + tx_count: fact.tx_count, + }, + fr, + ); + avet_map.insert( + AvetKey { + attribute: fact.attribute.clone(), + value_bytes: encode_value(&fact.value), + valid_from: fact.valid_from, + valid_to: fact.valid_to, + entity: fact.entity, + tx_count: fact.tx_count, + }, + fr, + ); + if let crate::graph::types::Value::Ref(target) = &fact.value { + vaet_map.insert( + VaetKey { + ref_target: *target, + attribute: fact.attribute.clone(), + valid_from: fact.valid_from, + valid_to: fact.valid_to, + source_entity: fact.entity, + tx_count: fact.tx_count, + }, + fr, + ); + } + } + (eavt_map, aevt_map, avet_map, vaet_map) + }; + + let mut backend = self + .backend + .lock() + .map_err(|_| err_coded!(ErrorCode::Stg016))?; + // Use the actual end of fact pages as the start for new index pages, NOT + // header.page_count — that field comes from the (possibly untrusted) file + // on disk and may be a huge fuzz-crafted value that causes build_btree to + // write leaf pages at a ~TB offset, then compute_page_checksum to loop + // over billions of pages. + let next_free = 1u64 + .checked_add(validated_num_fact_pages) + .ok_or_else(|| err_coded!(ErrorCode::Stg018))?; + + let (eavt_root, next_free2) = build_btree( + btree_entries(eavt.into_iter())?.into_iter(), + &mut *backend, + &self.page_cache, + next_free, + )?; + let (aevt_root, next_free3) = build_btree( + btree_entries(aevt.into_iter())?.into_iter(), + &mut *backend, + &self.page_cache, + next_free2, + )?; + let (avet_root, next_free4) = build_btree( + btree_entries(avet.into_iter())?.into_iter(), + &mut *backend, + &self.page_cache, + next_free3, + )?; + let (vaet_root, final_next_free) = build_btree( + btree_entries(vaet.into_iter())?.into_iter(), + &mut *backend, + &self.page_cache, + next_free4, + )?; + + let mut new_header = FileHeader::new(); // version=7 + new_header.page_count = final_next_free; + new_header.node_count = header.node_count; + new_header.last_checkpointed_tx_count = header.last_checkpointed_tx_count; + new_header.eavt_root_page = eavt_root; + new_header.aevt_root_page = aevt_root; + new_header.avet_root_page = avet_root; + new_header.vaet_root_page = vaet_root; + // Checksum over all data pages (facts + indexes) + let total_data_pages = final_next_free.saturating_sub(1); + let computed_checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; + new_header.index_checksum = computed_checksum; + new_header.fact_page_format = header.fact_page_format; + new_header.fact_page_count = validated_num_fact_pages; + new_header.header_checksum = compute_header_checksum(&new_header); + + let mut header_page = new_header.to_bytes(); + header_page.resize(PAGE_SIZE, 0); + backend.write_page(0, &header_page)?; + backend.sync()?; + drop(backend); + + self.last_checkpointed_tx_count = header.last_checkpointed_tx_count; + + let loader: Arc = + Arc::new(CommittedFactLoaderImpl { + backend: self.backend.clone(), + backend_adapter: MutexStorageBackend(self.backend.clone()), + page_cache: self.page_cache.clone(), + committed_fact_pages: self.committed_fact_pages.clone(), + first_fact_page: 1, + }); + self.storage.set_committed_reader(loader); + + let index_reader: Arc = + Arc::new(OnDiskIndexReader::new( + self.backend.clone(), + self.page_cache.clone(), + eavt_root, + aevt_root, + avet_root, + vaet_root, + )); + self.storage.set_committed_index_reader(index_reader); + + self.storage + .restore_tx_counter_from(header.last_checkpointed_tx_count); + self.dirty = false; + Ok(()) + } + /// Consume this storage and return the underlying backend. /// /// Useful in tests to inspect or reuse the backend after saving. @@ -561,7 +965,7 @@ impl PersistentFactStorage { let checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; // ── Step E: write header (last write = crash-safe boundary) ───────────── - let mut header = FileHeader::new(); // version=FORMAT_VERSION + let mut header = FileHeader::new(); // version=7 header.page_count = next4; let pending_len = u64::try_from(pending_facts.len()).map_err(|_| err_coded!(ErrorCode::Stg024))?; @@ -778,28 +1182,78 @@ fn build_sorted_index_entries( let mut eavt: Vec<(EavtKey, FactRef)> = facts .iter() .zip(refs.iter()) - .map(|(f, &fr)| (EavtKey::from_fact(f), fr)) + .map(|(f, &fr)| { + ( + EavtKey { + entity: f.entity, + attribute: f.attribute.clone(), + valid_from: f.valid_from, + valid_to: f.valid_to, + tx_count: f.tx_count, + }, + fr, + ) + }) .collect(); eavt.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let mut aevt: Vec<(AevtKey, FactRef)> = facts .iter() .zip(refs.iter()) - .map(|(f, &fr)| (AevtKey::from_fact(f), fr)) + .map(|(f, &fr)| { + ( + AevtKey { + attribute: f.attribute.clone(), + entity: f.entity, + valid_from: f.valid_from, + valid_to: f.valid_to, + tx_count: f.tx_count, + }, + fr, + ) + }) .collect(); aevt.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let mut avet: Vec<(AvetKey, FactRef)> = facts .iter() .zip(refs.iter()) - .map(|(f, &fr)| (AvetKey::from_fact(f), fr)) + .map(|(f, &fr)| { + ( + AvetKey { + attribute: f.attribute.clone(), + value_bytes: encode_value(&f.value), + valid_from: f.valid_from, + valid_to: f.valid_to, + entity: f.entity, + tx_count: f.tx_count, + }, + fr, + ) + }) .collect(); avet.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let mut vaet: Vec<(VaetKey, FactRef)> = facts .iter() .zip(refs.iter()) - .filter_map(|(f, &fr)| VaetKey::from_fact(f).map(|k| (k, fr))) + .filter_map(|(f, &fr)| { + if let crate::graph::types::Value::Ref(target) = &f.value { + Some(( + VaetKey { + ref_target: *target, + attribute: f.attribute.clone(), + valid_from: f.valid_from, + valid_to: f.valid_to, + source_entity: f.entity, + tx_count: f.tx_count, + }, + fr, + )) + } else { + None + } + }) .collect(); vaet.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); @@ -832,29 +1286,6 @@ mod tests { assert_eq!(storage.storage().fact_count(), 0); } - #[test] - fn same_tx_multi_value_visible_through_entity_and_attribute_lookups_before_save() { - let pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); - let e = Uuid::from_u128(42); - pfs.storage() - .transact( - vec![ - (e, ":kind".to_string(), Value::Keyword(":k/a".to_string())), - (e, ":kind".to_string(), Value::Keyword(":k/b".to_string())), - ], - None, - ) - .unwrap(); - assert_eq!(pfs.storage().get_facts_by_entity(&e).unwrap().len(), 2); - assert_eq!( - pfs.storage() - .get_facts_by_attribute(&":kind".to_string()) - .unwrap() - .len(), - 2 - ); - } - #[test] fn test_persistent_fact_storage_save_load() { // Create separate scopes to test persistence @@ -920,6 +1351,60 @@ mod tests { // This test verifies the pattern, actual persistence is tested above } + // ----------------------------------------------------------------------- + // Migration helpers + // ----------------------------------------------------------------------- + + /// Build a MemoryBackend that contains a v1-format file with two FactV1 facts. + fn make_v1_backend() -> MemoryBackend { + use crate::storage::{MAGIC_NUMBER, PAGE_SIZE}; + + let alice = Uuid::new_v4(); + + #[derive(serde::Serialize)] + struct FactV1Ser { + entity: Uuid, + attribute: String, + value: Value, + tx_id: u64, + asserted: bool, + } + + let fact1 = FactV1Ser { + entity: alice, + attribute: ":person/name".to_string(), + value: Value::String("Alice".to_string()), + tx_id: 1000, + asserted: true, + }; + let fact2 = FactV1Ser { + entity: alice, + attribute: ":person/age".to_string(), + value: Value::Integer(30), + tx_id: 1000, + asserted: true, + }; + + let mut backend = MemoryBackend::new(); + + // Write v1 header (version=1, page_count=3) + let mut header_bytes = vec![0u8; PAGE_SIZE]; + header_bytes[0..4].copy_from_slice(&MAGIC_NUMBER); + header_bytes[4..8].copy_from_slice(&1u32.to_le_bytes()); // version = 1 + header_bytes[8..16].copy_from_slice(&3u64.to_le_bytes()); // page_count = 3 + backend.write_page(0, &header_bytes).unwrap(); + + // Write facts (one per page) + for (i, fact) in [&fact1, &fact2].iter().enumerate() { + let data = postcard::to_allocvec(*fact).unwrap(); + let mut page = vec![0u8; PAGE_SIZE]; + page[..data.len()].copy_from_slice(&data); + backend.write_page((i + 1) as u64, &page).unwrap(); + } + + backend + } + #[test] fn test_load_preserves_original_tx_id() { let mut pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); @@ -952,6 +1437,30 @@ mod tests { ); } + #[test] + fn test_migrate_v1_to_v2_assigns_defaults() { + use crate::graph::types::VALID_TIME_FOREVER; + + let backend = make_v1_backend(); + let pfs = PersistentFactStorage::new(backend, 256).unwrap(); + let facts = pfs.storage().get_all_facts().unwrap(); + + assert_eq!(facts.len(), 2); + // Both facts share tx_id=1000 → same tx_count + assert_eq!( + facts[0].tx_count, facts[1].tx_count, + "facts from the same tx_id batch must get the same tx_count" + ); + assert_eq!( + facts[0].valid_to, VALID_TIME_FOREVER, + "migrated fact must have open-ended valid_to" + ); + assert_eq!( + facts[0].valid_from, 1000_i64, + "migrated fact valid_from must equal original tx_id" + ); + } + #[test] fn test_save_writes_v4_header() { use crate::storage::FORMAT_VERSION; @@ -1189,6 +1698,34 @@ mod tests { assert_eq!(c1, c2, "Checksum must be order-independent"); } + #[test] + fn test_migrate_v1_tx_counter_set_correctly() { + let backend = make_v1_backend(); + let pfs = PersistentFactStorage::new(backend, 256).unwrap(); + + let alice = Uuid::new_v4(); + pfs.storage() + .transact( + vec![(alice, ":new/fact".to_string(), Value::Boolean(true))], + None, + ) + .unwrap(); + + let new_fact = pfs + .storage() + .get_all_facts() + .unwrap() + .into_iter() + .find(|f| f.attribute == ":new/fact") + .unwrap(); + + // After migrating 1 unique tx_id (tx_count=1), next tx should get tx_count=2 + assert_eq!( + new_fact.tx_count, 2, + "first new transaction after migration must get tx_count=2" + ); + } + #[test] fn test_save_writes_packed_pages() { use crate::storage::backend::FileBackend; @@ -1212,12 +1749,12 @@ mod tests { pfs.save().unwrap(); } - // Verify: header says current version, fact_page_format = PACKED + // Verify: header says v6, fact_page_format = PACKED { let backend = FileBackend::open(&path).unwrap(); let header_bytes = backend.read_page(0).unwrap(); let header = crate::storage::FileHeader::from_bytes(&header_bytes).unwrap(); - assert_eq!(header.version, FORMAT_VERSION); + assert_eq!(header.version, 7); assert_eq!( header.fact_page_format, crate::storage::FACT_PAGE_FORMAT_PACKED @@ -1267,6 +1804,70 @@ mod tests { } } + #[test] + fn test_v4_database_migrates_to_v5_on_open() { + use crate::storage::backend::FileBackend; + use crate::storage::{FACT_PAGE_FORMAT_PACKED, PAGE_SIZE}; + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let alice = Uuid::new_v4(); + + // Write a "v4-style" file: version=4, fact_page_format byte = 0 (legacy padding) + { + use crate::storage::FileHeader; + let fact = crate::graph::types::Fact::with_valid_time( + alice, + ":name".to_string(), + Value::String("Alice".to_string()), + 1u64, + 1u64, + 0i64, + i64::MAX, + ); + let mut backend = FileBackend::open(&path).unwrap(); + + // Write fact at page 1 (one-per-page format) + let data = postcard::to_allocvec(&fact).unwrap(); + let mut page = vec![0u8; PAGE_SIZE]; + page[..data.len()].copy_from_slice(&data); + backend.write_page(1, &page).unwrap(); + + // Write v4 header (fact_page_format byte will be 0) + let mut header = FileHeader::new(); + header.page_count = 2; + header.node_count = 1; + let mut hbytes = header.to_bytes(); + // Force version to 4 + hbytes[4..8].copy_from_slice(&4u32.to_le_bytes()); + // Force fact_page_format byte (offset 68) to 0 + hbytes[68] = 0; + hbytes.resize(PAGE_SIZE, 0); + backend.write_page(0, &hbytes).unwrap(); + backend.sync().unwrap(); + } + + // Open — should auto-migrate to v7 + { + let pfs = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + assert_eq!( + pfs.storage().fact_count(), + 1, + "migrated fact must be loaded" + ); + } + + // Verify file is now v7 + { + let backend = FileBackend::open(&path).unwrap(); + let header_bytes = backend.read_page(0).unwrap(); + let header = crate::storage::FileHeader::from_bytes(&header_bytes).unwrap(); + assert_eq!(header.version, 7, "file must be upgraded to v7"); + assert_eq!(header.fact_page_format, FACT_PAGE_FORMAT_PACKED); + } + } + #[test] fn test_v5_load_fast_path_indexes_loaded() { use crate::storage::backend::FileBackend; @@ -1328,11 +1929,8 @@ mod tests { let backend = storage.into_backend().unwrap(); let header_page = backend.read_page(0).unwrap(); let header = crate::storage::FileHeader::from_bytes(&header_page).unwrap(); - assert_eq!( - header.version, FORMAT_VERSION, - "save() must write current-version header" - ); - assert_eq!(header.to_bytes().len(), 84, "header must be 84 bytes"); + assert_eq!(header.version, 7, "save() must write v7 header"); + assert_eq!(header.to_bytes().len(), 84, "v7 header must be 84 bytes"); assert!(header.fact_page_count > 0, "fact_page_count must be set"); assert!( header.eavt_root_page > 0, @@ -1340,91 +1938,6 @@ mod tests { ); } - #[test] - fn v7_header_forces_rebuild_and_upgrades_to_v8() { - use crate::storage::FileHeader; - let e = Uuid::from_u128(7); - let mut backend = { - let mut s = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); - s.storage() - .transact(vec![(e, ":n".to_string(), Value::Integer(1))], None) - .unwrap(); - s.mark_dirty(); - s.save().unwrap(); - s.into_backend().unwrap() - }; - // Downgrade the header to v7 (keys on disk are now v8-shaped, which is - // fine: the rebuild never reads them). - let mut h = FileHeader::from_bytes(&backend.read_page(0).unwrap()).unwrap(); - h.version = 7; - h.header_checksum = compute_header_checksum(&h); - let mut page = h.to_bytes(); - page.resize(PAGE_SIZE, 0); - backend.write_page(0, &page).unwrap(); - - let s = PersistentFactStorage::new(backend, 256).unwrap(); - assert_eq!(s.storage().get_facts_by_entity(&e).unwrap().len(), 1); - let b = s.into_backend().unwrap(); - let h2 = FileHeader::from_bytes(&b.read_page(0).unwrap()).unwrap(); - assert_eq!(h2.version, 8, "v7 file must be upgraded to v8 on open"); - } - - /// #371/#287 review finding: empty transactions (`(transact [])`, - /// `(retract [])`) still allocate a tx_count without producing any facts - /// (see `FactStorage::transact` — `fetch_add` happens unconditionally). - /// The v7->v8 rebuild path must not compute the restored tx_counter as - /// `max(fact.tx_count)` alone, or trailing empty transactions before the - /// last checkpoint would be forgotten, rewinding the counter and letting - /// new transactions reuse tx_counts that `:as-of N` already served facts - /// for. - #[test] - fn v7_migration_does_not_rewind_tx_counter_past_empty_transactions() { - use crate::storage::FileHeader; - let e = Uuid::from_u128(70); - let mut backend = { - let mut s = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); - s.storage() - .transact(vec![(e, ":n".to_string(), Value::Integer(1))], None) - .unwrap(); - // Empty transactions still allocate tx_count without producing facts. - s.storage().transact(vec![], None).unwrap(); - s.storage().transact(vec![], None).unwrap(); - s.mark_dirty(); - s.save().unwrap(); - s.into_backend().unwrap() - }; - let pre_migration_last_checkpointed = { - let h = FileHeader::from_bytes(&backend.read_page(0).unwrap()).unwrap(); - h.last_checkpointed_tx_count - }; - assert_eq!( - pre_migration_last_checkpointed, 3, - "one fact-bearing transact plus two empty transacts" - ); - - // Downgrade the header to v7 (keys on disk are now v8-shaped, which is - // fine: the rebuild never reads them). - let mut h = FileHeader::from_bytes(&backend.read_page(0).unwrap()).unwrap(); - h.version = 7; - h.header_checksum = compute_header_checksum(&h); - let mut page = h.to_bytes(); - page.resize(PAGE_SIZE, 0); - backend.write_page(0, &page).unwrap(); - - let s = PersistentFactStorage::new(backend, 256).unwrap(); - assert!( - s.storage().current_tx_count() >= pre_migration_last_checkpointed, - "tx_counter must not rewind below the pre-migration last_checkpointed_tx_count" - ); - - let b = s.into_backend().unwrap(); - let h2 = FileHeader::from_bytes(&b.read_page(0).unwrap()).unwrap(); - assert!( - h2.last_checkpointed_tx_count >= pre_migration_last_checkpointed, - "v8 header's last_checkpointed_tx_count must not decrease across migration" - ); - } - #[test] fn test_load_v6_wires_committed_index_reader() { let alice = Uuid::new_v4(); @@ -1500,6 +2013,53 @@ mod tests { ); } + #[test] + fn test_v6_migration_from_v5_unit() { + let mut backend = MemoryBackend::new(); + let mut page = vec![0u8; PAGE_SIZE]; + page[0..4].copy_from_slice(b"MGRF"); + page[4..8].copy_from_slice(&5u32.to_le_bytes()); // version = 5 + page[8..16].copy_from_slice(&2u64.to_le_bytes()); // page_count = 2 (header + 1 empty page) + page[68] = 0x02; // fact_page_format = PACKED + backend.write_page(0, &page).unwrap(); + // Write an empty fact page so page_count > 1 triggers load() + backend.write_page(1, &vec![0u8; PAGE_SIZE]).unwrap(); + + let s = PersistentFactStorage::new(backend, 256).unwrap(); + let b = s.into_backend().unwrap(); + let header_page = b.read_page(0).unwrap(); + let header = crate::storage::FileHeader::from_bytes(&header_page).unwrap(); + assert_eq!(header.version, 7, "migration must upgrade header to v7"); + assert_eq!(header.to_bytes().len(), 84, "v7 header must be 84 bytes"); + // page_count=2 means 1 fact page (page 1), even if empty + assert_eq!( + header.fact_page_count, 1, + "fact_page_count must reflect page layout" + ); + } + + #[test] + fn test_v6_migration_crash_safe_unit() { + let mut backend = MemoryBackend::new(); + let mut page = vec![0u8; PAGE_SIZE]; + page[0..4].copy_from_slice(b"MGRF"); + page[4..8].copy_from_slice(&5u32.to_le_bytes()); + page[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 + page[68] = 0x02; + backend.write_page(0, &page).unwrap(); + backend.write_page(1, &vec![0xFF_u8; PAGE_SIZE]).unwrap(); + backend.write_page(2, &vec![0xFF_u8; PAGE_SIZE]).unwrap(); + + let s = PersistentFactStorage::new(backend, 256).unwrap(); + let b = s.into_backend().unwrap(); + let header_bytes = b.read_page(0).unwrap(); + let header = crate::storage::FileHeader::from_bytes(&header_bytes).unwrap(); + assert_eq!( + header.version, 7, + "migration must complete despite prior partial run" + ); + } + /// Regression test for fuzz-discovered timeout (artifact: /// `timeout-23b2b7e0aa43d92c12c49f123a48d6f4ace6ce33`). /// @@ -1509,7 +2069,8 @@ mod tests { /// compute_page_checksum looped over 3.6 billion zero-filled sparse pages /// (each read_exact returns zeros, no error), hanging the process. /// - /// Since v3.0.0 a v5 header is rejected before any page is touched. + /// After the fix, next_free = 1 + validated_num_fact_pages (= 1 here), + /// so migration completes in constant time and produces a sane v7 header. #[test] fn test_v5_migration_large_page_count_does_not_hang() { use crate::storage::backend::FileBackend; @@ -1527,16 +2088,19 @@ mod tests { page[68] = 0x20; // fact_page_format std::fs::write(&path, &page).unwrap(); - // `FileBackend::open` itself reads and validates the header, so the - // pre-v7 rejection surfaces there rather than in - // `PersistentFactStorage::new`. - let err = match FileBackend::open(&path) { - Ok(_) => panic!("v5 header must be rejected"), - Err(e) => e, - }; + // Must complete without hanging; next_free must be computed from actual + // fact pages (= 1 + 0 = 1), not from the crafted header.page_count. + let s = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256) + .expect("migration must complete"); + let b = s.into_backend().unwrap(); + let header_bytes = b.read_page(0).unwrap(); + let header = crate::storage::FileHeader::from_bytes(&header_bytes).unwrap(); + assert_eq!(header.version, 7, "migration must upgrade to v7"); + // Resulting page_count must be small (0 fact pages + 4 index pages + header), + // NOT the crafted 3.6-billion-page value. assert!( - err.to_string().contains("no longer supported"), - "must fail with STG-028" + header.page_count < 100, + "page_count must be sane after migration" ); } diff --git a/tests/btree_v6_test.rs b/tests/btree_v6_test.rs index d3cd770504..b2a813132e 100644 --- a/tests/btree_v6_test.rs +++ b/tests/btree_v6_test.rs @@ -110,29 +110,49 @@ fn test_v6_pending_plus_committed_merge() { } #[test] -fn test_v5_file_is_rejected() { - // v3.0.0 dropped support for formats v1–v6. Write a minimal v5 header - // directly to the .graph file (raw bytes) and verify that opening it - // fails with STG-028 and leaves the file unmodified. +fn test_v6_migration_from_v5_eager() { + // Write a minimal v5 header directly to the .graph file (raw bytes), + // then open with v6 code — migrate_v5_to_v6 must run and produce a v6 file. + // We cannot use OpenOptions to produce a v5 file (v6 code always writes v6), + // so we write the raw 72-byte v5 header at offset 0. let (_tmp, path) = tmp_path(); - let mut page = vec![0u8; 4096]; // PAGE_SIZE - page[0..4].copy_from_slice(b"MGRF"); - page[4..8].copy_from_slice(&5u32.to_le_bytes()); // version = 5 - page[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 - page[68] = 0x02; // fact_page_format = PACKED - std::fs::write(&path, &page).unwrap(); + // Create an empty v5 .graph file with only a header (no facts, no index pages) + { + let mut f = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&path) + .unwrap(); + let mut page = vec![0u8; 4096]; // PAGE_SIZE + page[0..4].copy_from_slice(b"MGRF"); + page[4..8].copy_from_slice(&5u32.to_le_bytes()); // version = 5 + page[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 + page[68] = 0x02; // fact_page_format = PACKED + use std::io::Write; + f.write_all(&page).unwrap(); + } - let err = match OpenOptions::new().path(&path).open() { - Ok(_) => panic!("v5 file must not open"), - Err(e) => e, - }; - assert_eq!(err.code(), "STG-028", "v5 file must fail with STG-028"); + // Open with v6 code — migration must run + let db = OpenOptions::new().path(&path).open().unwrap(); + // Query on empty DB must return empty result (not an error) + let _result = db + .execute("(query [:find ?e :where [?e :any :any]])") + .unwrap(); + drop(db); + + // Re-open and verify header was upgraded to v6 + let mut f = std::fs::File::open(&path).unwrap(); + let mut header_bytes = vec![0u8; 4096]; + use std::io::Read; + f.read_exact(&mut header_bytes).unwrap(); + let version = u32::from_le_bytes(header_bytes[4..8].try_into().unwrap()); assert_eq!( - std::fs::read(&path).unwrap(), - page, - "rejected file must be left unmodified" + version, 7, + "header must be upgraded from v5 to v7; got version={}", + version ); } diff --git a/tests/cross_platform_compat_test.rs b/tests/cross_platform_compat_test.rs index fbc61c6a0d..0d1c3a8d08 100644 --- a/tests/cross_platform_compat_test.rs +++ b/tests/cross_platform_compat_test.rs @@ -82,25 +82,12 @@ fn native_raw_page_bytes_round_trip() { /// Load the committed fixture (produced by `cargo run --example generate_compat_fixture`) /// via `Minigraf::open` and assert that the known facts are present. /// -/// The fixture is a frozen v7 file written by Minigraf v2.0.0; opening it -/// here exercises v7→v8 migration (#371, #287). It is also loaded by the -/// browser WASM tests via `include_bytes!` + `BrowserDb::import_graph`, -/// completing the cross-boundary coverage. +/// This fixture is also loaded by the browser WASM tests via `include_bytes!` + +/// `BrowserDb::import_graph`, completing the cross-boundary coverage. #[test] fn fixture_readable_by_native() { let fixture: &[u8] = include_bytes!("fixtures/compat.graph"); - // The whole point of this fixture is exercising v7->v8 migration on open; - // catch a silent regeneration (e.g. someone running - // `cargo run --example generate_compat_fixture` against the old output - // path and overwriting the frozen fixture) before it opens successfully - // for the wrong reason. - let embedded_version = u32::from_le_bytes(fixture[4..8].try_into().unwrap()); - assert_eq!( - embedded_version, 7, - "compat.graph fixture must remain the frozen v7 file" - ); - // Write to a temp path so Minigraf::open can use it. let path = tmp("fixture"); cleanup(&path); @@ -128,10 +115,5 @@ fn fixture_readable_by_native() { _ => panic!("expected QueryResults for age query"), } - drop(db); - let raw = std::fs::read(&path).expect("read migrated fixture"); - let version = u32::from_le_bytes(raw[4..8].try_into().unwrap()); - assert_eq!(version, 8, "v7 fixture must be upgraded to v8 on open"); - cleanup(&path); } diff --git a/tests/error_codes_storage_test.rs b/tests/error_codes_storage_test.rs index edebe0ea5c..9920ac06bc 100644 --- a/tests/error_codes_storage_test.rs +++ b/tests/error_codes_storage_test.rs @@ -69,23 +69,6 @@ fn open_unsupported_version_returns_stg_006() { assert_eq!(err.category(), ErrorCategory::Storage); } -#[test] -fn open_pre_v7_version_returns_stg_028() { - let dir = scratch_dir("pre_v7"); - let path = dir.join("db.graph"); - write_file( - &path, - &header_page(|b| { - b[4..8].copy_from_slice(&6u32.to_le_bytes()); // version = 6 - }), - ); - let err = match Minigraf::open(&path) { - Ok(_) => panic!("opening a v6 file should fail"), - Err(e) => e, - }; - assert_eq!(err.code(), "STG-028"); -} - #[test] fn open_zero_page_count_returns_stg_007() { let dir = scratch_dir("zero_pages"); diff --git a/tests/fixtures/v7_multivalue.graph b/tests/fixtures/v7_multivalue.graph deleted file mode 100644 index fb0d07c7c228dc262093333ca1be6476867f3def..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 49152 zcmeI5dvMg%702&QHt)y&5Ri9BAR(|MB&^v00YjiOwiSbsM<5ai2?$smUeeOeKnoER zaL9lKv_2w=;#6%_1_VJW0y3Z|sk}f8L#hVWGFUTVt&a<-t#Lv1Fsx;;ro&nv8&q3jL+W8zOJPli^-N4iZHWo0 zg|L>wng;8c=%CsZ9a0Nn9SUn2tf!(vYEx8DEr4|>tSPXb@&(lfUq~&0wFK4_SU-;p zsSS}qRSjzitnDN7)MpWS>WzpTwHPFi|Fr|o`%5yL@BPQn;hXA@UHGcQk-ffxnUxcx zd<7G8s?HvLcj=P6NOk!ed}wtIyz=|uRc9KiM;zI{;+e-MecM7`tA*>H>3M_ttbGytpB>?$l1D~GuC&V-oI`BD`V|#=nfBUj`UcxVCsspy-#)g@N#_B zs(`%>-Ql5)E=%m|n|0Txk0$?Xa85@2Gxj!ghle&(x1YOMRWx$@>)U!h*|%`lQ}#A= zhle%~6n|5^Z$jy7n|*z^6s+tq-QI@o@X+R~K^t04-!p87054oxolW7g?>dmFmLLz{xY!QR24!^XXm z@sqsHGgLc!8@j_oo2idnnDgYp5qCWH$c)9qp01x`Z$o!@Xfx^QwL9u}4QsYO`1ib+ zNx$&$|1`S8Lz{LB7p-o&X5_s~(x;3*aP+fc`>UZlJhZ9aR-2Xg^N}lG$WK3cym?Z( zy$#*rq0PL0FHUiK-(B+Pxo1}2_1?fi_BM2fhc<&RrLx1l>cw0VAR#@#m^AF_XW-L&I1h3|Wq0(6ImHggN> zHlICER`+~$bGy>@3#3i^r8u zeq!l_DW49i-ezw@cX()XG^S;b{awZ!u00erYEi}Jz4kVAhle%~oj>ur((M)g+SO|D z+S(J%>}}`{4{e_9GHr71hvf^l^jbA1wJ7dQdmFmLLz_z-GSBV4fB3N#Z*_RR|K7xY z_BM2fhc<5?%1^jw_s}uRUwOB%wC-oO+uP6`9@^Bj&8W_LZp=rIo$OeeKJSH)y$#*r zp-pU0Rr0}~hA&_Jd4BAlhittvY$9jQn(F%Yp6+RCRQ2C%s&w0wrUo{B*MHC-P%ne3 zK_fvuKt9l4lLP8i(1W0IP!JRgI+7Gn>p+V^V?p_#M9?RR0ksiS0~!x106Cx&2?6yc z=n>E)P$8%V=wy5V&IJN$CFnlTKu~MY>9~N}0eT!X4Rjl*J*Yl5pmu?3K{G+OgHk~k zVghOp=vmM&L4!eom~<5l`df52wGOm68lEdCACw6CBnqA@s0K71Q~;8PI3Ny)1J`!I z;Xme*^BtYz(EkXA&x9x$%aGx@PstdE9_R2i`!OV6v#&$n;qWz+8IrG=VY1Z0Ss{+J`O#6;cI3wBww?;LqB8jHE(7}zGh2@o<;FBdoUzlGuxr> z(D|D249VAwbLc4(Uo*gve9dkSeN)WW^f4q~Gt!|Sg7}(=49V9_aOmkLUo*&%e9fK? zJxk$hrZ6O5vx7qqPWYPb8IrHr&Y>Up`I;>llCRm^p+_Qo&DIRb*KFm`??U*Rg$&8p zyw#zH>wL{ThU9DB;?RRfzGgZ@@-;g<^pugW*@+?fnrRL_?&WKCWk|kerbAEM_?m4Q zlCRmuq36YX%`Obd*UWI}aVlRkl_B|>9Uc0)oUa+dkbF(;nN$w_Hj3|_&yalgP+C)R zU#alte|d-l;($0H4u}KdfH)uyhy&t)I3Ny)1LD9B#R2BKDgLA&w-bE+zjCdA_&WaZ z<1hZMjX%i$iUZ<+I3Ny)1LA-iKOL?C zxCA->I?x)>Y|s!;SI}k1`tJw52zoFoT~&elM0LBNf3Nc7!~t>O25_L^?i3^wPDAMZ zdGNd0au3Xa#;eoFT&LFt9-ry^WS-tgM(QxR@F&_LR! zVic10sn`ce`&8_XqyL~D)N76nOTOnzmink(ZUVlMf`z=>U;|4I4?2M#&tpp|EGW&+t zZl8*oNZO}j8zk*hF#}2aRP2bPd1V28Mana&kuj*FM6D1LA-o1F2|$|ar{eH=*718sqUCY-U5X z#)LcvoAFSsF(l8yWRe-*mJNMFx47k_8e^HLAA#Gd^q-5WhTFl|Q@O0Q>8sCMd^AD0x1z6(zWFQ)We zcshGAtM9_o@r#Ll7oJXF%X5--V}h7&H7XJRQZDNZS-gunUwEm`TIaO16>8lFexwF?hu{EErv zn>lN$hfnEvKNRjrmmbmL?VmW9p3laH^Z!@A|BuCA-~oU4>-MXZ#^QiDAP$HF;($0H z4u}KdfH)uyhy&t)IN;F%_W&z70Z8*$QZR8q91sV@0dYVa5C_BoaX=gp2gCt!Kpgnt GI`AJiwFlP# diff --git a/tests/migration_matrix_test.rs b/tests/migration_matrix_test.rs index 3c3c66819d..787b5b2953 100644 --- a/tests/migration_matrix_test.rs +++ b/tests/migration_matrix_test.rs @@ -34,34 +34,24 @@ fn v7_round_trip_is_idempotent() { assert_eq!(n, 1, "v7 round-trip: Alice must survive close/reopen"); } -/// Formats v1–v6 are no longer readable (v3.0.0 file-format policy). Opening -/// one must fail with STG-028 and must not modify the file. #[test] -fn pre_v7_versions_are_rejected_with_stg_028() { - for version in 1u32..=6 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("old.graph"); - let mut page = vec![0u8; PAGE_SIZE]; - page[0..4].copy_from_slice(&MAGIC_NUMBER); - page[4..8].copy_from_slice(&version.to_le_bytes()); - page[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 - std::fs::write(&path, &page).unwrap(); - - let err = match Minigraf::open(&path) { - Ok(_) => panic!("pre-v7 file must not open"), - Err(e) => e, - }; - assert_eq!(err.code(), "STG-028", "pre-v7 file must fail with STG-028"); - assert!( - err.to_string().contains("v2.x"), - "STG-028 message must tell the user how to upgrade" - ); - assert_eq!( - std::fs::read(&path).unwrap(), - page, - "rejected file must be left unmodified" - ); - } +fn v3_empty_migrates_without_error() { + use std::io::Write; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("v3.graph"); + // Build a minimally valid v3 header: magic + version=3 + page_count=1 + // All other fields zero (roots=0 means empty index, fact_count=0). + let mut page = vec![0u8; PAGE_SIZE]; + page[0..4].copy_from_slice(&MAGIC_NUMBER); + page[4..8].copy_from_slice(&3u32.to_le_bytes()); // version + page[8..16].copy_from_slice(&1u64.to_le_bytes()); // page_count = 1 + let mut f = std::fs::File::create(&path).unwrap(); + f.write_all(&page).unwrap(); + drop(f); + assert!( + Minigraf::open(&path).is_ok(), + "v3 empty file should open without error" + ); } #[test] @@ -93,16 +83,8 @@ fn unsupported_version_fails_loudly() { page[4..8].copy_from_slice(&99u32.to_le_bytes()); let mut f = std::fs::File::create(&path).unwrap(); f.write_all(&page).unwrap(); - drop(f); - let before = std::fs::read(&path).unwrap(); let result = Minigraf::open(&path); assert!(result.is_err(), "unsupported version must produce an error"); - assert_eq!(result.err().unwrap().code(), "STG-006"); - let after = std::fs::read(&path).unwrap(); - assert_eq!( - before, after, - "a rejected future-version file must be left unmodified" - ); } #[test] diff --git a/tests/multi_value_test.rs b/tests/multi_value_test.rs deleted file mode 100644 index eb41859b88..0000000000 --- a/tests/multi_value_test.rs +++ /dev/null @@ -1,551 +0,0 @@ -//! Same-transaction multi-valued facts (#371, #287): every query path must -//! return every value, before checkpoint, after WAL replay, and after -//! checkpoint + reopen. -#![cfg(not(target_arch = "wasm32"))] - -use minigraf::{Minigraf, QueryResult, Value}; -use std::collections::BTreeSet; - -mod common; -use common::{CrashTx, run_crashing_child}; - -fn rows(r: QueryResult) -> Vec> { - match r { - QueryResult::QueryResults { results, .. } => results, - _ => panic!("expected QueryResults"), - } -} - -fn kw(v: &Value) -> String { - match v { - Value::Keyword(k) => k.clone(), - _ => panic!("expected keyword value"), - } -} - -/// Values of `attr` on the entity `ent` through EAVT (entity bound), AEVT -/// (attribute bound, joined to a marker attribute) and a full scan. -fn three_paths(db: &Minigraf, ent: &str, attr: &str, marker: &str) -> [BTreeSet; 3] { - let eavt = rows( - db.execute(&format!("(query [:find ?v :where [{ent} {attr} ?v]])")) - .unwrap(), - ) - .iter() - .map(|r| kw(&r[0])) - .collect(); - let aevt = rows( - db.execute(&format!( - r#"(query [:find ?v :where [?e {attr} ?v] [?e :note "{marker}"]])"# - )) - .unwrap(), - ) - .iter() - .map(|r| kw(&r[0])) - .collect(); - let scan = rows( - db.execute(&format!( - r#"(query [:find ?a ?v :where [?e ?a ?v] [?e :note "{marker}"]])"# - )) - .unwrap(), - ) - .iter() - .filter(|r| matches!(&r[0], Value::Keyword(a) if a == attr)) - .map(|r| kw(&r[1])) - .collect(); - [eavt, aevt, scan] -} - -fn setup_statements(i: usize) -> Vec { - let mut facts = vec![ - format!("[:t/x{i} :kind :k/a]"), - format!("[:t/x{i} :kind :k/b]"), - format!(r#"[:t/x{i} :note "two{i}"]"#), - ]; - for f in 0..30 { - facts.push(format!("[:t/f{i}-{f} :kind :k/c]")); - facts.push(format!(r#"[:t/f{i}-{f} :note "f"]"#)); - } - vec![format!("(transact [{}])", facts.join(" "))] -} - -fn both() -> BTreeSet { - [":k/a", ":k/b"].iter().map(|s| s.to_string()).collect() -} - -#[test] -fn multi_value_visible_on_every_path_before_checkpoint() { - let db = Minigraf::in_memory().unwrap(); - for s in setup_statements(0) { - db.execute(&s).unwrap(); - } - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, ":t/x0", ":kind", "two0")) - { - assert_eq!( - got, - both(), - "{name} path must return both values before checkpoint" - ); - } -} - -#[test] -fn multi_value_visible_on_every_path_after_checkpoint_and_reopen() { - // Several graph shapes: the issue saw EAVT and AEVT disagree depending on content. - for i in 0..8 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - { - let db = Minigraf::open(&path).unwrap(); - for s in setup_statements(i) { - db.execute(&s).unwrap(); - } - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let x = format!(":t/x{i}"); - let marker = format!("two{i}"); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, &x, ":kind", &marker)) - { - assert_eq!( - got, - both(), - "{name} path must return both values after reopen" - ); - } - } -} - -#[test] -fn multi_value_visible_on_every_path_after_wal_replay() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - let stmts = setup_statements(0); - let refs: Vec<&str> = stmts.iter().map(String::as_str).collect(); - run_crashing_child(&path, 1_000_000, &refs, CrashTx::Implicit); - let db = Minigraf::open(&path).unwrap(); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, ":t/x0", ":kind", "two0")) - { - assert_eq!( - got, - both(), - "{name} path must return both values after WAL replay" - ); - } -} - -#[test] -fn batched_retract_of_both_values_hides_both_on_every_path() { - for i in 0..8 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - { - let db = Minigraf::open(&path).unwrap(); - for s in setup_statements(i) { - db.execute(&s).unwrap(); - } - db.checkpoint().unwrap(); - } - { - let db = Minigraf::open(&path).unwrap(); - db.execute(&format!( - "(retract [[:t/x{i} :kind :k/a] [:t/x{i} :kind :k/b]])" - )) - .unwrap(); - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let x = format!(":t/x{i}"); - let marker = format!("two{i}"); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, &x, ":kind", &marker)) - { - assert!( - got.is_empty(), - "{name} path must hide both retracted values" - ); - } - } -} - -/// Retracting exactly the values that were asserted together (same -/// order or not, one transaction or split across several) is not, on its -/// own, a sufficient regression guard for the pre-fix dedup key: the old -/// `(entity, attribute, tx_count, asserted)` key — no value — collapses -/// every fact sharing that tuple down to whichever one sorts first by -/// encoded value bytes. When the retracted set is *exactly* the asserted -/// set, that same smallest-byte member wins the collision on both the -/// assert side and the retract side, so it always cancels itself out -/// correctly and the net result is empty either way — this was verified -/// empirically (reordering the retract list, and splitting the two -/// retracts across separate transactions, both still pass against the -/// pre-fix key). What actually exercises the retract-side collision is a -/// batched retract whose value set is *not* identical to what was -/// asserted: here, `:k/A` (uppercase — encodes to fewer bytes than -/// `:k/a`) is retracted alongside the two real values but was never -/// itself asserted. Under the pre-fix key, `:k/A` wins the three-way -/// collision in the retract batch, and the real retraction records for -/// `:k/a` and `:k/b` are silently dropped — leaving `:k/a` visible. -#[test] -fn batched_retract_with_non_asserted_colliding_value_hides_real_values_on_every_path() { - for i in 0..8 { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("db.graph"); - { - let db = Minigraf::open(&path).unwrap(); - for s in setup_statements(i) { - db.execute(&s).unwrap(); - } - db.checkpoint().unwrap(); - } - { - let db = Minigraf::open(&path).unwrap(); - db.execute(&format!( - "(retract [[:t/x{i} :kind :k/A] [:t/x{i} :kind :k/a] [:t/x{i} :kind :k/b]])" - )) - .unwrap(); - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let x = format!(":t/x{i}"); - let marker = format!("two{i}"); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, &x, ":kind", &marker)) - { - assert!( - got.is_empty(), - "{name} path must hide real values even when the retract batch also \ - contains a never-asserted colliding value" - ); - } - } -} - -/// Same construction as -/// `batched_retract_with_non_asserted_colliding_value_hides_real_values_on_every_path`, -/// checked on the same handle immediately after the retract — no -/// intervening checkpoint or reopen — to cover the pending in-memory -/// index path as well as the on-disk one. -#[test] -fn batched_retract_with_non_asserted_colliding_value_hides_real_values_before_checkpoint() { - let db = Minigraf::in_memory().unwrap(); - for s in setup_statements(0) { - db.execute(&s).unwrap(); - } - db.execute("(retract [[:t/x0 :kind :k/A] [:t/x0 :kind :k/a] [:t/x0 :kind :k/b]])") - .unwrap(); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, ":t/x0", ":kind", "two0")) - { - assert!( - got.is_empty(), - "{name} path must hide real values even when the retract batch also \ - contains a never-asserted colliding value, before checkpoint" - ); - } -} - -/// `tests/fixtures/v7_multivalue.graph` was written by the v7 writer at -/// 84bf375 (v2.0.0 format) with this generator: -/// -/// ```ignore -/// fn main() -> anyhow::Result<()> { -/// use std::path::PathBuf; -/// let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); -/// let tmp = dir.join("v7_multivalue.graph.tmp"); -/// let _ = std::fs::remove_file(&tmp); -/// let _ = std::fs::remove_file(dir.join("v7_multivalue.graph.tmp.wal")); -/// -/// let db = minigraf::Minigraf::open(&tmp)?; -/// let mut facts = vec![ -/// "[:t/x :kind :k/a]".to_string(), -/// "[:t/x :kind :k/b]".to_string(), -/// r#"[:t/x :note "two"]"#.to_string(), -/// ]; -/// for i in 0..30 { -/// facts.push(format!("[:t/f-{i} :kind :k/c]")); -/// facts.push(format!(r#"[:t/f-{i} :note "f"]"#)); -/// } -/// db.execute(&format!("(transact [{}])", facts.join(" ")))?; -/// db.execute("(transact [[:t/y :tag :g/a] [:t/y :tag :g/b]])")?; -/// db.execute("(retract [[:t/y :tag :g/a] [:t/y :tag :g/b]])")?; -/// db.checkpoint()?; -/// drop(db); -/// let _ = std::fs::remove_file(dir.join("v7_multivalue.graph.tmp.wal")); -/// std::fs::rename(&tmp, dir.join("v7_multivalue.graph"))?; -/// Ok(()) -/// } -/// ``` -/// -/// Opening it must migrate to v8 and return both values on every path, and -/// the batched retract of `:t/y`'s two `:tag` values must hide both. -#[test] -fn v7_multivalue_fixture_migrates_and_reads_correctly() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("v7.graph"); - std::fs::write(&path, include_bytes!("fixtures/v7_multivalue.graph")).unwrap(); - - { - let db = Minigraf::open(&path).unwrap(); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, ":t/x", ":kind", "two")) - { - assert_eq!( - got, - both(), - "{name} path must return both values after migration" - ); - } - let tags = rows( - db.execute("(query [:find ?v :where [:t/y :tag ?v]])") - .unwrap(), - ); - assert!( - tags.is_empty(), - "batched retract in the v7 file must hide both values" - ); - } - - let raw = std::fs::read(&path).unwrap(); - assert_eq!( - u32::from_le_bytes(raw[4..8].try_into().unwrap()), - 8, - "file must be v8 after open" - ); - - // Second open is a plain v8 open (no rebuild) and reads the same. - let db = Minigraf::open(&path).unwrap(); - for (name, got) in ["eavt", "aevt", "scan"] - .iter() - .zip(three_paths(&db, ":t/x", ":kind", "two")) - { - assert_eq!( - got, - both(), - "{name} path must return both values on second open" - ); - } -} - -/// Two valid-time stints for the *same* (entity, attribute, value) written in -/// one transaction must both be individually visible: a `:valid-at` query -/// inside the second window's range must return that stint (not the first, -/// not neither), and `:any-valid-time` must return both, on every selective -/// query path. The pre-fix `selective_fact_fetch` dedup key -/// `(entity, attribute, tx_count, asserted, value_bytes)` has no valid-time -/// component, so these two facts — same tx_count (one transact call), same -/// value bytes (`true`), same asserted flag — collapse into a single -/// arbitrary survivor in the entity-bound and attribute+join loops. A full -/// scan does not go through `selective_fact_fetch` (its patterns are not -/// selective — unbound attribute/value), so it is unaffected and serves as -/// the control. -#[test] -fn multi_value_time_stints_visible_at_correct_valid_time_on_every_path() { - let db = Minigraf::in_memory().unwrap(); - // The `:note` marker's valid-time window must cover both stints' windows — - // otherwise the query's `:valid-at "2022-06-01"` filter (applied to every - // candidate fact, including the join marker) would drop `:note` itself, - // since its default valid_from is the real transaction wall-clock time. - db.execute( - r#"(transact [[:vt/alice :employed true {:valid-from "2020-01-01" :valid-to "2021-01-01"}] [:vt/alice :employed true {:valid-from "2022-01-01" :valid-to "2023-01-01"}] [:vt/alice :note "vt" {:valid-from "2000-01-01"}]])"#, - ) - .unwrap(); - - // Inside the second window only: exactly one row on every path. - let eavt = rows( - db.execute(r#"(query [:find ?v :valid-at "2022-06-01" :where [:vt/alice :employed ?v]])"#) - .unwrap(), - ); - assert_eq!( - eavt.len(), - 1, - "entity-bound path must return the stint valid in the second window" - ); - - let aevt = rows( - db.execute( - r#"(query [:find ?v :valid-at "2022-06-01" :where [?e :employed ?v] [?e :note "vt"]])"#, - ) - .unwrap(), - ); - assert_eq!( - aevt.len(), - 1, - "attribute+join path must return the stint valid in the second window" - ); - - let scan = rows( - db.execute( - r#"(query [:find ?a ?v :valid-at "2022-06-01" :where [?e ?a ?v] [?e :note "vt"]])"#, - ) - .unwrap(), - ) - .into_iter() - .filter(|r| matches!(&r[0], Value::Keyword(a) if a == ":employed")) - .count(); - assert_eq!( - scan, 1, - "full scan path must return the stint valid in the second window" - ); - - // Under :any-valid-time, both stints must be visible on the selective paths. - let eavt_any = rows( - db.execute(r#"(query [:find ?v :any-valid-time :where [:vt/alice :employed ?v]])"#) - .unwrap(), - ); - assert_eq!( - eavt_any.len(), - 2, - "entity-bound path must return both stints under :any-valid-time" - ); - - let aevt_any = rows( - db.execute( - r#"(query [:find ?v :any-valid-time :where [?e :employed ?v] [?e :note "vt"]])"#, - ) - .unwrap(), - ); - assert_eq!( - aevt_any.len(), - 2, - "attribute+join path must return both stints under :any-valid-time" - ); -} - -/// Index entries now carry the value bytes; a near-maximum string value must -/// still checkpoint and read back through every index. -#[test] -fn near_max_size_values_checkpoint_and_read_back() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("big.graph"); - let big_a = "a".repeat(3_800); - let big_b = "b".repeat(3_800); - { - let db = Minigraf::open(&path).unwrap(); - db.execute(&format!( - r#"(transact [[:t/big :blob "{big_a}"] [:t/big :blob "{big_b}"] [:t/big :note "big"]])"# - )) - .unwrap(); - db.checkpoint().unwrap(); - } - let db = Minigraf::open(&path).unwrap(); - let by_entity = rows( - db.execute("(query [:find ?v :where [:t/big :blob ?v]])") - .unwrap(), - ); - assert_eq!( - by_entity.len(), - 2, - "EAVT path must return both large values" - ); - let by_attr = rows( - db.execute(r#"(query [:find ?v :where [?e :blob ?v] [?e :note "big"]])"#) - .unwrap(), - ); - assert_eq!(by_attr.len(), 2, "AEVT path must return both large values"); - let by_value = rows( - db.execute(&format!( - r#"(query [:find ?e :where [?e :blob "{big_a}"]])"# - )) - .unwrap(), - ); - assert_eq!(by_value.len(), 1, "AVET path must find the large value"); -} - -/// `near_max_size_values_checkpoint_and_read_back` above leaves roughly 230 -/// bytes of headroom below the real per-fact maximum (`MAX_FACT_BYTES` minus -/// the fixed postcard overhead of the rest of the `Fact` struct, which isn't -/// a public constant). This test locates the exact edge at runtime — the -/// largest value length that still transacts — by searching downward from a -/// length guaranteed to fail, and checkpoints/reads back exactly there. -#[test] -fn max_size_value_at_exact_boundary_checkpoint_and_read_back() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("boundary.graph"); - let db = Minigraf::open(&path).unwrap(); - - // Binary search the largest ASCII string length that still transacts as - // a single fact value, using a throwaway probe entity (and a fill - // character distinct from the real assertions below) so the search - // doesn't interfere with them — the AVET lookup below is by exact value, - // and the search's last successful probe value would otherwise collide - // with `big_a`. Same attribute name (`:blob`) as the real facts below: - // the attribute string is part of the encoded fact size, so a - // differently-sized attribute name would find a different boundary. - // `hi` starts comfortably above MAX_FACT_BYTES (4080), so it is - // guaranteed to fail. - let mut lo = 0usize; - let mut hi = 4096usize; - while lo + 1 < hi { - let mid = lo + (hi - lo) / 2; - let s = "z".repeat(mid); - let stmt = format!(r#"(transact [[:t/probe :blob "{s}"]])"#); - match db.execute(&stmt) { - Ok(_) => lo = mid, - Err(_) => hi = mid, - } - } - let max_len = lo; - assert!( - max_len > 3_800, - "boundary search must find a length past the near-max test's headroom" - ); - - let big_a = "a".repeat(max_len); - let big_b = "b".repeat(max_len); - db.execute(&format!( - r#"(transact [[:t/boundary :blob "{big_a}"] [:t/boundary :blob "{big_b}"] [:t/boundary :note "boundary"]])"# - )) - .unwrap(); - db.checkpoint().unwrap(); - drop(db); - - let db = Minigraf::open(&path).unwrap(); - let by_entity = rows( - db.execute("(query [:find ?v :where [:t/boundary :blob ?v]])") - .unwrap(), - ); - assert_eq!( - by_entity.len(), - 2, - "EAVT path must return both exact-boundary values" - ); - let by_attr = rows( - db.execute(r#"(query [:find ?v :where [?e :blob ?v] [?e :note "boundary"]])"#) - .unwrap(), - ); - assert_eq!( - by_attr.len(), - 2, - "AEVT path must return both exact-boundary values" - ); - let by_value = rows( - db.execute(&format!( - r#"(query [:find ?e :where [?e :blob "{big_a}"]])"# - )) - .unwrap(), - ); - assert_eq!( - by_value.len(), - 1, - "AVET path must find the exact-boundary value" - ); - - // One byte past the boundary must fail cleanly, confirming max_len is - // really the edge and not just some length that happens to work. - let over = "a".repeat(max_len + 1); - let over_result = db.execute(&format!(r#"(transact [[:t/over :blob "{over}"]])"#)); - assert!( - over_result.is_err(), - "one byte past the boundary must be rejected" - ); -} diff --git a/tests/wal_test.rs b/tests/wal_test.rs index 58d8bd3036..02cdc4a206 100644 --- a/tests/wal_test.rs +++ b/tests/wal_test.rs @@ -9,7 +9,7 @@ //! - Auto-checkpoint threshold //! - Explicit transaction commit and rollback //! - Concurrent reads while writer holds the write lock -//! - Pre-v7 file formats (e.g. V2) are rejected, not silently upgraded +//! - V2 → V3 file format upgrade on first checkpoint #![cfg(not(target_arch = "wasm32"))] use minigraf::QueryResult; @@ -720,13 +720,13 @@ fn write_state_clean_after_drop() { assert_eq!(n, 1, "only committed fact visible after dropped tx"); } -// ── 12. V2 file is rejected ────────────────────────────────────────────────── +// ── 12. V2 file upgrades to V3 on checkpoint ───────────────────────────────── -/// v3.0.0 dropped support for formats v1–v6. Create a v2-format `.graph` -/// file manually (version field = 2) and verify that opening it fails with -/// STG-028 rather than silently upgrading it. +/// Create a v2-format `.graph` file manually (version field = 2, no +/// `last_checkpointed_tx_count`), open it with `Minigraf`, write a fact, +/// checkpoint, then read the raw header and verify it is now v3. #[test] -fn test_v2_file_is_rejected() { +fn test_v2_file_opens_and_upgrades_to_v3_on_checkpoint() { use std::io::Write; let dir = tempfile::tempdir().unwrap(); @@ -734,6 +734,14 @@ fn test_v2_file_is_rejected() { // ── Build a minimal v2 `.graph` file ────────────────────────────────── // + // V2 and V3 have identical binary layouts. + // The only difference is the version field (bytes 4-7): 2 vs 3. + // In V2, bytes 24-31 were the unused `edge_count` field (always 0). + // Phase 5 repurposed that slot as `last_checkpointed_tx_count` without + // changing the wire layout. Opening a V2 file with Phase 5 code works + // transparently because `last_checkpointed_tx_count` reads as 0 from + // the old `edge_count` slot. + // // The file contains exactly 1 page (the header page, 4096 bytes). { let mut page = vec![0u8; PAGE_SIZE]; @@ -743,15 +751,39 @@ fn test_v2_file_is_rejected() { page[4..8].copy_from_slice(&2u32.to_le_bytes()); // page_count = 1 page[8..16].copy_from_slice(&1u64.to_le_bytes()); + // node_count = 0 (bytes 16..24 already zero) + // last_checkpointed_tx_count = 0 (bytes 24..32 already zero) + // reserved = 0 (bytes 32..64 already zero) let mut file = std::fs::File::create(&db_path).unwrap(); file.write_all(&page).unwrap(); file.sync_all().unwrap(); } - let err = match Minigraf::open(&db_path) { - Ok(_) => panic!("v2 file must not open"), - Err(e) => e, - }; - assert_eq!(err.code(), "STG-028", "v2 file must fail with STG-028"); + // ── Open, write a fact, and checkpoint ──────────────────────────────── + { + let db = Minigraf::open(&db_path).unwrap(); + db.execute(r#"(transact [[:alice :name "Alice"]])"#) + .unwrap(); + db.checkpoint().unwrap(); + // Drop runs another checkpoint, but that's idempotent. + } + + // ── Read the raw header and assert version = 7 ─────────────────────── + // Reads raw bytes: magic at 0..4, version at 4..8 (u32 LE), + // last_checkpointed_tx_count at 24..32 (u64 LE). + let raw = std::fs::read(&db_path).unwrap(); + assert!( + raw.len() >= PAGE_SIZE, + "file must be at least one page after checkpoint" + ); + let magic = &raw[0..4]; + let version = u32::from_le_bytes(raw[4..8].try_into().unwrap()); + let last_checkpointed_tx_count = u64::from_le_bytes(raw[24..32].try_into().unwrap()); + assert_eq!(version, 7, "file must be upgraded to v7 on checkpoint"); + assert_eq!(magic, b"MGRF", "magic number must be preserved"); + assert!( + last_checkpointed_tx_count > 0, + "last_checkpointed_tx_count must be set after checkpoint on v2→v6 upgrade" + ); } From ab653506b54b79ae2add16171139ecf523368b6a Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Fri, 25 Sep 2026 10:46:34 +0530 Subject: [PATCH 02/10] =?UTF-8?q?release:=20v2.0.1=20=E2=80=94=20fix=20ind?= =?UTF-8?q?ex=20rebuild=20on=20open=20(#370)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch release on the v2.x line; file format unchanged (v7), no API changes. Ships #372: the index rebuild on open now uses each fact's real on-disk location, so entity-bound lookups no longer come back empty after a crash-triggered rebuild of a multi-checkpoint file. Doc sync: CHANGELOG (v2.0.1 entry + known issues #371/#287, #373/#374), CLAUDE.md, README.md, ROADMAP.md (v2.x on main, v3.0.0 on the `v3` branch), docs/TEST_COVERAGE.md (1154 tests). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy --- CHANGELOG.md | 13 +++++++++++++ CLAUDE.md | 4 ++-- Cargo.toml | 2 +- README.md | 4 ++-- ROADMAP.md | 5 +++-- docs/TEST_COVERAGE.md | 8 ++++---- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7484edbc60..1da1272b15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v2.0.1 — 2026-09-25 + +Patch release on the v2.x line. File format is unchanged (v7); no API changes. + +### Fixed + +- **Entity-bound lookups could silently return nothing after an index rebuild on open (#370).** When the index checksum does not match on open (for example after a process was killed during `save()`), Minigraf rebuilds all four indexes from the fact pages. The rebuild computed each fact's location by re-packing all facts contiguously, but a file written by several checkpoints has partially filled pages that a contiguous re-pack does not reproduce. From the second checkpoint onward, rebuilt index entries pointed at the wrong page/slot: entity-bound queries (`[:some/ident ?a ?v]`) returned `[]` or errored while attribute-driven queries still worked, and `checkpoint()` copied the bad entries forward. The rebuild now reads each fact's real on-disk location (#372). + +### Known issues + +- Files whose indexes were already rebuilt with wrong locations by v2.0.0 are not repaired by this release. A public integrity check and rebuild-indexes-from-fact-pages operation is tracked in #373; a crash-atomic `save()` in #374. +- Two values of the same attribute for one entity written in a single `transact` (or retracted in a single `retract`) can read back as one value, differently per query path (#371, #287). The fix changes the index key layout (file format v8) and ships in v3.0.0. Workaround on v2.x: write or retract each value of a multi-valued attribute in its own call. + ## v2.0.0 — 2026-08-26 This was originally slated as v1.3.0, matching the GitHub milestone name under diff --git a/CLAUDE.md b/CLAUDE.md index 2b83e26fa9..bcec61ef7f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -167,14 +167,14 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1153 tests passing** (1145 passing, 8 ignored; unit + integration + doc). +**1154 tests passing** (1146 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. ## Current Maintainer Context -**v2.0.0 released** — it introduced kernel file locking (#317, #304), structured runtime error codes (#277), `OpenOptions` `#[non_exhaustive]`, and an MSRV of Rust 1.89. See `CHANGELOG.md` for the full rationale and release history. +**v2.0.1 released** (2026-09-25) — patch fixing index rebuild on open (#370). **v2.0.0** introduced kernel file locking (#317, #304), structured runtime error codes (#277), `OpenOptions` `#[non_exhaustive]`, and an MSRV of Rust 1.89. See `CHANGELOG.md` for the full rationale and release history. Relevant areas for the planned query-profiler work (see `ROADMAP.md`): - `src/query/datalog/` — query executor where profiling hooks will go (#185) diff --git a/Cargo.toml b/Cargo.toml index b9533f3916..480c6b9bf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [package] name = "minigraf" -version = "2.0.0" +version = "2.0.1" edition = "2024" rust-version = "1.89" description = "Zero-config, single-file, embedded graph database with bi-temporal Datalog queries" diff --git a/README.md b/README.md index 82414ce6dd..7e2a6e24ad 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Minigraf is a **single-file embedded graph database** that lets you: ```toml [dependencies] -minigraf = "2.0.0" +minigraf = "2.0.1" ``` Or via cargo: @@ -92,7 +92,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1153 tests +cargo test # run 1154 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/ROADMAP.md b/ROADMAP.md index 1876689185..45b27a6b23 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -105,7 +105,8 @@ When evaluating features, ask: ## Current Focus -- The v3.0.0 file-format policy and its scoped work above +- v2.x minor releases (v2.1.0–v2.3.0) on `main` +- The v3.0.0 file-format policy and its scoped work above, on the `v3` branch - Ecosystem work tracked in [`minigraf-examples`](https://github.com/project-minigraf/minigraf-examples) - Developer tools tracked in `minigraf-inspector` and `minigraf-visualizer` @@ -113,4 +114,4 @@ See [GitHub Issues](https://github.com/project-minigraf/minigraf/issues) for spe --- -**Last updated**: August 2026 — v2.0.0 is the current release; v3.0.0 is the next core milestone described here. +**Last updated**: September 2026 — v2.0.1 is the current release. v2.x minor releases (v2.1.0–v2.3.0) ship from `main` first; v3.0.0 work, including the file-format v8 fix for #287/#371, is developed on the long-lived `v3` branch and merges into `main` at the v3.0.0 cut. diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 2ccaaadcec..3e5a9a8739 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -1,12 +1,12 @@ # Minigraf Test Coverage -## v2.0.0 Coverage Summary +## v2.0.1 Coverage Summary -**Verified**: 2026-08-26 with `cargo test --quiet` +**Verified**: 2026-09-25 with `cargo test` -**Result**: 1,145 passing tests and 8 ignored tests (1,153 total) +**Result**: 1,146 passing tests and 8 ignored tests (1,154 total) -The v2.0.0 suite covers the public database API, the Datalog engine, storage and recovery paths, and the release’s locking and structured-error-code changes. +The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). ### Covered Areas From 402790dfa183efa9a489c86b0a9c3d652574bfd6 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 09:51:12 +0530 Subject: [PATCH 03/10] perf: narrow bound-entity point queries to the queried attribute (#323 mitigation) (#380) * docs: design spec for point-query history scan mitigation (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * docs: implementation plan for #323; spec: drop dedup, threshold collapse Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: add in-crate FxHash hasher for hot-path maps (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: borrow keys and use FxHash in net_asserted_facts (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: add (entity, attribute) EAVT range lookup (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: narrow bound-entity fetch to queried attributes, drop redundant dedup (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: rotate FxHasher output so hashbrown's low-bit bucket index is mixed (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * bench: point_query_chain_depth (#323) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * docs: changelog, benchmarks and test counts for #323; revise spec acceptance to measured Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * fix: bound (entity, attribute) EAVT scan exactly; pin same-tx multi-window fix (#323) End the committed range at attribute + NUL (exact successor) instead of incrementing the last byte, which returned None for attributes ending in 0x7F or a 0xBF UTF-8 byte and left the scan unbounded to the end of the index. Also stops resolving prefix siblings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: keep std's keyed hasher in net_asserted_facts; drop FxHash (#323) Fact values are often untrusted text (agent memory); a fixed-seed fast hash would let crafted colliding values make queries quadratic. The borrowed-key / encode-once rewrite stays. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * docs: final #323 benchmark numbers (std hasher, AC) and test counts Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 14 + CLAUDE.md | 4 +- README.md | 2 +- benches/minigraf_bench.rs | 51 +- docs/BENCHMARKS.md | 20 + docs/TEST_COVERAGE.md | 6 +- .../2026-09-25-point-query-history-scan.md | 1096 +++++++++++++++++ ...6-09-25-point-query-history-scan-design.md | 184 +++ src/graph/storage.rs | 561 ++++++++- src/query/datalog/executor.rs | 110 +- tests/point_query_history_test.rs | 209 ++++ 11 files changed, 2150 insertions(+), 107 deletions(-) create mode 100644 docs/superpowers/plans/2026-09-25-point-query-history-scan.md create mode 100644 docs/superpowers/specs/2026-09-25-point-query-history-scan-design.md create mode 100644 tests/point_query_history_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1da1272b15..f5ad6d863e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Performance + +- **Bound-entity point queries no longer pay for other attributes' history (#323).** `[:e :attr ?v]` now range-scans only `(e, :attr)` in the EAVT index instead of every record the entity has ever written. Reading a rarely changed attribute of a heavily rewritten entity no longer slows down as that entity's history grows: with 2,000 retract/reassert cycles on another attribute of the same entity, it drops from 4.58 ms to 19.0 µs. Reading the heavily rewritten attribute itself, and attribute scans (`[?e :attr ?v]`), are about 1.35–1.4× faster (4.64 ms → 3.36 ms), from cheaper net-assert grouping and removing a redundant dedup pass. The file format is unchanged. + +### Fixed + +- **Bound-entity queries could drop rows when one `WriteTransaction` wrote the same attribute in several valid-time windows (#323).** Both facts share a `tx_count`, and the selective lookup path de-duplicated on `(entity, attribute, tx_count, asserted)`, so one window's value was silently lost, while the same query via a full scan returned both. That de-duplication has been removed, and bound-entity queries now always match a full scan. + +### Notes + +- On v2.x, reading a heavily rewritten attribute still costs time proportional to its history, because v7 index keys carry neither the value nor the assert/retract flag; the structural fix needs the v8 keys and is tracked for v3.0.0 in #379. + ## v2.0.1 — 2026-09-25 Patch release on the v2.x line. File format is unchanged (v7); no API changes. diff --git a/CLAUDE.md b/CLAUDE.md index bcec61ef7f..f0359ec370 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,7 +101,7 @@ cargo run < demos/demo_negation.txt 1. **`src/graph/`** — EAV fact store with bi-temporal support - `types.rs`: `Fact`, `Value`, `EntityId`, `TxId`, `VALID_TIME_FOREVER` - - `storage.rs`: `FactStorage` — in-memory store, `transact_batch`, `retract`, `get_facts_as_of`, `get_facts_valid_at`, `net_asserted_facts` + - `storage.rs`: `FactStorage` — in-memory store, `transact_batch`, `retract`, `get_facts_as_of`, `get_facts_valid_at`, `net_asserted_facts`, `get_facts_by_entity_attribute_indexed` 2. **`src/storage/`** — Persistence layer - `mod.rs`: `StorageBackend` trait, `FileHeader` v7 (84 bytes), `CommittedFactReader` / `CommittedIndexReader` traits @@ -167,7 +167,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1154 tests passing** (1146 passing, 8 ignored; unit + integration + doc). +**1173 tests passing** (1165 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 7e2a6e24ad..93af3e1e0f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1154 tests +cargo test # run 1173 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/benches/minigraf_bench.rs b/benches/minigraf_bench.rs index e8f27bc8d8..d4b901a7c1 100644 --- a/benches/minigraf_bench.rs +++ b/benches/minigraf_bench.rs @@ -1582,6 +1582,54 @@ fn bench_btree_lookup(c: &mut Criterion) { } } +// ── point_query_chain_depth (Issue #323) ───────────────────────────────────── +// +// One entity whose :hash is retracted/reasserted `depth` times (exactly one live +// value), plus a never-churned :other on the same entity; checkpointed file DB. + +fn bench_point_query_chain_depth(c: &mut Criterion) { + const DEPTHS: &[usize] = &[1, 500, 2000]; + const QUERIES: &[(&str, &str)] = &[ + ( + "churned_attr", + "(query [:find ?v :where [:e/hot :hash ?v]])", + ), + ( + "sibling_attr", + "(query [:find ?v :where [:e/hot :other ?v]])", + ), + ("attr_scan", "(query [:find ?v :where [?e :hash ?v]])"), + ]; + for &(name, q) in QUERIES { + let mut group = c.benchmark_group(format!("point_query_chain_depth/{name}")); + group.sample_size(20); + for &depth in DEPTHS { + let dir = tempfile::tempdir().unwrap(); + let db = minigraf::Minigraf::open(dir.path().join("b.graph")).unwrap(); + for i in 0..2000 { + db.execute(&format!("(transact [[:f/{i} :x {i}]])")) + .unwrap(); + } + db.execute("(transact [[:e/hot :other \"o\"]])").unwrap(); + db.execute("(transact {:valid-from \"2020-01-01T00:00:00Z\"} [[:e/hot :hash \"h0\"]])") + .unwrap(); + for i in 1..depth { + db.execute(&format!("(retract [[:e/hot :hash \"h{}\"]])", i - 1)) + .unwrap(); + db.execute(&format!( + "(transact {{:valid-from \"2020-01-01T00:00:00Z\"}} [[:e/hot :hash \"h{i}\"]])" + )) + .unwrap(); + } + db.checkpoint().unwrap(); + group.bench_with_input(BenchmarkId::from_parameter(depth), &depth, |b, _| { + b.iter(|| black_box(db.execute(q).unwrap())); + }); + } + group.finish(); + } +} + // ── query/predicate_pushdown ────────────────────────────────────────────────── fn bench_predicate_pushdown(c: &mut Criterion) { @@ -1752,6 +1800,7 @@ criterion_group!( bench_retract, bench_btree_lookup, bench_predicate_pushdown, - bench_simd, // Issue #229 + bench_point_query_chain_depth, // Issue #323 + bench_simd, // Issue #229 ); criterion_main!(benches); diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index af74770b4e..8b13948d99 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -30,6 +30,26 @@ Each value is Criterion's estimated per-query latency; the range is its 95% conf `point_entity` uses a selective index-backed lookup. `point_attribute` and `join_3pattern` return larger result sets and therefore scale with the number of matching facts. Do not compare these local measurements directly with CI runs or earlier releases: host load, CPU-frequency policy, toolchain, and implementation all affect the result. +### Point Query vs. Version-Chain Depth (#323) + +**Date**: 2026-09-26 · **Command**: `cargo bench --bench minigraf_bench -- point_query_chain_depth` · same host as above, on AC power and otherwise idle. + +One entity's `:hash` is retracted and reasserted `depth` times (exactly one live value), plus a never-changed `:other` on the same entity and 2,000 filler facts; checkpointed file database. "Before" is v2.0.1; "after" is the #323 change. + +| Query | Depth | Before | After | +|---|---:|---:|---:| +| `[:e/hot :hash ?v]` (churned attribute) | 1 | 19.5 µs | 18.3 µs | +| | 500 | 1.18 ms | 855 µs | +| | 2000 | 4.64 ms | 3.36 ms | +| `[:e/hot :other ?v]` (sibling attribute) | 1 | 20.0 µs | 19.2 µs | +| | 500 | 1.17 ms | 20.6 µs | +| | 2000 | 4.58 ms | 19.0 µs | +| `[?e :hash ?v]` (attribute scan) | 1 | 17.5 µs | 17.8 µs | +| | 500 | 1.14 ms | 841 µs | +| | 2000 | 4.49 ms | 3.33 ms | + +The churned attribute still scales with its own history on v2.x; see #379 for the v3.0.0 fix. + ## Measurement Method Criterion warms up each benchmark, collects ten samples for this query group, and estimates per-call latency from repeated iterations. The reported confidence intervals describe measurement uncertainty on this host; they are not cross-machine performance guarantees. diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 3e5a9a8739..86fd20e154 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -2,11 +2,11 @@ ## v2.0.1 Coverage Summary -**Verified**: 2026-09-25 with `cargo test` +**Verified**: 2026-09-26 with `cargo test` -**Result**: 1,146 passing tests and 8 ignored tests (1,154 total) +**Result**: 1,165 passing tests and 8 ignored tests (1,173 total) -The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). +The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). ### Covered Areas diff --git a/docs/superpowers/plans/2026-09-25-point-query-history-scan.md b/docs/superpowers/plans/2026-09-25-point-query-history-scan.md new file mode 100644 index 0000000000..cc44585e10 --- /dev/null +++ b/docs/superpowers/plans/2026-09-25-point-query-history-scan.md @@ -0,0 +1,1096 @@ +# Point-Query History Scan Mitigation (#323) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop bound-entity point queries from paying for other attributes' history, and cut the per-history-record constant on the churned attribute itself by ≥3×, without changing file format, API, or results. + +**Architecture:** (1) an in-crate FxHash hasher; (2) `net_asserted_facts` rewritten to encode each value once, borrow keys, and use FxHash; (3) a new EAVT `(entity, attribute)` range lookup in `FactStorage`; (4) `selective_fact_fetch` narrows bound-entity lookups to the attributes the query references and drops its redundant dedup set. + +**Tech Stack:** Rust 2024 edition, MSRV 1.89, criterion benches, cargo test. + +**Spec:** `docs/superpowers/specs/2026-09-25-point-query-history-scan-design.md` + +## Global Constraints + +- No file-format change (stays v7), no public API change, no new dependencies (runtime or dev). +- Clippy workspace lints deny `unwrap_used`, `expect_used`, `panic`, `indexing_slicing`, `cast_possible_truncation`, `cast_possible_wrap`, `cast_sign_loss` in non-test code. Tests are exempt (`src/lib.rs` `cfg_attr(test, allow(...))`; integration tests are separate crates, where these lints still apply only as configured — existing tests use `.unwrap()` freely). +- Never use `{:?}` of `Result`/`Fact`/`Value`/`EdnValue`/UUID-bearing types in `assert!`/`assert_eq!` message strings (CodeQL `rust/cleartext-logging`). `{:?}` elsewhere (e.g. as a sort key) is fine. +- `#![forbid(unsafe_code)]` — no `unsafe`. +- Query results must be identical to before for every query. +- Work in `.worktrees/perf-323-point-query` on branch `perf/323-point-query-history-scan`. `cargo test --release` is broken by design (panic=abort); use `cargo test` and `cargo bench`. +- Commit messages end with: + ``` + Co-Authored-By: Claude Opus 5.5 + Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy + ``` + Commits must reference `#323` but never use a closing keyword for it until the final PR body. + +## Review Focus + +1. **Attribute prefix collision** (`:a` vs `:ab`, `:hash` vs `:hash2`) on a bound entity — only the exact attribute's facts may be returned, in pending *and* committed indexes. → Task 3 test `entity_attribute_indexed_excludes_prefix_sibling`, Task 4 test `prefix_sibling_attribute_not_leaked`. +2. **Bound entity whose query also uses a variable or pseudo-attribute** (`[:e ?a ?v]`, `[:e :db/valid-from ?vf]`) — must still see all of the entity's facts. → Task 4 tests `bound_entity_variable_attribute_sees_all`, `bound_entity_pseudo_attribute_matches_full_scan`. +3. **Many attributes on one bound entity (> threshold)** — must not regress to a full scan, and results must be correct. → Task 4 test `five_attributes_on_one_entity`. +4. **Facts visible in both pending and committed indexes / duplicated input records** — `net_asserted_facts` must be idempotent under duplicates since the executor dedup is removed. → Task 2 test `net_asserted_idempotent_under_duplicates`. +5. **Same-transaction assert + retract, and retraction followed by reassertion of the same value** across a checkpoint boundary. → Task 2 randomized equivalence test; Task 4 test `churned_attribute_live_value_file_backed`. + +--- + +## File Structure + +| File | Change | Responsibility | +|---|---|---| +| `src/graph/fxhash.rs` | Create | `FxHasher`, `FxBuildHasher` — fast non-cryptographic hasher | +| `src/graph/mod.rs` | Modify | register `pub(crate) mod fxhash;` | +| `src/graph/storage.rs` | Modify | rewrite `net_asserted_facts`; add `get_facts_by_entity_attribute_indexed`; tests | +| `src/query/datalog/executor.rs` | Modify | `selective_fact_fetch` narrowing + dedup removal | +| `tests/point_query_history_test.rs` | Create | end-to-end result-equivalence tests | +| `benches/minigraf_bench.rs` | Modify | `point_query_chain_depth` group | +| `docs/BENCHMARKS.md`, `CHANGELOG.md`, `docs/TEST_COVERAGE.md`, `CLAUDE.md` | Modify | doc sync | + +--- + +### Task 1: FxHash hasher + +**Files:** +- Create: `src/graph/fxhash.rs` +- Modify: `src/graph/mod.rs` + +**Interfaces:** +- Produces: `crate::graph::fxhash::FxBuildHasher` (type alias `BuildHasherDefault`), used as `HashMap` / `HashSet` created with `::default()`. + +- [ ] **Step 1: Write the module with its failing-by-absence tests** + +`src/graph/mod.rs` becomes: +```rust +pub(crate) mod fxhash; +pub(crate) mod storage; +pub(crate) mod types; +``` + +`src/graph/fxhash.rs`: +```rust +//! Small non-cryptographic hasher for internal hot-path hash maps (#323). +//! +//! Same multiply-rotate scheme as rustc's `FxHasher`. Not HashDoS-resistant; +//! only use it for maps keyed by data the embedding application already +//! controls (facts in its own database), never for untrusted network input. + +use std::hash::{BuildHasherDefault, Hasher}; + +const SEED: u64 = 0x51_7c_c1_b7_27_22_0a_95; + +/// Word-at-a-time multiply-rotate hasher. +#[derive(Default, Clone, Copy)] +pub(crate) struct FxHasher { + hash: u64, +} + +impl FxHasher { + #[inline] + fn add_to_hash(&mut self, word: u64) { + self.hash = (self.hash.rotate_left(5) ^ word).wrapping_mul(SEED); + } +} + +impl Hasher for FxHasher { + #[inline] + fn write(&mut self, bytes: &[u8]) { + let mut chunks = bytes.chunks_exact(8); + for chunk in &mut chunks { + let mut buf = [0u8; 8]; + buf.copy_from_slice(chunk); + self.add_to_hash(u64::from_le_bytes(buf)); + } + let rem = chunks.remainder(); + if !rem.is_empty() { + let mut buf = [0u8; 8]; + for (dst, src) in buf.iter_mut().zip(rem) { + *dst = *src; + } + self.add_to_hash(u64::from_le_bytes(buf)); + } + } + + #[inline] + fn write_u8(&mut self, i: u8) { + self.add_to_hash(u64::from(i)); + } + + #[inline] + fn write_u16(&mut self, i: u16) { + self.add_to_hash(u64::from(i)); + } + + #[inline] + fn write_u32(&mut self, i: u32) { + self.add_to_hash(u64::from(i)); + } + + #[inline] + fn write_u64(&mut self, i: u64) { + self.add_to_hash(i); + } + + #[inline] + fn finish(&self) -> u64 { + self.hash + } +} + +/// `BuildHasher` for [`FxHasher`]; use with `HashMap::default()`. +pub(crate) type FxBuildHasher = BuildHasherDefault; + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::{HashMap, HashSet}; + use std::hash::BuildHasher; + + #[test] + fn equal_inputs_hash_equal() { + let b = FxBuildHasher::default(); + assert_eq!(b.hash_one(("abc", 1u64)), b.hash_one(("abc", 1u64))); + } + + #[test] + fn remainder_bytes_affect_hash() { + let b = FxBuildHasher::default(); + // 9 bytes: one full word + 1 remainder byte that differs. + assert_ne!(b.hash_one(b"abcdefghX"), b.hash_one(b"abcdefghY")); + assert_ne!(b.hash_one(":a"), b.hash_one(":ab")); + } + + #[test] + fn usable_as_map_and_set_hasher() { + let mut m: HashMap<(&str, u64), u64, FxBuildHasher> = HashMap::default(); + m.insert((":x", 1), 10); + m.insert((":x", 2), 20); + assert_eq!(m.get(&(":x", 2)).copied(), Some(20)); + let s: HashSet = (0..1000).collect(); + assert_eq!(s.len(), 1000); + } +} +``` + +- [ ] **Step 2: Run tests** + +Run: `cargo test --lib graph::fxhash` +Expected: 3 passed. (Until Task 2 uses it, non-test builds warn `dead_code` for `FxBuildHasher`; that is resolved in Task 2 — do not add `#[allow(dead_code)]`.) + +- [ ] **Step 3: Commit** +```bash +git add src/graph/fxhash.rs src/graph/mod.rs +git commit -m "perf: add in-crate FxHash hasher for hot-path maps (#323)" +``` + +--- + +### Task 2: Faster `net_asserted_facts` + +**Files:** +- Modify: `src/graph/storage.rs` (function at ~line 528; tests module at ~line 827) + +**Interfaces:** +- Consumes: `crate::graph::fxhash::FxBuildHasher` (Task 1). +- Produces: `pub(crate) fn net_asserted_facts(facts: Vec) -> Vec` — same signature and semantics; output now preserves input order of surviving facts. Idempotent under duplicated input records (Task 4 relies on this). + +- [ ] **Step 1: Move the current implementation into the test module as the reference oracle** + +Cut the current body of `net_asserted_facts` and paste it into `mod tests` as: +```rust + /// Pre-#323 implementation, kept verbatim as the oracle for the + /// randomized equivalence test. + fn net_asserted_facts_reference(facts: Vec) -> Vec { + use std::collections::HashMap; + + type EavKey = (EntityId, Attribute, Vec); + type WindowKey = (EntityId, Attribute, Vec, i64, i64); + + let mut max_retract_tx: HashMap = HashMap::new(); + let mut by_window: HashMap = HashMap::new(); + + for fact in facts { + let eav_key = ( + fact.entity, + fact.attribute.clone(), + encode_value(&fact.value), + ); + + if fact.asserted { + let window_key = ( + eav_key.0, + eav_key.1, + eav_key.2, + fact.valid_from, + fact.valid_to, + ); + match by_window.get(&window_key) { + None => { + by_window.insert(window_key, fact); + } + Some(existing) if fact.tx_count > existing.tx_count => { + by_window.insert(window_key, fact); + } + _ => {} + } + } else { + let tx_count = fact.tx_count; + max_retract_tx + .entry(eav_key) + .and_modify(|max_tx| *max_tx = (*max_tx).max(tx_count)) + .or_insert(tx_count); + } + } + + by_window + .into_iter() + .filter_map(|((entity, attribute, value, _, _), fact)| { + let retract_tx = max_retract_tx + .get(&(entity, attribute, value)) + .copied() + .unwrap_or(0); + (fact.tx_count > retract_tx).then_some(fact) + }) + .collect() + } +``` +(Copy it from the file rather than from here if they differ — the file is the source of truth.) + +- [ ] **Step 2: Add the new tests to `mod tests`** + +```rust + /// Order-independent comparison key for a fact set. + fn sorted_keys(facts: &[Fact]) -> Vec { + let mut v: Vec = facts + .iter() + .map(|f| { + format!( + "{}|{}|{:?}|{}|{}|{}|{}", + f.entity, f.attribute, f.value, f.tx_count, f.valid_from, f.valid_to, f.asserted + ) + }) + .collect(); + v.sort(); + v + } + + /// Deterministic xorshift so the test needs no RNG dependency. + struct XorShift(u64); + impl XorShift { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + } + + #[test] + fn net_asserted_matches_reference_on_random_histories() { + let entities = [uuid::Uuid::from_u128(1), uuid::Uuid::from_u128(2)]; + let attrs = [":a", ":ab", ":b"]; + let windows = [ + (0_i64, VALID_TIME_FOREVER), + (1_000, 2_000), + (1_500, VALID_TIME_FOREVER), + ]; + let mut rng = XorShift(0x9E37_79B9_7F4A_7C15); + for _case in 0..500 { + let n = rng.below(40) + 1; + let mut facts = Vec::new(); + let mut tx = 0_u64; + for _ in 0..n { + // ~25% of records share the previous tx_count (same-transaction batches). + if rng.below(4) != 0 { + tx += 1; + } + let e = entities[rng.below(2) as usize]; + let a = attrs[rng.below(3) as usize]; + let v = Value::Integer(i64::try_from(rng.below(3)).unwrap()); + if rng.below(3) == 0 { + facts.push(make_retract(e, a, v, tx)); + } else { + let (vf, vt) = windows[rng.below(3) as usize]; + facts.push(make_assert(e, a, v, tx, vf, vt)); + } + } + let expected = sorted_keys(&net_asserted_facts_reference(facts.clone())); + let actual = sorted_keys(&net_asserted_facts(facts)); + assert_eq!(actual, expected, "new net_asserted_facts diverged from reference"); + } + } + + /// The executor's selective fetch no longer dedups (#323); it relies on + /// net_asserted_facts collapsing duplicated input records. + #[test] + fn net_asserted_idempotent_under_duplicates() { + let e = uuid::Uuid::from_u128(7); + let facts = vec![ + make_assert(e, ":hash", Value::String("h0".into()), 1, 0, VALID_TIME_FOREVER), + make_retract(e, ":hash", Value::String("h0".into()), 2), + make_assert(e, ":hash", Value::String("h1".into()), 3, 0, VALID_TIME_FOREVER), + make_assert(e, ":other", Value::String("o".into()), 3, 0, VALID_TIME_FOREVER), + ]; + let mut doubled = facts.clone(); + doubled.extend(facts.clone()); + let once = sorted_keys(&net_asserted_facts(facts)); + let twice = sorted_keys(&net_asserted_facts(doubled)); + assert_eq!(once.len(), 2, "h1 and o are live"); + assert_eq!(twice, once, "duplicated records must not change the result"); + } + + #[test] + fn net_asserted_preserves_input_order() { + let e = uuid::Uuid::from_u128(9); + let facts = vec![ + make_assert(e, ":z", Value::Integer(1), 1, 0, VALID_TIME_FOREVER), + make_assert(e, ":a", Value::Integer(2), 2, 0, VALID_TIME_FOREVER), + make_assert(e, ":m", Value::Integer(3), 3, 0, VALID_TIME_FOREVER), + ]; + let out = net_asserted_facts(facts); + let attrs: Vec<&str> = out.iter().map(|f| f.attribute.as_str()).collect(); + assert_eq!(attrs, vec![":z", ":a", ":m"]); + } +``` + +- [ ] **Step 3: Run the new tests against the unchanged production function** + +At this point the production `net_asserted_facts` still has the old body (Step 1 *copied* it into the test module; do not delete it from production yet). +Run: `cargo test --lib graph::storage::tests::net_asserted` +Expected: `net_asserted_preserves_input_order` FAILS (old code returns HashMap order); `net_asserted_matches_reference_on_random_histories` and `net_asserted_idempotent_under_duplicates` PASS. + +- [ ] **Step 4: Replace the production body** + +Replace the whole `pub(crate) fn net_asserted_facts` body (keep the existing doc comment, but replace its `# Implementation note` section with the text below): +```rust +/// # Implementation note +/// +/// Hot path for every non-`:as-of` query (#323). Each value is encoded once; +/// the two group maps borrow `(entity, attribute, value_bytes)` from the input +/// instead of cloning them, and use [`FxBuildHasher`]. `by_window` stores the +/// index and `tx_count` of the winning assertion per validity window; survivors +/// are moved out of `facts` at the end, preserving input order. +/// +/// Idempotent under duplicated input records (a duplicate assertion ties with +/// the original and loses; a duplicate retraction leaves the max unchanged). +/// `selective_fact_fetch` relies on this instead of deduplicating. +pub(crate) fn net_asserted_facts(facts: Vec) -> Vec { + use crate::graph::fxhash::FxBuildHasher; + use std::collections::HashMap; + + type EavKey<'a> = (&'a EntityId, &'a str, &'a [u8]); + type WindowKey<'a> = (&'a EntityId, &'a str, &'a [u8], i64, i64); + + let encoded: Vec> = facts.iter().map(|f| encode_value(&f.value)).collect(); + let mut keep = vec![false; facts.len()]; + + { + let mut max_retract_tx: HashMap, u64, FxBuildHasher> = HashMap::default(); + let mut by_window: HashMap, (usize, u64), FxBuildHasher> = + HashMap::default(); + + for (idx, (fact, value_bytes)) in facts.iter().zip(encoded.iter()).enumerate() { + let entity = &fact.entity; + let attribute = fact.attribute.as_str(); + let value = value_bytes.as_slice(); + if fact.asserted { + by_window + .entry((entity, attribute, value, fact.valid_from, fact.valid_to)) + .and_modify(|winner| { + if fact.tx_count > winner.1 { + *winner = (idx, fact.tx_count); + } + }) + .or_insert((idx, fact.tx_count)); + } else { + max_retract_tx + .entry((entity, attribute, value)) + .and_modify(|max_tx| *max_tx = (*max_tx).max(fact.tx_count)) + .or_insert(fact.tx_count); + } + } + + for ((entity, attribute, value, _, _), (idx, tx_count)) in &by_window { + let retract_tx = max_retract_tx + .get(&(*entity, *attribute, *value)) + .copied() + .unwrap_or(0); + if *tx_count > retract_tx + && let Some(slot) = keep.get_mut(*idx) + { + *slot = true; + } + } + } + + facts + .into_iter() + .zip(keep) + .filter_map(|(fact, kept)| kept.then_some(fact)) + .collect() +} +``` +Also add `use crate::graph::fxhash::FxBuildHasher;` only inside the function as shown (keeps the top-level imports untouched). Update the doc-comment link to `[`FxBuildHasher`](crate::graph::fxhash::FxBuildHasher)` if rustdoc complains about the intra-doc link. + +- [ ] **Step 5: Run the storage tests, then the whole suite** + +Run: `cargo test --lib graph::storage` +Expected: all pass, including the 3 new tests. +Run: `cargo test 2>&1 | grep -E "^test result|FAILED|panicked"` +Expected: no failures. If any test fails because it indexed `net_asserted_facts` output assuming HashMap order, that test was already order-fragile: fix the test to compare sorted/set results, never the production code. + +- [ ] **Step 6: Clippy + commit** + +Run: `cargo clippy --all-targets -- -D warnings` → clean. +```bash +git add src/graph/storage.rs +git commit -m "perf: borrow keys and use FxHash in net_asserted_facts (#323)" +``` + +--- + +### Task 3: `(entity, attribute)` EAVT range lookup + +**Files:** +- Modify: `src/graph/storage.rs` — add method inside `impl FactStorage` block that starts with `/// Production helpers on FactStorage` (~line 617), right after `get_facts_by_entity`; tests in `mod tests`. + +**Interfaces:** +- Produces: `pub(crate) fn get_facts_by_entity_attribute_indexed(&self, entity_id: &EntityId, attribute: &Attribute) -> Result>` — every stored record (assert and retract, all versions) whose `entity == entity_id && attribute == attribute`, pending + committed. Task 4 calls it. + +- [ ] **Step 1: Write failing tests in `mod tests`** + +```rust + #[test] + fn entity_attribute_indexed_pending_only() { + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + let other = uuid::Uuid::from_u128(2); + storage + .transact(vec![ + (e, ":hash".to_string(), Value::String("h0".into())), + (e, ":other".to_string(), Value::String("o".into())), + (other, ":hash".to_string(), Value::String("x".into())), + ], None) + .unwrap(); + storage + .retract(vec![(e, ":hash".to_string(), Value::String("h0".into()))]) + .unwrap(); + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":hash".to_string()) + .unwrap(); + assert_eq!(facts.len(), 2, "assert + retract of :hash for e only"); + assert!(facts.iter().all(|f| f.entity == e && f.attribute == ":hash")); + } + + #[test] + fn entity_attribute_indexed_excludes_prefix_sibling() { + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + storage + .transact(vec![ + (e, ":a".to_string(), Value::Integer(1)), + (e, ":ab".to_string(), Value::Integer(2)), + (e, ":a/b".to_string(), Value::Integer(3)), + ], None) + .unwrap(); + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":a".to_string()) + .unwrap(); + assert_eq!(facts.len(), 1, "only :a, not :ab or :a/b"); + assert_eq!(facts[0].value, Value::Integer(1)); + } + + #[test] + fn entity_attribute_indexed_no_index_fallback() { + // FactStorage with facts but empty pending indexes and no committed reader + // exercises the fallback branch. + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + storage + .transact(vec![ + (e, ":a".to_string(), Value::Integer(1)), + (e, ":b".to_string(), Value::Integer(2)), + ], None) + .unwrap(); + storage.replace_pending_indexes(crate::storage::index::Indexes::new()); + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":b".to_string()) + .unwrap(); + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].value, Value::Integer(2)); + } + + #[test] + fn entity_attribute_indexed_committed_and_pending() { + use crate::storage::CommittedFactReader; + use crate::storage::index::{FactRef, Indexes}; + use std::sync::Arc; + + struct MockLoader { + facts: Vec, + } + impl CommittedFactReader for MockLoader { + fn resolve(&self, fr: FactRef) -> anyhow::Result { + self.facts + .get(fr.slot_index as usize) + .cloned() + .ok_or_else(|| anyhow::anyhow!("no fact at slot")) + } + fn stream_all(&self) -> anyhow::Result> { + Ok(self.facts.clone()) + } + fn committed_page_count(&self) -> u64 { + 1 + } + } + + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + let committed = vec![ + make_assert(e, ":a", Value::Integer(1), 1, 0, VALID_TIME_FOREVER), + make_assert(e, ":ab", Value::Integer(9), 1, 0, VALID_TIME_FOREVER), + ]; + let mut indexes = Indexes::new(); + for (slot, f) in committed.iter().enumerate() { + indexes.insert( + f, + FactRef { + page_id: 1, + slot_index: u16::try_from(slot).unwrap(), + }, + ); + } + storage.replace_pending_indexes(indexes); + storage.set_committed_reader(Arc::new(MockLoader { facts: committed })); + + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":a".to_string()) + .unwrap(); + assert_eq!(facts.len(), 1, "committed :a only, :ab excluded"); + assert_eq!(facts[0].value, Value::Integer(1)); + } +``` +`FactStorage::transact(&self, Vec<(EntityId, Attribute, Value)>, Option) -> Result` and `FactStorage::retract(&self, Vec<(EntityId, Attribute, Value)>) -> Result<(TxId, u64)>` are the real signatures (verified). Each test transacts distinct attributes per entity, so the v2 same-transaction multi-value key collision (#371) cannot interfere. + +Note: the mock uses `page_id: 1` entries in the *pending* index map; this exercises the committed-resolve branch of `resolve_fact_ref` through the pending range loop, mirroring `test_committed_reader_resolves_facts`. The on-disk committed index path (`range_scan_eavt`) is covered end-to-end by Task 4's file-backed tests. + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test --lib entity_attribute_indexed` +Expected: compile error `no method named get_facts_by_entity_attribute_indexed`. + +- [ ] **Step 3: Implement** + +Insert after `get_facts_by_entity`: +```rust + /// Get every stored record for one `(entity, attribute)` pair (index-driven, #323). + /// + /// Range-scans EAVT over `[(e, a, …), (e, next_prefix(a), …))` so other + /// attributes of the same entity — and their version history — are never + /// resolved. Both index sources post-filter on the exact attribute because a + /// prefix range also covers longer attributes (`:a` → `:ab`). + pub(crate) fn get_facts_by_entity_attribute_indexed( + &self, + entity_id: &EntityId, + attribute: &Attribute, + ) -> Result> { + use crate::storage::index::EavtKey; + let d = self.data.read().unwrap_or_else(|e| e.into_inner()); + let matches = |f: &Fact| &f.entity == entity_id && &f.attribute == attribute; + + // Fallback: no indexes built yet + if d.pending_indexes.eavt.is_empty() && d.committed_index_reader.is_none() { + let mut result: Vec = d.facts.iter().filter(|f| matches(f)).cloned().collect(); + if let Some(loader) = &d.committed { + result.extend(loader.stream_all()?.into_iter().filter(|f| matches(f))); + } + return Ok(result); + } + + let start = EavtKey { + entity: *entity_id, + attribute: attribute.clone(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + // None only for the empty attribute; then the committed scan is unbounded + // above and relies on the post-filter. + let end_opt: Option = next_string_prefix(attribute).map(|next_attr| EavtKey { + entity: *entity_id, + attribute: next_attr, + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }); + + let mut facts = Vec::new(); + + // Pending: walk forward from `start` while still on this exact (e, a). + for (key, &fr) in d.pending_indexes.eavt.range(start.clone()..) { + if key.entity != *entity_id || key.attribute != *attribute { + break; + } + facts.push(resolve_fact_ref(&d, fr)?); + } + + // Committed: on-disk B+tree range scan, post-filtered on the exact pair. + if let Some(reader) = &d.committed_index_reader { + for fr in reader.range_scan_eavt(&start, end_opt.as_ref())? { + let fact = resolve_fact_ref(&d, fr)?; + if matches(&fact) { + facts.push(fact); + } + } + } + + Ok(facts) + } +``` +Why the pending loop can `break`: EAVT orders by `(entity, attribute, …)`, so all keys for the exact pair are contiguous starting at `start`; the first key with a different entity or attribute ends the run (`:ab` sorts after every `:a` key). + +- [ ] **Step 4: Run tests** + +Run: `cargo test --lib entity_attribute_indexed` +Expected: 4 passed. + +- [ ] **Step 5: Clippy + commit** + +Run: `cargo clippy --all-targets -- -D warnings`. `get_facts_by_entity_attribute_indexed` is unused outside tests until Task 4 → a `dead_code` warning is expected here; if clippy's `-D warnings` blocks the commit hook, fold this commit into Task 4 instead of adding `#[allow(dead_code)]`. +```bash +git add src/graph/storage.rs +git commit -m "perf: add (entity, attribute) EAVT range lookup (#323)" +``` + +--- + +### Task 4: Narrow `selective_fact_fetch` and drop its dedup + +**Files:** +- Modify: `src/query/datalog/executor.rs` — `selective_fact_fetch` (~lines 369–455) and the doc comment on `filter_facts_for_query` if it mentions dedup +- Create: `tests/point_query_history_test.rs` + +**Interfaces:** +- Consumes: `FactStorage::get_facts_by_entity_attribute_indexed` (Task 3); `FactStorage::get_facts_by_entity`, `get_facts_by_attribute` (existing); `net_asserted_facts` idempotence (Task 2). +- Produces: same signature `fn selective_fact_fetch(&self, patterns: &[Pattern], threshold: usize) -> Option>`. + +- [ ] **Step 1: Write the integration tests** + +`tests/point_query_history_test.rs`: +```rust +//! #323 — bound-entity point queries must return exactly what a full scan returns, +//! while only reading the queried attributes' history. +//! +//! Oracle: the same query with `:as-of 1000000000` goes through the full-scan +//! `get_facts_as_of` path (never the selective path) and, since every tx_count in +//! these tests is far below that bound, must produce identical rows. +#![cfg(not(target_arch = "wasm32"))] + +use minigraf::{Minigraf, QueryResult}; + +const ORACLE_AS_OF: &str = ":as-of 1000000000"; + +fn rows(r: QueryResult) -> Vec { + match r { + QueryResult::QueryResults { results, .. } => { + let mut v: Vec = results.iter().map(|row| format!("{row:?}")).collect(); + v.sort(); + v + } + _ => panic!("expected QueryResults"), + } +} + +/// Run `(query [:find :where ])` via the selective path and via +/// the full-scan oracle; assert equal; return the selective rows. +fn same_as_full_scan(db: &Minigraf, find: &str, opts: &str, body: &str) -> Vec { + let selective = rows( + db.execute(&format!("(query [:find {find} {opts} :where {body}])")) + .unwrap(), + ); + let oracle = rows( + db.execute(&format!( + "(query [:find {find} {ORACLE_AS_OF} {opts} :where {body}])" + )) + .unwrap(), + ); + assert_eq!(selective, oracle, "selective path diverged from full scan"); + selective +} + +fn churn(db: &Minigraf, depth: usize) { + db.execute("(transact [[:e/hot :other \"o\"]])").unwrap(); + db.execute("(transact {:valid-from \"2020-01-01T00:00:00Z\"} [[:e/hot :hash \"h0\"]])") + .unwrap(); + for i in 1..depth { + db.execute(&format!("(retract [[:e/hot :hash \"h{}\"]])", i - 1)) + .unwrap(); + db.execute(&format!( + "(transact {{:valid-from \"2020-01-01T00:00:00Z\"}} [[:e/hot :hash \"h{i}\"]])" + )) + .unwrap(); + } +} + +#[test] +fn churned_attribute_live_value_in_memory() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 50); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v]"); + assert_eq!(r.len(), 1, "exactly one live value"); + assert!(r[0].contains("h49")); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :other ?v]"); + assert_eq!(r.len(), 1); +} + +#[test] +fn churned_attribute_live_value_file_backed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.graph"); + { + let db = Minigraf::open(&path).unwrap(); + churn(&db, 30); + db.checkpoint().unwrap(); + // Pending on top of committed: more churn after the checkpoint, + // including reasserting a previously retracted value. + db.execute("(retract [[:e/hot :hash \"h29\"]])").unwrap(); + db.execute("(transact {:valid-from \"2020-01-01T00:00:00Z\"} [[:e/hot :hash \"h3\"]])") + .unwrap(); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("\"h3\"")); + } + let db = Minigraf::open(&path).unwrap(); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("\"h3\"")); +} + +#[test] +fn two_attributes_on_one_entity() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 10); + let r = same_as_full_scan(&db, "?h ?o", "", "[:e/hot :hash ?h] [:e/hot :other ?o]"); + assert_eq!(r.len(), 1); +} + +#[test] +fn five_attributes_on_one_entity() { + let db = Minigraf::in_memory().unwrap(); + db.execute("(transact [[:e/x :a 1] [:e/x :b 2] [:e/x :c 3] [:e/x :d 4] [:e/x :f 5]])") + .unwrap(); + let r = same_as_full_scan( + &db, + "?a ?b ?c ?d ?f", + "", + "[:e/x :a ?a] [:e/x :b ?b] [:e/x :c ?c] [:e/x :d ?d] [:e/x :f ?f]", + ); + assert_eq!(r.len(), 1); +} + +#[test] +fn bound_entity_variable_attribute_sees_all() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + let r = same_as_full_scan(&db, "?a ?v", "", "[:e/hot ?a ?v]"); + assert_eq!(r.len(), 2, ":hash h4 and :other o"); + // Mixed: one narrowed pattern + one variable-attribute pattern on the same entity. + let r = same_as_full_scan(&db, "?a ?h", "", "[:e/hot :hash ?h] [:e/hot ?a _]"); + assert_eq!(r.len(), 2); +} + +#[test] +fn bound_entity_pseudo_attribute_matches_full_scan() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + same_as_full_scan( + &db, + "?v ?vf", + ":any-valid-time", + "[:e/hot :hash ?v] [:e/hot :db/valid-from ?vf]", + ); +} + +#[test] +fn not_and_or_on_narrowed_entity() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + db.execute("(transact [[:e/hot :flag true]])").unwrap(); + same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v] (not [:e/hot :flag false])"); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v] (not [:e/hot :flag true])"); + assert!(r.is_empty(), "not must see :flag although only :hash is projected"); + same_as_full_scan( + &db, + "?v", + "", + "(or [:e/hot :hash ?v] [:e/hot :other ?v])", + ); +} + +#[test] +fn prefix_sibling_attribute_not_leaked() { + let dir = tempfile::tempdir().unwrap(); + let db = Minigraf::open(dir.path().join("p.graph")).unwrap(); + db.execute("(transact [[:e/x :hash \"a\"] [:e/x :hash2 \"b\"] [:e/x :hash/sub \"c\"]])") + .unwrap(); + db.checkpoint().unwrap(); + db.execute("(transact [[:e/x :hashx \"d\"]])").unwrap(); + let r = same_as_full_scan(&db, "?v", "", "[:e/x :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("\"a\"")); +} + +#[test] +fn entity_and_attribute_patterns_mixed() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + db.execute("(transact [[:e/a :hash \"z\"]])").unwrap(); + same_as_full_scan(&db, "?e ?v ?o", "", "[?e :hash ?v] [:e/hot :other ?o]"); +} + +#[test] +fn valid_at_past_on_churned_attribute() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + "(transact {:valid-from \"2020-01-01T00:00:00Z\" :valid-to \"2021-01-01T00:00:00Z\"} [[:e/v :hash \"old\"]])", + ) + .unwrap(); + db.execute("(transact {:valid-from \"2021-01-01T00:00:00Z\"} [[:e/v :hash \"new\"]])") + .unwrap(); + db.execute("(transact [[:e/v :other 1]])").unwrap(); + let r = same_as_full_scan(&db, "?v", ":valid-at \"2020-06-01T00:00:00Z\"", "[:e/v :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("old")); +} +``` +Adjust syntax only if the parser rejects something (e.g. option order `:as-of` / `:valid-at` / `:any-valid-time` — check `tests/bitemporal_test.rs` for accepted forms); never weaken an assertion to make it pass. + +- [ ] **Step 2: Run tests against the current code** + +Run: `cargo test --test point_query_history_test` +Expected: all PASS already (current code is correct, just slow). These are regression guards for Step 3. If one fails now, stop and investigate — it is a pre-existing bug, report it rather than fixing it inside this task. + +- [ ] **Step 3: Rewrite `selective_fact_fetch`** + +Replace the function (doc comment included) with: +```rust + /// Attempt a selective index-backed fact fetch for the given patterns. + /// + /// Patterns with a bound entity literal (UUID or keyword → deterministic UUID) are + /// fetched by entity; if every pattern on that entity also binds a concrete attribute + /// keyword, only those `(entity, attribute)` EAVT ranges are read, so other attributes' + /// version history is never resolved (#323). An entity also referenced with a variable + /// or pseudo-attribute is read whole. Patterns without a bound entity are fetched by + /// attribute. If any pattern has neither, returns `None` (full scan). + /// + /// Lookup budget: if narrowing needs more than `threshold` lookups, every entity falls + /// back to a single whole-entity scan; only if that still exceeds `threshold` does this + /// return `None`. Narrowing therefore never turns a selective query into a full scan. + /// + /// Results are not deduplicated: the sole caller feeds them to `net_asserted_facts`, + /// which is idempotent under duplicated records. + fn selective_fact_fetch(&self, patterns: &[Pattern], threshold: usize) -> Option> { + use std::collections::{BTreeMap, BTreeSet}; + + // Per bound entity: Some(attrs) = only these attributes are referenced; + // None = the whole entity is needed. BTree* for deterministic lookup order. + let mut entity_attrs: BTreeMap>> = BTreeMap::new(); + let mut attributes: BTreeSet = BTreeSet::new(); + + for pattern in patterns { + let bound_entity = match &pattern.entity { + EdnValue::Uuid(u) => Some(*u), + EdnValue::Keyword(_) => edn_to_entity_id(&pattern.entity).ok(), + _ => None, + }; + + if let Some(uid) = bound_entity { + let slot = entity_attrs + .entry(uid) + .or_insert_with(|| Some(BTreeSet::new())); + match &pattern.attribute { + AttributeSpec::Real(EdnValue::Keyword(attr)) => { + if let Some(attrs) = slot { + attrs.insert(attr.clone()); + } + } + _ => *slot = None, + } + continue; + } + + if let AttributeSpec::Real(EdnValue::Keyword(attr)) = &pattern.attribute { + attributes.insert(attr.clone()); + } else { + return None; + } + } + + let narrowed_lookups: usize = entity_attrs + .values() + .map(|a| a.as_ref().map_or(1, BTreeSet::len)) + .sum::() + + attributes.len(); + let narrow = narrowed_lookups <= threshold; + let total = if narrow { + narrowed_lookups + } else { + entity_attrs.len() + attributes.len() + }; + if total == 0 || total > threshold { + return None; + } + + let mut all_facts: Vec = Vec::new(); + + for (uid, attrs) in &entity_attrs { + match attrs { + Some(attrs) if narrow => { + for attr in attrs { + all_facts.extend( + self.storage + .get_facts_by_entity_attribute_indexed(uid, attr) + .ok()?, + ); + } + } + _ => all_facts.extend(self.storage.get_facts_by_entity(uid).ok()?), + } + } + + for attr in &attributes { + all_facts.extend(self.storage.get_facts_by_attribute(attr).ok()?); + } + + Some(all_facts) + } +``` +Behaviour kept from the old version: any storage error → `None` (caller falls back to full scan). + +- [ ] **Step 4: Run the targeted and full suites** + +Run: `cargo test --test point_query_history_test` → all pass. +Run: `cargo test 2>&1 | grep -E "^test result|FAILED|panicked"` → no failures. +Run: `cargo clippy --all-targets -- -D warnings` and `cargo fmt --check` → clean. + +- [ ] **Step 5: Commit** +```bash +git add src/query/datalog/executor.rs tests/point_query_history_test.rs +git commit -m "perf: narrow bound-entity fetch to queried attributes, drop redundant dedup (#323)" +``` + +--- + +### Task 5: Benchmark, measure, docs, follow-up issue + +**Files:** +- Modify: `benches/minigraf_bench.rs`, `docs/BENCHMARKS.md`, `CHANGELOG.md`, `docs/TEST_COVERAGE.md`, `CLAUDE.md` + +- [ ] **Step 1: Add the bench group** + +Before `// ── query/predicate_pushdown` add: +```rust +// ── point_query_chain_depth (Issue #323) ───────────────────────────────────── +// +// One entity whose :hash is retracted/reasserted `depth` times (exactly one live +// value), plus a never-churned :other on the same entity; checkpointed file DB. + +fn bench_point_query_chain_depth(c: &mut Criterion) { + const DEPTHS: &[usize] = &[1, 500, 2000]; + const QUERIES: &[(&str, &str)] = &[ + ("churned_attr", "(query [:find ?v :where [:e/hot :hash ?v]])"), + ("sibling_attr", "(query [:find ?v :where [:e/hot :other ?v]])"), + ("attr_scan", "(query [:find ?v :where [?e :hash ?v]])"), + ]; + for &(name, q) in QUERIES { + let mut group = c.benchmark_group(format!("point_query_chain_depth/{name}")); + group.sample_size(20); + for &depth in DEPTHS { + let dir = tempfile::tempdir().unwrap(); + let db = minigraf::Minigraf::open(dir.path().join("b.graph")).unwrap(); + for i in 0..2000 { + db.execute(&format!("(transact [[:f/{i} :x {i}]])")).unwrap(); + } + db.execute("(transact [[:e/hot :other \"o\"]])").unwrap(); + db.execute( + "(transact {:valid-from \"2020-01-01T00:00:00Z\"} [[:e/hot :hash \"h0\"]])", + ) + .unwrap(); + for i in 1..depth { + db.execute(&format!("(retract [[:e/hot :hash \"h{}\"]])", i - 1)) + .unwrap(); + db.execute(&format!( + "(transact {{:valid-from \"2020-01-01T00:00:00Z\"}} [[:e/hot :hash \"h{i}\"]])" + )) + .unwrap(); + } + db.checkpoint().unwrap(); + group.bench_with_input(BenchmarkId::from_parameter(depth), &depth, |b, _| { + b.iter(|| black_box(db.execute(q).unwrap())); + }); + } + group.finish(); + } +} +``` +Register `bench_point_query_chain_depth, // Issue #323` in `criterion_group!` after `bench_predicate_pushdown`. Confirm `tempfile` is usable from benches (it is a non-wasm dev-dependency; benches are native). + +- [ ] **Step 2: Baseline vs after** + +Commit the bench first, then measure the baseline in a throwaway worktree at the branch's merge-base (pre-Task-1 `src/`) with the new bench file copied in. Do not use `git stash` (shared across worktrees). +```bash +git add benches/minigraf_bench.rs && git commit -m "bench: point_query_chain_depth (#323)" +BASE=$(git merge-base HEAD origin/main) +git worktree add /tmp/claude-1000/-home-aditya-Work-AMC-Minigraf-minigraf/008662fe-0225-4a06-a393-915f375e4da0/scratchpad/base "$BASE" +cp benches/minigraf_bench.rs /tmp/claude-1000/-home-aditya-Work-AMC-Minigraf-minigraf/008662fe-0225-4a06-a393-915f375e4da0/scratchpad/base/benches/ +(cd /tmp/claude-1000/-home-aditya-Work-AMC-Minigraf-minigraf/008662fe-0225-4a06-a393-915f375e4da0/scratchpad/base && cargo bench --bench minigraf_bench -- point_query_chain_depth --save-baseline before) +cargo bench --bench minigraf_bench -- point_query_chain_depth +git worktree remove --force /tmp/claude-1000/-home-aditya-Work-AMC-Minigraf-minigraf/008662fe-0225-4a06-a393-915f375e4da0/scratchpad/base +``` +Record the median for each (query, depth) before and after. + +Acceptance (spec §5): `sibling_attr` at depth 2000 within 2× of depth 1; `churned_attr` and `attr_scan` at depth 2000 ≥3× faster than before. If not met, stop and report the numbers — do not tune further without discussing. + +- [ ] **Step 3: Docs** + +- `CHANGELOG.md`: add at top, above `## v2.0.1`: + ```markdown + ## Unreleased + + ### Performance + + - **Bound-entity point queries no longer pay for other attributes' history (#323).** `[:e :attr ?v]` now range-scans only `(e, :attr)` in the EAVT index instead of every record the entity has ever had. The remaining per-record cost on a heavily retracted/reasserted attribute is cut by × (cheaper net-assert grouping, redundant dedup removed); attribute scans (`[?e :attr ?v]`) benefit from the same change. File format and results unchanged. Resolve cost on a churned attribute is still proportional to its history on v2.x; the structural fix needs the v8 index keys and is tracked in #. + ``` + Fill `` from Step 2 and `` from Step 4. +- `docs/BENCHMARKS.md`: add a "Point query vs. version-chain depth (#323)" table (before/after medians) under the query latency section, following that file's existing table style. +- `docs/TEST_COVERAGE.md` and `CLAUDE.md` test count: run `cargo test 2>&1 | grep "^test result"` and sum passed/ignored; update both to the new totals and add `tests/point_query_history_test.rs` to the per-file breakdown. +- Also add `fxhash.rs` to the `src/graph/` module list in `CLAUDE.md`'s Architecture section. + +- [ ] **Step 4: File the v3 follow-up issue** + +```bash +gh issue create --milestone "v3.0.0" --title "perf: compute net-assert on v8 index keys before resolving facts (O(live) point queries)" --body "$(cat <<'EOF' +Follow-up to #323 (v2.x mitigation). + +v8 `EavtKey`/`AevtKey` carry `value_bytes` and `asserted`. `net_asserted_facts` semantics can therefore be computed over index keys **before** `resolve_fact_ref`, and only surviving records resolved. Resolve cost becomes O(live facts); the key walk stays O(history) but over B+tree leaf pages, which is far cheaper than a page resolve + postcard decode per record. + +Scope: `get_facts_by_entity_attribute_indexed`, `get_facts_by_entity`, `get_facts_by_attribute` on the `v3` branch; the executor's selective path. `:as-of` needs `tx_count <= N` applied on keys first. + +Refs #323 +EOF +)" +``` +If the milestone name differs, run `gh api repos/:owner/:repo/milestones --jq '.[].title'` and use the v3.0.0 one. + +- [ ] **Step 5: Commit docs** +```bash +git add CHANGELOG.md docs/BENCHMARKS.md docs/TEST_COVERAGE.md CLAUDE.md +git commit -m "docs: changelog, benchmarks and test counts for #323" +``` + +- [ ] **Step 6: PR** — push the branch and open a PR against `main` whose body ends with `Closes #323` and the attribution footer; then monitor CI until green (never merge without explicit confirmation). diff --git a/docs/superpowers/specs/2026-09-25-point-query-history-scan-design.md b/docs/superpowers/specs/2026-09-25-point-query-history-scan-design.md new file mode 100644 index 0000000000..6621c10581 --- /dev/null +++ b/docs/superpowers/specs/2026-09-25-point-query-history-scan-design.md @@ -0,0 +1,184 @@ +# Point-query history scan mitigation (#323) — Design + +**Issue:** #323 — point query on a bound entity scans the entity's whole version history +**Milestone:** v2.3.0 (`main`, file format v7 — no format change) +**Follow-up:** structural O(live) fix on the `v3` branch (separate issue, see §6) + +## 1. Problem + +`[:e/x :attr ?v]` costs O(every record ever written for `:e/x`), not O(live facts). +An entity whose attribute is retracted and reasserted N times gets ~N× slower to read. +Downstream (temporal_reasoning#239) this is ~⅓ of ingestion wall clock. Attribute-only +scans (`[?e :attr ?v]`) show the same history-bound behaviour. + +Reproduced locally (2000 filler facts, checkpointed, median of 20): + +| depth | `[:e/hot :hash ?v]` | `[:e/hot :other ?v]` | `[?e :hash ?v]` | +|---:|---:|---:|---:| +| 1 | 0.026 ms | 0.024 ms | 0.022 ms | +| 500 | 1.769 ms | 1.749 ms | 1.688 ms | +| 2000 | 7.156 ms | 7.250 ms | 7.067 ms | + +`:other` is written once and never churned, yet costs the same as `:hash`: the scan is +entity-wide, not attribute-wide. + +## 2. Where the time goes + +`perf` at depth 2000 (4001 EAVT records for the entity): + +| component | share | +|---|---:| +| `net_asserted_facts` — SipHash on cloned `(Uuid, String, Vec, i64, i64)` keys | ~40% | +| `get_facts_by_entity` — `resolve_fact_ref` + postcard decode per record | ~30% | +| `selective_fact_fetch` dedup `HashSet<(Uuid, String, u64, bool)>` (redundant — §4.2) | ~16% | + +It is CPU overhead per dead record, not I/O. + +**Why v2 cannot be O(live):** v7 `EavtKey`/`AevtKey` carry neither the value nor the +`asserted` flag. Deciding whether a record is live requires knowing its `(e, a, v)` and +whether a later retraction of that `v` exists — only obtainable by resolving the record. +Retraction records also have their own validity window (`[tx_id, ∞)`), so key-level +valid-time pruning cannot safely discard them. v2 can therefore only shrink the constant +and narrow the scan to the queried attribute. + +## 3. Goals / non-goals + +**Goals** +- Other attributes on a churned entity no longer pay for that entity's churn. +- Constant factor per history record cut on the churned attribute itself (original target ≥3×; + revised to the measured ~1.6× — see §5 Acceptance). +- The attribute-scan and full-scan paths benefit from the cheaper `net_asserted_facts`. +- Zero change to query results, file format, public API, or dependencies. + +**Non-goals** +- O(live) resolve cost (needs v8 keys — §6). +- Changes to `retract`/`transact` write paths. +- Changes to the `:as-of` path beyond the shared `net_asserted_facts` speed-up. + +## 4. Design + +### 4.1 Attribute-narrowed entity scan + +**Storage** (`src/graph/storage.rs`): new production method + +```rust +pub(crate) fn get_facts_by_entity_attribute_indexed( + &self, entity_id: &EntityId, attribute: &Attribute, +) -> Result> +``` + +- EAVT range `[(e, a, i64::MIN, i64::MIN, 0), (e, next_string_prefix(a), i64::MIN, i64::MIN, 0))`. + If `next_string_prefix` returns `None`, upper bound is `(e+1, "", …)` and results are + filtered by `key.attribute == a`. +- Pending (in-memory `BTreeMap`) and committed (`range_scan_eavt`) merged exactly as + `get_facts_by_entity` does; both paths post-filter `attribute == a` so prefix-sharing + attributes (`:a` vs `:ab`) never leak. +- No-index fallback: filter `d.facts` / `loader.stream_all()` on entity **and** attribute. +- The existing `#[cfg(test)] get_facts_by_entity_attribute` (full-scan) helper stays as is. + +**Executor** (`selective_fact_fetch` in `src/query/datalog/executor.rs`): + +- Replace `entity_ids: HashSet` with `entity_attrs: HashMap>>`. + - Pattern with bound entity and `AttributeSpec::Real(EdnValue::Keyword(a))` → add `a` + to that entity's set (unless it is already `None`). + - Pattern with bound entity and any other attribute (variable, pseudo-attribute) → + set that entity to `None` (whole-entity scan). +- Narrowed lookup count = Σ over entities of (1 if `None` else number of attributes) + + number of attribute-only lookups. If it exceeds the existing `threshold` (4), every + entity collapses back to one whole-entity scan (today's behaviour) and the count is + recomputed; only if *that* still exceeds the threshold → `None` (full scan), as today. + This guarantees narrowing never turns a query that used the selective path into a + full scan (e.g. five attributes of one bound entity). +- Fetch `get_facts_by_entity` for `None` entries and + `get_facts_by_entity_attribute_indexed` per `(e, a)` otherwise. + +**Correctness argument:** `net_asserted_facts` groups by `(e, a, v)` and the valid-time +filter is per-fact, so restricting input to one `(e, a)` never splits a group. +`collect_all_patterns` already includes patterns inside `not`, `not-join`, `or`, +`or-join`, so every attribute any clause could match on that entity is fetched. +Rule-using queries never reach this path. + +### 4.2 Remove the executor dedup + +`selective_fact_fetch` has exactly one caller, `filter_facts_for_query`, which passes its +output straight into `net_asserted_facts`. That function is idempotent under duplicated +input records: an identical assertion lands in the same `(e, a, v, valid_from, valid_to)` +window and the first one wins the tie; a duplicated retraction leaves the per-`(e, a, v)` +max `tx_count` unchanged. So the `seen: HashSet<(Uuid, String, u64, bool)>` pass is +redundant and is deleted. It cannot simply be skipped "for single-source lookups" instead: +between `set_committed_index_reader` and `post_checkpoint_clear` a concurrent reader can +see a fact in both the pending and the committed index. The idempotence is pinned by a +unit test (§5) so a future change to `net_asserted_facts` cannot silently break it. + +### 4.3 Cheaper `net_asserted_facts` + +- ~~Add a small in-crate FxHash-style hasher.~~ **Dropped after final review + (2026-09-26):** a fixed-seed fast hash lets crafted colliding values (untrusted text in + agent memory) make every query quadratic. It bought ~13%; the maps keep std's randomly + keyed `RandomState`. +- Restructure `net_asserted_facts` so each fact's value is encoded once and group keys + borrow from the input rather than cloning `attribute`/`value_bytes` into every key: + 1. `encoded: Vec> = facts.iter().map(|f| encode_value(&f.value)).collect()`. + 2. `max_retract_tx: HashMap<(&Uuid, &str, &[u8]), u64>`. + 3. `by_window: HashMap<(&Uuid, &str, &[u8], i64, i64), (usize, u64)>` storing the + index of the winning assertion. + 4. Collect surviving indices, then move the winning facts out of `facts` + (e.g. mark survivors in a `Vec` and `into_iter().zip(...).filter`). +- Output **order**: callers must not depend on it today (current output is HashMap + iteration order). Tests that compare results already sort or use sets; verify during + implementation. +- HashDoS: fact values can be untrusted text, so hashing stays randomly keyed (see above). + +## 5. Testing + +TDD; correctness first, then measurement. + +**Storage unit tests** (`src/graph/storage.rs`), for `get_facts_by_entity_attribute_indexed`: +- pending-only, committed-only, pending+committed after checkpoint; +- no-index fallback path; +- `:a` vs `:ab` prefix isolation; +- other entities / other attributes excluded. + +**`net_asserted_facts` tests**: existing tests keep passing; add a duplicate-idempotence +test (`net_asserted_facts(xs ++ dup) == net_asserted_facts(xs)` as sorted sets); add a property-style test +comparing the new implementation against the old one (kept as a `#[cfg(test)]` reference +function) on randomized assert/retract sequences incl. multiple windows, same-tx +assert+retract, and multi-valued attributes — compared as sorted sets. + +**Executor integration tests** (`tests/point_query_history_test.rs`, new): +- churned attribute returns exactly the live value after N retract/reassert cycles + (in-memory and file-backed, before and after checkpoint); +- `[:e :a ?x] [:e :b ?y]` on one entity (two narrowed ranges, dedup path); +- bound entity with variable attribute `[:e ?a ?v]` (whole-entity fallback); +- bound entity with pseudo-attribute under `:any-valid-time`; +- `not` / `not-join` / `or` referencing another attribute of the same bound entity; +- entity + attribute-only pattern mix exceeding threshold → full-scan path, same results; +- `:valid-at` past timestamp on a churned attribute. + +All assert messages follow the CLAUDE.md testing convention (no `{:?}` of UUID-bearing types). + +**Benchmark** (`benches/minigraf_bench.rs`): `point_query_chain_depth/{1,500,2000}` for +`[:e/hot :hash ?v]`, `[:e/hot :other ?v]`, `[?e :hash ?v]` on a checkpointed file DB. +Record before/after numbers in the PR and `docs/BENCHMARKS.md`. + +**Acceptance:** +- `[:e/hot :other ?v]` at depth 2000 within 2× of depth 1. **Met**: 19.0 µs vs 19.2 µs. +- `[:e/hot :hash ?v]` and `[?e :hash ?v]` at depth 2000 faster than baseline. Original + target ≥3×; **measured 1.38× / 1.35×** (criterion on AC, bench profile with LTO and + `opt-level = "z"`: 4.64 → 3.36 ms, 4.49 → 3.33 ms; 1.6× with the dropped FxHash). + Revised to the measured numbers on 2026-09-26 and shipped as a mitigation. The remaining + cost is mostly per-record fact resolve + decode and EAVT leaf-key decode — inherent to + v7 keys; see §6 / #379. +- Full suite green; clippy/fmt clean. + +## 6. Follow-up: v3 structural fix (#379, milestone v3.0.0) + +v8 `EavtKey`/`AevtKey` (on `v3`) carry `value_bytes` and `asserted`. Net-assert can then be +computed over index keys *before* `resolve_fact_ref`, resolving only surviving records: +resolve cost O(live), key walk O(history) but on leaf pages already in cache. File as a +new issue referencing #323; not part of this change. + +## 7. Docs + +On merge: `CHANGELOG.md` (Unreleased → Performance), `docs/BENCHMARKS.md`, +`docs/TEST_COVERAGE.md` + `CLAUDE.md` test count. No wiki or file-format docs change. diff --git a/src/graph/storage.rs b/src/graph/storage.rs index 74c9405d78..e4fbf0491b 100644 --- a/src/graph/storage.rs +++ b/src/graph/storage.rs @@ -512,70 +512,68 @@ pub(crate) fn filter_facts_as_of(facts: Vec, as_of: &AsOf) -> Vec { /// /// # Implementation note /// -/// This function uses two flat `HashMap`s in a single pass over the input: +/// Hot path for every non-`:as-of` query (#323). Each value is encoded once; +/// the two group maps borrow `(entity, attribute, value_bytes)` from the input +/// instead of cloning them. They keep std's randomly keyed hasher on purpose: +/// values are often untrusted text (agent memory), and a fixed-seed fast hash +/// would let crafted colliding values make every query quadratic. `by_window` stores the +/// index and `tx_count` of the winning assertion per validity window; survivors +/// are moved out of `facts` at the end, preserving input order. /// -/// - `max_retract_tx`: EAV → highest retraction `tx_count` seen so far. -/// - `by_window`: (EAV + valid_from + valid_to) → highest-`tx_count` assertion -/// for that time window. -/// -/// The retraction filter (`fact.tx_count > max_retract_tx`) is applied in the -/// final `filter_map` rather than eagerly. This means `by_window` temporarily -/// holds assertions that will be filtered out, so on workloads where most -/// EAV triples are immediately retracted it uses more peak memory than the -/// previous per-group approach. The trade-off is intentional: eliminating the -/// per-group `Vec` allocation yields a measurable throughput gain on large -/// mostly-unique fact sets (see issue #227). +/// Idempotent under duplicated input records (a duplicate assertion ties with +/// the original and loses; a duplicate retraction leaves the max unchanged). +/// `selective_fact_fetch` relies on this instead of deduplicating. pub(crate) fn net_asserted_facts(facts: Vec) -> Vec { use std::collections::HashMap; - type EavKey = (EntityId, Attribute, Vec); - type WindowKey = (EntityId, Attribute, Vec, i64, i64); - - let mut max_retract_tx: HashMap = HashMap::new(); - let mut by_window: HashMap = HashMap::new(); - - for fact in facts { - let eav_key = ( - fact.entity, - fact.attribute.clone(), - encode_value(&fact.value), - ); - - if fact.asserted { - let window_key = ( - eav_key.0, - eav_key.1, - eav_key.2, - fact.valid_from, - fact.valid_to, - ); - match by_window.get(&window_key) { - None => { - by_window.insert(window_key, fact); - } - Some(existing) if fact.tx_count > existing.tx_count => { - by_window.insert(window_key, fact); - } - _ => {} + type EavKey<'a> = (&'a EntityId, &'a str, &'a [u8]); + type WindowKey<'a> = (&'a EntityId, &'a str, &'a [u8], i64, i64); + + let encoded: Vec> = facts.iter().map(|f| encode_value(&f.value)).collect(); + let mut keep = vec![false; facts.len()]; + + { + let mut max_retract_tx: HashMap, u64> = HashMap::new(); + let mut by_window: HashMap, (usize, u64)> = HashMap::new(); + + for (idx, (fact, value_bytes)) in facts.iter().zip(encoded.iter()).enumerate() { + let entity = &fact.entity; + let attribute = fact.attribute.as_str(); + let value = value_bytes.as_slice(); + if fact.asserted { + by_window + .entry((entity, attribute, value, fact.valid_from, fact.valid_to)) + .and_modify(|winner| { + if fact.tx_count > winner.1 { + *winner = (idx, fact.tx_count); + } + }) + .or_insert((idx, fact.tx_count)); + } else { + max_retract_tx + .entry((entity, attribute, value)) + .and_modify(|max_tx| *max_tx = (*max_tx).max(fact.tx_count)) + .or_insert(fact.tx_count); } - } else { - let tx_count = fact.tx_count; - max_retract_tx - .entry(eav_key) - .and_modify(|max_tx| *max_tx = (*max_tx).max(tx_count)) - .or_insert(tx_count); } - } - by_window - .into_iter() - .filter_map(|((entity, attribute, value, _, _), fact)| { + for ((entity, attribute, value, _, _), (idx, tx_count)) in &by_window { let retract_tx = max_retract_tx - .get(&(entity, attribute, value)) + .get(&(*entity, *attribute, *value)) .copied() .unwrap_or(0); - (fact.tx_count > retract_tx).then_some(fact) - }) + if *tx_count > retract_tx + && let Some(slot) = keep.get_mut(*idx) + { + *slot = true; + } + } + } + + facts + .into_iter() + .zip(keep) + .filter_map(|(fact, kept)| kept.then_some(fact)) .collect() } @@ -684,6 +682,73 @@ impl FactStorage { Ok(facts) } + /// Get every stored record for one `(entity, attribute)` pair (index-driven, #323). + /// + /// Range-scans EAVT over exactly `[(e, a, …), (e, a + "\0", …))`, so other + /// attributes of the same entity — including prefix siblings such as `:ab` for + /// `:a` — and their version history are never resolved. + pub(crate) fn get_facts_by_entity_attribute_indexed( + &self, + entity_id: &EntityId, + attribute: &Attribute, + ) -> Result> { + use crate::storage::index::EavtKey; + let d = self.data.read().unwrap_or_else(|e| e.into_inner()); + let matches = |f: &Fact| &f.entity == entity_id && &f.attribute == attribute; + + // Fallback: no indexes built yet + if d.pending_indexes.eavt.is_empty() && d.committed_index_reader.is_none() { + let mut result: Vec = d.facts.iter().filter(|f| matches(f)).cloned().collect(); + if let Some(loader) = &d.committed { + result.extend(loader.stream_all()?.into_iter().filter(|f| matches(f))); + } + return Ok(result); + } + + let start = EavtKey { + entity: *entity_id, + attribute: attribute.clone(), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + // Exact successor of `attribute`: the smallest string that sorts after it is + // `attribute + "\0"`, so `[start, end)` holds exactly this (e, a) pair. Always + // valid UTF-8 — unlike incrementing the last byte, which fails for attributes + // ending in 0x7F or a character whose last UTF-8 byte is 0xBF (e.g. `:丿`). + let end = EavtKey { + entity: *entity_id, + attribute: format!("{attribute}\0"), + valid_from: i64::MIN, + valid_to: i64::MIN, + tx_count: 0, + }; + + let mut facts = Vec::new(); + + // Pending: EAVT keys for the exact (e, a) are contiguous from `start`; + // the first key with a different entity or attribute ends the run. + for (key, &fr) in d.pending_indexes.eavt.range(start.clone()..) { + if key.entity != *entity_id || key.attribute != *attribute { + break; + } + facts.push(resolve_fact_ref(&d, fr)?); + } + + // Committed: on-disk B+tree range scan over exactly this (e, a) pair; the + // post-filter is a cheap guard, not something the range relies on. + if let Some(reader) = &d.committed_index_reader { + for fr in reader.range_scan_eavt(&start, Some(&end))? { + let fact = resolve_fact_ref(&d, fr)?; + if matches(&fact) { + facts.push(fact); + } + } + } + + Ok(facts) + } + /// Get all facts for a specific attribute (index-driven). pub(crate) fn get_facts_by_attribute(&self, attribute: &Attribute) -> Result> { use crate::storage::index::AevtKey; @@ -1893,4 +1958,392 @@ mod tests { "retraction should wipe all windows for the EAV triple" ); } + + /// Pre-#323 implementation, kept verbatim as the oracle for the + /// randomized equivalence test. + fn net_asserted_facts_reference(facts: Vec) -> Vec { + use std::collections::HashMap; + + type EavKey = (EntityId, Attribute, Vec); + type WindowKey = (EntityId, Attribute, Vec, i64, i64); + + let mut max_retract_tx: HashMap = HashMap::new(); + let mut by_window: HashMap = HashMap::new(); + + for fact in facts { + let eav_key = ( + fact.entity, + fact.attribute.clone(), + encode_value(&fact.value), + ); + + if fact.asserted { + let window_key = ( + eav_key.0, + eav_key.1, + eav_key.2, + fact.valid_from, + fact.valid_to, + ); + match by_window.get(&window_key) { + None => { + by_window.insert(window_key, fact); + } + Some(existing) if fact.tx_count > existing.tx_count => { + by_window.insert(window_key, fact); + } + _ => {} + } + } else { + let tx_count = fact.tx_count; + max_retract_tx + .entry(eav_key) + .and_modify(|max_tx| *max_tx = (*max_tx).max(tx_count)) + .or_insert(tx_count); + } + } + + by_window + .into_iter() + .filter_map(|((entity, attribute, value, _, _), fact)| { + let retract_tx = max_retract_tx + .get(&(entity, attribute, value)) + .copied() + .unwrap_or(0); + (fact.tx_count > retract_tx).then_some(fact) + }) + .collect() + } + + /// Order-independent comparison key for a fact set. + fn sorted_keys(facts: &[Fact]) -> Vec { + let mut v: Vec = facts + .iter() + .map(|f| { + format!( + "{}|{}|{:?}|{}|{}|{}|{}", + f.entity, + f.attribute, + f.value, + f.tx_count, + f.valid_from, + f.valid_to, + f.asserted + ) + }) + .collect(); + v.sort(); + v + } + + /// Deterministic xorshift so the test needs no RNG dependency. + struct XorShift(u64); + impl XorShift { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + } + + #[test] + fn net_asserted_matches_reference_on_random_histories() { + let entities = [uuid::Uuid::from_u128(1), uuid::Uuid::from_u128(2)]; + let attrs = [":a", ":ab", ":b"]; + let windows = [ + (0_i64, VALID_TIME_FOREVER), + (1_000, 2_000), + (1_500, VALID_TIME_FOREVER), + ]; + let mut rng = XorShift(0x9E37_79B9_7F4A_7C15); + for _case in 0..500 { + let n = rng.below(40) + 1; + let mut facts = Vec::new(); + let mut tx = 0_u64; + for _ in 0..n { + // ~25% of records share the previous tx_count (same-transaction batches). + if rng.below(4) != 0 { + tx += 1; + } + let e = entities[rng.below(2) as usize]; + let a = attrs[rng.below(3) as usize]; + let v = Value::Integer(i64::try_from(rng.below(3)).unwrap()); + if rng.below(3) == 0 { + facts.push(make_retract(e, a, v, tx)); + } else { + let (vf, vt) = windows[rng.below(3) as usize]; + facts.push(make_assert(e, a, v, tx, vf, vt)); + } + } + let expected = sorted_keys(&net_asserted_facts_reference(facts.clone())); + let actual = sorted_keys(&net_asserted_facts(facts)); + assert_eq!( + actual, expected, + "new net_asserted_facts diverged from reference" + ); + } + } + + /// The executor's selective fetch no longer dedups (#323); it relies on + /// net_asserted_facts collapsing duplicated input records. + #[test] + fn net_asserted_idempotent_under_duplicates() { + let e = uuid::Uuid::from_u128(7); + let facts = vec![ + make_assert( + e, + ":hash", + Value::String("h0".into()), + 1, + 0, + VALID_TIME_FOREVER, + ), + make_retract(e, ":hash", Value::String("h0".into()), 2), + make_assert( + e, + ":hash", + Value::String("h1".into()), + 3, + 0, + VALID_TIME_FOREVER, + ), + make_assert( + e, + ":other", + Value::String("o".into()), + 3, + 0, + VALID_TIME_FOREVER, + ), + ]; + let mut doubled = facts.clone(); + doubled.extend(facts.clone()); + let once = sorted_keys(&net_asserted_facts(facts)); + let twice = sorted_keys(&net_asserted_facts(doubled)); + assert_eq!(once.len(), 2, "h1 and o are live"); + assert_eq!(twice, once, "duplicated records must not change the result"); + } + + #[test] + fn net_asserted_preserves_input_order() { + let e = uuid::Uuid::from_u128(9); + let facts = vec![ + make_assert(e, ":z", Value::Integer(1), 1, 0, VALID_TIME_FOREVER), + make_assert(e, ":a", Value::Integer(2), 2, 0, VALID_TIME_FOREVER), + make_assert(e, ":m", Value::Integer(3), 3, 0, VALID_TIME_FOREVER), + ]; + let out = net_asserted_facts(facts); + let attrs: Vec<&str> = out.iter().map(|f| f.attribute.as_str()).collect(); + assert_eq!(attrs, vec![":z", ":a", ":m"]); + } + + #[test] + fn entity_attribute_indexed_pending_only() { + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + let other = uuid::Uuid::from_u128(2); + storage + .transact( + vec![ + (e, ":hash".to_string(), Value::String("h0".into())), + (e, ":other".to_string(), Value::String("o".into())), + (other, ":hash".to_string(), Value::String("x".into())), + ], + None, + ) + .unwrap(); + storage + .retract(vec![(e, ":hash".to_string(), Value::String("h0".into()))]) + .unwrap(); + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":hash".to_string()) + .unwrap(); + assert_eq!(facts.len(), 2, "assert + retract of :hash for e only"); + assert!( + facts + .iter() + .all(|f| f.entity == e && f.attribute == ":hash") + ); + } + + #[test] + fn entity_attribute_indexed_excludes_prefix_sibling() { + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + storage + .transact( + vec![ + (e, ":a".to_string(), Value::Integer(1)), + (e, ":ab".to_string(), Value::Integer(2)), + (e, ":a/b".to_string(), Value::Integer(3)), + ], + None, + ) + .unwrap(); + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":a".to_string()) + .unwrap(); + assert_eq!(facts.len(), 1, "only :a, not :ab or :a/b"); + assert_eq!(facts[0].value, Value::Integer(1)); + } + + #[test] + fn entity_attribute_indexed_no_index_fallback() { + // FactStorage with facts but empty pending indexes and no committed reader + // exercises the fallback branch. + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + storage + .transact( + vec![ + (e, ":a".to_string(), Value::Integer(1)), + (e, ":b".to_string(), Value::Integer(2)), + ], + None, + ) + .unwrap(); + storage.replace_pending_indexes(crate::storage::index::Indexes::new()); + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":b".to_string()) + .unwrap(); + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].value, Value::Integer(2)); + } + + #[test] + fn entity_attribute_indexed_committed_and_pending() { + use crate::storage::CommittedFactReader; + use crate::storage::index::{FactRef, Indexes}; + use std::sync::Arc; + + struct MockLoader { + facts: Vec, + } + impl CommittedFactReader for MockLoader { + fn resolve(&self, fr: FactRef) -> anyhow::Result { + self.facts + .get(fr.slot_index as usize) + .cloned() + .ok_or_else(|| anyhow::anyhow!("no fact at slot")) + } + fn stream_all(&self) -> anyhow::Result> { + Ok(self.facts.clone()) + } + fn committed_page_count(&self) -> u64 { + 1 + } + } + + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(1); + let committed = vec![ + make_assert(e, ":a", Value::Integer(1), 1, 0, VALID_TIME_FOREVER), + make_assert(e, ":ab", Value::Integer(9), 1, 0, VALID_TIME_FOREVER), + ]; + let mut indexes = Indexes::new(); + for (slot, f) in committed.iter().enumerate() { + indexes.insert( + f, + FactRef { + page_id: 1, + slot_index: u16::try_from(slot).unwrap(), + }, + ); + } + storage.replace_pending_indexes(indexes); + storage.set_committed_reader(Arc::new(MockLoader { facts: committed })); + + let facts = storage + .get_facts_by_entity_attribute_indexed(&e, &":a".to_string()) + .unwrap(); + assert_eq!(facts.len(), 1, "committed :a only, :ab excluded"); + assert_eq!(facts[0].value, Value::Integer(1)); + } + + /// #323 review: the committed EAVT range must end at the exact successor of + /// `(e, a)` for every attribute — including ones whose last UTF-8 byte is 0xBF + /// (e.g. `:丿`), where incrementing the last byte is not valid UTF-8 and the scan + /// used to run unbounded to the end of the index. + #[test] + fn entity_attribute_indexed_committed_range_is_bounded_for_any_attribute() { + use crate::storage::CommittedIndexReader; + use crate::storage::index::{AevtKey, AvetKey, EavtKey, FactRef, VaetKey}; + use std::sync::{Arc, Mutex}; + + struct RecordingReader { + eavt_bounds: Mutex)>>, + } + impl CommittedIndexReader for RecordingReader { + fn range_scan_eavt( + &self, + start: &EavtKey, + end: Option<&EavtKey>, + ) -> anyhow::Result> { + self.eavt_bounds + .lock() + .unwrap() + .push((start.clone(), end.cloned())); + Ok(vec![]) + } + fn range_scan_aevt( + &self, + _: &AevtKey, + _: Option<&AevtKey>, + ) -> anyhow::Result> { + Ok(vec![]) + } + fn range_scan_avet( + &self, + _: &AvetKey, + _: Option<&AvetKey>, + ) -> anyhow::Result> { + Ok(vec![]) + } + fn range_scan_vaet( + &self, + _: &VaetKey, + _: Option<&VaetKey>, + ) -> anyhow::Result> { + Ok(vec![]) + } + } + + let reader = Arc::new(RecordingReader { + eavt_bounds: Mutex::new(Vec::new()), + }); + let storage = FactStorage::new(); + storage.set_committed_index_reader(reader.clone()); + let e = uuid::Uuid::from_u128(5); + + for attr in [":plain", ":\u{4e3f}", ":\u{bf}", ":a\u{7f}"] { + storage + .get_facts_by_entity_attribute_indexed(&e, &attr.to_string()) + .unwrap(); + } + + let bounds = reader.eavt_bounds.lock().unwrap(); + assert_eq!(bounds.len(), 4, "one committed scan per lookup"); + for (start, end) in bounds.iter() { + let end = end + .as_ref() + .expect("committed EAVT scan must have an upper bound"); + assert_eq!(end.entity, e, "upper bound must stay within the entity"); + assert!(start < end, "range must be non-empty"); + // No other attribute may sort between start and end. + let sibling = EavtKey { + attribute: format!("{}x", start.attribute), + ..start.clone() + }; + assert!( + sibling >= *end, + "prefix sibling must fall outside the range" + ); + } + } } diff --git a/src/query/datalog/executor.rs b/src/query/datalog/executor.rs index dfaa16aacc..1b33b03541 100644 --- a/src/query/datalog/executor.rs +++ b/src/query/datalog/executor.rs @@ -368,17 +368,26 @@ impl DatalogExecutor { /// Attempt a selective index-backed fact fetch for the given patterns. /// - /// For each pattern, prefer a bound entity literal (UUID or keyword -> deterministic UUID) - /// over a bound attribute keyword for that same pattern. Entity lookups are usually more - /// selective, but multi-pattern joins still need attribute candidates for patterns that do not - /// bind an entity. If any pattern has neither a bound entity nor a bound attribute, or if the - /// distinct lookup count exceeds `threshold`, returns `None` to use a full scan. Otherwise - /// returns `Some(facts)`, deduplicated by `(entity, attribute, tx_count, asserted)`. + /// Patterns with a bound entity literal (UUID or keyword → deterministic UUID) are + /// fetched by entity; if every pattern on that entity also binds a concrete attribute + /// keyword, only those `(entity, attribute)` EAVT ranges are read, so other attributes' + /// version history is never resolved (#323). An entity also referenced with a variable + /// or pseudo-attribute is read whole. Patterns without a bound entity are fetched by + /// attribute. If any pattern has neither, returns `None` (full scan). + /// + /// Lookup budget: if narrowing needs more than `threshold` lookups, every entity falls + /// back to a single whole-entity scan; only if that still exceeds `threshold` does this + /// return `None`. Narrowing therefore never turns a selective query into a full scan. + /// + /// Results are not deduplicated: the sole caller feeds them to `net_asserted_facts`, + /// which is idempotent under duplicated records. fn selective_fact_fetch(&self, patterns: &[Pattern], threshold: usize) -> Option> { - use std::collections::HashSet; + use std::collections::{BTreeMap, BTreeSet}; - let mut entity_ids: HashSet = HashSet::new(); - let mut attributes: HashSet = HashSet::new(); + // Per bound entity: Some(attrs) = only these attributes are referenced; + // None = the whole entity is needed. BTree* for deterministic lookup order. + let mut entity_attrs: BTreeMap>> = BTreeMap::new(); + let mut attributes: BTreeSet = BTreeSet::new(); for pattern in patterns { let bound_entity = match &pattern.entity { @@ -388,7 +397,17 @@ impl DatalogExecutor { }; if let Some(uid) = bound_entity { - entity_ids.insert(uid); + let slot = entity_attrs + .entry(uid) + .or_insert_with(|| Some(BTreeSet::new())); + match &pattern.attribute { + AttributeSpec::Real(EdnValue::Keyword(attr)) => { + if let Some(attrs) = slot { + attrs.insert(attr.clone()); + } + } + _ => *slot = None, + } continue; } @@ -399,54 +418,40 @@ impl DatalogExecutor { } } - let total = entity_ids.len() + attributes.len(); + let narrowed_lookups: usize = entity_attrs + .values() + .map(|a| a.as_ref().map_or(1, BTreeSet::len)) + .sum::() + + attributes.len(); + let narrow = narrowed_lookups <= threshold; + let total = if narrow { + narrowed_lookups + } else { + entity_attrs.len() + attributes.len() + }; if total == 0 || total > threshold { return None; } - // Dedup key: (entity uuid, attribute string, tx_count, asserted) — avoids Value debug - // formatting. Including `asserted` ensures that a retraction and an assertion committed in - // the same WriteTransaction (same tx_count) are both retained, since they differ only by - // the `asserted` flag. - let mut seen: HashSet<(uuid::Uuid, String, u64, bool)> = HashSet::new(); let mut all_facts: Vec = Vec::new(); - for uid in &entity_ids { - match self.storage.get_facts_by_entity(uid) { - Ok(facts) => { - for fact in facts { - let key = ( - fact.entity, - fact.attribute.clone(), - fact.tx_count, - fact.asserted, + for (uid, attrs) in &entity_attrs { + match attrs { + Some(attrs) if narrow => { + for attr in attrs { + all_facts.extend( + self.storage + .get_facts_by_entity_attribute_indexed(uid, attr) + .ok()?, ); - if seen.insert(key) { - all_facts.push(fact); - } } } - Err(_) => return None, + _ => all_facts.extend(self.storage.get_facts_by_entity(uid).ok()?), } } for attr in &attributes { - match self.storage.get_facts_by_attribute(attr) { - Ok(facts) => { - for fact in facts { - let key = ( - fact.entity, - fact.attribute.clone(), - fact.tx_count, - fact.asserted, - ); - if seen.insert(key) { - all_facts.push(fact); - } - } - } - Err(_) => return None, - } + all_facts.extend(self.storage.get_facts_by_attribute(attr).ok()?); } Some(all_facts) @@ -3891,11 +3896,24 @@ mod tests { _ => panic!("expected query"), }; + let facts = executor.filter_facts_for_query(&query).unwrap(); + assert_eq!( + facts.len(), + 1, + "bound entity + attribute query should fetch only that (entity, attribute) pair (#323)" + ); + + // A variable attribute on the same bound entity still needs the whole entity. + let query = match parse_datalog_command("(query [:find ?a ?v :where [:e0 ?a ?v]])").unwrap() + { + DatalogCommand::Query(query) => query, + _ => panic!("expected query"), + }; let facts = executor.filter_facts_for_query(&query).unwrap(); assert_eq!( facts.len(), 2, - "bound entity + attribute query should fetch only the bound entity's facts" + "bound entity + variable attribute should fetch all of the entity's facts" ); } diff --git a/tests/point_query_history_test.rs b/tests/point_query_history_test.rs new file mode 100644 index 0000000000..26878311bc --- /dev/null +++ b/tests/point_query_history_test.rs @@ -0,0 +1,209 @@ +//! #323 — bound-entity point queries must return exactly what a full scan returns, +//! while only reading the queried attributes' history. +//! +//! Oracle: the same query with `:as-of 1000000000` goes through the full-scan +//! `get_facts_as_of` path (never the selective path) and, since every tx_count in +//! these tests is far below that bound, must produce identical rows. +#![cfg(not(target_arch = "wasm32"))] + +use minigraf::{Minigraf, QueryResult}; + +const ORACLE_AS_OF: &str = ":as-of 1000000000"; + +fn rows(r: QueryResult) -> Vec { + match r { + QueryResult::QueryResults { results, .. } => { + let mut v: Vec = results.iter().map(|row| format!("{row:?}")).collect(); + v.sort(); + v + } + _ => panic!("expected QueryResults"), + } +} + +/// Run `(query [:find :where ])` via the selective path and via +/// the full-scan oracle; assert equal; return the selective rows. +fn same_as_full_scan(db: &Minigraf, find: &str, opts: &str, body: &str) -> Vec { + let selective = rows( + db.execute(&format!("(query [:find {find} {opts} :where {body}])")) + .unwrap(), + ); + let oracle = rows( + db.execute(&format!( + "(query [:find {find} {ORACLE_AS_OF} {opts} :where {body}])" + )) + .unwrap(), + ); + assert_eq!(selective, oracle, "selective path diverged from full scan"); + selective +} + +fn churn(db: &Minigraf, depth: usize) { + db.execute("(transact [[:e/hot :other \"o\"]])").unwrap(); + db.execute("(transact {:valid-from \"2020-01-01T00:00:00Z\"} [[:e/hot :hash \"h0\"]])") + .unwrap(); + for i in 1..depth { + db.execute(&format!("(retract [[:e/hot :hash \"h{}\"]])", i - 1)) + .unwrap(); + db.execute(&format!( + "(transact {{:valid-from \"2020-01-01T00:00:00Z\"}} [[:e/hot :hash \"h{i}\"]])" + )) + .unwrap(); + } +} + +#[test] +fn churned_attribute_live_value_in_memory() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 50); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v]"); + assert_eq!(r.len(), 1, "exactly one live value"); + assert!(r[0].contains("h49")); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :other ?v]"); + assert_eq!(r.len(), 1); +} + +#[test] +fn churned_attribute_live_value_file_backed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.graph"); + { + let db = Minigraf::open(&path).unwrap(); + churn(&db, 30); + db.checkpoint().unwrap(); + // Pending on top of committed: more churn after the checkpoint, + // including reasserting a previously retracted value. + db.execute("(retract [[:e/hot :hash \"h29\"]])").unwrap(); + db.execute("(transact {:valid-from \"2020-01-01T00:00:00Z\"} [[:e/hot :hash \"h3\"]])") + .unwrap(); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("\"h3\"")); + } + let db = Minigraf::open(&path).unwrap(); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("\"h3\"")); +} + +#[test] +fn two_attributes_on_one_entity() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 10); + let r = same_as_full_scan(&db, "?h ?o", "", "[:e/hot :hash ?h] [:e/hot :other ?o]"); + assert_eq!(r.len(), 1); +} + +#[test] +fn five_attributes_on_one_entity() { + let db = Minigraf::in_memory().unwrap(); + db.execute("(transact [[:e/x :a 1] [:e/x :b 2] [:e/x :c 3] [:e/x :d 4] [:e/x :f 5]])") + .unwrap(); + let r = same_as_full_scan( + &db, + "?a ?b ?c ?d ?f", + "", + "[:e/x :a ?a] [:e/x :b ?b] [:e/x :c ?c] [:e/x :d ?d] [:e/x :f ?f]", + ); + assert_eq!(r.len(), 1); +} + +#[test] +fn bound_entity_variable_attribute_sees_all() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + let r = same_as_full_scan(&db, "?a ?v", "", "[:e/hot ?a ?v]"); + assert_eq!(r.len(), 2, ":hash h4 and :other o"); + // Mixed: one narrowed pattern + one variable-attribute pattern on the same entity. + let r = same_as_full_scan(&db, "?a ?h", "", "[:e/hot :hash ?h] [:e/hot ?a _]"); + assert_eq!(r.len(), 2); +} + +#[test] +fn bound_entity_pseudo_attribute_matches_full_scan() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + same_as_full_scan( + &db, + "?v ?vf", + ":any-valid-time", + "[:e/hot :hash ?v] [:e/hot :db/valid-from ?vf]", + ); +} + +#[test] +fn not_and_or_on_narrowed_entity() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + db.execute("(transact [[:e/hot :flag true]])").unwrap(); + same_as_full_scan( + &db, + "?v", + "", + "[:e/hot :hash ?v] (not [:e/hot :flag false])", + ); + let r = same_as_full_scan(&db, "?v", "", "[:e/hot :hash ?v] (not [:e/hot :flag true])"); + assert!( + r.is_empty(), + "not must see :flag although only :hash is projected" + ); + same_as_full_scan(&db, "?v", "", "(or [:e/hot :hash ?v] [:e/hot :other ?v])"); +} + +#[test] +fn prefix_sibling_attribute_not_leaked() { + let dir = tempfile::tempdir().unwrap(); + let db = Minigraf::open(dir.path().join("p.graph")).unwrap(); + db.execute("(transact [[:e/x :hash \"a\"] [:e/x :hash2 \"b\"] [:e/x :hash/sub \"c\"]])") + .unwrap(); + db.checkpoint().unwrap(); + db.execute("(transact [[:e/x :hashx \"d\"]])").unwrap(); + let r = same_as_full_scan(&db, "?v", "", "[:e/x :hash ?v]"); + assert_eq!(r.len(), 1); + assert!(r[0].contains("\"a\"")); +} + +#[test] +fn entity_and_attribute_patterns_mixed() { + let db = Minigraf::in_memory().unwrap(); + churn(&db, 5); + db.execute("(transact [[:e/a :hash \"z\"]])").unwrap(); + same_as_full_scan(&db, "?e ?v ?o", "", "[?e :hash ?v] [:e/hot :other ?o]"); +} + +#[test] +fn valid_at_past_on_churned_attribute() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + "(transact {:valid-from \"2020-01-01T00:00:00Z\" :valid-to \"2021-01-01T00:00:00Z\"} [[:e/v :hash \"old\"]])", + ) + .unwrap(); + db.execute("(transact {:valid-from \"2021-01-01T00:00:00Z\"} [[:e/v :hash \"new\"]])") + .unwrap(); + db.execute("(transact [[:e/v :other 1]])").unwrap(); + let r = same_as_full_scan( + &db, + "?v", + ":valid-at \"2020-06-01T00:00:00Z\"", + "[:e/v :hash ?v]", + ); + assert_eq!(r.len(), 1); + assert!(r[0].contains("old")); +} + +/// One WriteTransaction writing the same (entity, attribute) in two valid-time +/// windows gives both facts the same tx_count. The selective path used to dedup on +/// (entity, attribute, tx_count, asserted) and silently drop one window's row; it +/// must match a full scan (#323 review). +#[test] +fn same_transaction_multi_window_matches_full_scan() { + let db = Minigraf::in_memory().unwrap(); + let mut tx = db.begin_write().unwrap(); + tx.execute(r#"(transact {:valid-from "2020-01-01T00:00:00Z" :valid-to "2021-01-01T00:00:00Z"} [[:e/w :x 1]])"#) + .unwrap(); + tx.execute(r#"(transact {:valid-from "2021-01-01T00:00:00Z"} [[:e/w :x 2]])"#) + .unwrap(); + tx.commit().unwrap(); + let r = same_as_full_scan(&db, "?v", ":any-valid-time", "[:e/w :x ?v]"); + assert_eq!(r.len(), 2, "both windows' values must be returned"); +} From 69da4b2bbc070bee8cd6cdee28116724b51eeeb7 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 10:10:18 +0530 Subject: [PATCH 04/10] perf: bound attribute scans to exactly the queried attribute (#381) (#382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_facts_by_attribute ended its AEVT range at next_string_prefix(attr), which increments the last byte. For attributes ending in 0x7F or 0xBF (e.g. `:丿`, `:ÿ`) that is not valid UTF-8, so the committed scan ran to the end of the AEVT index. It also covered prefix siblings (`:ab` for `:a`). End the range at `attribute + "\0"`, the exact successor, as #380 does for the entity path. Remove the now-unused next_string_prefix. Sparse `:丿` scan (100 of 40.1k facts, checkpointed): 45.4 ms -> 251 µs. Closes #381 Refs #323 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CLAUDE.md | 2 +- README.md | 2 +- docs/TEST_COVERAGE.md | 4 +- src/graph/storage.rs | 155 ++++++++++++++++++++++-------- tests/point_query_history_test.rs | 31 ++++++ 6 files changed, 150 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5ad6d863e..4cb3be6768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **Bound-entity point queries no longer pay for other attributes' history (#323).** `[:e :attr ?v]` now range-scans only `(e, :attr)` in the EAVT index instead of every record the entity has ever written. Reading a rarely changed attribute of a heavily rewritten entity no longer slows down as that entity's history grows: with 2,000 retract/reassert cycles on another attribute of the same entity, it drops from 4.58 ms to 19.0 µs. Reading the heavily rewritten attribute itself, and attribute scans (`[?e :attr ?v]`), are about 1.35–1.4× faster (4.64 ms → 3.36 ms), from cheaper net-assert grouping and removing a redundant dedup pass. The file format is unchanged. +- **Attribute scans are bounded to exactly the queried attribute (#381).** `[?e :attr ?v]` computed the end of its AEVT range by incrementing the attribute's last byte. For attributes whose last byte is `0x7F` or `0xBF` — including about 1 in 64 non-ASCII characters, such as `:丿` or `:ÿ` — the result was not valid UTF-8, so the scan ran to the end of the whole AEVT index and read every later fact before filtering. It also read facts for prefix siblings (`:ab`, `:a/b` when scanning `:a`). The range now ends at `attribute + "\0"`, which covers exactly one attribute. Scanning 100 `:丿` facts in a checkpointed file with 40,000 facts on neighbouring attributes drops from 45.4 ms to 251 µs. Results are unchanged. ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index f0359ec370..78c9a85788 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -167,7 +167,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1173 tests passing** (1165 passing, 8 ignored; unit + integration + doc). +**1176 tests passing** (1168 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 93af3e1e0f..874758f154 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1173 tests +cargo test # run 1176 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 86fd20e154..beb7cb9c04 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,9 +4,9 @@ **Verified**: 2026-09-26 with `cargo test` -**Result**: 1,165 passing tests and 8 ignored tests (1,173 total) +**Result**: 1,168 passing tests and 8 ignored tests (1,176 total) -The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). +The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). ### Covered Areas diff --git a/src/graph/storage.rs b/src/graph/storage.rs index e4fbf0491b..82d54d71b7 100644 --- a/src/graph/storage.rs +++ b/src/graph/storage.rs @@ -595,23 +595,6 @@ fn resolve_fact_ref(d: &FactData, fr: FactRef) -> Result { } } -/// Increment the last byte of a string for prefix upper-bound construction. -/// Returns `None` if all bytes are 0xFF (true unbounded scan needed). -/// Used by the production index-driven lookup methods (`get_facts_by_attribute`). -fn next_string_prefix(s: &str) -> Option { - let mut bytes = s.as_bytes().to_vec(); - for i in (0..bytes.len()).rev() { - if let Some(b) = bytes.get_mut(i) - && *b < 0xFF - { - *b += 1; - bytes.truncate(i + 1); - return String::from_utf8(bytes).ok(); - } - } - None -} - /// Production helpers on FactStorage: index-driven entity/attribute lookups used by the query executor. impl FactStorage { /// Get all facts for a specific entity (index-driven). @@ -771,41 +754,28 @@ impl FactStorage { valid_to: i64::MIN, tx_count: 0, }; - let end_opt: Option = next_string_prefix(attribute).map(|next_attr| AevtKey { - attribute: next_attr, + // Exact successor of `attribute` (see `get_facts_by_entity_attribute_indexed`): + // `[start, end)` holds exactly this attribute, never prefix siblings such as + // `:ab` for `:a`, and is valid UTF-8 for any attribute (#381). + let end = AevtKey { + attribute: format!("{attribute}\0"), entity: uuid::Uuid::nil(), valid_from: i64::MIN, valid_to: i64::MIN, tx_count: 0, - }); + }; let mut facts = Vec::new(); - // Pending - let pending_range: Vec = match &end_opt { - Some(end) => d - .pending_indexes - .aevt - .range(start.clone()..end.clone()) - .filter(|(k, _)| k.attribute == *attribute) - .map(|(_, &r)| r) - .collect(), - None => d - .pending_indexes - .aevt - .range(start.clone()..) - .take_while(|(k, _)| k.attribute == *attribute) - .map(|(_, &r)| r) - .collect(), - }; - for fr in pending_range { + // Pending: AEVT keys for the exact attribute are contiguous from `start`. + for (_, &fr) in d.pending_indexes.aevt.range(start.clone()..end.clone()) { facts.push(resolve_fact_ref(&d, fr)?); } - // Committed + // Committed: on-disk B+tree range scan over exactly this attribute; the + // post-filter is a cheap guard, not something the range relies on. if let Some(reader) = &d.committed_index_reader { - let committed_refs = reader.range_scan_aevt(&start, end_opt.as_ref())?; - for fr in committed_refs { + for fr in reader.range_scan_aevt(&start, Some(&end))? { let fact = resolve_fact_ref(&d, fr)?; if &fact.attribute == attribute { facts.push(fact); @@ -2346,4 +2316,107 @@ mod tests { ); } } + + #[test] + fn attribute_committed_range_is_bounded_for_any_attribute() { + use crate::storage::CommittedIndexReader; + use crate::storage::index::{AevtKey, AvetKey, EavtKey, FactRef, VaetKey}; + use std::sync::{Arc, Mutex}; + + struct RecordingReader { + aevt_bounds: Mutex)>>, + } + impl CommittedIndexReader for RecordingReader { + fn range_scan_eavt( + &self, + _: &EavtKey, + _: Option<&EavtKey>, + ) -> anyhow::Result> { + Ok(vec![]) + } + fn range_scan_aevt( + &self, + start: &AevtKey, + end: Option<&AevtKey>, + ) -> anyhow::Result> { + self.aevt_bounds + .lock() + .unwrap() + .push((start.clone(), end.cloned())); + Ok(vec![]) + } + fn range_scan_avet( + &self, + _: &AvetKey, + _: Option<&AvetKey>, + ) -> anyhow::Result> { + Ok(vec![]) + } + fn range_scan_vaet( + &self, + _: &VaetKey, + _: Option<&VaetKey>, + ) -> anyhow::Result> { + Ok(vec![]) + } + } + + let reader = Arc::new(RecordingReader { + aevt_bounds: Mutex::new(Vec::new()), + }); + let storage = FactStorage::new(); + storage.set_committed_index_reader(reader.clone()); + + for attr in [":plain", ":\u{4e3f}", ":\u{bf}", ":a\u{7f}"] { + storage.get_facts_by_attribute(&attr.to_string()).unwrap(); + } + + let bounds = reader.aevt_bounds.lock().unwrap(); + assert_eq!(bounds.len(), 4, "one committed scan per lookup"); + for (start, end) in bounds.iter() { + let end = end + .as_ref() + .expect("committed AEVT scan must have an upper bound"); + assert!(start < end, "range must be non-empty"); + // No other attribute may sort between start and end. + let sibling = AevtKey { + attribute: format!("{}x", start.attribute), + ..start.clone() + }; + assert!( + sibling >= *end, + "prefix sibling must fall outside the range" + ); + } + } + + #[test] + fn attribute_scan_excludes_prefix_siblings_and_non_ascii_neighbours() { + let storage = FactStorage::new(); + let e = uuid::Uuid::from_u128(7); + for attr in [ + ":a", + ":ab", + ":a/b", + ":\u{4e3f}", + ":\u{4e40}", + ":\u{bf}", + ":\u{c0}", + ] { + storage + .transact( + vec![(e, attr.to_string(), Value::String(attr.to_string()))], + None, + ) + .unwrap(); + } + for attr in [":a", ":\u{4e3f}", ":\u{bf}"] { + let facts = storage.get_facts_by_attribute(&attr.to_string()).unwrap(); + assert_eq!(facts.len(), 1, "exactly one fact for the attribute"); + assert!( + facts.iter().all(|f| f.attribute == attr), + "no sibling facts" + ); + } + } } diff --git a/tests/point_query_history_test.rs b/tests/point_query_history_test.rs index 26878311bc..68d9ff3bc1 100644 --- a/tests/point_query_history_test.rs +++ b/tests/point_query_history_test.rs @@ -207,3 +207,34 @@ fn same_transaction_multi_window_matches_full_scan() { let r = same_as_full_scan(&db, "?v", ":any-valid-time", "[:e/w :x ?v]"); assert_eq!(r.len(), 2, "both windows' values must be returned"); } + +/// #381 — attribute-bound scans (`[?e :a ?v]`) over a checkpointed file return exactly +/// the queried attribute, including non-ASCII names whose last UTF-8 byte is 0xBF +/// (`:丿`, `:ÿ`) and ordinary names with prefix siblings (`:a` vs `:ab`, `:a/b`). +#[test] +fn attribute_scan_exact_for_non_ascii_and_prefix_siblings_file_backed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.graph"); + let attrs = [":a", ":ab", ":a/b", ":丿", ":丿x", ":乀", ":ÿ", ":ÿx", ":Ā"]; + let db = Minigraf::open(&path).unwrap(); + for (i, attr) in attrs.iter().enumerate() { + db.execute(&format!("(transact [[:e/c{i} {attr} \"c{i}\"]])")) + .unwrap(); + } + db.checkpoint().unwrap(); + // Pending on top of committed. + for (i, attr) in attrs.iter().enumerate() { + db.execute(&format!("(transact [[:e/p{i} {attr} \"p{i}\"]])")) + .unwrap(); + } + for (i, attr) in attrs.iter().enumerate() { + let r = same_as_full_scan(&db, "?e ?v", "", &format!("[?e {attr} ?v]")); + assert_eq!( + r.len(), + 2, + "one committed and one pending fact per attribute" + ); + assert!(r.iter().any(|row| row.contains(&format!("\"c{i}\"")))); + assert!(r.iter().any(|row| row.contains(&format!("\"p{i}\"")))); + } +} From 525c2a984bc3b10fa8d775ff71a736986756703c Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 11:28:42 +0530 Subject: [PATCH 05/10] ci: report nightly bench alerts only when they repeat (#401) Shared runners make single-night Bencher alerts mostly noise. From Jul 1 to Sep 25, 31 of 87 nightly runs raised alerts, and none were real regressions (see #301). The report step now: - saves the Bencher output and reads the alerted benchmark names - reads the same suite's alerts from the previous completed run - reports only benchmarks that alerted on both runs - comments on an open regression issue instead of skipping it, so an old open issue no longer hides new alerts - still reports at once when Bencher fails without listing alerts Adds the actions: read permission to read the previous run's job logs. Claude-Session: https://claude.ai/code/session_019dotPmboGEcQoQWSngbceY Co-authored-by: Claude --- .github/workflows/bench.yml | 117 +++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 23 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d3efe0b37e..2b28408a02 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,6 +8,7 @@ on: permissions: contents: read issues: write + actions: read # read the previous night's job logs to confirm an alert repeats jobs: compile: @@ -212,6 +213,7 @@ jobs: id: bencher continue-on-error: true run: | + set -o pipefail bencher run \ --project minigraf \ --token "${{ secrets.BENCHER_API_TOKEN }}" \ @@ -222,42 +224,111 @@ jobs: --threshold-upper-boundary ${{ matrix.threshold }} \ --err \ --adapter rust_criterion \ - --file ${{ matrix.file }} - - - name: Open regression issue + --file ${{ matrix.file }} \ + 2>&1 | tee bencher_out.txt + + # Shared CI runners are noisy, so one alert on its own is usually a slow + # runner, not a regression. Report only benchmarks that also alerted in + # this suite on the previous run. If a regression issue is already open, + # add a comment to it instead of skipping, so later alerts stay visible. + - name: Report regression if: steps.bencher.outcome == 'failure' uses: actions/github-script@v6 + env: + SUITE: ${{ matrix.name }} with: script: | - const today = new Date().toISOString().slice(0, 10); - const title = `Benchmark regression - ${today}`; + const fs = require('fs'); + const { owner, repo } = context.repo; + const suite = process.env.SUITE; + const jobName = `bench / ${suite}`; + + // Bencher lists each alert as "- (Latency): <.../alerts/>". + // The "View results" lines use /perf? URLs, so they do not match. + const alertRe = /- (\S+) \(Latency\): https:\/\/bencher\.dev\/console\/projects\/[^/\s]+\/alerts\//g; + const parseAlerts = text => new Set([...text.matchAll(alertRe)].map(m => m[1])); + + const current = parseAlerts(fs.readFileSync('bencher_out.txt', 'utf8')); + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + + let body; + if (current.size === 0) { + // Bencher failed without listing alerts (API or upload error). + // Report it at once so a broken setup does not go unnoticed. + body = [ + `Bencher failed in the nightly benchmark run (${suite}) but listed no alerts. This is likely an API or upload error.`, + '', + `**Run:** ${runUrl}`, + ].join('\n'); + } else { + // Find this suite's job in the previous completed run and read its alerts. + let previous = new Set(); + let previousUrl = null; + try { + const { data: runs } = await github.rest.actions.listWorkflowRuns({ + owner, repo, + workflow_id: 'bench.yml', + branch: 'main', + status: 'completed', + per_page: 10, + }); + const prevRun = runs.workflow_runs.find(r => r.id !== context.runId); + if (prevRun) { + previousUrl = prevRun.html_url; + const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({ + owner, repo, run_id: prevRun.id, per_page: 100, + }); + const prevJob = jobs.jobs.find(j => j.name === jobName); + if (prevJob) { + const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({ + owner, repo, job_id: prevJob.id, + }); + previous = parseAlerts(typeof log === 'string' ? log : Buffer.from(log).toString('utf8')); + } + } + } catch (err) { + core.warning(`Could not read the previous run's alerts: ${err.message}`); + } + + const repeated = [...current].filter(b => previous.has(b)).sort(); + console.log(`Alerts this run: ${[...current].join(', ')}`); + console.log(`Alerts in previous run: ${[...previous].join(', ') || 'none'}`); + if (repeated.length === 0) { + core.notice(`No benchmark in ${suite} alerted on two runs in a row. Not reporting.`); + return; + } + + body = [ + `Bencher flagged these benchmarks in the nightly run (${suite}) on two runs in a row:`, + '', + ...repeated.map(b => `- \`${b}\``), + '', + `**Run:** ${runUrl}`, + `**Previous run:** ${previousUrl}`, + '', + 'Please review the Bencher dashboard to confirm the regression.', + ].join('\n'); + } const { data: openIssues } = await github.rest.issues.listForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, + owner, repo, labels: 'performance', state: 'open', per_page: 100, }); - const existing = openIssues.filter(i => - i.title.startsWith('Benchmark regression') - ); - if (existing.length > 0) { - console.log(`Skipping: open regression issue already exists (#${existing[0].number})`); + const existing = openIssues.find(i => i.title.startsWith('Benchmark regression')); + if (existing) { + await github.rest.issues.createComment({ + owner, repo, issue_number: existing.number, body, + }); + console.log(`Commented on open regression issue #${existing.number}`); return; } - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const today = new Date().toISOString().slice(0, 10); await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: title, - body: [ - `Bencher detected a performance regression in the nightly benchmark run (${{ matrix.name }}).`, - '', - `**Run:** ${runUrl}`, - '', - 'Please review the Bencher dashboard for details on which benchmarks regressed.', - ].join('\n'), + owner, repo, + title: `Benchmark regression - ${today}`, + body, labels: ['performance'], }); From 12b1ec310d99a64114ebabc46621bc8be4bedd3d Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 12:33:03 +0530 Subject: [PATCH 06/10] perf: checkpoint copies untouched index leaves instead of rebuilding every index (#315) (#402) * docs: design spec for incremental checkpoint index rebuild (#315) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * docs: implementation plan for incremental checkpoint (#315) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * bench: add checkpoint/after_1_fact group (#315) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * bench: run one dirty fact per checkpoint iteration (#315) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * refactor(btree_v6): extract leaf encoding, fill rule, internal-level build (#315) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * feat(btree_v6): incremental rebuild that copies untouched leaves (#315) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * perf: checkpoint copies untouched index leaves and skips re-hashing fact pages (#315) save() snapshots each index's raw leaf pages and rebuilds via rebuild_btree_incremental: leaves no pending entry touches are copied verbatim, touched leaves split balanced so repeated checkpoints do not fragment the tree. The CRC32 state over unchanged fact pages is cached so each save hashes only new fact pages and the index region. File format and stored checksum are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * docs: checkpoint cost and durability semantics; #315 benchmarks and doc sync Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy * fix: reject cyclic leaf chains and damaged leaves before copying them (#315) Untouched leaves are now copied verbatim and sealed under a fresh file checksum, so collect_leaf_pages bounds-checks each leaf's slot directory and caps the chain at the file's page count instead of looping. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 3 + CLAUDE.md | 4 +- README.md | 2 +- benches/minigraf_bench.rs | 35 + docs/BENCHMARKS.md | 13 + docs/TEST_COVERAGE.md | 4 +- .../2026-09-26-incremental-checkpoint.md | 1076 +++++++++++++++++ ...026-09-26-incremental-checkpoint-design.md | 175 +++ src/db.rs | 24 +- src/storage/btree_v6.rs | 641 +++++++++- src/storage/persistent_facts.rs | 336 ++++- 11 files changed, 2211 insertions(+), 102 deletions(-) create mode 100644 docs/superpowers/plans/2026-09-26-incremental-checkpoint.md create mode 100644 docs/superpowers/specs/2026-09-26-incremental-checkpoint-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cb3be6768..d479cb9cab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Bound-entity point queries no longer pay for other attributes' history (#323).** `[:e :attr ?v]` now range-scans only `(e, :attr)` in the EAVT index instead of every record the entity has ever written. Reading a rarely changed attribute of a heavily rewritten entity no longer slows down as that entity's history grows: with 2,000 retract/reassert cycles on another attribute of the same entity, it drops from 4.58 ms to 19.0 µs. Reading the heavily rewritten attribute itself, and attribute scans (`[?e :attr ?v]`), are about 1.35–1.4× faster (4.64 ms → 3.36 ms), from cheaper net-assert grouping and removing a redundant dedup pass. The file format is unchanged. - **Attribute scans are bounded to exactly the queried attribute (#381).** `[?e :attr ?v]` computed the end of its AEVT range by incrementing the attribute's last byte. For attributes whose last byte is `0x7F` or `0xBF` — including about 1 in 64 non-ASCII characters, such as `:丿` or `:ÿ` — the result was not valid UTF-8, so the scan ran to the end of the whole AEVT index and read every later fact before filtering. It also read facts for prefix siblings (`:ab`, `:a/b` when scanning `:a`). The range now ends at `attribute + "\0"`, which covers exactly one attribute. Scanning 100 `:丿` facts in a checkpointed file with 40,000 facts on neighbouring attributes drops from 45.4 ms to 251 µs. Results are unchanged. +- **Checkpoints no longer re-encode the whole index (#315).** `checkpoint()` decoded and re-serialised every entry of all four covering indexes and re-read every page of the file for its checksum, so checkpointing after one new fact cost almost as much as after thousands. Index leaves that receive no new entries are now copied verbatim, only the leaves that do are decoded and re-split, and unchanged fact pages are no longer re-hashed. Leaves are still bounds-checked and the leaf chain is checked for cycles, so a damaged index fails the checkpoint instead of being copied forward. Checkpoint after one new fact drops from 246 ms to 26.3 ms on a 100k-fact file and from 23.9 ms to 4.0 ms at 10k. Cost still grows with graph size, because the index pages are copied on every checkpoint; the file format is unchanged. The `checkpoint()` and `wal_checkpoint_threshold` docs now state that a checkpoint is not a durability boundary: the WAL is already crash-durable. + ### Fixed - **Bound-entity queries could drop rows when one `WriteTransaction` wrote the same attribute in several valid-time windows (#323).** Both facts share a `tx_count`, and the selective lookup path de-duplicated on `(entity, attribute, tx_count, asserted)`, so one window's value was silently lost, while the same query via a full scan returned both. That de-duplication has been removed, and bound-entity queries now always match a full scan. @@ -19,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Notes - On v2.x, reading a heavily rewritten attribute still costs time proportional to its history, because v7 index keys carry neither the value nor the assert/retract flag; the structural fix needs the v8 keys and is tracked for v3.0.0 in #379. +- Checkpoints on v2.x still copy every index page, so their cost grows with graph size. Copy-on-write index pages on the v3 branch, which also make `save()` crash-atomic, are needed for checkpoints proportional to the change alone (#374). ## v2.0.1 — 2026-09-25 diff --git a/CLAUDE.md b/CLAUDE.md index 78c9a85788..ff620b95b0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,7 +108,7 @@ cargo run < demos/demo_negation.txt - `backend/file.rs`: Single `.graph` file backend (4KB pages, cross-platform) - `backend/memory.rs`: In-memory backend for testing - `index.rs`: EAVT / AEVT / AVET / VAET key types, `FactRef`, `encode_value` - - `btree_v6.rs`: On-disk B+tree (`build_btree`, `OnDiskIndexReader`, `MutexStorageBackend`) + - `btree_v6.rs`: On-disk B+tree (`build_btree`, `rebuild_btree_incremental`, `OnDiskIndexReader`, `MutexStorageBackend`) - `btree.rs`: Legacy v5 B+tree (migration only) - `cache.rs`: LRU page cache (`PageCache`, default 256 pages) - `packed_pages.rs`: Packed fact pages (~25 facts/4KB page), `MAX_FACT_BYTES` @@ -167,7 +167,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1176 tests passing** (1168 passing, 8 ignored; unit + integration + doc). +**1190 tests passing** (1182 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 874758f154..aad4e86be5 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1176 tests +cargo test # run 1190 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/benches/minigraf_bench.rs b/benches/minigraf_bench.rs index d4b901a7c1..2de35d982a 100644 --- a/benches/minigraf_bench.rs +++ b/benches/minigraf_bench.rs @@ -415,6 +415,40 @@ fn bench_checkpoint(c: &mut Criterion) { group.finish(); } +// ── checkpoint/after_1_fact (#315) ──────────────────────────────────────────── + +/// Checkpoint cost when only one fact is dirty, on an already-checkpointed graph. +/// Before #315 this was flat in dirty bytes and proportional to graph size. +fn bench_checkpoint_after_1_fact(c: &mut Criterion) { + use criterion::BatchSize; + use tempfile::NamedTempFile; + + let mut group = c.benchmark_group("checkpoint/after_1_fact"); + group.sample_size(20); + for &(label, n) in &[("10k", 10_000usize), ("100k", 100_000)] { + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + helpers::populate_file_no_checkpoint(n, &path); + let db = helpers::open_file_no_checkpoint(&path); + db.checkpoint().unwrap(); + let mut i = 0u64; + group.bench_function(BenchmarkId::from_parameter(label), |b| { + b.iter_batched( + || { + i += 1; + db.execute(&format!("(transact [[:ck{i} :val {i}]])")) + .unwrap(); + }, + // One dirty fact per checkpoint: batched setups would leave + // every checkpoint after the first with nothing to do. + |()| db.checkpoint().unwrap(), + BatchSize::PerIteration, + ); + }); + } + group.finish(); +} + // ── Task 10: concurrent/ ───────────────────────────────────────────────────── fn bench_concurrent(c: &mut Criterion) { @@ -1793,6 +1827,7 @@ criterion_group!( bench_query_extras, bench_open, bench_checkpoint, + bench_checkpoint_after_1_fact, bench_concurrent, bench_concurrent_file, bench_concurrent_btree_scan, diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 8b13948d99..d67494cbab 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -50,6 +50,19 @@ One entity's `:hash` is retracted and reasserted `depth` times (exactly one live The churned attribute still scales with its own history on v2.x; see #379 for the v3.0.0 fix. +### Checkpoint After One Dirty Fact (#315) + +**Date**: 2026-09-26 · **Command**: `cargo bench --bench minigraf_bench -- checkpoint/after_1_fact` · same host as above; file on tmpfs, so fsync cost is excluded. + +An already-checkpointed file database of `n` facts receives one new fact, then `checkpoint()` runs; every iteration has exactly one dirty fact. "Before" is v2.0.1, which decoded and re-encoded every index entry. "After" copies index leaves that receive no new entries. + +| Facts | Before | After | +|---:|---:|---:| +| 10k | 23.9 ms | 4.01 ms | +| 100k | 246 ms | 26.3 ms | + +Cost still grows with graph size, because index pages are copied on every checkpoint. Checkpoints proportional to the change alone need copy-on-write pages, tracked with #374 for v3.0.0. + ## Measurement Method Criterion warms up each benchmark, collects ten samples for this query group, and estimates per-call latency from repeated iterations. The reported confidence intervals describe measurement uncertainty on this host; they are not cross-machine performance guarantees. diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index beb7cb9c04..5850fcb4e5 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,9 +4,9 @@ **Verified**: 2026-09-26 with `cargo test` -**Result**: 1,168 passing tests and 8 ignored tests (1,176 total) +**Result**: 1,182 passing tests and 8 ignored tests (1,190 total) -The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). +The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a cyclic leaf chain or damaged slot directory is rejected rather than copied (#315). ### Covered Areas diff --git a/docs/superpowers/plans/2026-09-26-incremental-checkpoint.md b/docs/superpowers/plans/2026-09-26-incremental-checkpoint.md new file mode 100644 index 0000000000..88eca25c69 --- /dev/null +++ b/docs/superpowers/plans/2026-09-26-incremental-checkpoint.md @@ -0,0 +1,1076 @@ +# Incremental Checkpoint Index Rebuild Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `checkpoint()` decode/re-encode only the index leaves the delta touches, and stop re-reading unchanged fact pages for the file checksum, without changing file format v7. + +**Architecture:** `save()` snapshots each old B+tree's raw leaf pages, then `rebuild_btree_incremental` copies untouched leaves verbatim (patching only `next_leaf`), repacks only leaves that receive pending entries, and rebuilds internal levels with a helper extracted from `build_btree`. `PersistentFactStorage` caches the CRC32 state over its (immutable) fact pages so each save hashes only new fact pages plus the index region. + +**Tech Stack:** Rust (MSRV 1.89), postcard, crc32fast, criterion. + +**Spec:** `docs/superpowers/specs/2026-09-26-incremental-checkpoint-design.md` + +## Global Constraints + +- File format stays v7: node layout, header layout, and `index_checksum` value (CRC32 over pages `1..page_count`) are unchanged; files must open on v2.0.1 without the index-rebuild path. +- No public API change; no new dependencies (dev or runtime). +- Test assert messages must never format `Result`, `Fact`, `Value`, `EdnValue`, `EavtKey`, `FactRef` or anything containing a `Uuid` with `{:?}` (CodeQL `rust/cleartext-logging`). Use plain string messages. +- Match the surrounding clippy style: `btree_v6.rs` functions that index/do arithmetic carry `#[allow(clippy::arithmetic_side_effects)]` / `#[allow(clippy::indexing_slicing)]`; errors use `err_coded!`/`bail_coded!` with `ErrorCode::Int049` for B+tree invariant violations. +- `cargo test --release` does not work (panic=abort profile); use `cargo test` and `cargo bench`. +- Pre-push hook runs `cargo fmt --check`, `cargo clippy`, `cargo test`; run `cargo fmt` and `cargo clippy --all-targets -- -D warnings` before each commit. +- Commit messages end with: + ``` + Co-Authored-By: Claude Opus 5.5 + Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy + ``` +- Success target: 50k-fact graph, checkpoint after 1 fact ≤ 15 ms (was ~80 ms). + +## Review Focus + +1. **New index region overlaps the old one** — `save()` writes new fact pages over the start of the old index, and the new tree starts where the old tree's leaves were. Expect: entries identical to a from-scratch build. Pinned by `test_incremental_new_tree_overlaps_old_pages` (Task 3). +2. **Repeated checkpoints on an incrementally built tree** — the second, third… checkpoint consumes a tree whose leaves were raw-copied/split by the previous one. Expect: still exact. Pinned by `test_incremental_repeated_rounds` (Task 3) and `test_incremental_saves_keep_indexes_and_checksum_exact` (Task 4). +3. **Pending keys outside the old key range** (below the first leaf's first key, above the last leaf's last key). Expect: land in first/last leaf, range scans find them. Pinned by `test_incremental_pending_outside_old_range` (Task 3). +4. **Stale or wrong checksum-prefix cache** (after a failed save, a rebuild-on-open, or a page-count mismatch). Expect: silent fallback to a full pass and a correct stored checksum. Pinned by `test_prefix_cache_page_count_mismatch_falls_back` and `test_prefix_cache_absent_after_rebuild_on_open` (Task 4). +5. **Many pending entries into one leaf** (hot key range, e.g. all new facts for one entity). Expect: leaf splits into several ≤75%-fill leaves, no overflow. Pinned by `test_incremental_many_pending_into_one_leaf_splits` (Task 3). + +--- + +### Task 1: Benchmark `checkpoint/after_1_fact` and record the baseline + +**Files:** +- Modify: `benches/minigraf_bench.rs` (new function after `bench_checkpoint`, ~line 416; register in `criterion_group!` at ~line 1778) + +**Interfaces:** +- Consumes: `helpers::populate_file_no_checkpoint(n, &path)`, `helpers::open_file_no_checkpoint(&path) -> Arc` +- Produces: criterion group `checkpoint/after_1_fact` with ids `10k`, `100k` + +- [ ] **Step 1: Add the bench function** after `bench_checkpoint`: + +```rust +// ── checkpoint/after_1_fact (#315) ──────────────────────────────────────────── + +/// Checkpoint cost when only one fact is dirty, on an already-checkpointed graph. +/// Before #315 this was flat in dirty bytes and proportional to graph size. +fn bench_checkpoint_after_1_fact(c: &mut Criterion) { + use criterion::BatchSize; + use tempfile::NamedTempFile; + + let mut group = c.benchmark_group("checkpoint/after_1_fact"); + group.sample_size(20); + for &(label, n) in &[("10k", 10_000usize), ("100k", 100_000)] { + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + helpers::populate_file_no_checkpoint(n, &path); + let db = helpers::open_file_no_checkpoint(&path); + db.checkpoint().unwrap(); + let mut i = 0u64; + group.bench_function(BenchmarkId::from_parameter(label), |b| { + b.iter_batched( + || { + i += 1; + db.execute(&format!("(transact [[:ck{i} :val {i}]])")) + .unwrap(); + }, + |()| db.checkpoint().unwrap(), + BatchSize::SmallInput, + ); + }); + } + group.finish(); +} +``` + +- [ ] **Step 2: Register it** — add `bench_checkpoint_after_1_fact,` to the `criterion_group!(benches, ...)` list directly after `bench_checkpoint,`. + +- [ ] **Step 3: Record the baseline** + +Run: `cargo bench --bench minigraf_bench -- checkpoint/after_1_fact` +Expected: completes; note the two `time:` medians (≈ tens of ms at 100k) in the task report — they are the "Before" column for Task 5. + +- [ ] **Step 4: Commit** + +```bash +cargo fmt && git add benches/minigraf_bench.rs +git commit -m "bench: add checkpoint/after_1_fact group (#315)" +``` + +--- + +### Task 2: Extract leaf encoding, fill rule and internal-level build from `build_btree` + +Pure refactor; existing `btree_v6` tests are the safety net. + +**Files:** +- Modify: `src/storage/btree_v6.rs:62-420` + +**Interfaces:** +- Produces (module-private, used by Task 3): + - `fn encode_leaf_page(entries: &[Vec], next_leaf: u64) -> Result>` + - `fn leaf_overflows(n_entries: usize, data_bytes: usize, entry_len: usize) -> bool` + - `fn build_internal_levels(leaf_infos: Vec<(u64, Vec)>, backend: &mut dyn StorageBackend, cache: &PageCache, next_page: u64) -> Result<(u64, u64)>` — returns `(root_page_id, next_free_page_id)` + +- [ ] **Step 1: Split `write_leaf_page`.** Rename the current body (everything up to but excluding `backend.write_page`) into `encode_leaf_page` returning the page, and make `write_leaf_page` a wrapper: + +```rust +/// Encode a leaf page: fixed header, slot directory, entries written end-to-start. +/// +/// `entries`: each element is the postcard-serialised `(K, FactRef)` bytes for +/// one index entry, in sort order. +#[allow(clippy::arithmetic_side_effects, clippy::indexing_slicing)] +fn encode_leaf_page(entries: &[Vec], next_leaf: u64) -> Result> { + // ... existing body of write_leaf_page from `let entry_count = ...` + // through the slot-directory loop, unchanged ... + Ok(page) +} + +/// Write a single leaf page and insert it into the cache. +fn write_leaf_page( + backend: &mut dyn StorageBackend, + cache: &PageCache, + page_id: u64, + entries: &[Vec], + next_leaf: u64, +) -> Result<()> { + let page = encode_leaf_page(entries, next_leaf)?; + backend.write_page(page_id, &page)?; + cache.put_dirty(page_id, page); + Ok(()) +} +``` + +- [ ] **Step 2: Extract the fill rule.** Add below `write_internal_page`: + +```rust +/// True when adding an entry of `entry_len` bytes to a leaf that already holds +/// `n_entries` entries totalling `data_bytes` would exceed the fill threshold. +#[allow(clippy::arithmetic_side_effects)] +fn leaf_overflows(n_entries: usize, data_bytes: usize, entry_len: usize) -> bool { + LEAF_HEADER_SIZE + (n_entries + 1) * SLOT_SIZE + data_bytes + entry_len > PAGE_FILL_BYTES +} +``` + +and in `build_btree` phase 1 replace + +```rust + let projected = LEAF_HEADER_SIZE + + (cur_entries.len() + 1) * SLOT_SIZE + + cur_data_bytes + + entry_bytes.len(); + + if projected > PAGE_FILL_BYTES && !cur_entries.is_empty() { +``` + +with + +```rust + if !cur_entries.is_empty() + && leaf_overflows(cur_entries.len(), cur_data_bytes, entry_bytes.len()) + { +``` + +- [ ] **Step 3: Extract phase 2.** Move everything in `build_btree` from `// Single leaf: it is the root` to the end of the function into: + +```rust +/// Build internal levels bottom-up over `leaf_infos` (`(page_id, first_key_bytes)` +/// per leaf, in key order), writing nodes from `next_page` onward. +/// +/// Returns `(root_page_id, next_free_page_id)`. A single leaf is its own root. +#[allow(clippy::arithmetic_side_effects, clippy::indexing_slicing)] +fn build_internal_levels( + leaf_infos: Vec<(u64, Vec)>, + backend: &mut dyn StorageBackend, + cache: &PageCache, + next_page: u64, +) -> Result<(u64, u64)> { + if leaf_infos.is_empty() { + bail_coded!( + ErrorCode::Int049, + "build_internal_levels: no leaves".to_string() + ); + } + let mut next_page = next_page; + let mut current_level = leaf_infos; + loop { + // ... existing phase-2 loop body, unchanged (it already returns + // `(current_level[0].0, next_page)` when current_level.len() == 1) ... + } +} +``` + +Delete the now-redundant "Single leaf: it is the root" block (the loop handles `len() == 1`). End `build_btree` with: + +```rust + build_internal_levels(leaf_infos, backend, cache, next_page) +``` + +- [ ] **Step 4: Run the B+tree and storage tests** + +Run: `cargo test --lib storage::` +Expected: all PASS (no behaviour change). + +- [ ] **Step 5: Commit** + +```bash +cargo fmt && cargo clippy --all-targets -- -D warnings +git add src/storage/btree_v6.rs +git commit -m "refactor(btree_v6): extract leaf encoding, fill rule, internal-level build (#315)" +``` + +--- + +### Task 3: `collect_leaf_pages` and `rebuild_btree_incremental` + +**Files:** +- Modify: `src/storage/btree_v6.rs` (new section after `merge_sorted_vecs`; tests in the existing `mod tests`) + +**Interfaces:** +- Consumes (Task 2): `encode_leaf_page`, `leaf_overflows`, `build_internal_levels`; existing `build_btree`, `btree_entries`, `merge_sorted_vecs`, `read_leaf_entries`, `find_leftmost_leaf`, `read_u16_at`, `read_u64_at`. +- Produces (used by Task 4): + - `pub fn collect_leaf_pages(root_page_id: u64, backend: &dyn StorageBackend, cache: &PageCache) -> Result>>>` + - `pub fn rebuild_btree_incremental(old_leaves: Vec>>, pending: Vec<(K, FactRef)>, backend: &mut dyn StorageBackend, cache: &PageCache, start_page_id: u64) -> Result<(u64, u64)> where K: Serialize + for<'de> Deserialize<'de> + Ord` — `pending` must be sorted; returns `(root_page_id, next_free_page_id)`. + +- [ ] **Step 1: Write the failing tests** in `mod tests` (add `use std::sync::Arc;` if not already imported there): + +```rust + /// Deterministic xorshift64 so randomized tests are reproducible without deps. + struct Rng(u64); + impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + } + + /// `n` EAVT entries with random entities; `counter` keeps keys unique. + fn random_eavt(rng: &mut Rng, n: usize, counter: &mut u64) -> Vec<(EavtKey, FactRef)> { + let mut v: Vec<(EavtKey, FactRef)> = (0..n) + .map(|_| { + *counter += 1; + let entity = (u128::from(rng.next()) << 64) | u128::from(*counter); + make_eavt(entity, ":attr", *counter) + }) + .collect(); + v.sort(); + v + } + + fn sorted_union( + a: &[(EavtKey, FactRef)], + b: &[(EavtKey, FactRef)], + ) -> Vec<(EavtKey, FactRef)> { + let mut v: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); + v.sort(); + v + } + + /// Build `committed` at page 1, then rebuild with `pending` starting at `start`. + fn build_then_incremental( + committed: &[(EavtKey, FactRef)], + pending: &[(EavtKey, FactRef)], + start: u64, + ) -> (MemoryBackend, PageCache, u64, u64) { + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let ser = btree_entries(committed.iter().cloned()).unwrap(); + let (old_root, _) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + let leaves = collect_leaf_pages(old_root, &backend, &cache).unwrap(); + let (root, next_free) = + rebuild_btree_incremental(leaves, pending.to_vec(), &mut backend, &cache, start) + .unwrap(); + (backend, cache, root, next_free) + } + + /// Stream equality, a strictly ascending next_leaf chain of non-empty leaves, + /// and range_scan agreement on random bounds. + fn assert_tree_exact( + root: u64, + backend: &MemoryBackend, + cache: &PageCache, + expected: &[(EavtKey, FactRef)], + rng: &mut Rng, + ) { + let got: Vec<(EavtKey, FactRef)> = stream_all_entries(root, backend, cache).unwrap(); + assert_eq!(got.len(), expected.len(), "entry count differs"); + assert!(got == expected, "streamed entries differ from expected"); + + let leaves = collect_leaf_pages(root, backend, cache).unwrap(); + if expected.is_empty() { + assert_eq!(leaves.len(), 1, "empty tree is one empty leaf"); + } else { + for page in &leaves { + assert!(read_u16_at(&page[..], 2).unwrap() > 0, "empty leaf in tree"); + } + } + + for _ in 0..20 { + if expected.is_empty() { + break; + } + let a = (rng.next() as usize) % expected.len(); + let b = (rng.next() as usize) % expected.len(); + let (lo, hi) = (a.min(b), a.max(b)); + let start = &expected[lo].0; + let end = &expected[hi].0; + let want: Vec = expected[lo..hi].iter().map(|(_, r)| *r).collect(); + let refs = range_scan(root, start, Some(end), backend, cache).unwrap(); + assert!(refs == want, "range_scan differs from expected slice"); + } + } + + #[test] + fn test_incremental_no_pending_copies_leaves_verbatim() { + let mut rng = Rng(0x315); + let mut ctr = 0; + let committed = random_eavt(&mut rng, 2000, &mut ctr); + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let ser = btree_entries(committed.iter().cloned()).unwrap(); + let (old_root, old_next) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + let old_leaves = collect_leaf_pages(old_root, &backend, &cache).unwrap(); + let (root, _) = rebuild_btree_incremental( + old_leaves.clone(), + Vec::new(), + &mut backend, + &cache, + old_next, + ) + .unwrap(); + let new_leaves = collect_leaf_pages(root, &backend, &cache).unwrap(); + assert_eq!(new_leaves.len(), old_leaves.len(), "leaf count changed"); + for (o, n) in old_leaves.iter().zip(new_leaves.iter()) { + assert!(o[..4] == n[..4], "leaf header prefix changed"); + assert!(o[12..] == n[12..], "leaf body changed"); + } + assert_tree_exact(root, &backend, &cache, &committed, &mut rng); + } + + #[test] + fn test_incremental_matches_merge_random() { + let mut rng = Rng(0xC0FFEE); + let mut ctr = 0; + for round in 0..40 { + let committed = random_eavt(&mut rng, (rng.next() % 3000) as usize, &mut ctr); + let pending = random_eavt(&mut rng, (rng.next() % 200) as usize, &mut ctr); + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 5000); + let expected = sorted_union(&committed, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + let _ = round; + } + } + + #[test] + fn test_incremental_pending_outside_old_range() { + let mut rng = Rng(7); + let committed: Vec<_> = (1000u128..3000).map(|n| make_eavt(n, ":a", n as u64)).collect(); + let pending = vec![ + make_eavt(1, ":a", 1), + make_eavt(2, ":a", 2), + make_eavt(9_000, ":a", 9_000), + make_eavt(9_001, ":a", 9_001), + ]; + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 5000); + assert_tree_exact(root, &backend, &cache, &sorted_union(&committed, &pending), &mut rng); + } + + #[test] + fn test_incremental_many_pending_into_one_leaf_splits() { + let mut rng = Rng(11); + // Committed entities are 0, 10_000, 20_000, ...: pending 0 < e < 10_000 all route to leaf 0. + let committed: Vec<_> = (0u128..2000) + .map(|n| make_eavt(n * 10_000, ":a", n as u64 + 1)) + .collect(); + let pending: Vec<_> = (1u128..3000) + .map(|n| make_eavt(n, ":a", 100_000 + n as u64)) + .collect(); + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 5000); + assert_tree_exact(root, &backend, &cache, &sorted_union(&committed, &pending), &mut rng); + } + + #[test] + fn test_incremental_empty_old_tree_falls_back_to_bulk_build() { + let mut rng = Rng(13); + let mut ctr = 0; + let pending = random_eavt(&mut rng, 500, &mut ctr); + let (backend, cache, root, _) = build_then_incremental(&[], &pending, 5000); + assert_tree_exact(root, &backend, &cache, &pending, &mut rng); + // Nothing old, nothing new: still a valid single empty leaf. + let (backend, cache, root, next) = build_then_incremental(&[], &[], 5000); + assert_eq!(next, root + 1, "empty result is one page"); + assert_tree_exact(root, &backend, &cache, &[], &mut rng); + } + + #[test] + fn test_incremental_new_tree_overlaps_old_pages() { + // save() writes the new tree over the old tree's pages; old leaves are + // snapshotted first, so the result must still be exact. + let mut rng = Rng(17); + let mut ctr = 0; + let committed = random_eavt(&mut rng, 3000, &mut ctr); + let pending = random_eavt(&mut rng, 100, &mut ctr); + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 3); + assert_tree_exact(root, &backend, &cache, &sorted_union(&committed, &pending), &mut rng); + } + + #[test] + fn test_incremental_repeated_rounds() { + // Each round consumes the previous round's incrementally built tree, + // writing the new tree at the same start page like save() does. + let mut rng = Rng(19); + let mut ctr = 0; + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let mut expected = random_eavt(&mut rng, 1500, &mut ctr); + let ser = btree_entries(expected.iter().cloned()).unwrap(); + let (mut root, _) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + for round in 0..30u64 { + let pending = random_eavt(&mut rng, 1 + (rng.next() % 150) as usize, &mut ctr); + let leaves = collect_leaf_pages(root, &backend, &cache).unwrap(); + let start = 1 + round % 3; + let (r, _) = + rebuild_btree_incremental(leaves, pending.clone(), &mut backend, &cache, start) + .unwrap(); + root = r; + expected = sorted_union(&expected, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + } + } +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test --lib btree_v6::tests::test_incremental` +Expected: compile error — `collect_leaf_pages` / `rebuild_btree_incremental` not found. + +- [ ] **Step 3: Implement.** Add after `merge_sorted_vecs`: + +```rust +// ─── Incremental rebuild (#315) ─────────────────────────────────────────────── + +/// Collect the raw leaf pages of the B+tree at `root_page_id`, in key order. +/// +/// `save()` calls this before writing anything: new fact pages overwrite the +/// start of the old index region, so the old leaves must be snapshotted first. +pub fn collect_leaf_pages( + root_page_id: u64, + backend: &dyn StorageBackend, + cache: &PageCache, +) -> Result>>> { + let mut leaves = Vec::new(); + let mut leaf_id = find_leftmost_leaf(root_page_id, backend, cache)?; + loop { + let page = cache.get_or_load(leaf_id, backend)?; + if page.first().copied() != Some(PAGE_TYPE_LEAF) { + bail_coded!( + ErrorCode::Int049, + format!("collect_leaf_pages: expected leaf page at page_id={leaf_id}") + ); + } + let next_leaf = read_u64_at(&page[..], 4)?; + leaves.push(page); + if next_leaf == 0 { + break; + } + leaf_id = next_leaf; + } + Ok(leaves) +} + +/// Decode the key of a leaf's first entry; `None` for an empty leaf. +#[allow(clippy::arithmetic_side_effects)] +fn leaf_first_key(page: &[u8]) -> Result> +where + K: for<'de> Deserialize<'de>, +{ + if read_u16_at(page, 2)? == 0 { + return Ok(None); + } + let offset = read_u16_at(page, LEAF_HEADER_SIZE)? as usize; + let length = read_u16_at(page, LEAF_HEADER_SIZE + 2)? as usize; + let slice = page + .get(offset..offset.saturating_add(length)) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("first entry out of bounds: offset={offset} len={length}") + ) + })?; + let (key, _): (K, FactRef) = postcard::from_bytes(slice)?; + Ok(Some(key)) +} + +/// Writes leaf pages at consecutive page ids, linking each to the next. +/// +/// One page is held back so its `next_leaf` can be set to the following page +/// id, or to 0 once it is known to be the last leaf. +struct LeafEmitter<'a> { + backend: &'a mut dyn StorageBackend, + cache: &'a PageCache, + next_page: u64, + held: Option<(u64, Vec)>, + infos: Vec<(u64, Vec)>, +} + +impl LeafEmitter<'_> { + #[allow(clippy::arithmetic_side_effects)] + fn push(&mut self, page: Vec, first_key_bytes: Vec) -> Result<()> { + let page_id = self.next_page; + self.next_page += 1; + self.release_held(page_id)?; + self.held = Some((page_id, page)); + self.infos.push((page_id, first_key_bytes)); + Ok(()) + } + + fn release_held(&mut self, next_leaf: u64) -> Result<()> { + if let Some((page_id, mut page)) = self.held.take() { + page.get_mut(4..12) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "page too small to write next_leaf".to_string() + ) + })? + .copy_from_slice(&next_leaf.to_le_bytes()); + self.backend.write_page(page_id, &page)?; + self.cache.put_dirty(page_id, page); + } + Ok(()) + } + + /// Write the last held leaf and return `(leaf_infos, next_free_page_id)`. + fn finish(mut self) -> Result<(Vec<(u64, Vec)>, u64)> { + self.release_held(0)?; + Ok((self.infos, self.next_page)) + } +} + +/// Serialise sorted entries and emit them as leaves using `build_btree`'s fill rule. +#[allow(clippy::arithmetic_side_effects)] +fn emit_packed( + emitter: &mut LeafEmitter<'_>, + entries: impl Iterator, +) -> Result<()> { + let mut cur: Vec> = Vec::new(); + let mut cur_bytes = 0usize; + let mut cur_first: Option> = None; + for (key, fact_ref) in entries { + let entry = postcard::to_allocvec(&(&key, &fact_ref))?; + if !cur.is_empty() && leaf_overflows(cur.len(), cur_bytes, entry.len()) { + let first = cur_first.take().ok_or_else(|| { + err_coded!(ErrorCode::Int049, "BUG: leaf without first key".to_string()) + })?; + emitter.push(encode_leaf_page(&cur, 0)?, first)?; + cur.clear(); + cur_bytes = 0; + } + if cur_first.is_none() { + cur_first = Some(postcard::to_allocvec(&key)?); + } + cur_bytes += entry.len(); + cur.push(entry); + } + if let Some(first) = cur_first { + emitter.push(encode_leaf_page(&cur, 0)?, first)?; + } + Ok(()) +} + +/// Rebuild a B+tree from its old leaves plus sorted `pending` entries. +/// +/// Leaves that receive no pending entry are copied verbatim (only `next_leaf` +/// is patched); only leaves that do are decoded, merged and repacked. Internal +/// levels are rebuilt from scratch. Pending keys route to leaf `i` when +/// `first_key[i] <= key < first_key[i + 1]`; keys below the first leaf's first +/// key go to leaf 0. With no non-empty old leaves this is a plain bulk build. +/// +/// `old_leaves` must be snapshotted (see [`collect_leaf_pages`]) before any +/// page in `start_page_id..` is written. Returns `(root_page_id, next_free_page_id)`. +pub fn rebuild_btree_incremental( + old_leaves: Vec>>, + pending: Vec<(K, FactRef)>, + backend: &mut dyn StorageBackend, + cache: &PageCache, + start_page_id: u64, +) -> Result<(u64, u64)> +where + K: Serialize + for<'de> Deserialize<'de> + Ord, +{ + let mut leaves: Vec<(Arc>, K)> = Vec::with_capacity(old_leaves.len()); + for page in old_leaves { + if let Some(first_key) = leaf_first_key::(&page[..])? { + leaves.push((page, first_key)); + } + } + if leaves.is_empty() { + return build_btree( + btree_entries(pending.into_iter())?.into_iter(), + backend, + cache, + start_page_id, + ); + } + + let mut emitter = LeafEmitter { + backend: &mut *backend, + cache, + next_page: start_page_id, + held: None, + infos: Vec::new(), + }; + let mut pending = pending.into_iter().peekable(); + for (i, (page, first_key)) in leaves.iter().enumerate() { + let upper = leaves.get(i.saturating_add(1)).map(|(_, k)| k); + let mut batch = Vec::new(); + while let Some((key, _)) = pending.peek() { + if upper.is_some_and(|u| key >= u) { + break; + } + if let Some(entry) = pending.next() { + batch.push(entry); + } + } + if batch.is_empty() { + emitter.push((**page).clone(), postcard::to_allocvec(first_key)?)?; + } else { + let old = read_leaf_entries::(&page[..])?; + emit_packed(&mut emitter, merge_sorted_vecs(old, batch))?; + } + } + let (leaf_infos, next_page) = emitter.finish()?; + build_internal_levels(leaf_infos, backend, cache, next_page) +} +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test --lib btree_v6::` +Expected: all PASS, including the seven `test_incremental_*`. + +- [ ] **Step 5: Commit** + +```bash +cargo fmt && cargo clippy --all-targets -- -D warnings +git add src/storage/btree_v6.rs +git commit -m "feat(btree_v6): incremental rebuild that copies untouched leaves (#315)" +``` + +--- + +### Task 4: Wire into `save()` and cache the fact-page checksum prefix + +**Files:** +- Modify: `src/storage/persistent_facts.rs` — struct (~line 145), `new` (~line 166), `load` checksum block (~lines 277-301), `save` Steps A/D/checksum (~lines 842-970), `compute_page_checksum` (~line 1102), imports (lines 10-13), tests module + +**Interfaces:** +- Consumes (Task 3): `collect_leaf_pages`, `rebuild_btree_incremental`. +- Produces: field `fact_prefix_crc: Option<(u64, Hasher)>`; `fn hash_pages(backend: &dyn StorageBackend, hasher: &mut Hasher, first_page: u64, num_pages: u64) -> Result<()>`. + +- [ ] **Step 1: Write the failing tests** in the `#[cfg(test)] mod tests` of `persistent_facts.rs`: + +```rust + /// All four on-disk indexes equal a from-scratch derivation from the fact pages, + /// and the stored checksum equals a full pass over pages 1..page_count. + fn assert_indexes_and_checksum_exact( + pfs: &PersistentFactStorage, + ) { + let backend = pfs.backend.lock().unwrap(); + let header = FileHeader::from_bytes(&backend.read_page(0).unwrap()).unwrap(); + let full = compute_page_checksum(&*backend, 1, header.page_count - 1).unwrap(); + assert_eq!(header.index_checksum, full, "stored checksum != full pass"); + + let (facts, refs) = crate::storage::packed_pages::read_all_with_refs( + &*backend, + 1, + header.fact_page_count, + ) + .unwrap(); + let (eavt, aevt, avet, vaet) = build_sorted_index_entries(&facts, &refs); + let cache = &pfs.page_cache; + let got_eavt: Vec<(EavtKey, FactRef)> = + stream_all_entries(header.eavt_root_page, &*backend, cache).unwrap(); + let got_aevt: Vec<(AevtKey, FactRef)> = + stream_all_entries(header.aevt_root_page, &*backend, cache).unwrap(); + let got_avet: Vec<(AvetKey, FactRef)> = + stream_all_entries(header.avet_root_page, &*backend, cache).unwrap(); + let got_vaet: Vec<(VaetKey, FactRef)> = + stream_all_entries(header.vaet_root_page, &*backend, cache).unwrap(); + assert!(got_eavt == eavt, "EAVT differs from full derivation"); + assert!(got_aevt == aevt, "AEVT differs from full derivation"); + assert!(got_avet == avet, "AVET differs from full derivation"); + assert!(got_vaet == vaet, "VAET differs from full derivation"); + } + + /// One transact of `n` facts: new and reused entities, strings and refs (VAET). + fn transact_mixed( + pfs: &mut PersistentFactStorage, + entities: &mut Vec, + n: usize, + seed: u64, + ) { + let mut batch = Vec::new(); + for i in 0..n { + let k = seed.wrapping_mul(31).wrapping_add(i as u64); + let e = if entities.is_empty() || k % 3 == 0 { + let e = Uuid::new_v4(); + entities.push(e); + e + } else { + entities[(k as usize) % entities.len()] + }; + let value = if k % 4 == 0 { + Value::Ref(entities[(k as usize / 4) % entities.len()]) + } else { + Value::String(format!("v{k}")) + }; + batch.push((e, format!(":a{}", k % 5), value)); + } + pfs.storage().transact(batch, None).unwrap(); + pfs.mark_dirty(); + } + + #[test] + fn test_incremental_saves_keep_indexes_and_checksum_exact() { + let mut pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); + let mut entities = Vec::new(); + for round in 0..25u64 { + let n = 1 + ((round * 37) % 60) as usize; + transact_mixed(&mut pfs, &mut entities, n, round); + pfs.save().unwrap(); + assert!(pfs.fact_prefix_crc.is_some(), "prefix cached after save"); + assert_indexes_and_checksum_exact(&pfs); + } + } + + #[test] + fn test_reopen_after_incremental_saves_takes_no_rebuild() { + use crate::storage::backend::FileBackend; + use tempfile::NamedTempFile; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let mut entities = Vec::new(); + { + let mut pfs = + PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + for round in 0..10u64 { + transact_mixed(&mut pfs, &mut entities, 40, round); + pfs.save().unwrap(); + } + } + let mut pfs = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + // Set only when the full checksum matched on load, i.e. no rebuild. + assert!(pfs.fact_prefix_crc.is_some(), "full checksum must match on reopen"); + transact_mixed(&mut pfs, &mut entities, 40, 99); + pfs.save().unwrap(); + assert_indexes_and_checksum_exact(&pfs); + } + + #[test] + fn test_prefix_cache_page_count_mismatch_falls_back() { + let mut pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); + let mut entities = Vec::new(); + transact_mixed(&mut pfs, &mut entities, 50, 1); + pfs.save().unwrap(); + pfs.fact_prefix_crc = Some((999, Hasher::new())); + transact_mixed(&mut pfs, &mut entities, 50, 2); + pfs.save().unwrap(); + assert_indexes_and_checksum_exact(&pfs); + } + + #[test] + fn test_prefix_cache_absent_after_rebuild_on_open() { + use crate::storage::backend::FileBackend; + use tempfile::NamedTempFile; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let mut entities = Vec::new(); + { + let mut pfs = + PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + for round in 0..3u64 { + transact_mixed(&mut pfs, &mut entities, 30, round); + pfs.save().unwrap(); + } + } + // Corrupt index_checksum (re-sealing the header) to force the rebuild path. + { + let mut backend = FileBackend::open(&path).unwrap(); + let mut page = backend.read_page(0).unwrap(); + page[64] ^= 0xFF; + let cs = compute_header_checksum_from_bytes(&page); + page[80..84].copy_from_slice(&cs.to_le_bytes()); + backend.write_page(0, &page).unwrap(); + backend.sync().unwrap(); + } + let mut pfs = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + assert!(pfs.fact_prefix_crc.is_none(), "no prefix after rebuild-on-open"); + transact_mixed(&mut pfs, &mut entities, 30, 7); + pfs.save().unwrap(); + assert_indexes_and_checksum_exact(&pfs); + } +``` + +Add any missing test-module imports (`FileHeader`, `StorageBackend`, `MemoryBackend`, `Value`, `Uuid`, `Hasher`, `EavtKey`/`AevtKey`/`AvetKey`/`VaetKey`, `FactRef`, `stream_all_entries`) — most are already imported via `use super::*;`; the compiler will list the rest. + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test --lib persistent_facts::tests::test_ -- incremental prefix reopen_after` +Expected: compile error — no field `fact_prefix_crc`. + +- [ ] **Step 3: Add the field.** In `PersistentFactStorage`: + +```rust + committed_fact_pages: Arc, + /// CRC32 state after hashing fact pages `1..=n` (`n` is the first element). + /// Fact pages are never rewritten, so `save()` extends this instead of + /// re-reading them; `None` (or a stale `n`) falls back to a full pass (#315). + fact_prefix_crc: Option<(u64, Hasher)>, +``` + +and `fact_prefix_crc: None,` in the constructor literal in `new`. + +- [ ] **Step 4: Add `hash_pages`** and re-express `compute_page_checksum` with it: + +```rust +/// Feed pages `first_page..first_page + num_pages` into `hasher`. +fn hash_pages( + backend: &dyn StorageBackend, + hasher: &mut Hasher, + first_page: u64, + num_pages: u64, +) -> Result<()> { + for i in 0..num_pages { + let page_id = first_page + .checked_add(i) + .ok_or_else(|| err_coded!(ErrorCode::Stg021))?; + hasher.update(&backend.read_page(page_id)?); + } + Ok(()) +} + +fn compute_page_checksum( + backend: &dyn StorageBackend, + first_page: u64, + num_pages: u64, +) -> Result { + let mut hasher = Hasher::new(); + hash_pages(backend, &mut hasher, first_page, num_pages)?; + Ok(hasher.finalize()) +} +``` + +- [ ] **Step 5: Capture the prefix in `load()`.** In the `needs_rebuild` block replace + +```rust + let full_checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; + if full_checksum == stored { + false // new-style checksum matches: facts + indexes verified +``` + +with + +```rust + // Hash fact pages, keep that state for save() (#315), then the index pages. + let (full_checksum, fact_prefix) = if total_data_pages >= num_fact_pages { + let mut hasher = Hasher::new(); + hash_pages(&*backend, &mut hasher, 1, num_fact_pages)?; + let prefix = hasher.clone(); + let index_start = num_fact_pages + .checked_add(1) + .ok_or_else(|| err_coded!(ErrorCode::Stg021))?; + hash_pages( + &*backend, + &mut hasher, + index_start, + total_data_pages - num_fact_pages, + )?; + (hasher.finalize(), Some(prefix)) + } else { + (compute_page_checksum(&*backend, 1, total_data_pages)?, None) + }; + if full_checksum == stored { + self.fact_prefix_crc = fact_prefix.map(|p| (num_fact_pages, p)); + false // new-style checksum matches: facts + indexes verified +``` + +(The subtraction is guarded by the `>=` check; add `#[allow(clippy::arithmetic_side_effects)]` only if clippy asks.) + +- [ ] **Step 6: Step A of `save()`** — replace the four `committed_* = stream_all_entries(...)` blocks with raw-leaf snapshots: + +```rust + // Snapshot old B+tree leaves BEFORE writing new pages that may overlap them. + let old_leaves = |root: u64| -> Result>>> { + if root == 0 { + Ok(Vec::new()) + } else { + collect_leaf_pages(root, &*backend, &self.page_cache) + } + }; + let old_eavt = old_leaves(curr_header.eavt_root_page)?; + let old_aevt = old_leaves(curr_header.aevt_root_page)?; + let old_avet = old_leaves(curr_header.avet_root_page)?; + let old_vaet = old_leaves(curr_header.vaet_root_page)?; +``` + +(If the closure's borrow of `backend` conflicts with the later `&mut *backend`, it ends at its last use; if the borrow checker still objects, inline the four `if` expressions.) + +- [ ] **Step 7: Step D of `save()`** — replace the four `*_ser` / `build_btree` pairs with: + +```rust + let (eavt_root, next1) = rebuild_btree_incremental( + old_eavt, + pending_eavt, + &mut *backend, + &self.page_cache, + index_start, + )?; + let (aevt_root, next2) = + rebuild_btree_incremental(old_aevt, pending_aevt, &mut *backend, &self.page_cache, next1)?; + let (avet_root, next3) = + rebuild_btree_incremental(old_avet, pending_avet, &mut *backend, &self.page_cache, next2)?; + let (vaet_root, next4) = + rebuild_btree_incremental(old_vaet, pending_vaet, &mut *backend, &self.page_cache, next3)?; +``` + +- [ ] **Step 8: Checksum in `save()`** — replace + +```rust + let total_data_pages = next4.saturating_sub(1); + let checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; +``` + +with + +```rust + // Fact pages 1..=old count are unchanged since the last save/load, so extend + // the cached CRC state instead of re-reading them (#315). Taking the cache + // means a failed save leaves None and the next save does a full pass. + let mut hasher = match self.fact_prefix_crc.take() { + Some((n, prefix)) if n == old_fact_page_count => { + let mut h = prefix; + hash_pages(&*backend, &mut h, new_fact_start, new_pages_len)?; + h + } + _ => { + let mut h = Hasher::new(); + hash_pages(&*backend, &mut h, 1, new_total_fact_pages)?; + h + } + }; + let new_prefix = hasher.clone(); + hash_pages( + &*backend, + &mut hasher, + index_start, + next4.saturating_sub(index_start), + )?; + let checksum = hasher.finalize(); +``` + +and after `self.committed_fact_pages.store(new_total_fact_pages, ...)` add: + +```rust + self.fact_prefix_crc = Some((new_total_fact_pages, new_prefix)); +``` + +- [ ] **Step 9: Fix imports** — add `collect_leaf_pages, rebuild_btree_incremental` to the `use crate::storage::btree_v6::{...}` list; drop names clippy reports unused in non-test code (e.g. `merge_sorted_vecs`; `stream_all_entries` if only tests use it — then import it inside `mod tests`). + +- [ ] **Step 10: Run the tests** + +Run: `cargo test --lib persistent_facts::` then `cargo test` +Expected: all PASS; total = baseline 1168 + 7 (Task 3) + 4 (this task) = 1179 passing, 8 ignored. + +- [ ] **Step 11: Measure** — `cargo bench --bench minigraf_bench -- checkpoint/after_1_fact`. Expected: substantially below the Task 1 baseline. Also rerun the scratch harness if still present (50k facts target ≤ 15 ms). Record numbers in the task report. + +- [ ] **Step 12: Commit** + +```bash +cargo fmt && cargo clippy --all-targets -- -D warnings +git add src/storage/persistent_facts.rs +git commit -m "perf: checkpoint copies untouched index leaves and skips re-hashing fact pages (#315)" +``` + +--- + +### Task 5: Docs, comment trim, doc sync + +**Files:** +- Modify: `src/db.rs` (`OpenOptions::wal_checkpoint_threshold` field doc ~line 112; `Minigraf::checkpoint` doc ~line 684; `do_checkpoint` guard comment ~lines 730-742) +- Modify: `docs/BENCHMARKS.md`, `CHANGELOG.md`, `CLAUDE.md`, `README.md`, `docs/TEST_COVERAGE.md` + +- [ ] **Step 1: `Minigraf::checkpoint` rustdoc** — replace the first paragraph with: + +```rust + /// Manually trigger a checkpoint: flush all in-memory facts to the main file + /// and delete the WAL sidecar. + /// + /// A checkpoint is **not** a durability boundary: committed writes are already + /// crash-durable in the `.wal` sidecar and are replayed on the next open. + /// Checkpointing less often only makes that replay (reopen) slower. + /// + /// Cost: every checkpoint rewrites the four covering indexes after the fact + /// pages, so it copies pages in proportion to the total index size; only the + /// index leaves that receive new entries are decoded and re-encoded. Prefer + /// checkpointing on a size or time budget over once per logical unit of work. + /// + /// No-op for in-memory databases. +``` + +- [ ] **Step 2: `wal_checkpoint_threshold` field doc** — append: + +```rust + /// + /// A checkpoint's cost grows with the total size of the database (see + /// [`Minigraf::checkpoint`]), while durability does not depend on it: the WAL + /// is crash-durable. Raise this for write-heavy workloads on large graphs. +``` + +- [ ] **Step 3: Trim the `do_checkpoint` guard comment** — replace the paragraph starting "The kernel file lock taken by FileBackend::open…" through "…concurrent writers." with: + +```rust + // Kernel file locking refuses a second writer (another process, or a + // second handle in this one — #304/#314). The guard still matters where + // the filesystem cannot lock and the caller set `allow_unlocked`. +``` + +- [ ] **Step 4: `docs/BENCHMARKS.md`** — add after the "Point Query vs. Version-Chain Depth (#323)" section: + +```markdown +### Checkpoint After One Dirty Fact (#315) + +**Date**: · **Command**: `cargo bench --bench minigraf_bench -- checkpoint/after_1_fact` · same host as above. + +An already-checkpointed file database of `n` facts receives one new fact, then `checkpoint()` runs. "Before" is v2.0.1 behaviour (every index entry decoded and re-encoded); "after" copies untouched index leaves verbatim. + +| Facts | Before | After | +|---:|---:|---:| +| 10k | | | +| 100k | | | + +Cost still grows with graph size (index pages are copied every checkpoint); O(delta) checkpoints need copy-on-write pages and are tracked with #374 for v3.0.0. +``` + +Fill `<…>` with the recorded numbers — no placeholders may remain. + +- [ ] **Step 5: `CHANGELOG.md`** — under `## Unreleased` → `### Performance` (create headings if absent), add: + +```markdown +- **Checkpoints no longer re-encode the whole index (#315).** `checkpoint()` used to decode and re-serialise every entry of all four covering indexes and re-read every page for the file checksum, so checkpointing after one new fact cost as much as after thousands. Index leaves that receive no new entries are now copied verbatim, and unchanged fact pages are not re-hashed. On a 100k-fact file, checkpoint after one fact drops from to . Cost still scales with graph size in page copies; the file format is unchanged. The `checkpoint()` docs now state that a checkpoint is not a durability boundary. +``` + +- [ ] **Step 6: Test counts** — run `cargo test 2>&1 | grep "^test result"` and sum; update `CLAUDE.md` (`**N tests passing** (P passing, 8 ignored…)`), `README.md` (`cargo test # run N tests`), `docs/TEST_COVERAGE.md` (`**Verified**` date, `**Result**` line, and one sentence appended to the suite summary: "Since v2.0.1, incremental B+tree rebuild tests check that checkpoints which copy untouched leaves produce exactly the indexes and checksum a full rebuild would (#315)."). + +- [ ] **Step 7: Verify and commit** + +Run: `cargo test` and `cargo doc --no-deps 2>&1 | grep -i warning` (expect none new) + +```bash +cargo fmt && cargo clippy --all-targets -- -D warnings +git add src/db.rs docs/BENCHMARKS.md CHANGELOG.md CLAUDE.md README.md docs/TEST_COVERAGE.md +git commit -m "docs: checkpoint cost and durability semantics; #315 benchmarks and doc sync" +``` + +- [ ] **Step 8 (at PR time, with the user's go-ahead):** comment on #374 that copy-on-write pages there also own #315's remaining O(size) page-copy term. The PR body uses "Mitigates #315" wording chosen with the user, never a closing keyword for #374. diff --git a/docs/superpowers/specs/2026-09-26-incremental-checkpoint-design.md b/docs/superpowers/specs/2026-09-26-incremental-checkpoint-design.md new file mode 100644 index 0000000000..5c6bf4fa0d --- /dev/null +++ b/docs/superpowers/specs/2026-09-26-incremental-checkpoint-design.md @@ -0,0 +1,175 @@ +# Incremental checkpoint index rebuild (#315) — Design + +**Issue:** #315 — `checkpoint()` is O(graph size) and flat in dirty bytes +**Milestone:** v2.3.0 (`main`, file format v7 — no format change) +**Follow-up:** true O(delta) checkpoints move to #374 on the `v3` branch (see §7) + +## 1. Problem + +Every `checkpoint()` streams all four covering indexes (EAVT/AEVT/AVET/VAET) off +disk, decodes every entry, merges in the pending entries, re-serialises everything +and bulk-builds four new B+trees. Cost scales with total graph size, not with the +number of dirty facts. Downstream (temporal_reasoning#241) this was ~half of +at-scale ingestion wall clock. The default `wal_checkpoint_threshold` (1000) makes +every default user pay `N_checkpoints × graph_size`. + +Reproduced locally (50k facts, 13 MB, checkpoint after **one** new fact, mean of 20): +**~80 ms**. + +## 2. Where the time goes + +Phase timing of `PersistentFactStorage::save` (tmpfs, same harness): + +| phase | ms | +|---|---:| +| merge + serialise + `build_btree` (Step D) | ~58 | +| `stream_all_entries` × 4 (Step A) | ~19 | +| full-file CRC re-read | ~3.5 | +| pack facts + fsyncs | <1 (a few ms on NVMe) | + +The cost is CPU in decode/re-encode of every index entry, not I/O. + +## 3. Why true O(delta) is out of scope here + +In format v7, fact pages must be contiguous at pages `1..=fact_page_count` +(`read_all_with_refs`, `CommittedFactLoaderImpl::first_fact_page`), and indexes live +directly after them. New fact pages therefore always overwrite the start of the old +index region, so every checkpoint must relocate the index. Avoiding that needs +copy-on-write B+trees plus free-space management — a format change, and the same +change #374 (save() is not crash-atomic) requires. That work belongs on `v3`. + +Dropping the unconditional `force_dirty()` (issue option 2) does not help: with one +dirty fact `dirty` is already true, and the zero-dirty case is already short-circuited +by the `wal_entry_count == 0 && !pfs.is_dirty()` guard. + +## 4. Design + +### 4.1 Leaf reuse — `src/storage/btree_v6.rs` + +`btree_v6.rs` is the B+tree for both v6 and v7 (v7 changed only the header). The +node layout is unchanged by this design, so v2.x readers are unaffected. + +New function: + +```rust +pub fn rebuild_btree_incremental( + old_leaves: Vec>>, // raw leaf pages of the old tree, in next_leaf order + pending: Vec<(K, FactRef)>, // sorted + backend: &mut dyn StorageBackend, + cache: &PageCache, + start_page_id: u64, +) -> Result<(u64, u64)> // (root_page_id, next_free_page_id) +where K: Serialize + for<'de> Deserialize<'de> + Ord +``` + +Algorithm: + +1. For each old leaf decode **only the first entry's key** → `first_key[i]`. +2. Route each pending entry to leaf `i` where `first_key[i] ≤ key < first_key[i+1]`; + keys below `first_key[0]` go to leaf 0. +3. Leaf with no pending entries → **raw copy**: write the 4 KB page verbatim to the + next page id, patching only bytes 4..12 (`next_leaf`). No decode/encode. +4. Leaf with pending entries → decode all its entries, merge with its pending + entries and serialise. If the result fits in one physical page it stays one + leaf; otherwise it splits into pages of at most `PAGE_FILL_BYTES`, balanced by + size (classic B+tree split). *Revised during implementation:* repacking with + `build_btree`'s "fill to 75%, then start a new leaf" rule left a near-empty + leaf on every checkpoint that inserted into a full leaf (43 → 596 leaves after + 600 single-fact checkpoints), so cost drifted upward with checkpoint count. +5. Leaves are written in order with `next_leaf = pid + 1` (0 for the last), so there + is no second patch pass. +6. Collect `(page_id, first_key_bytes)` per new leaf (postcard of the first key) and + build internal levels bottom-up via a helper **extracted from `build_btree`'s + phase 2** (`build_internal_levels`), shared by both paths. + +`build_btree` remains the path for the first save, for an empty or missing old tree +(root 0 or a single empty leaf), and for load-time index rebuild / migrations. + +Collecting raw leaves: `collect_leaf_pages(root, backend, cache) -> Vec>>` +walks `find_leftmost_leaf` + the `next_leaf` chain, validating page type as +`stream_all_entries` does. + +Leaf fill: untouched leaves keep their existing fill; touched leaves stay between +roughly 37% and 100% full. The tree shape differs from a fresh bulk build but is a +valid B+tree for `range_scan` / `stream_all_entries` / `find_leaf_for_key`. No +merging of under-filled neighbours (inserts only; nothing is ever removed from an +index). + +### 4.2 `save()` ordering — `src/storage/persistent_facts.rs` + +Step A reads the raw leaf pages of all four old trees into memory **before** any +write (new fact pages overwrite the start of the old index region). This replaces +the decoded `Vec<(K, FactRef)>` streams; memory drops (raw pages ≤ decoded entries ++ serialised copies held today). + +Step D calls `rebuild_btree_incremental` per index when the old root is non-zero, +otherwise `build_btree` as today. + +Crash semantics are unchanged: the WAL is only deleted after the header write, and a +checksum mismatch on open rebuilds indexes from fact pages (#370 path). #374 remains +the real fix for atomicity. + +### 4.3 Checksum prefix cache + +`PersistentFactStorage` gains `fact_prefix_crc: Option` — CRC32 +state after hashing fact pages `1..=committed_fact_pages`. + +- **Set** in `load()`: the existing full-checksum pass is split into fact pages then + index pages; the hasher is cloned between the two segments. Only set when the + new-style full checksum matched. +- **Set** at the end of `save()`: prefix + the new fact pages (hashed from the + in-memory page buffers). +- **Cleared (None)** on any other path that rewrites the header/indexes (index + rebuild in `load()`, legacy/v5 migrations). +- **Use** in `save()`: clone the prefix, update with the new fact pages from memory, + then read back and hash only the index pages. `None` → today's full pass. + +The stored `index_checksum` value is identical to today's (CRC over pages +`1..page_count`), so older v2.x readers verify it unchanged. + +### 4.4 Docs and cleanup + +- `Minigraf::checkpoint` rustdoc: cost is O(graph size) page copies plus O(delta) + decode; checkpoint is **not** a durability boundary (the WAL is crash-durable); + batching trades only reopen latency (WAL replay). +- Same note on `OpenOptions::wal_checkpoint_threshold`. +- Trim the obsolete "same-process double-opens" rationale in `do_checkpoint`'s + guard comment (now refused outright since #304/#314). +- At release: `CHANGELOG.md`, `ROADMAP.md`, `docs/BENCHMARKS.md`, + `docs/TEST_COVERAGE.md`, `CLAUDE.md` test count. + +## 5. Testing + +No new dependencies; deterministic seeded loops. + +- **Equivalence (unit, `btree_v6.rs`):** for random committed sets and random + pending batches (including batches that force leaf splits, keys below the first + leaf, keys after the last leaf, and many pending entries in one leaf), + `stream_all_entries` of the incremental tree equals the merged sorted set. +- **Structure:** `next_leaf` chain is strictly ascending across leaves; `range_scan` + over random bounds matches a filter over the expected set. +- **Multi-checkpoint (`persistent_facts.rs`):** many save cycles on a file backend; + after each, all four indexes stream equal to a from-scratch build over all facts. +- **Checksum:** after incremental saves `header.index_checksum == + compute_page_checksum(1, page_count-1)`; reopening does not take the rebuild path. + Prefix cache is `None` after a rebuild/migration path and a subsequent save still + writes a correct checksum. +- **Regression:** full existing suite. +- **Benchmark:** new `checkpoint/after_1_fact` group (10k / 100k facts) in + `benches/minigraf_bench.rs`; before/after numbers in `docs/BENCHMARKS.md`. + +Test assert messages follow the CodeQL rule (no `{:?}` of UUID-bearing types). + +## 6. Success criteria + +- 50k facts, checkpoint after 1 fact: ~80 ms → **≤ 15 ms** on the §1 harness. +- No file-format or public-API change; files written by the new code open without + index rebuild on v2.0.1. +- All existing tests pass. + +## 7. Follow-up + +Comment on #374 (v3.0.0): copy-on-write B+tree pages with a free list make save() +crash-atomic **and** give O(delta · log n) checkpoints; #315's remaining O(size) +page-copy term is owned there. #315 is closed by this work as mitigated, not fully +resolved (no closing keyword on #374). diff --git a/src/db.rs b/src/db.rs index 4d0a4c4b6b..eb1dc65480 100644 --- a/src/db.rs +++ b/src/db.rs @@ -113,6 +113,10 @@ pub struct OpenOptions { /// /// Defaults to 1000. Lower values mean more frequent checkpoints (smaller WAL, /// more I/O). Higher values mean less frequent checkpoints (larger WAL, less I/O). + /// + /// A checkpoint's cost grows with the total size of the database (see + /// [`Minigraf::checkpoint`]), while durability does not depend on it: the WAL + /// is crash-durable. Raise this for write-heavy workloads on large graphs. pub wal_checkpoint_threshold: usize, /// Number of pages to hold in the LRU page cache. Default: 256 (= 1MB at 4KB pages). /// @@ -684,6 +688,15 @@ impl Minigraf { /// Manually trigger a checkpoint: flush all in-memory facts to the main file /// and delete the WAL sidecar. /// + /// A checkpoint is **not** a durability boundary: committed writes are already + /// crash-durable in the `.wal` sidecar and are replayed on the next open. + /// Checkpointing less often only makes that replay (reopen) slower. + /// + /// Cost: every checkpoint rewrites the four covering indexes after the fact + /// pages, so it copies pages in proportion to the total index size; only the + /// index leaves that receive new entries are decoded and re-encoded. Prefer + /// checkpointing on a size or time budget over once per logical unit of work. + /// /// No-op for in-memory databases. /// /// # Errors @@ -733,13 +746,10 @@ impl Minigraf { // replayed WAL entries on open (crash-recovery path). `pfs.is_dirty()` // catches any facts marked dirty via the normal write path. // - // The kernel file lock taken by FileBackend::open covers both a - // second process and a second handle in this one, since flock - // and OFD locks attach to the open file description. This guard - // remains for environments where the filesystem cannot lock at all — - // for example, NFSv3 without lockd or some FUSE mounts — where the - // caller set `allow_unlocked` to accept that Minigraf cannot detect - // concurrent writers. + // Kernel file locking refuses a second writer (another process, or a + // second handle in this one — #304/#314). The guard still matters where + // the filesystem cannot lock (e.g. NFSv3 without lockd, some FUSE + // mounts) and the caller set `allow_unlocked`. if *wal_entry_count == 0 && !pfs.is_dirty() { return Ok(()); } diff --git a/src/storage/btree_v6.rs b/src/storage/btree_v6.rs index 5c5a9b9e14..684d0ca9a5 100644 --- a/src/storage/btree_v6.rs +++ b/src/storage/btree_v6.rs @@ -61,18 +61,12 @@ fn read_u64_at(page: &[u8], offset: usize) -> Result { // ─── Low-level page writers ─────────────────────────────────────────────────── -/// Write a single leaf page and insert it into the cache. +/// Encode a leaf page: fixed header, slot directory, entries written end-to-start. /// /// `entries`: each element is the postcard-serialised `(K, FactRef)` bytes for -/// one index entry, in sort order. Written end-to-start in the page. +/// one index entry, in sort order. #[allow(clippy::arithmetic_side_effects, clippy::indexing_slicing)] -fn write_leaf_page( - backend: &mut dyn StorageBackend, - cache: &PageCache, - page_id: u64, - entries: &[Vec], - next_leaf: u64, -) -> Result<()> { +fn encode_leaf_page(entries: &[Vec], next_leaf: u64) -> Result> { let entry_count = u16::try_from(entries.len()).map_err(|_| { err_coded!( ErrorCode::Int049, @@ -108,7 +102,18 @@ fn write_leaf_page( page[slot_off..slot_off + 2].copy_from_slice(&write_pos_u16.to_le_bytes()); page[slot_off + 2..slot_off + 4].copy_from_slice(&entry_len_u16.to_le_bytes()); } + Ok(page) +} +/// Write a single leaf page and insert it into the cache. +fn write_leaf_page( + backend: &mut dyn StorageBackend, + cache: &PageCache, + page_id: u64, + entries: &[Vec], + next_leaf: u64, +) -> Result<()> { + let page = encode_leaf_page(entries, next_leaf)?; backend.write_page(page_id, &page)?; cache.put_dirty(page_id, page); Ok(()) @@ -188,6 +193,13 @@ fn write_internal_page( Ok(()) } +/// True when adding an entry of `entry_len` bytes to a leaf that already holds +/// `n_entries` entries totalling `data_bytes` would exceed the fill threshold. +#[allow(clippy::arithmetic_side_effects)] +fn leaf_overflows(n_entries: usize, data_bytes: usize, entry_len: usize) -> bool { + LEAF_HEADER_SIZE + (n_entries + 1) * SLOT_SIZE + data_bytes + entry_len > PAGE_FILL_BYTES +} + // ─── build_btree ────────────────────────────────────────────────────────────── /// Serialize `(key, fact_ref)` pairs into the byte format expected by [`build_btree`]. @@ -235,12 +247,9 @@ pub fn build_btree( let mut next_page = start_page_id; for (entry_bytes, key_bytes) in sorted_entries { - let projected = LEAF_HEADER_SIZE - + (cur_entries.len() + 1) * SLOT_SIZE - + cur_data_bytes - + entry_bytes.len(); - - if projected > PAGE_FILL_BYTES && !cur_entries.is_empty() { + if !cur_entries.is_empty() + && leaf_overflows(cur_entries.len(), cur_data_bytes, entry_bytes.len()) + { write_leaf_page(backend, cache, next_page, &cur_entries, 0)?; let first_key = cur_first_key.take().ok_or_else(|| { err_coded!( @@ -310,23 +319,27 @@ pub fn build_btree( cache.put_dirty(pid, page); } - // Single leaf: it is the root - if leaf_infos.len() == 1 { - return Ok(( - leaf_infos - .first() - .ok_or_else(|| { - err_coded!( - ErrorCode::Int049, - "leaf_infos unexpectedly empty".to_string() - ) - })? - .0, - next_page, - )); - } + build_internal_levels(leaf_infos, backend, cache, next_page) +} - // ── Phase 2: build internal levels bottom-up ────────────────────────────── +/// Build internal levels bottom-up over `leaf_infos` (`(page_id, first_key_bytes)` +/// per leaf, in key order), writing nodes from `next_page` onward. +/// +/// Returns `(root_page_id, next_free_page_id)`. A single leaf is its own root. +#[allow(clippy::arithmetic_side_effects, clippy::indexing_slicing)] +fn build_internal_levels( + leaf_infos: Vec<(u64, Vec)>, + backend: &mut dyn StorageBackend, + cache: &PageCache, + next_page: u64, +) -> Result<(u64, u64)> { + if leaf_infos.is_empty() { + bail_coded!( + ErrorCode::Int049, + "build_internal_levels: no leaves".to_string() + ); + } + let mut next_page = next_page; let mut current_level = leaf_infos; loop { @@ -437,6 +450,264 @@ pub fn merge_sorted_vecs(a: Vec, b: Vec) -> impl Iterator Result>>> { + // A chain longer than the file has pages must contain a cycle. + let max_leaves = backend.page_count()?; + let mut leaves = Vec::new(); + let mut leaf_id = find_leftmost_leaf(root_page_id, backend, cache)?; + loop { + let page = cache.get_or_load(leaf_id, backend)?; + if page.first().copied() != Some(PAGE_TYPE_LEAF) { + bail_coded!( + ErrorCode::Int049, + format!("collect_leaf_pages: expected leaf page at page_id={leaf_id}") + ); + } + if u64::try_from(leaves.len()).map_or(true, |n| n >= max_leaves) { + bail_coded!( + ErrorCode::Int049, + format!("collect_leaf_pages: leaf chain longer than the file at page_id={leaf_id}") + ); + } + validate_leaf_slots(&page[..], leaf_id)?; + let next_leaf = read_u64_at(&page[..], 4)?; + leaves.push(page); + if next_leaf == 0 { + break; + } + leaf_id = next_leaf; + } + Ok(leaves) +} + +/// Check a leaf's slot directory stays inside the page without decoding entries. +/// +/// Untouched leaves are copied verbatim and sealed under a fresh file checksum, +/// so damage must be caught here instead of being carried forward. +#[allow(clippy::arithmetic_side_effects)] +fn validate_leaf_slots(page: &[u8], page_id: u64) -> Result<()> { + let count = read_u16_at(page, 2)? as usize; + let data_start = LEAF_HEADER_SIZE + count * SLOT_SIZE; + if data_start > page.len() { + bail_coded!( + ErrorCode::Int049, + format!("leaf page_id={page_id}: slot directory overflows the page") + ); + } + for i in 0..count { + let slot_off = LEAF_HEADER_SIZE + i * SLOT_SIZE; + let offset = read_u16_at(page, slot_off)? as usize; + let length = read_u16_at(page, slot_off + 2)? as usize; + if offset < data_start || offset + length > page.len() { + bail_coded!( + ErrorCode::Int049, + format!("leaf page_id={page_id}: slot {i} out of bounds") + ); + } + } + Ok(()) +} + +/// Decode the key of a leaf's first entry; `None` for an empty leaf. +#[allow(clippy::arithmetic_side_effects)] +fn leaf_first_key(page: &[u8]) -> Result> +where + K: for<'de> Deserialize<'de>, +{ + if read_u16_at(page, 2)? == 0 { + return Ok(None); + } + let offset = read_u16_at(page, LEAF_HEADER_SIZE)? as usize; + let length = read_u16_at(page, LEAF_HEADER_SIZE + 2)? as usize; + let slice = page + .get(offset..offset.saturating_add(length)) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + format!("first entry out of bounds: offset={offset} len={length}") + ) + })?; + let (key, _): (K, FactRef) = postcard::from_bytes(slice)?; + Ok(Some(key)) +} + +/// Writes leaf pages at consecutive page ids, linking each to the next. +/// +/// One page is held back so its `next_leaf` can be set to the following page +/// id, or to 0 once it is known to be the last leaf. +struct LeafEmitter<'a> { + backend: &'a mut dyn StorageBackend, + cache: &'a PageCache, + next_page: u64, + held: Option<(u64, Vec)>, + infos: Vec<(u64, Vec)>, +} + +impl LeafEmitter<'_> { + #[allow(clippy::arithmetic_side_effects)] + fn push(&mut self, page: Vec, first_key_bytes: Vec) -> Result<()> { + let page_id = self.next_page; + self.next_page += 1; + self.release_held(page_id)?; + self.held = Some((page_id, page)); + self.infos.push((page_id, first_key_bytes)); + Ok(()) + } + + fn release_held(&mut self, next_leaf: u64) -> Result<()> { + if let Some((page_id, mut page)) = self.held.take() { + page.get_mut(4..12) + .ok_or_else(|| { + err_coded!( + ErrorCode::Int049, + "page too small to write next_leaf".to_string() + ) + })? + .copy_from_slice(&next_leaf.to_le_bytes()); + self.backend.write_page(page_id, &page)?; + self.cache.put_dirty(page_id, page); + } + Ok(()) + } + + /// Write the last held leaf and return `(leaf_infos, next_free_page_id)`. + #[allow(clippy::type_complexity)] + fn finish(mut self) -> Result<(Vec<(u64, Vec)>, u64)> { + self.release_held(0)?; + Ok((self.infos, self.next_page)) + } +} + +/// Serialise sorted entries and emit them as leaves. +/// +/// Entries that fit in one physical page stay in one leaf. Otherwise they are +/// split into pages of at most `PAGE_FILL_BYTES`, balanced by size, so a leaf that +/// overflows by one entry splits roughly in half. Splitting "full page + remainder" +/// instead left a near-empty leaf behind on every checkpoint that inserted into a +/// full leaf, growing the index without bound (#315). +#[allow(clippy::arithmetic_side_effects)] +fn emit_packed( + emitter: &mut LeafEmitter<'_>, + entries: impl Iterator, +) -> Result<()> { + // Each entry costs its bytes plus one slot-directory entry. + let mut serialised: Vec<(Vec, K)> = Vec::new(); + let mut total = 0usize; + for (key, fact_ref) in entries { + let entry = postcard::to_allocvec(&(&key, &fact_ref))?; + total += entry.len() + SLOT_SIZE; + serialised.push((entry, key)); + } + if serialised.is_empty() { + return Ok(()); + } + let target = if LEAF_HEADER_SIZE + total <= PAGE_SIZE { + total + } else { + let usable = PAGE_FILL_BYTES - LEAF_HEADER_SIZE; + total.div_ceil(total.div_ceil(usable)) + }; + + let mut cur: Vec> = Vec::new(); + let mut cur_size = 0usize; + let mut cur_first: Option> = None; + for (entry, key) in serialised { + let size = entry.len() + SLOT_SIZE; + if !cur.is_empty() && cur_size + size > target { + let first = cur_first.take().ok_or_else(|| { + err_coded!(ErrorCode::Int049, "BUG: leaf without first key".to_string()) + })?; + emitter.push(encode_leaf_page(&cur, 0)?, first)?; + cur.clear(); + cur_size = 0; + } + if cur_first.is_none() { + cur_first = Some(postcard::to_allocvec(&key)?); + } + cur_size += size; + cur.push(entry); + } + if let Some(first) = cur_first { + emitter.push(encode_leaf_page(&cur, 0)?, first)?; + } + Ok(()) +} + +/// Rebuild a B+tree from its old leaves plus sorted `pending` entries. +/// +/// Leaves that receive no pending entry are copied verbatim (only `next_leaf` +/// is patched); only leaves that do are decoded, merged and repacked. Internal +/// levels are rebuilt from scratch. Pending keys route to leaf `i` when +/// `first_key[i] <= key < first_key[i + 1]`; keys below the first leaf's first +/// key go to leaf 0. With no non-empty old leaves this is a plain bulk build. +/// +/// `old_leaves` must be snapshotted (see [`collect_leaf_pages`]) before any +/// page in `start_page_id..` is written. Returns `(root_page_id, next_free_page_id)`. +pub fn rebuild_btree_incremental( + old_leaves: Vec>>, + pending: Vec<(K, FactRef)>, + backend: &mut dyn StorageBackend, + cache: &PageCache, + start_page_id: u64, +) -> Result<(u64, u64)> +where + K: Serialize + for<'de> Deserialize<'de> + Ord, +{ + let mut leaves: Vec<(Arc>, K)> = Vec::with_capacity(old_leaves.len()); + for page in old_leaves { + if let Some(first_key) = leaf_first_key::(&page[..])? { + leaves.push((page, first_key)); + } + } + if leaves.is_empty() { + return build_btree( + btree_entries(pending.into_iter())?.into_iter(), + backend, + cache, + start_page_id, + ); + } + + let mut emitter = LeafEmitter { + backend: &mut *backend, + cache, + next_page: start_page_id, + held: None, + infos: Vec::new(), + }; + let mut pending = pending.into_iter().peekable(); + for (i, (page, first_key)) in leaves.iter().enumerate() { + let upper = leaves.get(i.saturating_add(1)).map(|(_, k)| k); + let mut batch = Vec::new(); + while let Some((key, _)) = pending.peek() { + if upper.is_some_and(|u| key >= u) { + break; + } + if let Some(entry) = pending.next() { + batch.push(entry); + } + } + if batch.is_empty() { + emitter.push((**page).clone(), postcard::to_allocvec(first_key)?)?; + } else { + let old = read_leaf_entries::(&page[..])?; + emit_packed(&mut emitter, merge_sorted_vecs(old, batch))?; + } + } + let (leaf_infos, next_page) = emitter.finish()?; + build_internal_levels(leaf_infos, backend, cache, next_page) +} + // ─── Leaf traversal helpers ─────────────────────────────────────────────────── /// Traverse internal nodes from `root` to find the leftmost (first) leaf page. @@ -571,6 +842,10 @@ where // ─── stream_all_entries ─────────────────────────────────────────────────────── /// Stream all `(K, FactRef)` entries from a B+tree in sorted order. +/// +/// Test-only since #315: `save()` snapshots raw leaves via [`collect_leaf_pages`] +/// instead of decoding every entry. +#[cfg(test)] pub fn stream_all_entries( root_page_id: u64, backend: &dyn StorageBackend, @@ -826,6 +1101,310 @@ mod tests { ) } + /// Deterministic xorshift64 so randomized tests are reproducible without deps. + struct Rng(u64); + impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + } + + /// `n` EAVT entries with random entities; `counter` keeps keys unique. + fn random_eavt(rng: &mut Rng, n: usize, counter: &mut u64) -> Vec<(EavtKey, FactRef)> { + let mut v: Vec<(EavtKey, FactRef)> = (0..n) + .map(|_| { + *counter += 1; + let entity = (u128::from(rng.next()) << 64) | u128::from(*counter); + make_eavt(entity, ":attr", *counter) + }) + .collect(); + v.sort(); + v + } + + fn sorted_union(a: &[(EavtKey, FactRef)], b: &[(EavtKey, FactRef)]) -> Vec<(EavtKey, FactRef)> { + let mut v: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); + v.sort(); + v + } + + /// Build `committed` at page 1, then rebuild with `pending` starting at `start`. + fn build_then_incremental( + committed: &[(EavtKey, FactRef)], + pending: &[(EavtKey, FactRef)], + start: u64, + ) -> (MemoryBackend, PageCache, u64, u64) { + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let ser = btree_entries(committed.iter().cloned()).unwrap(); + let (old_root, _) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + let leaves = collect_leaf_pages(old_root, &backend, &cache).unwrap(); + let (root, next_free) = + rebuild_btree_incremental(leaves, pending.to_vec(), &mut backend, &cache, start) + .unwrap(); + (backend, cache, root, next_free) + } + + /// Stream equality, a next_leaf chain of non-empty leaves, and range_scan + /// agreement on random bounds. + fn assert_tree_exact( + root: u64, + backend: &MemoryBackend, + cache: &PageCache, + expected: &[(EavtKey, FactRef)], + rng: &mut Rng, + ) { + let got: Vec<(EavtKey, FactRef)> = stream_all_entries(root, backend, cache).unwrap(); + assert_eq!(got.len(), expected.len(), "entry count differs"); + assert!(got == expected, "streamed entries differ from expected"); + + let leaves = collect_leaf_pages(root, backend, cache).unwrap(); + if expected.is_empty() { + assert_eq!(leaves.len(), 1, "empty tree is one empty leaf"); + } else { + for page in &leaves { + assert!(read_u16_at(&page[..], 2).unwrap() > 0, "empty leaf in tree"); + } + } + + for _ in 0..20 { + if expected.is_empty() { + break; + } + let a = (rng.next() as usize) % expected.len(); + let b = (rng.next() as usize) % expected.len(); + let (lo, hi) = (a.min(b), a.max(b)); + let start = &expected[lo].0; + let end = &expected[hi].0; + let want: Vec = expected[lo..hi].iter().map(|(_, r)| *r).collect(); + let refs = range_scan(root, start, Some(end), backend, cache).unwrap(); + assert!(refs == want, "range_scan differs from expected slice"); + } + } + + /// Build a multi-leaf tree and return (backend, cache, root, leaf page ids). + fn multi_leaf_tree() -> (MemoryBackend, PageCache, u64, Vec) { + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let entries: Vec<_> = (0u128..500).map(|n| make_eavt(n, ":a", n as u64)).collect(); + let ser = btree_entries(entries.into_iter()).unwrap(); + let (root, _) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + let mut ids = vec![find_leftmost_leaf(root, &backend, &cache).unwrap()]; + loop { + let page = cache.get_or_load(*ids.last().unwrap(), &backend).unwrap(); + let next = read_u64_at(&page[..], 4).unwrap(); + if next == 0 { + break; + } + ids.push(next); + } + assert!(ids.len() >= 3, "fixture needs several leaves"); + (backend, cache, root, ids) + } + + fn overwrite_page(backend: &mut MemoryBackend, cache: &PageCache, id: u64, page: Vec) { + backend.write_page(id, &page).unwrap(); + cache.put_dirty(id, page); + } + + #[test] + fn test_collect_leaf_pages_rejects_cyclic_chain() { + // A corrupt next_leaf pointing back must be an error, not an endless loop. + let (mut backend, cache, root, ids) = multi_leaf_tree(); + let mut page = cache.get_or_load(ids[1], &backend).unwrap().to_vec(); + page[4..12].copy_from_slice(&ids[0].to_le_bytes()); + overwrite_page(&mut backend, &cache, ids[1], page); + assert!( + collect_leaf_pages(root, &backend, &cache).is_err(), + "cyclic leaf chain must be rejected" + ); + } + + #[test] + fn test_collect_leaf_pages_rejects_corrupt_slot_directory() { + // Untouched leaves are copied without decoding, so a damaged slot must be + // caught here rather than sealed under a fresh checksum. + let (mut backend, cache, root, ids) = multi_leaf_tree(); + let mut page = cache.get_or_load(ids[1], &backend).unwrap().to_vec(); + let bad_offset = u16::try_from(PAGE_SIZE - 2).unwrap(); + page[LEAF_HEADER_SIZE..LEAF_HEADER_SIZE + 2].copy_from_slice(&bad_offset.to_le_bytes()); + overwrite_page(&mut backend, &cache, ids[1], page); + assert!( + collect_leaf_pages(root, &backend, &cache).is_err(), + "slot pointing past the page end must be rejected" + ); + + let (mut backend, cache, root, ids) = multi_leaf_tree(); + let mut page = cache.get_or_load(ids[2], &backend).unwrap().to_vec(); + page[2..4].copy_from_slice(&u16::MAX.to_le_bytes()); + overwrite_page(&mut backend, &cache, ids[2], page); + assert!( + collect_leaf_pages(root, &backend, &cache).is_err(), + "entry count overflowing the page must be rejected" + ); + } + + #[test] + fn test_incremental_no_pending_copies_leaves_verbatim() { + let mut rng = Rng(0x315); + let mut ctr = 0; + let committed = random_eavt(&mut rng, 2000, &mut ctr); + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let ser = btree_entries(committed.iter().cloned()).unwrap(); + let (old_root, old_next) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + let old_leaves = collect_leaf_pages(old_root, &backend, &cache).unwrap(); + let (root, _) = rebuild_btree_incremental( + old_leaves.clone(), + Vec::<(EavtKey, FactRef)>::new(), + &mut backend, + &cache, + old_next, + ) + .unwrap(); + let new_leaves = collect_leaf_pages(root, &backend, &cache).unwrap(); + assert_eq!(new_leaves.len(), old_leaves.len(), "leaf count changed"); + for (o, n) in old_leaves.iter().zip(new_leaves.iter()) { + assert!(o[..4] == n[..4], "leaf header prefix changed"); + assert!(o[12..] == n[12..], "leaf body changed"); + } + assert_tree_exact(root, &backend, &cache, &committed, &mut rng); + } + + #[test] + fn test_incremental_matches_merge_random() { + let mut rng = Rng(0xC0FFEE); + let mut ctr = 0; + for _ in 0..40 { + let n_committed = (rng.next() % 3000) as usize; + let committed = random_eavt(&mut rng, n_committed, &mut ctr); + let n_pending = (rng.next() % 200) as usize; + let pending = random_eavt(&mut rng, n_pending, &mut ctr); + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 5000); + let expected = sorted_union(&committed, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + } + } + + #[test] + fn test_incremental_pending_outside_old_range() { + let mut rng = Rng(7); + let committed: Vec<_> = (1000u128..3000) + .map(|n| make_eavt(n, ":a", n as u64)) + .collect(); + let pending = vec![ + make_eavt(1, ":a", 1), + make_eavt(2, ":a", 2), + make_eavt(9_000, ":a", 9_000), + make_eavt(9_001, ":a", 9_001), + ]; + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 5000); + let expected = sorted_union(&committed, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + } + + #[test] + fn test_incremental_many_pending_into_one_leaf_splits() { + let mut rng = Rng(11); + // Committed entities are 0, 10_000, 20_000, …: every pending 0 < e < 10_000 + // routes to leaf 0, which must split into several leaves. + let committed: Vec<_> = (0u128..2000) + .map(|n| make_eavt(n * 10_000, ":a", n as u64 + 1)) + .collect(); + let pending: Vec<_> = (1u128..3000) + .map(|n| make_eavt(n, ":a", 100_000 + n as u64)) + .collect(); + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 5000); + let expected = sorted_union(&committed, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + } + + #[test] + fn test_incremental_empty_old_tree_falls_back_to_bulk_build() { + let mut rng = Rng(13); + let mut ctr = 0; + let pending = random_eavt(&mut rng, 500, &mut ctr); + let (backend, cache, root, _) = build_then_incremental(&[], &pending, 5000); + assert_tree_exact(root, &backend, &cache, &pending, &mut rng); + // Nothing old, nothing new: still a valid single empty leaf. + let (backend, cache, root, next) = build_then_incremental(&[], &[], 5000); + assert_eq!(next, root + 1, "empty result is one page"); + assert_tree_exact(root, &backend, &cache, &[], &mut rng); + } + + #[test] + fn test_incremental_new_tree_overlaps_old_pages() { + // save() writes the new tree over the old tree's pages; old leaves are + // snapshotted first, so the result must still be exact. + let mut rng = Rng(17); + let mut ctr = 0; + let committed = random_eavt(&mut rng, 3000, &mut ctr); + let pending = random_eavt(&mut rng, 100, &mut ctr); + let (backend, cache, root, _) = build_then_incremental(&committed, &pending, 3); + let expected = sorted_union(&committed, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + } + + #[test] + fn test_incremental_single_inserts_do_not_fragment_leaves() { + // One random insert per round into a bulk-built (75%-full) tree, like one + // checkpoint per new fact. Splitting "75% + remainder" left a near-empty + // leaf per insert; leaves must stay reasonably full instead. + let mut rng = Rng(23); + let mut ctr = 0; + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let mut expected = random_eavt(&mut rng, 3000, &mut ctr); + let ser = btree_entries(expected.iter().cloned()).unwrap(); + let (mut root, _) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + let bulk_leaves = collect_leaf_pages(root, &backend, &cache).unwrap().len(); + for _ in 0..600 { + let pending = random_eavt(&mut rng, 1, &mut ctr); + let leaves = collect_leaf_pages(root, &backend, &cache).unwrap(); + root = rebuild_btree_incremental(leaves, pending.clone(), &mut backend, &cache, 1) + .unwrap() + .0; + expected = sorted_union(&expected, &pending); + } + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + // 20% more entries; at >= ~37% fill that is at most ~2.5x the bulk leaf count. + let leaves = collect_leaf_pages(root, &backend, &cache).unwrap().len(); + assert!( + leaves <= bulk_leaves * 5 / 2, + "leaf count grew from {bulk_leaves} to {leaves}" + ); + } + + #[test] + fn test_incremental_repeated_rounds() { + // Each round consumes the previous round's incrementally built tree, + // writing the new tree over the old pages like save() does. + let mut rng = Rng(19); + let mut ctr = 0; + let mut backend = MemoryBackend::new(); + let cache = PageCache::new(4096); + let mut expected = random_eavt(&mut rng, 1500, &mut ctr); + let ser = btree_entries(expected.iter().cloned()).unwrap(); + let (mut root, _) = build_btree(ser.into_iter(), &mut backend, &cache, 1).unwrap(); + for round in 0..30u64 { + let n_pending = 1 + (rng.next() % 150) as usize; + let pending = random_eavt(&mut rng, n_pending, &mut ctr); + let leaves = collect_leaf_pages(root, &backend, &cache).unwrap(); + let start = 1 + round % 3; + let (r, _) = + rebuild_btree_incremental(leaves, pending.clone(), &mut backend, &cache, start) + .unwrap(); + root = r; + expected = sorted_union(&expected, &pending); + assert_tree_exact(root, &backend, &cache, &expected, &mut rng); + } + } + #[test] fn test_read_u16_at_oob_rejected() { let page = vec![0u8; 4]; diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index 2a057513f4..d7147375da 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -8,8 +8,8 @@ use crate::graph::types::Fact; use crate::storage::FACT_PAGE_FORMAT_PACKED; use crate::storage::btree::{read_aevt_index, read_avet_index, read_eavt_index, read_vaet_index}; use crate::storage::btree_v6::{ - MutexStorageBackend, OnDiskIndexReader, btree_entries, build_btree, merge_sorted_vecs, - stream_all_entries, + MutexStorageBackend, OnDiskIndexReader, btree_entries, build_btree, collect_leaf_pages, + rebuild_btree_incremental, }; use crate::storage::cache::PageCache; use crate::storage::index::{AevtKey, AvetKey, EavtKey, FactRef, VaetKey, encode_value}; @@ -149,6 +149,10 @@ pub struct PersistentFactStorage { dirty: bool, last_checkpointed_tx_count: u64, committed_fact_pages: Arc, + /// CRC32 state after hashing fact pages `1..=n` (`n` is the first element). + /// Fact pages are never rewritten, so `save()` extends this instead of + /// re-reading them; `None` (or a stale `n`) falls back to a full pass (#315). + fact_prefix_crc: Option<(u64, Hasher)>, } impl PersistentFactStorage { @@ -170,6 +174,7 @@ impl PersistentFactStorage { dirty: false, last_checkpointed_tx_count: 0, committed_fact_pages, + fact_prefix_crc: None, }; // Try to load existing data. @@ -288,8 +293,26 @@ impl PersistentFactStorage { let stored = header.index_checksum; // Total data pages: pages 1 through page_count-1 (everything except header) let total_data_pages = header.page_count.saturating_sub(1); - let full_checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; + // Hash fact pages, keep that state for save() (#315), then the index pages. + let (full_checksum, fact_prefix) = if total_data_pages >= num_fact_pages { + let mut hasher = Hasher::new(); + hash_pages(&*backend, &mut hasher, 1, num_fact_pages)?; + let prefix = hasher.clone(); + let index_start = num_fact_pages + .checked_add(1) + .ok_or_else(|| err_coded!(ErrorCode::Stg021))?; + hash_pages( + &*backend, + &mut hasher, + index_start, + total_data_pages.saturating_sub(num_fact_pages), + )?; + (hasher.finalize(), Some(prefix)) + } else { + (compute_page_checksum(&*backend, 1, total_data_pages)?, None) + }; if full_checksum == stored { + self.fact_prefix_crc = fact_prefix.map(|p| (num_fact_pages, p)); false // new-style checksum matches: facts + indexes verified } else { // Fall back: old files stored checksum over fact pages only @@ -863,27 +886,18 @@ impl PersistentFactStorage { Err(e) => bail_coded!(ErrorCode::Stg010, e), }; - // Stream committed B+tree entries BEFORE writing new pages that may overlap - let committed_eavt: Vec<(EavtKey, FactRef)> = if curr_header.eavt_root_page != 0 { - stream_all_entries(curr_header.eavt_root_page, &*backend, &self.page_cache)? - } else { - Vec::new() - }; - let committed_aevt: Vec<(AevtKey, FactRef)> = if curr_header.aevt_root_page != 0 { - stream_all_entries(curr_header.aevt_root_page, &*backend, &self.page_cache)? - } else { - Vec::new() - }; - let committed_avet: Vec<(AvetKey, FactRef)> = if curr_header.avet_root_page != 0 { - stream_all_entries(curr_header.avet_root_page, &*backend, &self.page_cache)? - } else { - Vec::new() - }; - let committed_vaet: Vec<(VaetKey, FactRef)> = if curr_header.vaet_root_page != 0 { - stream_all_entries(curr_header.vaet_root_page, &*backend, &self.page_cache)? - } else { - Vec::new() + // Snapshot old B+tree leaves BEFORE writing new pages that may overlap them. + let old_leaves = |root: u64| -> Result>>> { + if root == 0 { + Ok(Vec::new()) + } else { + collect_leaf_pages(root, &*backend, &self.page_cache) + } }; + let old_eavt = old_leaves(curr_header.eavt_root_page)?; + let old_aevt = old_leaves(curr_header.aevt_root_page)?; + let old_avet = old_leaves(curr_header.avet_root_page)?; + let old_vaet = old_leaves(curr_header.vaet_root_page)?; // Invalidate cached pages that will be overwritten (old index pages) self.page_cache.invalidate_from(new_fact_start); @@ -917,41 +931,35 @@ impl PersistentFactStorage { .checked_add(new_total_fact_pages) .ok_or_else(|| err_coded!(ErrorCode::Stg017))?; - let eavt_ser = if !committed_eavt.is_empty() { - btree_entries(merge_sorted_vecs(committed_eavt, pending_eavt))? - } else { - btree_entries(pending_eavt.into_iter())? - }; - let (eavt_root, next1) = build_btree( - eavt_ser.into_iter(), + // Copy untouched leaves, repack only leaves that receive pending entries (#315). + let (eavt_root, next1) = rebuild_btree_incremental( + old_eavt, + pending_eavt, &mut *backend, &self.page_cache, index_start, )?; - - let aevt_ser = if !committed_aevt.is_empty() { - btree_entries(merge_sorted_vecs(committed_aevt, pending_aevt))? - } else { - btree_entries(pending_aevt.into_iter())? - }; - let (aevt_root, next2) = - build_btree(aevt_ser.into_iter(), &mut *backend, &self.page_cache, next1)?; - - let avet_ser = if !committed_avet.is_empty() { - btree_entries(merge_sorted_vecs(committed_avet, pending_avet))? - } else { - btree_entries(pending_avet.into_iter())? - }; - let (avet_root, next3) = - build_btree(avet_ser.into_iter(), &mut *backend, &self.page_cache, next2)?; - - let vaet_ser = if !committed_vaet.is_empty() { - btree_entries(merge_sorted_vecs(committed_vaet, pending_vaet))? - } else { - btree_entries(pending_vaet.into_iter())? - }; - let (vaet_root, next4) = - build_btree(vaet_ser.into_iter(), &mut *backend, &self.page_cache, next3)?; + let (aevt_root, next2) = rebuild_btree_incremental( + old_aevt, + pending_aevt, + &mut *backend, + &self.page_cache, + next1, + )?; + let (avet_root, next3) = rebuild_btree_incremental( + old_avet, + pending_avet, + &mut *backend, + &self.page_cache, + next2, + )?; + let (vaet_root, next4) = rebuild_btree_incremental( + old_vaet, + pending_vaet, + &mut *backend, + &self.page_cache, + next3, + )?; // Sync index pages to disk before writing the header. // The header update is the atomic commit point: once it's durable, @@ -961,8 +969,29 @@ impl PersistentFactStorage { // CRC32 over ALL data pages (facts + indexes), excluding page 0 (header). // This detects corruption in both fact pages and B+tree index pages. - let total_data_pages = next4.saturating_sub(1); - let checksum = compute_page_checksum(&*backend, 1, total_data_pages)?; + // Fact pages 1..=old count are unchanged since the last save/load, so extend + // the cached CRC state instead of re-reading them (#315). Taking the cache + // means a failed save leaves None and the next save does a full pass. + let mut hasher = match self.fact_prefix_crc.take() { + Some((n, prefix)) if n == old_fact_page_count => { + let mut h = prefix; + hash_pages(&*backend, &mut h, new_fact_start, new_pages_len)?; + h + } + _ => { + let mut h = Hasher::new(); + hash_pages(&*backend, &mut h, 1, new_total_fact_pages)?; + h + } + }; + let new_prefix = hasher.clone(); + hash_pages( + &*backend, + &mut hasher, + index_start, + next4.saturating_sub(index_start), + )?; + let checksum = hasher.finalize(); // ── Step E: write header (last write = crash-safe boundary) ───────────── let mut header = FileHeader::new(); // version=7 @@ -991,6 +1020,7 @@ impl PersistentFactStorage { self.committed_fact_pages .store(new_total_fact_pages, Ordering::SeqCst); + self.fact_prefix_crc = Some((new_total_fact_pages, new_prefix)); self.last_checkpointed_tx_count = self.storage.current_tx_count(); self.dirty = false; @@ -1105,14 +1135,24 @@ fn compute_page_checksum( num_pages: u64, ) -> Result { let mut hasher = Hasher::new(); + hash_pages(backend, &mut hasher, first_page, num_pages)?; + Ok(hasher.finalize()) +} + +/// Feed pages `first_page..first_page + num_pages` into `hasher`. +fn hash_pages( + backend: &dyn StorageBackend, + hasher: &mut Hasher, + first_page: u64, + num_pages: u64, +) -> Result<()> { for i in 0..num_pages { let page_id = first_page .checked_add(i) .ok_or_else(|| err_coded!(ErrorCode::Stg021))?; - let page = backend.read_page(page_id)?; - hasher.update(&page); + hasher.update(&backend.read_page(page_id)?); } - Ok(hasher.finalize()) + Ok(()) } /// Compute CRC32 checksum over header bytes 0-79 (header_checksum field zeroed). @@ -1265,9 +1305,187 @@ mod tests { use super::*; use crate::graph::types::Value; use crate::storage::backend::MemoryBackend; + use crate::storage::btree_v6::stream_all_entries; use std::io::Write; use uuid::Uuid; + /// All four on-disk indexes equal a from-scratch derivation from the fact pages, + /// and the stored checksum equals a full pass over pages 1..page_count. + fn assert_indexes_and_checksum_exact( + pfs: &PersistentFactStorage, + ) { + let backend = pfs.backend.lock().unwrap(); + let header = FileHeader::from_bytes(&backend.read_page(0).unwrap()).unwrap(); + let full = compute_page_checksum(&*backend, 1, header.page_count - 1).unwrap(); + assert_eq!(header.index_checksum, full, "stored checksum != full pass"); + + let (facts, refs) = + crate::storage::packed_pages::read_all_with_refs(&*backend, 1, header.fact_page_count) + .unwrap(); + let (eavt, aevt, avet, vaet) = build_sorted_index_entries(&facts, &refs); + let cache = &pfs.page_cache; + let got_eavt: Vec<(EavtKey, FactRef)> = + stream_all_entries(header.eavt_root_page, &*backend, cache).unwrap(); + let got_aevt: Vec<(AevtKey, FactRef)> = + stream_all_entries(header.aevt_root_page, &*backend, cache).unwrap(); + let got_avet: Vec<(AvetKey, FactRef)> = + stream_all_entries(header.avet_root_page, &*backend, cache).unwrap(); + let got_vaet: Vec<(VaetKey, FactRef)> = + stream_all_entries(header.vaet_root_page, &*backend, cache).unwrap(); + assert!( + got_eavt.is_sorted_by(|a, b| a.0 <= b.0), + "EAVT not in key order" + ); + assert!( + got_aevt.is_sorted_by(|a, b| a.0 <= b.0), + "AEVT not in key order" + ); + assert!( + got_avet.is_sorted_by(|a, b| a.0 <= b.0), + "AVET not in key order" + ); + assert!( + got_vaet.is_sorted_by(|a, b| a.0 <= b.0), + "VAET not in key order" + ); + // One transaction can write the same (entity, attribute) twice, giving equal + // v7 keys (#371); their relative order is unspecified, so compare as sets. + fn by_key_then_ref(mut v: Vec<(K, FactRef)>) -> Vec<(K, FactRef)> { + v.sort(); + v + } + let (eavt, aevt, avet, vaet) = ( + by_key_then_ref(eavt), + by_key_then_ref(aevt), + by_key_then_ref(avet), + by_key_then_ref(vaet), + ); + let (got_eavt, got_aevt, got_avet, got_vaet) = ( + by_key_then_ref(got_eavt), + by_key_then_ref(got_aevt), + by_key_then_ref(got_avet), + by_key_then_ref(got_vaet), + ); + assert!(got_eavt == eavt, "EAVT differs from full derivation"); + assert!(got_aevt == aevt, "AEVT differs from full derivation"); + assert!(got_avet == avet, "AVET differs from full derivation"); + assert!(got_vaet == vaet, "VAET differs from full derivation"); + } + + /// One transact of `n` facts: new and reused entities, strings and refs (VAET). + fn transact_mixed( + pfs: &mut PersistentFactStorage, + entities: &mut Vec, + n: usize, + seed: u64, + ) { + let mut batch = Vec::new(); + for i in 0..n { + let k = seed.wrapping_mul(31).wrapping_add(i as u64); + let e = if entities.is_empty() || k % 3 == 0 { + let e = Uuid::new_v4(); + entities.push(e); + e + } else { + entities[(k as usize) % entities.len()] + }; + let value = if k % 4 == 0 { + Value::Ref(entities[(k as usize / 4) % entities.len()]) + } else { + Value::String(format!("v{k}")) + }; + batch.push((e, format!(":a{}", k % 5), value)); + } + pfs.storage().transact(batch, None).unwrap(); + pfs.mark_dirty(); + } + + #[test] + fn test_incremental_saves_keep_indexes_and_checksum_exact() { + let mut pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); + let mut entities = Vec::new(); + for round in 0..25u64 { + let n = 1 + ((round * 37) % 60) as usize; + transact_mixed(&mut pfs, &mut entities, n, round); + pfs.save().unwrap(); + assert!(pfs.fact_prefix_crc.is_some(), "prefix cached after save"); + assert_indexes_and_checksum_exact(&pfs); + } + } + + #[test] + fn test_reopen_after_incremental_saves_takes_no_rebuild() { + use crate::storage::backend::FileBackend; + use tempfile::NamedTempFile; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let mut entities = Vec::new(); + { + let mut pfs = + PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + for round in 0..10u64 { + transact_mixed(&mut pfs, &mut entities, 40, round); + pfs.save().unwrap(); + } + } + let mut pfs = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + // Set only when the full checksum matched on load, i.e. no rebuild. + assert!( + pfs.fact_prefix_crc.is_some(), + "full checksum must match on reopen" + ); + transact_mixed(&mut pfs, &mut entities, 40, 99); + pfs.save().unwrap(); + assert_indexes_and_checksum_exact(&pfs); + } + + #[test] + fn test_prefix_cache_page_count_mismatch_falls_back() { + let mut pfs = PersistentFactStorage::new(MemoryBackend::new(), 256).unwrap(); + let mut entities = Vec::new(); + transact_mixed(&mut pfs, &mut entities, 50, 1); + pfs.save().unwrap(); + pfs.fact_prefix_crc = Some((999, Hasher::new())); + transact_mixed(&mut pfs, &mut entities, 50, 2); + pfs.save().unwrap(); + assert_indexes_and_checksum_exact(&pfs); + } + + #[test] + fn test_prefix_cache_absent_after_rebuild_on_open() { + use crate::storage::backend::FileBackend; + use tempfile::NamedTempFile; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let mut entities = Vec::new(); + { + let mut pfs = + PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + for round in 0..3u64 { + transact_mixed(&mut pfs, &mut entities, 30, round); + pfs.save().unwrap(); + } + } + // Corrupt index_checksum (re-sealing the header) to force the rebuild path. + { + let mut backend = FileBackend::open(&path).unwrap(); + let mut page = backend.read_page(0).unwrap(); + page[64] ^= 0xFF; + let cs = compute_header_checksum_from_bytes(&page); + page[80..84].copy_from_slice(&cs.to_le_bytes()); + backend.write_page(0, &page).unwrap(); + backend.sync().unwrap(); + } + let mut pfs = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + assert!( + pfs.fact_prefix_crc.is_none(), + "no prefix after rebuild-on-open" + ); + transact_mixed(&mut pfs, &mut entities, 30, 7); + pfs.save().unwrap(); + assert_indexes_and_checksum_exact(&pfs); + } + #[test] fn test_page_cache_capacity_reflects_constructed_value() { let zero_cap = PersistentFactStorage::new(MemoryBackend::new(), 0).unwrap(); From a07573646008d3916a6f1838f178798cdb9d4b40 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 12:54:04 +0530 Subject: [PATCH 07/10] docs(roadmap): make v2.0.2 the final v2.x release, split v3.0.0/v3.1.0 (#403) v2.3.0 becomes v2.0.2: the merged #323/#381/#315 mitigations plus #389, #393, #395, #397, #399, #400. The new production-readiness issues (#383 tracker) go into v3.0.0 with format v8 and #371. Features from v2.1.0/v2.2.0 and the old v3.0.0 scope move to a new v3.1.0. Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy Co-authored-by: Claude Opus 5.5 --- ROADMAP.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 45b27a6b23..3e228b32a8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -8,15 +8,62 @@ Completed releases and their implementation details live in the [CHANGELOG](CHAN --- -## v3.0.0 — File Format Policy +## v2.0.2 — Final Planned v2.x Release -**v3.0.0 supports file format v8 only.** Fixing #287's `EavtKey`/`AevtKey` missing-value-bytes data-loss bug bumps the format from v7 to v8. Support for v1–v6 is dropped at the same time. The v1–v6 → v7 auto-migration code in `persistent_facts.rs` can be removed when cutting this release; a v7→v8 migration replaces it. Any database opened at least once under a v1.x release will already be on v7; there are no known users on older formats. - -This was the GitHub milestone named “2.0” before v2.0.0 used that version number for the kernel-locking and structured-error-code breaking changes; it was consequently renumbered to v3.0.0. +**The v2.x line (`main`) ends with a patch.** File format stays v7 and the public API does not change. All new feature work has moved to v3.x. Tracker: #383. Scope: -- `EavtKey`/`AevtKey` missing-value-bytes fix and file-format v7→v8 migration (#287) +- Already merged: bound-entity point queries narrowed to the queried attribute (#323 mitigation), exact attribute-scan bounds (#381), and checkpoints that copy untouched index leaves (#315 mitigation) +- fsync the parent directory after creating the `.graph`/WAL file and after deleting the WAL (#389) +- Benchmark regression alerting that is not silently muted (#393) +- CI: MSRV build, public-API semver check, dependency policy (#395) +- Stability and support policy, including how long v2.x gets data-integrity fixes after v3.0.0 (#397) +- Known-issues process: data-integrity issues stay open until the fix is in a published release (#399) +- Support tiers for language bindings and platforms (#400) + +**Known issue on all v2.x releases:** same-transaction multi-valued facts read back as one value (#371, #287). The fix needs format v8 and ships in v3.0.0. Workaround: write or retract each value of a multi-valued attribute in its own `transact`/`retract` call. + +--- + +## v3.0.0 — File Format v8 and Data Integrity + +**v3.0.0 supports file format v8 only.** Fixing #287/#371 (`EavtKey`/`AevtKey` carry no value bytes, so same-transaction multi-values collapse) bumps the format from v7 to v8. Support for v1–v6 is dropped at the same time. The v1–v6 → v7 auto-migration code in `persistent_facts.rs` can be removed when cutting this release; a v7→v8 migration replaces it. Any database opened at least once under a v1.x release will already be on v7; there are no known users on older formats. Every other layout change this release needs goes into v8 too, so there is no separate v9. + +This was the GitHub milestone named “2.0” before v2.0.0 used that version number for the kernel-locking and structured-error-code breaking changes; it was consequently renumbered to v3.0.0. It is developed on the long-lived `v3` branch and merges into `main` at the cut. + +Scope — format and storage integrity: + +- v8 index keys with value bytes and v7→v8 migration (#371, #287; done on `v3`) +- Per-page CRC32 checksums on fact pages and B+tree pages (#388) +- Crash-atomic `save()`, with checkpoints proportional to the change (#374) +- Index `verify` and public `rebuild_indexes()` (#373) +- Net-assert on v8 keys before resolving facts, for O(live) point queries (#379) +- `btree_page` fuzz target that reaches node decoding (#375) + +Scope — release-gate testing and documentation: + +- Golden-file compatibility corpus for every format version, frozen before v8 work goes further (#391) +- SIGKILL crash test that checks committed data through EAVT, AEVT and full scan (#384) +- Model-based test of transact/retract/checkpoint/reopen (#385) +- Wider property tests: file-backed, batched, retracts, temporal, joins (#386) +- Persistent fuzz corpus and an operation-sequence target (#387) +- Fault injection across save/WAL/recovery (#390) +- Long-running soak test at scale (#392) +- Benchmarks at 100K–10M facts and a published performance envelope (#394) +- Higher storage/WAL coverage gates and branch-coverage gates (#396) +- Durability, recovery and operations guide (#398) + +--- + +## v3.1.0 — Features + +Additive features, kept out of v3.0.0 so the #371 fix is not held up by them: + +- Rule persistence — design first (#241) +- Query profiler (#185) +- Native `:limit N` / `:offset` for plain `:find` queries (#306, #310) +- Set membership predicates (#316) - `lag`/`lead` window functions (#182) - Sliding row frames — `:rows N preceding` (#183; builds on #182) - `PreparedQuery` over UniFFI (#181) @@ -33,7 +80,7 @@ Scope: ### Developer Tools - 🎯 Database inspector/debugger (separate repository: [minigraf-inspector](https://github.com/project-minigraf/minigraf-inspector)) -- 🎯 Query profiler +- 🎯 Query profiler (#185, v3.1.0) - 🎯 Time-travel visualizer (separate repository: [minigraf-visualizer](https://github.com/project-minigraf/minigraf-visualizer)) ### Integration Examples @@ -105,8 +152,9 @@ When evaluating features, ask: ## Current Focus -- v2.x minor releases (v2.1.0–v2.3.0) on `main` -- The v3.0.0 file-format policy and its scoped work above, on the `v3` branch +- v2.0.2, the final planned v2.x release, on `main` +- v3.0.0 (format v8 and data integrity) on the `v3` branch, then v3.1.0 features +- Production readiness for high-stakes deployments, tracked in #383 - Ecosystem work tracked in [`minigraf-examples`](https://github.com/project-minigraf/minigraf-examples) - Developer tools tracked in `minigraf-inspector` and `minigraf-visualizer` @@ -114,4 +162,4 @@ See [GitHub Issues](https://github.com/project-minigraf/minigraf/issues) for spe --- -**Last updated**: September 2026 — v2.0.1 is the current release. v2.x minor releases (v2.1.0–v2.3.0) ship from `main` first; v3.0.0 work, including the file-format v8 fix for #287/#371, is developed on the long-lived `v3` branch and merges into `main` at the v3.0.0 cut. +**Last updated**: September 2026 — v2.0.1 is the current release. v2.0.2 is the final planned v2.x release and ships from `main` first; v3.0.0 work, including the file-format v8 fix for #287/#371, is developed on the long-lived `v3` branch and merges into `main` at the v3.0.0 cut. From da6c4ffe46aba107ecc8d02760d3424504c73d3f Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 14:04:43 +0530 Subject: [PATCH 08/10] fix(durability): fsync the parent directory after creating the .graph/WAL and deleting the WAL (#389) (#404) On POSIX a created or deleted file survives a power loss only once its parent directory is fsynced. Minigraf fsynced file contents but never a directory, so a power loss could lose a freshly created .graph or .wal, or resurrect a WAL that checkpoint() had deleted. Add storage::dir_sync::sync_parent_dir (open parent dir + sync_all on Unix; no-op elsewhere) and call it: - in FileBackend::open_with after writing the header of a new/empty file - in WalWriter::open_or_create after creating the WAL, and after re-initializing a headerless WAL left by a crash - in WalWriter::delete_file after a successful remove A test-only thread-local log records each sync and whether the child existed at that moment, so unit tests check both that the sync happens and that it follows the create/remove. Resurrected-WAL replay idempotence was already covered by tests/edge_cases_test.rs::test_stale_wal_after_checkpoint_is_idempotent. Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CLAUDE.md | 3 +- README.md | 2 +- docs/TEST_COVERAGE.md | 4 +- src/storage/backend/file.rs | 54 ++++++++++++++- src/storage/dir_sync.rs | 126 ++++++++++++++++++++++++++++++++++ src/storage/mod.rs | 1 + src/wal.rs | 131 +++++++++++++++++++++++++++++++++++- 8 files changed, 315 insertions(+), 7 deletions(-) create mode 100644 src/storage/dir_sync.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d479cb9cab..1e4e2b955a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A power loss could lose a new database or WAL file, or bring back a deleted WAL (#389).** On Linux and other POSIX systems, a created or deleted file survives a power loss only once its parent directory is fsynced; Minigraf fsynced file contents but never a directory. Creating the `.graph` file or the `.wal` sidecar, and deleting the WAL after a checkpoint, now fsync the parent directory after the file operation. A resurrected WAL was already harmless, since replay skips entries at or below the last checkpointed transaction, but a lost `.graph` or WAL lost committed data. Windows needs no directory sync and is unchanged. Process kills were never affected, because they leave the OS page cache intact. - **Bound-entity queries could drop rows when one `WriteTransaction` wrote the same attribute in several valid-time windows (#323).** Both facts share a `tx_count`, and the selective lookup path de-duplicated on `(entity, attribute, tx_count, asserted)`, so one window's value was silently lost, while the same query via a full scan returned both. That de-duplication has been removed, and bound-entity queries now always match a full scan. ### Notes diff --git a/CLAUDE.md b/CLAUDE.md index ff620b95b0..b5c4609f4a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,6 +111,7 @@ cargo run < demos/demo_negation.txt - `btree_v6.rs`: On-disk B+tree (`build_btree`, `rebuild_btree_incremental`, `OnDiskIndexReader`, `MutexStorageBackend`) - `btree.rs`: Legacy v5 B+tree (migration only) - `cache.rs`: LRU page cache (`PageCache`, default 256 pages) + - `dir_sync.rs`: `sync_parent_dir` — fsyncs the parent directory after creating the `.graph`/WAL or deleting the WAL (no-op off Unix) - `packed_pages.rs`: Packed fact pages (~25 facts/4KB page), `MAX_FACT_BYTES` - `persistent_facts.rs`: `PersistentFactStorage` — v7 save/load, auto-migration v1–v6→v7 @@ -167,7 +168,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1190 tests passing** (1182 passing, 8 ignored; unit + integration + doc). +**1200 tests passing** (1192 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index aad4e86be5..17778e53b7 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1190 tests +cargo test # run 1200 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 5850fcb4e5..605e313171 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,9 +4,9 @@ **Verified**: 2026-09-26 with `cargo test` -**Result**: 1,182 passing tests and 8 ignored tests (1,190 total) +**Result**: 1,192 passing tests and 8 ignored tests (1,200 total) -The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a cyclic leaf chain or damaged slot directory is rejected rather than copied (#315). +The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a cyclic leaf chain or damaged slot directory is rejected rather than copied (#315). Directory-sync unit tests check that creating the `.graph` file or the WAL, and deleting the WAL, fsync the parent directory after the file operation, and that reopening an existing file does not (#389). ### Covered Areas diff --git a/src/storage/backend/file.rs b/src/storage/backend/file.rs index 9db2a66199..f521ee292f 100644 --- a/src/storage/backend/file.rs +++ b/src/storage/backend/file.rs @@ -1,5 +1,6 @@ /// File-based storage backend for native platforms. use crate::error::{ErrorCode, bail_coded, err_coded}; +use crate::storage::dir_sync::sync_parent_dir; use crate::storage::{FileHeader, PAGE_SIZE, StorageBackend}; use anyhow::Result; use std::fs::{File, OpenOptions}; @@ -285,9 +286,14 @@ impl FileBackend { } } } else { - // New file or empty file: write initial header + // New file or empty file: write initial header, then make the + // file's directory entry durable. Without the directory sync a + // power loss can lose the whole file despite the header fsync + // (#389). An empty file left by a crash before this point takes + // this branch again on the next open, so the sync is repeated. let header = FileHeader::new(); Self::write_header(&mut file, &header)?; + sync_parent_dir(&path)?; header }; @@ -855,4 +861,50 @@ mod tests { "exactly one concurrent claimant should be admitted" ); } + + // ── #389: parent-directory fsync ──────────────────────────────────────── + + #[test] + fn test_create_syncs_parent_dir_after_header_write() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("new.graph"); + take_sync_log(); + + let backend = FileBackend::open(&path).unwrap(); + + let log = take_sync_log(); + assert_eq!( + log.len(), + 1, + "creating the file must sync its directory once" + ); + assert_eq!( + log[0].dir, + dir.path(), + "must sync the file's parent directory" + ); + assert!( + log[0].child_existed, + "directory sync must follow the create" + ); + drop(backend); + } + + #[test] + fn test_reopen_existing_does_not_sync_parent_dir() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("existing.graph"); + drop(FileBackend::open(&path).unwrap()); + take_sync_log(); + + let backend = FileBackend::open(&path).unwrap(); + + assert!( + take_sync_log().is_empty(), + "reopening an initialized file creates nothing, so needs no directory sync" + ); + drop(backend); + } } diff --git a/src/storage/dir_sync.rs b/src/storage/dir_sync.rs new file mode 100644 index 0000000000..21f9d8d9ce --- /dev/null +++ b/src/storage/dir_sync.rs @@ -0,0 +1,126 @@ +//! Parent-directory fsync (#389). +//! +//! On POSIX systems a newly created, deleted or renamed file survives a power +//! loss only once its parent directory has been fsynced: `fsync` on the file +//! itself makes the file's *contents* durable, not the directory entry that +//! names it. Without this, a power loss (not a process kill -- a kill leaves +//! the OS page cache intact) can lose a freshly created `.graph` or `.wal`, +//! or bring back a WAL that `checkpoint()` deleted. +//! +//! Durability order: +//! +//! - Create: create file → write + `sync_all` its first bytes → sync parent dir. +//! - Delete: remove file → sync parent dir. +//! +//! Windows needs no directory sync (NTFS journals directory metadata), and +//! cannot open a directory as a `File` without extra flags, so this is a no-op +//! there and on every other non-Unix target. + +use std::io; +use std::path::Path; + +/// Fsync the directory that contains `child`, making `child`'s creation or +/// removal durable. +/// +/// `child` is the path of the file that was just created or removed, not the +/// directory itself. A bare file name (no parent component) refers to the +/// current directory. +pub(crate) fn sync_parent_dir(child: &Path) -> io::Result<()> { + let dir = match child.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + }; + + #[cfg(test)] + record(dir, child); + + #[cfg(unix)] + std::fs::File::open(dir)?.sync_all()?; + #[cfg(not(unix))] + let _ = dir; + Ok(()) +} + +// ─── Test-only call log ───────────────────────────────────────────────────── +// +// A power loss cannot be simulated in a test, so tests instead check that the +// sync happens at all and in the right order relative to the file operation: +// each call records the directory and whether `child` existed at that moment. +// After a create it must exist; after a delete it must not. Thread-local, so +// concurrently running tests do not see each other's calls. + +#[cfg(test)] +thread_local! { + static SYNC_LOG: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; +} + +/// One recorded `sync_parent_dir` call. +#[cfg(test)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DirSync { + /// The directory that was synced. + pub dir: std::path::PathBuf, + /// The file the sync was made on behalf of. + pub child: std::path::PathBuf, + /// Whether `child` existed when the directory was synced. + pub child_existed: bool, +} + +#[cfg(test)] +fn record(dir: &Path, child: &Path) { + SYNC_LOG.with(|log| { + log.borrow_mut().push(DirSync { + dir: dir.to_path_buf(), + child: child.to_path_buf(), + child_existed: child.exists(), + }) + }); +} + +/// Return and clear this thread's recorded directory syncs. +#[cfg(test)] +pub(crate) fn take_sync_log() -> Vec { + SYNC_LOG.with(|log| std::mem::take(&mut *log.borrow_mut())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_syncs_parent_of_child() { + let dir = tempfile::tempdir().unwrap(); + let child = dir.path().join("a.graph"); + std::fs::write(&child, b"x").unwrap(); + take_sync_log(); + + sync_parent_dir(&child).unwrap(); + + let log = take_sync_log(); + assert_eq!(log.len(), 1, "expected exactly one directory sync"); + assert_eq!(log[0].dir, dir.path(), "must sync the child's parent"); + assert!(log[0].child_existed, "child existed at sync time"); + } + + #[test] + fn test_bare_file_name_syncs_current_dir() { + take_sync_log(); + sync_parent_dir(Path::new("no-such-file.graph")).unwrap(); + let log = take_sync_log(); + assert_eq!(log.len(), 1, "expected exactly one directory sync"); + assert_eq!(log[0].dir, Path::new("."), "bare name means current dir"); + assert!(!log[0].child_existed, "child does not exist"); + } + + #[cfg(unix)] + #[test] + fn test_missing_parent_dir_is_an_error() { + let dir = tempfile::tempdir().unwrap(); + let child = dir.path().join("gone").join("a.graph"); + assert!( + sync_parent_dir(&child).is_err(), + "a directory that cannot be opened must surface as an error" + ); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 3145504265..9a409e653f 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -9,6 +9,7 @@ pub mod backend; pub mod btree; pub mod btree_v6; pub mod cache; +pub(crate) mod dir_sync; pub mod index; pub mod packed_pages; pub mod persistent_facts; diff --git a/src/wal.rs b/src/wal.rs index 781bbf77a1..3c611c940a 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -22,6 +22,7 @@ use crate::db::SyncMode; use crate::error::{ErrorCode, bail_coded, err_coded}; use crate::graph::types::Fact; +use crate::storage::dir_sync::sync_parent_dir; use crate::storage::packed_pages::MAX_FACT_BYTES; use anyhow::Result; use std::fs::{File, OpenOptions}; @@ -148,6 +149,9 @@ impl WalWriter { { Ok(mut file) => { write_wal_header(&mut file)?; + // The header is durable; make the WAL's directory entry + // durable too, or a power loss can lose the whole file (#389). + sync_parent_dir(path)?; file.seek(SeekFrom::End(0))?; return Ok(WalWriter { file, sync_mode }); } @@ -162,7 +166,12 @@ impl WalWriter { // A previous crash landed between this file's creation and its // header write completing; no entry could have been appended // yet, so re-initialize it as a fresh, empty WAL. - WalHeaderState::Absent => write_wal_header(&mut file)?, + // The crash may also have preceded the directory sync that + // follows creation, so repeat it (#389). + WalHeaderState::Absent => { + write_wal_header(&mut file)?; + sync_parent_dir(path)?; + } } file.seek(SeekFrom::End(0))?; Ok(WalWriter { file, sync_mode }) @@ -192,6 +201,9 @@ impl WalWriter { /// Delete the WAL file at `path`. Called after a successful checkpoint. /// + /// Fsyncs the parent directory after the remove, so a power loss cannot + /// bring the deleted WAL back (#389). + /// /// Uses a short retry loop to tolerate Windows races where the OS file handle /// is not immediately released after the `WalWriter` is dropped. pub fn delete_file(path: &Path) -> Result<()> { @@ -200,7 +212,10 @@ impl WalWriter { let mut last_err = None; for i in 0..retries { match std::fs::remove_file(path) { - Ok(()) => return Ok(()), + Ok(()) => { + sync_parent_dir(path)?; + return Ok(()); + } Err(e) => { last_err = Some(e); if i.saturating_add(1) < retries { @@ -740,4 +755,116 @@ mod tests { std::fs::remove_dir(&path).unwrap(); } + + // ── #389: parent-directory fsync ──────────────────────────────────────── + + #[test] + fn test_wal_create_syncs_parent_dir_after_create() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.wal"); + take_sync_log(); + + let _writer = WalWriter::open_or_create(&path, SyncMode::Full).unwrap(); + + let log = take_sync_log(); + assert_eq!( + log.len(), + 1, + "creating the WAL must sync its directory once" + ); + assert_eq!( + log[0].dir, + dir.path(), + "must sync the WAL's parent directory" + ); + assert!( + log[0].child_existed, + "directory sync must follow the create" + ); + } + + #[test] + fn test_wal_reopen_existing_does_not_sync_parent_dir() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.wal"); + drop(WalWriter::open_or_create(&path, SyncMode::Full).unwrap()); + take_sync_log(); + + let _writer = WalWriter::open_or_create(&path, SyncMode::Full).unwrap(); + + assert!( + take_sync_log().is_empty(), + "reopening an intact WAL creates nothing, so needs no directory sync" + ); + } + + #[test] + fn test_wal_reinit_headerless_syncs_parent_dir() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.wal"); + // A crash between `create_new` and the header write leaves a short + // file whose directory entry may never have been synced either. + std::fs::write(&path, b"MW").unwrap(); + take_sync_log(); + + let _writer = WalWriter::open_or_create(&path, SyncMode::Full).unwrap(); + + let log = take_sync_log(); + assert_eq!( + log.len(), + 1, + "re-initializing the WAL must sync its directory" + ); + assert!( + log[0].child_existed, + "directory sync must follow the header write" + ); + } + + #[test] + fn test_wal_delete_syncs_parent_dir_after_remove() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.wal"); + drop(WalWriter::open_or_create(&path, SyncMode::Full).unwrap()); + take_sync_log(); + + WalWriter::delete_file(&path).unwrap(); + + let log = take_sync_log(); + assert_eq!( + log.len(), + 1, + "deleting the WAL must sync its directory once" + ); + assert_eq!( + log[0].dir, + dir.path(), + "must sync the WAL's parent directory" + ); + assert!( + !log[0].child_existed, + "directory sync must follow the remove" + ); + } + + #[test] + fn test_wal_failed_delete_does_not_sync_parent_dir() { + use crate::storage::dir_sync::take_sync_log; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("not-a-file.wal"); + std::fs::create_dir(&path).unwrap(); + take_sync_log(); + + assert!(WalWriter::delete_file(&path).is_err(), "delete must fail"); + assert!( + take_sync_log().is_empty(), + "nothing was removed, nothing to sync" + ); + + std::fs::remove_dir(&path).unwrap(); + } } From dee008ffa12dc304fef24cb1755e9d18fc468be3 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 14:36:24 +0530 Subject: [PATCH 09/10] docs: fix unsafe WAL recovery advice and align docs with milestones (#408) - ERROR_REFERENCE: WAL-001/WAL-002/STG-011 said deleting the .wal loses no data, or that a database can be rebuilt from the WAL alone. Both are wrong. Replace with safe steps. Fix ~50 wiki links that pointed at a local .wiki path. - ROADMAP/CLAUDE.md: match the v2.0.2, v3.0.0 and v3.1.0 milestones. Document error.rs, magic_sets.rs, fault_inject.rs and src/browser/. - README: state MSRV 1.89, show the #371 known issue, fix Maven coordinates (io.github.project-minigraf), Android on Maven Central, and point binding downloads to the binding repos. - SECURITY.md: 2.x is the supported line. Refs #383, #397, #399 Claude-Session: https://claude.ai/code/session_01BBxgR4mnF8VMVGa42idcxx Co-authored-by: Claude --- .github/SECURITY.md | 4 +- CHANGELOG.md | 7 +++ CLAUDE.md | 14 +++-- README.md | 25 +++++---- ROADMAP.md | 2 + docs/ERROR_REFERENCE.md | 111 +++++++++++++++++++++------------------- 6 files changed, 94 insertions(+), 69 deletions(-) diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 9509777b88..41c5befe88 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -9,8 +9,8 @@ patched unless a separate security advisory explicitly says otherwise. | Version | Supported | | ------- | --------- | -| latest 1.x minor line | ✅ | -| older 1.x minor lines | ❌ | +| 2.x (latest release) | ✅ | +| 1.x | ❌ | | 0.x | ❌ | ## Reporting a Vulnerability diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e4e2b955a..3bbbe16e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A power loss could lose a new database or WAL file, or bring back a deleted WAL (#389).** On Linux and other POSIX systems, a created or deleted file survives a power loss only once its parent directory is fsynced; Minigraf fsynced file contents but never a directory. Creating the `.graph` file or the `.wal` sidecar, and deleting the WAL after a checkpoint, now fsync the parent directory after the file operation. A resurrected WAL was already harmless, since replay skips entries at or below the last checkpointed transaction, but a lost `.graph` or WAL lost committed data. Windows needs no directory sync and is unchanged. Process kills were never affected, because they leave the OS page cache intact. - **Bound-entity queries could drop rows when one `WriteTransaction` wrote the same attribute in several valid-time windows (#323).** Both facts share a `tx_count`, and the selective lookup path de-duplicated on `(entity, attribute, tx_count, asserted)`, so one window's value was silently lost, while the same query via a full scan returned both. That de-duplication has been removed, and bound-entity queries now always match a full scan. +### Documentation + +- **Unsafe WAL recovery advice removed from `docs/ERROR_REFERENCE.md`.** The WAL-001, WAL-002 and STG-011 resolutions said a `.wal` file could be deleted with no data loss, or that a database could be rebuilt from the WAL alone. Both are wrong: transactions committed since the last checkpoint exist only in the WAL, and the WAL holds nothing older than that checkpoint. The resolutions now say to copy both files first, to checkpoint with the version that wrote the WAL, and what is lost if the WAL is deleted. +- CLAUDE.md now matches the milestones (v2.0.2 final v2.x release, v3.0.0 format v8 and data integrity, v3.1.0 features) and documents `error.rs`, `magic_sets.rs`, `fault_inject.rs` and `src/browser/`. ROADMAP lists #405 and #407 in the v2.0.2 scope. +- README: MSRV (1.89) stated, v2.x known issue (#371) shown near the top, Maven coordinates corrected to `io.github.project-minigraf`, Android listed on Maven Central, and binding download locations point to the binding repos. +- `.github/SECURITY.md` lists 2.x as the supported line. About 50 broken wiki links in `docs/ERROR_REFERENCE.md` now use full wiki URLs. + ### Notes - On v2.x, reading a heavily rewritten attribute still costs time proportional to its history, because v7 index keys carry neither the value nor the assert/retract flag; the structural fix needs the v8 keys and is tracked for v3.0.0 in #379. diff --git a/CLAUDE.md b/CLAUDE.md index b5c4609f4a..240f51ef6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,6 +107,7 @@ cargo run < demos/demo_negation.txt - `mod.rs`: `StorageBackend` trait, `FileHeader` v7 (84 bytes), `CommittedFactReader` / `CommittedIndexReader` traits - `backend/file.rs`: Single `.graph` file backend (4KB pages, cross-platform) - `backend/memory.rs`: In-memory backend for testing + - `backend/fault_inject.rs`: `FaultInjectingBackend` — injects I/O errors for durability tests (test builds only) - `index.rs`: EAVT / AEVT / AVET / VAET key types, `FactRef`, `encode_value` - `btree_v6.rs`: On-disk B+tree (`build_btree`, `rebuild_btree_incremental`, `OnDiskIndexReader`, `MutexStorageBackend`) - `btree.rs`: Legacy v5 B+tree (migration only) @@ -125,6 +126,7 @@ cargo run < demos/demo_negation.txt - `types.rs`: `EdnValue`, `Pattern`, `DatalogQuery`, `AsOf`, `ValidAt`, `WhereClause` (incl. `Not`, `NotJoin`); `PseudoAttr` enum, `AttributeSpec` wrapper - `optimizer.rs`: Selectivity-based join reordering; disabled under `wasm` feature - `prepared.rs`: `BindValue`, `PreparedQuery` — parse-once/execute-many with named `$slot` bind slots; `prepare_query`, `substitute` + - `magic_sets.rs`: Magic-sets rewriting for demand-driven recursive rule evaluation (not applied to rules with `not`/`not-join`) 4. **`src/temporal.rs`** — UTC-only timestamp parsing (avoids chrono CVE GHSA-wcg3-cvx6-7396) @@ -134,6 +136,10 @@ cargo run < demos/demo_negation.txt 7. **`src/wal.rs`** — Fact-level sidecar WAL, CRC32-protected entries, crash recovery +8. **`src/error.rs`** — Structured error codes: `MinigrafError`, `ErrorCategory`, `ErrorCode` registry (PRS/QRY/STG/WAL/API/INT codes); must match `docs/ERROR_REFERENCE.md` + +9. **`src/browser/`** — Browser WASM backend (`browser` feature): `buffer.rs` (`BrowserBufferBackend`, in-memory pages with dirty tracking), `indexeddb.rs` (IndexedDB persistence) + ### Data Model ```rust @@ -177,9 +183,11 @@ See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **v2.0.1 released** (2026-09-25) — patch fixing index rebuild on open (#370). **v2.0.0** introduced kernel file locking (#317, #304), structured runtime error codes (#277), `OpenOptions` `#[non_exhaustive]`, and an MSRV of Rust 1.89. See `CHANGELOG.md` for the full rationale and release history. -Relevant areas for the planned query-profiler work (see `ROADMAP.md`): -- `src/query/datalog/` — query executor where profiling hooks will go (#185) -- `docs/BENCHMARKS.md` — post-1.0 performance baseline updates +Current plan (see `ROADMAP.md` and the milestones): +- **v2.0.2** (final planned v2.x release, format v7, no API change) ships from `main`. +- **v3.0.0** (format v8 and data integrity, tracker #383) is developed on the long-lived `v3` branch. Merge `main` into `v3` regularly. +- **v3.1.0** holds new features (query profiler #185, `:limit` #306/#310, lag/lead #182, UniFFI additions). +- Known v2.x data issue: same-transaction multi-valued facts (#371). Fix ships in v3.0.0. ## Testing Conventions diff --git a/README.md b/README.md index 17778e53b7..5c83eb0e21 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ [![Clippy Status](https://github.com/project-minigraf/minigraf/actions/workflows/rust-clippy.yml/badge.svg)](https://github.com/project-minigraf/minigraf/actions/workflows/rust-clippy.yml) [![Coverage](https://codecov.io/gh/project-minigraf/minigraf/branch/main/graph/badge.svg)](https://codecov.io/gh/project-minigraf/minigraf) [![License: MIT OR Apache-2.0](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-blue.svg)](https://github.com/project-minigraf/minigraf#license) -[![Rust Edition](https://img.shields.io/badge/rust-2024-orange.svg)](https://blog.rust-lang.org/2024/10/17/Rust-1.82.0.html) +[![Rust Edition](https://img.shields.io/badge/rust-2024-orange.svg)](https://doc.rust-lang.org/edition-guide/rust-2024/index.html) +[![MSRV](https://img.shields.io/badge/MSRV-1.89-orange.svg)](Cargo.toml) > **Embedded graph memory for AI agents, mobile apps, and the browser** — the SQLite of bi-temporal graph databases @@ -50,6 +51,10 @@ Or via cargo: cargo add minigraf ``` +Requires Rust 1.89 or newer (the minimum supported Rust version, set in `Cargo.toml`). + +**Known issue on v2.x:** two values of the same attribute for one entity written in a single `transact` (or retracted in a single `retract`) can read back as one value (#371). Write or retract each value of a multi-valued attribute in its own call. The fix changes the file format and ships in v3.0.0. See the [CHANGELOG](CHANGELOG.md). + ## Quick Start ```rust @@ -114,7 +119,7 @@ No other database offers this combination: | **Embedded** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ No | ✅ Yes | | **Graph Native** | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | ❌ No | | **Rust** | ✅ Yes | ❌ Clojure | ✅ Yes | ❌ Java | ❌ C | -| **WASM Ready** | ✅ Yes (browser + WASI + 6 targets) | ❌ No | ⚠️ Limited | ❌ No | ✅ Yes | +| **WASM Ready** | ✅ Yes (browser + WASI) | ❌ No | ⚠️ Limited | ❌ No | ✅ Yes | ## Platform support @@ -122,13 +127,13 @@ No other database offers this combination: |---|---|---| | Rust (native) | `minigraf` on crates.io | `cargo add minigraf` | | Browser WASM | `@minigraf/browser` on npm | `npm install @minigraf/browser` | -| WASI | `@minigraf/wasi` on npm, `.wasm` on GitHub Releases | `npm install @minigraf/wasi` | +| WASI | `@minigraf/wasi` on npm | `npm install @minigraf/wasi` | | Node.js | `minigraf` on npm | `npm install minigraf` | | Python | `minigraf` on PyPI | `pip install minigraf` | -| Java/JVM | `io.github.adityamukho:minigraf-jvm` on Maven Central | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | -| Android | `.aar` on GitHub Packages | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | -| iOS / macOS | `.xcframework` via Swift Package Manager | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | -| C / FFI | header + tarball on GitHub Releases | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | +| Java/JVM | `io.github.project-minigraf:minigraf-jvm` on Maven Central | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | +| Android | `io.github.project-minigraf:minigraf-android` (`.aar`) on Maven Central | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | +| iOS / macOS | `.xcframework` via Swift Package Manager ([minigraf-swift](https://github.com/project-minigraf/minigraf-swift)) | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | +| C / FFI | header + tarball on [minigraf-c releases](https://github.com/project-minigraf/minigraf-c/releases) | see [wiki](https://github.com/project-minigraf/minigraf/wiki/Use-Cases) | **Embedded graph memory for agents, mobile, and the browser — SQLite's simplicity + Datomic's temporal model.** @@ -140,8 +145,8 @@ No other database offers this combination: | Node.js | [`minigraf` on npm](https://www.npmjs.com/package/minigraf) | [minigraf-node](https://github.com/project-minigraf/minigraf-node) | | Browser WASM | [`@minigraf/browser` on npm](https://www.npmjs.com/package/@minigraf/browser) | [minigraf-wasm](https://github.com/project-minigraf/minigraf-wasm) | | WASI | [`@minigraf/wasi` on npm](https://www.npmjs.com/package/@minigraf/wasi) | [minigraf-wasm](https://github.com/project-minigraf/minigraf-wasm) | -| Java | `minigraf-jvm` on Maven Central | [minigraf-java](https://github.com/project-minigraf/minigraf-java) | -| Android | Android bindings | [minigraf-android](https://github.com/project-minigraf/minigraf-android) | +| Java | `io.github.project-minigraf:minigraf-jvm` on Maven Central | [minigraf-java](https://github.com/project-minigraf/minigraf-java) | +| Android | `io.github.project-minigraf:minigraf-android` on Maven Central | [minigraf-android](https://github.com/project-minigraf/minigraf-android) | | iOS/macOS | Swift bindings | [minigraf-swift](https://github.com/project-minigraf/minigraf-swift) | | C | C bindings | [minigraf-c](https://github.com/project-minigraf/minigraf-c) | @@ -155,7 +160,7 @@ Pairs well with vector stores (GraphRAG pattern): the vector store answers "what ### For Mobile Apps -Offline-first storage with retroactive corrections — the bi-temporal model lets you correct a mis-entered value while preserving the original record. Native Kotlin and Swift bindings ship as an Android `.aar` (GitHub Packages) and an iOS `.xcframework` (Swift Package Manager) via [UniFFI](https://github.com/mozilla/uniffi-rs). No Rust required. +Offline-first storage with retroactive corrections — the bi-temporal model lets you correct a mis-entered value while preserving the original record. Native Kotlin and Swift bindings ship as an Android `.aar` (Maven Central) and an iOS `.xcframework` (Swift Package Manager) via [UniFFI](https://github.com/mozilla/uniffi-rs). No Rust required. ```kotlin // Android (Kotlin) diff --git a/ROADMAP.md b/ROADMAP.md index 3e228b32a8..24270938e8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -16,6 +16,8 @@ Scope: - Already merged: bound-entity point queries narrowed to the queried attribute (#323 mitigation), exact attribute-scan bounds (#381), and checkpoints that copy untouched index leaves (#315 mitigation) - fsync the parent directory after creating the `.graph`/WAL file and after deleting the WAL (#389) +- `or-join` returns an empty result, not INT-031, when the clauses before it match no rows (#405) +- A clear user error for `$slot` queries run through `execute()`, not internal INT-025 (#407) - Benchmark regression alerting that is not silently muted (#393) - CI: MSRV build, public-API semver check, dependency policy (#395) - Stability and support policy, including how long v2.x gets data-integrity fixes after v3.0.0 (#397) diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index d4b1fe9dfc..84538bd10c 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -215,7 +215,7 @@ with no `CodedError` anywhere in its chain. Parser errors occur when Minigraf cannot parse the Datalog/EDN input string. They are returned immediately from `db.execute()` before any fact is read or written. -See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidance. +See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference) for syntax guidance. ### PRS-001 Unexpected end of input @@ -259,7 +259,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Check surrounding delimiters for balance. -- Consult the [Datalog Reference](../../.wiki/Datalog-Reference.md) for expected syntax at that position. +- Consult the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference) for expected syntax at that position. **Example**: ```datalog @@ -431,7 +431,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add a value: `:as-of 5` (transaction count) or `:as-of "2024-01-01T00:00:00Z"` (ISO 8601 wall-clock time). -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -448,7 +448,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use a non-negative integer. - `:as-of 0` returns the database before any transaction; `:as-of 1` returns the state after the first transaction. -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -464,7 +464,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `42` (tx-count) or `"2024-01-01T00:00:00Z"` (wall-clock). -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -480,7 +480,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add an ISO 8601 timestamp: `:valid-at "2024-06-01T00:00:00Z"`. -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -496,7 +496,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `"2024-06-01T00:00:00Z"` or `:any-valid-time`. -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -512,7 +512,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `"2024-01-01T00:00:00Z"` format. -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -529,7 +529,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `"2024-12-31T23:59:59Z"` format. - To express "forever", omit `:valid-to` (the default is `VALID_TIME_FOREVER`). -- See the [Datalog Reference — Time Travel](../../.wiki/Datalog-Reference.md#time-travel). +- See the [Datalog Reference — Time Travel](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#bi-temporal-queries). **Example**: ```datalog @@ -545,7 +545,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add an aggregate to `:find`, or remove `:with` if no aggregation is needed. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -563,7 +563,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add a `:where` clause that binds the variable, or remove it from `:with`. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -581,7 +581,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add a `:where` clause that binds the variable. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -598,7 +598,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `(count ?e)`, `(sum ?amount)`, `(avg ?score)` — one function name, one variable. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -615,7 +615,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `sum`, not `:sum`. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -632,7 +632,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use a `?variable`, not a literal value. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -649,7 +649,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add an `:over` clause: `(rank :over (:order-by ?score))`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -666,7 +666,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Provide at least `:order-by` or `:partition-by` inside the `:over` list. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -683,7 +683,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `rank`, not `:rank`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -717,7 +717,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Drop the `:over` clause and use the function as a plain aggregate. -- See the [Datalog Reference — Aggregates](../../.wiki/Datalog-Reference.md#aggregates). +- See the [Datalog Reference — Aggregates](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#aggregation). **Example**: ```datalog @@ -734,7 +734,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add the variable argument before `:over`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -751,7 +751,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add `:over` after the variable. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -768,7 +768,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Remove the variable: `(rank :over (:order-by ?score))`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -785,7 +785,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Always follow `:over` with a list: `(rank :over (:order-by ?score))`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -802,7 +802,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - The window expression must end after the `:over` clause. Remove trailing tokens. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -819,7 +819,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use a `?variable`: `(rank :over (:partition-by ?dept :order-by ?score))`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -836,7 +836,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use a `?variable`: `(rank :over (:order-by ?score))`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -853,7 +853,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use only `:order-by` and `:partition-by`. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -870,7 +870,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - The `:over` list must contain only `:order-by` / `:partition-by` followed by variables. -- See the [Datalog Reference — Window Functions](../../.wiki/Datalog-Reference.md#window-functions). +- See the [Datalog Reference — Window Functions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#window-functions). **Example**: ```datalog @@ -1024,7 +1024,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Remove the empty list or replace it with a valid clause. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference). **Example**: ```datalog @@ -1042,7 +1042,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Double negation is logically equivalent to the positive pattern; use the pattern directly. - For complex negation use `not-join` with explicit join variables. -- See the [Datalog Reference — Negation](../../.wiki/Datalog-Reference.md#negation). +- See the [Datalog Reference — Negation](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#negation). **Example**: ```datalog @@ -1059,7 +1059,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add at least one pattern inside `(not ...)`. -- See the [Datalog Reference — Negation](../../.wiki/Datalog-Reference.md#negation). +- See the [Datalog Reference — Negation](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#negation). **Example**: ```datalog @@ -1076,7 +1076,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add at least one branch. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md#disjunction). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#disjunction). **Example**: ```datalog @@ -1093,7 +1093,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use the form `(or-join [?e] [[?e :a 1]] [[?e :b 2]])`. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md#disjunction). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#disjunction). **Example**: ```datalog @@ -1110,7 +1110,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - The form is `(or-join [?var1 ?var2] ...)`. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md#disjunction). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#disjunction). **Example**: ```datalog @@ -1127,7 +1127,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `?variable` not `:keyword` or a string. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md#disjunction). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#disjunction). **Example**: ```datalog @@ -1145,7 +1145,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Restructure branches so they all bind the same new variables. - Or use `or-join` to specify exactly which variables to share. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md#disjunction). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#disjunction). **Example**: ```datalog @@ -1163,7 +1163,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Add at least one clause inside `(and ...)`. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md#disjunction). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#disjunction). **Example**: ```datalog @@ -1180,7 +1180,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `(not-join [?e] [?e :deleted true])`. -- See the [Datalog Reference — Negation](../../.wiki/Datalog-Reference.md#negation). +- See the [Datalog Reference — Negation](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#negation). **Example**: ```datalog @@ -1197,7 +1197,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Each `:where` element must be a vector `[e a v]`, a `(not ...)`, `(not-join ...)`, `(or ...)`, `(or-join ...)`, or expression `[(expr) ?out]`. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference). **Example**: ```datalog @@ -1215,7 +1215,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Valid query map keys are `:find`, `:where`, `:with`, `:as-of`, `:valid-at`. - Check spelling. -- See the [Datalog Reference](../../.wiki/Datalog-Reference.md). +- See the [Datalog Reference](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference). **Example**: ```datalog @@ -1231,7 +1231,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Provide a function: `[(+ ?a ?b) ?sum]`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1248,7 +1248,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use `+` not `:+`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1266,7 +1266,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Check the argument count. - Single-argument operators: `abs`, `not`, `str` (for coercion). -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1284,7 +1284,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Check the argument count. - Two-argument operators: `+`, `-`, `*`, `/`, `mod`, `quot`, `=`, `!=`, `<`, `<=`, `>`, `>=`, `starts-with?`, `ends-with?`, `contains?`, `matches?`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1301,7 +1301,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Pass the pattern as a string literal: `[(matches? ?name "alice.*")]`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1319,7 +1319,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Check spelling against the supported operator list above. - For missing operators, register a custom predicate via `db.register_predicate()`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1336,7 +1336,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Ensure the expression is wrapped in exactly one outer `[...]`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1353,7 +1353,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Use a `?variable`: `[(+ ?a ?b) ?sum]`. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1371,7 +1371,7 @@ See the [Datalog Reference](../../.wiki/Datalog-Reference.md) for syntax guidanc **Resolution**: - Expressions accept variables (`?x`), integers, floats, strings, booleans, and keywords. - Restructure to avoid passing complex types. -- See the [Datalog Reference — Expressions](../../.wiki/Datalog-Reference.md#expressions). +- See the [Datalog Reference — Expressions](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#arithmetic--predicate-expressions). **Example**: ```datalog @@ -1791,7 +1791,9 @@ See the [file format section in README](../README.md#file-format) for version hi **Cause**: An internal B+tree page was found with an empty children list, which violates B+tree invariants. This indicates index corruption. **Resolution**: -- Restore from backup. If the corruption occurred after a recent write, the WAL file may contain a recoverable state — delete the `.graph` file and try opening from the WAL only (by restoring the last good checkpoint and replaying the WAL). See the [file format section in README](../README.md#file-format). +- Stop writing to the database and copy both the `.graph` file and its `.wal` sidecar before trying anything else. +- Restore from backup. Do not delete the `.graph` file: the WAL holds only the transactions written since the last checkpoint, so it cannot rebuild the database on its own. If you restore an older `.graph` file, you can keep the matching `.wal` only if it was written after that file's last checkpoint. +- Index integrity checking and rebuilding from fact pages are tracked in #373. **Scenario**: A crash mid-checkpoint left an internal B+tree page in an inconsistent state. @@ -1998,7 +2000,8 @@ The WAL is replayed on open and deleted on checkpoint. **Cause**: The sidecar `.wal` file does not start with the expected WAL magic bytes. The file may have been replaced, corrupted, or created by an incompatible tool. **Resolution**: -- If the WAL file is stale or corrupt, delete `.wal` and reopen the database — Minigraf will replay only from the committed state in the `.graph` file. +- Copy both the `.graph` file and the `.wal` file before changing anything. +- Deleting `.wal` makes the database open, but **every transaction committed since the last checkpoint is lost**, because those transactions exist only in the WAL. Do this only if you have a backup or accept that loss. - Do not manually create or edit `.wal` files. **Scenario**: `my-db.wal` was accidentally replaced with an empty file before `Minigraf::open("my-db.graph")` was called. @@ -2010,8 +2013,8 @@ The WAL is replayed on open and deleted on checkpoint. **Cause**: The `.wal` file was written by a version of Minigraf with a different WAL format. This can occur when downgrading the library after a WAL was written by a newer version. **Resolution**: -- Delete the `.wal` file if it is from an incomplete or stale session (no data is lost — committed facts are in the `.graph` file). -- If the WAL contains uncommitted in-flight data you need to recover, upgrade the library to the version that wrote the WAL before reopening. +- Open the database with the Minigraf version that wrote the WAL and call `checkpoint()`. This moves the WAL's transactions into the `.graph` file and removes the WAL. Then open it with the new version. +- Deleting the `.wal` file instead loses **every transaction committed since the last checkpoint**, because those transactions exist only in the WAL. Do this only if you have a backup or accept that loss. **Scenario**: A `.wal` file written by a pre-release version of Minigraf is opened with the stable release, which uses a different WAL version number — e.g. `Unsupported WAL version: 3 (expected 2)`. @@ -2025,7 +2028,7 @@ The WAL is replayed on open and deleted on checkpoint. - Store large payloads in an external file or object store. - Store the file path or URL as a `Value::String` attribute on the entity. - Or create a dedicated entity for the content and reference it with `Value::Ref`. -- See [BENCHMARKS.md](../BENCHMARKS.md) for size guidance. +- See the [Datalog Reference — Constraints and Limits](https://github.com/project-minigraf/minigraf/wiki/Datalog-Reference#constraints-and-limits) for the size limit. **Example**: ```datalog From e78d2876c3479e542586bed8f38db4854a6bf327 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 26 Sep 2026 14:46:55 +0530 Subject: [PATCH 10/10] fix(magic-sets): keep preceding rule calls in magic propagation rules (#297) (#409) A recursive rule whose intermediate variable is bound by a call to another user rule, e.g. (chain ?x ?y) :- (link ?x ?mid) (chain ?mid ?y), failed with "Unbound variable in rule head: ?mid" (INT-022) when queried with a literal first argument such as (chain :a ?y). build_propagation_rules built the magic rule for the recursive call from the guard plus only the non-RuleInvocation clauses preceding it, so it emitted (__magic_chain_bf ?mid) :- (__magic_chain_bf ?x) with ?mid unbound. #300 fixed only the case where a fact pattern binds the intermediate. Keep every preceding clause, including rule invocations, as standard magic-sets rewriting does. All rewritten rules are positive, so the added dependency on the (guarded) called predicate is safe. Adds integration tests for the reopened repro and a left-recursive variant, and a unit test pinning the propagation-rule body. Claude-Session: https://claude.ai/code/session_01BedF9ZmqVZLkvDwn91fuKy Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CLAUDE.md | 2 +- README.md | 2 +- docs/TEST_COVERAGE.md | 2 +- src/query/datalog/magic_sets.rs | 45 ++++++++++++++++++---- tests/magic_sets_test.rs | 67 +++++++++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bbbe16e4e..52ec33a6ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **A power loss could lose a new database or WAL file, or bring back a deleted WAL (#389).** On Linux and other POSIX systems, a created or deleted file survives a power loss only once its parent directory is fsynced; Minigraf fsynced file contents but never a directory. Creating the `.graph` file or the `.wal` sidecar, and deleting the WAL after a checkpoint, now fsync the parent directory after the file operation. A resurrected WAL was already harmless, since replay skips entries at or below the last checkpointed transaction, but a lost `.graph` or WAL lost committed data. Windows needs no directory sync and is unchanged. Process kills were never affected, because they leave the OS page cache intact. +- **Recursive rules failed with a literal start when the recursion goes through another rule (#297).** `(query [:find ?y :where (chain :a ?y)])` over `(chain ?x ?y) :- (link ?x ?mid) (chain ?mid ?y)` failed with `Unbound variable in rule head: ?mid` (INT-022). The magic-sets rewrite, which runs when a rule argument is given as a literal, dropped earlier rule calls from the rules that pass bindings to the next call, so a variable bound only by such a call (here `?mid`, bound by `link`) was never bound. #300 fixed the case where a fact pattern binds it. Those calls are now kept, and results match the same query without a literal start. - **Bound-entity queries could drop rows when one `WriteTransaction` wrote the same attribute in several valid-time windows (#323).** Both facts share a `tx_count`, and the selective lookup path de-duplicated on `(entity, attribute, tx_count, asserted)`, so one window's value was silently lost, while the same query via a full scan returned both. That de-duplication has been removed, and bound-entity queries now always match a full scan. ### Documentation diff --git a/CLAUDE.md b/CLAUDE.md index 240f51ef6e..08814bf337 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -174,7 +174,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1200 tests passing** (1192 passing, 8 ignored; unit + integration + doc). +**1203 tests passing** (1195 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 5c83eb0e21..cd30c7035b 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1200 tests +cargo test # run 1203 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 605e313171..905c1b043b 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,7 +4,7 @@ **Verified**: 2026-09-26 with `cargo test` -**Result**: 1,192 passing tests and 8 ignored tests (1,200 total) +**Result**: 1,195 passing tests and 8 ignored tests (1,203 total) The v2.0.1 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a cyclic leaf chain or damaged slot directory is rejected rather than copied (#315). Directory-sync unit tests check that creating the `.graph` file or the WAL, and deleting the WAL, fsync the parent directory after the file operation, and that reopening an existing file does not (#389). diff --git a/src/query/datalog/magic_sets.rs b/src/query/datalog/magic_sets.rs index 2ba77b1408..48660baa43 100644 --- a/src/query/datalog/magic_sets.rs +++ b/src/query/datalog/magic_sets.rs @@ -188,7 +188,7 @@ pub(crate) fn inject_magic_guard(rule: &Rule, predicate: &str, adornment: &[char /// Build magic propagation rules for adorned recursive calls within a rule body. /// /// For each `RuleInvocation` in the body that calls an adorned predicate, -/// emits a rule: (magic-p-ad ?bound_arg) :- (magic-p-ad ?head_bound_var) +/// emits a rule: (magic-p-ad ?bound_arg) :- (magic-p-ad ?head_bound_var) /// /// Returns Vec<(predicate_name, Rule)> to be registered with `register_rule_unchecked`. #[allow(dead_code)] @@ -284,13 +284,12 @@ pub(crate) fn build_propagation_rules( .filter_map(|(pos, _)| called_args.get(pos).cloned()) .collect(); - // Propagation body = guard + all non-RuleInvocation clauses before this call. - let mut prop_body = vec![guard.clone()]; - for preceding in rule.body.iter().take(i) { - if !matches!(preceding, WhereClause::RuleInvocation { .. }) { - prop_body.push(preceding.clone()); - } - } + // Propagation body = guard + every clause before this call, including rule + // invocations: a preceding call may be the only binder of a variable passed + // in a bound position (e.g. `(link ?x ?mid) (chain ?mid ?y)`, #297). + let mut prop_body = Vec::with_capacity(1 + i); + prop_body.push(guard.clone()); + prop_body.extend(rule.body.iter().take(i).cloned()); // Build the propagation rule head. // For fb-adorned called predicates: head = [pred, sentinel, bound_val_var] @@ -695,6 +694,36 @@ mod tests { } } + #[test] + fn test_propagation_rule_keeps_preceding_rule_invocation() { + // (chain ?x ?y) :- (link ?x ?mid) (chain ?mid ?y) + // bf → (__magic_chain_bf ?mid) :- (__magic_chain_bf ?x) (link ?x ?mid) + // The preceding (link ...) call is the only binder of ?mid (#297). + let rule = make_rule( + "chain", + &["?x", "?y"], + vec![ + rule_inv("link", &["?x", "?mid"]), + rule_inv("chain", &["?mid", "?y"]), + ], + ); + let adorned: HashMap> = [ + ("chain".to_string(), vec!['b', 'f']), + ("link".to_string(), vec!['b', 'f']), + ] + .into_iter() + .collect(); + let prop_rules = build_propagation_rules(&rule, "chain", &adorned); + let chain_magic = magic_pred_name("chain", &['b', 'f']); + let (_, prop) = prop_rules + .iter() + .find(|(pred, _)| *pred == chain_magic) + .expect("propagation rule for the recursive chain call"); + assert_eq!(prop.head[1], EdnValue::Symbol("?mid".to_string())); + assert_eq!(prop.body.len(), 2, "guard + preceding link call"); + assert_eq!(prop.body[1], rule_inv("link", &["?x", "?mid"])); + } + #[test] fn test_propagation_rule_bb_adornment() { // (reachable ?a ?b) :- (reachable ?a ?c) [?c :edge/to ?b] diff --git a/tests/magic_sets_test.rs b/tests/magic_sets_test.rs index 2191b4ab8f..7f035b46fb 100644 --- a/tests/magic_sets_test.rs +++ b/tests/magic_sets_test.rs @@ -306,3 +306,70 @@ fn test_recursive_intermediate_variable_with_keyword_entity() { _ => panic!("expected QueryResults"), } } + +fn extract_keywords(result: QueryResult) -> Vec { + match result { + QueryResult::QueryResults { results, .. } => { + let mut kws: Vec = results + .into_iter() + .map(|row| match &row[0] { + Value::Keyword(k) => k.clone(), + _ => panic!("Expected Keyword in result row"), + }) + .collect(); + kws.sort(); + kws + } + _ => panic!("Expected QueryResults"), + } +} + +/// #297 (reopened) — the intermediate variable is bound by a call to another +/// user-defined rule rather than a fact pattern. The magic propagation rule +/// must keep that call in its body, otherwise `?mid` is unbound in its head. +#[test] +fn test_recursive_intermediate_variable_bound_by_rule_call() { + let db = open_db(); + + exec( + &db, + r#"(transact [[:a :next :b] [:b :next :c] [:x :next :y]])"#, + ); + exec(&db, r#"(rule [(link ?x ?y) [?x :next ?y]])"#); + exec(&db, r#"(rule [(chain ?x ?y) (link ?x ?y)])"#); + exec( + &db, + r#"(rule [(chain ?x ?y) (link ?x ?mid) (chain ?mid ?y)])"#, + ); + + let bound = extract_keywords(exec(&db, r#"(query [:find ?y :where (chain :a ?y)])"#)); + assert_eq!(bound, vec![":b".to_string(), ":c".to_string()]); + + // Same answer as the pattern-bound form, which does not go through magic sets. + let via_pattern = extract_keywords(exec( + &db, + r#"(query [:find ?y :where [?a :next :b] (chain ?a ?y)])"#, + )); + assert_eq!(bound, via_pattern, "magic sets must not change results"); +} + +/// #297 (reopened) — left-recursive variant: the intermediate variable is bound +/// by a preceding recursive call and then fed into another user rule. +#[test] +fn test_left_recursive_intermediate_variable_bound_by_rule_call() { + let db = open_db(); + + exec( + &db, + r#"(transact [[:a :next :b] [:b :next :c] [:x :next :y]])"#, + ); + exec(&db, r#"(rule [(link ?x ?y) [?x :next ?y]])"#); + exec(&db, r#"(rule [(path ?x ?y) (link ?x ?y)])"#); + exec( + &db, + r#"(rule [(path ?x ?y) (path ?x ?mid) (link ?mid ?y)])"#, + ); + + let bound = extract_keywords(exec(&db, r#"(query [:find ?y :where (path :a ?y)])"#)); + assert_eq!(bound, vec![":b".to_string(), ":c".to_string()]); +}