From de8db7693424aff16e8fddc85bd433006a220bbe Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 7 Oct 2026 11:51:56 +0530 Subject: [PATCH 1/2] ci: test and lint the wasm32 targets again; fix browser panics and dropped pages The repo split (#231) moved the browser and WASI CI jobs to minigraf-wasm, which only builds the package. Core's own wasm tests stopped running and the wasm32 test build broke unnoticed. - New .github/workflows/wasm.yml: wasm-pack test in headless Firefox and Chrome; unit and integration tests on wasm32-wasip1 under Wasmtime; clippy -D warnings on both wasm targets. - Gate the seven file-backed integration test files that lacked #![cfg(not(target_arch = "wasm32"))], and pid_ns_helper's main. - Ignore on WASI the eight tests that spawn threads or a child process. - BrowserDb: a dirty page that cannot be read is now an error instead of being skipped from the IndexedDB flush while reporting success. No unwrap() in the IndexedDB upgrade callback, the backend handle or the page-buffer lock (a wasm32 panic aborts the instance). IndexedDB keys that are not whole page numbers are rejected, not truncated. - The nine BrowserBufferBackend tests used #[test], which the wasm test runner never runs; they are #[wasm_bindgen_test] now and pass. - wal, storage::verify and storage::dir_sync, and the file-only PersistentFactStorage methods, are not compiled for wasm32, so the wasm32 builds have no warnings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015GgLEs8YozJjYhwPEYgTUn --- .github/workflows/wasm.yml | 85 ++++++++++++++++++++++++ CHANGELOG.md | 1 + CLAUDE.md | 4 +- docs/TEST_COVERAGE.md | 2 +- examples/pid_ns_helper.rs | 4 ++ src/browser/buffer.rs | 21 +++--- src/browser/indexeddb.rs | 68 ++++++++++++++++--- src/browser/mod.rs | 97 +++++++++++++++------------- src/graph/types.rs | 8 ++- src/lib.rs | 2 + src/query/datalog/evaluator.rs | 12 ++++ src/query/datalog/executor.rs | 12 ++++ src/repl.rs | 4 ++ src/storage/backend/mod.rs | 3 - src/storage/mod.rs | 3 + src/storage/persistent_facts.rs | 29 +++++++-- tests/concurrency_test.rs | 3 + tests/cursor_test.rs | 2 + tests/error_codes_foundation_test.rs | 3 + tests/error_codes_storage_test.rs | 2 + tests/error_codes_wal_test.rs | 2 + tests/fact_log_test.rs | 2 + tests/log_writer_test.rs | 2 + tests/udf_test.rs | 1 + 24 files changed, 299 insertions(+), 73 deletions(-) create mode 100644 .github/workflows/wasm.yml diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 000000000..003f5cf51 --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,85 @@ +name: WASM + +# Core's own tests and lints on the two wasm32 targets. The browser (`BrowserDb`, +# `browser` feature) and WASI builds share the engine with native, but their +# cfg-gated code only compiles for wasm32, so native CI never sees it. These jobs +# ran here before the repo split (#231) moved packaging to minigraf-wasm, which +# only builds; without them the wasm32 test build broke unnoticed for months. + +on: + push: + branches: [ "main", "v[0-9]*" ] + pull_request: + branches: [ "main", "v[0-9]*" ] + +env: + CARGO_TERM_COLOR: always + +permissions: + contents: read + +jobs: + browser: + name: Browser tests (headless ${{ matrix.browser }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Both browsers and their WebDriver binaries are preinstalled on + # ubuntu-latest; wasm-pack finds them on PATH. + browser: [firefox, chrome] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: wasm-bindgen tests (lib, incl. BrowserDb and IndexedDB) + run: wasm-pack test --headless --${{ matrix.browser }} --features browser + + wasi: + name: WASI tests (Wasmtime) + runs-on: ubuntu-latest + env: + WASMTIME_VERSION: v49.0.2 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 + - name: Install Wasmtime + run: | + curl -sSfL \ + "https://github.com/bytecodealliance/wasmtime/releases/download/${WASMTIME_VERSION}/wasmtime-${WASMTIME_VERSION}-x86_64-linux.tar.xz" \ + -o /tmp/wasmtime.tar.xz + tar -xf /tmp/wasmtime.tar.xz -C /tmp + echo "/tmp/wasmtime-${WASMTIME_VERSION}-x86_64-linux" >> "$GITHUB_PATH" + - name: Unit and integration tests + # `--dir .` preopens the package root, where tests read fixtures + # such as tests/grammar/. File-backed tests are cfg'd off wasm32. + env: + CARGO_TARGET_WASM32_WASIP1_RUNNER: wasmtime run --dir . + run: cargo test --target wasm32-wasip1 --lib --tests + - name: Build the REPL binary + run: cargo build --target wasm32-wasip1 --release --bin minigraf + + clippy: + name: Clippy (${{ matrix.target }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - target: wasm32-unknown-unknown + features: --features browser + - target: wasm32-wasip1 + features: "" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + components: clippy + - name: Clippy (strict — fails on warnings) + run: cargo clippy --target ${{ matrix.target }} ${{ matrix.features }} -- -D warnings diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b002a5c..321f49d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The wasm32 builds are tested again, and the browser layer no longer panics or drops pages.** The repo split (#231) moved the browser and WASI CI jobs to minigraf-wasm, which only builds the package, so core's own wasm tests stopped running and the wasm32 test build broke unnoticed. A new `wasm.yml` workflow runs the `wasm-bindgen` tests in headless Firefox and Chrome, runs the unit and integration tests on `wasm32-wasip1` under Wasmtime, and lints both targets with `clippy -D warnings`. Fixes found on the way: `BrowserDb` skipped any dirty page it could not read when flushing to IndexedDB and still reported success, and now returns the error. The IndexedDB upgrade callback, the IndexedDB backend handle and the page-buffer lock no longer `unwrap()` (on wasm32 a panic aborts the whole instance). IndexedDB keys that are not whole page numbers are rejected instead of truncated. The nine `BrowserBufferBackend` tests used `#[test]`, which the wasm test runner never runs, and now run. File-only modules (`wal`, `storage::verify`, `storage::dir_sync`) are no longer compiled for wasm32, so the wasm32 builds have no warnings. - Two values of the same attribute written in one transaction are both returned by every query path. Previously only one came back, and entity-bound and attribute-bound queries could return different ones (#371, #287). - A retract that removes several values of one attribute in one call now hides all of them (#371). - Queries no longer lose or duplicate facts when more than 65,535 facts are waiting for a checkpoint (#445; also fixed in v2.0.3). diff --git a/CLAUDE.md b/CLAUDE.md index 4bd63d651..ab18c0226 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -143,11 +143,11 @@ cargo run < demos/demo_negation.txt - `src/fact_log.rs`: `FactLog` / `FactFilter` / `FactRecord` / `FactOrder` — every fact version streamed by `Minigraf::fact_log` (#430): tx order (multi-pass, bounded window) or storage order, key-level filters; an open log pins the committed generation, deferring checkpoints (`API-013`) - `src/log_writer.rs`: `LogWriter` — builds a new file from `FactRecord`s with their tx and valid-time kept (#431): tx-ordered appends with holes (`API-015`/`API-016`), batched copy-on-write checkpoints at tx boundaries, no WAL, built at `.partial` and renamed by `finish` (`STG-043` if the target exists) -7. **`src/wal.rs`** — Fact-level sidecar WAL, CRC32-protected entries, crash recovery +7. **`src/wal.rs`** — Fact-level sidecar WAL, CRC32-protected entries, crash recovery. Like `storage::verify` and `storage::dir_sync`, not compiled for wasm32 (no file backend there) 8. **`src/error.rs`** — Structured error codes: `MinigrafError`, `ErrorCategory`, `ErrorCode` registry (PRS/QRY/STG/WAL/API/INT codes); must match `docs/ERROR_REFERENCE.md`. `MinigrafError::invalid_argument` (API-017) and `MinigrafError::closed` (API-018) exist for language bindings -9. **`src/browser/`** — Browser WASM backend (`browser` feature): `buffer.rs` (`BrowserBufferBackend`, in-memory pages with dirty tracking), `indexeddb.rs` (IndexedDB persistence); `BrowserDb.query` returns a `BrowserCursor` (JSON row batches) +9. **`src/browser/`** — Browser WASM backend (`browser` feature): `buffer.rs` (`BrowserBufferBackend`, in-memory pages with dirty tracking), `indexeddb.rs` (IndexedDB persistence); `BrowserDb.query` returns a `BrowserCursor` (JSON row batches). Tests here must be `#[wasm_bindgen_test]` (the module only builds for wasm32); `.github/workflows/wasm.yml` runs them in headless browsers, runs the WASI tests under Wasmtime, and lints both wasm targets. New file-backed integration test files need `#![cfg(not(target_arch = "wasm32"))]` ### Data Model diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 6bfbd330c..13c6b3401 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -6,7 +6,7 @@ **Result**: 1,337 passing tests and 9 ignored tests (1,346 total) -This covers the unreleased file-format-v8 work (#287, #371): the index-key value-collision fix, the v7→v8 migration, and the drop of formats v1–v6. It otherwise still covers the public database API, the Datalog engine, storage and recovery paths, v2.0.0's locking and structured-error-code changes, and v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Merged from the v2.x line: `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a damaged slot directory is rejected rather than copied (#315). B+tree cursor tests check that every scan walks the tree, not the leaf `next_leaf` chain: full scans and start positions match the input, `seek` from the current position lands where a fresh descent from the root would (randomized, plus within a leaf, to a sibling, across subtrees, past the end, into the gap after a leaf, and backwards as a no-op), a seek to the sibling leaf reads exactly one page, reads are unchanged with every `next_leaf` zeroed, a child pointer into a non-B+tree page is STG-013, and a cycle in child pointers is an error, not a hang (v8 storage format PR 1, #374). Directory-sync unit tests check that creating the `.graph` file or the WAL, and deleting the WAL, fsync the parent directory after the file operation, and that reopening an existing file does not (#389). Disjunction tests check that `or-join` returns an empty result, not INT-031, when no rows reach it: on an empty database, after a pattern that matches nothing, after a `not` that removes every row, and in a rule body (#405). A stratification unit test builds the same two-rule negative cycle 64 times and checks that the INT-054 message names the same predicate pair every time (#410). Magic-sets tests check that a recursive rule with a literal start that recurses through another rule returns the same results as without the literal (#297). API tests check that `execute()` and `WriteTransaction::execute()` reject `$slot` bind tokens in every position with API-010 (#407). The REPL `run()` test now runs in a child process with null stdin, so it no longer hangs when the test runner's stdin is an open non-TTY stream (#415). v8 page-format tests (PR 2, #374, #388) check the 24-byte page header (a flipped bit, a wrong page id and a future generation each give their own STG code, legacy page types fail before the CRC, and a failed page is never cached), meta-page encoding, the free-list chain, WAL v2's base generation, and every row of the meta-selection table: both slots valid, a torn newest slot with the WAL's base generation (opens at the older one), a WAL newer than the valid meta, pages of the next generation on the free list or past `page_count` with no WAL (all STG-033, with the file unmodified), an unknown feature bit (STG-034), and a pre-release v8 single header (STG-032). Fault-injection tests crash `save()` at every page write (cleanly and with torn pages, including the meta write) and every sync, and crash a v7 migration at every write; each reopen finds exactly the old or the new checkpoint, and every reachable page plus the free list covers the file exactly. Twenty-five consecutive saves never write a page the previous meta references. Covering-key tests (PR 3, #433) check that byte order equals logical order for every key component and value type on random tuples (negative, zero and large integers, NaN and ±0.0, strings with 0x00, 64- and 65-byte strings, FOREVER, newest-transaction-first) and that every component round-trips; that prefix-compressed leaves and shortest-separator internal nodes round-trip, agree with a linear scan, keep `max(left) < s ≤ min(right)`, and reject an overflow; that the byte-key B+tree's builds, incremental rebuilds, cursor and `seek` match their expected sets; that dictionary ids are assigned once, never reused, shared between a keyword attribute and value, and deterministic; that a re-asserted or retracted long value writes no new value page and a forced hash collision still finds the right value; that one transaction with two timestamps is STG-037 and a missing dictionary entry STG-036; that committed reads by entity, entity and attribute, attribute, and in full equal an in-memory model across eight checkpoints and a reopen; that short-valued reads touch no value page; that a v7 file with every value type, long strings, refs and retractions migrates to exactly the same facts; that a pre-covering-key v8 file is STG-032; and that string values over 4,068 bytes and attributes or keywords over 1,024 bytes are WAL-003. A WAL test checks that reopening next to a fully checkpointed WAL does not reuse transaction numbers (#447). Copy-on-write checkpoint tests (PR 4, #434) check that lazy free-list pops and pushes conserve every page across page boundaries, partial head pages and chain exhaustion, never hand out a page freed in the same generation, and share unread chain pages unchanged; that a copy-on-write insert equals a sorted merge for random batches (including keys below the first and above the last, many into one leaf, root growth to depth 3, and 1 KB separators that split internal nodes), leaves the old tree readable and unchanged, frees exactly the replaced nodes, writes only new pages, writes exactly one leaf and two internal nodes for one entry in a depth-3 tree, and does not fragment leaves under single inserts; that a checkpoint after 1 or 100 right-edge facts writes at most the tree-depth difference more pages at 100,000 facts than at 10,000, and at most one path per new entry per tree for random keys; and that four reader threads querying throughout 40 checkpoints never fail and never see the fact count go backwards. Hardening tests (PR 5) damage an index leaf, an internal node, a dictionary leaf, a value page and a free-list page in turn. Every committed read or checkpoint that reaches the damaged page fails with STG-029, never with wrong or fewer rows. A query through `Minigraf` on a file with damaged leaves fails with STG-029, and the file still opens at its last checkpoint. A size test measures at most 200 bytes per fact (142 measured) on 144k facts in #433's shape; its 1.15M-fact variant is ignored locally and runs in the nightly smoke workflow. A shared dictionary-cache test checks that cached ident and timestamp lookups return the same facts and that the cache stays bounded. Merged from the v2.x line: storage and `edge_cases_test.rs` tests check that more than 65,535 uncheckpointed facts each resolve to their own fact through entity lookups and attribute scans, in memory and file-backed (#445). Merged from the v2.x line: `tests/golden_corpus_test.rs` opens a copy of each committed golden file in `tests/golden/` (v7 files written by minigraf 2.0.3: one checkpoint, several checkpoints, indexes rebuilt on open, a crash-left WAL, an already-checkpointed WAL, same-transaction multi-values) and checks its manifest's CRC32, `tx_count` and query results after open, after a checkpoint and reopen, and after one more write; a second test checks that every golden file has a manifest (#391). On `v3` the same test checks that each v7 file has a v8 meta page after its first open and gives the same results after reopening the migrated file, and adds seven v8 golden files (one checkpoint, seven copy-on-write checkpoints, a crash-left v2 WAL, an already-checkpointed v2 WAL, same-transaction multi-values, a v7 file migrated in place, and long values at the size limits); the #371 queries, skipped on v2.x, run and pass. Integrity tests (#373) check that intact files verify with no problems and an exact fact and page count, and that `verify` reports each kind of damage with its code: an EAVT, AEVT, AVET or VAET leaf missing an entry (STG-038), keys out of order and a page reached from two trees (STG-039), a rotted index leaf (STG-029), a free page still in use and a leaked page (STG-035), a missing transaction entry (STG-036), and an entity id whose UUID maps to another id (STG-040). After each index or free-list damage, `rebuild_indexes` leaves a file that verifies clean, reads back every fact and accounts for every page. The source is an index that agrees with another one, or the only intact one; three intact indexes that all disagree, or a damaged dictionary, give STG-041 with the file unchanged. A rebuild commits pending facts, including new ids and long values. A crash at every write (clean and torn) and every sync of a rebuild reopens either at the old generation with only its old damage or at the new one with none, never losing a fact. Through `Minigraf` (`tests/integrity_test.rs`), a rebuild commits WAL entries and deletes the WAL, and flipping a byte in each leaf of a file in turn is always reported, then either repaired (index leaves) or refused with STG-041 and the file unchanged (dictionary leaves). Live-read tests (#379) check that `triple_len` ends exactly where `tx↓` begins for every value type and index; that on random histories (several valid-time windows, retractions, re-assertions, several values per attribute, long values, 12 transactions per checkpoint) the committed live read for every scan kind and `:as-of` point is exactly the net-asserted set, that it stays equivalent with pending facts on top, and after reopen; and that a point read on a triple asserted and retracted 2,000 times reads under a third of the pages a history read does, with or without `:as-of`. `tests/live_reads_test.rs` runs the same history in a checkpointed file database and in memory and checks that point, attribute, full-scan, join, valid-time, `:as-of` and rule queries return the same rows, before and after reopen. One-window tests (#435) check that `net_asserted_facts` keeps each triple's newest transaction only (a later window replaces an earlier one, closing an open window, a retraction in the newest transaction hides it, records with two windows in one transaction keep both, once each) against a per-triple model on random histories; that the on-disk key walk agrees at every `:as-of` before and after reopen; and that `API-011` rejects two windows of one fact in one transaction without taking a `tx_count`. `tests/valid_time_window_test.rs` checks replacing, closing, extending, reopening and retract-then-reassert through entity-bound and attribute queries in memory, on a checkpointed file and after reopen, and `API-011` from `transact` and from `WriteTransaction::commit`. Cursor tests (#432, `tests/cursor_test.rs`) check that `query()` batches partition the `execute()` answer for several batch sizes (never empty, never over `max_rows`, `max_rows == 0` still progresses, `None` stays `None`), that the iterator and aggregates, rules, `:as-of` and `:any-valid-time` match `execute()`, that a cursor opened before a `transact`, a `WriteTransaction` commit, a retract or a checkpoint returns the pre-write answer to the end (in memory and file-backed), that `transact`/`retract`/`rule` are `API-012` and bind slots `API-010`, that a `WriteTransaction` thread gets `INT-001`, that rule limits give `INT-020` as from `execute()`, that `PreparedQuery::query` matches `PreparedQuery::execute`, and that a cursor is `Send` and outlives its database handle. Fact-log tests (#430, `tests/fact_log_test.rs`) check that `fact_log()` returns exactly the records written (assertions and retractions, with `tx_count` and `tx_id`) in memory, with everything pending, half checkpointed, fully checkpointed and after reopen, in both orders, with windows of 1 to 1,000 records and batch sizes of 1 to 1,000, in non-decreasing `tx_count` in Tx order; that every layout gives identical records, including valid-time windows and long string values; that a transaction larger than the window is split with no loss or duplicate; that attribute, attribute-prefix, entity and tx-range filters and their combinations match a model, and unknown names match nothing; that writes after open are not in the log; that `checkpoint()` and `rebuild_indexes()` are `API-013` while a log is open (a checkpoint with nothing to write is allowed), that automatic checkpoints from `execute()` and `WriteTransaction::commit` wait for the log, and that closing, dropping or finishing the log releases the database; that a `WriteTransaction` thread gets `INT-001`; and that a log is `Send`, outlives its database handle, and the deferred writes replay on the next open. Read-only open tests (#429, `tests/read_only_open_test.rs`) check that a read-only handle on a file with checkpointed facts and a pending WAL returns the same query results, `tx_count` and fact log as a read-write open; that queries, a cursor, a fact log, `verify`, `prepare`, rules and a UDF leave the `.graph` and WAL bytes and the `.graph` mtime unchanged, and no WAL is created; that `transact`, `retract`, `begin_write`, `checkpoint` and `rebuild_indexes` are `API-014` without taking a `tx_count`, also on a read-only in-memory database and through the builder path; that a missing file is `STG-042` and nothing is created; that an empty file is an empty database and stays 0 bytes; that read-only handles share a file while a read-write one is refused with `STG-025`, both ways; that a file without write permission opens read-only; and that small and whole-file page caches work. File-backend unit tests check that a shared-mode backend refuses page writes and syncs with `INT-056`, that the in-process registry counts readers and admits readers or one writer (also without kernel locks), and that a reader in another process shares the file with a reader here and refuses a writer here with `STG-026`. Storage unit tests check that a read-only open of a v7 file at every point of an interrupted migration (clean and torn writes) sees every fact and writes nothing, and that an empty backend stays empty. `tests/golden_corpus_test.rs` opens a copy of every golden file read-only first and checks its `tx_count` and queries, and that neither the file nor its WAL changes. Log-writer tests (#431, `tests/log_writer_test.rs`) copy a source's fact log (assertions, retractions, an explicit valid-time window, long strings, refs, keywords, checkpointed and pending facts) through `LogWriter` and check that the new file has no `.partial` or WAL, verifies clean, and gives the source's `tx_count`, fact log and query results (current, `:valid-at`, `:any-valid-time`, every `:as-of`, and entity-, attribute- and value-bound lookups) read-write, read-only and after reopen (#370); that it takes new writes at the source's next `tx_count`; that 90,000 facts over two batch commits verify clean and count correctly at `:as-of`; that a purged middle transaction is a hole whose `:as-of` equals the one before, and trailing holes keep the counter through `advance_tx_count`; that an empty log is an empty database with the counter; that `tx_count` 0, a lower or closed transaction and a backwards advance are `API-015`, a second `tx_id` `API-016`, a second window `API-011` and oversize values or idents `WAL-003`, each leaving the writer usable and the output exact, with repeats written once; that an existing target or WAL is `STG-043`, a second writer `STG-025` and `read_only` `API-014`; that a stale `.partial` is reused; that dropping an unfinished writer leaves nothing; that `finish` refuses a target created during the build and leaves it untouched; and that `LogWriter` is `Send`. Unit tests check that a batch commits only when a transaction opens past the batch size, never splitting one, and that `FileBackend::truncate` empties a file. A unit test in `error.rs` checks that `MinigrafError::invalid_argument` and `MinigrafError::closed`, the constructors for language bindings, give API-017 and API-018 with their text. Three `BrowserDb` tests (wasm32, run with `wasm-bindgen-test` in a browser, not counted above) check that `query()` batches of 1, 7 and 1,000 give the rows of `execute()`, that a cursor keeps the answer from when it opened and returns the end after `close()`, and that a `transact` is API-012. +This covers the unreleased file-format-v8 work (#287, #371): the index-key value-collision fix, the v7→v8 migration, and the drop of formats v1–v6. It otherwise still covers the public database API, the Datalog engine, storage and recovery paths, v2.0.0's locking and structured-error-code changes, and v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Merged from the v2.x line: `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a damaged slot directory is rejected rather than copied (#315). B+tree cursor tests check that every scan walks the tree, not the leaf `next_leaf` chain: full scans and start positions match the input, `seek` from the current position lands where a fresh descent from the root would (randomized, plus within a leaf, to a sibling, across subtrees, past the end, into the gap after a leaf, and backwards as a no-op), a seek to the sibling leaf reads exactly one page, reads are unchanged with every `next_leaf` zeroed, a child pointer into a non-B+tree page is STG-013, and a cycle in child pointers is an error, not a hang (v8 storage format PR 1, #374). Directory-sync unit tests check that creating the `.graph` file or the WAL, and deleting the WAL, fsync the parent directory after the file operation, and that reopening an existing file does not (#389). Disjunction tests check that `or-join` returns an empty result, not INT-031, when no rows reach it: on an empty database, after a pattern that matches nothing, after a `not` that removes every row, and in a rule body (#405). A stratification unit test builds the same two-rule negative cycle 64 times and checks that the INT-054 message names the same predicate pair every time (#410). Magic-sets tests check that a recursive rule with a literal start that recurses through another rule returns the same results as without the literal (#297). API tests check that `execute()` and `WriteTransaction::execute()` reject `$slot` bind tokens in every position with API-010 (#407). The REPL `run()` test now runs in a child process with null stdin, so it no longer hangs when the test runner's stdin is an open non-TTY stream (#415). v8 page-format tests (PR 2, #374, #388) check the 24-byte page header (a flipped bit, a wrong page id and a future generation each give their own STG code, legacy page types fail before the CRC, and a failed page is never cached), meta-page encoding, the free-list chain, WAL v2's base generation, and every row of the meta-selection table: both slots valid, a torn newest slot with the WAL's base generation (opens at the older one), a WAL newer than the valid meta, pages of the next generation on the free list or past `page_count` with no WAL (all STG-033, with the file unmodified), an unknown feature bit (STG-034), and a pre-release v8 single header (STG-032). Fault-injection tests crash `save()` at every page write (cleanly and with torn pages, including the meta write) and every sync, and crash a v7 migration at every write; each reopen finds exactly the old or the new checkpoint, and every reachable page plus the free list covers the file exactly. Twenty-five consecutive saves never write a page the previous meta references. Covering-key tests (PR 3, #433) check that byte order equals logical order for every key component and value type on random tuples (negative, zero and large integers, NaN and ±0.0, strings with 0x00, 64- and 65-byte strings, FOREVER, newest-transaction-first) and that every component round-trips; that prefix-compressed leaves and shortest-separator internal nodes round-trip, agree with a linear scan, keep `max(left) < s ≤ min(right)`, and reject an overflow; that the byte-key B+tree's builds, incremental rebuilds, cursor and `seek` match their expected sets; that dictionary ids are assigned once, never reused, shared between a keyword attribute and value, and deterministic; that a re-asserted or retracted long value writes no new value page and a forced hash collision still finds the right value; that one transaction with two timestamps is STG-037 and a missing dictionary entry STG-036; that committed reads by entity, entity and attribute, attribute, and in full equal an in-memory model across eight checkpoints and a reopen; that short-valued reads touch no value page; that a v7 file with every value type, long strings, refs and retractions migrates to exactly the same facts; that a pre-covering-key v8 file is STG-032; and that string values over 4,068 bytes and attributes or keywords over 1,024 bytes are WAL-003. A WAL test checks that reopening next to a fully checkpointed WAL does not reuse transaction numbers (#447). Copy-on-write checkpoint tests (PR 4, #434) check that lazy free-list pops and pushes conserve every page across page boundaries, partial head pages and chain exhaustion, never hand out a page freed in the same generation, and share unread chain pages unchanged; that a copy-on-write insert equals a sorted merge for random batches (including keys below the first and above the last, many into one leaf, root growth to depth 3, and 1 KB separators that split internal nodes), leaves the old tree readable and unchanged, frees exactly the replaced nodes, writes only new pages, writes exactly one leaf and two internal nodes for one entry in a depth-3 tree, and does not fragment leaves under single inserts; that a checkpoint after 1 or 100 right-edge facts writes at most the tree-depth difference more pages at 100,000 facts than at 10,000, and at most one path per new entry per tree for random keys; and that four reader threads querying throughout 40 checkpoints never fail and never see the fact count go backwards. Hardening tests (PR 5) damage an index leaf, an internal node, a dictionary leaf, a value page and a free-list page in turn. Every committed read or checkpoint that reaches the damaged page fails with STG-029, never with wrong or fewer rows. A query through `Minigraf` on a file with damaged leaves fails with STG-029, and the file still opens at its last checkpoint. A size test measures at most 200 bytes per fact (142 measured) on 144k facts in #433's shape; its 1.15M-fact variant is ignored locally and runs in the nightly smoke workflow. A shared dictionary-cache test checks that cached ident and timestamp lookups return the same facts and that the cache stays bounded. Merged from the v2.x line: storage and `edge_cases_test.rs` tests check that more than 65,535 uncheckpointed facts each resolve to their own fact through entity lookups and attribute scans, in memory and file-backed (#445). Merged from the v2.x line: `tests/golden_corpus_test.rs` opens a copy of each committed golden file in `tests/golden/` (v7 files written by minigraf 2.0.3: one checkpoint, several checkpoints, indexes rebuilt on open, a crash-left WAL, an already-checkpointed WAL, same-transaction multi-values) and checks its manifest's CRC32, `tx_count` and query results after open, after a checkpoint and reopen, and after one more write; a second test checks that every golden file has a manifest (#391). On `v3` the same test checks that each v7 file has a v8 meta page after its first open and gives the same results after reopening the migrated file, and adds seven v8 golden files (one checkpoint, seven copy-on-write checkpoints, a crash-left v2 WAL, an already-checkpointed v2 WAL, same-transaction multi-values, a v7 file migrated in place, and long values at the size limits); the #371 queries, skipped on v2.x, run and pass. Integrity tests (#373) check that intact files verify with no problems and an exact fact and page count, and that `verify` reports each kind of damage with its code: an EAVT, AEVT, AVET or VAET leaf missing an entry (STG-038), keys out of order and a page reached from two trees (STG-039), a rotted index leaf (STG-029), a free page still in use and a leaked page (STG-035), a missing transaction entry (STG-036), and an entity id whose UUID maps to another id (STG-040). After each index or free-list damage, `rebuild_indexes` leaves a file that verifies clean, reads back every fact and accounts for every page. The source is an index that agrees with another one, or the only intact one; three intact indexes that all disagree, or a damaged dictionary, give STG-041 with the file unchanged. A rebuild commits pending facts, including new ids and long values. A crash at every write (clean and torn) and every sync of a rebuild reopens either at the old generation with only its old damage or at the new one with none, never losing a fact. Through `Minigraf` (`tests/integrity_test.rs`), a rebuild commits WAL entries and deletes the WAL, and flipping a byte in each leaf of a file in turn is always reported, then either repaired (index leaves) or refused with STG-041 and the file unchanged (dictionary leaves). Live-read tests (#379) check that `triple_len` ends exactly where `tx↓` begins for every value type and index; that on random histories (several valid-time windows, retractions, re-assertions, several values per attribute, long values, 12 transactions per checkpoint) the committed live read for every scan kind and `:as-of` point is exactly the net-asserted set, that it stays equivalent with pending facts on top, and after reopen; and that a point read on a triple asserted and retracted 2,000 times reads under a third of the pages a history read does, with or without `:as-of`. `tests/live_reads_test.rs` runs the same history in a checkpointed file database and in memory and checks that point, attribute, full-scan, join, valid-time, `:as-of` and rule queries return the same rows, before and after reopen. One-window tests (#435) check that `net_asserted_facts` keeps each triple's newest transaction only (a later window replaces an earlier one, closing an open window, a retraction in the newest transaction hides it, records with two windows in one transaction keep both, once each) against a per-triple model on random histories; that the on-disk key walk agrees at every `:as-of` before and after reopen; and that `API-011` rejects two windows of one fact in one transaction without taking a `tx_count`. `tests/valid_time_window_test.rs` checks replacing, closing, extending, reopening and retract-then-reassert through entity-bound and attribute queries in memory, on a checkpointed file and after reopen, and `API-011` from `transact` and from `WriteTransaction::commit`. Cursor tests (#432, `tests/cursor_test.rs`) check that `query()` batches partition the `execute()` answer for several batch sizes (never empty, never over `max_rows`, `max_rows == 0` still progresses, `None` stays `None`), that the iterator and aggregates, rules, `:as-of` and `:any-valid-time` match `execute()`, that a cursor opened before a `transact`, a `WriteTransaction` commit, a retract or a checkpoint returns the pre-write answer to the end (in memory and file-backed), that `transact`/`retract`/`rule` are `API-012` and bind slots `API-010`, that a `WriteTransaction` thread gets `INT-001`, that rule limits give `INT-020` as from `execute()`, that `PreparedQuery::query` matches `PreparedQuery::execute`, and that a cursor is `Send` and outlives its database handle. Fact-log tests (#430, `tests/fact_log_test.rs`) check that `fact_log()` returns exactly the records written (assertions and retractions, with `tx_count` and `tx_id`) in memory, with everything pending, half checkpointed, fully checkpointed and after reopen, in both orders, with windows of 1 to 1,000 records and batch sizes of 1 to 1,000, in non-decreasing `tx_count` in Tx order; that every layout gives identical records, including valid-time windows and long string values; that a transaction larger than the window is split with no loss or duplicate; that attribute, attribute-prefix, entity and tx-range filters and their combinations match a model, and unknown names match nothing; that writes after open are not in the log; that `checkpoint()` and `rebuild_indexes()` are `API-013` while a log is open (a checkpoint with nothing to write is allowed), that automatic checkpoints from `execute()` and `WriteTransaction::commit` wait for the log, and that closing, dropping or finishing the log releases the database; that a `WriteTransaction` thread gets `INT-001`; and that a log is `Send`, outlives its database handle, and the deferred writes replay on the next open. Read-only open tests (#429, `tests/read_only_open_test.rs`) check that a read-only handle on a file with checkpointed facts and a pending WAL returns the same query results, `tx_count` and fact log as a read-write open; that queries, a cursor, a fact log, `verify`, `prepare`, rules and a UDF leave the `.graph` and WAL bytes and the `.graph` mtime unchanged, and no WAL is created; that `transact`, `retract`, `begin_write`, `checkpoint` and `rebuild_indexes` are `API-014` without taking a `tx_count`, also on a read-only in-memory database and through the builder path; that a missing file is `STG-042` and nothing is created; that an empty file is an empty database and stays 0 bytes; that read-only handles share a file while a read-write one is refused with `STG-025`, both ways; that a file without write permission opens read-only; and that small and whole-file page caches work. File-backend unit tests check that a shared-mode backend refuses page writes and syncs with `INT-056`, that the in-process registry counts readers and admits readers or one writer (also without kernel locks), and that a reader in another process shares the file with a reader here and refuses a writer here with `STG-026`. Storage unit tests check that a read-only open of a v7 file at every point of an interrupted migration (clean and torn writes) sees every fact and writes nothing, and that an empty backend stays empty. `tests/golden_corpus_test.rs` opens a copy of every golden file read-only first and checks its `tx_count` and queries, and that neither the file nor its WAL changes. Log-writer tests (#431, `tests/log_writer_test.rs`) copy a source's fact log (assertions, retractions, an explicit valid-time window, long strings, refs, keywords, checkpointed and pending facts) through `LogWriter` and check that the new file has no `.partial` or WAL, verifies clean, and gives the source's `tx_count`, fact log and query results (current, `:valid-at`, `:any-valid-time`, every `:as-of`, and entity-, attribute- and value-bound lookups) read-write, read-only and after reopen (#370); that it takes new writes at the source's next `tx_count`; that 90,000 facts over two batch commits verify clean and count correctly at `:as-of`; that a purged middle transaction is a hole whose `:as-of` equals the one before, and trailing holes keep the counter through `advance_tx_count`; that an empty log is an empty database with the counter; that `tx_count` 0, a lower or closed transaction and a backwards advance are `API-015`, a second `tx_id` `API-016`, a second window `API-011` and oversize values or idents `WAL-003`, each leaving the writer usable and the output exact, with repeats written once; that an existing target or WAL is `STG-043`, a second writer `STG-025` and `read_only` `API-014`; that a stale `.partial` is reused; that dropping an unfinished writer leaves nothing; that `finish` refuses a target created during the build and leaves it untouched; and that `LogWriter` is `Send`. Unit tests check that a batch commits only when a transaction opens past the batch size, never splitting one, and that `FileBackend::truncate` empties a file. A unit test in `error.rs` checks that `MinigrafError::invalid_argument` and `MinigrafError::closed`, the constructors for language bindings, give API-017 and API-018 with their text. Three `BrowserDb` tests (wasm32, run with `wasm-bindgen-test` in a browser, not counted above) check that `query()` batches of 1, 7 and 1,000 give the rows of `execute()`, that a cursor keeps the answer from when it opened and returns the end after `close()`, and that a `transact` is API-012. The wasm32 targets run in CI (`.github/workflows/wasm.yml`). In a browser (`wasm-pack test --headless`, Firefox and Chrome): 23 `wasm_bindgen_test` tests: twelve for `BrowserDb` (including IndexedDB persistence and the cursor), nine for `BrowserBufferBackend` (which used `#[test]` and never ran before), and two checking that IndexedDB page keys accept only whole numbers from 0 to 2^53. On `wasm32-wasip1` under Wasmtime: the unit and integration tests that do not use a file run, 935 passing; nine that spawn threads or a child process are ignored on WASI. These are not counted above. ### Covered Areas diff --git a/examples/pid_ns_helper.rs b/examples/pid_ns_helper.rs index 567b3aaa7..95550023f 100644 --- a/examples/pid_ns_helper.rs +++ b/examples/pid_ns_helper.rs @@ -34,6 +34,10 @@ //! release by the library, nothing but whatever the kernel does on its own. //! That is the exact state #317 is about. +#[cfg(target_arch = "wasm32")] +fn main() {} + +#[cfg(not(target_arch = "wasm32"))] fn main() { let args: Vec = std::env::args().collect(); let path = &args[1]; diff --git a/src/browser/buffer.rs b/src/browser/buffer.rs index a7f71f2ac..64958d95e 100644 --- a/src/browser/buffer.rs +++ b/src/browser/buffer.rs @@ -101,13 +101,16 @@ impl StorageBackend for BrowserBufferBackend { #[cfg(test)] mod tests { + // The browser module only builds for wasm32, where the test runner runs + // `#[wasm_bindgen_test]` functions, not `#[test]` ones. use super::*; + use wasm_bindgen_test::wasm_bindgen_test; fn page(byte: u8) -> Vec { vec![byte; PAGE_SIZE] } - #[test] + #[wasm_bindgen_test] fn write_marks_dirty() { let mut buf = BrowserBufferBackend::new(); buf.write_page(0, &page(1)).unwrap(); @@ -115,7 +118,7 @@ mod tests { assert!(dirty.contains(&0)); } - #[test] + #[wasm_bindgen_test] fn take_dirty_clears_set() { let mut buf = BrowserBufferBackend::new(); buf.write_page(0, &page(1)).unwrap(); @@ -123,7 +126,7 @@ mod tests { assert!(buf.take_dirty().is_empty()); } - #[test] + #[wasm_bindgen_test] fn read_after_write_returns_same_bytes() { let mut buf = BrowserBufferBackend::new(); let p = page(42); @@ -131,7 +134,7 @@ mod tests { assert_eq!(buf.read_page(3).unwrap(), p); } - #[test] + #[wasm_bindgen_test] fn page_count_is_high_water_mark() { let mut buf = BrowserBufferBackend::new(); buf.write_page(0, &page(0)).unwrap(); @@ -142,14 +145,14 @@ mod tests { assert_eq!(buf.page_count().unwrap(), 6); } - #[test] + #[wasm_bindgen_test] fn load_pages_starts_with_no_dirty() { let pages = HashMap::from([(0u64, page(0)), (1u64, page(1))]); let mut buf = BrowserBufferBackend::load_pages(pages); assert!(buf.take_dirty().is_empty()); } - #[test] + #[wasm_bindgen_test] fn load_pages_all_dirty_marks_all() { let pages = HashMap::from([(0u64, page(0)), (1u64, page(1))]); let mut buf = BrowserBufferBackend::load_pages_all_dirty(pages); @@ -158,18 +161,18 @@ mod tests { assert!(dirty.contains(&1)); } - #[test] + #[wasm_bindgen_test] fn empty_buffer_has_no_pages() { assert_eq!(BrowserBufferBackend::new().page_count().unwrap(), 0); } - #[test] + #[wasm_bindgen_test] fn wrong_page_size_errors() { let mut buf = BrowserBufferBackend::new(); assert!(buf.write_page(0, &[0u8; 100]).is_err()); } - #[test] + #[wasm_bindgen_test] fn read_missing_page_errors() { let buf = BrowserBufferBackend::new(); assert!(buf.read_page(99).is_err()); diff --git a/src/browser/indexeddb.rs b/src/browser/indexeddb.rs index 8ca8a0957..428b27440 100644 --- a/src/browser/indexeddb.rs +++ b/src/browser/indexeddb.rs @@ -85,12 +85,11 @@ impl IndexedDbBackend { // Create the object store if this is a fresh database (version upgrade). let on_upgrade: Closure = Closure::once(move |event: web_sys::Event| { - let target = event.target().unwrap(); - let request: web_sys::IdbOpenDbRequest = target.dyn_into().unwrap(); - let db: IdbDatabase = request.result().unwrap().dyn_into().unwrap(); - if !db.object_store_names().contains(&store_name_upgrade) { - db.create_object_store(&store_name_upgrade).unwrap(); - } + // A callback cannot return the error. If the store is not + // created, the first transaction on it fails instead, and + // that error reaches the caller; a panic would abort the + // whole wasm instance. + let _ = create_store(&event, &store_name_upgrade); }); open_request.set_onupgradeneeded(Some(on_upgrade.as_ref().unchecked_ref())); on_upgrade.forget(); @@ -126,8 +125,8 @@ impl IndexedDbBackend { let key = keys_arr.get(i); let page_id = key .as_f64() - .ok_or_else(|| JsValue::from_str("page_id is not a number"))? - as u64; + .and_then(page_id_from_f64) + .ok_or_else(|| JsValue::from_str("page_id is not a valid page number"))?; let val = vals_arr.get(i); let arr: Uint8Array = val.dyn_into()?; pages.insert(page_id, arr.to_vec()); @@ -173,3 +172,56 @@ impl IndexedDbBackend { Ok(()) } } + +/// Create the object store `name` in the database an `upgradeneeded` event +/// opens, unless it exists. +fn create_store(event: &web_sys::Event, name: &str) -> Result<(), JsValue> { + let target = event + .target() + .ok_or_else(|| JsValue::from_str("upgradeneeded event has no target"))?; + let request: web_sys::IdbOpenDbRequest = target.dyn_into()?; + let db: IdbDatabase = request.result()?.dyn_into()?; + if !db.object_store_names().contains(name) { + db.create_object_store(name)?; + } + Ok(()) +} + +/// A page id stored as an IndexedDB key (a JS number): a whole, non-negative +/// number that a JS number holds exactly (at most 2^53). +fn page_id_from_f64(key: f64) -> Option { + const MAX_SAFE: f64 = 9_007_199_254_740_992.0; // 2^53 + if key.fract() != 0.0 || !(0.0..=MAX_SAFE).contains(&key) { + return None; + } + // Checked above: whole and in range, so the cast is exact. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + Some(key as u64) +} + +#[cfg(test)] +mod tests { + use super::page_id_from_f64; + use wasm_bindgen_test::*; + + #[wasm_bindgen_test] + fn page_id_accepts_whole_non_negative_numbers() { + assert_eq!(page_id_from_f64(0.0), Some(0)); + assert_eq!(page_id_from_f64(42.0), Some(42)); + assert_eq!(page_id_from_f64(9_007_199_254_740_992.0), Some(1 << 53)); + } + + #[wasm_bindgen_test] + fn page_id_rejects_fractions_negatives_and_non_finite() { + for bad in [ + -1.0, + 0.5, + 9_007_199_254_740_994.0, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + ] { + assert_eq!(page_id_from_f64(bad), None); + } + } +} diff --git a/src/browser/mod.rs b/src/browser/mod.rs index 0a2625290..8f63a30b7 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -161,26 +161,22 @@ impl BrowserDb { /// so `checkpoint()` is only needed after `import_graph()` or explicit bulk ops. /// No-op for in-memory databases. pub async fn checkpoint(&self) -> Result<(), JsValue> { - let (dirty_pages, has_idb) = { + let (dirty_pages, idb) = { let mut inner = self.inner.borrow_mut(); inner .pfs .save() .map_err(|e| JsValue::from_str(&e.to_string()))?; - let dirty_ids = inner.pfs.with_backend_mut(|b| b.take_dirty()); - let pages: Vec<(u64, Vec)> = dirty_ids - .into_iter() - .filter_map(|id| { - inner - .pfs - .with_backend(|b| b.read_page_raw(id).ok().map(|d| (id, d))) - }) - .collect(); - (pages, inner.idb.is_some()) + let pages = take_dirty_pages(&mut inner.pfs)?; + ( + pages, + inner.idb.as_ref().map(IndexedDbBackend::clone_handle), + ) }; - if has_idb && !dirty_pages.is_empty() { - let idb = self.inner.borrow().idb.as_ref().unwrap().clone_handle(); + if let Some(idb) = idb + && !dirty_pages.is_empty() + { idb.write_pages(dirty_pages).await?; } Ok(()) @@ -199,10 +195,14 @@ impl BrowserDb { let page_count = inner .pfs .with_backend(|b| b.page_count_raw()) - .map_err(|e| JsValue::from_str(&e.to_string()))? as usize; + .map_err(|e| JsValue::from_str(&e.to_string()))?; + let capacity = usize::try_from(page_count) + .ok() + .and_then(|n| n.checked_mul(crate::storage::PAGE_SIZE)) + .ok_or_else(|| JsValue::from_str("database is too large to export"))?; - let mut blob = Vec::with_capacity(page_count * crate::storage::PAGE_SIZE); - for id in 0..page_count as u64 { + let mut blob = Vec::with_capacity(capacity); + for id in 0..page_count { let page = inner .pfs .with_backend(|b| b.read_page_raw(id)) @@ -220,7 +220,7 @@ impl BrowserDb { #[wasm_bindgen(js_name = importGraph)] pub async fn import_graph(&self, data: js_sys::Uint8Array) -> Result<(), JsValue> { let bytes = data.to_vec(); - if bytes.len() % crate::storage::PAGE_SIZE != 0 { + if !bytes.len().is_multiple_of(crate::storage::PAGE_SIZE) { return Err(JsValue::from_str( "import data length is not a multiple of PAGE_SIZE", )); @@ -232,7 +232,7 @@ impl BrowserDb { } // ── Sync section ────────────────────────────────────────────────────────── - let (dirty_pages, has_idb) = { + let (dirty_pages, idb) = { let mut inner = self.inner.borrow_mut(); let buffer = BrowserBufferBackend::load_pages_all_dirty(pages); // Page cache capacity 0 — see comment in `open_in_memory` above. @@ -241,23 +241,21 @@ impl BrowserDb { let new_fact_storage = new_pfs.storage().clone(); // Drain dirty set and collect owned page bytes before swapping inner. - let dirty_ids = new_pfs.with_backend_mut(|b| b.take_dirty()); - let dirty_pages: Vec<(u64, Vec)> = dirty_ids - .into_iter() - .filter_map(|id| { - new_pfs.with_backend(|b| b.read_page_raw(id).ok().map(|d| (id, d))) - }) - .collect(); + let dirty_pages = take_dirty_pages(&mut new_pfs)?; inner.pfs = new_pfs; inner.fact_storage = new_fact_storage; - (dirty_pages, inner.idb.is_some()) + ( + dirty_pages, + inner.idb.as_ref().map(IndexedDbBackend::clone_handle), + ) }; // ── Borrow dropped ──────────────────────────────────────────────────────── - if has_idb && !dirty_pages.is_empty() { - let idb = self.inner.borrow().idb.as_ref().unwrap().clone_handle(); + if let Some(idb) = idb + && !dirty_pages.is_empty() + { idb.write_pages(dirty_pages).await?; } Ok(()) @@ -316,7 +314,7 @@ impl BrowserDb { f.tx_id = tx_id; f.tx_count = tx_count; if f.asserted && f.valid_from == VALID_FROM_USE_TX_TIME { - f.valid_from = tx_id as i64; + f.valid_from = tx_id.cast_signed(); } f }) @@ -336,15 +334,7 @@ impl BrowserDb { .map_err(|e| JsValue::from_str(&e.to_string()))?; // Collect dirty pages as owned Vec<(u64, Vec)> — no borrows escape - let dirty_ids = inner.pfs.with_backend_mut(|b| b.take_dirty()); - let dirty_pages: Vec<(u64, Vec)> = dirty_ids - .into_iter() - .filter_map(|id| { - inner - .pfs - .with_backend(|b| b.read_page_raw(id).ok().map(|d| (id, d))) - }) - .collect(); + let dirty_pages = take_dirty_pages(&mut inner.pfs)?; let json = if is_retract { format!(r#"{{"retracted":{}}}"#, tx_id) @@ -357,12 +347,16 @@ impl BrowserDb { // ── Borrow dropped here ─────────────────────────────────────────────── // ── Async section: flush to IDB (no RefCell borrow held) ───────────── - if !dirty_pages.is_empty() { - let has_idb = self.inner.borrow().idb.is_some(); - if has_idb { - let idb = self.inner.borrow().idb.as_ref().unwrap().clone_handle(); - idb.write_pages(dirty_pages).await?; - } + let idb = self + .inner + .borrow() + .idb + .as_ref() + .map(IndexedDbBackend::clone_handle); + if let Some(idb) = idb + && !dirty_pages.is_empty() + { + idb.write_pages(dirty_pages).await?; } Ok(result_json) @@ -419,6 +413,21 @@ impl BrowserCursor { } } +/// The bytes of every page the last `save()` dirtied, clearing the dirty set. +/// A page that cannot be read is an error, never a page silently left out of +/// the flush to IndexedDB. +fn take_dirty_pages( + pfs: &mut PersistentFactStorage, +) -> Result)>, JsValue> { + let ids = pfs.with_backend_mut(|b| b.take_dirty()); + pfs.with_backend(|b| { + ids.into_iter() + .map(|id| b.read_page_raw(id).map(|d| (id, d))) + .collect::>>() + }) + .map_err(to_js_error) +} + fn to_js_error(e: anyhow::Error) -> JsValue { JsValue::from_str(&crate::MinigrafError::from(e).to_string()) } diff --git a/src/graph/types.rs b/src/graph/types.rs index 23c3b948e..6f6d14af8 100644 --- a/src/graph/types.rs +++ b/src/graph/types.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; -use std::time::{SystemTime, UNIX_EPOCH}; use uuid::Uuid; // ============================================================================ @@ -20,10 +19,15 @@ pub(crate) fn tx_id_now() -> TxId { // On WASM browser targets, `std::time::SystemTime::now()` panics. // Use `js_sys::Date::now()` which returns milliseconds since the Unix // epoch as an f64 (same precision as `Date.now()` in JavaScript). - js_sys::Date::now() as u64 + // It is a whole, non-negative number far below 2^53, and an `f64` to + // `u64` cast saturates (NaN gives 0), so the cast cannot misbehave. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let ms = js_sys::Date::now() as u64; + ms } #[cfg(not(all(target_arch = "wasm32", feature = "browser")))] { + use std::time::{SystemTime, UNIX_EPOCH}; u64::try_from( SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/src/lib.rs b/src/lib.rs index 3bf378e7c..3b65b1fa1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -89,6 +89,8 @@ pub(crate) mod query; pub mod repl; pub(crate) mod storage; pub(crate) mod temporal; +/// The sidecar WAL of file-backed databases; wasm32 targets have no file backend. +#[cfg(not(target_arch = "wasm32"))] pub(crate) mod wal; #[cfg(all(target_arch = "wasm32", feature = "browser"))] diff --git a/src/query/datalog/evaluator.rs b/src/query/datalog/evaluator.rs index 866de0c49..aedd9a879 100644 --- a/src/query/datalog/evaluator.rs +++ b/src/query/datalog/evaluator.rs @@ -1156,6 +1156,10 @@ mod tests { } #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "poisons the lock from a spawned thread; WASI has no threads" + )] fn evaluate_iteration_rule_registry_lock_poisoned_error() { let storage = create_test_storage(); let rules = Arc::new(RwLock::new(RuleRegistry::new())); @@ -1185,6 +1189,10 @@ mod tests { } #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "poisons the lock from a spawned thread; WASI has no threads" + )] fn evaluate_rule_function_registry_lock_poisoned_error() { // Exercises evaluate_rule's function registry read (used to apply // Expr clauses in a rule body). @@ -1217,6 +1225,10 @@ mod tests { } #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "poisons the lock from a spawned thread; WASI has no threads" + )] fn stratified_evaluate_rule_registry_lock_poisoned_error() { let storage = create_test_storage(); let rules = Arc::new(RwLock::new(RuleRegistry::new())); diff --git a/src/query/datalog/executor.rs b/src/query/datalog/executor.rs index 942c9b0e3..cf0cfa1ab 100644 --- a/src/query/datalog/executor.rs +++ b/src/query/datalog/executor.rs @@ -4698,6 +4698,10 @@ mod expr_eval_tests { } #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "poisons the lock from a spawned thread; WASI has no threads" + )] fn execute_query_functions_lock_poisoned_error() { let storage = FactStorage::new(); let rules = Arc::new(RwLock::new(RuleRegistry::new())); @@ -4712,6 +4716,10 @@ mod expr_eval_tests { } #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "poisons the lock from a spawned thread; WASI has no threads" + )] fn execute_rule_rules_lock_poisoned_error() { let storage = FactStorage::new(); let rules = Arc::new(RwLock::new(RuleRegistry::new())); @@ -4726,6 +4734,10 @@ mod expr_eval_tests { } #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "poisons the lock from a spawned thread; WASI has no threads" + )] fn execute_query_with_rules_rules_lock_poisoned_error() { // Covers the magic-sets rewrite's rules.read() in execute_query_with_rules // (distinct call site from execute_rule's rules.write()). diff --git a/src/repl.rs b/src/repl.rs index 601c3a4d9..694e76f4e 100644 --- a/src/repl.rs +++ b/src/repl.rs @@ -235,6 +235,10 @@ mod tests { use crate::db::Minigraf; #[test] + #[cfg_attr( + target_os = "wasi", + ignore = "spawns a child process, which WASI cannot" + )] fn run_public_method_exits_on_non_tty_stdin() { // Exercises the two otherwise-uncovered lines in the public `run()` wrapper // (is_terminal + run_impl call). `run()` reads the process's real stdin, which diff --git a/src/storage/backend/mod.rs b/src/storage/backend/mod.rs index fd8bf92ab..f69b749d1 100644 --- a/src/storage/backend/mod.rs +++ b/src/storage/backend/mod.rs @@ -21,6 +21,3 @@ pub use memory::MemoryBackend; #[cfg(not(target_arch = "wasm32"))] #[allow(unused_imports)] pub use file::FileBackend; - -#[cfg(all(target_arch = "wasm32", feature = "browser"))] -pub use crate::browser::buffer::BrowserBufferBackend; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index a22ae7e2b..63e05b423 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -9,6 +9,7 @@ pub mod backend; pub(crate) mod btree; pub mod cache; pub(crate) mod dict; +#[cfg(not(target_arch = "wasm32"))] pub(crate) mod dir_sync; pub(crate) mod freelist; pub mod index; @@ -20,6 +21,8 @@ pub(crate) mod page; pub mod persistent_facts; pub(crate) mod reader; pub(crate) mod value_pages; +// `Minigraf::verify` / `rebuild_indexes` walk a file; not built for wasm32. +#[cfg(not(target_arch = "wasm32"))] pub(crate) mod verify; use crate::error::{ErrorCode, bail_coded, err_coded}; diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index 6cca5cda9..cccb7252e 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -9,13 +9,17 @@ use crate::error::{ErrorCode, bail_coded, err_coded}; use crate::graph::FactStorage; use crate::graph::types::Fact; -use crate::storage::btree::{MutexStorageBackend, build_btree, cow_insert}; +#[cfg(not(target_arch = "wasm32"))] +use crate::storage::btree::MutexStorageBackend; +use crate::storage::btree::{build_btree, cow_insert}; use crate::storage::cache::PageCache; use crate::storage::dict::{DictReader, Encoded, Encoder}; use crate::storage::meta::{MetaPage, SlotState, slot_page}; use crate::storage::page::PageAllocator; use crate::storage::reader::OnDiskReader; -use crate::storage::{LegacyHeaderV7, PAGE_SIZE, StorageBackend, freelist, page, verify}; +#[cfg(not(target_arch = "wasm32"))] +use crate::storage::verify; +use crate::storage::{LegacyHeaderV7, PAGE_SIZE, StorageBackend, freelist, page}; use anyhow::Result; use std::sync::{Arc, Mutex}; @@ -35,6 +39,7 @@ pub struct PersistentFactStorage { /// Page cache size for the integrity walk: enough to keep a value page hot /// while its values are checked, without displacing the readers' cache. +#[cfg(not(target_arch = "wasm32"))] const VERIFY_CACHE_PAGES: usize = 64; /// What open found in the meta slots. @@ -80,6 +85,7 @@ impl PersistentFactStorage { Ok(pfs) } + #[cfg(not(target_arch = "wasm32"))] /// Open storage on `backend` without writing to it (#429). /// /// Meta selection is the same as [`open`](Self::open), but nothing is @@ -119,6 +125,7 @@ impl PersistentFactStorage { } } + #[cfg(not(target_arch = "wasm32"))] /// Read a format v7 file's facts into memory as uncheckpointed facts, on an /// empty committed state, for a read-only open. Returns the meta to /// activate: empty, with the v7 header's tx counter so WAL replay skips @@ -431,6 +438,7 @@ impl PersistentFactStorage { Ok(()) } + #[cfg(not(target_arch = "wasm32"))] /// Check the committed file (#373). Never modifies it, and holds the /// backend lock only for each page read, so queries keep running. pub(crate) fn verify(&self) -> Result { @@ -440,6 +448,7 @@ impl PersistentFactStorage { Ok(verify::verify(&backend, &cache, self.meta)) } + #[cfg(not(target_arch = "wasm32"))] /// Rebuild EAVT, AEVT, AVET and VAET from an intact index and commit them, /// with the pending facts, as the next generation (#373). /// @@ -625,6 +634,7 @@ impl PersistentFactStorage { &self.storage } + #[cfg(not(target_arch = "wasm32"))] /// The `last_checkpointed_tx_count` recorded in the active meta page. /// /// Used by WAL replay to skip entries already present in the main file. @@ -632,6 +642,7 @@ impl PersistentFactStorage { self.meta.last_checkpointed_tx_count } + #[cfg(not(target_arch = "wasm32"))] /// The active meta page's generation. A WAL created now records it as its /// base generation. pub fn generation(&self) -> u64 { @@ -644,11 +655,13 @@ impl PersistentFactStorage { self.meta } + #[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] /// Mark storage as dirty (needs saving) pub fn mark_dirty(&mut self) { self.dirty = true; } + #[cfg(not(target_arch = "wasm32"))] /// Force the dirty flag to true regardless of current state. /// /// Used by checkpoint to ensure save() always writes even if no new @@ -657,6 +670,7 @@ impl PersistentFactStorage { self.mark_dirty(); } + #[cfg(not(target_arch = "wasm32"))] /// Check if storage has unsaved changes pub fn is_dirty(&self) -> bool { self.dirty @@ -678,7 +692,11 @@ impl PersistentFactStorage { where F: FnOnce(&B) -> R, { - let guard = self.backend.lock().unwrap(); + // wasm32 aborts on panic, so the lock is never poisoned. + let guard = self + .backend + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); f(&*guard) } @@ -690,7 +708,10 @@ impl PersistentFactStorage { where F: FnOnce(&mut B) -> R, { - let mut guard = self.backend.lock().unwrap(); + let mut guard = self + .backend + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); f(&mut *guard) } } diff --git a/tests/concurrency_test.rs b/tests/concurrency_test.rs index 7c183d275..76756945b 100644 --- a/tests/concurrency_test.rs +++ b/tests/concurrency_test.rs @@ -1,3 +1,6 @@ +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] + use minigraf::{Minigraf, QueryResult}; use std::sync::Arc; use std::thread; diff --git a/tests/cursor_test.rs b/tests/cursor_test.rs index dcd066140..b5c26fca6 100644 --- a/tests/cursor_test.rs +++ b/tests/cursor_test.rs @@ -2,6 +2,8 @@ //! the answer in batches (#432). These tests pin the public contract the //! streaming engine must keep: batches partition the `execute()` answer, the //! answer is fixed when the cursor opens, and non-queries are rejected. +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] use minigraf::{BindValue, Cursor, Minigraf, OpenOptions, QueryResult, Value}; diff --git a/tests/error_codes_foundation_test.rs b/tests/error_codes_foundation_test.rs index 60aa78673..943f9598a 100644 --- a/tests/error_codes_foundation_test.rs +++ b/tests/error_codes_foundation_test.rs @@ -1,3 +1,6 @@ +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] + use minigraf::{ErrorCategory, Minigraf}; #[test] diff --git a/tests/error_codes_storage_test.rs b/tests/error_codes_storage_test.rs index edebe0ea5..ec361b0a2 100644 --- a/tests/error_codes_storage_test.rs +++ b/tests/error_codes_storage_test.rs @@ -8,6 +8,8 @@ //! crate-private, so these tests cannot call `FileHeader`/`StorageBackend` //! directly — see the unit tests inside `src/storage/*.rs` for codes that //! are only reachable at that lower level). +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] use minigraf::{ErrorCategory, Minigraf}; use std::io::Write; diff --git a/tests/error_codes_wal_test.rs b/tests/error_codes_wal_test.rs index 648653ed4..65ceed28c 100644 --- a/tests/error_codes_wal_test.rs +++ b/tests/error_codes_wal_test.rs @@ -11,6 +11,8 @@ //! and `error::tests::every_error_code_variant_has_a_registry_entry`; there is //! no way to trigger them from the public API without an unrealistic (multi-GB) //! fixture, so they are intentionally not exercised here. +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] use minigraf::{ErrorCategory, Minigraf, OpenOptions}; use std::io::Write; diff --git a/tests/fact_log_test.rs b/tests/fact_log_test.rs index b6e3238e7..16025cda3 100644 --- a/tests/fact_log_test.rs +++ b/tests/fact_log_test.rs @@ -2,6 +2,8 @@ //! valid-time bounds (#430). These tests check the records against a model of //! the writes, in every split between checkpointed and pending facts, both //! orders and small windows, the filters, the snapshot, and the checkpoint pin. +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] use minigraf::{FactFilter, FactLog, FactOrder, FactRecord, Minigraf, OpenOptions, QueryResult}; use std::path::Path; diff --git a/tests/log_writer_test.rs b/tests/log_writer_test.rs index 8af2fc278..58b8abc3d 100644 --- a/tests/log_writer_test.rs +++ b/tests/log_writer_test.rs @@ -3,6 +3,8 @@ //! every query, `:as-of` and fact-log read as the source does, survives a //! reopen with intact indexes (#370), and that holes, rejections, and the //! `.partial` build file behave as documented. +// File-backed (Minigraf::open, tempfile): native only, like the other file tests. +#![cfg(not(target_arch = "wasm32"))] use minigraf::{FactFilter, FactRecord, LogWriter, Minigraf, OpenOptions, QueryResult, Value}; use std::path::{Path, PathBuf}; diff --git a/tests/udf_test.rs b/tests/udf_test.rs index 1c576e32b..45006e6ec 100644 --- a/tests/udf_test.rs +++ b/tests/udf_test.rs @@ -259,6 +259,7 @@ fn unknown_predicate_runtime_error() { // ─── Test 9: thread safety ─────────────────────────────────────────────────── #[test] +#[cfg_attr(target_os = "wasi", ignore = "spawns threads; WASI has no threads")] fn thread_safety() { use std::sync::Arc; From df0ac61cf017b06e14bb67d17747257c286332a3 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 7 Oct 2026 12:04:42 +0530 Subject: [PATCH 2/2] refactor: move the browser page accessors into a wasm-only file PersistentFactStorage::with_backend and with_backend_mut exist only for wasm32 with the browser feature, but tarpaulin counted their lines in the native coverage report as uncovered (codecov/patch failed on #469). They now live in a child module, storage/persistent_facts/browser_access.rs, that only browser builds compile, like src/browser itself. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015GgLEs8YozJjYhwPEYgTUn --- src/storage/persistent_facts.rs | 37 +++--------------- .../persistent_facts/browser_access.rs | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 32 deletions(-) create mode 100644 src/storage/persistent_facts/browser_access.rs diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index cccb7252e..d8fe958cc 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -21,6 +21,11 @@ use crate::storage::reader::OnDiskReader; use crate::storage::verify; use crate::storage::{LegacyHeaderV7, PAGE_SIZE, StorageBackend, freelist, page}; use anyhow::Result; + +// Page access for the browser layer, in its own file so that native builds, +// and the native coverage report, never see it. +#[cfg(all(target_arch = "wasm32", feature = "browser"))] +mod browser_access; use std::sync::{Arc, Mutex}; /// Persistent fact storage with page-based persistence. @@ -682,38 +687,6 @@ impl PersistentFactStorage { pub(crate) fn discard(&mut self) { self.dirty = false; } - - /// Run a closure with read access to the underlying storage backend. - /// - /// Used by the browser WASM layer to read pages after `save()` without - /// exposing the `Arc>` directly. - #[cfg(all(target_arch = "wasm32", feature = "browser"))] - pub(crate) fn with_backend(&self, f: F) -> R - where - F: FnOnce(&B) -> R, - { - // wasm32 aborts on panic, so the lock is never poisoned. - let guard = self - .backend - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - f(&*guard) - } - - /// Run a closure with mutable access to the underlying storage backend. - /// - /// Used by the browser WASM layer to drain dirty pages after `save()`. - #[cfg(all(target_arch = "wasm32", feature = "browser"))] - pub(crate) fn with_backend_mut(&mut self, f: F) -> R - where - F: FnOnce(&mut B) -> R, - { - let mut guard = self - .backend - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - f(&mut *guard) - } } impl Drop for PersistentFactStorage { diff --git a/src/storage/persistent_facts/browser_access.rs b/src/storage/persistent_facts/browser_access.rs new file mode 100644 index 000000000..18ed1a86a --- /dev/null +++ b/src/storage/persistent_facts/browser_access.rs @@ -0,0 +1,38 @@ +//! Direct page access for the browser layer (`src/browser`), which flushes +//! the pages `save()` dirtied to IndexedDB. Built only for wasm32 with the +//! `browser` feature. + +use super::PersistentFactStorage; +use crate::storage::StorageBackend; + +impl PersistentFactStorage { + /// Run a closure with read access to the underlying storage backend. + /// + /// Used by the browser WASM layer to read pages after `save()` without + /// exposing the `Arc>` directly. + pub(crate) fn with_backend(&self, f: F) -> R + where + F: FnOnce(&B) -> R, + { + // wasm32 aborts on panic, so the lock is never poisoned. + let guard = self + .backend + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f(&*guard) + } + + /// Run a closure with mutable access to the underlying storage backend. + /// + /// Used by the browser WASM layer to drain dirty pages after `save()`. + pub(crate) fn with_backend_mut(&mut self, f: F) -> R + where + F: FnOnce(&mut B) -> R, + { + let mut guard = self + .backend + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f(&mut *guard) + } +}