diff --git a/.github/workflows/crash-kill.yml b/.github/workflows/crash-kill.yml index 3a78dfc1b1..fa72f14c7a 100644 --- a/.github/workflows/crash-kill.yml +++ b/.github/workflows/crash-kill.yml @@ -1,5 +1,13 @@ name: SIGKILL crash-safety fuzz (nightly) +# Two branches alternate every 12 hours: main at 03:17 UTC, v3 at 15:17 UTC. +# Scheduled runs always use this file from the default branch, so the branch +# under test is chosen from the cron string that fired (as in bench.yml). +# Keep the cron strings here and in CRASH_REF/run-name identical. +run-name: >- + SIGKILL crash-safety fuzz (${{ github.event_name == 'workflow_dispatch' && inputs.ref + || (github.event.schedule == '17 15 * * *' && 'v3' || 'main') }}) + # #308: an abrupt kill landing mid-checkpoint had a ~26% chance of leaving # the header on disk with a bumped page_count but a stale checksum -- # permanent corruption with no recovery path. The bug was in the ordering @@ -8,17 +16,29 @@ name: SIGKILL crash-safety fuzz (nightly) # matrix rather than Linux only. tests/crash_kill_test.rs runs a small # smoke count in the per-PR suite (rust.yml, already cross-platform); this # workflow runs a much larger round count nightly for statistical -# confidence on each OS. +# confidence on each OS. Since #384 each round also checks that the reopened +# file holds exactly the transactions the child confirmed (EAVT, AEVT, AVET, +# full scan, :as-of; verify on v3). The test prints its base seed; replay a +# failure with MINIGRAF_CRASH_KILL_SEED. on: schedule: - - cron: "17 3 * * *" + - cron: "17 3 * * *" # main + - cron: "17 15 * * *" # v3 workflow_dispatch: + inputs: + ref: + description: Branch to test + required: true + default: main permissions: contents: read issues: write +env: + CRASH_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || (github.event.schedule == '17 15 * * *' && 'v3' || 'main') }} + jobs: crash-kill: name: SIGKILL fuzz suite @@ -31,6 +51,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + ref: ${{ env.CRASH_REF }} - name: Install Rust stable uses: dtolnay/rust-toolchain@stable @@ -49,16 +71,16 @@ jobs: shell: bash env: GH_TOKEN: ${{ github.token }} - ISSUE_TITLE: "Nightly SIGKILL crash-safety failure (${{ matrix.os }})" + ISSUE_TITLE: "Nightly SIGKILL crash-safety failure (${{ env.CRASH_REF }}, ${{ matrix.os }})" run: | report_file="crash-kill-failure.md" { - echo "The nightly SIGKILL crash-safety fuzz suite failed on ${{ matrix.os }}." + echo "The nightly SIGKILL crash-safety fuzz suite failed on ${{ env.CRASH_REF }} (${{ matrix.os }})." echo echo "- Run: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" - echo "- Commit: $GITHUB_SHA" + echo "- Commit: $(git rev-parse HEAD)" echo "- Reproduce: cargo test --test crash_kill_test -- --nocapture" - echo " (set MINIGRAF_CRASH_KILL_ROUNDS to raise the round count)" + echo " (set MINIGRAF_CRASH_KILL_ROUNDS to raise the round count; MINIGRAF_CRASH_KILL_SEED to the printed base seed to replay)" echo echo "## Test output" echo diff --git a/CHANGELOG.md b/CHANGELOG.md index 2707f3e862..268f269963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Tests - **Golden-file compatibility corpus (#391).** `tests/golden/` holds v7 files written by minigraf 2.0.3, each with a JSON manifest of its CRC32, `tx_count` and expected query results. They cover one checkpoint, several checkpoints, indexes rebuilt on open, a WAL left by a crash, a WAL whose entries were already checkpointed, and same-transaction multi-values (#371). `tests/golden_corpus_test.rs` opens a copy of each file and checks its manifest after open, after a checkpoint and reopen, and after one more write. Golden files are never regenerated, only added; a new `golden-corpus` job in `policy.yml` fails a pull request that changes one. The generator is a standalone crate pinned to `minigraf = "=2.0.3"` (`tests/golden/gen/v7/`). +- **SIGKILL crash test checks committed data (#384).** `tests/crash_kill_test.rs` used to check only that the file reopened and one attribute query ran, so it missed #370. The child now runs a seeded workload (batches over several entities and attributes, batched retracts, re-assertions after a retract) and logs each transaction after `execute` returns; it checkpoints after every transaction, every third, never (WAL replay), automatically every 2 writes, or every third with `SyncMode::Normal`. After a `SIGKILL` at a random point, the reopened file must hold exactly the model after the last logged transaction or the one in flight, through a full scan, attribute (AEVT), attribute+value (AVET) and entity-bound (EAVT) queries and `:as-of`; again after a second reopen and after a checkpoint plus reopen. On v2.x the workload avoids the known issues #371 (two values of one attribute in one transaction) and #435 (asserting a triple that is already live returns one row per assertion); the v3 version covers both. Per-PR CI runs 5 rounds; the nightly crash-kill workflow runs 100 per OS. `MINIGRAF_CRASH_KILL_SEED` replays a failure. ## v2.0.3 — 2026-10-06 diff --git a/ROADMAP.md b/ROADMAP.md index e0b14f6be7..55b17844b9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -63,7 +63,7 @@ Scope — format and storage integrity: Scope — release-gate testing and documentation: - Golden-file compatibility corpus for every format version, frozen before v8 work goes further (#391) -- SIGKILL crash test that checks committed data through EAVT, AEVT and full scan (#384) +- SIGKILL crash test that checks committed data through EAVT, AEVT and full scan (#384; also on v2.x, without the #371/#435 cases) - Model-based test of transact/retract/checkpoint/reopen (#385) - Wider property tests: file-backed, batched, retracts, temporal, joins (#386) - Persistent fuzz corpus and an operation-sequence target (#387) diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 6c6bb0443e..f3aeb1138d 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -6,6 +6,8 @@ **Result**: 1,213 passing tests and 8 ignored tests (1,221 total), including the golden-file corpus added after v2.0.3 +`tests/crash_kill_test.rs` kills a child process running a seeded workload under five checkpoint modes at a random point and checks that the reopened file holds exactly the transactions the child logged as committed, or those plus the one in flight, through full-scan, AEVT, AVET, EAVT and `:as-of` queries, again after a second reopen and after a checkpoint plus reopen (#384). The workload leaves out the v2.x known issues #371 and #435. + Not counted above: the `BrowserDb` tests run only on wasm32 with `wasm-bindgen-test` in a browser. Two of them check that the IndexedDB flush returns every page a write dirtied, and that a dirty page that cannot be read is an error rather than left out. The v2.0.3 suite covers the public database API, the Datalog engine, storage and recovery paths, and the v2.0.0 locking and structured-error-code changes, plus v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Since v2.0.1, `tests/point_query_history_test.rs` and new storage unit tests check that attribute-narrowed bound-entity lookups return exactly what a full scan returns, and that `net_asserted_facts` matches its previous implementation on randomized histories (#323). Attribute-scan tests check that the AEVT range covers exactly the queried attribute, including non-ASCII names such as `:丿` and prefix siblings such as `:ab` for `:a` (#381). Incremental B+tree rebuild tests check that checkpoints which copy untouched index leaves produce exactly the entries and file checksum a full rebuild would, including over repeated checkpoints, that single-fact checkpoints do not fragment the index, and that a cyclic leaf chain or damaged slot directory is rejected rather than copied (#315). Directory-sync unit tests check that creating the `.graph` file or the WAL, and deleting the WAL, fsync the parent directory after the file operation, and that reopening an existing file does not (#389). Disjunction tests check that `or-join` returns an empty result, not INT-031, when no rows reach it: on an empty database, after a pattern that matches nothing, after a `not` that removes every row, and in a rule body (#405). A stratification unit test builds the same two-rule negative cycle 64 times and checks that the INT-054 message names the same predicate pair every time (#410). Magic-sets tests check that a recursive rule with a literal start that recurses through another rule returns the same results as without the literal (#297). API tests check that `execute()` and `WriteTransaction::execute()` reject `$slot` bind tokens in every position with API-010 (#407). The REPL `run()` test now runs in a child process with null stdin, so it no longer hangs when the test runner's stdin is an open non-TTY stream (#415). A fault-injection test crashes `save()` at each of its 47 page writes and syncs and checks that every reopen finds exactly the old or the new checkpointed facts (#374). For v2.0.3, storage unit tests and two `edge_cases_test.rs` tests check that more than 65,535 uncheckpointed facts each resolve to their own fact through entity lookups and attribute scans, in memory and file-backed, before and after a checkpoint, and that the widened `FactRef` encodes to the same bytes as before (#445). A WAL test reopens a database next to a WAL whose entries are all checkpointed, writes again, and checks that `:as-of` history is unchanged before and after another reopen; a unit test checks that restoring the transaction counter never lowers it (#447). Since v2.0.3, `tests/golden_corpus_test.rs` opens a copy of each committed golden file in `tests/golden/` (v7 files written by minigraf 2.0.3: one checkpoint, several checkpoints, indexes rebuilt on open, a crash-left WAL, an already-checkpointed WAL, same-transaction multi-values) and checks its manifest's CRC32, `tx_count` and query results after open, after a checkpoint and reopen, and after one more write; a second test checks that every golden file has a manifest (#391). diff --git a/tests/crash_kill_test.rs b/tests/crash_kill_test.rs index b330990f97..b11278ceab 100644 --- a/tests/crash_kill_test.rs +++ b/tests/crash_kill_test.rs @@ -1,39 +1,48 @@ -//! #308 regression: a `SIGKILL` during an active `transact`/`checkpoint` -//! must never permanently corrupt the database (header checksum mismatch, -//! no recovery path). +//! A `SIGKILL` at any point while a child process writes must leave the +//! database holding exactly the transactions the child had committed (#384). //! -//! Two fixes landed together, both closing the same class of bug -- a -//! multi-step on-disk state transition that a kill could land in the -//! middle of: -//! - `src/storage/backend/file.rs`: `FileBackend::write_page` now -//! recomputes `header_checksum` every time it persists a `page_count` -//! bump, so the on-disk header is self-consistent at every single-page -//! write, not just at the deliberate commits in -//! `PersistentFactStorage::save`. -//! - `src/wal.rs`: a WAL sidecar file shorter than its own header (a crash -//! between `create_new` and `write_wal_header` completing) is now -//! treated as an empty WAL instead of a hard read error, since no entry -//! could ever have been appended at that point. +//! The child runs a deterministic workload from a seed: batches across several +//! entities and attributes, batched retracts and re-assertions, with +//! checkpoints at a rate set by the round's variant. After each `execute` returns, it appends the +//! transaction's number to a confirmation log. The parent kills it at a random +//! point, replays the same workload into a reference model, and checks the +//! reopened file through the entity-bound (EAVT), attribute (AEVT), value +//! (AVET) and full-scan paths, and `:as-of`. The checks run again +//! after a second reopen and after a checkpoint plus reopen, to catch damage a +//! checkpoint copies forward. //! -//! This test reproduces the original report directly: spawn a child that -//! loops `transact` + `checkpoint()` on a fresh `.graph` file, kill it with -//! a real `SIGKILL` at a random point mid-loop, and confirm a fresh process -//! can still open the file afterward. +//! A transaction can reach the WAL or file before the kill and still miss the +//! log, so the reopened state is the model after `N` or `N + 1` transactions, +//! where `N` is the last confirmed one. Nothing else is allowed. //! -//! Round count is controlled by `MINIGRAF_CRASH_KILL_ROUNDS` (default: a -//! small smoke count so this runs unconditionally, cheaply, in the per-PR -//! suite). `.github/workflows/crash-kill.yml` runs a much larger nightly -//! count for statistical confidence, matching the ~26% per-round corruption -//! rate observed in the original report before the fix. +//! v2.x: a transaction never writes two values of one entity and attribute +//! (known issue #371), and never asserts a triple that is already live: on +//! v2.x each assertion keeps its own valid-time window, so a re-asserted +//! triple reads back as one row per assertion (#435). Both are fixed in +//! v3.0.0, where the `v3` version of this test covers them, and `verify()`. +//! +//! The log is not fsynced: a process kill keeps the OS page cache, so a +//! written line survives exactly as the WAL entry does. +//! +//! This also covers the original #308 report (a kill mid-checkpoint left a +//! header checksum mismatch): every round must reopen. +//! +//! Round count: `MINIGRAF_CRASH_KILL_ROUNDS` (default 5, one per variant; +//! `.github/workflows/crash-kill.yml` runs 100 nightly on each OS). Replay a +//! failure with the base seed it prints: `MINIGRAF_CRASH_KILL_SEED`. +//! +//! Design: docs/superpowers/specs/2026-10-08-crash-kill-data-check-design.md #![cfg(not(target_arch = "wasm32"))] -use minigraf::db::Minigraf; +use minigraf::db::{Minigraf, OpenOptions, SyncMode}; +use minigraf::{QueryResult, Value}; +use std::collections::{BTreeSet, HashMap}; +use std::io::Write; +use std::path::Path; use std::process::Command; use std::time::{Duration, Instant}; +use uuid::Uuid; -/// Per-PR default: cheap enough to run on every CI build, but with a ~1 - -/// 0.74^5 ≈ 78% chance of catching a reintroduced bug at the original 26% -/// per-round corruption rate. const DEFAULT_ROUNDS: usize = 5; fn rounds() -> usize { @@ -43,36 +52,366 @@ fn rounds() -> usize { .unwrap_or(DEFAULT_ROUNDS) } -/// Cheap, dependency-free jitter for the kill delay. No cryptographic -/// requirement here -- just enough spread across the checkpoint loop's -/// write window to land the kill at varying points mid-`save()`. -fn jitter_millis(seed: u64, max: u64) -> u64 { - let mut x = seed ^ 0x9E37_79B9_7F4A_7C15; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - x % max.max(1) +// ── Workload ────────────────────────────────────────────────────────────────── + +const ENTITIES: u8 = 4; +const ATTRS: [&str; 3] = [":a0", ":a1", ":tag/丿"]; +/// Values 0..8 are integers; 8 and 9 are strings over the 64-byte inline limit +/// that share a 40-byte prefix. +const VALUES: u8 = 10; + +fn long_string(i: u8) -> String { + format!( + "{}{}", + "p".repeat(40), + if i == 8 { "A" } else { "B" }.repeat(40) + ) +} + +fn value_edn(v: u8) -> String { + if v < 8 { + v.to_string() + } else { + format!("\"{}\"", long_string(v)) + } +} + +fn value_render(v: u8) -> String { + if v < 8 { + format!("i{v}") + } else { + format!("s{v}") + } +} + +/// An `(entity, attribute, value)` triple, as pool indices. +type Triple = (u8, u8, u8); + +fn triple_edn((e, a, v): Triple) -> String { + format!("[:e{e} {} {}]", ATTRS[a as usize], value_edn(v)) +} + +/// xorshift64: deterministic, dependency-free. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng(seed ^ 0x9E37_79B9_7F4A_7C15 | 1) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n.max(1) + } + + fn below_u8(&mut self, n: u8) -> u8 { + self.below(u64::from(n)) as u8 + } +} + +/// Keep one triple per entity and attribute (#371 on v2.x). +fn one_per_entity_attr(picked: BTreeSet) -> BTreeSet { + let mut seen = BTreeSet::new(); + picked + .into_iter() + .filter(|&(e, a, _)| seen.insert((e, a))) + .collect() +} + +/// The next transaction of the workload, and the live triples after it. +fn next_tx(rng: &mut Rng, live: &BTreeSet) -> (String, BTreeSet) { + let mut next = live.clone(); + if !live.is_empty() && rng.below(10) < 3 { + // Batched retract of 1-4 live triples. + let pool: Vec = live.iter().copied().collect(); + let mut picked = BTreeSet::new(); + for _ in 0..1 + rng.below(4) { + picked.insert(pool[rng.below(pool.len() as u64) as usize]); + } + let picked = one_per_entity_attr(picked); + let body: Vec = picked.iter().map(|t| triple_edn(*t)).collect(); + for t in &picked { + next.remove(t); + } + return (format!("(retract [{}])", body.join(" ")), next); + } + // Batch of 1-6 triples, at most one per entity and attribute, none live + // already. Over several transactions an attribute gathers several values. + // Triples retracted earlier come back this way. + let mut picked = BTreeSet::new(); + for _ in 0..1 + rng.below(6) { + picked.insert(( + rng.below_u8(ENTITIES), + rng.below_u8(ATTRS.len() as u8), + rng.below_u8(VALUES), + )); + } + let picked = one_per_entity_attr(picked); + let picked: BTreeSet = picked.difference(live).copied().collect(); + if picked.is_empty() { + // Every pick is live already: retract one instead. + let t = *live.iter().next().expect("a live pick"); + next.remove(&t); + return (format!("(retract [{}])", triple_edn(t)), next); + } + let body: Vec = picked.iter().map(|t| triple_edn(*t)).collect(); + next.extend(picked); + (format!("(transact [{}])", body.join(" ")), next) +} + +/// Live triples after each transaction: `history[k]` is the state after `k`. +fn model(seed: u64, txs: u64) -> Vec> { + let mut rng = Rng::new(seed); + let mut history = vec![BTreeSet::new()]; + for _ in 0..txs { + let (_, next) = next_tx(&mut rng, history.last().unwrap()); + history.push(next); + } + history +} + +// ── Variants ────────────────────────────────────────────────────────────────── + +#[derive(Clone, Copy, Debug)] +enum Variant { + /// `checkpoint()` after every transaction: the kill mostly lands in a save. + CheckpointEach, + /// `checkpoint()` after every third transaction. + CheckpointEvery3, + /// Never checkpoints: reopen replays the WAL. + WalOnly, + /// `wal_checkpoint_threshold(2)`: checkpoints inside `execute`. + AutoCheckpoint, + /// As `CheckpointEvery3`, with `SyncMode::Normal` (no WAL fsync). + NormalSync, +} + +const VARIANTS: [Variant; 5] = [ + Variant::CheckpointEach, + Variant::CheckpointEvery3, + Variant::WalOnly, + Variant::AutoCheckpoint, + Variant::NormalSync, +]; + +impl Variant { + fn options(self) -> OpenOptions { + let opts = OpenOptions::new(); + match self { + Variant::WalOnly => opts.wal_checkpoint_threshold(usize::MAX), + Variant::AutoCheckpoint => opts.wal_checkpoint_threshold(2), + Variant::NormalSync => opts.synchronous(SyncMode::Normal), + Variant::CheckpointEach | Variant::CheckpointEvery3 => opts, + } + } + + fn checkpoint_after(self, k: u64) -> bool { + match self { + Variant::CheckpointEach => true, + Variant::CheckpointEvery3 | Variant::NormalSync => k % 3 == 0, + Variant::WalOnly | Variant::AutoCheckpoint => false, + } + } +} + +// ── Checks ──────────────────────────────────────────────────────────────────── + +fn entity_uuid(e: u8) -> Uuid { + Uuid::new_v5(&Uuid::NAMESPACE_OID, format!(":e{e}").as_bytes()) +} + +struct Checker { + uuids: HashMap, + /// Names the round in every failure message. + ctx: String, } +impl Checker { + fn render(&self, v: &Value) -> String { + match v { + Value::Integer(n) => format!("i{n}"), + Value::String(s) => match (8..VALUES).find(|i| long_string(*i) == *s) { + Some(i) => format!("s{i}"), + None => format!("s?{}", s.len()), + }, + Value::Keyword(k) => k.clone(), + Value::Ref(u) => match self.uuids.get(u) { + Some(e) => format!("r{e}"), + None => "r?".to_string(), + }, + _ => "?".to_string(), + } + } + + fn rows(&self, db: &Minigraf, query: &str) -> Vec { + let result = db + .execute(query) + .unwrap_or_else(|e| panic!("{}: query failed ({}): {query}", self.ctx, e.code())); + let QueryResult::QueryResults { results, .. } = result else { + panic!("{}: not a query result: {query}", self.ctx); + }; + let mut rows: Vec = results + .iter() + .map(|row| { + row.iter() + .map(|v| self.render(v)) + .collect::>() + .join(" ") + }) + .collect(); + rows.sort(); + rows + } + + fn expect(&self, db: &Minigraf, query: &str, want: impl Iterator) { + let mut want: Vec = want.collect(); + want.sort(); + assert_eq!(self.rows(db, query), want, "{}: {query}", self.ctx); + } + + /// The reopened state must be the model after `confirmed` or + /// `confirmed + 1` transactions, through every index path. + fn check(&self, db: &Minigraf, history: &[BTreeSet], confirmed: u64) { + let t = db.current_tx_count(); + assert!( + t == confirmed || t == confirmed + 1, + "{}: tx count {t} after {confirmed} confirmed transactions", + self.ctx + ); + let live = &history[t as usize]; + let full = |s: &BTreeSet| -> Vec { + s.iter() + .map(|&(e, a, v)| format!("r{e} {} {}", ATTRS[a as usize], value_render(v))) + .collect() + }; + + // Full scan. + self.expect( + db, + "(query [:find ?e ?a ?v :where [?e ?a ?v]])", + full(live).into_iter(), + ); + // AEVT: attribute scans; AVET: attribute and value bound. + for (ai, attr) in ATTRS.iter().enumerate() { + let ai = ai as u8; + self.expect( + db, + &format!("(query [:find ?e ?v :where [?e {attr} ?v]])"), + live.iter() + .filter(|t| t.1 == ai) + .map(|&(e, _, v)| format!("r{e} {}", value_render(v))), + ); + for v in 0..VALUES { + self.expect( + db, + &format!("(query [:find ?e :where [?e {attr} {}]])", value_edn(v)), + live.iter() + .filter(|t| t.1 == ai && t.2 == v) + .map(|&(e, _, _)| format!("r{e}")), + ); + } + } + // EAVT: entity-bound lookups (the path #370 broke). + for e in 0..ENTITIES { + self.expect( + db, + &format!("(query [:find ?a ?v :where [:e{e} ?a ?v]])"), + live.iter() + .filter(|t| t.0 == e) + .map(|&(_, a, v)| format!("{} {}", ATTRS[a as usize], value_render(v))), + ); + } + // Earlier states, through :as-of. + let samples: BTreeSet = [1, t / 2, t.saturating_sub(1)] + .into_iter() + .filter(|k| (1..t).contains(k)) + .collect(); + for k in samples { + self.expect( + db, + &format!("(query [:find ?e ?a ?v :as-of {k} :where [?e ?a ?v]])"), + full(&history[k as usize]).into_iter(), + ); + } + } +} + +fn open(path: &Path, ctx: &str) -> Minigraf { + Minigraf::open(path).unwrap_or_else(|e| panic!("{ctx}: reopen failed ({})", e.code())) +} + +/// Complete lines in the confirmation log; the last is the newest confirmed +/// transaction. A partial last line is ignored. +fn confirmed(log: &Path) -> u64 { + let text = std::fs::read_to_string(log).unwrap_or_default(); + let complete = &text[..text.rfind('\n').map_or(0, |i| i + 1)]; + let mut n = 0; + for (i, line) in complete.lines().enumerate() { + let k: u64 = line.parse().expect("log line is a number"); + assert_eq!(k, i as u64 + 1, "log lines are 1, 2, 3, ..."); + n = k; + } + n +} + +// ── Test ────────────────────────────────────────────────────────────────────── + #[test] -fn sigkill_during_checkpoint_never_permanently_corrupts_file() { +fn sigkill_keeps_exactly_the_committed_transactions() { let exe = std::env::current_exe().expect("test binary path"); + let base_seed: u64 = std::env::var("MINIGRAF_CRASH_KILL_SEED") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + }); + println!("crash-kill base seed {base_seed} (replay: MINIGRAF_CRASH_KILL_SEED={base_seed})"); + let uuids: HashMap = (0..ENTITIES).map(|e| (entity_uuid(e), e)).collect(); for round in 0..rounds() { + let seed = base_seed ^ (round as u64 + 1).wrapping_mul(0x2545_F491_4F6C_DD1D); + let variant_idx = round % VARIANTS.len(); + let variant = VARIANTS[variant_idx]; let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("kill.graph"); + let log_path = dir.path().join("confirmed.log"); let mut child = Command::new(&exe) .args(["crash_kill_child_entrypoint", "--exact", "--nocapture"]) .env("MINIGRAF_KILL_DB", &db_path) + .env("MINIGRAF_KILL_LOG", &log_path) + .env("MINIGRAF_KILL_SEED", seed.to_string()) + .env("MINIGRAF_KILL_VARIANT", variant_idx.to_string()) .spawn() .expect("spawn crash-kill child"); - let seed = Instant::now().elapsed().as_nanos() as u64 - ^ (round as u64).wrapping_mul(0x2545_F491_4F6C_DD1D); - let delay = Duration::from_millis(5 + jitter_millis(seed, 150)); - std::thread::sleep(delay); - + // Kill after a random number of confirmed transactions, plus jitter so + // the kill lands anywhere inside the next execute or checkpoint. + let mut rng = Rng::new(seed.rotate_left(17)); + let target = 1 + rng.below(60); + let deadline = Instant::now() + Duration::from_secs(30); + while confirmed(&log_path) < target { + if let Some(status) = child.try_wait().expect("poll child") { + panic!("round {round} seed {seed}: child exited early ({status})"); + } + assert!( + Instant::now() < deadline, + "round {round} seed {seed}: child too slow" + ); + std::thread::sleep(Duration::from_millis(1)); + } + std::thread::sleep(Duration::from_micros(rng.below(20_000))); child.kill().expect("SIGKILL crash-kill child"); let status = child.wait().expect("reap crash-kill child"); assert!( @@ -80,32 +419,60 @@ fn sigkill_during_checkpoint_never_permanently_corrupts_file() { "round {round}: child must not exit cleanly before being killed" ); - // A fresh handle in a fresh process must always be able to open the - // file after the kill -- this is the exact guarantee #308 broke. - let db = match Minigraf::open(&db_path) { - Ok(db) => db, - Err(e) => panic!("round {round}: reopen after SIGKILL must not fail: {e}"), + let n = confirmed(&log_path); + let history = model(seed, n + 1); + let ctx = |stage: &str| { + format!("round {round} seed {seed} variant {variant:?} confirmed {n}: {stage}") + }; + let checker = |stage: &str| Checker { + uuids: uuids.clone(), + ctx: ctx(stage), }; - db.execute("(query [:find ?e :where [?e :k ?v]])") - .unwrap_or_else(|e| panic!("round {round}: query after reopen must not fail: {e}")); + + let db = open(&db_path, &ctx("first reopen")); + checker("first reopen").check(&db, &history, n); + drop(db); + + let db = open(&db_path, &ctx("second reopen")); + checker("second reopen").check(&db, &history, n); + db.checkpoint() + .unwrap_or_else(|e| panic!("{}: checkpoint failed ({})", ctx("checkpoint"), e.code())); + drop(db); + + let db = open(&db_path, &ctx("reopen after checkpoint")); + checker("reopen after checkpoint").check(&db, &history, n); } } -/// Child half of the SIGKILL fuzz loop above: opens `MINIGRAF_KILL_DB` and -/// loops transact + checkpoint until killed. A no-op when the marker -/// environment variable is absent -- same convention as -/// `tests/common::crash_child_entrypoint`. +/// Child half of the test above: runs the seeded workload until killed. A no-op +/// when `MINIGRAF_KILL_DB` is absent, as in `tests/common::crash_child_entrypoint`. #[test] fn crash_kill_child_entrypoint() { let Ok(db_path) = std::env::var("MINIGRAF_KILL_DB") else { return; }; - let db = Minigraf::open(&db_path).expect("child opens db"); - let mut i: u64 = 0; - loop { - db.execute(&format!("(transact [[:e{i} :k {i}]])")) - .expect("child transact"); - db.checkpoint().expect("child checkpoint"); - i += 1; + let env = |k: &str| std::env::var(k).unwrap_or_else(|_| panic!("child needs {k}")); + let seed: u64 = env("MINIGRAF_KILL_SEED").parse().expect("seed"); + let variant = VARIANTS[env("MINIGRAF_KILL_VARIANT") + .parse::() + .expect("variant")]; + let mut log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(env("MINIGRAF_KILL_LOG")) + .expect("child opens log"); + + let db = Minigraf::open_with_options(&db_path, variant.options()).expect("child opens db"); + let mut rng = Rng::new(seed); + let mut live = BTreeSet::new(); + for k in 1.. { + let (stmt, next) = next_tx(&mut rng, &live); + db.execute(&stmt).expect("child transaction"); + log.write_all(format!("{k}\n").as_bytes()) + .expect("child logs transaction"); + live = next; + if variant.checkpoint_after(k) { + db.checkpoint().expect("child checkpoint"); + } } }