From a20b7bac131c80b4fbea1fe94be9f4bffa8a5223 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Mon, 21 Sep 2026 10:00:10 +0530 Subject: [PATCH] fix: derive FactRefs from on-disk layout when rebuilding indexes on open (#370) The rebuild-on-load path (index checksum mismatch, e.g. after a crash mid-save) re-packed all facts contiguously with pack_facts() to derive FactRefs. Every save() starts a fresh page, so a file written by multiple checkpoints has partially filled pages that a contiguous re-pack does not reproduce. From the second checkpoint onward, all four rebuilt indexes pointed at the wrong page/slot, so entity-bound lookups returned nothing (or errored) while the file opened cleanly. Read facts together with their real (page, slot) via the new packed_pages::read_all_with_refs and build the indexes from those. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PuKbXn6NaHBMDVkScdBNQZ --- src/storage/packed_pages.rs | 24 +++++++++++-- src/storage/persistent_facts.rs | 64 ++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/src/storage/packed_pages.rs b/src/storage/packed_pages.rs index 120568e6db..4bb874aa3e 100644 --- a/src/storage/packed_pages.rs +++ b/src/storage/packed_pages.rs @@ -256,9 +256,25 @@ pub fn read_all_from_pages( first_page_id: u64, num_pages: u64, ) -> Result> { + Ok(read_all_with_refs(backend, first_page_id, num_pages)?.0) +} + +/// Read every fact from `num_pages` packed pages, together with the `FactRef` that +/// addresses each fact at its actual on-disk location. +/// +/// Unlike re-packing the facts with [`pack_facts`], the returned refs reflect the real +/// page layout, which is not contiguous when the file was written by several `save()` +/// calls (each starts a fresh page). +pub fn read_all_with_refs( + backend: &dyn StorageBackend, + first_page_id: u64, + num_pages: u64, +) -> Result<(Vec, Vec)> { let mut facts = Vec::new(); + let mut refs = Vec::new(); for i in 0..num_pages { - let page = backend.read_page(first_page_id.saturating_add(i))?; + let page_id = first_page_id.saturating_add(i); + let page = backend.read_page(page_id)?; let page_type = page.first().copied().unwrap_or(0); if page.len() < PAGE_SIZE || page_type != PAGE_TYPE_PACKED { continue; @@ -268,9 +284,13 @@ pub fn read_all_from_pages( let record_count = u16::from_le_bytes([b2, b3]); for slot in 0..record_count { facts.push(read_slot(&page, slot)?); + refs.push(FactRef { + page_id, + slot_index: slot, + }); } } - Ok(facts) + Ok((facts, refs)) } // ── helpers ────────────────────────────────────────────────────────────────── diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index 9e116ecede..2a057513f4 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -319,15 +319,16 @@ impl PersistentFactStorage { if needs_rebuild { // Checksum mismatch: rebuild indexes by re-reading all packed facts - let all_facts = { + // FactRefs must come from the actual on-disk layout: each save() starts a + // fresh page, so re-packing all facts contiguously would yield refs that + // point at the wrong page/slot (#370). + let (all_facts, real_refs) = { let backend = self .backend .lock() .map_err(|_| err_coded!(ErrorCode::Stg016))?; - crate::storage::packed_pages::read_all_from_pages(&*backend, 1, num_fact_pages)? + crate::storage::packed_pages::read_all_with_refs(&*backend, 1, num_fact_pages)? }; - // Re-pack to derive correct FactRefs (same deterministic layout as on disk) - let (_, real_refs) = pack_facts(&all_facts, 1)?; // Build sorted index entries let (eavt_entries, aevt_entries, avet_entries, vaet_entries) = @@ -1609,6 +1610,61 @@ mod tests { } } + /// Regression test for #370: the rebuild-on-load path must derive `FactRef`s from the + /// on-disk fact-page layout, not by re-packing all facts as a single batch. Each + /// `save()` starts a fresh page, so a file written by several checkpoints has partially + /// filled pages that a contiguous re-pack would not reproduce. + #[test] + fn test_rebuild_after_multiple_checkpoints_keeps_entity_lookups() { + use crate::graph::types::Value; + use crate::storage::StorageBackend; + use crate::storage::backend::FileBackend; + use tempfile::NamedTempFile; + use uuid::Uuid; + + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let entities: Vec = (0..5).map(|_| Uuid::new_v4()).collect(); + + // One checkpoint per entity: five fact batches, each on its own partial page. + { + let mut pfs = + PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + for (i, e) in entities.iter().enumerate() { + pfs.storage() + .transact( + vec![(*e, ":name".to_string(), Value::String(format!("n{i}")))], + None, + ) + .unwrap(); + pfs.dirty = true; + pfs.save().unwrap(); + } + } + + // Corrupt index_checksum (and re-seal the header) to force the rebuild path. + { + let mut backend = FileBackend::open(&path).unwrap(); + let mut page = backend.read_page(0).unwrap(); + page[64] ^= 0xFF; + let cs = compute_header_checksum_from_bytes(&page); + page[80..84].copy_from_slice(&cs.to_le_bytes()); + backend.write_page(0, &page).unwrap(); + backend.sync().unwrap(); + } + + let pfs = PersistentFactStorage::new(FileBackend::open(&path).unwrap(), 256).unwrap(); + for e in &entities { + let facts = pfs.storage().get_facts_by_entity(e).unwrap(); + assert_eq!( + facts.len(), + 1, + "entity lookup must survive index rebuild after multiple checkpoints" + ); + assert_eq!(facts[0].entity, *e, "lookup returned a different entity"); + } + } + #[test] fn test_compute_index_checksum_stable() { use crate::graph::types::{Fact, VALID_TIME_FOREVER, Value};