From b912f2dbc4a022a7ba2b7ed3e8af675738d869da Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Fri, 9 Oct 2026 09:26:31 +0530 Subject: [PATCH 1/3] ci(fuzz): fuzz main and v3 nightly in parallel jobs, keep the corpus (#387) Scheduled workflows run main's copy of the file, so this is what makes the nightly fuzz run reach v3. Same files as on v3 (#498): - fuzz.yml: main at 04:00 UTC, v3 at 16:00 UTC (the cron string picks the branch, as in bench.yml); one parallel job per target from the branch's own fuzz/Cargo.toml; corpus restored from and saved to the Actions cache per branch and target; -timeout=60 for hangs. - fuzz-corpus.yml: weekly cargo fuzz cmin per target, committed seed_* files kept, proposed as a pull request from bot/fuzz-corpus-. Workflow-only; no code or test changes. Refs #387 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012wXxZguv7PgKQjMMH4yJ5h --- .github/workflows/fuzz-corpus.yml | 179 +++++++++++++++++++++++++++ .github/workflows/fuzz.yml | 193 +++++++++++++++++------------- 2 files changed, 288 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/fuzz-corpus.yml diff --git a/.github/workflows/fuzz-corpus.yml b/.github/workflows/fuzz-corpus.yml new file mode 100644 index 0000000000..98fe931365 --- /dev/null +++ b/.github/workflows/fuzz-corpus.yml @@ -0,0 +1,179 @@ +name: Fuzz corpus (weekly) + +# Minimizes each target's cached fuzz corpus (see fuzz.yml) with +# `cargo fuzz cmin`, saves the result back to the cache, and proposes it as a +# commit to the branch in a pull request from bot/fuzz-corpus- (#387). +# Committed seed_* files are always kept: they are regression seeds. +# +# Two branches, as in fuzz.yml: main on Monday 08:00 UTC, v3 on Monday 20:00 +# UTC. Keep the cron strings here and in CORPUS_REF/run-name identical, and +# keep this file the same on main and v3. +# +# Opening the pull request needs the repository setting "Allow GitHub Actions +# to create and approve pull requests". Without it, the branch is still +# pushed and the job summary links to it. Pull requests opened with +# GITHUB_TOKEN do not start CI; they change only fuzz/corpus/. +run-name: >- + Fuzz corpus (${{ github.event_name == 'workflow_dispatch' && inputs.ref + || (github.event.schedule == '0 20 * * 1' && 'v3' || 'main') }}) + +on: + schedule: + - cron: "0 8 * * 1" # main + - cron: "0 20 * * 1" # v3 + workflow_dispatch: + inputs: + ref: + description: Branch whose corpus to minimize + required: true + default: main + +permissions: + contents: read + +env: + CORPUS_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || (github.event.schedule == '0 20 * * 1' && 'v3' || 'main') }} + # Larger corpus files stay in the cache only. + MAX_COMMITTED_BYTES: "65536" + +jobs: + targets: + name: List fuzz targets + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.list.outputs.targets }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.CORPUS_REF }} + + - name: Read the [[bin]] names + id: list + shell: bash + run: | + targets="$(sed -n 's/^name = "\(.*\)"$/\1/p' fuzz/Cargo.toml | grep -vx minigraf-fuzz \ + | jq -R . | jq -sc .)" + echo "targets=$targets" >> "$GITHUB_OUTPUT" + + cmin: + name: Minimize ${{ matrix.target }} + needs: targets + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.targets.outputs.targets) }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.CORPUS_REF }} + + - name: Install Rust nightly + uses: dtolnay/rust-toolchain@nightly + + - name: Cache cargo-fuzz binary + uses: actions/cache@v4 + with: + path: ~/.cargo/bin/cargo-fuzz + key: cargo-fuzz-${{ runner.os }}-${{ hashFiles('fuzz/Cargo.toml') }} + + - name: Install cargo-fuzz + run: which cargo-fuzz || cargo install cargo-fuzz + + - name: Restore corpus + uses: actions/cache/restore@v4 + with: + path: fuzz/corpus/${{ matrix.target }} + key: fuzz-corpus-${{ env.CORPUS_REF }}-${{ matrix.target }}-cmin-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: fuzz-corpus-${{ env.CORPUS_REF }}-${{ matrix.target }}- + + - name: Minimize + shell: bash + env: + TARGET: ${{ matrix.target }} + run: | + dir="fuzz/corpus/$TARGET" + echo "Before: $(find "$dir" -type f | wc -l) files" + cargo fuzz cmin -O "$TARGET" "$dir" + # Put back every committed seed cmin dropped. + git ls-files -z "$dir" | grep -z '/seed_' | xargs -0 -r git checkout -- + echo "After: $(find "$dir" -type f | wc -l) files" + + - name: Save corpus + uses: actions/cache/save@v4 + with: + path: fuzz/corpus/${{ matrix.target }} + key: fuzz-corpus-${{ env.CORPUS_REF }}-${{ matrix.target }}-cmin-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Keep committable files + shell: bash + env: + TARGET: ${{ matrix.target }} + run: | + mkdir -p "committed/$TARGET" + find "fuzz/corpus/$TARGET" -maxdepth 1 -type f -size -"$((MAX_COMMITTED_BYTES + 1))"c \ + -exec cp {} "committed/$TARGET/" \; + + - uses: actions/upload-artifact@v4 + with: + name: corpus-${{ matrix.target }} + path: committed/ + retention-days: 7 + + propose: + name: Propose the minimized corpus + needs: cmin + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.CORPUS_REF }} + + - uses: actions/download-artifact@v4 + with: + pattern: corpus-* + path: minimized + merge-multiple: true + + - name: Commit, push and open the pull request + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + branch="bot/fuzz-corpus-$CORPUS_REF" + for d in minimized/*/; do + target="$(basename "$d")" + rm -rf "fuzz/corpus/$target" + cp -r "$d" "fuzz/corpus/$target" + done + git add -A fuzz/corpus + if git diff --cached --quiet; then + echo "Corpus unchanged." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -q -m "fuzz: minimized corpus for $CORPUS_REF (weekly cmin)" \ + -m "Run: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + -m "Refs #387" + git push -f origin "HEAD:refs/heads/$branch" + + body="Weekly \`cargo fuzz cmin\` of the cached corpus of every fuzz target on \`$CORPUS_REF\`, with all committed \`seed_*\` files kept. Files over $MAX_COMMITTED_BYTES bytes stay in the cache only. Generated by \`.github/workflows/fuzz-corpus.yml\` (#387). + + Run: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + existing="$(gh pr list --head "$branch" --base "$CORPUS_REF" --state open --json number --jq '.[0].number')" + if [ -n "$existing" ]; then + echo "Updated pull request #$existing." | tee -a "$GITHUB_STEP_SUMMARY" + elif gh pr create --base "$CORPUS_REF" --head "$branch" \ + --title "fuzz: minimized corpus for $CORPUS_REF" --body "$body"; then + echo "Opened a pull request from $branch." | tee -a "$GITHUB_STEP_SUMMARY" + else + compare="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/$CORPUS_REF...$branch?expand=1" + echo "Could not open the pull request (is \"Allow GitHub Actions to create and approve pull requests\" off?). Open it from $compare" \ + | tee -a "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index eef1af0410..50160be718 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -1,22 +1,75 @@ name: Fuzz (nightly) +# Two branches alternate every 12 hours: main at 04:00 UTC, v3 at 16:00 UTC. +# Scheduled runs always use this file from the default branch, so the branch +# under test is chosen from the cron string that fired (as in bench.yml). +# Keep the cron strings here and in FUZZ_REF/run-name identical, and keep this +# file the same on main and v3. +# +# Each target runs in its own job, in parallel. The corpus a run grows is saved +# in the Actions cache per branch and target, and the next run starts from it +# (#387). fuzz-corpus.yml minimizes it weekly and proposes it as a commit. +run-name: >- + Fuzz (${{ github.event_name == 'workflow_dispatch' && inputs.ref + || (github.event.schedule == '0 16 * * *' && 'v3' || 'main') }}) + on: schedule: - - cron: "0 4 * * *" + - cron: "0 4 * * *" # main + - cron: "0 16 * * *" # v3 workflow_dispatch: + inputs: + ref: + description: Branch to fuzz + required: true + default: main + seconds: + description: Seconds per target + required: true + default: "3300" permissions: contents: read - issues: write + +env: + FUZZ_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || (github.event.schedule == '0 16 * * *' && 'v3' || 'main') }} + FUZZ_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.seconds || '3300' }} jobs: + targets: + name: List fuzz targets + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.list.outputs.targets }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.FUZZ_REF }} + + # Each branch fuzzes the targets its own fuzz/Cargo.toml declares. + - name: Read the [[bin]] names + id: list + shell: bash + run: | + targets="$(sed -n 's/^name = "\(.*\)"$/\1/p' fuzz/Cargo.toml | grep -vx minigraf-fuzz \ + | jq -R . | jq -sc .)" + echo "targets=$targets" >> "$GITHUB_OUTPUT" + echo "Targets on $FUZZ_REF: $targets" + fuzz: - name: Fuzz targets + name: Fuzz ${{ matrix.target }} + needs: targets runs-on: ubuntu-latest - timeout-minutes: 360 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.targets.outputs.targets) }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ env.FUZZ_REF }} - name: Install Rust nightly uses: dtolnay/rust-toolchain@nightly @@ -30,102 +83,77 @@ jobs: - name: Install cargo-fuzz run: which cargo-fuzz || cargo install cargo-fuzz - - name: Fuzz wal_entry (55m) - id: wal_entry - shell: bash - run: | - set -o pipefail - cargo fuzz run wal_entry -- -max_total_time=3300 2>&1 | tee fuzz-wal_entry.log - continue-on-error: true - - - name: Fuzz file_header (55m) - id: file_header - shell: bash - run: | - set -o pipefail - cargo fuzz run file_header -- -max_total_time=3300 2>&1 | tee fuzz-file_header.log - continue-on-error: true - - - name: Fuzz fact_page (55m) - id: fact_page - shell: bash - run: | - set -o pipefail - cargo fuzz run fact_page -- -max_total_time=3300 2>&1 | tee fuzz-fact_page.log - continue-on-error: true + - name: Restore corpus + uses: actions/cache/restore@v4 + with: + path: fuzz/corpus/${{ matrix.target }} + key: fuzz-corpus-${{ env.FUZZ_REF }}-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: fuzz-corpus-${{ env.FUZZ_REF }}-${{ matrix.target }}- - - name: Fuzz btree_page (55m) - id: btree_page - shell: bash - run: | - set -o pipefail - cargo fuzz run btree_page -- -max_total_time=3300 2>&1 | tee fuzz-btree_page.log - continue-on-error: true + - name: Build + run: cargo fuzz build -O ${{ matrix.target }} - - name: Fuzz datalog_parser (55m) - id: datalog_parser + # -timeout: one input running this long is a hang (libFuzzer's default + # of 1200 s would spend a third of the budget on it). + - name: Fuzz ${{ matrix.target }} shell: bash run: | set -o pipefail - cargo fuzz run datalog_parser -- -max_total_time=3300 2>&1 | tee fuzz-datalog_parser.log - continue-on-error: true + cargo fuzz run -O ${{ matrix.target }} fuzz/corpus/${{ matrix.target }} -- \ + -max_total_time="$FUZZ_SECONDS" -timeout=60 2>&1 | tee fuzz-${{ matrix.target }}.log - - name: Fuzz datalog_eval (55m) - id: datalog_eval - shell: bash - run: | - set -o pipefail - cargo fuzz run datalog_eval -- -max_total_time=3300 2>&1 | tee fuzz-datalog_eval.log - continue-on-error: true + - name: Save corpus + if: ${{ always() }} + uses: actions/cache/save@v4 + with: + path: fuzz/corpus/${{ matrix.target }} + key: fuzz-corpus-${{ env.FUZZ_REF }}-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} - - name: Upload fuzz failures - if: ${{ always() && (steps.wal_entry.outcome == 'failure' || steps.file_header.outcome == 'failure' || steps.fact_page.outcome == 'failure' || steps.btree_page.outcome == 'failure' || steps.datalog_parser.outcome == 'failure' || steps.datalog_eval.outcome == 'failure') }} + - name: Upload failure + if: ${{ failure() }} uses: actions/upload-artifact@v4 with: - name: fuzz-failures-${{ github.run_id }} + name: fuzz-failure-${{ matrix.target }} path: | - fuzz/artifacts/ - fuzz-*.log + fuzz/artifacts/${{ matrix.target }}/ + fuzz-${{ matrix.target }}.log if-no-files-found: warn retention-days: 30 - - name: Record fuzz failures - if: ${{ always() && (steps.wal_entry.outcome == 'failure' || steps.file_header.outcome == 'failure' || steps.fact_page.outcome == 'failure' || steps.btree_page.outcome == 'failure' || steps.datalog_parser.outcome == 'failure' || steps.datalog_eval.outcome == 'failure') }} + report: + name: Record fuzz failures + needs: fuzz + if: ${{ failure() }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: fuzz-failure-* + path: failures + + - name: Open or update the issue shell: bash env: GH_TOKEN: ${{ github.token }} - ISSUE_TITLE: "Nightly fuzz failure" - FAILED_TARGETS: >- - wal_entry=${{ steps.wal_entry.outcome }}, - file_header=${{ steps.file_header.outcome }}, - fact_page=${{ steps.fact_page.outcome }}, - btree_page=${{ steps.btree_page.outcome }}, - datalog_parser=${{ steps.datalog_parser.outcome }}, - datalog_eval=${{ steps.datalog_eval.outcome }} + ISSUE_TITLE: "Nightly fuzz failure (${{ env.FUZZ_REF }})" run: | report_file="fuzz-failure.md" - artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" { - echo "The nightly fuzz workflow found one or more failures." + echo "The nightly fuzz workflow found one or more failures on \`$FUZZ_REF\`." echo - echo "- Run and artifacts: $artifact_url" - echo "- Commit: $GITHUB_SHA" - echo "- Target outcomes: $FAILED_TARGETS" - echo "- Reproduce: \`cargo +nightly fuzz run fuzz/artifacts//\`" - echo "- Full logs: see the \`fuzz-failures-$GITHUB_RUN_ID\` artifact on the run above" + echo "- Run and artifacts: $run_url" + echo "- Reproduce: check out \`$FUZZ_REF\` at the run's commit, then" + echo " \`cargo +nightly fuzz run fuzz/artifacts//\`" echo echo "## Fuzz logs (failed targets only, last 100 lines each)" echo - - IFS=',' read -ra target_outcomes <<< "$FAILED_TARGETS" - for entry in "${target_outcomes[@]}"; do - entry="$(echo "$entry" | xargs)" - target="${entry%%=*}" - outcome="${entry#*=}" - [ "$outcome" = "failure" ] || continue - log="fuzz-${target}.log" - [ -e "$log" ] || continue - echo "### $log" + shopt -s nullglob + for log in failures/fuzz-failure-*/fuzz-*.log; do + echo "### $(basename "$log")" echo echo '```text' tail -n 100 "$log" @@ -134,7 +162,7 @@ jobs: done } > "$report_file" - # Defensive hard cap: GitHub issue/comment bodies max out at 65536 chars. + # GitHub issue and comment bodies max out at 65536 characters. max_bytes=60000 if [ "$(wc -c < "$report_file")" -gt "$max_bytes" ]; then head -c "$max_bytes" "$report_file" > "${report_file}.tmp" @@ -142,7 +170,7 @@ jobs: echo echo "..." echo - echo "_Report truncated to fit GitHub's body size limit. See the run and artifact linked above for full logs._" + echo "_Report truncated to fit GitHub's body size limit. See the run linked above for full logs._" } >> "${report_file}.tmp" mv "${report_file}.tmp" "$report_file" fi @@ -153,7 +181,7 @@ jobs: --search "$ISSUE_TITLE in:title" \ --json number,title \ --limit 100 \ - --jq '.[] | select(.title == "Nightly fuzz failure") | .number' \ + --jq ".[] | select(.title == \"$ISSUE_TITLE\") | .number" \ | head -n 1)" if [ -n "$existing_issue" ]; then @@ -161,7 +189,4 @@ jobs: else gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$report_file" fi - - - name: Fail workflow when fuzzing finds a failure - if: ${{ always() && (steps.wal_entry.outcome == 'failure' || steps.file_header.outcome == 'failure' || steps.fact_page.outcome == 'failure' || steps.btree_page.outcome == 'failure' || steps.datalog_parser.outcome == 'failure' || steps.datalog_eval.outcome == 'failure') }} - run: exit 1 + exit 1 From e8450be85fef127e60b2eaae91e720b73762396d Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Fri, 9 Oct 2026 18:34:42 +0530 Subject: [PATCH 2/3] ci(fuzz): drop the hashed copy of each seed after cmin (#387) cargo fuzz cmin names the inputs it keeps by the SHA-1 of their content, so after putting the committed seed_* files back every kept seed was in the corpus twice. The first run proposed 33 such copies and nothing else. Refs #387 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012wXxZguv7PgKQjMMH4yJ5h --- .github/workflows/fuzz-corpus.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fuzz-corpus.yml b/.github/workflows/fuzz-corpus.yml index 98fe931365..9cbb6285ab 100644 --- a/.github/workflows/fuzz-corpus.yml +++ b/.github/workflows/fuzz-corpus.yml @@ -97,8 +97,14 @@ jobs: dir="fuzz/corpus/$TARGET" echo "Before: $(find "$dir" -type f | wc -l) files" cargo fuzz cmin -O "$TARGET" "$dir" - # Put back every committed seed cmin dropped. + # Put back every committed seed cmin dropped. cmin names the files + # it keeps by the SHA-1 of their content, so a seed it kept is also + # there under that name: drop that copy. git ls-files -z "$dir" | grep -z '/seed_' | xargs -0 -r git checkout -- + shopt -s nullglob + for seed in "$dir"/seed_*; do + rm -f "$dir/$(sha1sum < "$seed" | cut -c1-40)" + done echo "After: $(find "$dir" -type f | wc -l) files" - name: Save corpus From e17c5ecbce219cba1c5223bbec43c0bf7d75fe48 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 10 Oct 2026 11:13:47 +0530 Subject: [PATCH 3/3] fix(storage): refuse a sub-page file that is not a torn first write (#506) Backport of #508 to v2.x. A file of 1..4095 bytes made FileBackend::open report a new database and write a fresh header over it. It now opens as new only when its bytes are zeros or a prefix of the initial header page; anything else is STG-045 (same number and meaning as on v3), file unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012wXxZguv7PgKQjMMH4yJ5h --- CHANGELOG.md | 6 ++++ CLAUDE.md | 2 +- README.md | 2 +- docs/ERROR_REFERENCE.md | 14 ++++++++++ docs/TEST_COVERAGE.md | 6 ++-- src/error.rs | 9 ++++++ src/storage/backend/file.rs | 22 +++++++++++++++ tests/edge_cases_test.rs | 56 +++++++++++++++++++++++++++++++++++++ 8 files changed, 113 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecf5edd41..6bd112c886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- **A file shorter than one page is refused at open with `STG-045` unless it is a torn first write (#506).** Open treated any file under 4 KiB as a new database and wrote a new header over it, so a database cut below one page by an interrupted copy, or an unrelated file at the path, was silently replaced. Such a file now opens as a new database only when its bytes are zeros or a prefix of the header a new database writes (what a crash during creation can leave); anything else is `STG-045` and the file is not modified. A zero-length file is still a new database. New code `STG-045`, numbered as in v3.0.0, which also fixes this. + ## v2.0.4 — 2026-10-08 Patch release on the v2.x line under the [support policy](PHILOSOPHY.md#support-policy): one `BrowserDb` durability hardening fix, plus new compatibility and crash tests. File format is unchanged (v7). No API changes. Native and WASI builds behave exactly as in v2.0.3. diff --git a/CLAUDE.md b/CLAUDE.md index afc183bd3b..b602f1fd17 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -174,7 +174,7 @@ Auto-migrates v1/v2/v3/v4/v5/v6 → v7 on open/checkpoint. ## Test Coverage -**1223 tests** (1215 passing, 8 ignored; unit + integration + doc). +**1224 tests** (1216 passing, 8 ignored; unit + integration + doc). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. **Testing conventions** — see the Testing Conventions section below before writing any tests. diff --git a/README.md b/README.md index 1fc5aa3747..4287b6a5dd 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ let r2 = pq.execute(&[("tx", BindValue::TxCount(2)), ("entity", BindValue::Entit ```bash cargo run # interactive Datalog REPL -cargo test # run 1223 tests +cargo test # run 1224 tests cargo run < demos/demo_recursive.txt # recursive rules demo ``` diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index 5dbfdceecb..ef85615ee5 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -136,6 +136,7 @@ with no `CodedError` anywhere in its chain. | STG-025 | Database already open in this process | Storage | | STG-026 | Database locked by another process | Storage | | STG-027 | Filesystem does not support file locking | Storage | +| STG-045 | File shorter than one page and not a new database | Storage | | WAL-001 | Invalid WAL magic number | WAL | | WAL-002 | Unsupported WAL version | WAL | | WAL-003 | Fact serialised size exceeds maximum | WAL | @@ -1987,6 +1988,19 @@ See the [file format section in README](../README.md#file-format) for version hi **NFSv3 `nolock` is not covered by this error — it is a silent, undetectable gap (#334).** An export explicitly mounted with `-o nolock` behaves differently from plain "no `lockd`": the Linux NFS client serves `flock`/OFD locks out of its own local, in-kernel lock table instead of going over NLM to the server. `try_lock` sees an ordinary local success, `classify` takes the same branch it would on a filesystem where locking genuinely works, and this error is never raised — `allow_unlocked` is never consulted because the code never learns the mount can't really lock. Two separate client hosts writing the same `nolock` export each get a false `Ok(())` from their own kernel with zero cross-host coordination, and can silently corrupt the file. There is no way to detect this from inside `try_lock`'s result, so `nolock` NFSv3 exports are an unsupported deployment for multi-writer use, the same way running mixed Minigraf versions against one file is unsupported (see the kernel-locking CHANGELOG entry) — avoid `nolock` exports rather than relying on this check to catch them. +### STG-045 File shorter than one page and not a new database + +**Error text**: `File is {} bytes, shorter than one page, and is not a new Minigraf database` + +**Cause**: The file is longer than zero bytes but shorter than one 4 KiB page, and its bytes are not a torn first write of a new database. Creating a database writes its header page in one go, so a crash during that write can leave only a prefix of that page (or zeros); such a file is opened as a new, empty database. Any other short file is something else: a different file at the path, a database cut short by an interrupted copy, or a damaged file. Earlier versions opened it as a new database and overwrote it. The file is not modified. (Codes STG-028 to STG-044 belong to v3.0.0; this code has the same number and meaning there.) + +**Resolution**: +- Check that the path points at the intended `.graph` file. +- Copy the database again from its source, or restore it from backup. +- If the file is not needed, delete it and open the path again to create a new database. + +**Scenario**: A copy of a `.graph` file stops after its first kilobyte, and the copy is opened. + --- ## WAL — Write-Ahead Log Errors diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index a1009fd755..896c61601a 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -2,9 +2,11 @@ ## v2.0.4 Coverage Summary -**Verified**: 2026-10-08 with `cargo test` +**Verified**: 2026-10-10 with `cargo test` -**Result**: 1,215 passing tests and 8 ignored tests (1,223 total), including the golden-file corpus (#391) +**Result**: 1,216 passing tests and 8 ignored tests (1,224 total), including the golden-file corpus (#391) + +Since v2.0.4, `tests/edge_cases_test.rs` checks that a file of 1 to 4,095 bytes is `STG-045`, unchanged and with no WAL, when it is a database cut below one page or foreign bytes, and opens as a new database that takes a write and reads it back after reopen when it is a prefix of the initial header or all zeros (#506). `tests/property_test.rs` writes random histories (batched transacts, retracts, write transactions, checkpoints) in memory and to a file, and compares random queries (one to three joined patterns, `not` or `not-join`, a comparison predicate, `:as-of`, `:valid-at` or `:any-valid-time`) with a reference evaluator, on the open file and after a reopen; a query error fails the test (#386). The histories leave out the v2.x known issues #371, #435 and #477. diff --git a/src/error.rs b/src/error.rs index 0763506bec..228e9234f3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -163,6 +163,7 @@ pub(crate) enum ErrorCode { Stg025, Stg026, Stg027, + Stg045, Wal001, Wal002, Wal003, @@ -937,6 +938,12 @@ pub(crate) const REGISTRY: &[(ErrorCode, &str, &str, ErrorCategory)] = &[ "Failed to lock database at {}: {}. This filesystem does not support file locking (common on NFSv3 without lockd, and on some FUSE mounts). Set `allow_unlocked` in `OpenOptions` to open anyway — that accepts the risk that concurrent writers corrupt the file.", ErrorCategory::Storage, ), + ( + ErrorCode::Stg045, + "STG-045", + "File is {} bytes, shorter than one page, and is not a new Minigraf database", + ErrorCategory::Storage, + ), ( ErrorCode::Wal001, "WAL-001", @@ -1751,6 +1758,7 @@ mod tests { | ErrorCode::Stg025 | ErrorCode::Stg026 | ErrorCode::Stg027 + | ErrorCode::Stg045 | ErrorCode::Wal001 | ErrorCode::Wal002 | ErrorCode::Wal003 @@ -1942,6 +1950,7 @@ mod tests { ErrorCode::Stg025, ErrorCode::Stg026, ErrorCode::Stg027, + ErrorCode::Stg045, ErrorCode::Wal001, ErrorCode::Wal002, ErrorCode::Wal003, diff --git a/src/storage/backend/file.rs b/src/storage/backend/file.rs index f521ee292f..8d772f1023 100644 --- a/src/storage/backend/file.rs +++ b/src/storage/backend/file.rs @@ -286,6 +286,17 @@ impl FileBackend { } } } else { + // A file shorter than one page is new only if it holds a torn + // write of the initial header below; anything else is not ours + // to overwrite (#506). + if file_len > 0 { + let mut tail = Vec::new(); + file.seek(SeekFrom::Start(0))?; + (&mut file).take(PAGE_SIZE as u64).read_to_end(&mut tail)?; + if !is_initial_header_prefix(&tail) { + bail_coded!(ErrorCode::Stg045, tail.len()); + } + } // New file or empty file: write initial header, then make the // file's directory entry durable. Without the directory sync a // power loss can lose the whole file despite the header fsync @@ -343,6 +354,17 @@ impl FileBackend { } } +/// True if `bytes` are a prefix of the initial header `open` writes to a new +/// file, then zeros: what a torn write of that header can leave. +fn is_initial_header_prefix(bytes: &[u8]) -> bool { + let expected = FileHeader::new().to_bytes(); + let written = bytes + .iter() + .rposition(|&b| b != 0) + .map_or(0, |i| i.saturating_add(1)); + bytes.get(..written) == expected.get(..written) +} + impl StorageBackend for FileBackend { fn write_page(&mut self, page_id: u64, data: &[u8]) -> Result<()> { if data.len() != PAGE_SIZE { diff --git a/tests/edge_cases_test.rs b/tests/edge_cases_test.rs index c97bfa264b..e992b3003d 100644 --- a/tests/edge_cases_test.rs +++ b/tests/edge_cases_test.rs @@ -5,6 +5,7 @@ //! - Oversized fact accepted in-memory (no page size constraint) //! - Stale WAL after checkpoint is replayed idempotently (no duplicate facts) //! - More than 65,535 uncheckpointed facts query correctly (#445) +//! - A file shorter than one page is refused unless a torn first write (#506) #![cfg(not(target_arch = "wasm32"))] use minigraf::{Minigraf, OpenOptions, QueryResult}; @@ -203,3 +204,58 @@ fn test_more_than_u16_pending_facts_file_backed() { db.checkpoint().unwrap(); assert_all_facts_visible(&db, "after checkpoint"); } + +// ── File shorter than one page (#506) ───────────────────────────────────────── + +fn open_code(path: &std::path::Path) -> String { + match Minigraf::open(path) { + Ok(_) => "opened".to_string(), + Err(e) => e.code().to_string(), + } +} + +/// A file shorter than one page opens as a new database only when it is a +/// torn first write: zeros, or a prefix of the initial header. Any other short +/// file (a database cut to under a page, or foreign bytes) is refused with +/// STG-045 and left as it was; it used to be overwritten (#506). +#[test] +fn test_sub_page_file_refused_unless_torn_first_write() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("short.graph"); + { + let db = Minigraf::open(&path).unwrap(); + db.execute("(transact [[:a :b 1] [:a :c 2]])").unwrap(); + db.checkpoint().unwrap(); + } + let saved = std::fs::read(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + let foreign = vec![0x5Au8; 100]; + for bytes in [&saved[..2222], &saved[..20], &foreign[..]] { + std::fs::write(&path, bytes).unwrap(); + assert_eq!(open_code(&path), "STG-045"); + assert_eq!(std::fs::read(&path).unwrap(), bytes, "file unchanged"); + assert!( + !dir.path().join("short.graph.wal").exists(), + "no WAL created" + ); + } + + // A torn first write: the initial header cut short, or zeros. + let fresh = dir.path().join("fresh.graph"); + drop(Minigraf::open(&fresh).unwrap()); + let initial = std::fs::read(&fresh).unwrap(); + for bytes in [&initial[..4], &initial[..10], &[0u8; 300][..]] { + std::fs::write(&path, bytes).unwrap(); + let db = Minigraf::open(&path).unwrap(); + db.execute("(transact [[:a :b 1]])").unwrap(); + db.checkpoint().unwrap(); + drop(db); + let db = Minigraf::open(&path).unwrap(); + match db.execute("(query [:find ?v :where [:a :b ?v]])").unwrap() { + QueryResult::QueryResults { results, .. } => assert_eq!(results.len(), 1), + _ => panic!("expected query results"), + } + drop(db); + std::fs::remove_file(&path).unwrap(); + } +}