From e4b742848b80bd5f5505bf8974327b1127b61047 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 11:24:03 +0000 Subject: [PATCH] fix(backups): back up only Lens's own files, so upgrading in a microsandbox works Back up and install in Settings stopped at the backup with "Permission denied: '/data/.lens/apt/lock'" in a microsandbox, and so did every backup there: since backups keep the files under data_dir, they walked into /data/.lens, the Lens the microsandbox runs (apt, the venv, the web build, some of it root's). Backups now leave out hidden entries at the top of data_dir (.lens, caches) and SurrealDB's own folder (surreal; the export holds the database), and a file Lens can't read is left out with a warning instead of stopping the backup. A backup that stops an upgrade marks "Back up the database" as the failed step, not "Download the new version". Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GMryXNWd5MJ3vsjM7RkV93 --- CHANGELOG.md | 8 ++++ docs/database.md | 9 +++-- fastapi_backend/app/domain/datafiles.py | 29 +++++++++++--- fastapi_backend/app/domain/updates.py | 5 ++- fastapi_backend/app/schemas/updates.py | 4 +- fastapi_backend/tests/api/test_backups.py | 38 +++++++++++++++++++ fastapi_backend/tests/api/test_updates.py | 1 + .../__tests__/settings-updates.test.tsx | 4 ++ .../app/openapi-client/types.gen.ts | 4 +- .../components/settings/updates-panel.tsx | 2 +- nextjs-frontend/openapi.json | 3 +- 11 files changed, 91 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c80f9ecb3..33e4da06f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Installing an update from the web app works again in a microsandbox.** *Back up and install* stopped at the + backup with `Permission denied: '/data/.lens/apt/lock'`, and so did every backup there, because backups had started + keeping everything under `/data`, including the Lens the microsandbox runs (`.lens`). Backups now keep only Lens's + own files: hidden folders at the top of `data_dir`, and SurrealDB's own folder (the backup's export already holds the + database), are left out, and a file Lens can't read is left out with a warning instead of stopping the backup. When + backing up stops an upgrade, Settings now marks *Back up the database* as the step that failed + ([Backups](docs/database.md#backups)). + ## 0.10.0 October 15, 2026 {id="0.10.0"} - **The assistant always answers.** When a question used up its tool steps (`ai.max_steps`), the assistant used to stop diff --git a/docs/database.md b/docs/database.md index 8a62d9f6a..4fe9857a8 100644 --- a/docs/database.md +++ b/docs/database.md @@ -149,9 +149,12 @@ first ([Updates and backups](updates.md)). Each is one entry in `/back reports, exports, podcasts, certificates) in `/backups/.files//`. A file that hasn't changed since the previous backup is a hard link to that backup's copy, so each backup takes only the room of what changed. Left out: what Lens fetches or rebuilds by itself (`models`, `bin`, `python`, `cache`, the search index), passing files - (`tmp`, `locks`, `logs`, unfinished uploads), and `secret.key`. A backup is refused when the disk hasn't room for - the files that changed. A backup with files, or of an embedded database, downloads as one `.tar.gz` holding - `` and `.files/`. Backups from before files were kept show *database only* in Settings. + (`tmp`, `locks`, `logs`, unfinished uploads), `secret.key`, the database's own folder (`surreal` in a microsandbox: + the export holds it), and hidden entries at the top of `data_dir` (in a microsandbox, `.lens` is the Lens it runs, + and the rest are caches). A file Lens can't read isn't its own: it is left out, with a warning in the log. A backup + is refused when the disk hasn't room for the files that changed. A backup with files, or of an embedded database, + downloads as one `.tar.gz` holding `` and `.files/`. Backups from before files were kept show *database + only* in Settings. **Restoring** replaces the whole database, and the files under `data_dir`, with the backup's (a *database only* backup leaves the files as they are). What was there is backed up first (`before-restore`) and diff --git a/fastapi_backend/app/domain/datafiles.py b/fastapi_backend/app/domain/datafiles.py index 2ff4e8752..59a05326a 100644 --- a/fastapi_backend/app/domain/datafiles.py +++ b/fastapi_backend/app/domain/datafiles.py @@ -4,19 +4,23 @@ A file that hasn't changed since the newest earlier backup is a hard link to that backup's copy, so backing up again takes no more room than what changed. Left out: what Lens fetches or rebuilds by itself (models, bin, python, cache, -search), what's passing (tmp, locks, logs, unfinished uploads), the backups, the embedded database (the backup's own -part) and secret.key (keep it apart from the backups, docs/encryption.md). +search), what's passing (tmp, locks, logs, unfinished uploads), the backups, the database's own folder (the embedded +database, or SurrealDB's in a microsandbox, surreal: the backup's own part), secret.key (keep it apart from the +backups, docs/encryption.md), and hidden entries at the top (.lens, where a microsandbox keeps the Lens it runs, and +caches). A file Lens can't read isn't its own, so it is left out too, with a warning. """ from __future__ import annotations import contextlib +import logging import os import pathlib import shutil import time -SKIP = {"backups", "secret.key", "tmp", "locks", "logs", "bin", "models", "python", "cache", "search"} +log = logging.getLogger("lens") +SKIP = {"backups", "secret.key", "tmp", "locks", "logs", "bin", "models", "python", "cache", "search", "surreal"} FOLDER = ".files" @@ -28,11 +32,16 @@ def _skipped(name, skip): return name in skip or name == ".partial" or name.endswith((".part", ".partial")) +def _top_skipped(name, skip): + """A top-level entry of data_dir that isn't backed up: SKIP's, and hidden ones (a microsandbox's .lens, caches).""" + return _skipped(name, skip) or name.startswith(".") + + def _entries(data, skip): """The top-level entries of data_dir that are backed up.""" if not data.is_dir(): return [] - return sorted(p for p in data.iterdir() if not _skipped(p.name, skip) and not p.is_symlink()) + return sorted(p for p in data.iterdir() if not _top_skipped(p.name, skip) and not p.is_symlink()) def _walk(data, skip): @@ -108,6 +117,7 @@ def snapshot(data_dir, name, db_path=None): raise RuntimeError( f"backing up the files needs {need >> 20} MB and the disk has {free >> 20} MB free; free some space, or delete older backups" ) + unread = [] try: for src, rel, old in plan: out = part / rel @@ -118,11 +128,18 @@ def snapshot(data_dir, name, db_path=None): continue except OSError: # another file system, or too many links pass - with contextlib.suppress(FileNotFoundError): # gone meanwhile + try: shutil.copy2(src, out) + except FileNotFoundError: # gone meanwhile + pass + except PermissionError: # not Lens's own (another user's file): left out + out.unlink(missing_ok=True) + unread.append(str(rel)) except BaseException: shutil.rmtree(part, ignore_errors=True) raise + if unread: + log.warning("backup %s: left out %d file(s) Lens can't read, such as %s", name, len(unread), unread[0]) part.rename(dst) return dst @@ -143,7 +160,7 @@ def restore(data_dir, name, db_path=None): p.rename(held / p.name) moved.append(p.name) for p in sorted(src.iterdir()): - if _skipped(p.name, skip): + if _top_skipped(p.name, skip): continue if p.is_dir(): shutil.copytree(p, data / p.name, symlinks=False) diff --git a/fastapi_backend/app/domain/updates.py b/fastapi_backend/app/domain/updates.py index 391fbc14f..944a6a7d2 100644 --- a/fastapi_backend/app/domain/updates.py +++ b/fastapi_backend/app/domain/updates.py @@ -374,8 +374,11 @@ def _hand_over(db, run, backup): run.update(state="queued", queued_at=_ts()) except Exception as e: # noqa: BLE001 - the admin reads why log.exception("couldn't start the upgrade") + backing = run["state"] == "backing-up" run.update( - state="failed", error=f"{'backing up' if run['state'] == 'backing-up' else 'handing the upgrade over'} failed: {e}"[:500] + state="failed", + step="backup" if backing else None, # the step the web app marks as failed + error=f"{'backing up' if backing else 'handing the upgrade over'} failed: {e}"[:500], ) _put(db, upgrade=run) diff --git a/fastapi_backend/app/schemas/updates.py b/fastapi_backend/app/schemas/updates.py index 1a91433c4..101a3f9de 100644 --- a/fastapi_backend/app/schemas/updates.py +++ b/fastapi_backend/app/schemas/updates.py @@ -29,8 +29,8 @@ class UpgradeRun(ResponseModel): state: Literal["backing-up", "queued", "running", "rolling-back", "done", "failed", "rolled-back"] = Field( description="rolling-back / rolled-back: the new version didn't start, so the agent went back to the one before" ) - step: Literal["download", "build", "start", "check"] | None = Field( - None, description="while running: what the agent is on (the backup comes before)" + step: Literal["backup", "download", "build", "start", "check"] | None = Field( + None, description="while running: what the agent is on (the backup comes before); backup: backing up failed" ) restore: str | None = Field(None, description="after a rollback: whether the backup was put back, and why") by: str | None = None diff --git a/fastapi_backend/tests/api/test_backups.py b/fastapi_backend/tests/api/test_backups.py index ff739c3dd..cfc2312a2 100644 --- a/fastapi_backend/tests/api/test_backups.py +++ b/fastapi_backend/tests/api/test_backups.py @@ -155,6 +155,44 @@ def test_backups_keep_and_restore_the_files_too(web): assert (second / "objects/ab/abcdef").read_bytes() == b"blob" # a link, so the later backup still has it +def test_a_microsandbox_backs_up_only_lenss_own_files(folder, monkeypatch, caplog): + """In a microsandbox, data_dir (/data) also holds the Lens it runs (.lens: apt, the venv, the web build, some of it + root's), caches and SurrealDB's own folder: none of it is backed up, and a file Lens can't read doesn't stop it.""" + data = folder / "sandbox-data" + for rel, body in { + "uploads/pods/1/talk.mp3": b"talk", + "surreal/lens.db/000001.vlog": b"database", + ".lens/apt/lock": b"", + ".lens/venv/bin/python": b"elf", + ".pnpm-store/v3/x": b"pkg", + ".cache/uv/y": b"cache", + ".DS_Store": b"finder", + "reports/1/locked.pdf": b"not ours", + }.items(): + (data / rel).parent.mkdir(parents=True, exist_ok=True) + (data / rel).write_bytes(body) + real = shutil.copy2 + + def copy2(src, dst, **kw): + if str(src).endswith("locked.pdf"): + raise PermissionError(13, "Permission denied", str(src)) + return real(src, dst, **kw) + + monkeypatch.setattr(datafiles.shutil, "copy2", copy2) + kept = datafiles.snapshot(data, "manual-1") + got = sorted(str(p.relative_to(kept)) for p in kept.rglob("*") if p.is_file()) + assert got == ["uploads/pods/1/talk.mp3"] + assert "left out 1 file(s) Lens can't read, such as reports/1/locked.pdf" in caplog.text + + # restoring leaves them where they are too + (data / "uploads/pods/1/talk.mp3").write_bytes(b"changed") + assert datafiles.restore(data, "manual-1") + assert (data / "uploads/pods/1/talk.mp3").read_bytes() == b"talk" + assert (data / ".lens/venv/bin/python").read_bytes() == b"elf" + assert (data / "surreal/lens.db/000001.vlog").read_bytes() == b"database" + assert (data / ".DS_Store").exists() and (data / ".pnpm-store/v3/x").exists() + + def test_a_backup_from_before_files_were_kept_leaves_them(web): client, db, cfg, h = web data = _files(cfg) diff --git a/fastapi_backend/tests/api/test_updates.py b/fastapi_backend/tests/api/test_updates.py index d6b0d1adf..738f54daf 100644 --- a/fastapi_backend/tests/api/test_updates.py +++ b/fastapi_backend/tests/api/test_updates.py @@ -176,6 +176,7 @@ def full(*_a, **_k): updates.upgrade(db, client.app.state.settings.current(), by="t", wait=True) run = client.get("/api/v1/admin/updates", headers=h).json()["upgrade"] assert run["state"] == "failed" and "backing up failed: No space left" in run["error"] + assert run["step"] == "backup" # the step the web app marks as failed assert not (control / "inbox" / "upgrade").exists() diff --git a/nextjs-frontend/__tests__/settings-updates.test.tsx b/nextjs-frontend/__tests__/settings-updates.test.tsx index c21eab7f5..6434ca6cf 100644 --- a/nextjs-frontend/__tests__/settings-updates.test.tsx +++ b/nextjs-frontend/__tests__/settings-updates.test.tsx @@ -131,4 +131,8 @@ describe("upgrade steps", () => { expect(at("rolling-back", "start")).toEqual(["done", "done", "done", "failed", "pending"]); expect(at("done")).toEqual(["done", "done", "done", "done", "done"]); }); + it("mark the backup failed when backing up stopped the upgrade", () => { + expect(at("failed", "backup")).toEqual(["failed", "pending", "pending", "pending", "pending"]); + expect(at("failed", "download")).toEqual(["done", "failed", "pending", "pending", "pending"]); + }); }); diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index b0d2a2f70..9470533cc 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -19061,9 +19061,9 @@ export type UpgradeRun = { /** * Step * - * while running: what the agent is on (the backup comes before) + * while running: what the agent is on (the backup comes before); backup: backing up failed */ - step?: "download" | "build" | "start" | "check" | null; + step?: "backup" | "download" | "build" | "start" | "check" | null; /** * Restore * diff --git a/nextjs-frontend/components/settings/updates-panel.tsx b/nextjs-frontend/components/settings/updates-panel.tsx index 9d259d649..03d628727 100644 --- a/nextjs-frontend/components/settings/updates-panel.tsx +++ b/nextjs-frontend/components/settings/updates-panel.tsx @@ -37,7 +37,7 @@ const STEPS = [ /** Where each step is, from the run's state and the step the machine says it is on. */ export function steps(run: UpgradeRun): StepState[] { const at = - run.state === "backing-up" + run.state === "backing-up" || run.step === "backup" ? 0 : run.state === "queued" ? 1 diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 49f9e7ffe..bf122af15 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -60245,6 +60245,7 @@ { "type": "string", "enum": [ + "backup", "download", "build", "start", @@ -60256,7 +60257,7 @@ } ], "title": "Step", - "description": "while running: what the agent is on (the backup comes before)" + "description": "while running: what the agent is on (the backup comes before); backup: backing up failed" }, "restore": { "anyOf": [