diff --git a/CHANGELOG.md b/CHANGELOG.md index 27439f1b3..4207fe4cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Installing an update from the web app works again in a microsandbox.** *Back up and install* stopped at the + backup with `Permission denied: '/data/.lens/apt/lock'`, and so did every backup there, because backups had started + keeping everything under `/data`, including the Lens the microsandbox runs (`.lens`). Backups now keep only Lens's + own files: hidden folders at the top of `data_dir`, and SurrealDB's own folder (the backup's export already holds the + database), are left out, and a file Lens can't read is left out with a warning instead of stopping the backup. When + backing up stops an upgrade, Settings now marks *Back up the database* as the step that failed + ([Backups](docs/database.md#backups)). - **The Ask button no longer covers buttons.** The floating Ask button at the bottom right used to sit on whatever was there, such as Import on the Import page. Now, when a button or field is under it and scrolling can't move it clear (a row of buttons pinned to the bottom of the screen, or the last button on a page scrolled to its end), the Ask diff --git a/docs/database.md b/docs/database.md index 8a62d9f6a..4fe9857a8 100644 --- a/docs/database.md +++ b/docs/database.md @@ -149,9 +149,12 @@ first ([Updates and backups](updates.md)). Each is one entry in `/back reports, exports, podcasts, certificates) in `/backups/.files//`. A file that hasn't changed since the previous backup is a hard link to that backup's copy, so each backup takes only the room of what changed. Left out: what Lens fetches or rebuilds by itself (`models`, `bin`, `python`, `cache`, the search index), passing files - (`tmp`, `locks`, `logs`, unfinished uploads), and `secret.key`. A backup is refused when the disk hasn't room for - the files that changed. A backup with files, or of an embedded database, downloads as one `.tar.gz` holding - `` and `.files/`. Backups from before files were kept show *database only* in Settings. + (`tmp`, `locks`, `logs`, unfinished uploads), `secret.key`, the database's own folder (`surreal` in a microsandbox: + the export holds it), and hidden entries at the top of `data_dir` (in a microsandbox, `.lens` is the Lens it runs, + and the rest are caches). A file Lens can't read isn't its own: it is left out, with a warning in the log. A backup + is refused when the disk hasn't room for the files that changed. A backup with files, or of an embedded database, + downloads as one `.tar.gz` holding `` and `.files/`. Backups from before files were kept show *database + only* in Settings. **Restoring** replaces the whole database, and the files under `data_dir`, with the backup's (a *database only* backup leaves the files as they are). What was there is backed up first (`before-restore`) and diff --git a/fastapi_backend/app/domain/datafiles.py b/fastapi_backend/app/domain/datafiles.py index 2ff4e8752..59a05326a 100644 --- a/fastapi_backend/app/domain/datafiles.py +++ b/fastapi_backend/app/domain/datafiles.py @@ -4,19 +4,23 @@ A file that hasn't changed since the newest earlier backup is a hard link to that backup's copy, so backing up again takes no more room than what changed. Left out: what Lens fetches or rebuilds by itself (models, bin, python, cache, -search), what's passing (tmp, locks, logs, unfinished uploads), the backups, the embedded database (the backup's own -part) and secret.key (keep it apart from the backups, docs/encryption.md). +search), what's passing (tmp, locks, logs, unfinished uploads), the backups, the database's own folder (the embedded +database, or SurrealDB's in a microsandbox, surreal: the backup's own part), secret.key (keep it apart from the +backups, docs/encryption.md), and hidden entries at the top (.lens, where a microsandbox keeps the Lens it runs, and +caches). A file Lens can't read isn't its own, so it is left out too, with a warning. """ from __future__ import annotations import contextlib +import logging import os import pathlib import shutil import time -SKIP = {"backups", "secret.key", "tmp", "locks", "logs", "bin", "models", "python", "cache", "search"} +log = logging.getLogger("lens") +SKIP = {"backups", "secret.key", "tmp", "locks", "logs", "bin", "models", "python", "cache", "search", "surreal"} FOLDER = ".files" @@ -28,11 +32,16 @@ def _skipped(name, skip): return name in skip or name == ".partial" or name.endswith((".part", ".partial")) +def _top_skipped(name, skip): + """A top-level entry of data_dir that isn't backed up: SKIP's, and hidden ones (a microsandbox's .lens, caches).""" + return _skipped(name, skip) or name.startswith(".") + + def _entries(data, skip): """The top-level entries of data_dir that are backed up.""" if not data.is_dir(): return [] - return sorted(p for p in data.iterdir() if not _skipped(p.name, skip) and not p.is_symlink()) + return sorted(p for p in data.iterdir() if not _top_skipped(p.name, skip) and not p.is_symlink()) def _walk(data, skip): @@ -108,6 +117,7 @@ def snapshot(data_dir, name, db_path=None): raise RuntimeError( f"backing up the files needs {need >> 20} MB and the disk has {free >> 20} MB free; free some space, or delete older backups" ) + unread = [] try: for src, rel, old in plan: out = part / rel @@ -118,11 +128,18 @@ def snapshot(data_dir, name, db_path=None): continue except OSError: # another file system, or too many links pass - with contextlib.suppress(FileNotFoundError): # gone meanwhile + try: shutil.copy2(src, out) + except FileNotFoundError: # gone meanwhile + pass + except PermissionError: # not Lens's own (another user's file): left out + out.unlink(missing_ok=True) + unread.append(str(rel)) except BaseException: shutil.rmtree(part, ignore_errors=True) raise + if unread: + log.warning("backup %s: left out %d file(s) Lens can't read, such as %s", name, len(unread), unread[0]) part.rename(dst) return dst @@ -143,7 +160,7 @@ def restore(data_dir, name, db_path=None): p.rename(held / p.name) moved.append(p.name) for p in sorted(src.iterdir()): - if _skipped(p.name, skip): + if _top_skipped(p.name, skip): continue if p.is_dir(): shutil.copytree(p, data / p.name, symlinks=False) diff --git a/fastapi_backend/app/domain/updates.py b/fastapi_backend/app/domain/updates.py index 391fbc14f..944a6a7d2 100644 --- a/fastapi_backend/app/domain/updates.py +++ b/fastapi_backend/app/domain/updates.py @@ -374,8 +374,11 @@ def _hand_over(db, run, backup): run.update(state="queued", queued_at=_ts()) except Exception as e: # noqa: BLE001 - the admin reads why log.exception("couldn't start the upgrade") + backing = run["state"] == "backing-up" run.update( - state="failed", error=f"{'backing up' if run['state'] == 'backing-up' else 'handing the upgrade over'} failed: {e}"[:500] + state="failed", + step="backup" if backing else None, # the step the web app marks as failed + error=f"{'backing up' if backing else 'handing the upgrade over'} failed: {e}"[:500], ) _put(db, upgrade=run) diff --git a/fastapi_backend/app/schemas/updates.py b/fastapi_backend/app/schemas/updates.py index 1a91433c4..101a3f9de 100644 --- a/fastapi_backend/app/schemas/updates.py +++ b/fastapi_backend/app/schemas/updates.py @@ -29,8 +29,8 @@ class UpgradeRun(ResponseModel): state: Literal["backing-up", "queued", "running", "rolling-back", "done", "failed", "rolled-back"] = Field( description="rolling-back / rolled-back: the new version didn't start, so the agent went back to the one before" ) - step: Literal["download", "build", "start", "check"] | None = Field( - None, description="while running: what the agent is on (the backup comes before)" + step: Literal["backup", "download", "build", "start", "check"] | None = Field( + None, description="while running: what the agent is on (the backup comes before); backup: backing up failed" ) restore: str | None = Field(None, description="after a rollback: whether the backup was put back, and why") by: str | None = None diff --git a/fastapi_backend/tests/api/test_backups.py b/fastapi_backend/tests/api/test_backups.py index ff739c3dd..cfc2312a2 100644 --- a/fastapi_backend/tests/api/test_backups.py +++ b/fastapi_backend/tests/api/test_backups.py @@ -155,6 +155,44 @@ def test_backups_keep_and_restore_the_files_too(web): assert (second / "objects/ab/abcdef").read_bytes() == b"blob" # a link, so the later backup still has it +def test_a_microsandbox_backs_up_only_lenss_own_files(folder, monkeypatch, caplog): + """In a microsandbox, data_dir (/data) also holds the Lens it runs (.lens: apt, the venv, the web build, some of it + root's), caches and SurrealDB's own folder: none of it is backed up, and a file Lens can't read doesn't stop it.""" + data = folder / "sandbox-data" + for rel, body in { + "uploads/pods/1/talk.mp3": b"talk", + "surreal/lens.db/000001.vlog": b"database", + ".lens/apt/lock": b"", + ".lens/venv/bin/python": b"elf", + ".pnpm-store/v3/x": b"pkg", + ".cache/uv/y": b"cache", + ".DS_Store": b"finder", + "reports/1/locked.pdf": b"not ours", + }.items(): + (data / rel).parent.mkdir(parents=True, exist_ok=True) + (data / rel).write_bytes(body) + real = shutil.copy2 + + def copy2(src, dst, **kw): + if str(src).endswith("locked.pdf"): + raise PermissionError(13, "Permission denied", str(src)) + return real(src, dst, **kw) + + monkeypatch.setattr(datafiles.shutil, "copy2", copy2) + kept = datafiles.snapshot(data, "manual-1") + got = sorted(str(p.relative_to(kept)) for p in kept.rglob("*") if p.is_file()) + assert got == ["uploads/pods/1/talk.mp3"] + assert "left out 1 file(s) Lens can't read, such as reports/1/locked.pdf" in caplog.text + + # restoring leaves them where they are too + (data / "uploads/pods/1/talk.mp3").write_bytes(b"changed") + assert datafiles.restore(data, "manual-1") + assert (data / "uploads/pods/1/talk.mp3").read_bytes() == b"talk" + assert (data / ".lens/venv/bin/python").read_bytes() == b"elf" + assert (data / "surreal/lens.db/000001.vlog").read_bytes() == b"database" + assert (data / ".DS_Store").exists() and (data / ".pnpm-store/v3/x").exists() + + def test_a_backup_from_before_files_were_kept_leaves_them(web): client, db, cfg, h = web data = _files(cfg) diff --git a/fastapi_backend/tests/api/test_updates.py b/fastapi_backend/tests/api/test_updates.py index d6b0d1adf..738f54daf 100644 --- a/fastapi_backend/tests/api/test_updates.py +++ b/fastapi_backend/tests/api/test_updates.py @@ -176,6 +176,7 @@ def full(*_a, **_k): updates.upgrade(db, client.app.state.settings.current(), by="t", wait=True) run = client.get("/api/v1/admin/updates", headers=h).json()["upgrade"] assert run["state"] == "failed" and "backing up failed: No space left" in run["error"] + assert run["step"] == "backup" # the step the web app marks as failed assert not (control / "inbox" / "upgrade").exists() diff --git a/nextjs-frontend/__tests__/settings-updates.test.tsx b/nextjs-frontend/__tests__/settings-updates.test.tsx index c21eab7f5..6434ca6cf 100644 --- a/nextjs-frontend/__tests__/settings-updates.test.tsx +++ b/nextjs-frontend/__tests__/settings-updates.test.tsx @@ -131,4 +131,8 @@ describe("upgrade steps", () => { expect(at("rolling-back", "start")).toEqual(["done", "done", "done", "failed", "pending"]); expect(at("done")).toEqual(["done", "done", "done", "done", "done"]); }); + it("mark the backup failed when backing up stopped the upgrade", () => { + expect(at("failed", "backup")).toEqual(["failed", "pending", "pending", "pending", "pending"]); + expect(at("failed", "download")).toEqual(["done", "failed", "pending", "pending", "pending"]); + }); }); diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index c497f38ab..abfbc6881 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -19131,9 +19131,9 @@ export type UpgradeRun = { /** * Step * - * while running: what the agent is on (the backup comes before) + * while running: what the agent is on (the backup comes before); backup: backing up failed */ - step?: "download" | "build" | "start" | "check" | null; + step?: "backup" | "download" | "build" | "start" | "check" | null; /** * Restore * diff --git a/nextjs-frontend/components/settings/updates-panel.tsx b/nextjs-frontend/components/settings/updates-panel.tsx index 9d259d649..03d628727 100644 --- a/nextjs-frontend/components/settings/updates-panel.tsx +++ b/nextjs-frontend/components/settings/updates-panel.tsx @@ -37,7 +37,7 @@ const STEPS = [ /** Where each step is, from the run's state and the step the machine says it is on. */ export function steps(run: UpgradeRun): StepState[] { const at = - run.state === "backing-up" + run.state === "backing-up" || run.step === "backup" ? 0 : run.state === "queued" ? 1 diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 9f5da21a8..d1af04669 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -60491,6 +60491,7 @@ { "type": "string", "enum": [ + "backup", "download", "build", "start", @@ -60502,7 +60503,7 @@ } ], "title": "Step", - "description": "while running: what the agent is on (the backup comes before)" + "description": "while running: what the agent is on (the backup comes before); backup: backing up failed" }, "restore": { "anyOf": [