Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version

## Unreleased

- **Installing an update from the web app works again in a microsandbox.** *Back up and install* stopped at the
backup with `Permission denied: '/data/.lens/apt/lock'`, and so did every backup there, because backups had started
keeping everything under `/data`, including the Lens the microsandbox runs (`.lens`). Backups now keep only Lens's
own files: hidden folders at the top of `data_dir`, and SurrealDB's own folder (the backup's export already holds the
database), are left out, and a file Lens can't read is left out with a warning instead of stopping the backup. When
backing up stops an upgrade, Settings now marks *Back up the database* as the step that failed
([Backups](docs/database.md#backups)).
- **The Ask button no longer covers buttons.** The floating Ask button at the bottom right used to sit on whatever was
there, such as Import on the Import page. Now, when a button or field is under it and scrolling can't move it clear
(a row of buttons pinned to the bottom of the screen, or the last button on a page scrolled to its end), the Ask
Expand Down
9 changes: 6 additions & 3 deletions docs/database.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,12 @@ first ([Updates and backups](updates.md)). Each is one entry in `<data_dir>/back
reports, exports, podcasts, certificates) in `<data_dir>/backups/.files/<name>/`. A file that hasn't changed since
the previous backup is a hard link to that backup's copy, so each backup takes only the room of what changed. Left
out: what Lens fetches or rebuilds by itself (`models`, `bin`, `python`, `cache`, the search index), passing files
(`tmp`, `locks`, `logs`, unfinished uploads), and `secret.key`. A backup is refused when the disk hasn't room for
the files that changed. A backup with files, or of an embedded database, downloads as one `.tar.gz` holding
`<name>` and `<name>.files/`. Backups from before files were kept show *database only* in Settings.
(`tmp`, `locks`, `logs`, unfinished uploads), `secret.key`, the database's own folder (`surreal` in a microsandbox:
the export holds it), and hidden entries at the top of `data_dir` (in a microsandbox, `.lens` is the Lens it runs,
and the rest are caches). A file Lens can't read isn't its own: it is left out, with a warning in the log. A backup
is refused when the disk hasn't room for the files that changed. A backup with files, or of an embedded database,
downloads as one `.tar.gz` holding `<name>` and `<name>.files/`. Backups from before files were kept show *database
only* in Settings.

**Restoring** replaces the whole database, and the files under `data_dir`, with the backup's (a *database only*
backup leaves the files as they are). What was there is backed up first (`before-restore`) and
Expand Down
29 changes: 23 additions & 6 deletions fastapi_backend/app/domain/datafiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,23 @@

A file that hasn't changed since the newest earlier backup is a hard link to that backup's copy, so backing up again
takes no more room than what changed. Left out: what Lens fetches or rebuilds by itself (models, bin, python, cache,
search), what's passing (tmp, locks, logs, unfinished uploads), the backups, the embedded database (the backup's own
part) and secret.key (keep it apart from the backups, docs/encryption.md).
search), what's passing (tmp, locks, logs, unfinished uploads), the backups, the database's own folder (the embedded
database, or SurrealDB's in a microsandbox, surreal: the backup's own part), secret.key (keep it apart from the
backups, docs/encryption.md), and hidden entries at the top (.lens, where a microsandbox keeps the Lens it runs, and
caches). A file Lens can't read isn't its own, so it is left out too, with a warning.
"""

from __future__ import annotations

import contextlib
import logging
import os
import pathlib
import shutil
import time

SKIP = {"backups", "secret.key", "tmp", "locks", "logs", "bin", "models", "python", "cache", "search"}
log = logging.getLogger("lens")
SKIP = {"backups", "secret.key", "tmp", "locks", "logs", "bin", "models", "python", "cache", "search", "surreal"}
FOLDER = ".files"


Expand All @@ -28,11 +32,16 @@ def _skipped(name, skip):
return name in skip or name == ".partial" or name.endswith((".part", ".partial"))


def _top_skipped(name, skip):
"""A top-level entry of data_dir that isn't backed up: SKIP's, and hidden ones (a microsandbox's .lens, caches)."""
return _skipped(name, skip) or name.startswith(".")


def _entries(data, skip):
"""The top-level entries of data_dir that are backed up."""
if not data.is_dir():
return []
return sorted(p for p in data.iterdir() if not _skipped(p.name, skip) and not p.is_symlink())
return sorted(p for p in data.iterdir() if not _top_skipped(p.name, skip) and not p.is_symlink())


def _walk(data, skip):
Expand Down Expand Up @@ -108,6 +117,7 @@ def snapshot(data_dir, name, db_path=None):
raise RuntimeError(
f"backing up the files needs {need >> 20} MB and the disk has {free >> 20} MB free; free some space, or delete older backups"
)
unread = []
try:
for src, rel, old in plan:
out = part / rel
Expand All @@ -118,11 +128,18 @@ def snapshot(data_dir, name, db_path=None):
continue
except OSError: # another file system, or too many links
pass
with contextlib.suppress(FileNotFoundError): # gone meanwhile
try:
shutil.copy2(src, out)
except FileNotFoundError: # gone meanwhile
pass
except PermissionError: # not Lens's own (another user's file): left out
out.unlink(missing_ok=True)
unread.append(str(rel))
except BaseException:
shutil.rmtree(part, ignore_errors=True)
raise
if unread:
log.warning("backup %s: left out %d file(s) Lens can't read, such as %s", name, len(unread), unread[0])
part.rename(dst)
return dst

Expand All @@ -143,7 +160,7 @@ def restore(data_dir, name, db_path=None):
p.rename(held / p.name)
moved.append(p.name)
for p in sorted(src.iterdir()):
if _skipped(p.name, skip):
if _top_skipped(p.name, skip):
continue
if p.is_dir():
shutil.copytree(p, data / p.name, symlinks=False)
Expand Down
5 changes: 4 additions & 1 deletion fastapi_backend/app/domain/updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -374,8 +374,11 @@ def _hand_over(db, run, backup):
run.update(state="queued", queued_at=_ts())
except Exception as e: # noqa: BLE001 - the admin reads why
log.exception("couldn't start the upgrade")
backing = run["state"] == "backing-up"
run.update(
state="failed", error=f"{'backing up' if run['state'] == 'backing-up' else 'handing the upgrade over'} failed: {e}"[:500]
state="failed",
step="backup" if backing else None, # the step the web app marks as failed
error=f"{'backing up' if backing else 'handing the upgrade over'} failed: {e}"[:500],
)
_put(db, upgrade=run)

Expand Down
4 changes: 2 additions & 2 deletions fastapi_backend/app/schemas/updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ class UpgradeRun(ResponseModel):
state: Literal["backing-up", "queued", "running", "rolling-back", "done", "failed", "rolled-back"] = Field(
description="rolling-back / rolled-back: the new version didn't start, so the agent went back to the one before"
)
step: Literal["download", "build", "start", "check"] | None = Field(
None, description="while running: what the agent is on (the backup comes before)"
step: Literal["backup", "download", "build", "start", "check"] | None = Field(
None, description="while running: what the agent is on (the backup comes before); backup: backing up failed"
)
restore: str | None = Field(None, description="after a rollback: whether the backup was put back, and why")
by: str | None = None
Expand Down
38 changes: 38 additions & 0 deletions fastapi_backend/tests/api/test_backups.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,44 @@ def test_backups_keep_and_restore_the_files_too(web):
assert (second / "objects/ab/abcdef").read_bytes() == b"blob" # a link, so the later backup still has it


def test_a_microsandbox_backs_up_only_lenss_own_files(folder, monkeypatch, caplog):
"""In a microsandbox, data_dir (/data) also holds the Lens it runs (.lens: apt, the venv, the web build, some of it
root's), caches and SurrealDB's own folder: none of it is backed up, and a file Lens can't read doesn't stop it."""
data = folder / "sandbox-data"
for rel, body in {
"uploads/pods/1/talk.mp3": b"talk",
"surreal/lens.db/000001.vlog": b"database",
".lens/apt/lock": b"",
".lens/venv/bin/python": b"elf",
".pnpm-store/v3/x": b"pkg",
".cache/uv/y": b"cache",
".DS_Store": b"finder",
"reports/1/locked.pdf": b"not ours",
}.items():
(data / rel).parent.mkdir(parents=True, exist_ok=True)
(data / rel).write_bytes(body)
real = shutil.copy2

def copy2(src, dst, **kw):
if str(src).endswith("locked.pdf"):
raise PermissionError(13, "Permission denied", str(src))
return real(src, dst, **kw)

monkeypatch.setattr(datafiles.shutil, "copy2", copy2)
kept = datafiles.snapshot(data, "manual-1")
got = sorted(str(p.relative_to(kept)) for p in kept.rglob("*") if p.is_file())
assert got == ["uploads/pods/1/talk.mp3"]
assert "left out 1 file(s) Lens can't read, such as reports/1/locked.pdf" in caplog.text

# restoring leaves them where they are too
(data / "uploads/pods/1/talk.mp3").write_bytes(b"changed")
assert datafiles.restore(data, "manual-1")
assert (data / "uploads/pods/1/talk.mp3").read_bytes() == b"talk"
assert (data / ".lens/venv/bin/python").read_bytes() == b"elf"
assert (data / "surreal/lens.db/000001.vlog").read_bytes() == b"database"
assert (data / ".DS_Store").exists() and (data / ".pnpm-store/v3/x").exists()


def test_a_backup_from_before_files_were_kept_leaves_them(web):
client, db, cfg, h = web
data = _files(cfg)
Expand Down
1 change: 1 addition & 0 deletions fastapi_backend/tests/api/test_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ def full(*_a, **_k):
updates.upgrade(db, client.app.state.settings.current(), by="t", wait=True)
run = client.get("/api/v1/admin/updates", headers=h).json()["upgrade"]
assert run["state"] == "failed" and "backing up failed: No space left" in run["error"]
assert run["step"] == "backup" # the step the web app marks as failed
assert not (control / "inbox" / "upgrade").exists()


Expand Down
4 changes: 4 additions & 0 deletions nextjs-frontend/__tests__/settings-updates.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -131,4 +131,8 @@ describe("upgrade steps", () => {
expect(at("rolling-back", "start")).toEqual(["done", "done", "done", "failed", "pending"]);
expect(at("done")).toEqual(["done", "done", "done", "done", "done"]);
});
it("mark the backup failed when backing up stopped the upgrade", () => {
expect(at("failed", "backup")).toEqual(["failed", "pending", "pending", "pending", "pending"]);
expect(at("failed", "download")).toEqual(["done", "failed", "pending", "pending", "pending"]);
});
});
4 changes: 2 additions & 2 deletions nextjs-frontend/app/openapi-client/types.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19131,9 +19131,9 @@ export type UpgradeRun = {
/**
* Step
*
* while running: what the agent is on (the backup comes before)
* while running: what the agent is on (the backup comes before); backup: backing up failed
*/
step?: "download" | "build" | "start" | "check" | null;
step?: "backup" | "download" | "build" | "start" | "check" | null;
/**
* Restore
*
Expand Down
2 changes: 1 addition & 1 deletion nextjs-frontend/components/settings/updates-panel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const STEPS = [
/** Where each step is, from the run's state and the step the machine says it is on. */
export function steps(run: UpgradeRun): StepState[] {
const at =
run.state === "backing-up"
run.state === "backing-up" || run.step === "backup"
? 0
: run.state === "queued"
? 1
Expand Down
3 changes: 2 additions & 1 deletion nextjs-frontend/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -60491,6 +60491,7 @@
{
"type": "string",
"enum": [
"backup",
"download",
"build",
"start",
Expand All @@ -60502,7 +60503,7 @@
}
],
"title": "Step",
"description": "while running: what the agent is on (the backup comes before)"
"description": "while running: what the agent is on (the backup comes before); backup: backing up failed"
},
"restore": {
"anyOf": [
Expand Down
Loading