From 38057905ffe55850c49d66376fba5036081ba8fe Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 07:11:54 +0000 Subject: [PATCH] feat(storage): show the space each source takes and what Lens made of it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 4 +- changelog.d/storage-per-source.md | 3 ++ docs/api.md | 5 +- docs/requirements.md | 1 + .../app/api/v1/routes/recordings.py | 12 +++-- fastapi_backend/app/domain/deletion.py | 10 ++-- fastapi_backend/app/domain/library.py | 46 +++++++++++++++++-- fastapi_backend/app/schemas/recordings.py | 6 +++ .../tests/api/test_recording_list.py | 33 +++++++++++++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 3 +- .../app/openapi-client/types.gen.ts | 18 ++++++++ nextjs-frontend/components/admin/health.tsx | 33 +++++++++++-- nextjs-frontend/openapi.json | 38 ++++++++++++++- 13 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 changelog.d/storage-per-source.md diff --git a/README.md b/README.md index 3c41880ae..5c42d045a 100644 --- a/README.md +++ b/README.md @@ -274,8 +274,10 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p same through the API and MCP. The Analytics page shows admins active people, churn, retention and which features get used, worked out on this machine; the same daily events can also go to a Google Analytics 4 property of your own (off unless you turn it on). + System health shows the space each source takes, its files and what Lens made of them. ([Logging](docs/logging.md), [telemetry](docs/telemetry.md), [Prometheus](docs/prometheus.md), - [usage analytics](docs/analytics.md); [checks](docs/requirements.md#watch-it-run)) + [usage analytics](docs/analytics.md), [storage per source](docs/api.md#resources); + [checks](docs/requirements.md#watch-it-run)) - **Stay private and safe.** Passkeys instead of passwords (one at Lens's remote address added from a signed-in one with Open signed in), encryption at rest with a key per namespace, roles per namespace (viewer, editor, owner, or custom roles made of permissions), who sees each asset, set for many at once from the Library, email invitations, removing diff --git a/changelog.d/storage-per-source.md b/changelog.d/storage-per-source.md new file mode 100644 index 000000000..1c246092d --- /dev/null +++ b/changelog.d/storage-per-source.md @@ -0,0 +1,3 @@ +- **Storage per source.** Admin › System health lists, under Disk, the space each source's assets take and what Lens + made of them (frames, pages, reports, caches); `GET /api/v1/resources/origins?storage=true` gives agents the same + ([API](docs/api.md#resources)). diff --git a/docs/api.md b/docs/api.md index f4fa98204..2502d5f47 100644 --- a/docs/api.md +++ b/docs/api.md @@ -651,7 +651,10 @@ of each); an asset of another namespace is a 400 (move it to that namespace firs their IIIF Manifests change (`partOf`), so harvesters see an Update. Audited as `recording.collection`. `GET /resources/origins`, `GET /resources/languages` and `GET /resources/objects` (`ns` for one namespace) list the origins, languages and kinds of object of the assets you can read with how many have each, most first, for the -Library's Source, Language and Objects filters. +Library's Source, Language and Objects filters. `GET /resources/origins?storage=true` adds the space each origin takes +on this server, read from the disk: `media_bytes`, the assets' own files (the media or document, and supplementary +files), and `made_bytes`, what Lens made of them (frames, pages, reports, exports, cached copies of files on storage +connections). Admin › System health shows it under Disk. `PATCH /resources/{rid}/segments/{idx}` corrects a line of text: its `text`, its `speaker` (a person id in the namespace, or `null`), or both (editors). `POST …/segments/{idx}/split {at, t?, speaker?}` splits a line in two at diff --git a/docs/requirements.md b/docs/requirements.md index c104e4ca9..f6988ae73 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -348,6 +348,7 @@ request, to another check that proves the same thing; loosening a requirement or | The job queue: clear it, reorder it by drag and drop, retry failed runs in bulk, in Activity and over the API and MCP | [test_jobs.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_jobs.py) `test_clearing_reordering_and_retrying_the_queue`, `test_agents_work_the_queue_over_mcp`; [queue-controls.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/queue-controls.test.tsx); [activity-model.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/activity-model.test.ts) | | The Analytics page: active people, churn, retention and feature use, worked out on this machine | [test_analytics.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_analytics.py) `test_metrics_active_churn_retention`; [analyticsPage.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/analyticsPage.test.tsx) | | The same daily events to a Google Analytics 4 property of your own, off unless turned on | [test_analytics.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_analytics.py) `test_events_kept_locally_with_ga_off`, `test_daily_events_once_a_day_counts_only` | +| The space each source takes, its assets' own files and what Lens made of them, through the API and on System health | [test_recording_list.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_recording_list.py) `test_the_space_each_origin_takes` | ## Stay private and safe diff --git a/fastapi_backend/app/api/v1/routes/recordings.py b/fastapi_backend/app/api/v1/routes/recordings.py index df19b9abc..a55cb2025 100644 --- a/fastapi_backend/app/api/v1/routes/recordings.py +++ b/fastapi_backend/app/api/v1/routes/recordings.py @@ -238,12 +238,18 @@ def list_tags(acl: Acl, user: CurrentUser, db: Db, ns: str | None = Query(None, @router.get("/origins") def list_origins( - acl: Acl, user: CurrentUser, db: Db, cfg: Cfg, ns: str | None = Query(None, description="one namespace") + acl: Acl, + user: CurrentUser, + db: Db, + cfg: Cfg, + ns: str | None = Query(None, description="one namespace"), + storage: bool = Query(False, description="also the disk space each takes (media_bytes, made_bytes), read from the disk"), ) -> list[OriginCount]: """Where the assets you can read (or one namespace's) came from, with how many came from each: connected sources - by name, uploads, pasted text, IIIF imports, the archive's own folders and other imported files.""" + by name, uploads, pasted text, IIIF imports, the archive's own folders and other imported files. With + ``storage=true``, the space each origin's files take and what Lens made of them.""" spaces, within = acl.scope(ns) - return [OriginCount.model_validate(o) for o in library.Origins(db, cfg).counts(spaces, within)] + return [OriginCount.model_validate(o) for o in library.Origins(db, cfg).counts(spaces, within, storage=storage)] @router.get("/languages") diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py index c82f5d087..58fbd0a53 100644 --- a/fastapi_backend/app/domain/deletion.py +++ b/fastapi_backend/app/domain/deletion.py @@ -132,6 +132,11 @@ def _files(db, cfg, rec, rid, ns): return out +def folders(cfg, rid): + """The folders Lens keeps for a recording: its frames, its supplementary files and an episode's audio.""" + return [video.frames_dir(cfg, rid), filemod.folder(cfg, rid), pathlib.Path(cfg["data_dir"]) / "podcasts" / str(int(rid))] + + def orphans(db, speakers, faces): """Speakers and faces nothing else has, which nobody named: they only existed for the deleted recording.""" for s in sorted(speakers): @@ -194,9 +199,8 @@ def delete(db, cfg, rid, by=None): rec=R("recording", rid), ) notebook.release(db, f"recording:{rid}") - shutil.rmtree(video.frames_dir(cfg, rid), ignore_errors=True) - shutil.rmtree(filemod.folder(cfg, rid), ignore_errors=True) - shutil.rmtree(pathlib.Path(cfg["data_dir"]) / "podcasts" / str(int(rid)), ignore_errors=True) # an episode's audio + for d in folders(cfg, rid): # its frames, its supplementary files and an episode's audio + shutil.rmtree(d, ignore_errors=True) for f in files: f.unlink(missing_ok=True) orphans(db, speakers, faces) diff --git a/fastapi_backend/app/domain/library.py b/fastapi_backend/app/domain/library.py index cfa15ee6d..576d50c35 100644 --- a/fastapi_backend/app/domain/library.py +++ b/fastapi_backend/app/domain/library.py @@ -98,7 +98,7 @@ def __init__(self, db, cfg): for name, spec in (cfg.get("namespaces") or {}).items() } self.roots = {sid: roots.get(name, []) for sid, name in names.items()} - self.db = db + self.db, self.cfg = db, cfg def of(self, r): rm = r.get("remote") or {} @@ -136,15 +136,55 @@ def names(self, keys): ) return {k: ORIGINS.get(k) or srcs.get(int(k.split(":", 1)[1])) or "A removed source" for k in keys} - def counts(self, spaces, within=None): + def storage(self, spaces, within=None): + """{origin: (media bytes, made bytes)}: the space each origin's assets take, their own files (the media or + document, and supplementary files) and what Lens made of them (frames, pages, reports, exports, cached copies + of files on storage connections). Read from the disk each time, so it is right however the files got there.""" + from . import deletion + + cond, p = in_scope(spaces, within) + rows = self.db.rows(f"SELECT record::id(id) AS id, space, path, remote, engine, size, title FROM recording WHERE {cond}", **p) + names = store.space_names(self.db) + out: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + for r in rows: + got = out[self.of(r)] + own = r.get("size") + if not own and r.get("path") and not r.get("remote"): + own = _size(store.resolve_path(self.cfg, r["path"])) + files = deletion._files(self.db, self.cfg, r, r["id"], names.get(r["space"])) + dirs = deletion.folders(self.cfg, r["id"]) + got[0] += int(own or 0) + _size(dirs[1]) + got[1] += sum(_size(f) for f in files) + _size(dirs[0]) + _size(dirs[2]) + return {k: (a, b) for k, (a, b) in out.items()} + + def counts(self, spaces, within=None, storage=False): """[{origin, name, recordings}], the most recordings first.""" n = Counter(self.recordings(spaces, within).values()) names = self.names(n) + sizes = self.storage(spaces, within) if storage else {} return sorted( - ({"origin": k, "name": names[k], "recordings": c} for k, c in n.items()), key=lambda x: (-x["recordings"], x["name"].casefold()) + ( + {"origin": k, "name": names[k], "recordings": c} + | ({"media_bytes": sizes.get(k, (0, 0))[0], "made_bytes": sizes.get(k, (0, 0))[1]} if storage else {}) + for k, c in n.items() + ), + key=lambda x: (-x["recordings"], x["name"].casefold()), ) +def _size(path): + """The bytes a file, or a folder and everything in it, takes; 0 when it isn't there.""" + p = pathlib.Path(path) if path else None + try: + if p is None or not p.exists(): + return 0 + if p.is_file(): + return p.stat().st_size + return sum(f.stat().st_size for f in p.rglob("*") if f.is_file()) + except OSError: + return 0 + + def check_origins(origins): for o in origins or []: kind, _, sid = o.partition(":") diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index d3358a82e..9a410ea02 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -136,6 +136,12 @@ class OriginCount(ResponseModel): origin: str = Field(description="source:, upload, paste, iiif, folder (the archive's own folders) or file") name: str recordings: int + media_bytes: int | None = Field( + None, description="with storage=true: the bytes their own files take (media or document, and supplementary files)" + ) + made_bytes: int | None = Field( + None, description="with storage=true: the bytes of what Lens made of them (frames, pages, reports, exports, caches)" + ) class LanguageCount(ResponseModel): diff --git a/fastapi_backend/tests/api/test_recording_list.py b/fastapi_backend/tests/api/test_recording_list.py index 812dc1131..2ffd12cf6 100644 --- a/fastapi_backend/tests/api/test_recording_list.py +++ b/fastapi_backend/tests/api/test_recording_list.py @@ -184,6 +184,39 @@ def test_where_they_came_from_and_their_language(client, db, cfg, archive, folde assert {x["origin"] for x in client.get("/api/v1/recordings/origins", headers=hv).json()} == {"upload", "paste", f"source:{sid}"} +def test_the_space_each_origin_takes(client, db, cfg, archive): + import pathlib + + from app.domain import files as filemod + from app.domain import video + + a, b, h = archive["a"], archive["b"], archive["h"] + sid = db.next_id("storage_source") + db.q("CREATE $r CONTENT {name: 'Team drive', type: 'local'}", r=R("storage_source", sid)) + db.q("UPDATE $r SET remote = {source: $s, path: 'talks/b.wav'}, size = 4000", r=R("recording", b), s=sid) + + def sizes(): + got = client.get("/api/v1/recordings/origins", params={"storage": True}, headers=h).json() + return {o["origin"]: (o["media_bytes"], o["made_bytes"]) for o in got} + + before = sizes() + assert before[f"source:{sid}"] == (4000, 0) + assert client.get("/api/v1/recordings/origins", headers=h).json()[0].get("media_bytes") is None # only when asked + data = pathlib.Path(cfg["data_dir"]) + frames = video.frames_dir(cfg, a) + frames.mkdir(parents=True, exist_ok=True) + (frames / "f1.jpg").write_bytes(b"x" * 1200) + (data / "renditions").mkdir(parents=True, exist_ok=True) + (data / "renditions" / f"{a}.pdf").write_bytes(b"x" * 300) + att = filemod.folder(cfg, a) / "1" + att.mkdir(parents=True, exist_ok=True) + (att / "notes.txt").write_bytes(b"x" * 700) + after = sizes() + media, made = after["file"] + assert (media - before["file"][0], made - before["file"][1]) == (700, 1500) # its own files; what Lens made + assert after[f"source:{sid}"] == (4000, 0) + + def test_edited_by_me(client, db, cfg, archive): from app.domain import metadata diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 421124bb6..adb0f34e2 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -5794,7 +5794,8 @@ export class Resources { * List Origins * * Where the assets you can read (or one namespace's) came from, with how many came from each: connected sources - * by name, uploads, pasted text, IIIF imports, the archive's own folders and other imported files. + * by name, uploads, pasted text, IIIF imports, the archive's own folders and other imported files. With + * ``storage=true``, the space each origin's files take and what Lens made of them. */ public static listOrigins( options?: Options, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 00011dda3..9a345489f 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -16977,6 +16977,18 @@ export type OriginCount = { * Recordings */ recordings: number; + /** + * Media Bytes + * + * with storage=true: the bytes their own files take (media or document, and supplementary files) + */ + media_bytes?: number | null; + /** + * Made Bytes + * + * with storage=true: the bytes of what Lens made of them (frames, pages, reports, exports, caches) + */ + made_bytes?: number | null; [key: string]: unknown; }; @@ -35966,6 +35978,12 @@ export type ListOriginsData = { * one namespace */ ns?: string | null; + /** + * Storage + * + * also the disk space each takes (media_bytes, made_bytes), read from the disk + */ + storage?: boolean; }; url: "/api/v1/resources/origins"; }; diff --git a/nextjs-frontend/components/admin/health.tsx b/nextjs-frontend/components/admin/health.tsx index e949842cc..b64c23802 100644 --- a/nextjs-frontend/components/admin/health.tsx +++ b/nextjs-frontend/components/admin/health.tsx @@ -5,7 +5,7 @@ import { Cpu, Database, HardDriveDownload, KeyRound, Sparkles, type LucideIcon } import Link from "next/link"; import type { ReactNode } from "react"; -import { Admin, Jobs } from "@/app/openapi-client"; +import { Admin, Jobs, Resources } from "@/app/openapi-client"; import { AdminFrame } from "@/components/admin/admin-frame"; import { grafanaAddress, MonitoringSwitch, useMonitoring } from "@/components/admin/monitoring-switch"; import { isUnreachable, ServerUnreachable } from "@/components/errors/error-states"; @@ -118,6 +118,12 @@ export function HealthPage() { const test = useMutation({ mutationFn: () => data(Admin.testLlm({ client })), }); + // read from the disk on the server, so asked for less often than the rest + const storage = useQuery({ + queryKey: ["recording-origins", "storage"], + queryFn: () => data(Resources.listOrigins({ client, query: { storage: true } })), + staleTime: 5 * 60_000, + }); if (health.isPending) return ( @@ -301,9 +307,30 @@ export function HealthPage() { /> - {used.toFixed(1)} GB used · free {h.disk.free_gb} GB of {h.disk.total_gb} GB. The split between audio, cache - and database isn’t reported yet. + {used.toFixed(1)} GB used · free {h.disk.free_gb} GB of {h.disk.total_gb} GB. + {storage.data && storage.data.length > 0 && ( + + + + + + + + + + + {storage.data.map((o) => ( + + + + + + + ))} + +
Came fromAssetsTheir filesMade by Lens
{o.name}{count(o.recordings)}{bytes(o.media_bytes)}{bytes(o.made_bytes)}
+ )}
Queue diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 1d73c34c9..c2213a071 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -10172,7 +10172,7 @@ "resources" ], "summary": "List Origins", - "description": "Where the assets you can read (or one namespace's) came from, with how many came from each: connected sources\nby name, uploads, pasted text, IIIF imports, the archive's own folders and other imported files.", + "description": "Where the assets you can read (or one namespace's) came from, with how many came from each: connected sources\nby name, uploads, pasted text, IIIF imports, the archive's own folders and other imported files. With\n``storage=true``, the space each origin's files take and what Lens made of them.", "operationId": "list_origins", "parameters": [ { @@ -10192,6 +10192,18 @@ "title": "Ns" }, "description": "one namespace" + }, + { + "name": "storage", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "also the disk space each takes (media_bytes, made_bytes), read from the disk", + "default": false, + "title": "Storage" + }, + "description": "also the disk space each takes (media_bytes, made_bytes), read from the disk" } ], "responses": { @@ -65192,6 +65204,30 @@ "recordings": { "type": "integer", "title": "Recordings" + }, + "media_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Media Bytes", + "description": "with storage=true: the bytes their own files take (media or document, and supplementary files)" + }, + "made_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Made Bytes", + "description": "with storage=true: the bytes of what Lens made of them (frames, pages, reports, exports, caches)" } }, "additionalProperties": true,