From d2a16f45855cf42bfd6bc296dd2609c97f411156 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:12:14 +0000 Subject: [PATCH 1/3] feat(tree): show the graph as folders and files Namespaces, collections, topics and entities become folders; each asset is its own file, a .json metadata sidecar and a .txt of its text. Shared layer (app/domain/filetree.py) for the S3, SMB and NFS servers, browsable now through GET /api/v1/tree and the MCP tools list_files and read_file. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- CHANGELOG.md | 5 + docs/api.md | 10 + docs/file-tree.md | 61 ++ docs/mcp.md | 2 + fastapi_backend/app/api/mcp_tools.py | 58 +- fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/filetree.py | 65 +++ fastapi_backend/app/domain/filetree.py | 546 ++++++++++++++++++ fastapi_backend/app/schemas/filetree.py | 20 + fastapi_backend/tests/api/test_file_tree.py | 154 +++++ mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 13 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 37 ++ .../app/openapi-client/types.gen.ts | 114 ++++ nextjs-frontend/openapi.json | 169 ++++++ 15 files changed, 1256 insertions(+), 1 deletion(-) create mode 100644 docs/file-tree.md create mode 100644 fastapi_backend/app/api/v1/routes/filetree.py create mode 100644 fastapi_backend/app/domain/filetree.py create mode 100644 fastapi_backend/app/schemas/filetree.py create mode 100644 fastapi_backend/tests/api/test_file_tree.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5112764c6..57d29ba56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **The graph as files.** Namespaces, collections, topics and entities are folders, and each asset is its own file, a + `.json` with its metadata and a `.txt` with its text, in every folder it belongs to; everyone sees what the web app + shows them. Browse and read it through the API (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` + ([The graph as files](docs/file-tree.md)). The S3, SMB and NFS servers that share it are in progress. + - **Install on Windows, or by double-clicking.** `install.ps1` installs Lens on Windows: it gets Python 3 if needed and opens the web installer, which installs Docker Desktop (and WSL) and runs Lens in it. Each release now carries launchers that need no terminal at all: **Install Lens.app** for macOS, **Install Lens.cmd** for Windows and diff --git a/docs/api.md b/docs/api.md index 8d862bc80..9859b4f87 100644 --- a/docs/api.md +++ b/docs/api.md @@ -594,6 +594,16 @@ page was before each change, and `restore` puts an earlier version back (what it lists the files and images attached to a page; `blobs` uploads (in chunks) and reads them, and `DELETE …/files?key=` removes one. +## the graph as files + +``` +GET /api/v1/tree?path= +GET /api/v1/tree/file?path= +``` + +The archive as folders and files ([The graph as files](file-tree.md)): `tree` lists a folder (an empty path lists your +namespaces), `tree/file` reads a file with byte ranges. Read-only; everyone sees what they can read in the web app. + ## comments ``` diff --git a/docs/file-tree.md b/docs/file-tree.md new file mode 100644 index 000000000..64c910317 --- /dev/null +++ b/docs/file-tree.md @@ -0,0 +1,61 @@ +# The graph as files + +Lens shows your archive as an ordinary tree of folders and files: namespaces, collections, topics and entities are +folders, and every asset (a recording, document or image) is a few files in each folder it belongs to. Agents and +tools that already understand files can read Lens without learning its API. + +| Where | Status | +| --- | --- | +| The API (`GET /api/v1/tree`, `GET /api/v1/tree/file`) and the MCP tools `list_files` and `read_file` | built | +| An S3 server (read-only, off until turned on) | in progress | +| SMB and NFS shares | in progress | + +Every way in serves the same tree, built by `app/domain/filetree.py`. + +## Layout + +```text +/ + namespace.json the namespace: its collections, topic and entity counts + Collections//<…>/ every asset in its collection's folder + Topics// the assets about a topic (accepted, not suggestions) + Entities/// the assets that mention an entity (hidden entities, dates and + numbers left out) +``` + +An asset's files are named after it, ` [<id>]`, so names are unique and stable: + +| File | What it holds | +| --- | --- | +| `<title> [<id>].<ext>` | its own file as it was imported: audio, video, PDF, Word file, image… (`.original.txt` for an imported transcript) | +| `<title> [<id>].json` | its metadata: Dublin Core terms (`identifier`, `title`, `type`, `format`, `created`, `date`, `subject` for its topics), its collection, its entities, and the description saved on it | +| `<title> [<id>].txt` | its text: the transcript (`Speaker: …` lines) or what was read off its pages | +| `<title> [<id>].pdf` | the PDF made of a document that isn't one (a Word file, an email…) | + +The same asset shows in its collection's folder, in each topic's folder and in each entity's folder. Files stored +encrypted ([Encryption](encryption.md)) are read decrypted, and a locked vault's files can't be read until one of its +owners unlocks it. Files kept on a storage connection are read from there. + +Names are made safe for every file system: `/ \ : * ? " < > |` and control characters become `_`, names are cut to +120 characters, and two names in a folder that differ only in case get the second one's id added (`Talks [12]`). + +## Who sees what + +Exactly what the web app shows them ([Access](access.md)): an admin sees every namespace; others see the namespaces +they have a role in, and in a namespace where they only have roles on some collections, those collections, and only +the topics and entities of the assets in them. Anything else answers "not found". The tree only reads. + +## The API + +`GET /api/v1/tree?path=pods/Topics/Gene therapy` lists a folder: `{path, entries: [{name, folder, modified, size, +content_type}]}`. An empty path lists your namespaces. `GET /api/v1/tree/file?path=…` reads a file, with byte ranges +(`Range: bytes=0-1023`) and an `ETag`. Asking for a file as a folder (or the other way round) answers 400. + +The MCP server has `list_files` and `read_file` ([MCP server](mcp.md)): `read_file` returns the text of `.txt` and +`.json` files, a page at a time, and the download link of everything else. + +## Refine later + +- Show supplementary files (captions, translations, attachments) beside each asset. +- Cache an asset's text and metadata so large folders list faster. +- `Speakers/<name>/` folders, and a folder of notes. diff --git a/docs/mcp.md b/docs/mcp.md index 5a8b2b154..3bbf8abc1 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -62,6 +62,8 @@ front of the web app also needs `TRUST_PROXY_HEADERS=true` ([Authentication](aut | `write_note` | write a note or a thing's page, marked as written by an assistant. Needs a write-scope token and editor access | | `create_podcast` | make a two-host learning [episode](podcasts.md) about recordings or lines in them, with an angle, a length and a style; it's made in the background. Needs a write-scope token and editor access where it goes | | `get_podcast` | an episode: how far it has got, its script with each line's speaker and the sources it cites (links), and what the fact-check changed | +| `list_files` | a folder of the archive [as files](file-tree.md): your namespaces, then each one's Collections, Topics and Entities folders holding its assets' files | +| `read_file` | a text file of that tree (an asset's `.txt` text or `.json` metadata, `namespace.json`), a page at a time; other files' download link | | `file_storage` | where Lens keeps its own files (notes' images and attachments) and the connections it could use; `check` tests it. Admins only | | `set_file_storage` | keep new files on this machine or a storage connection (folder, rclone crypt); checked before it's saved. Admins only, with a write-scope token | | `propose_graph_change` | ask for two entities to be merged or linked; it waits in Proposed changes unless `apply` makes it at once (it can be undone). Needs a write-scope token and editor access | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 8015a11fb..7defd3efc 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -12,6 +12,7 @@ import html import json import re +import urllib.parse from collections.abc import Callable from dataclasses import dataclass, field from typing import Any, Literal @@ -26,7 +27,7 @@ from app.api.v1.routes import recordings as recording_routes from app.api.v1.routes import search as search_routes from app.api.v1.routes import topics as topic_routes -from app.domain import analyze, auth, blobs, feeds, graph_history, library, notebook, rdf, render, settings, sources, store +from app.domain import analyze, auth, blobs, feeds, filetree, graph_history, library, notebook, rdf, render, settings, sources, store from app.domain import entities as ents from app.domain import speakers as spk from app.domain import topics as topicmod @@ -1357,6 +1358,61 @@ def get_podcast(ctx: Context, episode_id: int) -> dict[str, Any]: ) +# ---------- the graph as files ---------- +FILE_CHARS = 100_000 # the most of a file read_file returns at once; offset pages through the rest + + +def _tree(ctx: Context) -> filetree.Tree: + return filetree.Tree(ctx.db, ctx.cfg, filetree.Viewer(ctx.user.admin, ctx.user.roles, ctx.user.collections)) + + +@tool( + "list_files", + "List a folder of the file tree", + "The archive as folders and files, the tree its S3 server serves: '' lists your namespaces; a namespace has " + "Collections/<collection>/…, Topics/<topic>/ and Entities/<type>/<name>/ holding its assets, and namespace.json. " + "Each asset is files named '<title> [<id>]': its own file (.mp3, .pdf, …), .json with its metadata, topics and " + "entities, and .txt with its text.", + Arg("path", "string", "the folder, from the top (e.g. pods/Topics/Gene therapy)", default="", max_length=4000), +) +def list_files(ctx: Context, path: str) -> dict[str, Any]: + t = _tree(ctx) + try: + entries = t.list(path) + except filetree.NotFound: + raise KeyError(path) from None + except filetree.NotAFolder: + raise ToolError("that's a file: read it with read_file") from None + return {"path": "/".join(t.split(path)), "entries": [e.out() for e in entries]} + + +@tool( + "read_file", + "Read a file of the file tree", + "A text file of the tree (an asset's .txt or .json, or namespace.json), from list_files. Media and documents' own " + f"files are only their size and type here: download them from the API (/api/v1/tree/file?path=). At most " + f"{FILE_CHARS} characters at a time; next_offset continues.", + Arg("path", "string", "the file, from the top", required=True, max_length=4000), + Arg("offset", "integer", "start at this character", default=0, minimum=0), +) +def read_file(ctx: Context, path: str, offset: int) -> dict[str, Any]: + try: + e = _tree(ctx).stat(path) + except filetree.NotFound: + raise KeyError(path) from None + if e.folder: + raise ToolError("that's a folder: list it with list_files") + out: dict[str, Any] = {"path": path, "size": e.size, "content_type": e.content_type, "modified": e.modified} + if e.data is None: + out["download"] = f"{store.API}/tree/file?path={urllib.parse.quote(path)}" + return out + text = e.data.decode("utf-8", "replace") + out["text"] = text[offset : offset + FILE_CHARS] + if offset + FILE_CHARS < len(text): + out["next_offset"] = offset + FILE_CHARS + return out + + # ---------- where files are kept (admins) ---------- def _admin(ctx: Context) -> None: if not ctx.user.admin: diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 49941c369..d3b6eef1d 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -21,6 +21,7 @@ fedora, fields, files, + filetree, graph, graph_history, hierarchy, @@ -84,6 +85,7 @@ notebook, comments, files, + filetree, fields, imports, uploads, diff --git a/fastapi_backend/app/api/v1/routes/filetree.py b/fastapi_backend/app/api/v1/routes/filetree.py new file mode 100644 index 000000000..35600704a --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/filetree.py @@ -0,0 +1,65 @@ +"""The graph as folders and files (docs/file-tree.md): browse the tree the S3, SMB and NFS servers serve, and read +its files. Everyone sees what the web app shows them; the tree only reads.""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, HTTPException, Query, Request +from fastapi.responses import Response, StreamingResponse + +from app.api.deps import Cfg, CurrentUser, Db, Principal +from app.api.streaming import LOCKED, byte_range +from app.domain import filetree, keyring +from app.domain.store import DB +from app.schemas.filetree import TreeEntry, TreeFolder + +router = APIRouter(prefix="/tree", tags=["tree"]) +PATH = Query("", max_length=4000, description="a folder or file, from the top: namespace/Collections/…") + + +def tree_of(db: DB, cfg: dict[str, Any], user: Principal) -> filetree.Tree: + return filetree.Tree(db, cfg, filetree.Viewer(user.admin, user.roles, user.collections)) + + +@router.get("") +def list_folder(user: CurrentUser, db: Db, cfg: Cfg, path: str = PATH) -> TreeFolder: + """A folder's files and folders: '' lists your namespaces; each has Collections, Topics, Entities and + namespace.json. A file answers 400 (read it at /tree/file).""" + t = tree_of(db, cfg, user) + try: + entries = t.list(path) + except filetree.NotFound: + raise HTTPException(404, "not found") from None + except filetree.NotAFolder: + raise HTTPException(400, "that's a file: read it at /tree/file") from None + return TreeFolder(path="/".join(t.split(path)), entries=[TreeEntry(**e.out()) for e in entries]) + + +@router.get( + "/file", + response_class=Response, + responses={200: {"content": {"application/octet-stream": {}}}, 206: {"description": "a byte range"}}, +) +def read_file(request: Request, user: CurrentUser, db: Db, cfg: Cfg, path: str = PATH) -> Response: + """A file of the tree, with byte ranges.""" + t = tree_of(db, cfg, user) + try: + e = t.stat(path) + except filetree.NotFound: + raise HTTPException(404, "not found") from None + if e.folder: + raise HTTPException(400, "that's a folder: list it at /tree") + r = byte_range(e.size, request) if e.size else (0, -1, 200) + if r is None: + return Response(status_code=416, headers={"Content-Range": f"bytes */{e.size}"}) + start, end, status = r + try: + if e.encrypted: # a locked vault says so now, not halfway through the body + keyring.plain_size(db, cfg, e.path) + except keyring.Locked: + raise HTTPException(423, LOCKED) from None + headers = {"Accept-Ranges": "bytes", "ETag": f'"{e.version}"', "Content-Length": str(max(0, end - start + 1))} + if status == 206: + headers["Content-Range"] = f"bytes {start}-{end}/{e.size}" + return StreamingResponse(e.read(db, cfg, start, end), status_code=status, media_type=e.content_type, headers=headers) diff --git a/fastapi_backend/app/domain/filetree.py b/fastapi_backend/app/domain/filetree.py new file mode 100644 index 000000000..40c68f447 --- /dev/null +++ b/fastapi_backend/app/domain/filetree.py @@ -0,0 +1,546 @@ +"""The graph as folders and files (docs/file-tree.md): what the S3, SMB and NFS servers serve, and the API browses. + + /<namespace>/ + namespace.json the namespace: its collections, topic and entity counts + Collections/<collection>/<…>/<asset> every asset in its collection's folder + Topics/<topic>/<asset> the assets about a topic (accepted ones) + Entities/<type>/<name>/<asset> the assets that mention an entity + +An asset (a recording, document or image) shows as up to four files named after it, `<title> [<id>]`: +its own file (`.mp3`, `.pdf`, …), `.json` beside it with its metadata (Dublin Core terms, its topics and entities), +`.txt` with its text (the transcript, or what was read off its pages), and `.pdf` for a document converted to one. +An own file that is itself `.txt` or `.json` (an imported transcript) is `<title> [<id>].original.txt`. +The same asset shows in every folder it belongs to. + +What someone sees is what the web app shows them: the namespaces they have a role in (admins: all of them), and in a +namespace where they only have roles on some collections, those collections and the topics and entities of the +assets in them. The tree only reads; names are made safe for every file system (no `/ \\ : * ? " < > |`) and unique +in their folder, ignoring case. +""" + +from __future__ import annotations + +import hashlib +import json +import mimetypes +import os +import pathlib +import re +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any + +from . import auth, convert, documents, entity_setup, hierarchy, keyring, metadata, render, store, video + +R = store.R +COLLECTIONS, TOPICS, ENTITIES = "Collections", "Topics", "Entities" +NS_FILE = "namespace.json" +NAME_MAX = 120 +CHUNK = 1 << 16 +ASSET_ID = re.compile(r" \[(\d+)\](?:\.[^.\[\]]+)+$") +UNSAFE = re.compile(r'[\x00-\x1f\x7f/\\:*?"<>|]') +REC_FIELDS = ( + "record::id(id) AS id, space, collection, title, path, source, remote, size, media, status, duration_ms, created_at, " + "updated_at, analyzed_at, recorded_at, meta_json, access, access_parts, featured, fields" +) + + +class NotFound(KeyError): + """No such file or folder, or not one this person may see.""" + + +class NotAFolder(ValueError): + """A file where a folder was asked for.""" + + +@dataclass(frozen=True) +class Viewer: + """Who is looking: an admin sees every namespace; others their namespaces (`roles`) and, in namespaces they have + no role in, the collections they were given a role on, with the ones inside them (`collections`).""" + + admin: bool = False + roles: dict[int, str] = field(default_factory=dict) + collections: dict[int, dict[int, str]] = field(default_factory=dict) + + @classmethod + def of(cls, db, account: dict[str, Any]) -> Viewer: + """For an account row ({id, admin}).""" + admin = bool(account.get("admin")) + return cls(admin, auth.roles(db, account), {} if admin else hierarchy.roles_of(db, account["id"])) + + def spaces(self, db) -> dict[int, set[int] | None]: + """{namespace: None (all of it) or the collections seen}.""" + out: dict[int, set[int] | None] = {} + for sid, cols in self.collections.items(): + if cols: + out[sid] = set(cols) + for sid in self.roles: + out[sid] = None + return out + + +@dataclass +class Entry: + """A file or a folder. A file's bytes come from `data` (made here) or `path` (stored, maybe encrypted) or a storage + source (`remote`: {source, path, space}).""" + + name: str + folder: bool + size: int = 0 + modified: str | None = None + content_type: str | None = None + data: bytes | None = None + path: str | None = None + encrypted: bool = False + remote: dict[str, Any] | None = None + + @property + def version(self) -> str: + """Changes when the file does (for ETags): its name, size and time.""" + return hashlib.sha256(f"{self.name}\0{self.size}\0{self.modified}".encode()).hexdigest()[:24] + + def out(self) -> dict[str, Any]: + d: dict[str, Any] = {"name": self.name, "folder": self.folder, "modified": self.modified} + if not self.folder: + d.update(size=self.size, content_type=self.content_type) + return d + + def read(self, db, cfg, start: int = 0, end: int | None = None) -> Iterator[bytes]: + """The bytes from `start` to `end` (inclusive; None: to the end). A locked vault raises keyring.Locked.""" + if self.folder: + raise NotAFolder(self.name) + end = self.size - 1 if end is None else min(end, self.size - 1) + if end < start: + return iter(()) + if self.data is not None: + return iter([self.data[start : end + 1]]) + if self.remote is not None: + from . import sources + + return sources.stream(db, cfg, self.remote["source"], self.remote["path"], start, end - start + 1, self.remote.get("space")) + return self._file(db, cfg, start, end) + + def _file(self, db, cfg, start: int, end: int) -> Iterator[bytes]: + assert self.path is not None + f = keyring.Reader(db, cfg, self.path) if self.encrypted else open(self.path, "rb") # noqa: SIM115 - closed below + with f: + f.seek(start) + left = end - start + 1 + while left > 0: + chunk = f.read(min(CHUNK, left)) + if not chunk: + break + left -= len(chunk) + yield chunk + + +def clean(name: Any) -> str: + """A name any file system takes: no separators or reserved characters, no trailing dots or spaces, not too long.""" + s = UNSAFE.sub("_", " ".join(str(name or "").split()))[:NAME_MAX].rstrip(". ") + return s if s and s not in (".", "..") else "_" + + +def _unique(items: list[tuple[str, Any]]) -> list[tuple[str, Any]]: + """Names made unique in their folder, ignoring case: a repeat gets its key added, ` [key]`.""" + seen: set[str] = set() + out = [] + for name, key in sorted(items, key=lambda x: (x[0].casefold(), str(x[1]))): + n = name + if n.casefold() in seen: + n = f"{name} [{key}]" + seen.add(n.casefold()) + out.append((n, key)) + return out + + +def _json(d: Any) -> bytes: + return (json.dumps(d, ensure_ascii=False, indent=2, default=str) + "\n").encode() + + +def _when(rec: dict[str, Any]) -> str | None: + return rec.get("updated_at") or rec.get("analyzed_at") or rec.get("created_at") + + +def _folder(name: str, modified: str | None = None) -> Entry: + return Entry(name, True, modified=modified) + + +def _type_of(name: str, kind: str) -> str: + ext = pathlib.PurePosixPath(name).suffix.lower() + return ( + {"video": video.VIDEO_TYPES.get(ext), "audio": render.AUDIO_TYPES.get(ext)}.get(kind) + or documents.content_type(name) + or mimetypes.guess_type(name)[0] + or "application/octet-stream" + ) + + +class Tree: + """One person's view of the tree, for one request: list folders, look files up, read them.""" + + def __init__(self, db, cfg, viewer: Viewer): + self.db, self.cfg, self.viewer = db, cfg, viewer + self._spaces = viewer.spaces(db) + self._names = {sid: n for sid, n in store.space_names(db).items() if sid in self._spaces} + self._cols: dict[int, list[dict[str, Any]]] = {} + self._seen: dict[int, set[int]] = {} + self._only: int | None = None + + # ---------- walking ---------- + @staticmethod + def split(path: str) -> list[str]: + return [p for p in str(path or "").split("/") if p] + + def list(self, path: str) -> list[Entry]: + """A folder's files and folders, by name; NotFound, or NotAFolder for a file.""" + try: + return self._list(self.split(path)) + except NotFound: + if not self.stat(path).folder: # NotFound again when there's nothing there at all + raise NotAFolder(path) from None + raise + + def _list(self, parts: list[str]) -> list[Entry]: + if not parts: + return [_folder(n) for n in sorted(self._names.values(), key=str.casefold)] + sid = self._space(parts[0]) + rest = parts[1:] + if not rest: + return [_folder(COLLECTIONS), _folder(ENTITIES), self._ns_file(sid), _folder(TOPICS)] + head, rest = rest[0], rest[1:] + if head == COLLECTIONS: + return self._list_collection(sid, rest) + if head == TOPICS: + topics = self._topics(sid) + if not rest: + return [_folder(n) for n in topics] + if len(rest) == 1 and rest[0] in topics: + return self._assets(self._topic_recordings(sid, topics[rest[0]])) + if head == ENTITIES: + types = self._types(sid) + if not rest: + return [_folder(n) for n in types] + if rest[0] in types: + ents = self._entities(sid, types[rest[0]]) + if len(rest) == 1: + return [_folder(n) for n in ents] + if len(rest) == 2 and rest[1] in ents: + return self._assets(self._entity_recordings(sid, ents[rest[1]])) + raise NotFound("/".join(parts)) + + def stat(self, path: str) -> Entry: + """One file or folder; NotFound when there's none (or it isn't this person's to see).""" + parts = self.split(path) + if not parts: + return _folder("") + sid = self._space(parts[0]) + if len(parts) == 1: + return _folder(parts[0]) + if len(parts) == 2 and parts[1] == NS_FILE: + return self._ns_file(sid) + *up, name = parts + m = ASSET_ID.search(name) + self._only = int(m.group(1)) if m else None # one asset's files: don't make the whole folder's + try: + listing = self._list(up) + except NotFound: + raise NotFound(path) from None + finally: + self._only = None + for e in listing: + if e.name == name: + return e + raise NotFound(path) + + def walk(self, path: str = "") -> Iterator[tuple[str, Entry]]: + """Every file under a folder, depth first: (its path from the root, the file).""" + base = "/".join(self.split(path)) + for e in self.list(base): + p = f"{base}/{e.name}" if base else e.name + if e.folder: + yield from self.walk(p) + else: + yield p, e + + def _space(self, name: str) -> int: + for sid, n in self._names.items(): + if n == name: + return sid + raise NotFound(name) + + # ---------- namespaces and collections ---------- + def _collections(self, sid: int) -> list[dict[str, Any]]: + if sid not in self._cols: + rows = hierarchy.of_space(self.db, sid) + seen = self._spaces[sid] + self._cols[sid] = [c for c in rows if seen is None or c["id"] in seen] + return self._cols[sid] + + def _children(self, sid: int, parent: int | None) -> dict[str, dict[str, Any]]: + """The collections in a collection (None: at the top of what this person sees), by their names here.""" + rows = self._collections(sid) + known = {c["id"] for c in rows} + if parent is None: + kids = [c for c in rows if c.get("parent") is None or c["parent"] not in known] + else: + kids = [c for c in rows if c.get("parent") == parent] + by_id = {c["id"]: c for c in kids} + return {n: by_id[cid] for n, cid in _unique([(clean(c["name"]), c["id"]) for c in kids])} + + def _list_collection(self, sid: int, names: list[str]) -> list[Entry]: + cid = None + for n in names: + kids = self._children(sid, cid) + if n not in kids: + raise NotFound(n) + cid = kids[n]["id"] + out = [_folder(n, c.get("updated_at") or c.get("created_at")) for n, c in self._children(sid, cid).items()] + if cid is not None: + out += self._assets(self.db.rows(f"SELECT {REC_FIELDS} FROM recording WHERE collection = $c", c=cid)) + return sorted(out, key=lambda e: e.name.casefold()) + + def _visible(self, sid: int, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + seen = self._spaces.get(sid) + return [r for r in rows if r.get("space") == sid and (seen is None or r.get("collection") in seen)] + + def _recordings(self, sid: int, rids: list[int]) -> list[dict[str, Any]]: + if not rids: + return [] + rows = self.db.rows(f"SELECT {REC_FIELDS} FROM recording WHERE id IN $ids", ids=[R("recording", int(r)) for r in set(rids)]) + return self._visible(sid, rows) + + def _ns_file(self, sid: int) -> Entry: + cols = hierarchy.tree(self.db, sid, only=self._spaces[sid]) + d = { + "namespace": self._names[sid], + "collections": [{"path": "/".join(c["path"]), "assets": c["recordings"], "id": c["id"]} for c in cols], + "topics": len(self._topics(sid)), + "entity_types": list(self._types(sid)), + "layout": { + COLLECTIONS: "every asset in its collection's folder", + TOPICS: "the assets about each topic", + ENTITIES: "the assets that mention each entity, by type", + }, + } + data = _json(d) + return Entry(NS_FILE, False, len(data), None, "application/json", data=data) + + # ---------- topics and entities ---------- + def _topics(self, sid: int) -> dict[str, int]: + rows = self.db.rows("SELECT record::id(id) AS id, label FROM topic WHERE space = $s", s=sid) + if self._spaces[sid] is not None: # seen in part: the topics of the assets they see + have = { + a["topic"] + for a in self.db.rows("SELECT topic, recording FROM topic_about WHERE space = $s AND status = 'accepted'", s=sid) + if a["recording"] in self._seen_recordings(sid) + } + rows = [t for t in rows if t["id"] in have] + return dict(_unique([(clean(t["label"]), t["id"]) for t in rows])) + + def _topic_recordings(self, sid: int, tid: int) -> list[dict[str, Any]]: + rids = self.db.values("SELECT VALUE recording FROM topic_about WHERE topic = $t AND status = 'accepted'", t=tid) + return self._recordings(sid, rids) + + def _seen_recordings(self, sid: int) -> set[int]: + if sid not in self._seen: + self._seen[sid] = hierarchy.recordings_in(self.db, self._spaces[sid] or []) + return self._seen[sid] + + def _entity_rows(self, sid: int) -> list[dict[str, Any]]: + from .entities import QUIET + + rows = [ + e + for e in self.db.rows("SELECT record::id(id) AS id, name, type, hidden FROM entity WHERE space = $s", s=sid) + if not e.get("hidden") and e.get("type") not in QUIET + ] + if self._spaces[sid] is not None: + seen = self._seen_recordings(sid) + have = { + m["entity"] + for m in self.db.rows("SELECT entity, recording FROM mentions WHERE space = $s GROUP BY entity, recording", s=sid) + if m["recording"] in seen + } + rows = [e for e in rows if e["id"] in have] + return rows + + def _types(self, sid: int) -> dict[str, str]: + """{folder name: entity type} for the types this namespace's entities have.""" + from .entities import TYPES + + own = entity_setup.labels(self.db, {sid}).get(sid, {}) + codes = sorted({e["type"] for e in self._entity_rows(sid)}) + return dict(_unique([(clean(TYPES.get(c) or own.get(c) or c.title()), c) for c in codes])) + + def _entities(self, sid: int, code: str) -> dict[str, int]: + return dict(_unique([(clean(e["name"]), e["id"]) for e in self._entity_rows(sid) if e["type"] == code])) + + def _entity_recordings(self, sid: int, eid: int) -> list[dict[str, Any]]: + rids = [m["recording"] for m in self.db.rows("SELECT recording FROM mentions WHERE entity = $e GROUP BY recording", e=eid)] + return self._recordings(sid, rids) + + # ---------- assets ---------- + def _primary(self, rec: dict[str, Any]) -> Entry | None: + """The asset's own file: stored here (maybe encrypted), cached from a storage source, or on one.""" + kind = render.kind(rec) + rm = rec.get("remote") or None + original = (rm or {}).get("path") or rec.get("path") or "" + name = pathlib.PurePosixPath(str(original)).name + if not name: + return None + ctype = _type_of(name, kind) + local = store.resolve_path(self.cfg, rec.get("path")) if not rm else None + if rm: + from . import sources + + for cached in ( + sources.cache_file(self.cfg, rm["source"], rm["path"], rec.get("space")), + sources.cache_file(self.cfg, rm["source"], rm["path"]), + ): + if cached.is_file(): + local = str(cached) + break + if local and os.path.isfile(local): + enc = keyring.is_encrypted(local) + try: + size = keyring.plain_size(self.db, self.cfg, local) if enc else os.path.getsize(local) + except (keyring.Locked, keyring.Damaged): + if not rec.get("size"): + return None + size = int(rec["size"]) + return Entry(name, False, size, _when(rec), ctype, path=local, encrypted=enc) + if rm and rec.get("size"): + return Entry(name, False, int(rec["size"]), _when(rec), ctype, remote={**rm, "space": rec.get("space")}) + return None + + def _texts(self, rids: list[int]) -> dict[int, bytes]: + """Each asset's text, a line per segment ("Speaker: …" when it has one).""" + if not rids: + return {} + segs = self.db.rows( + "SELECT recording, idx, t0, text, local_speaker, speaker FROM segment WHERE recording IN $r", r=sorted(set(rids)) + ) + spk = { + s["id"]: s.get("name") or s.get("label") + for s in ( + self.db.rows( + "SELECT record::id(id) AS id, name, label FROM speaker WHERE id IN $ids", + ids=[R("speaker", x) for x in {s["speaker"] for s in segs if s.get("speaker")}], + ) + if segs + else [] + ) + } + lines: dict[int, list[tuple[Any, str]]] = {} + for s in segs: + text = " ".join(str(s.get("text") or "").split()) + if not text: + continue + who = s.get("local_speaker") or spk.get(s.get("speaker")) + lines.setdefault(s["recording"], []).append(((s.get("idx") or 0, s.get("t0") or 0), f"{who}: {text}" if who else text)) + return {rid: ("\n".join(t for _, t in sorted(ls)) + "\n").encode() for rid, ls in lines.items()} + + def _about(self, rids: list[int]) -> tuple[dict[int, list[str]], dict[int, list[dict[str, str]]]]: + """Each asset's accepted topics, and the entities it mentions.""" + from .entities import QUIET, TYPES + + if not rids: + return {}, {} + ids = sorted(set(rids)) + about = self.db.rows("SELECT topic, recording FROM topic_about WHERE recording IN $r AND status = 'accepted'", r=ids) + labels = { + t["id"]: t["label"] + for t in ( + self.db.rows("SELECT record::id(id) AS id, label FROM topic WHERE id IN $t", t=[R("topic", a["topic"]) for a in about]) + if about + else [] + ) + } + topics: dict[int, list[str]] = {} + for a in about: + if a["topic"] in labels: + topics.setdefault(a["recording"], []).append(labels[a["topic"]]) + ments = self.db.rows("SELECT recording, entity FROM mentions WHERE recording IN $r GROUP BY recording, entity", r=ids) + ents = { + e["id"]: e + for e in ( + self.db.rows( + "SELECT record::id(id) AS id, space, name, type, hidden FROM entity WHERE id IN $e", + e=[R("entity", m["entity"]) for m in ments], + ) + if ments + else [] + ) + if not e.get("hidden") and e.get("type") not in QUIET + } + own = entity_setup.labels(self.db, {e["space"] for e in ents.values()}) + named: dict[int, list[dict[str, str]]] = {} + for m in ments: + e = ents.get(m["entity"]) + if e: + kind = TYPES.get(e["type"]) or own.get(e["space"], {}).get(e["type"]) or e["type"].title() + named.setdefault(m["recording"], []).append({"name": e["name"], "type": kind}) + return ( + {k: sorted(v, key=str.casefold) for k, v in topics.items()}, + {k: sorted(v, key=lambda x: (x["type"], x["name"].casefold())) for k, v in named.items()}, + ) + + def _assets(self, rows: list[dict[str, Any]]) -> list[Entry]: + """The files of these assets: own file, metadata, text and PDF, each named `<title> [<id>]`.""" + if self._only is not None: + rows = [r for r in rows if r["id"] == self._only] + rids = [r["id"] for r in rows] + texts, (topics, ents) = self._texts(rids), self._about(rids) + cols = {c["id"]: c for sid in {r["space"] for r in rows} for c in hierarchy.of_space(self.db, sid)} if rows else {} + out: list[Entry] = [] + for rec in sorted(rows, key=lambda r: r["id"]): + rid, when = rec["id"], _when(rec) + base = f"{clean(rec.get('title') or 'Untitled')[: NAME_MAX - 20]} [{rid}]" + own, own_name = self._primary(rec), None + files = [] + if own: + ext = pathlib.PurePosixPath(own.name).suffix.lower() + own_name = own.name + own.name = base + (ext if ext not in (".json", ".txt") else f".original{ext}") + files.append(own) + text = texts.get(rid) + if text is not None: + files.append(Entry(f"{base}.txt", False, len(text), when, "text/plain; charset=utf-8", data=text)) + pdf = convert.rendition_path(self.cfg, rid) + if rec.get("source") == "document" and pdf.is_file() and not (own and own.name.endswith(".pdf")): + enc = keyring.is_encrypted(pdf) + try: + size = keyring.plain_size(self.db, self.cfg, pdf) if enc else pdf.stat().st_size + files.append(Entry(f"{base}.pdf", False, size, when, "application/pdf", path=str(pdf), encrypted=enc)) + except (keyring.Locked, keyring.Damaged): + pass + col = [] + c = cols.get(rec.get("collection")) + while c and len(col) < hierarchy.MAX_DEPTH + 1: + col.append(c["name"]) + c = cols.get(c.get("parent")) if c.get("parent") is not None else None + meta = { + "identifier": f"{store.API}/recordings/{rid}", + "id": rid, + "title": rec.get("title"), + "type": render.kind(rec), + "format": own.content_type if own else None, + "file": own.name if own else None, + "original_name": own_name if own else None, + "size": own.size if own else None, + "namespace": self._names.get(rec["space"]), + "collection": "/".join(col[::-1]) or None, + "created": rec.get("created_at"), + "date": rec.get("recorded_at"), + "modified": when, + "duration_seconds": round(rec["duration_ms"] / 1000, 3) if rec.get("duration_ms") else None, + "status": rec.get("status"), + "subject": topics.get(rid, []), + "entities": ents.get(rid, []), + "text": f"{base}.txt" if text is not None else None, + "metadata": metadata._split(rec), + } + data = _json({k: v for k, v in meta.items() if v not in (None, {}, [])}) + files.append(Entry(f"{base}.json", False, len(data), when, "application/json", data=data)) + out += files + return sorted(out, key=lambda e: e.name.casefold()) diff --git a/fastapi_backend/app/schemas/filetree.py b/fastapi_backend/app/schemas/filetree.py new file mode 100644 index 000000000..4ab032d3b --- /dev/null +++ b/fastapi_backend/app/schemas/filetree.py @@ -0,0 +1,20 @@ +"""The graph as folders and files (docs/file-tree.md).""" + +from __future__ import annotations + +from pydantic import Field + +from app.schemas.common import ResponseModel + + +class TreeEntry(ResponseModel): + name: str + folder: bool + modified: str | None = None + size: int | None = Field(None, description="a file's size in bytes") + content_type: str | None = None + + +class TreeFolder(ResponseModel): + path: str = Field(description="the folder, from the top: '' for the namespaces") + entries: list[TreeEntry] diff --git a/fastapi_backend/tests/api/test_file_tree.py b/fastapi_backend/tests/api/test_file_tree.py new file mode 100644 index 000000000..5d019c683 --- /dev/null +++ b/fastapi_backend/tests/api/test_file_tree.py @@ -0,0 +1,154 @@ +"""The graph as folders and files (docs/file-tree.md): namespaces, collections, topics and entities as folders, assets as +files with a metadata sidecar and their text, seen exactly as the web app shows them to each person.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import filetree, hierarchy, keyring, store, topics +from tests.api.test_mcp import tool, tool_error +from tests.api.test_oauth import _bearer, _grant +from tests.helpers import login, make_user, seed + +URL = "/api/v1/tree" + + +@pytest.fixture +def env(client, db, cfg, folder): + a, b, call = seed(db, cfg, folder) + t = topics.create(db, store.ns_id(db, "pods"), "Gene therapy") + topics.tag(db, t, [a]) + make_user(db, "admin@x.io", "admin password 1", admin=True) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + make_user(db, "out@x.io", "outsider password 1") + return { + "a": a, + "b": b, + "call": call, + "folder": folder, + "ha": login(client, "admin@x.io", "admin password 1"), + "hv": login(client, "vi@x.io", "viewer password 1"), + "hx": login(client, "out@x.io", "outsider password 1"), + } + + +def _names(client, h, path=""): + r = client.get(URL, headers=h, params={"path": path}) + assert r.status_code == 200, r.text + return [e["name"] for e in r.json()["entries"]] + + +def _read(client, h, path, **headers): + return client.get(f"{URL}/file", headers={**h, **headers}, params={"path": path}) + + +def test_namespaces_collections_topics_and_entities_are_folders(client, env): + ha = env["ha"] + assert _names(client, ha) == ["calls", "pods"] + assert _names(client, ha, "pods") == ["Collections", "Entities", "namespace.json", "Topics"] + assert _names(client, ha, "pods/Collections") == ["General"] + assert _names(client, ha, "pods/Collections/General") == [ + "ep1 [1].json", + "ep1 [1].original.txt", + "ep1 [1].txt", + "ep2 [2].json", + "ep2 [2].original.txt", + "ep2 [2].txt", + ] + assert _names(client, ha, "pods/Topics") == ["Gene therapy"] + assert _names(client, ha, "pods/Topics/Gene therapy") == ["ep1 [1].json", "ep1 [1].original.txt", "ep1 [1].txt"] + assert "Organisation" in _names(client, ha, "pods/Entities") + assert "Dyno Therapeutics" in _names(client, ha, "pods/Entities/Organisation") + assert "ep2 [2].txt" in _names(client, ha, "pods/Entities/Organisation/Dyno Therapeutics") + # a file isn't a folder, nor the other way round; what isn't there is 404 + assert client.get(URL, headers=ha, params={"path": "pods/namespace.json"}).status_code == 400 + assert client.get(URL, headers=ha, params={"path": "pods/Collections/General/ep1 [1].txt"}).status_code == 400 + assert _read(client, ha, "pods/Collections").status_code == 400 + assert client.get(URL, headers=ha, params={"path": "pods/Nothing"}).status_code == 404 + assert _read(client, ha, "pods/Collections/General/nope.txt").status_code == 404 + + +def test_an_assets_files(client, env, folder): + ha = env["ha"] + base = "pods/Collections/General/ep1 [1]" + text = _read(client, ha, f"{base}.txt") + assert text.status_code == 200 and text.headers["content-type"].startswith("text/plain") + assert text.text.startswith("Alice: Welcome back.") + meta = _read(client, ha, f"{base}.json").json() + assert meta["id"] == env["a"] and meta["title"] == "ep1" and meta["namespace"] == "pods" and meta["collection"] == "General" + assert meta["subject"] == ["Gene therapy"] + assert {"name": "Dyno Therapeutics", "type": "Organisation"} in meta["entities"] + assert meta["text"] == "ep1 [1].txt" and meta["original_name"] == "ep1.txt" + # its own file, as it was imported, with byte ranges + own = (folder / "ep1.txt").read_bytes() + assert _read(client, ha, f"{base}.original.txt").content == own + part = _read(client, ha, f"{base}.original.txt", range="bytes=5-14") + assert part.status_code == 206 and part.content == own[5:15] and part.headers["content-range"] == f"bytes 5-14/{len(own)}" + # the size listed is the size read + listed = {e["name"]: e["size"] for e in client.get(URL, headers=ha, params={"path": "pods/Collections/General"}).json()["entries"]} + for name, size in listed.items(): + assert len(_read(client, ha, f"pods/Collections/General/{name}").content) == size, name + ns = _read(client, ha, "pods/namespace.json").json() + assert ns["namespace"] == "pods" and ns["topics"] == 1 and ns["collections"][0]["path"] == "General" + + +def test_people_see_what_the_web_app_shows_them(client, env, db, folder): + hv, hx = env["hv"], env["hx"] + assert _names(client, hv) == ["pods"] + assert client.get(URL, headers=hv, params={"path": "calls"}).status_code == 404 + assert _read(client, hv, "calls/namespace.json").status_code == 404 + assert _names(client, hx) == [] + assert client.get(URL).status_code == 401 + # a role on one collection: just that collection, and the topics and entities of what's in it + pods = store.ns_id(db, "pods") + talks = hierarchy.create(db, pods, "Talks") + db.q("UPDATE $r SET collection = $c", r=store.R("recording", env["b"]), c=talks) + hierarchy.give(db, talks, _account(db, "out@x.io"), "viewer") + assert _names(client, hx) == ["pods"] + assert _names(client, hx, "pods/Collections") == ["Talks"] + assert _names(client, hx, "pods/Collections/Talks") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] + assert _names(client, hx, "pods/Topics") == [] # Gene therapy is about ep1, which they can't see + assert _names(client, hx, "pods/Entities/Organisation/Dyno Therapeutics") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] + assert _read(client, hx, "pods/Collections/General/ep1 [1].txt").status_code == 404 + + +def _account(db, email): + return db.one("SELECT record::id(id) AS id FROM account WHERE email = $e", e=email)["id"] + + +def test_names_are_safe_and_unique(db, cfg, env): + pods = store.ns_id(db, "pods") + hierarchy.create(db, pods, 'a/b: "c"?') + db.q("UPDATE $r SET title = $t", r=store.R("recording", env["a"]), t="Q3: plans / ideas?") + t = filetree.Tree(db, cfg, filetree.Viewer(admin=True, roles={pods: "owner"})) + assert [e.name for e in t.list("pods/Collections")] == ["a_b_ _c__", "General"] + assert "Q3_ plans _ ideas_ [1].txt" in [e.name for e in t.list("pods/Collections/General")] + assert filetree._unique([("Talks", 1), ("talks", 2)]) == [("Talks", 1), ("talks [2]", 2)] + assert filetree.clean("..") == "_" and filetree.clean("x. ") == "x" and len(filetree.clean("y" * 500)) == filetree.NAME_MAX + + +def test_encrypted_files_are_read_plain(client, db, cfg, env, folder): + """An asset's own file kept encrypted (a namespace's key) reads as it was, and is listed at its plain size.""" + own = (folder / "ep1.txt").read_bytes() + keyring.encrypt_file(db, cfg, store.ns_id(db, "pods"), str(folder / "ep1.txt"), force=True) + assert keyring.is_encrypted(folder / "ep1.txt") + got = _read(client, env["ha"], "pods/Collections/General/ep1 [1].original.txt") + assert got.status_code == 200 and got.content == own + sizes = { + e["name"]: e["size"] for e in client.get(URL, headers=env["ha"], params={"path": "pods/Collections/General"}).json()["entries"] + } + assert sizes["ep1 [1].original.txt"] == len(own) + + +def test_agents_browse_it_over_mcp(client, env): + _, tokens = _grant(client, env["hv"], scope="read") + h = _bearer(tokens) + assert [e["name"] for e in tool(client, h, "list_files", path="")["entries"]] == ["pods"] + got = tool(client, h, "read_file", path="pods/Topics/Gene therapy/ep1 [1].json") + assert json.loads(got["text"])["subject"] == ["Gene therapy"] + own = tool(client, h, "read_file", path="pods/Collections/General/ep1 [1].original.txt") + assert "text" not in own and own["download"].startswith("/api/v1/tree/file?path=") + assert "a file" in tool_error(client, h, "list_files", path="pods/namespace.json") + assert "not found" in tool_error(client, h, "read_file", path="calls/namespace.json") diff --git a/mkdocs.yml b/mkdocs.yml index d1f90077a..896a32299 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - Topics: topics.md - Notes: notes.md - Storage: storage.md + - The graph as files: file-tree.md - Linked data (RDF): rdf.md - Fedora repository: fedora.md - API: api.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 32ab2f424..7c989f4b3 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -42,6 +42,7 @@ export { Templates, Tokens, Topics, + Tree, Updates, Uploads, Users, @@ -1449,6 +1450,11 @@ export type { ListFilesErrors, ListFilesResponse, ListFilesResponses, + ListFolderData, + ListFolderError, + ListFolderErrors, + ListFolderResponse, + ListFolderResponses, ListGrantsData, ListGrantsResponse, ListGrantsResponses, @@ -2017,6 +2023,11 @@ export type { RdfImportItem, RdfImportResult, RdfUnmatched, + ReadFileData, + ReadFileError, + ReadFileErrors, + ReadFileResponse, + ReadFileResponses, Reading, Recording, RecordingAccess, @@ -2617,6 +2628,8 @@ export type { TranscribeClipData, TranscribeClipResponse, TranscribeClipResponses, + TreeEntry, + TreeFolder, TryWorkflowData, TryWorkflowError, TryWorkflowErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 6e5801f34..8e10dea79 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -727,6 +727,9 @@ import type { ListFilesData, ListFilesErrors, ListFilesResponses, + ListFolderData, + ListFolderErrors, + ListFolderResponses, ListGrantsData, ListGrantsResponses, ListGraphChangesData, @@ -988,6 +991,9 @@ import type { QueueStepData, QueueStepErrors, QueueStepResponses, + ReadFileData, + ReadFileErrors, + ReadFileResponses, RecordingTopicsData, RecordingTopicsErrors, RecordingTopicsResponses, @@ -4676,6 +4682,37 @@ export class Files { } } +export class Tree { + /** + * List Folder + * + * A folder's files and folders: '' lists your namespaces; each has Collections, Topics, Entities and + * namespace.json. A file answers 400 (read it at /tree/file). + */ + public static listFolder<ThrowOnError extends boolean = false>( + options?: Options<ListFolderData, ThrowOnError>, + ): RequestResult<ListFolderResponses, ListFolderErrors, ThrowOnError> { + return (options?.client ?? client).get<ListFolderResponses, ListFolderErrors, ThrowOnError>({ + url: "/api/v1/tree", + ...options, + }); + } + + /** + * Read File + * + * A file of the tree, with byte ranges. + */ + public static readFile<ThrowOnError extends boolean = false>( + options?: Options<ReadFileData, ThrowOnError>, + ): RequestResult<ReadFileResponses, ReadFileErrors, ThrowOnError> { + return (options?.client ?? client).get<ReadFileResponses, ReadFileErrors, ThrowOnError>({ + url: "/api/v1/tree/file", + ...options, + }); + } +} + export class Fields { /** * List Fields diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index fd83bbb05..9214b4c32 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -15847,6 +15847,52 @@ export type TopicUpdate = { related?: Array<number> | null; }; +/** + * TreeEntry + */ +export type TreeEntry = { + /** + * Name + */ + name: string; + /** + * Folder + */ + folder: boolean; + /** + * Modified + */ + modified?: string | null; + /** + * Size + * + * a file's size in bytes + */ + size?: number | null; + /** + * Content Type + */ + content_type?: string | null; + [key: string]: unknown; +}; + +/** + * TreeFolder + */ +export type TreeFolder = { + /** + * Path + * + * the folder, from the top: '' for the namespaces + */ + path: string; + /** + * Entries + */ + entries: Array<TreeEntry>; + [key: string]: unknown; +}; + /** * TunnelStatus */ @@ -23277,6 +23323,74 @@ export type ListFileLinesResponses = { export type ListFileLinesResponse = ListFileLinesResponses[keyof ListFileLinesResponses]; +export type ListFolderData = { + body?: never; + path?: never; + query?: { + /** + * Path + * + * a folder or file, from the top: namespace/Collections/… + */ + path?: string; + }; + url: "/api/v1/tree"; +}; + +export type ListFolderErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ListFolderError = ListFolderErrors[keyof ListFolderErrors]; + +export type ListFolderResponses = { + /** + * Successful Response + */ + 200: TreeFolder; +}; + +export type ListFolderResponse = ListFolderResponses[keyof ListFolderResponses]; + +export type ReadFileData = { + body?: never; + path?: never; + query?: { + /** + * Path + * + * a folder or file, from the top: namespace/Collections/… + */ + path?: string; + }; + url: "/api/v1/tree/file"; +}; + +export type ReadFileErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ReadFileError = ReadFileErrors[keyof ReadFileErrors]; + +export type ReadFileResponses = { + /** + * Successful Response + */ + 200: Blob | File; + /** + * a byte range + */ + 206: unknown; +}; + +export type ReadFileResponse = ReadFileResponses[keyof ReadFileResponses]; + export type ListFieldsData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index cfe128052..72ffa8601 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -9469,6 +9469,99 @@ } } }, + "/api/v1/tree": { + "get": { + "tags": [ + "tree" + ], + "summary": "List Folder", + "description": "A folder's files and folders: '' lists your namespaces; each has Collections, Topics, Entities and\nnamespace.json. A file answers 400 (read it at /tree/file).", + "operationId": "list_folder", + "parameters": [ + { + "name": "path", + "in": "query", + "required": false, + "schema": { + "type": "string", + "maxLength": 4000, + "description": "a folder or file, from the top: namespace/Collections/\u2026", + "default": "", + "title": "Path" + }, + "description": "a folder or file, from the top: namespace/Collections/\u2026" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeFolder" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/tree/file": { + "get": { + "tags": [ + "tree" + ], + "summary": "Read File", + "description": "A file of the tree, with byte ranges.", + "operationId": "read_file", + "parameters": [ + { + "name": "path", + "in": "query", + "required": false, + "schema": { + "type": "string", + "maxLength": 4000, + "description": "a folder or file, from the top: namespace/Collections/\u2026", + "default": "", + "title": "Path" + }, + "description": "a folder or file, from the top: namespace/Collections/\u2026" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/octet-stream": {} + } + }, + "206": { + "description": "a byte range" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/fields": { "get": { "tags": [ @@ -51754,6 +51847,82 @@ "type": "object", "title": "TopicUpdate" }, + "TreeEntry": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "folder": { + "type": "boolean", + "title": "Folder" + }, + "modified": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Modified" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Size", + "description": "a file's size in bytes" + }, + "content_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Content Type" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "name", + "folder" + ], + "title": "TreeEntry" + }, + "TreeFolder": { + "properties": { + "path": { + "type": "string", + "title": "Path", + "description": "the folder, from the top: '' for the namespaces" + }, + "entries": { + "items": { + "$ref": "#/components/schemas/TreeEntry" + }, + "type": "array", + "title": "Entries" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "path", + "entries" + ], + "title": "TreeFolder" + }, "TunnelStatus": { "properties": { "mode": { From 46bdfd082b6446ca87dad4bd87a522809f3c91b3 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad <noreply@anthropic.com> Date: Sat, 10 Oct 2026 20:26:44 +0000 Subject: [PATCH 2/3] refactor(tree): assets by namespace and collection, not graph folders Per Adeel: drop the Topics and Entities folders and namespace.json; the tree is namespaces and collections holding each asset's files and sidecar. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- CHANGELOG.md | 8 +- docs/api.md | 4 +- docs/file-tree.md | 29 ++-- docs/mcp.md | 4 +- fastapi_backend/app/api/mcp_tools.py | 13 +- fastapi_backend/app/api/v1/routes/filetree.py | 10 +- fastapi_backend/app/domain/filetree.py | 135 ++---------------- fastapi_backend/app/schemas/filetree.py | 2 +- fastapi_backend/tests/api/test_file_tree.py | 63 ++++---- mkdocs.yml | 2 +- nextjs-frontend/app/openapi-client/sdk.gen.ts | 4 +- .../app/openapi-client/types.gen.ts | 4 +- nextjs-frontend/openapi.json | 10 +- 13 files changed, 79 insertions(+), 209 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57d29ba56..456d00ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,10 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased -- **The graph as files.** Namespaces, collections, topics and entities are folders, and each asset is its own file, a - `.json` with its metadata and a `.txt` with its text, in every folder it belongs to; everyone sees what the web app - shows them. Browse and read it through the API (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` - ([The graph as files](docs/file-tree.md)). The S3, SMB and NFS servers that share it are in progress. +- **Assets as files.** Namespaces and collections are folders, and each asset is its own file with a `.json` sidecar + of its metadata and a `.txt` of its text, in its collection's folder; everyone sees what the web app shows them. + Browse and read it through the API (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` + ([Assets as files](docs/file-tree.md)). S3, WebDAV, SMB and NFS servers that share it are in progress. - **Install on Windows, or by double-clicking.** `install.ps1` installs Lens on Windows: it gets Python 3 if needed and opens the web installer, which installs Docker Desktop (and WSL) and runs Lens in it. Each release now carries diff --git a/docs/api.md b/docs/api.md index 9859b4f87..68f2b20bd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -594,14 +594,14 @@ page was before each change, and `restore` puts an earlier version back (what it lists the files and images attached to a page; `blobs` uploads (in chunks) and reads them, and `DELETE …/files?key=` removes one. -## the graph as files +## assets as files ``` GET /api/v1/tree?path= GET /api/v1/tree/file?path= ``` -The archive as folders and files ([The graph as files](file-tree.md)): `tree` lists a folder (an empty path lists your +The archive as folders and files ([Assets as files](file-tree.md)): `tree` lists a folder (an empty path lists your namespaces), `tree/file` reads a file with byte ranges. Read-only; everyone sees what they can read in the web app. ## comments diff --git a/docs/file-tree.md b/docs/file-tree.md index 64c910317..d0f169e51 100644 --- a/docs/file-tree.md +++ b/docs/file-tree.md @@ -1,13 +1,13 @@ -# The graph as files +# Assets as files -Lens shows your archive as an ordinary tree of folders and files: namespaces, collections, topics and entities are -folders, and every asset (a recording, document or image) is a few files in each folder it belongs to. Agents and -tools that already understand files can read Lens without learning its API. +Lens shows your archive's assets as an ordinary tree of folders and files: namespaces and collections are folders, +and every asset (a recording, document or image) is a few files in its collection's folder, its metadata in a +sidecar beside it. Agents and tools that already understand files can read Lens without learning its API. | Where | Status | | --- | --- | | The API (`GET /api/v1/tree`, `GET /api/v1/tree/file`) and the MCP tools `list_files` and `read_file` | built | -| An S3 server (read-only, off until turned on) | in progress | +| S3 and WebDAV servers (read-only, off until turned on) | in progress | | SMB and NFS shares | in progress | Every way in serves the same tree, built by `app/domain/filetree.py`. @@ -15,12 +15,7 @@ Every way in serves the same tree, built by `app/domain/filetree.py`. ## Layout ```text -<namespace>/ - namespace.json the namespace: its collections, topic and entity counts - Collections/<collection>/<…>/<asset files> every asset in its collection's folder - Topics/<topic>/<asset files> the assets about a topic (accepted, not suggestions) - Entities/<type>/<name>/<asset files> the assets that mention an entity (hidden entities, dates and - numbers left out) +<namespace>/<collection>/<collection inside it>/…/<asset files> ``` An asset's files are named after it, `<title> [<id>]`, so names are unique and stable: @@ -32,9 +27,8 @@ An asset's files are named after it, `<title> [<id>]`, so names are unique and s | `<title> [<id>].txt` | its text: the transcript (`Speaker: …` lines) or what was read off its pages | | `<title> [<id>].pdf` | the PDF made of a document that isn't one (a Word file, an email…) | -The same asset shows in its collection's folder, in each topic's folder and in each entity's folder. Files stored -encrypted ([Encryption](encryption.md)) are read decrypted, and a locked vault's files can't be read until one of its -owners unlocks it. Files kept on a storage connection are read from there. +Files stored encrypted ([Encryption](encryption.md)) are read decrypted, and a locked vault's files can't be read until +one of its owners unlocks it. Files kept on a storage connection are read from there. Names are made safe for every file system: `/ \ : * ? " < > |` and control characters become `_`, names are cut to 120 characters, and two names in a folder that differ only in case get the second one's id added (`Talks [12]`). @@ -42,12 +36,11 @@ Names are made safe for every file system: `/ \ : * ? " < > |` and control chara ## Who sees what Exactly what the web app shows them ([Access](access.md)): an admin sees every namespace; others see the namespaces -they have a role in, and in a namespace where they only have roles on some collections, those collections, and only -the topics and entities of the assets in them. Anything else answers "not found". The tree only reads. +they have a role in, and in a namespace where they only have roles on some collections, those collections. Anything else answers "not found". The tree only reads. ## The API -`GET /api/v1/tree?path=pods/Topics/Gene therapy` lists a folder: `{path, entries: [{name, folder, modified, size, +`GET /api/v1/tree?path=pods/General` lists a folder: `{path, entries: [{name, folder, modified, size, content_type}]}`. An empty path lists your namespaces. `GET /api/v1/tree/file?path=…` reads a file, with byte ranges (`Range: bytes=0-1023`) and an `ETag`. Asking for a file as a folder (or the other way round) answers 400. @@ -58,4 +51,4 @@ The MCP server has `list_files` and `read_file` ([MCP server](mcp.md)): `read_fi - Show supplementary files (captions, translations, attachments) beside each asset. - Cache an asset's text and metadata so large folders list faster. -- `Speakers/<name>/` folders, and a folder of notes. +- A sidecar for each collection, and a folder of notes. diff --git a/docs/mcp.md b/docs/mcp.md index 3bbf8abc1..14be9df58 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -62,8 +62,8 @@ front of the web app also needs `TRUST_PROXY_HEADERS=true` ([Authentication](aut | `write_note` | write a note or a thing's page, marked as written by an assistant. Needs a write-scope token and editor access | | `create_podcast` | make a two-host learning [episode](podcasts.md) about recordings or lines in them, with an angle, a length and a style; it's made in the background. Needs a write-scope token and editor access where it goes | | `get_podcast` | an episode: how far it has got, its script with each line's speaker and the sources it cites (links), and what the fact-check changed | -| `list_files` | a folder of the archive [as files](file-tree.md): your namespaces, then each one's Collections, Topics and Entities folders holding its assets' files | -| `read_file` | a text file of that tree (an asset's `.txt` text or `.json` metadata, `namespace.json`), a page at a time; other files' download link | +| `list_files` | a folder of the archive's [assets as files](file-tree.md): your namespaces, then their collections, each holding its assets' files | +| `read_file` | a text file of that tree (an asset's `.txt` text or `.json` metadata), a page at a time; other files' download link | | `file_storage` | where Lens keeps its own files (notes' images and attachments) and the connections it could use; `check` tests it. Admins only | | `set_file_storage` | keep new files on this machine or a storage connection (folder, rclone crypt); checked before it's saved. Admins only, with a write-scope token | | `propose_graph_change` | ask for two entities to be merged or linked; it waits in Proposed changes unless `apply` makes it at once (it can be undone). Needs a write-scope token and editor access | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 7defd3efc..93130fad3 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -1358,7 +1358,7 @@ def get_podcast(ctx: Context, episode_id: int) -> dict[str, Any]: ) -# ---------- the graph as files ---------- +# ---------- assets as files ---------- FILE_CHARS = 100_000 # the most of a file read_file returns at once; offset pages through the rest @@ -1369,11 +1369,10 @@ def _tree(ctx: Context) -> filetree.Tree: @tool( "list_files", "List a folder of the file tree", - "The archive as folders and files, the tree its S3 server serves: '' lists your namespaces; a namespace has " - "Collections/<collection>/…, Topics/<topic>/ and Entities/<type>/<name>/ holding its assets, and namespace.json. " - "Each asset is files named '<title> [<id>]': its own file (.mp3, .pdf, …), .json with its metadata, topics and " - "entities, and .txt with its text.", - Arg("path", "string", "the folder, from the top (e.g. pods/Topics/Gene therapy)", default="", max_length=4000), + "The archive's assets as folders and files: '' lists your namespaces, then each one's collections and the " + "collections inside them. Each asset is files named '<title> [<id>]' in its collection's folder: its own file " + "(.mp3, .pdf, …), .json with its metadata, topics and entities, and .txt with its text.", + Arg("path", "string", "the folder, from the top (e.g. pods/General)", default="", max_length=4000), ) def list_files(ctx: Context, path: str) -> dict[str, Any]: t = _tree(ctx) @@ -1389,7 +1388,7 @@ def list_files(ctx: Context, path: str) -> dict[str, Any]: @tool( "read_file", "Read a file of the file tree", - "A text file of the tree (an asset's .txt or .json, or namespace.json), from list_files. Media and documents' own " + "A text file of the tree (an asset's .txt or .json), from list_files. Media and documents' own " f"files are only their size and type here: download them from the API (/api/v1/tree/file?path=). At most " f"{FILE_CHARS} characters at a time; next_offset continues.", Arg("path", "string", "the file, from the top", required=True, max_length=4000), diff --git a/fastapi_backend/app/api/v1/routes/filetree.py b/fastapi_backend/app/api/v1/routes/filetree.py index 35600704a..b4369faf6 100644 --- a/fastapi_backend/app/api/v1/routes/filetree.py +++ b/fastapi_backend/app/api/v1/routes/filetree.py @@ -1,5 +1,5 @@ -"""The graph as folders and files (docs/file-tree.md): browse the tree the S3, SMB and NFS servers serve, and read -its files. Everyone sees what the web app shows them; the tree only reads.""" +"""Assets as folders and files (docs/file-tree.md): browse the tree the S3, WebDAV, SMB and NFS servers serve, and +read its files. Everyone sees what the web app shows them; the tree only reads.""" from __future__ import annotations @@ -15,7 +15,7 @@ from app.schemas.filetree import TreeEntry, TreeFolder router = APIRouter(prefix="/tree", tags=["tree"]) -PATH = Query("", max_length=4000, description="a folder or file, from the top: namespace/Collections/…") +PATH = Query("", max_length=4000, description="a folder or file, from the top: namespace/collection/…") def tree_of(db: DB, cfg: dict[str, Any], user: Principal) -> filetree.Tree: @@ -24,8 +24,8 @@ def tree_of(db: DB, cfg: dict[str, Any], user: Principal) -> filetree.Tree: @router.get("") def list_folder(user: CurrentUser, db: Db, cfg: Cfg, path: str = PATH) -> TreeFolder: - """A folder's files and folders: '' lists your namespaces; each has Collections, Topics, Entities and - namespace.json. A file answers 400 (read it at /tree/file).""" + """A folder's files and folders: '' lists your namespaces, then their collections, each holding its assets' + files. A file answers 400 (read it at /tree/file).""" t = tree_of(db, cfg, user) try: entries = t.list(path) diff --git a/fastapi_backend/app/domain/filetree.py b/fastapi_backend/app/domain/filetree.py index 40c68f447..3d796205b 100644 --- a/fastapi_backend/app/domain/filetree.py +++ b/fastapi_backend/app/domain/filetree.py @@ -1,21 +1,16 @@ -"""The graph as folders and files (docs/file-tree.md): what the S3, SMB and NFS servers serve, and the API browses. +"""Assets as folders and files (docs/file-tree.md): what the S3, WebDAV, SMB and NFS servers serve, and the API browses. - /<namespace>/ - namespace.json the namespace: its collections, topic and entity counts - Collections/<collection>/<…>/<asset> every asset in its collection's folder - Topics/<topic>/<asset> the assets about a topic (accepted ones) - Entities/<type>/<name>/<asset> the assets that mention an entity + /<namespace>/<collection>/<sub-collection>/…/<asset files> -An asset (a recording, document or image) shows as up to four files named after it, `<title> [<id>]`: -its own file (`.mp3`, `.pdf`, …), `.json` beside it with its metadata (Dublin Core terms, its topics and entities), -`.txt` with its text (the transcript, or what was read off its pages), and `.pdf` for a document converted to one. -An own file that is itself `.txt` or `.json` (an imported transcript) is `<title> [<id>].original.txt`. -The same asset shows in every folder it belongs to. +An asset (a recording, document or image) shows as up to four files in its collection's folder, named after it, +`<title> [<id>]`: its own file (`.mp3`, `.pdf`, …), `.json` beside it with its metadata (Dublin Core terms, its +collection, topics and entities, the description saved on it), `.txt` with its text (the transcript, or what was read +off its pages), and `.pdf` for a document converted to one. An own file that is itself `.txt` or `.json` (an imported +transcript) is `<title> [<id>].original.txt`. What someone sees is what the web app shows them: the namespaces they have a role in (admins: all of them), and in a -namespace where they only have roles on some collections, those collections and the topics and entities of the -assets in them. The tree only reads; names are made safe for every file system (no `/ \\ : * ? " < > |`) and unique -in their folder, ignoring case. +namespace where they only have roles on some collections, those collections. The tree only reads; names are made +safe for every file system (no `/ \\ : * ? " < > |`) and unique in their folder, ignoring case. """ from __future__ import annotations @@ -33,8 +28,6 @@ from . import auth, convert, documents, entity_setup, hierarchy, keyring, metadata, render, store, video R = store.R -COLLECTIONS, TOPICS, ENTITIES = "Collections", "Topics", "Entities" -NS_FILE = "namespace.json" NAME_MAX = 120 CHUNK = 1 << 16 ASSET_ID = re.compile(r" \[(\d+)\](?:\.[^.\[\]]+)+$") @@ -183,7 +176,6 @@ def __init__(self, db, cfg, viewer: Viewer): self._spaces = viewer.spaces(db) self._names = {sid: n for sid, n in store.space_names(db).items() if sid in self._spaces} self._cols: dict[int, list[dict[str, Any]]] = {} - self._seen: dict[int, set[int]] = {} self._only: int | None = None # ---------- walking ---------- @@ -203,41 +195,16 @@ def list(self, path: str) -> list[Entry]: def _list(self, parts: list[str]) -> list[Entry]: if not parts: return [_folder(n) for n in sorted(self._names.values(), key=str.casefold)] - sid = self._space(parts[0]) - rest = parts[1:] - if not rest: - return [_folder(COLLECTIONS), _folder(ENTITIES), self._ns_file(sid), _folder(TOPICS)] - head, rest = rest[0], rest[1:] - if head == COLLECTIONS: - return self._list_collection(sid, rest) - if head == TOPICS: - topics = self._topics(sid) - if not rest: - return [_folder(n) for n in topics] - if len(rest) == 1 and rest[0] in topics: - return self._assets(self._topic_recordings(sid, topics[rest[0]])) - if head == ENTITIES: - types = self._types(sid) - if not rest: - return [_folder(n) for n in types] - if rest[0] in types: - ents = self._entities(sid, types[rest[0]]) - if len(rest) == 1: - return [_folder(n) for n in ents] - if len(rest) == 2 and rest[1] in ents: - return self._assets(self._entity_recordings(sid, ents[rest[1]])) - raise NotFound("/".join(parts)) + return self._list_collection(self._space(parts[0]), parts[1:]) def stat(self, path: str) -> Entry: """One file or folder; NotFound when there's none (or it isn't this person's to see).""" parts = self.split(path) if not parts: return _folder("") - sid = self._space(parts[0]) + self._space(parts[0]) if len(parts) == 1: return _folder(parts[0]) - if len(parts) == 2 and parts[1] == NS_FILE: - return self._ns_file(sid) *up, name = parts m = ASSET_ID.search(name) self._only = int(m.group(1)) if m else None # one asset's files: don't make the whole folder's @@ -299,86 +266,6 @@ def _list_collection(self, sid: int, names: list[str]) -> list[Entry]: out += self._assets(self.db.rows(f"SELECT {REC_FIELDS} FROM recording WHERE collection = $c", c=cid)) return sorted(out, key=lambda e: e.name.casefold()) - def _visible(self, sid: int, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - seen = self._spaces.get(sid) - return [r for r in rows if r.get("space") == sid and (seen is None or r.get("collection") in seen)] - - def _recordings(self, sid: int, rids: list[int]) -> list[dict[str, Any]]: - if not rids: - return [] - rows = self.db.rows(f"SELECT {REC_FIELDS} FROM recording WHERE id IN $ids", ids=[R("recording", int(r)) for r in set(rids)]) - return self._visible(sid, rows) - - def _ns_file(self, sid: int) -> Entry: - cols = hierarchy.tree(self.db, sid, only=self._spaces[sid]) - d = { - "namespace": self._names[sid], - "collections": [{"path": "/".join(c["path"]), "assets": c["recordings"], "id": c["id"]} for c in cols], - "topics": len(self._topics(sid)), - "entity_types": list(self._types(sid)), - "layout": { - COLLECTIONS: "every asset in its collection's folder", - TOPICS: "the assets about each topic", - ENTITIES: "the assets that mention each entity, by type", - }, - } - data = _json(d) - return Entry(NS_FILE, False, len(data), None, "application/json", data=data) - - # ---------- topics and entities ---------- - def _topics(self, sid: int) -> dict[str, int]: - rows = self.db.rows("SELECT record::id(id) AS id, label FROM topic WHERE space = $s", s=sid) - if self._spaces[sid] is not None: # seen in part: the topics of the assets they see - have = { - a["topic"] - for a in self.db.rows("SELECT topic, recording FROM topic_about WHERE space = $s AND status = 'accepted'", s=sid) - if a["recording"] in self._seen_recordings(sid) - } - rows = [t for t in rows if t["id"] in have] - return dict(_unique([(clean(t["label"]), t["id"]) for t in rows])) - - def _topic_recordings(self, sid: int, tid: int) -> list[dict[str, Any]]: - rids = self.db.values("SELECT VALUE recording FROM topic_about WHERE topic = $t AND status = 'accepted'", t=tid) - return self._recordings(sid, rids) - - def _seen_recordings(self, sid: int) -> set[int]: - if sid not in self._seen: - self._seen[sid] = hierarchy.recordings_in(self.db, self._spaces[sid] or []) - return self._seen[sid] - - def _entity_rows(self, sid: int) -> list[dict[str, Any]]: - from .entities import QUIET - - rows = [ - e - for e in self.db.rows("SELECT record::id(id) AS id, name, type, hidden FROM entity WHERE space = $s", s=sid) - if not e.get("hidden") and e.get("type") not in QUIET - ] - if self._spaces[sid] is not None: - seen = self._seen_recordings(sid) - have = { - m["entity"] - for m in self.db.rows("SELECT entity, recording FROM mentions WHERE space = $s GROUP BY entity, recording", s=sid) - if m["recording"] in seen - } - rows = [e for e in rows if e["id"] in have] - return rows - - def _types(self, sid: int) -> dict[str, str]: - """{folder name: entity type} for the types this namespace's entities have.""" - from .entities import TYPES - - own = entity_setup.labels(self.db, {sid}).get(sid, {}) - codes = sorted({e["type"] for e in self._entity_rows(sid)}) - return dict(_unique([(clean(TYPES.get(c) or own.get(c) or c.title()), c) for c in codes])) - - def _entities(self, sid: int, code: str) -> dict[str, int]: - return dict(_unique([(clean(e["name"]), e["id"]) for e in self._entity_rows(sid) if e["type"] == code])) - - def _entity_recordings(self, sid: int, eid: int) -> list[dict[str, Any]]: - rids = [m["recording"] for m in self.db.rows("SELECT recording FROM mentions WHERE entity = $e GROUP BY recording", e=eid)] - return self._recordings(sid, rids) - # ---------- assets ---------- def _primary(self, rec: dict[str, Any]) -> Entry | None: """The asset's own file: stored here (maybe encrypted), cached from a storage source, or on one.""" diff --git a/fastapi_backend/app/schemas/filetree.py b/fastapi_backend/app/schemas/filetree.py index 4ab032d3b..28aa8e629 100644 --- a/fastapi_backend/app/schemas/filetree.py +++ b/fastapi_backend/app/schemas/filetree.py @@ -1,4 +1,4 @@ -"""The graph as folders and files (docs/file-tree.md).""" +"""Assets as folders and files (docs/file-tree.md).""" from __future__ import annotations diff --git a/fastapi_backend/tests/api/test_file_tree.py b/fastapi_backend/tests/api/test_file_tree.py index 5d019c683..a08f46dad 100644 --- a/fastapi_backend/tests/api/test_file_tree.py +++ b/fastapi_backend/tests/api/test_file_tree.py @@ -1,5 +1,5 @@ -"""The graph as folders and files (docs/file-tree.md): namespaces, collections, topics and entities as folders, assets as -files with a metadata sidecar and their text, seen exactly as the web app shows them to each person.""" +"""Assets as folders and files (docs/file-tree.md): namespaces and collections as folders, assets as files with a +metadata sidecar and their text, seen exactly as the web app shows them to each person.""" from __future__ import annotations @@ -44,12 +44,15 @@ def _read(client, h, path, **headers): return client.get(f"{URL}/file", headers={**h, **headers}, params={"path": path}) -def test_namespaces_collections_topics_and_entities_are_folders(client, env): +def test_namespaces_and_collections_are_folders(client, env, db): ha = env["ha"] + pods = store.ns_id(db, "pods") + talks = hierarchy.create(db, pods, "Talks") + hierarchy.create(db, pods, "2024", parent=talks) assert _names(client, ha) == ["calls", "pods"] - assert _names(client, ha, "pods") == ["Collections", "Entities", "namespace.json", "Topics"] - assert _names(client, ha, "pods/Collections") == ["General"] - assert _names(client, ha, "pods/Collections/General") == [ + assert _names(client, ha, "pods") == ["General", "Talks"] + assert _names(client, ha, "pods/Talks") == ["2024"] + assert _names(client, ha, "pods/General") == [ "ep1 [1].json", "ep1 [1].original.txt", "ep1 [1].txt", @@ -57,22 +60,16 @@ def test_namespaces_collections_topics_and_entities_are_folders(client, env): "ep2 [2].original.txt", "ep2 [2].txt", ] - assert _names(client, ha, "pods/Topics") == ["Gene therapy"] - assert _names(client, ha, "pods/Topics/Gene therapy") == ["ep1 [1].json", "ep1 [1].original.txt", "ep1 [1].txt"] - assert "Organisation" in _names(client, ha, "pods/Entities") - assert "Dyno Therapeutics" in _names(client, ha, "pods/Entities/Organisation") - assert "ep2 [2].txt" in _names(client, ha, "pods/Entities/Organisation/Dyno Therapeutics") # a file isn't a folder, nor the other way round; what isn't there is 404 - assert client.get(URL, headers=ha, params={"path": "pods/namespace.json"}).status_code == 400 - assert client.get(URL, headers=ha, params={"path": "pods/Collections/General/ep1 [1].txt"}).status_code == 400 - assert _read(client, ha, "pods/Collections").status_code == 400 + assert client.get(URL, headers=ha, params={"path": "pods/General/ep1 [1].txt"}).status_code == 400 + assert _read(client, ha, "pods/General").status_code == 400 assert client.get(URL, headers=ha, params={"path": "pods/Nothing"}).status_code == 404 - assert _read(client, ha, "pods/Collections/General/nope.txt").status_code == 404 + assert _read(client, ha, "pods/General/nope.txt").status_code == 404 def test_an_assets_files(client, env, folder): ha = env["ha"] - base = "pods/Collections/General/ep1 [1]" + base = "pods/General/ep1 [1]" text = _read(client, ha, f"{base}.txt") assert text.status_code == 200 and text.headers["content-type"].startswith("text/plain") assert text.text.startswith("Alice: Welcome back.") @@ -87,18 +84,16 @@ def test_an_assets_files(client, env, folder): part = _read(client, ha, f"{base}.original.txt", range="bytes=5-14") assert part.status_code == 206 and part.content == own[5:15] and part.headers["content-range"] == f"bytes 5-14/{len(own)}" # the size listed is the size read - listed = {e["name"]: e["size"] for e in client.get(URL, headers=ha, params={"path": "pods/Collections/General"}).json()["entries"]} + listed = {e["name"]: e["size"] for e in client.get(URL, headers=ha, params={"path": "pods/General"}).json()["entries"]} for name, size in listed.items(): - assert len(_read(client, ha, f"pods/Collections/General/{name}").content) == size, name - ns = _read(client, ha, "pods/namespace.json").json() - assert ns["namespace"] == "pods" and ns["topics"] == 1 and ns["collections"][0]["path"] == "General" + assert len(_read(client, ha, f"pods/General/{name}").content) == size, name def test_people_see_what_the_web_app_shows_them(client, env, db, folder): hv, hx = env["hv"], env["hx"] assert _names(client, hv) == ["pods"] assert client.get(URL, headers=hv, params={"path": "calls"}).status_code == 404 - assert _read(client, hv, "calls/namespace.json").status_code == 404 + assert _read(client, hv, "calls/General/call [3].txt").status_code == 404 assert _names(client, hx) == [] assert client.get(URL).status_code == 401 # a role on one collection: just that collection, and the topics and entities of what's in it @@ -107,11 +102,9 @@ def test_people_see_what_the_web_app_shows_them(client, env, db, folder): db.q("UPDATE $r SET collection = $c", r=store.R("recording", env["b"]), c=talks) hierarchy.give(db, talks, _account(db, "out@x.io"), "viewer") assert _names(client, hx) == ["pods"] - assert _names(client, hx, "pods/Collections") == ["Talks"] - assert _names(client, hx, "pods/Collections/Talks") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] - assert _names(client, hx, "pods/Topics") == [] # Gene therapy is about ep1, which they can't see - assert _names(client, hx, "pods/Entities/Organisation/Dyno Therapeutics") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] - assert _read(client, hx, "pods/Collections/General/ep1 [1].txt").status_code == 404 + assert _names(client, hx, "pods") == ["Talks"] + assert _names(client, hx, "pods/Talks") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] + assert _read(client, hx, "pods/General/ep1 [1].txt").status_code == 404 def _account(db, email): @@ -123,8 +116,8 @@ def test_names_are_safe_and_unique(db, cfg, env): hierarchy.create(db, pods, 'a/b: "c"?') db.q("UPDATE $r SET title = $t", r=store.R("recording", env["a"]), t="Q3: plans / ideas?") t = filetree.Tree(db, cfg, filetree.Viewer(admin=True, roles={pods: "owner"})) - assert [e.name for e in t.list("pods/Collections")] == ["a_b_ _c__", "General"] - assert "Q3_ plans _ ideas_ [1].txt" in [e.name for e in t.list("pods/Collections/General")] + assert [e.name for e in t.list("pods")] == ["a_b_ _c__", "General"] + assert "Q3_ plans _ ideas_ [1].txt" in [e.name for e in t.list("pods/General")] assert filetree._unique([("Talks", 1), ("talks", 2)]) == [("Talks", 1), ("talks [2]", 2)] assert filetree.clean("..") == "_" and filetree.clean("x. ") == "x" and len(filetree.clean("y" * 500)) == filetree.NAME_MAX @@ -134,11 +127,9 @@ def test_encrypted_files_are_read_plain(client, db, cfg, env, folder): own = (folder / "ep1.txt").read_bytes() keyring.encrypt_file(db, cfg, store.ns_id(db, "pods"), str(folder / "ep1.txt"), force=True) assert keyring.is_encrypted(folder / "ep1.txt") - got = _read(client, env["ha"], "pods/Collections/General/ep1 [1].original.txt") + got = _read(client, env["ha"], "pods/General/ep1 [1].original.txt") assert got.status_code == 200 and got.content == own - sizes = { - e["name"]: e["size"] for e in client.get(URL, headers=env["ha"], params={"path": "pods/Collections/General"}).json()["entries"] - } + sizes = {e["name"]: e["size"] for e in client.get(URL, headers=env["ha"], params={"path": "pods/General"}).json()["entries"]} assert sizes["ep1 [1].original.txt"] == len(own) @@ -146,9 +137,9 @@ def test_agents_browse_it_over_mcp(client, env): _, tokens = _grant(client, env["hv"], scope="read") h = _bearer(tokens) assert [e["name"] for e in tool(client, h, "list_files", path="")["entries"]] == ["pods"] - got = tool(client, h, "read_file", path="pods/Topics/Gene therapy/ep1 [1].json") + got = tool(client, h, "read_file", path="pods/General/ep1 [1].json") assert json.loads(got["text"])["subject"] == ["Gene therapy"] - own = tool(client, h, "read_file", path="pods/Collections/General/ep1 [1].original.txt") + own = tool(client, h, "read_file", path="pods/General/ep1 [1].original.txt") assert "text" not in own and own["download"].startswith("/api/v1/tree/file?path=") - assert "a file" in tool_error(client, h, "list_files", path="pods/namespace.json") - assert "not found" in tool_error(client, h, "read_file", path="calls/namespace.json") + assert "a file" in tool_error(client, h, "list_files", path="pods/General/ep1 [1].txt") + assert "not found" in tool_error(client, h, "read_file", path="calls/General/call [3].txt") diff --git a/mkdocs.yml b/mkdocs.yml index 896a32299..ae71ea409 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,7 +87,7 @@ nav: - Topics: topics.md - Notes: notes.md - Storage: storage.md - - The graph as files: file-tree.md + - Assets as files: file-tree.md - Linked data (RDF): rdf.md - Fedora repository: fedora.md - API: api.md diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 8e10dea79..8165d8950 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -4686,8 +4686,8 @@ export class Tree { /** * List Folder * - * A folder's files and folders: '' lists your namespaces; each has Collections, Topics, Entities and - * namespace.json. A file answers 400 (read it at /tree/file). + * A folder's files and folders: '' lists your namespaces, then their collections, each holding its assets' + * files. A file answers 400 (read it at /tree/file). */ public static listFolder<ThrowOnError extends boolean = false>( options?: Options<ListFolderData, ThrowOnError>, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 9214b4c32..70ebd3385 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -23330,7 +23330,7 @@ export type ListFolderData = { /** * Path * - * a folder or file, from the top: namespace/Collections/… + * a folder or file, from the top: namespace/collection/… */ path?: string; }; @@ -23362,7 +23362,7 @@ export type ReadFileData = { /** * Path * - * a folder or file, from the top: namespace/Collections/… + * a folder or file, from the top: namespace/collection/… */ path?: string; }; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 72ffa8601..5d23fd81d 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -9475,7 +9475,7 @@ "tree" ], "summary": "List Folder", - "description": "A folder's files and folders: '' lists your namespaces; each has Collections, Topics, Entities and\nnamespace.json. A file answers 400 (read it at /tree/file).", + "description": "A folder's files and folders: '' lists your namespaces, then their collections, each holding its assets'\nfiles. A file answers 400 (read it at /tree/file).", "operationId": "list_folder", "parameters": [ { @@ -9485,11 +9485,11 @@ "schema": { "type": "string", "maxLength": 4000, - "description": "a folder or file, from the top: namespace/Collections/\u2026", + "description": "a folder or file, from the top: namespace/collection/\u2026", "default": "", "title": "Path" }, - "description": "a folder or file, from the top: namespace/Collections/\u2026" + "description": "a folder or file, from the top: namespace/collection/\u2026" } ], "responses": { @@ -9532,11 +9532,11 @@ "schema": { "type": "string", "maxLength": 4000, - "description": "a folder or file, from the top: namespace/Collections/\u2026", + "description": "a folder or file, from the top: namespace/collection/\u2026", "default": "", "title": "Path" }, - "description": "a folder or file, from the top: namespace/Collections/\u2026" + "description": "a folder or file, from the top: namespace/collection/\u2026" } ], "responses": { From a6b56598e6685949a2e92cdb149bb46f73a949b9 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad <noreply@anthropic.com> Date: Sat, 10 Oct 2026 20:41:05 +0000 Subject: [PATCH 3/3] feat(tree): graph nodes as nested folders Collections nest by parent, topics by broader/narrower, entities by type; each folder holds its assets' files. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- CHANGELOG.md | 9 +- docs/api.md | 6 +- docs/file-tree.md | 24 ++- docs/mcp.md | 2 +- fastapi_backend/app/api/mcp_tools.py | 10 +- fastapi_backend/app/api/v1/routes/filetree.py | 4 +- fastapi_backend/app/domain/filetree.py | 149 ++++++++++++++++-- fastapi_backend/tests/api/test_file_tree.py | 71 ++++++--- mkdocs.yml | 2 +- .../app/openapi-client/types.gen.ts | 4 +- nextjs-frontend/openapi.json | 8 +- 11 files changed, 225 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c434b444..ca20c7a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased -- **Assets as files.** Namespaces and collections are folders, and each asset is its own file with a `.json` sidecar - of its metadata and a `.txt` of its text, in its collection's folder; everyone sees what the web app shows them. - Browse and read it through the API (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` - ([Assets as files](docs/file-tree.md)). S3, WebDAV, SMB and NFS servers that share it are in progress. +- **The graph as files.** Graph nodes are folders, nested along its links: collections inside collections, topics + inside their broader topics, entities by type. Each folder holds its assets' files, with a `.json` sidecar of + metadata and a `.txt` of the text; everyone sees what the web app shows them. Browse and read it through the API + (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` ([The graph as files](docs/file-tree.md)). + S3, WebDAV, SMB and NFS servers that share it are in progress. - **Prometheus metrics.** Lens can serve `/metrics` for Prometheus and Grafana: jobs by status and pipeline, imports, model calls, tokens, cost and failures by model, budgets, workers and disk. Off by default; turned on in Settings → diff --git a/docs/api.md b/docs/api.md index 02b550b24..4ae926995 100644 --- a/docs/api.md +++ b/docs/api.md @@ -594,15 +594,15 @@ page was before each change, and `restore` puts an earlier version back (what it lists the files and images attached to a page; `blobs` uploads (in chunks) and reads them, and `DELETE …/files?key=` removes one. -## assets as files +## the graph as files ``` GET /api/v1/tree?path= GET /api/v1/tree/file?path= ``` -The archive as folders and files ([Assets as files](file-tree.md)): `tree` lists a folder (an empty path lists your -namespaces), `tree/file` reads a file with byte ranges. Read-only; everyone sees what they can read in the web app. +The archive as folders and files ([The graph as files](file-tree.md)): `tree` lists a folder (an empty path lists your +namespaces; in each, `Collections`, `Topics` and `Entities`), `tree/file` reads a file with byte ranges. Read-only; everyone sees what they can read in the web app. ## comments diff --git a/docs/file-tree.md b/docs/file-tree.md index d0f169e51..6ae8d42df 100644 --- a/docs/file-tree.md +++ b/docs/file-tree.md @@ -1,8 +1,8 @@ -# Assets as files +# The graph as files -Lens shows your archive's assets as an ordinary tree of folders and files: namespaces and collections are folders, -and every asset (a recording, document or image) is a few files in its collection's folder, its metadata in a -sidecar beside it. Agents and tools that already understand files can read Lens without learning its API. +Lens shows its graph as an ordinary tree of folders and files: graph nodes (collections, topics, entities) are +folders, nested along the graph's links, and every asset (a recording, document or image) is a few files inside the +folders it belongs to, its metadata in a sidecar beside it. Agents and tools that already understand files can read Lens without learning its API. | Where | Status | | --- | --- | @@ -15,9 +15,15 @@ Every way in serves the same tree, built by `app/domain/filetree.py`. ## Layout ```text -<namespace>/<collection>/<collection inside it>/…/<asset files> +<namespace>/ + Collections/<collection>/<collection inside it>/… every asset, in its collection's folder + Topics/<topic>/<narrower topic>/… the assets about each topic (accepted ones) + Entities/<type>/<name>/ the assets that mention each entity ``` +A folder holds the folders under it and its assets' files side by side. The same asset shows in every folder it +belongs to, and a topic with several broader topics shows under each of them. + An asset's files are named after it, `<title> [<id>]`, so names are unique and stable: | File | What it holds | @@ -36,11 +42,12 @@ Names are made safe for every file system: `/ \ : * ? " < > |` and control chara ## Who sees what Exactly what the web app shows them ([Access](access.md)): an admin sees every namespace; others see the namespaces -they have a role in, and in a namespace where they only have roles on some collections, those collections. Anything else answers "not found". The tree only reads. +they have a role in, and in a namespace where they only have roles on some collections, those collections and the +topics and entities of the assets in them (with the topics broader than those). Anything else answers "not found". The tree only reads. ## The API -`GET /api/v1/tree?path=pods/General` lists a folder: `{path, entries: [{name, folder, modified, size, +`GET /api/v1/tree?path=pods/Topics/Gene therapy` lists a folder: `{path, entries: [{name, folder, modified, size, content_type}]}`. An empty path lists your namespaces. `GET /api/v1/tree/file?path=…` reads a file, with byte ranges (`Range: bytes=0-1023`) and an `ETag`. Asking for a file as a folder (or the other way round) answers 400. @@ -51,4 +58,5 @@ The MCP server has `list_files` and `read_file` ([MCP server](mcp.md)): `read_fi - Show supplementary files (captions, translations, attachments) beside each asset. - Cache an asset's text and metadata so large folders list faster. -- A sidecar for each collection, and a folder of notes. +- A sidecar for each collection, topic and entity, and a folder of notes. +- Nest entities by how they relate (people under organisations) once the graph records relations between them. diff --git a/docs/mcp.md b/docs/mcp.md index dc8b84d33..52330102b 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -65,7 +65,7 @@ front of the web app also needs `TRUST_PROXY_HEADERS=true` ([Authentication](aut | `write_note` | write a note or a thing's page, marked as written by an assistant. Needs a write-scope token and editor access | | `create_podcast` | make a two-host learning [episode](podcasts.md) about recordings or lines in them, with an angle, a length and a style; it's made in the background. Needs a write-scope token and editor access where it goes | | `get_podcast` | an episode: how far it has got, its script with each line's speaker and the sources it cites (links), and what the fact-check changed | -| `list_files` | a folder of the archive's [assets as files](file-tree.md): your namespaces, then their collections, each holding its assets' files | +| `list_files` | a folder of [the graph as files](file-tree.md): your namespaces, then their collections, topics (nested by narrower topics) and entities, each holding its assets' files | | `read_file` | a text file of that tree (an asset's `.txt` text or `.json` metadata), a page at a time; other files' download link | | `file_storage` | where Lens keeps its own files (notes' images and attachments) and the connections it could use; `check` tests it. Admins only | | `set_file_storage` | keep new files on this machine or a storage connection (folder, rclone crypt); checked before it's saved. Admins only, with a write-scope token | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 93130fad3..ba4b8141a 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -1358,7 +1358,7 @@ def get_podcast(ctx: Context, episode_id: int) -> dict[str, Any]: ) -# ---------- assets as files ---------- +# ---------- the graph as files ---------- FILE_CHARS = 100_000 # the most of a file read_file returns at once; offset pages through the rest @@ -1369,10 +1369,10 @@ def _tree(ctx: Context) -> filetree.Tree: @tool( "list_files", "List a folder of the file tree", - "The archive's assets as folders and files: '' lists your namespaces, then each one's collections and the " - "collections inside them. Each asset is files named '<title> [<id>]' in its collection's folder: its own file " - "(.mp3, .pdf, …), .json with its metadata, topics and entities, and .txt with its text.", - Arg("path", "string", "the folder, from the top (e.g. pods/General)", default="", max_length=4000), + "The graph as folders and files: '' lists your namespaces; each has Collections/<collection>/<inside it>/…, " + "Topics/<topic>/<narrower topic>/… and Entities/<type>/<name>/, each folder holding its assets' files, named " + "'<title> [<id>]': its own file (.mp3, .pdf, …), .json with its metadata, topics and entities, and .txt with its text.", + Arg("path", "string", "the folder, from the top (e.g. pods/Topics/Gene therapy)", default="", max_length=4000), ) def list_files(ctx: Context, path: str) -> dict[str, Any]: t = _tree(ctx) diff --git a/fastapi_backend/app/api/v1/routes/filetree.py b/fastapi_backend/app/api/v1/routes/filetree.py index b4369faf6..bb170d4d7 100644 --- a/fastapi_backend/app/api/v1/routes/filetree.py +++ b/fastapi_backend/app/api/v1/routes/filetree.py @@ -1,4 +1,4 @@ -"""Assets as folders and files (docs/file-tree.md): browse the tree the S3, WebDAV, SMB and NFS servers serve, and +"""The graph as folders and files (docs/file-tree.md): browse the tree the S3, WebDAV, SMB and NFS servers serve, and read its files. Everyone sees what the web app shows them; the tree only reads.""" from __future__ import annotations @@ -15,7 +15,7 @@ from app.schemas.filetree import TreeEntry, TreeFolder router = APIRouter(prefix="/tree", tags=["tree"]) -PATH = Query("", max_length=4000, description="a folder or file, from the top: namespace/collection/…") +PATH = Query("", max_length=4000, description="a folder or file, from the top: namespace/Collections|Topics|Entities/…") def tree_of(db: DB, cfg: dict[str, Any], user: Principal) -> filetree.Tree: diff --git a/fastapi_backend/app/domain/filetree.py b/fastapi_backend/app/domain/filetree.py index 3d796205b..79692c413 100644 --- a/fastapi_backend/app/domain/filetree.py +++ b/fastapi_backend/app/domain/filetree.py @@ -1,16 +1,23 @@ -"""Assets as folders and files (docs/file-tree.md): what the S3, WebDAV, SMB and NFS servers serve, and the API browses. +"""The graph as folders and files (docs/file-tree.md): what the S3, WebDAV, SMB and NFS servers serve, and the API browses. - /<namespace>/<collection>/<sub-collection>/…/<asset files> +Graph nodes are folders, nested along the graph's links, with assets' files inside them: -An asset (a recording, document or image) shows as up to four files in its collection's folder, named after it, -`<title> [<id>]`: its own file (`.mp3`, `.pdf`, …), `.json` beside it with its metadata (Dublin Core terms, its -collection, topics and entities, the description saved on it), `.txt` with its text (the transcript, or what was read -off its pages), and `.pdf` for a document converted to one. An own file that is itself `.txt` or `.json` (an imported -transcript) is `<title> [<id>].original.txt`. + /<namespace>/ + Collections/<collection>/<collection inside it>/… every asset in its collection's folder + Topics/<topic>/<narrower topic>/… the assets about each topic (accepted ones) + Entities/<type>/<name>/ the assets that mention an entity + +An asset (a recording, document or image) shows as up to four files named after it, `<title> [<id>]`: its own file +(`.mp3`, `.pdf`, …), `.json` beside it with its metadata (Dublin Core terms, its collection, topics and entities, the +description saved on it), `.txt` with its text (the transcript, or what was read off its pages), and `.pdf` for a +document converted to one. An own file that is itself `.txt` or `.json` (an imported transcript) is +`<title> [<id>].original.txt`. The same asset shows in every folder it belongs to; a topic with several broader +topics shows under each. What someone sees is what the web app shows them: the namespaces they have a role in (admins: all of them), and in a -namespace where they only have roles on some collections, those collections. The tree only reads; names are made -safe for every file system (no `/ \\ : * ? " < > |`) and unique in their folder, ignoring case. +namespace where they only have roles on some collections, those collections and the topics and entities of the +assets in them. The tree only reads; names are made safe for every file system (no `/ \\ : * ? " < > |`) and unique +in their folder, ignoring case. """ from __future__ import annotations @@ -31,6 +38,8 @@ NAME_MAX = 120 CHUNK = 1 << 16 ASSET_ID = re.compile(r" \[(\d+)\](?:\.[^.\[\]]+)+$") +COLLECTIONS, TOPICS, ENTITIES = "Collections", "Topics", "Entities" +TOPIC_DEPTH = 32 # topics can't loop (topics._would_loop), but a path stops here anyway UNSAFE = re.compile(r'[\x00-\x1f\x7f/\\:*?"<>|]') REC_FIELDS = ( "record::id(id) AS id, space, collection, title, path, source, remote, size, media, status, duration_ms, created_at, " @@ -177,6 +186,8 @@ def __init__(self, db, cfg, viewer: Viewer): self._names = {sid: n for sid, n in store.space_names(db).items() if sid in self._spaces} self._cols: dict[int, list[dict[str, Any]]] = {} self._only: int | None = None + self._seen: dict[int, set[int]] = {} + self._topic_rows: dict[int, list[dict[str, Any]]] = {} # ---------- walking ---------- @staticmethod @@ -195,7 +206,26 @@ def list(self, path: str) -> list[Entry]: def _list(self, parts: list[str]) -> list[Entry]: if not parts: return [_folder(n) for n in sorted(self._names.values(), key=str.casefold)] - return self._list_collection(self._space(parts[0]), parts[1:]) + sid = self._space(parts[0]) + rest = parts[1:] + if not rest: + return [_folder(COLLECTIONS), _folder(ENTITIES), _folder(TOPICS)] + head, rest = rest[0], rest[1:] + if head == COLLECTIONS: + return self._list_collection(sid, rest) + if head == TOPICS: + return self._list_topic(sid, rest) + if head == ENTITIES: + types = self._types(sid) + if not rest: + return [_folder(n) for n in types] + if rest[0] in types: + ents = self._entities(sid, types[rest[0]]) + if len(rest) == 1: + return [_folder(n) for n in ents] + if len(rest) == 2 and rest[1] in ents: + return self._assets(self._entity_recordings(sid, ents[rest[1]])) + raise NotFound("/".join(parts)) def stat(self, path: str) -> Entry: """One file or folder; NotFound when there's none (or it isn't this person's to see).""" @@ -266,6 +296,105 @@ def _list_collection(self, sid: int, names: list[str]) -> list[Entry]: out += self._assets(self.db.rows(f"SELECT {REC_FIELDS} FROM recording WHERE collection = $c", c=cid)) return sorted(out, key=lambda e: e.name.casefold()) + def _visible(self, sid: int, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + seen = self._spaces.get(sid) + return [r for r in rows if r.get("space") == sid and (seen is None or r.get("collection") in seen)] + + def _recordings(self, sid: int, rids: list[int]) -> list[dict[str, Any]]: + if not rids: + return [] + rows = self.db.rows(f"SELECT {REC_FIELDS} FROM recording WHERE id IN $ids", ids=[R("recording", int(r)) for r in set(rids)]) + return self._visible(sid, rows) + + # ---------- topics and entities ---------- + def _topics(self, sid: int) -> list[dict[str, Any]]: + """The namespace's topics this person sees: all of them, or (seen in part) the topics of the assets they see and + the topics broader than those.""" + if sid not in self._topic_rows: + rows = self.db.rows("SELECT record::id(id) AS id, label, broader FROM topic WHERE space = $s", s=sid) + if self._spaces[sid] is not None: + keep = { + a["topic"] + for a in self.db.rows("SELECT topic, recording FROM topic_about WHERE space = $s AND status = 'accepted'", s=sid) + if a["recording"] in self._seen_recordings(sid) + } + up = {t["id"]: t.get("broader") or [] for t in rows} + todo = list(keep) + while todo: + for b in up.get(todo.pop(), []): + if b not in keep: + keep.add(b) + todo.append(b) + rows = [t for t in rows if t["id"] in keep] + self._topic_rows[sid] = rows + return self._topic_rows[sid] + + def _narrower(self, sid: int, parent: int | None) -> dict[str, int]: + """The topics just under a topic (None: the ones with no broader topic this person sees), by their names here.""" + rows = self._topics(sid) + known = {t["id"] for t in rows} + if parent is None: + kids = [t for t in rows if not any(b in known for b in t.get("broader") or [])] + else: + kids = [t for t in rows if parent in (t.get("broader") or [])] + return dict(_unique([(clean(t["label"]), t["id"]) for t in kids])) + + def _list_topic(self, sid: int, names: list[str]) -> list[Entry]: + if len(names) > TOPIC_DEPTH: + raise NotFound("/".join(names)) + tid = None + for n in names: + kids = self._narrower(sid, tid) + if n not in kids: + raise NotFound(n) + tid = kids[n] + out = [_folder(n) for n in self._narrower(sid, tid)] + if tid is not None: + out += self._assets(self._topic_recordings(sid, tid)) + return sorted(out, key=lambda e: e.name.casefold()) + + def _topic_recordings(self, sid: int, tid: int) -> list[dict[str, Any]]: + rids = self.db.values("SELECT VALUE recording FROM topic_about WHERE topic = $t AND status = 'accepted'", t=tid) + return self._recordings(sid, rids) + + def _seen_recordings(self, sid: int) -> set[int]: + if sid not in self._seen: + self._seen[sid] = hierarchy.recordings_in(self.db, self._spaces[sid] or []) + return self._seen[sid] + + def _entity_rows(self, sid: int) -> list[dict[str, Any]]: + from .entities import QUIET + + rows = [ + e + for e in self.db.rows("SELECT record::id(id) AS id, name, type, hidden FROM entity WHERE space = $s", s=sid) + if not e.get("hidden") and e.get("type") not in QUIET + ] + if self._spaces[sid] is not None: + seen = self._seen_recordings(sid) + have = { + m["entity"] + for m in self.db.rows("SELECT entity, recording FROM mentions WHERE space = $s GROUP BY entity, recording", s=sid) + if m["recording"] in seen + } + rows = [e for e in rows if e["id"] in have] + return rows + + def _types(self, sid: int) -> dict[str, str]: + """{folder name: entity type} for the types this namespace's entities have.""" + from .entities import TYPES + + own = entity_setup.labels(self.db, {sid}).get(sid, {}) + codes = sorted({e["type"] for e in self._entity_rows(sid)}) + return dict(_unique([(clean(TYPES.get(c) or own.get(c) or c.title()), c) for c in codes])) + + def _entities(self, sid: int, code: str) -> dict[str, int]: + return dict(_unique([(clean(e["name"]), e["id"]) for e in self._entity_rows(sid) if e["type"] == code])) + + def _entity_recordings(self, sid: int, eid: int) -> list[dict[str, Any]]: + rids = [m["recording"] for m in self.db.rows("SELECT recording FROM mentions WHERE entity = $e GROUP BY recording", e=eid)] + return self._recordings(sid, rids) + # ---------- assets ---------- def _primary(self, rec: dict[str, Any]) -> Entry | None: """The asset's own file: stored here (maybe encrypted), cached from a storage source, or on one.""" diff --git a/fastapi_backend/tests/api/test_file_tree.py b/fastapi_backend/tests/api/test_file_tree.py index a08f46dad..21b18acd6 100644 --- a/fastapi_backend/tests/api/test_file_tree.py +++ b/fastapi_backend/tests/api/test_file_tree.py @@ -1,5 +1,5 @@ -"""Assets as folders and files (docs/file-tree.md): namespaces and collections as folders, assets as files with a -metadata sidecar and their text, seen exactly as the web app shows them to each person.""" +"""The graph as folders and files (docs/file-tree.md): collections, topics and entities as nested folders, assets as +files with a metadata sidecar and their text, seen exactly as the web app shows them to each person.""" from __future__ import annotations @@ -50,9 +50,10 @@ def test_namespaces_and_collections_are_folders(client, env, db): talks = hierarchy.create(db, pods, "Talks") hierarchy.create(db, pods, "2024", parent=talks) assert _names(client, ha) == ["calls", "pods"] - assert _names(client, ha, "pods") == ["General", "Talks"] - assert _names(client, ha, "pods/Talks") == ["2024"] - assert _names(client, ha, "pods/General") == [ + assert _names(client, ha, "pods") == ["Collections", "Entities", "Topics"] + assert _names(client, ha, "pods/Collections") == ["General", "Talks"] + assert _names(client, ha, "pods/Collections/Talks") == ["2024"] + assert _names(client, ha, "pods/Collections/General") == [ "ep1 [1].json", "ep1 [1].original.txt", "ep1 [1].txt", @@ -61,15 +62,33 @@ def test_namespaces_and_collections_are_folders(client, env, db): "ep2 [2].txt", ] # a file isn't a folder, nor the other way round; what isn't there is 404 - assert client.get(URL, headers=ha, params={"path": "pods/General/ep1 [1].txt"}).status_code == 400 - assert _read(client, ha, "pods/General").status_code == 400 - assert client.get(URL, headers=ha, params={"path": "pods/Nothing"}).status_code == 404 - assert _read(client, ha, "pods/General/nope.txt").status_code == 404 + assert client.get(URL, headers=ha, params={"path": "pods/Collections/General/ep1 [1].txt"}).status_code == 400 + assert _read(client, ha, "pods/Collections/General").status_code == 400 + assert _read(client, ha, "pods/Collections").status_code == 400 + assert client.get(URL, headers=ha, params={"path": "pods/Collections/Nothing"}).status_code == 404 + assert _read(client, ha, "pods/Collections/General/nope.txt").status_code == 404 + + +def test_graph_nodes_are_nested_folders(client, env, db): + """Topics nest under their broader topics, entities under their type, each with the assets about it.""" + ha = env["ha"] + pods = store.ns_id(db, "pods") + gt = db.one("SELECT record::id(id) AS id FROM topic WHERE label = 'Gene therapy'")["id"] + aav = topics.create(db, pods, "AAV vectors", broader=[gt]) + topics.tag(db, aav, [env["b"]]) + assert _names(client, ha, "pods/Topics") == ["Gene therapy"] + assert _names(client, ha, "pods/Topics/Gene therapy") == ["AAV vectors", "ep1 [1].json", "ep1 [1].original.txt", "ep1 [1].txt"] + assert _names(client, ha, "pods/Topics/Gene therapy/AAV vectors") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] + assert client.get(URL, headers=ha, params={"path": "pods/Topics/AAV vectors"}).status_code == 404 + assert "Organisation" in _names(client, ha, "pods/Entities") + assert "Dyno Therapeutics" in _names(client, ha, "pods/Entities/Organisation") + assert "ep2 [2].txt" in _names(client, ha, "pods/Entities/Organisation/Dyno Therapeutics") + assert _read(client, ha, "pods/Topics/Gene therapy/AAV vectors/ep2 [2].txt").status_code == 200 def test_an_assets_files(client, env, folder): ha = env["ha"] - base = "pods/General/ep1 [1]" + base = "pods/Collections/General/ep1 [1]" text = _read(client, ha, f"{base}.txt") assert text.status_code == 200 and text.headers["content-type"].startswith("text/plain") assert text.text.startswith("Alice: Welcome back.") @@ -84,16 +103,16 @@ def test_an_assets_files(client, env, folder): part = _read(client, ha, f"{base}.original.txt", range="bytes=5-14") assert part.status_code == 206 and part.content == own[5:15] and part.headers["content-range"] == f"bytes 5-14/{len(own)}" # the size listed is the size read - listed = {e["name"]: e["size"] for e in client.get(URL, headers=ha, params={"path": "pods/General"}).json()["entries"]} + listed = {e["name"]: e["size"] for e in client.get(URL, headers=ha, params={"path": "pods/Collections/General"}).json()["entries"]} for name, size in listed.items(): - assert len(_read(client, ha, f"pods/General/{name}").content) == size, name + assert len(_read(client, ha, f"pods/Collections/General/{name}").content) == size, name def test_people_see_what_the_web_app_shows_them(client, env, db, folder): hv, hx = env["hv"], env["hx"] assert _names(client, hv) == ["pods"] assert client.get(URL, headers=hv, params={"path": "calls"}).status_code == 404 - assert _read(client, hv, "calls/General/call [3].txt").status_code == 404 + assert _read(client, hv, "calls/Collections/General/call [3].txt").status_code == 404 assert _names(client, hx) == [] assert client.get(URL).status_code == 401 # a role on one collection: just that collection, and the topics and entities of what's in it @@ -102,9 +121,11 @@ def test_people_see_what_the_web_app_shows_them(client, env, db, folder): db.q("UPDATE $r SET collection = $c", r=store.R("recording", env["b"]), c=talks) hierarchy.give(db, talks, _account(db, "out@x.io"), "viewer") assert _names(client, hx) == ["pods"] - assert _names(client, hx, "pods") == ["Talks"] - assert _names(client, hx, "pods/Talks") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] - assert _read(client, hx, "pods/General/ep1 [1].txt").status_code == 404 + assert _names(client, hx, "pods/Collections") == ["Talks"] + assert _names(client, hx, "pods/Topics") == [] # Gene therapy is about ep1, which they can't see + assert _names(client, hx, "pods/Entities/Organisation/Dyno Therapeutics") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] + assert _names(client, hx, "pods/Collections/Talks") == ["ep2 [2].json", "ep2 [2].original.txt", "ep2 [2].txt"] + assert _read(client, hx, "pods/Collections/General/ep1 [1].txt").status_code == 404 def _account(db, email): @@ -116,8 +137,8 @@ def test_names_are_safe_and_unique(db, cfg, env): hierarchy.create(db, pods, 'a/b: "c"?') db.q("UPDATE $r SET title = $t", r=store.R("recording", env["a"]), t="Q3: plans / ideas?") t = filetree.Tree(db, cfg, filetree.Viewer(admin=True, roles={pods: "owner"})) - assert [e.name for e in t.list("pods")] == ["a_b_ _c__", "General"] - assert "Q3_ plans _ ideas_ [1].txt" in [e.name for e in t.list("pods/General")] + assert [e.name for e in t.list("pods/Collections")] == ["a_b_ _c__", "General"] + assert "Q3_ plans _ ideas_ [1].txt" in [e.name for e in t.list("pods/Collections/General")] assert filetree._unique([("Talks", 1), ("talks", 2)]) == [("Talks", 1), ("talks [2]", 2)] assert filetree.clean("..") == "_" and filetree.clean("x. ") == "x" and len(filetree.clean("y" * 500)) == filetree.NAME_MAX @@ -127,9 +148,11 @@ def test_encrypted_files_are_read_plain(client, db, cfg, env, folder): own = (folder / "ep1.txt").read_bytes() keyring.encrypt_file(db, cfg, store.ns_id(db, "pods"), str(folder / "ep1.txt"), force=True) assert keyring.is_encrypted(folder / "ep1.txt") - got = _read(client, env["ha"], "pods/General/ep1 [1].original.txt") + got = _read(client, env["ha"], "pods/Collections/General/ep1 [1].original.txt") assert got.status_code == 200 and got.content == own - sizes = {e["name"]: e["size"] for e in client.get(URL, headers=env["ha"], params={"path": "pods/General"}).json()["entries"]} + sizes = { + e["name"]: e["size"] for e in client.get(URL, headers=env["ha"], params={"path": "pods/Collections/General"}).json()["entries"] + } assert sizes["ep1 [1].original.txt"] == len(own) @@ -137,9 +160,9 @@ def test_agents_browse_it_over_mcp(client, env): _, tokens = _grant(client, env["hv"], scope="read") h = _bearer(tokens) assert [e["name"] for e in tool(client, h, "list_files", path="")["entries"]] == ["pods"] - got = tool(client, h, "read_file", path="pods/General/ep1 [1].json") + got = tool(client, h, "read_file", path="pods/Collections/General/ep1 [1].json") assert json.loads(got["text"])["subject"] == ["Gene therapy"] - own = tool(client, h, "read_file", path="pods/General/ep1 [1].original.txt") + own = tool(client, h, "read_file", path="pods/Collections/General/ep1 [1].original.txt") assert "text" not in own and own["download"].startswith("/api/v1/tree/file?path=") - assert "a file" in tool_error(client, h, "list_files", path="pods/General/ep1 [1].txt") - assert "not found" in tool_error(client, h, "read_file", path="calls/General/call [3].txt") + assert "a file" in tool_error(client, h, "list_files", path="pods/Collections/General/ep1 [1].txt") + assert "not found" in tool_error(client, h, "read_file", path="calls/Collections/General/call [3].txt") diff --git a/mkdocs.yml b/mkdocs.yml index 63ac301de..8c7fd52e9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -89,7 +89,7 @@ nav: - Notes: notes.md - Web clipper: web-clipper.md - Storage: storage.md - - Assets as files: file-tree.md + - The graph as files: file-tree.md - Linked data (RDF): rdf.md - Fedora repository: fedora.md - API: api.md diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 7eb0b4415..548a63513 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -23642,7 +23642,7 @@ export type ListFolderData = { /** * Path * - * a folder or file, from the top: namespace/collection/… + * a folder or file, from the top: namespace/Collections|Topics|Entities/… */ path?: string; }; @@ -23674,7 +23674,7 @@ export type ReadFileData = { /** * Path * - * a folder or file, from the top: namespace/collection/… + * a folder or file, from the top: namespace/Collections|Topics|Entities/… */ path?: string; }; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index c634292a2..cd87956f9 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -9485,11 +9485,11 @@ "schema": { "type": "string", "maxLength": 4000, - "description": "a folder or file, from the top: namespace/collection/\u2026", + "description": "a folder or file, from the top: namespace/Collections|Topics|Entities/\u2026", "default": "", "title": "Path" }, - "description": "a folder or file, from the top: namespace/collection/\u2026" + "description": "a folder or file, from the top: namespace/Collections|Topics|Entities/\u2026" } ], "responses": { @@ -9532,11 +9532,11 @@ "schema": { "type": "string", "maxLength": 4000, - "description": "a folder or file, from the top: namespace/collection/\u2026", + "description": "a folder or file, from the top: namespace/Collections|Topics|Entities/\u2026", "default": "", "title": "Path" }, - "description": "a folder or file, from the top: namespace/collection/\u2026" + "description": "a folder or file, from the top: namespace/Collections|Topics|Entities/\u2026" } ], "responses": {