From 226c9673996dcbbf74864b0c8475b2112d1b2daf Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 12:48:16 +0000 Subject: [PATCH] feat(topics): # links in notes point at topics, and topic tools for the assistant and MCP of their own. The assistant gains find_topics, topic_recordings and suggest_topic; the MCP server gains list_topics, get_topic and suggest_topic. Their suggestions wait for someone to accept. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012Jo1JNziunMs8MgWJSGyyM --- CHANGELOG.md | 4 + DEVLOG.md | 2 +- docs/graph.md | 3 +- docs/mcp.md | 8 +- docs/notes.md | 16 +-- docs/topics.md | 15 ++- fastapi_backend/app/api/mcp_tools.py | 116 ++++++++++++++++-- fastapi_backend/app/api/v1/routes/notebook.py | 2 +- fastapi_backend/app/domain/ai_tools.py | 93 +++++++++++++- fastapi_backend/app/domain/notebook.py | 23 ++-- fastapi_backend/app/domain/topics.py | 29 ++++- fastapi_backend/app/schemas/notebook.py | 2 +- fastapi_backend/app/schemas/topics.py | 2 +- fastapi_backend/tests/api/test_chats.py | 23 +++- fastapi_backend/tests/api/test_mcp.py | 32 ++++- fastapi_backend/tests/api/test_notebook.py | 18 +-- nextjs-frontend/__tests__/notes-links.test.ts | 1 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 2 +- .../app/openapi-client/types.gen.ts | 4 +- nextjs-frontend/components/notes/links.ts | 4 +- .../components/topics/topic-drawer.tsx | 22 ++-- nextjs-frontend/openapi.json | 6 +- 22 files changed, 358 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfd3eb305..bdfb0d038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,10 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version `ABOUT`, `NARROWER` and `RELATED`; TERM entities are now labelled `Term` there) and in RDF (`/id/topic/`). See docs/topics.md. +- **Topics in notes and for agents.** `#` in a note links a topic, found by any of its labels, and every topic has + a page of its own. The assistant can find topics, read one with its recordings and suggest a topic for recordings; + the MCP server has `list_topics`, `get_topic` and `suggest_topic`. Their suggestions wait for someone to accept. + - **Topic suggestions.** Analysis suggests the vocabulary's topics for recordings whose summary or transcript says one of their labels, for people to accept or dismiss (a dismissed one isn't suggested again). The Topics page lists what summaries say recordings are about that no topic covers yet, to add as a topic or skip. No model is called. diff --git a/DEVLOG.md b/DEVLOG.md index d6c4257e7..589f67995 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -148,7 +148,7 @@ Todo: - [x] Topics page: the tree of broader and narrower topics, a topic's recordings, edit and merge; a recording's topics - [x] Analysis suggests topics (summary topics and keywords matched to the vocabulary; new ones as suggestions to accept); a namespace can keep its vocabulary fixed or open, as entities do -- [ ] # links in notes point at topics; assistant and MCP tools for topics +- [x] # links in notes point at topics; assistant and MCP tools for topics Refine later: shared vocabularies across namespaces and imported schemes (LCSH, Wikidata) with exactMatch; entities as authority records (variant names, external identifiers); topic history and undo for merges; a namespace setting diff --git a/docs/graph.md b/docs/graph.md index f1efca325..f9b26e8b9 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -86,7 +86,8 @@ namespaces; `ids` lists the entities behind it, and `e` finds the merged nod ## The assistant and MCP The assistant in Chat has the graph as tools, over the namespaces of the chat (or every shared one): `graph_schema`, -`graph_query` (Cypher), `graph_related` and `graph_paths`. They only read. +`graph_query` (Cypher), `graph_related` and `graph_paths`. They only read. For topics it has `find_topics`, +`topic_recordings` and `suggest_topic` ([Topics](topics.md)). The MCP server (docs/mcp.md) has the same four, marked read-only, and `propose_graph_change`, which needs a write-scope token and editor access; the change waits in Proposed changes unless `apply` makes it at once, and can be diff --git a/docs/mcp.md b/docs/mcp.md index 76167c243..82f970a53 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -44,7 +44,7 @@ front of the web app also needs `TRUST_PROXY_HEADERS=true` ([Authentication](aut | `get_transcript` | a recording's lines in order (speaker, time, text, link), a page at a time, from a line or a time window | | `fetch` | a recording's whole text at once, as `[m:ss] Speaker: …` lines | | `cite` | a citation for a moment (a line or a second, and a few lines from there): the words, who said them, the recording and date, the time, the link, and all of it as Markdown | -| `list_entities` | people, organisations, products, places, events, works and topics mentioned, by name, type, namespace or recording | +| `list_entities` | people, organisations, products, places, events, works and terms mentioned, by name, type, namespace or recording | | `get_entity` | one entity: its names, how often and when it is mentioned, what it is mentioned with, who mentions it most, and the lines that mention it | | `explore_graph` | the knowledge graph around an entity (`e12`) or a speaker (`s3`), one or two steps out | | `find_path` | the shortest chain of links between two of them, with lines that show each link | @@ -52,6 +52,9 @@ front of the web app also needs `TRUST_PROXY_HEADERS=true` ([Authentication](aut | `graph_query` | read-only Cypher over the graph of one namespace or every shared one: columns, rows, and the nodes and relationships found | | `graph_related` | a node's parents, children, ancestors, descendants or neighbours | | `graph_paths` | the paths between two nodes, shortest first | +| `list_topics` | each namespace's vocabulary of topics ([Topics](topics.md)), by any label, top topics or the narrower ones of a topic | +| `get_topic` | one topic: its labels, definition, broader, narrower and related topics, and the recordings about it, with links | +| `suggest_topic` | suggest that recordings are about a topic; it waits for someone to accept it. Needs a write-scope token and editor access | | `propose_graph_change` | ask for two entities to be merged or linked; it waits in Proposed changes unless `apply` makes it at once (it can be undone). Needs a write-scope token and editor access | Results are JSON (as `structuredContent`, and the same as text). Links open the recording's page in the web app at @@ -60,7 +63,8 @@ argument, something it can't read) come back as a tool result with `isError`, so person's to read answers "not found", as the API does. `search` and `fetch` follow the shape ChatGPT's connectors expect (`results` with `id`, `title`, `url`; `fetch` by that `id`). -Every tool reads, except `propose_graph_change`, which records an undoable change (proposed by default). +Every tool reads, except `suggest_topic`, which adds suggestions people accept or dismiss, and `propose_graph_change`, +which records an undoable change (proposed by default). Importing, editing and curating stay in the web app and the API. ## Protocol diff --git a/docs/notes.md b/docs/notes.md index 996974f44..5d22a38d3 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -3,10 +3,11 @@ Every resource, entity and topic in Lens has a page of its own, and next to them sit free notes that people and the assistant write. The assistant is the main writer and organiser; people can do everything it can. -Status: **built**: pages (free notes in a tree, a page per recording, entity, collection or speaker), @ and # links -with backlinks, the tree in the left navigation, the page view with the BlockSuite editor, the model keeping titles -and summaries up to date, and assistant tools to find, read, write and update notes. **Planned**: MCP tools, page -history, linking notes to entities and topics and nesting them under their project or area on their own, attachments on encrypted object storage, pages in the graph, # links to SKOS topics. +Status: **built**: pages (free notes in a tree, a page per recording, entity, topic, collection or speaker), @ and # +links with backlinks (# links topics), the tree in the left navigation, the page view with the BlockSuite editor, the +model keeping titles and summaries up to date, and assistant tools to find, read, write and update notes. +**Planned**: MCP tools, page history, linking notes to entities and topics and nesting them under their project or +area on their own, attachments on encrypted object storage, pages in the graph. ## A page @@ -32,11 +33,12 @@ nothing written on it is lost. The body links with mention tokens, which the editor and the assistant both write: ```text -@[Weekly call](recording:12) @[Ada Lovelace](entity:5) @[Plans](page:3) #[Capsids](entity:9) +@[Weekly call](recording:12) @[Ada Lovelace](entity:5) @[Plans](page:3) #[Capsid design](topic:9) ``` -`@` links resources, people and other pages; `#` links topics (entities of type Topic, until topics become a SKOS -vocabulary of their own; see [the graph](graph.md)). `GET /api/v1/notes/targets?ns=&sign=&q=` offers what to link. +`@` links resources, people and other pages; `#` links topics, the namespace's vocabulary ([Topics](topics.md)), +found by any of their labels. Links written to terms (`#[Capsids](entity:9)`) before topics existed still work. +`GET /api/v1/notes/targets?ns=&sign=&q=` offers what to link. Links are kept as rows (`note_link`), so every page lists its backlinks; links to things that are gone or that the reader can't see show without a name. diff --git a/docs/topics.md b/docs/topics.md index 5ad3c2f21..5b1363248 100644 --- a/docs/topics.md +++ b/docs/topics.md @@ -6,8 +6,8 @@ subject someone chose for the vocabulary, like "Gene therapy" or "Funding". Status: **built**: the vocabulary (labels, definitions, broader, narrower and related topics), recordings about topics, topics made from topic-like entities, the API, the Topics page, a recording's topics, topics in the graph -(and on its canvas) and in RDF, and topic suggestions from analysis. **Planned**: `#` links in notes pointing at -topics, and assistant and MCP tools for topics. +(and on its canvas) and in RDF, topic suggestions from analysis, `#` links in notes, a page per topic, and +assistant and MCP tools for topics. ## In the web app @@ -109,3 +109,14 @@ In RDF ([Linked data](rdf.md)) a topic is `/id/topic/`, a `skos:Concept` in - Vocabularies shared by several namespaces, and imported schemes (LCSH, Wikidata) linked with `skos:exactMatch`. - Entities as authority records: variant names and external identifiers. - History and undo for topic merges and deletes. + +## Notes, the assistant and agents + +In notes, `#` links a topic (`#[Capsid design](topic:9)`), found by any of its labels, and every topic has a page of +its own (**Page** in its drawer) that lists the notes linking to it. Deleting or merging a topic keeps its page, as a +free note. + +The assistant in Chat finds topics (`find_topics`), reads one with its recordings (`topic_recordings`) and, for +editors, suggests a topic for recordings (`suggest_topic`). The MCP server has the same as `list_topics`, `get_topic` +and `suggest_topic` (with a write-scope token). Suggestions from either wait for someone to accept them, with +`assistant` as their source; what people accepted or dismissed stays as it is. diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 634002dcd..af789b44f 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -24,9 +24,11 @@ from app.api.v1.routes import namespaces as namespace_routes from app.api.v1.routes import recordings as recording_routes from app.api.v1.routes import search as search_routes +from app.api.v1.routes import topics as topic_routes from app.domain import analyze, graph_history, library, rdf, render, store from app.domain import entities as ents from app.domain import speakers as spk +from app.domain import topics as topicmod from app.domain.store import DB from app.schemas.entities import EntityList from app.schemas.search import SearchResults @@ -75,6 +77,8 @@ class Arg: def schema(self) -> dict[str, Any]: if self.type == "string[]": s: dict[str, Any] = {"type": "array", "items": {"type": "string", **({"enum": list(self.enum)} if self.enum else {})}} + elif self.type == "integer[]": + s = {"type": "array", "items": {"type": "integer"}, "maxItems": self.maximum or 500} else: s = {"type": self.type} if self.enum: @@ -82,7 +86,7 @@ def schema(self) -> dict[str, Any]: s["description"] = self.description if self.minimum is not None: s["minimum"] = self.minimum - if self.maximum is not None: + if self.maximum is not None and self.type != "integer[]": s["maximum"] = self.maximum if self.max_length is not None: s["maxLength"] = self.max_length @@ -137,6 +141,19 @@ def parse(self, value: Any) -> Any: if bad: raise ToolError(f"{self.name}: {', '.join(bad)} isn't one of {', '.join(self.enum)}") return value or self.default + elif t == "integer[]": + if isinstance(value, str): + value = [x.strip() for x in value.split(",") if x.strip()] + if not isinstance(value, list) or not all( + (isinstance(x, int) and not isinstance(x, bool)) or (isinstance(x, str) and re.fullmatch(r"\d+", x)) for x in value + ): + raise ToolError(f"{self.name} is a list of whole numbers") + value = [int(x) for x in value] + if self.maximum is not None and len(value) > self.maximum: + raise ToolError(f"{self.name} has at most {self.maximum} items") + if self.required and not value: + raise ToolError(f"{self.name} is required") + return value or self.default if self.enum and t != "string[]" and value not in self.enum: raise ToolError(f"{self.name} is one of {', '.join(self.enum)}") if self.minimum is not None and value < self.minimum: @@ -689,7 +706,7 @@ def _entity_row(e: dict[str, Any]) -> dict[str, Any]: @tool( "list_entities", "Find entities", - "Entities: the people, organisations, products, places, events, works and topics mentioned in recordings, the most " + "Entities: the people, organisations, products, places, events, works and terms mentioned in recordings, the most " "mentioned first. Find one by name with query; its entity_id opens it in get_entity and the graph.", Arg("query", "string", "a name, or part of one", max_length=200), Arg("types", "string[]", "only these types", enum=ENTITY_TYPES), @@ -940,9 +957,10 @@ def _gscope(namespace: str | None) -> str: @tool( "graph_schema", "What the graph holds", - "The archive as a property graph: node labels (Namespace, Collection, Recording, Speaker, Entity and its type) with " - "their properties and counts, relationship types (CONTAINS, HAS_SPEAKER, MENTIONS, SAID, MENTIONED_WITH, " - "SPOKE_WITH, SAME_AS, SAME_THING) and what they join, and example Cypher. Read it before graph_query.", + "The archive as a property graph: node labels (Namespace, Collection, Recording, Speaker, Entity and its type, " + "Topic) with their properties and counts, relationship types (CONTAINS, HAS_SPEAKER, MENTIONS, SAID, " + "MENTIONED_WITH, SPOKE_WITH, SAME_AS, SAME_THING, ABOUT, NARROWER, RELATED) and what they join, and example " + "Cypher. Read it before graph_query.", Arg("namespace", "string", "one namespace (else every namespace that shares its graph)"), ) def graph_schema(ctx: Context, namespace: str | None) -> dict[str, Any]: @@ -969,9 +987,10 @@ def graph_query(ctx: Context, query: str, namespace: str | None, limit: int) -> @tool( "graph_related", "Walk the graph from a node", - "A node's parents, children, ancestors or descendants (along CONTAINS, HAS_SPEAKER, MENTIONS and SAID: namespace " - "> collection > recording > speaker and entity), or its neighbours over any relationship, nearest first. Nodes " - "are n, c, r, s and e (e: for an entity across namespaces).", + "A node's parents, children, ancestors or descendants (along CONTAINS, HAS_SPEAKER, MENTIONS, SAID, ABOUT and " + "NARROWER: namespace > collection > recording > speaker, entity and topic > narrower topic), or its neighbours " + "over any relationship, nearest first. Nodes are n, c, r, s, e (e: for an entity across " + "namespaces) and t.", Arg("node", "string", "the node id", required=True, max_length=200), Arg("relation", "string", "which way to walk", required=True, enum=("parents", "children", "ancestors", "descendants", "neighbours")), Arg("depth", "integer", "steps out (ancestors, descendants, neighbours)", default=1, minimum=1, maximum=8), @@ -1027,6 +1046,87 @@ def graph_paths(ctx: Context, from_node: str, to_node: str, max_depth: int, shor ) +@tool( + "list_topics", + "Find topics", + "Topics: each namespace's controlled vocabulary of what its recordings are about (SKOS concepts), by label, with " + "other labels, a definition, broader topics and how many recordings are about each. query matches any label.", + Arg("query", "string", "a label, or part of one", max_length=200), + Arg("namespace", "string", "only this namespace"), + Arg("top", "boolean", "only topics with no broader topic", default=False), + Arg("broader_id", "integer", "only the narrower topics of this one"), + Arg("limit", "integer", "how many", default=50, minimum=1, maximum=200), + Arg("offset", "integer", "skip this many (for the next page)", default=0, minimum=0), +) +def list_topics( + ctx: Context, query: str | None, namespace: str | None, top: bool, broader_id: int | None, limit: int, offset: int +) -> dict[str, Any]: + res = topic_routes.list_topics(ctx.user, ctx.acl, ctx.db, query or "", namespace or "", top, broader_id, limit, offset) + out: dict[str, Any] = { + "total": res.total, + "topics": [ + { + "topic_id": t.id, + "label": t.label, + "namespace": t.namespace, + "also": t.alt or None, + "definition": t.definition, + "broader_ids": t.broader or None, + "narrower": t.narrower, + "recordings": t.recordings, + } + for t in res.items + ], + } + if offset + len(res.items) < res.total: + out["next_offset"] = offset + len(res.items) + return out + + +@tool( + "get_topic", + "Get a topic", + "One topic: its labels and definition, its broader, narrower and related topics, and the recordings about it " + "(accepted, and suggested ones waiting for someone), each with a url.", + Arg("topic_id", "integer", "the topic (from list_topics or the graph's t)", required=True), +) +def get_topic(ctx: Context, topic_id: int) -> dict[str, Any]: + d = topic_routes.get_topic(topic_id, ctx.user, ctx.acl, ctx.db) + return { + "topic_id": d.id, + "label": d.label, + "namespace": d.namespace, + "also": d.alt or None, + "definition": d.definition, + "broader": [{"topic_id": x.id, "label": x.label} for x in d.broader], + "narrower": [{"topic_id": x.id, "label": x.label} for x in d.narrower], + "related": [{"topic_id": x.id, "label": x.label} for x in d.related], + "recordings": [{"recording_id": a.recording, "title": a.title, "status": a.status, "url": ctx.link(a.recording)} for a in d.about], + } + + +@tool( + "suggest_topic", + "Suggest a topic for recordings", + "Say recordings are about a topic, as a suggestion that waits for someone to accept it on the recording or the " + "topic. What they accepted or dismissed stays. Needs a token with the write scope and editor access.", + Arg("topic_id", "integer", "the topic", required=True), + Arg("recording_ids", "integer[]", "the recordings, in the topic's namespace", required=True, maximum=500), + writes=True, +) +def suggest_topic(ctx: Context, topic_id: int, recording_ids: list[int]) -> dict[str, Any]: + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to suggest topics") + row = ctx.db.one("SELECT space FROM $r", r=R("topic", int(topic_id))) + if not row: + raise KeyError(topic_id) + ctx.acl.need(row["space"], "editor") + for rid in recording_ids: + ctx.acl.recording(rid) + made = topicmod.propose(ctx.db, topic_id, recording_ids, "assistant", ctx.user.email) + return {"suggested_for": made, "waiting": "for someone to accept on the recording or the topic"} + + @tool( "propose_graph_change", "Propose a change to the graph", diff --git a/fastapi_backend/app/api/v1/routes/notebook.py b/fastapi_backend/app/api/v1/routes/notebook.py index 7d04b4655..85bcf0b25 100644 --- a/fastapi_backend/app/api/v1/routes/notebook.py +++ b/fastapi_backend/app/api/v1/routes/notebook.py @@ -107,7 +107,7 @@ def link_targets( @router.get("/about/{kind}/{key}") def page_about(kind: str, key: int, user: CurrentUser, acl: Acl, db: Db) -> NotePage | NotePageDraft: - """The page of a recording, entity, collection or speaker: its page, or a draft while nobody has written one.""" + """The page of a recording, entity, topic, collection or speaker: its page, or a draft while nobody has written one.""" thing = f"{kind}:{key}" with domain_errors(): notebook._about(thing) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index e03a17b28..b807820af 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -27,6 +27,7 @@ speakers as spk, store, templates, + topics, ) R = store.R @@ -64,7 +65,7 @@ ), ( "find_entities", - "Find people, organisations, products, places and topics mentioned in scope.", + "Find people, organisations, products, places and terms mentioned in scope (for subjects, use find_topics).", {"query": _S, "type": _S, "namespace": _S, "limit": _I}, [], False, @@ -77,6 +78,29 @@ False, ), ("entity_timeline", "How often an entity was mentioned per month.", {"entity_id": _I}, ["entity_id"], False), + ( + "find_topics", + "Find topics: each namespace's controlled vocabulary of what recordings are about, with other labels, a " + "definition, broader topics and how many recordings are about each.", + {"query": _S, "namespace": _S, "limit": _I}, + [], + False, + ), + ( + "topic_recordings", + "One topic: its broader, narrower and related topics and the recordings about it (accepted, and suggested).", + {"topic_id": _I}, + ["topic_id"], + False, + ), + ( + "suggest_topic", + "Suggest that recordings are about a topic. It waits on the recording and the topic for someone to accept; " + "nothing they accepted or dismissed changes.", + {"topic_id": _I, "recording_ids": {"type": "array", "items": _I}}, + ["topic_id", "recording_ids"], + True, + ), ( "graph_neighbours", "Who and what is connected to an entity or speaker in the knowledge graph.", @@ -107,8 +131,8 @@ 'Write a new note, or the page of a thing (about, like "entity:5"; one each). You keep notes as you learn: ' "a specific title, a one-line summary of what it holds, Markdown text, and where it's filed: project (an " "outcome with an end), area (a responsibility kept up), resource (a topic of interest) or archive (done). Link " - "with @[label](recording:12), @[label](entity:5), @[label](page:3), and topics with #[label](entity:9) (ids " - "from find_entities, find_notes and list_recordings). Put it inside another note with parent_id.", + "with @[label](recording:12), @[label](entity:5), @[label](page:3), and topics with #[label](topic:9) (ids " + "from find_entities, find_topics, find_notes and list_recordings). Put it inside another note with parent_id.", { "namespace": _S, "title": _S, @@ -139,7 +163,7 @@ ), ( "graph_schema", - "What the graph holds (namespaces, collections, recordings, speakers, entities and how they link), with example " + "What the graph holds (namespaces, collections, recordings, speakers, entities, topics and how they link), with example " "Cypher. Read it before graph_query.", {"namespace": _S}, [], @@ -156,7 +180,7 @@ ( "graph_related", "A node's parents, children, ancestors, descendants (recording, collection, namespace...) or neighbours. Nodes " - "are n namespaces, c collections, r recordings, s speakers, e entities.", + "are n namespaces, c collections, r recordings, s speakers, e entities, t topics.", { "node": _S, "relation": {"type": "string", "enum": ["parents", "children", "ancestors", "descendants", "neighbours"]}, @@ -587,6 +611,65 @@ def t_entity_mentions(self, entity_id, limit=8): def t_entity_timeline(self, entity_id): return entities.timeline(self.db, [int(entity_id)], self.readable), "Counted mentions by month" + def t_find_topics(self, query=None, namespace=None, limit=20): + spaces = self.readable + if namespace: + sid = {v: k for k, v in store.space_names(self.db).items()}.get(namespace) + if sid not in self.readable: + raise ValueError(f"no namespace called {namespace} in scope") + spaces = {sid} + res = topics.list_topics(self.db, spaces, query or "", limit=max(1, min(int(limit or 20), 50))) + up = sorted({b for t in res["items"] for b in t["broader"]}) + label = { + t["id"]: t["label"] + for t in ( + self.db.rows("SELECT record::id(id) AS id, label FROM topic WHERE id IN $ids", ids=[R("topic", b) for b in up]) + if up + else [] + ) + } + out = [ + store.clean( + { + "id": t["id"], + "label": t["label"], + "namespace": t.get("namespace"), + "also": t["alt"] or None, + "definition": t.get("definition"), + "broader": [label.get(b, b) for b in t["broader"]] or None, + "recordings": t["recordings"], + } + ) + for t in res["items"] + ] + return {"total": res["total"], "topics": out}, f"Found {res['total']} topic(s)" + + def t_topic_recordings(self, topic_id): + t = topics.detail(self.db, int(topic_id), self.readable) + about = [a for a in t["about"] if self.allowed is None or a["recording"] in self.allowed] + out = { + "id": t["id"], + "label": t["label"], + "namespace": t.get("namespace"), + "definition": t.get("definition"), + "broader": t["broader"], + "narrower": t["narrower"], + "related": t["related"], + "recordings": [{"recording_id": a["recording"], "title": a["title"], "status": a["status"]} for a in about], + } + return out, f"Read the topic {t['label']}" + + def t_suggest_topic(self, topic_id, recording_ids): + row = self.db.one("SELECT space, label FROM $r", r=R("topic", int(topic_id))) + if not row or row["space"] not in self.readable: + raise ValueError(f"no topic {topic_id} in scope") + if row["space"] not in self.editable: + raise ValueError("suggesting topics needs editor access to the namespace") + for rid in recording_ids or []: + self._ok(rid) + made = topics.propose(self.db, int(topic_id), recording_ids, "assistant", self.user.get("email")) + return {"suggested_for": made, "note": "waiting for someone to accept"}, f"Suggested {row['label']} for {len(made)} recording(s)" + def t_graph_neighbours(self, entity_id=None, speaker_id=None, limit=15): if not entity_id and not speaker_id: raise ValueError("give an entity_id or a speaker_id") diff --git a/fastapi_backend/app/domain/notebook.py b/fastapi_backend/app/domain/notebook.py index fff6457b2..33d14af60 100644 --- a/fastapi_backend/app/domain/notebook.py +++ b/fastapi_backend/app/domain/notebook.py @@ -7,10 +7,9 @@ The body links to anything with mention tokens, which the editor and the assistant both write: - @[Weekly call](recording:12) @[Ada Lovelace](entity:5) @[Plans](page:3) #[Capsids](entity:9) + @[Weekly call](recording:12) @[Ada Lovelace](entity:5) @[Plans](page:3) #[Capsid design](topic:9) -@ is for resources, people and other pages; # is for topics (entities of type TERM until topics become a SKOS -vocabulary of their own). Links are kept as note_link rows, so a page shows its backlinks and the graph sees them. +@ is for resources, people and other pages; # is for topics, the namespace's vocabulary (topics.py). Links are kept as note_link rows, so a page shows its backlinks and the graph sees them. The editor may also keep its own document state (`doc`, opaque: a BlockSuite/Yjs snapshot) next to the Markdown, which holds what Markdown can't, like drawings on the edgeless canvas. A change to the Markdown alone (the assistant's, say) @@ -30,8 +29,8 @@ PLACES = ("project", "area", "resource", "archive") AUTHORS = ("person", "assistant") VIEWS = ("page", "edgeless") # how the editor shows it: a document, or the endless canvas (drawings, diagrams) -KINDS = ("recording", "entity", "collection", "speaker", "page") # what a page can link to -ABOUT = ("recording", "entity", "collection", "speaker") # what can have a page of its own +KINDS = ("recording", "entity", "topic", "collection", "speaker", "page") # what a page can link to +ABOUT = ("recording", "entity", "topic", "collection", "speaker") # what can have a page of its own TITLE_MAX = 200 SUMMARY_MAX = 300 BODY_MAX = 200_000 @@ -374,7 +373,7 @@ def labels(db, targets): out = {} for kind, keys in by_kind.items(): table = "note_page" if kind == "page" else kind - name = {"recording": "title ?? path", "page": "title", "speaker": "name ?? label"}.get(kind, "name") + name = {"recording": "title ?? path", "page": "title", "speaker": "name ?? label", "topic": "label"}.get(kind, "name") for r in db.rows( f"SELECT record::id(id) AS id, space, {name} AS name FROM {table} WHERE id IN $ids", ids=[R(table, k) for k in keys] ): @@ -383,15 +382,15 @@ def labels(db, targets): def search_targets(db, sid, q="", sign="@", limit=20): - """What a mention can link to in namespace sid, best matches first: for #, topics; for @, pages, recordings, - people and other entities, collections and speakers.""" + """What a mention can link to in namespace sid, best matches first: for #, topics (by any of their labels); for @, + pages, recordings, people and other entities, collections and speakers.""" q = " ".join(str(q or "").split()).casefold() hit = (lambda s: q in str(s or "").casefold()) if q else (lambda s: True) out = [] - if sign == "#": - for e in db.rows("SELECT record::id(id) AS id, name FROM entity WHERE space = $s AND type = 'TERM' AND hidden != true", s=sid): - if hit(e["name"]): - out.append({"target": f"entity:{e['id']}", "label": e["name"], "kind": "topic"}) + if sign == "#": # the namespace's vocabulary, found by any of a topic's labels + for t in db.rows("SELECT record::id(id) AS id, label, alt FROM topic WHERE space = $s", s=sid): + if any(hit(x) for x in [t["label"], *(t.get("alt") or [])]): + out.append({"target": f"topic:{t['id']}", "label": t["label"], "kind": "topic"}) return sorted(out, key=lambda x: (not x["label"].casefold().startswith(q), x["label"].casefold()))[:limit] for p in db.rows("SELECT record::id(id) AS id, title FROM note_page WHERE space = $s AND about = NONE", s=sid): if hit(p["title"]): diff --git a/fastapi_backend/app/domain/topics.py b/fastapi_backend/app/domain/topics.py index f655b09b9..afdba2e55 100644 --- a/fastapi_backend/app/domain/topics.py +++ b/fastapi_backend/app/domain/topics.py @@ -3,7 +3,7 @@ Entities are the named things a transcript mentions (people, organisations, places...). A topic is a subject someone chose for the vocabulary, as a SKOS concept: a preferred label, other labels it goes by, a definition, broader and related topics. Narrower topics are the ones that name it as broader. Recordings are about topics (`topic_about`), -put there by a person, by turning an entity of type TERM into a topic, or (suggested) by analysis. +put there by a person, by turning an entity of type TERM into a topic, or (suggested) by analysis or an assistant. topic {space, key, tkey: ":", label, alt: [labels], definition, broader: [ids], related: [ids], origin: {entity}, created, updated, by} @@ -21,7 +21,7 @@ from collections import Counter -from . import analyze, graph_history, store +from . import analyze, graph_history, notebook, store R = store.R FIELDS = "record::id(id) AS id, space, key, label, alt, definition, broader, related, origin, created, updated, by" @@ -29,7 +29,7 @@ ALT_MAX = 50 DEFINITION_MAX = 2000 LINKS_MAX = 50 -SOURCES = ("person", "entity", "analysis") +SOURCES = ("person", "entity", "analysis", "assistant") STATUSES = ("accepted", "suggested", "dismissed") SUGGEST_MIN = 2 # times a transcript says a label before analysis suggests its topic SUMMARY_WEIGHT = 5 # what a summary naming a topic adds to its weight @@ -227,6 +227,7 @@ def delete(db, tid): _set_related(db, tid, t.get("related") or [], []) db.q("DELETE topic_about WHERE topic = $t", t=tid) db.q("DELETE $r", r=R("topic", tid)) + notebook.release(db, f"topic:{tid}") # its page stays, as a free note eid = (t.get("origin") or {}).get("entity") if eid and db.one("SELECT id FROM $r", r=R("entity", int(eid))): with graph_history.change(db, "entity.show", entities=[int(eid)], why=f"topic {tid} was deleted"): @@ -271,6 +272,7 @@ def merge(db, keep, others, user=None): db.q("UPDATE $r SET broader = $b, related = $rel", r=R("topic", t["id"]), b=nb, rel=nr) for o in others: db.q("DELETE $r", r=R("topic", o)) + notebook.release(db, f"topic:{o}") broader = [b for b in dict.fromkeys(broader) if b not in ids and b != keep] if _would_loop(db, k["space"], keep, broader): broader = list(k.get("broader") or []) @@ -328,6 +330,27 @@ def tag(db, tid, recordings, remove=False, user=None): return len(rids) +def propose(db, tid, recordings, source="assistant", user=None): + """Suggest a topic for recordings of its namespace, for someone to accept: what already holds, waits or was + dismissed stays as it is. Returns the recordings it is now suggested for.""" + t = _topic(db, tid) + rids = [int(r) for r in dict.fromkeys(recordings or [])] + if rids: + found = { + r["id"]: r["space"] + for r in db.rows("SELECT record::id(id) AS id, space FROM recording WHERE id IN $ids", ids=[R("recording", i) for i in rids]) + } + for r in rids: + if found.get(r) != t["space"]: + raise KeyError(r) + made = [] + for r in rids: + if not db.one("SELECT status FROM $r", r=_about_id(r, t["id"])): + _about(db, t["space"], r, t["id"], source, None, "suggested", user) + made.append(r) + return made + + def from_entity(db, eid, user=None): """Make an entity of type TERM a topic: its name the label, its aliases other labels, its description the definition, and the recordings that mention it about it. The entity is hidden (deleting the topic shows it again). diff --git a/fastapi_backend/app/schemas/notebook.py b/fastapi_backend/app/schemas/notebook.py index eca0a22fa..5b4ce8b5a 100644 --- a/fastapi_backend/app/schemas/notebook.py +++ b/fastapi_backend/app/schemas/notebook.py @@ -53,7 +53,7 @@ class NoteBacklink(ResponseModel): class NotePage(NotePageItem): namespace: str - body: str = Field("", description="Markdown, with mentions written @[label](kind:id) and #[label](entity:id)") + body: str = Field("", description="Markdown, with mentions written @[label](kind:id) and #[label](topic:id)") doc: str | None = Field(None, description="the editor's own document state, if it kept one") doc_stale: bool | None = Field(None, description="the body changed without the editor: its text follows the body") created_by: str | None = Field(None, description="its writer's email") diff --git a/fastapi_backend/app/schemas/topics.py b/fastapi_backend/app/schemas/topics.py index 665011e66..1f1734964 100644 --- a/fastapi_backend/app/schemas/topics.py +++ b/fastapi_backend/app/schemas/topics.py @@ -17,7 +17,7 @@ class TopicRef(ResponseModel): class TopicAbout(ResponseModel): recording: int title: str | None = None - source: str | None = Field(None, description="how it got there: person, entity or analysis") + source: str | None = Field(None, description="how it got there: person, entity, analysis or assistant") weight: float | None = None status: Literal["accepted", "suggested", "dismissed"] | None = None diff --git a/fastapi_backend/tests/api/test_chats.py b/fastapi_backend/tests/api/test_chats.py index 6896fd561..159fe8268 100644 --- a/fastapi_backend/tests/api/test_chats.py +++ b/fastapi_backend/tests/api/test_chats.py @@ -7,7 +7,7 @@ import pytest -from app.domain import ai_tools, auth, chat, store +from app.domain import ai_tools, auth, chat, store, topics from app.domain import llm as llm_mod from tests import fake_llm from tests.api._assist import Assist, sse, start_llm @@ -556,3 +556,24 @@ def test_the_assistant_queries_the_graph(app, db, cfg, folder, new_client, llm): one = ai_tools.Toolbox(db, cfg, {"id": 1, "email": "e"}, pods, set(), {"recordings": [s.b]}, None) recs = json.loads(one.call("graph_query", {"query": "MATCH (r:Recording) RETURN id(r)"})[0])["rows"] assert recs == [[f"r{s.b}"]] + + +def test_the_assistant_finds_and_suggests_topics(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + pods = {s.pods} + bio = topics.create(db, s.pods, "Biology") + gene = topics.create(db, s.pods, "Gene therapy", alt=["GT"], broader=[bio]) + topics.tag(db, gene, [s.a]) + viewer = ai_tools.Toolbox(db, cfg, {"id": 1, "email": "e"}, pods, set(), {}, None) + names = {sp["function"]["name"] for sp in viewer.specs()} + assert {"find_topics", "topic_recordings"} <= names and "suggest_topic" not in names # suggesting needs an editor + found = json.loads(viewer.call("find_topics", {"query": "gt"})[0]) + assert found["topics"] == [ + {"id": gene, "label": "Gene therapy", "namespace": "pods", "also": ["GT"], "broader": ["Biology"], "recordings": 1} + ] + t = json.loads(viewer.call("topic_recordings", {"topic_id": gene})[0]) + assert [r["recording_id"] for r in t["recordings"]] == [s.a] and t["broader"] == [{"id": bio, "label": "Biology"}] + editor = ai_tools.Toolbox(db, cfg, {"id": 1, "email": "e"}, pods, pods, {}, None) + out = json.loads(editor.call("suggest_topic", {"topic_id": gene, "recording_ids": [s.a, s.b]})[0]) + assert out["suggested_for"] == [s.b] # what holds stays; the rest waits for someone to accept + assert [(x["label"], x["source"], x["status"]) for x in topics.of_recording(db, s.b)] == [("Gene therapy", "assistant", "suggested")] diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index f99112f70..38d9626e9 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -10,6 +10,7 @@ from fastapi.testclient import TestClient from app.api import mcp, mcp_tools +from app.domain import store, topics from tests.api.test_oauth import O, _bearer, _grant from tests.helpers import login, make_user, seed @@ -69,8 +70,8 @@ def test_clients_are_sent_to_sign_in(client, app, env): tools = rpc(client, env["h"], "tools/list")["result"]["tools"] assert [t["name"] for t in tools] == list(mcp_tools.TOOLS) assert all(t["inputSchema"]["type"] == "object" for t in tools) - # every tool only reads, except asking for a graph change (which needs the write scope) - assert [t["name"] for t in tools if not t["annotations"]["readOnlyHint"]] == ["propose_graph_change"] + # every tool only reads, except suggesting a topic and asking for a graph change (which need the write scope) + assert [t["name"] for t in tools if not t["annotations"]["readOnlyHint"]] == ["suggest_topic", "propose_graph_change"] assert next(t for t in tools if t["name"] == "search")["inputSchema"]["required"] == ["query"] # API tokens and sessions work too assert rpc(client, env["hv"], "ping")["result"] == {} @@ -361,3 +362,30 @@ def test_graph_tools(client, db, env): assert tool(client, h, "graph_paths", from_node=alice, to_node=f"e{dyno}", namespace="pods")["paths"] assert tool_error(client, h, "graph_schema", namespace="calls") # not theirs assert "read-only" in tool_error(client, h, "propose_graph_change", kind="merge", a=f"e{dyno}", b="e1") + + +def test_topic_tools(client, db, env): + h = env["h"] # a read-only app token for a pods viewer + a, b, _ = env["ids"] + pods = store.ns_id(db, "pods") + bio = topics.create(db, pods, "Biology") + gene = topics.create(db, pods, "Gene therapy", alt=["GT"], broader=[bio]) + topics.tag(db, gene, [a]) + found = tool(client, h, "list_topics", query="gt") + assert [(t["topic_id"], t["label"], t["broader_ids"], t["recordings"]) for t in found["topics"]] == [(gene, "Gene therapy", [bio], 1)] + assert [t["label"] for t in tool(client, h, "list_topics", top=True)["topics"]] == ["Biology"] + t = tool(client, h, "get_topic", topic_id=gene) + assert t["broader"] == [{"topic_id": bio, "label": "Biology"}] + assert [(r["recording_id"], r["status"]) for r in t["recordings"]] == [(a, "accepted")] and t["recordings"][0]["url"].endswith( + f"/resources/{a}" + ) + assert "read-only" in tool_error(client, h, "suggest_topic", topic_id=gene, recording_ids=[b]) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + he = login(client, "ed@x.io", "editor password 1") + assert tool(client, he, "suggest_topic", topic_id=gene, recording_ids=[a, b])["suggested_for"] == [b] # a already holds + assert [(r["recording_id"], r["status"]) for r in tool(client, h, "get_topic", topic_id=gene)["recordings"]] == [ + (a, "accepted"), + (b, "suggested"), + ] + assert "list of whole numbers" in tool_error(client, he, "suggest_topic", topic_id=gene, recording_ids=["x"]) + assert tool_error(client, he, "suggest_topic", topic_id=gene, recording_ids=[env["ids"][2]]) # calls isn't theirs diff --git a/fastapi_backend/tests/api/test_notebook.py b/fastapi_backend/tests/api/test_notebook.py index a2bc0cb8e..de587fc79 100644 --- a/fastapi_backend/tests/api/test_notebook.py +++ b/fastapi_backend/tests/api/test_notebook.py @@ -5,7 +5,7 @@ import pytest -from app.domain import deletion, entities, moving, notebook, store +from app.domain import deletion, entities, moving, notebook, store, topics from tests.helpers import login, make_user, seed R = store.R @@ -83,21 +83,23 @@ def test_tree(client, env): def test_links_and_backlinks(client, env, db): he, a = env["he"], env["a"] - topic, person = _entity(db, "Capsids", "TERM"), _entity(db, "Ada Lovelace", "PERSON") - # what @ and # offer - hits = client.get("/api/v1/notes/targets?ns=pods&sign=%23", headers=he).json() - assert hits and {h["kind"] for h in hits} == {"topic"} + term, person = _entity(db, "Capsids", "TERM"), _entity(db, "Ada Lovelace", "PERSON") + tid = topics.create(db, store.ns_id(db, "pods"), "Capsid design", alt=["AAV capsids"]) + # what @ and # offer: # the topics, found by any of their labels + hits = client.get("/api/v1/notes/targets?ns=pods&sign=%23&q=aav", headers=he).json() + assert hits == [{"target": f"topic:{tid}", "label": "Capsid design", "kind": "topic"}] hits = client.get(f"/api/v1/notes/targets?ns=pods&q={person['name'][:3]}", headers=he).json() assert f"entity:{person['id']}" in [h["target"] for h in hits] other = _new(client, he, title="Background") body = ( - f"Met @[{person['name']}](entity:{person['id']}) about #[{topic['name']}](entity:{topic['id']}) in " + f"Met @[{person['name']}](entity:{person['id']}) about #[Capsid design](topic:{tid}) and #[{term['name']}](entity:{term['id']}) in " f"@[the call](recording:{a}); see @[bg](page:{other['id']}) and @[gone](recording:999999)." ) p = _new(client, he, title="Meeting", body=body) assert [(x["sign"], x["target"], x["name"] is not None) for x in p["links"]] == [ ("@", f"entity:{person['id']}", True), - ("#", f"entity:{topic['id']}", True), + ("#", f"topic:{tid}", True), + ("#", f"entity:{term['id']}", True), # links to terms written before topics still work ("@", f"recording:{a}", True), ("@", f"page:{other['id']}", True), ("@", "recording:999999", False), @@ -106,6 +108,8 @@ def test_links_and_backlinks(client, env, db): assert [b["title"] for b in client.get(f"/api/v1/notes/{other['id']}", headers=he).json()["backlinks"]] == ["Meeting"] draft = client.get(f"/api/v1/notes/about/recording/{a}", headers=he).json() assert draft["id"] is None and draft["about"] == f"recording:{a}" and [b["title"] for b in draft["backlinks"]] == ["Meeting"] + t = client.get(f"/api/v1/notes/about/topic/{tid}", headers=he).json() # a topic has a page of its own too + assert (t["title"], [b["title"] for b in t["backlinks"]]) == ("Capsid design", ["Meeting"]) # the body's plain text drops the link syntax assert db.one("SELECT text FROM $r", r=R("note_page", p["id"]))["text"].startswith(f"Met {person['name']} about") # a new body replaces the links diff --git a/nextjs-frontend/__tests__/notes-links.test.ts b/nextjs-frontend/__tests__/notes-links.test.ts index 5db7b8f33..65a8b52da 100644 --- a/nextjs-frontend/__tests__/notes-links.test.ts +++ b/nextjs-frontend/__tests__/notes-links.test.ts @@ -5,6 +5,7 @@ describe("links in notes", () => { expect(hrefFor("page:3")).toBe("/notes/3"); expect(hrefFor("recording:12")).toBe("/resources/12"); expect(hrefFor("entity:5", "pods")).toBe("/entities?entity=5&ns=pods"); + expect(hrefFor("topic:2", "pods")).toBe("/topics/2"); expect(hrefFor("speaker:7")).toBe("/speakers/7"); expect(hrefFor("collection:4", "pods")).toBe("/library?namespace=pods&collection=4"); }); diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 0e702be0a..d1bd08d4a 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -3767,7 +3767,7 @@ export class Notes { /** * Page About * - * The page of a recording, entity, collection or speaker: its page, or a draft while nobody has written one. + * The page of a recording, entity, topic, collection or speaker: its page, or a draft while nobody has written one. */ public static pageAbout( options: Options, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 9987a7b90..9b9e300ae 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -7324,7 +7324,7 @@ export type NotePage = { /** * Body * - * Markdown, with mentions written @[label](kind:id) and #[label](entity:id) + * Markdown, with mentions written @[label](kind:id) and #[label](topic:id) */ body?: string; /** @@ -13858,7 +13858,7 @@ export type TopicAbout = { /** * Source * - * how it got there: person, entity or analysis + * how it got there: person, entity, analysis or assistant */ source?: string | null; /** diff --git a/nextjs-frontend/components/notes/links.ts b/nextjs-frontend/components/notes/links.ts index ecd47a14b..9ee3e991c 100644 --- a/nextjs-frontend/components/notes/links.ts +++ b/nextjs-frontend/components/notes/links.ts @@ -1,4 +1,4 @@ -/** Where a link in a note goes in the app: "recording:12", "entity:5", "page:3", "collection:4", "speaker:7". */ +/** Where a link in a note goes in the app: "recording:12", "entity:5", "topic:2", "page:3", "collection:4", "speaker:7". */ export function hrefFor(target: string, ns?: string | null): string | null { const [kind, id] = target.split(":"); if (!id || !/^\d+$/.test(id)) return null; @@ -10,6 +10,8 @@ export function hrefFor(target: string, ns?: string | null): string | null { return `/resources/${id}`; case "entity": return `/entities?entity=${id}${q}`; + case "topic": + return `/topics/${id}`; case "speaker": return `/speakers/${id}`; case "collection": diff --git a/nextjs-frontend/components/topics/topic-drawer.tsx b/nextjs-frontend/components/topics/topic-drawer.tsx index 4236422cd..541212947 100644 --- a/nextjs-frontend/components/topics/topic-drawer.tsx +++ b/nextjs-frontend/components/topics/topic-drawer.tsx @@ -1,7 +1,7 @@ "use client"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Check, Merge, Trash2, X } from "lucide-react"; +import { Check, Merge, NotebookPen, Trash2, X } from "lucide-react"; import Link from "next/link"; import { useEffect, useState } from "react"; @@ -128,15 +128,21 @@ export function TopicDrawer({ title={d?.label ?? "Topic"} width={460} actions={ - d && - canEdit && ( + d && ( <> - - + {canEdit && ( + <> + + + + )} ) } diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index f14cf615b..94236cafe 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -7125,7 +7125,7 @@ "notes" ], "summary": "Page About", - "description": "The page of a recording, entity, collection or speaker: its page, or a draft while nobody has written one.", + "description": "The page of a recording, entity, topic, collection or speaker: its page, or a draft while nobody has written one.", "operationId": "page_about", "parameters": [ { @@ -34559,7 +34559,7 @@ "body": { "type": "string", "title": "Body", - "description": "Markdown, with mentions written @[label](kind:id) and #[label](entity:id)", + "description": "Markdown, with mentions written @[label](kind:id) and #[label](topic:id)", "default": "" }, "doc": { @@ -45941,7 +45941,7 @@ } ], "title": "Source", - "description": "how it got there: person, entity or analysis" + "description": "how it got there: person, entity, analysis or assistant" }, "weight": { "anyOf": [