From 6b5dae543d1f5deb915d6510a53d27f8ab1063d4 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:42:06 +0000 Subject: [PATCH 1/5] docs(devlog): plan for a versioned, replayable graph Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCB28xsaiCpiDKKk2JjRpe --- DEVLOG.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/DEVLOG.md b/DEVLOG.md index e6e642a46..b97e261af 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,40 @@ Plans and progress for work in flight. Newest first. +## 2026-10-04 · Graph history: versions, replay, rollback + +Goal (Adeel): make the graph replayable and versioned. + +Model (fastapi_backend/app/domain/graph_history.py, docs/graph.md "History"): + +- Every change to the curated entity graph is one append-only `graph_event`: its number is the graph's version. It + says when, who (a person, a routine, the assistant, analysis), through what (web app, API token, OAuth app, + assistant, MCP, routine, analysis), why, and for every record it touched (entity, alias, cross-namespace link, + "not the same" pair) the row before and after. +- Built on what is there: graph changes (organize.py: proposed, applied, dismissed, undone) and merges (entity_merge) + are recorded through the same entity functions, and their events point back at them. +- As of: the graph at any version, rebuilt by walking back from today's rows. Diff between two versions. An entity's + history. Versions can be named ("before the cleanup"). +- Rollback to a version: a preview first, then later changes are undone as new events, so a rollback can itself be + undone. Merges are undone with their mentions. +- Replay: checkpoints (the graph at a version) plus the events after them rebuild the graph; verify compares the + replay with the live rows and records any difference as an event, so nothing stays unrecorded. +- Not versioned here: mentions (analysis makes them from transcripts, whose edits have their own history). As-of views + use today's mention counts. + +Todo: + +- [ ] Event log with who, through what and why; every curation path records (entity page, assistant, MCP, routines + and graph changes, analysis, the entity map) +- [ ] Versions API: list, the graph as of a version, diff, an entity's history, named versions +- [ ] Rollback to a version, with a preview +- [ ] Checkpoints, replay and verify (`lens graph verify`, nightly checkpoint) +- [ ] The explorer and Cypher as of a version (with the graph explorer thread) +- [ ] Web app: a history timeline with diffs and rollback +- [ ] Graph events in each resource's activity history (with the activity history thread) + +Refine later: speakers (same-person links, names) in the history; a mention-level history. + ## 2026-10-04 · The graph, end to end Goal (Adeel): make the graph the one focus and nail it: a human explorer canvas, agent queries with rights, questions From 6396a57c21869ba74791bf7ce53603e70cb16bb8 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:51:45 +0000 Subject: [PATCH 2/5] feat(graph): record every entity graph change as a versioned event, with as-of, diffs and named versions Every change to entities, their other names, links across namespaces and distinct pairs is one append-only graph_event with who, through what and why, and the rows before and after. The graph can be read as of any version, compared between versions, and an entity's history listed. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCB28xsaiCpiDKKk2JjRpe --- CHANGELOG.md | 5 + DEVLOG.md | 2 +- docs/graph-history.md | 73 +++ fastapi_backend/app/api/deps.py | 9 +- fastapi_backend/app/api/mcp_tools.py | 5 +- fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/entities.py | 5 +- .../app/api/v1/routes/graph_history.py | 116 +++++ fastapi_backend/app/core/middleware.py | 5 +- fastapi_backend/app/domain/ai_tools.py | 23 +- fastapi_backend/app/domain/analyze.py | 74 +-- fastapi_backend/app/domain/entities.py | 61 ++- fastapi_backend/app/domain/entity_map.py | 73 +-- fastapi_backend/app/domain/graph_history.py | 479 ++++++++++++++++++ fastapi_backend/app/domain/organize.py | 63 ++- fastapi_backend/app/domain/store.py | 3 + .../tests/api/test_graph_history.py | 56 ++ .../tests/domain/test_graph_history.py | 156 ++++++ mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 34 ++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 124 +++++ .../app/openapi-client/types.gen.ts | 278 ++++++++++ nextjs-frontend/openapi.json | 440 ++++++++++++++++ 23 files changed, 1974 insertions(+), 113 deletions(-) create mode 100644 docs/graph-history.md create mode 100644 fastapi_backend/app/api/v1/routes/graph_history.py create mode 100644 fastapi_backend/app/domain/graph_history.py create mode 100644 fastapi_backend/tests/api/test_graph_history.py create mode 100644 fastapi_backend/tests/domain/test_graph_history.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fc1854751..74f4a1a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **The entity graph keeps its history.** Every change to entities, their other names, links across namespaces and + "not the same" pairs is recorded as a numbered version: what changed, who (a person, a routine, the assistant, + analysis), through what (the web app, a token, an app, MCP) and why. `GET /api/v1/graph/history` lists versions + (one entity's with `entity`), `/graph/as-of/{version}` shows the graph as it was, `/graph/diff` what changed + between two versions, and `/graph/tags` names versions to come back to. See docs/graph-history.md. - **Query the graph in Cypher, and walk it.** The archive is now a property graph of namespaces, collections, recordings, speakers and entities. `POST /api/v1/graph/query` runs read-only Cypher (the language of Neo4j and ISO GQL) over the namespaces you can read; `/graph/related` gives a node's parents, children, ancestors, descendants or diff --git a/DEVLOG.md b/DEVLOG.md index b97e261af..8eb34d74a 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -6,7 +6,7 @@ Plans and progress for work in flight. Newest first. Goal (Adeel): make the graph replayable and versioned. -Model (fastapi_backend/app/domain/graph_history.py, docs/graph.md "History"): +Model (fastapi_backend/app/domain/graph_history.py, docs/graph-history.md): - Every change to the curated entity graph is one append-only `graph_event`: its number is the graph's version. It says when, who (a person, a routine, the assistant, analysis), through what (web app, API token, OAuth app, diff --git a/docs/graph-history.md b/docs/graph-history.md new file mode 100644 index 000000000..95ca29da5 --- /dev/null +++ b/docs/graph-history.md @@ -0,0 +1,73 @@ +# Graph history + +Every change to the entity graph is kept, in order, as an event that is never changed or deleted. Its number is the +graph's version. So the graph can be seen as it was at any version, two versions compared, and one entity's history +read: who renamed it, who merged what into it, which routine linked it, and why. + +Built: the event log, versions, as-of, diffs, entity history, named versions. Planned: rollback to a version, +checkpoints with replay and verify, the explorer and Cypher as of a version, a history timeline in the web app +(DEVLOG.md). + +## What is versioned + +What people and agents curate: + +- **Entities**: name, type, description, hidden (and why), on the fixed list, Unknown and Unlabeled. +- **Other names** (`entity_alias`): the ways a name is said that point at an entity, from merges, renames, the + entity page and names the model placed. +- **Links**: the same thing in two namespaces. +- **Distinct pairs**: two entities someone said are not one, so they aren't suggested again. + +Mentions are not versioned: analysis makes them from transcripts, and transcript edits have their own history. An +as-of view uses today's mentions. A moved or removed mention is still an event (`mention.move`, `mention.remove`), so +it shows in both entities' histories. + +## An event + +| Field | What it holds | +| --- | --- | +| `version` | The graph's version after this change. | +| `at` | When. | +| `op` | What happened: `entity.rename`, `entity.describe`, `entity.retype`, `entity.hide`, `entity.show`, `entity.define`, `entity.undefine`, `entity.aliases`, `entity.delete`, `entity.merge`, `entity.unmerge`, `link.add`, `link.remove`, `distinct.add`, `mention.move`, `mention.remove`, `merge.apply`, `link.apply`, `merge.accept`, `link.accept`, `merge.dismiss`, `merge.undo`, `link.undo`, `analysis`. | +| `actor` | Who: the person's email, `routine:`, `analysis`, or `system`. | +| `via` | Through what: `web`, `token` (an API token), `oauth` (an app), `assistant`, `mcp`, `routine`, `workflow`, `analysis`, `cli`, `system`. | +| `why` | The reason given: a graph change's reason, the model's verdict, why an entity was hidden. | +| `origin` | What it came from: `graph_change`, `merge` (the entity_merge to undo), `routine`, `run`, `workflow`, `recording`, `approval` (an assistant approval), `tool` (an MCP tool), `mention`. | +| `spaces`, `entities` | The namespaces and entities it touched. | +| `ops` | Every record it changed: `{t: table, k: key, b: the row before, a: the row after, s: namespaces}`; no `b` means it was made, no `a` that it went. | + +One action is one event: a merge (the other entity going, its names becoming other names of the one kept, its links +moving) is one `entity.merge`, and a routine's merge is one `merge.apply` that points at its graph change. + +## The API + +All calls see only the namespaces the caller can read (`namespace` narrows it to one). An event that touched a +namespace the caller can't read (a link across namespaces) isn't listed for them. A version is a number, a version's +name, or `head`. + +| Call | What it does | +| --- | --- | +| `GET /api/v1/graph/history?namespace=&entity=&before=&limit=` | Versions, newest first, with what changed, who, through what, why and the entities' names; `entity` gives one entity's history; `next` pages back. | +| `GET /api/v1/graph/history/{version}` | One version in full, every record before and after. | +| `GET /api/v1/graph/as-of/{version}?namespace=` | The graph at a version: entities with their other names, links and distinct pairs. | +| `GET /api/v1/graph/diff?from=&to=&namespace=` | What changed between two versions: entities added, removed and changed field by field; other names, links and distinct pairs added and removed. | +| `GET /api/v1/graph/tags` | Named versions. | +| `POST /api/v1/graph/tags` `{name, version, note}` | Name a version (default: the current one), such as "before the cleanup". Editors and admins. | +| `DELETE /api/v1/graph/tags/{name}` | Drop a name; the version stays. | + +## How it works + +Code that changes the graph wraps its writes in `graph_history.change(db, op, entities=...)`. It reads the rows of +those entities (and the other names, links and distinct pairs that name them) before and after, and records what +differs as one event; a change made inside another joins it. Who and why come from `graph_history.acting(...)`: each +API request sets the caller and whether they came through the web app, a token or an app; the assistant, MCP tools, +routines, workflows and analysis set their own. + +The graph as of a version is today's rows with every later event walked back, newest first. A diff composes the +events between two versions: for each record, how it was before the first and after the last. + +## Refine later + +- Speakers (same-person links, names) in the history. +- A mention-level history. +- Checkpoints so very old versions don't walk back through every later event. diff --git a/fastapi_backend/app/api/deps.py b/fastapi_backend/app/api/deps.py index b10a4e821..7c9f6517f 100644 --- a/fastapi_backend/app/api/deps.py +++ b/fastapi_backend/app/api/deps.py @@ -24,7 +24,7 @@ from app.core import security from app.domain import access as acc -from app.domain import auth, hierarchy, ipgroups, oauth, store +from app.domain import auth, graph_history, hierarchy, ipgroups, oauth, store from app.domain.store import DB Config = dict[str, Any] @@ -64,6 +64,13 @@ def as_audit(self) -> dict[str, Any]: def _principal(request: Request, db: DB) -> Principal | None: + p = _resolve(request, db) + if p: # graph changes made in this request are recorded as theirs (graph_history.py) + graph_history.note(actor=p.email, via={"access": "web", "token": "token", "oauth": "oauth"}[p.via]) + return p + + +def _resolve(request: Request, db: DB) -> Principal | None: if hasattr(request.state, "principal"): return request.state.principal p: Principal | None = None diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index a54347de6..e94cd5d9b 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -23,7 +23,7 @@ from app.api.v1.routes import namespaces as namespace_routes from app.api.v1.routes import recordings as recording_routes from app.api.v1.routes import search as search_routes -from app.domain import analyze, library, rdf, render, store +from app.domain import analyze, graph_history, library, rdf, render, store from app.domain import entities as ents from app.domain import speakers as spk from app.domain.store import DB @@ -206,7 +206,8 @@ def call(ctx: Context, name: str, arguments: dict[str, Any]) -> dict[str, Any]: if unknown: raise ToolError(f"unknown argument {', '.join(unknown)}; {name} takes {', '.join(a.name for a in t.args) or 'none'}") values = {a.name: a.parse(arguments.get(a.name)) for a in t.args} - return _text(t.run(ctx, **values)) + with graph_history.acting(via="mcp", tool=name): + return _text(t.run(ctx, **values)) except ToolError as e: return _text({"error": str(e)}, error=True) except HTTPException as e: diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 85fb24028..ae13cac91 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -20,6 +20,7 @@ fields, files, graph, + graph_history, hierarchy, iiif, imports, @@ -78,6 +79,7 @@ speakers, entities, graph, + graph_history, entity_setup, metadata, rdf, diff --git a/fastapi_backend/app/api/v1/routes/entities.py b/fastapi_backend/app/api/v1/routes/entities.py index f3a54812e..b7499c5e4 100644 --- a/fastapi_backend/app/api/v1/routes/entities.py +++ b/fastapi_backend/app/api/v1/routes/entities.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, HTTPException, Request from app.api.deps import Access, Acl, CurrentUser, Db, Writer, domain_errors -from app.domain import auth, entity_map, entity_setup, hierarchy, jobs +from app.domain import auth, entity_map, entity_setup, graph_history, hierarchy, jobs from app.domain import entities as ents from app.domain.store import DB, R from app.schemas.common import Ok @@ -258,7 +258,8 @@ def update_entity(eid: int, body: EntityUpdate, request: Request, user: Writer, if body.defined is not None: if db.one("SELECT builtin FROM $r", r=R("entity", eid)).get("builtin"): raise ValueError("Unknown and Unlabeled are always there.") - db.q("UPDATE $r SET defined = $d", r=R("entity", eid), d=body.defined) + with graph_history.change(db, "entity.define" if body.defined else "entity.undefine", entities=[eid]): + db.q("UPDATE $r SET defined = $d", r=R("entity", eid), d=body.defined) if body.defined: entity_map.builtins(db, sid) auth.audit(db, user.as_audit(), "entity.update", f"entity:{eid}", changes) diff --git a/fastapi_backend/app/api/v1/routes/graph_history.py b/fastapi_backend/app/api/v1/routes/graph_history.py new file mode 100644 index 000000000..d7b1ba0be --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/graph_history.py @@ -0,0 +1,116 @@ +"""The entity graph's history (docs/graph-history.md): every change as a numbered version, with who made it, through +what and why; the graph as of any version; what changed between two versions; and names for versions. Every call +sees only the namespaces its caller can read (one with `namespace`), and an event only when all its namespaces are +readable. A version is a number, a version's name, or `head`. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, HTTPException, Query +from pydantic import BaseModel, Field + +from app.api.deps import Acl, CurrentUser, Db, Writer, domain_errors +from app.domain import auth, graph_history + +router = APIRouter(tags=["graph"]) +NAMESPACE = Query(None, description="one namespace (default: every namespace you can read)") + + +def _spaces(acl, namespace: str | None) -> set[int] | None: + if namespace: + return {acl.namespace(namespace)} + return acl.readable() + + +def _version(db, ref: str | None) -> int: + try: + return graph_history.resolve(db, ref) + except KeyError: + raise HTTPException(404, f"no version is called {ref}") from None + + +@router.get("/graph/history") +def graph_history_list( + user: CurrentUser, + acl: Acl, + db: Db, + namespace: str | None = NAMESPACE, + entity: int | None = Query(None, description="only the changes that touched this entity"), + before: int | None = Query(None, ge=1, description="older than this version (the next page)"), + limit: int = Query(50, ge=1, le=500), +) -> dict[str, Any]: + """The graph's versions, newest first: what changed, who changed it, through what (web, token, oauth, assistant, + mcp, routine, workflow, analysis, cli, system) and why, with the entities each touched.""" + spaces = _spaces(acl, namespace) + rows = graph_history.versions(db, spaces, entity, before, limit) + return {"head": graph_history.head(db), "versions": rows, "next": rows[-1]["version"] if len(rows) == limit else None} + + +@router.get("/graph/history/{version}") +def graph_history_event(version: str, user: CurrentUser, acl: Acl, db: Db) -> dict[str, Any]: + """One version in full: every record it changed, as it was before and after.""" + try: + return graph_history.event(db, _version(db, version), acl.readable()) + except KeyError: + raise HTTPException(404, "not found") from None + + +@router.get("/graph/as-of/{version}") +def graph_as_of(version: str, user: CurrentUser, acl: Acl, db: Db, namespace: str | None = NAMESPACE) -> dict[str, Any]: + """The graph as it was at a version: entities (with their other names), cross-namespace links and the pairs someone + said are different. Mentions aren't versioned: counts elsewhere are today's.""" + with domain_errors(): + return graph_history.as_of(db, _version(db, version), _spaces(acl, namespace)) + + +@router.get("/graph/diff") +def graph_diff( + user: CurrentUser, + acl: Acl, + db: Db, + from_: str = Query(..., alias="from", description="a version, a version's name, or head"), + to: str = Query("head", description="a version, a version's name, or head"), + namespace: str | None = NAMESPACE, +) -> dict[str, Any]: + """What changed between two versions: entities added, removed and changed (field by field), and other names, + links and distinct pairs added and removed.""" + with domain_errors(): + return graph_history.diff(db, _version(db, from_), _version(db, to), _spaces(acl, namespace)) + + +class TagAsk(BaseModel): + name: str = Field(min_length=1, max_length=80) + version: str | None = Field(None, description="a version or a version's name (default: head)") + note: str | None = Field(None, max_length=300) + + +@router.get("/graph/tags") +def graph_tags(user: CurrentUser, db: Db) -> list[dict[str, Any]]: + """Named versions, newest first.""" + return graph_history.tags(db) + + +@router.post("/graph/tags") +def graph_tag(body: TagAsk, user: Writer, acl: Acl, db: Db) -> dict[str, Any]: + """Name a version ("before the cleanup") to come back to it; naming again moves the name. Editors of a namespace + (or admins) can.""" + if not (user.admin or acl.editable()): + raise HTTPException(403, "naming versions needs editor access to a namespace") + with domain_errors(): + out = graph_history.tag(db, body.name, _version(db, body.version), body.note, user.email) + auth.audit(db, user.as_audit(), "graph.tag", f"graph_tag:{out['name']}", out) + return out + + +@router.delete("/graph/tags/{name}") +def graph_untag(name: str, user: Writer, acl: Acl, db: Db) -> dict[str, Any]: + if not (user.admin or acl.editable()): + raise HTTPException(403, "naming versions needs editor access to a namespace") + try: + graph_history.untag(db, name) + except KeyError: + raise HTTPException(404, "not found") from None + auth.audit(db, user.as_audit(), "graph.untag", f"graph_tag:{name}") + return {"ok": True} diff --git a/fastapi_backend/app/core/middleware.py b/fastapi_backend/app/core/middleware.py index 9a838dbbc..5abac260f 100644 --- a/fastapi_backend/app/core/middleware.py +++ b/fastapi_backend/app/core/middleware.py @@ -10,6 +10,8 @@ from fastapi.responses import PlainTextResponse from starlette.types import ASGIApp, Receive, Scope, Send +from app.domain import graph_history + IIIF_PRIVATE = ("/iiif/auth/access", "/iiif/auth/token", "/iiif/auth/logout") RESOURCES, RECORDINGS = "/api/v1/resources", "/api/v1/recordings" @@ -70,7 +72,8 @@ async def guard(request: Request, call_next: Callable[[Request], Awaitable[Respo frames = " ".join(server["embed_frame_ancestors"]) or "'none'" else: frames = "'none'" - resp = await call_next(request) + with graph_history.acting(via="web"): # the caller fills in who (deps._principal) + resp = await call_next(request) path = request.url.path # Built-in reports carry their player inline; reports rendered from people's templates may not run scripts at all. script = ("'none'" if "--" in path else "'self' 'unsafe-inline'") if path.startswith("/reports/") else "'self'" diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 983dabc5d..4b121e9c5 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -15,6 +15,7 @@ extensions, entity_map, entity_setup, + graph_history, ops_tools, recsets, render, @@ -646,6 +647,19 @@ def approve(db, cfg, aid, user, editable, decision="approve", base=None, admin=F "UPDATE $r SET status = 'declined', decided_at = $t, decided_by = $u", r=R("approval", a["id"]), t=store.now(), u=user["email"] ) return {"status": "declined"} + with graph_history.acting(actor=user["email"], via="assistant", approval=a["id"]): + result = _carry_out(db, cfg, a, user, editable, decision, base, admin, readable) + db.q( + "UPDATE $r SET status = 'done', decided_at = $t, decided_by = $u, result = $res", + r=R("approval", a["id"]), + t=store.now(), + u=user["email"], + res=result, + ) + return {"status": "done", **result} + + +def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable): args = a["args"] if a["tool"] == "extension": box = Toolbox(db, cfg, user, readable if readable is not None else editable, editable, None, a["chat"], base, admin) @@ -697,14 +711,7 @@ def approve(db, cfg, aid, user, editable, decision="approve", base=None, admin=F else: entities.retype(db, [eid], args.get("new_type")) result = {"type": args.get("new_type")} - db.q( - "UPDATE $r SET status = 'done', decided_at = $t, decided_by = $u, result = $res", - r=R("approval", a["id"]), - t=store.now(), - u=user["email"], - res=result, - ) - return {"status": "done", **result} + return result def run_extension(db, cfg, args, user, admin, readable, toolbox=None): diff --git a/fastapi_backend/app/domain/analyze.py b/fastapi_backend/app/domain/analyze.py index 4c1c07a43..09c29bab6 100644 --- a/fastapi_backend/app/domain/analyze.py +++ b/fastapi_backend/app/domain/analyze.py @@ -9,7 +9,7 @@ import urllib.request from collections import Counter, defaultdict -from . import entity_map, entity_setup, store, telemetry +from . import entity_map, entity_setup, graph_history, store, telemetry STOP = set( """a about above after again against all almost also am an and any are aren as at be because been before being @@ -345,6 +345,44 @@ def talk_stats(segs): } +def _place(db, cfg, nid, rec, setup, listed, first, segs, ch): + """{key: entity id} for the names found in a recording: existing entities, their other names, or new ones.""" + if listed: # mapped onto the defined entities, Unlabeled, Unknown or (hybrid) new entities (entity_map.py) + known = entity_map.place(db, cfg, nid, rec.get("collection"), setup, first, [s["text"] for s in segs]) + else: + known = ( + { + r["key"]: r["id"] + for r in db.rows("SELECT record::id(id) AS id, key FROM entity WHERE space = $s AND key IN $k", s=nid, k=sorted(first)) + } + if first + else {} + ) + if first: # names merged into another entity keep pointing at it + known.update( + { + r["key"]: r["entity"] + for r in db.rows("SELECT key, entity FROM entity_alias WHERE space = $s AND key IN $k", s=nid, k=sorted(first)) + } + ) + new = {k: v for k, v in first.items() if k and k not in known} + if new and setup["matching"] == "model": # the model may know a new name as a described entity + picked = entity_map.judge(db, cfg, nid, setup, new, entity_map.described(db, nid), ("new",), [s["text"] for s in segs]) + placed = {k: v for k, v in picked.items() if isinstance(v, int)} + entity_map.learn(db, nid, placed) + known.update(placed) + for key, (name, typ) in first.items(): + if key and key not in known: + known[key] = db.next_id("entity") + ch.created(known[key]) + db.q( + "CREATE $r CONTENT $d", + r=R("entity", known[key]), + d={"space": nid, "key": key, "ekey": f"{nid}:{key}", "name": name, "type": typ}, + ) + return known + + def analyze_recording(db, cfg, rid, seg_ents=None): """Entities, keywords, sections and talk statistics. `seg_ents` (per line, [(name, type)]) replaces the extractor: a workflow's save entities node passes what its nodes found.""" @@ -385,38 +423,8 @@ def analyze_recording(db, cfg, rid, seg_ents=None): for es in seg_ents: for name, typ in es: first.setdefault(alias.get(ent_key(name), ent_key(name)), (name, typ)) - if listed: # mapped onto the defined entities, Unlabeled, Unknown or (hybrid) new entities (entity_map.py) - known = entity_map.place(db, cfg, nid, rec.get("collection"), setup, first, [s["text"] for s in segs]) - else: - known = ( - { - r["key"]: r["id"] - for r in db.rows("SELECT record::id(id) AS id, key FROM entity WHERE space = $s AND key IN $k", s=nid, k=sorted(first)) - } - if first - else {} - ) - if first: # names merged into another entity keep pointing at it - known.update( - { - r["key"]: r["entity"] - for r in db.rows("SELECT key, entity FROM entity_alias WHERE space = $s AND key IN $k", s=nid, k=sorted(first)) - } - ) - new = {k: v for k, v in first.items() if k and k not in known} - if new and setup["matching"] == "model": # the model may know a new name as a described entity - picked = entity_map.judge(db, cfg, nid, setup, new, entity_map.described(db, nid), ("new",), [s["text"] for s in segs]) - placed = {k: v for k, v in picked.items() if isinstance(v, int)} - entity_map.learn(db, nid, placed) - known.update(placed) - for key, (name, typ) in first.items(): - if key and key not in known: - known[key] = db.next_id("entity") - db.q( - "CREATE $r CONTENT $d", - r=R("entity", known[key]), - d={"space": nid, "key": key, "ekey": f"{nid}:{key}", "name": name, "type": typ}, - ) + with graph_history.acting(via="analysis", recording=rid), graph_history.change(db, "analysis", recording=rid) as ch: + known = _place(db, cfg, nid, rec, setup, listed, first, segs, ch) over = { (o["segment"], o["key"]): o["target"] for o in db.rows("SELECT segment, key, target FROM entity_override WHERE recording = $r", r=rid) diff --git a/fastapi_backend/app/domain/entities.py b/fastapi_backend/app/domain/entities.py index 3991259cd..84d7c25ab 100644 --- a/fastapi_backend/app/domain/entities.py +++ b/fastapi_backend/app/domain/entities.py @@ -13,7 +13,7 @@ import re from collections import Counter, defaultdict, deque -from . import analyze, entity_setup, render, store +from . import analyze, entity_setup, graph_history, render, store R = store.R TYPES = { @@ -656,7 +656,8 @@ def suggestions(db, spaces, eid=None, limit=50): # ---------- curation ---------- def not_same(db, a, b): lo, hi = sorted((int(a), int(b))) - db.q("UPSERT $r CONTENT $d", r=R("entity_distinct", f"{lo}-{hi}"), d={"a": lo, "b": hi, "at": store.now()}) + with graph_history.change(db, "distinct.add", entities=[lo, hi]): + db.q("UPSERT $r CONTENT $d", r=R("entity_distinct", f"{lo}-{hi}"), d={"a": lo, "b": hi, "at": store.now()}) def _check_type(db, space, typ): @@ -670,7 +671,8 @@ def retype(db, eids, typ): _not_builtin(e, "retyped") for sp in {e["space"] for e in db.rows("SELECT space FROM entity WHERE id IN $ids", ids=[R("entity", int(i)) for i in eids])}: _check_type(db, sp, typ) - db.q("UPDATE $ids SET type = $t", ids=[R("entity", int(i)) for i in eids], t=typ) + with graph_history.change(db, "entity.retype", entities=eids): + db.q("UPDATE $ids SET type = $t", ids=[R("entity", int(i)) for i in eids], t=typ) def describe(db, eid, description): @@ -679,14 +681,19 @@ def describe(db, eid, description): text = str(description or "").strip() if len(text) > DESCRIPTION_MAX: raise ValueError(f"A description can have up to {DESCRIPTION_MAX} characters.") - db.q("UPDATE $r SET description = $d", r=R("entity", int(eid)), d=text or None) + with graph_history.change(db, "entity.describe", entities=[eid]): + db.q("UPDATE $r SET description = $d", r=R("entity", int(eid)), d=text or None) def hide(db, eid, hidden=True, reason=None): _not_builtin(_entity(db, eid), "hidden") - db.q( - "UPDATE $r SET hidden = $h, hidden_reason = $why", r=R("entity", int(eid)), h=bool(hidden), why=(reason or None) if hidden else None - ) + with graph_history.change(db, "entity.hide" if hidden else "entity.show", entities=[eid], why=reason if hidden else None): + db.q( + "UPDATE $r SET hidden = $h, hidden_reason = $why", + r=R("entity", int(eid)), + h=bool(hidden), + why=(reason or None) if hidden else None, + ) def _add_alias(db, space, key, eid): @@ -733,10 +740,11 @@ def rename(db, eid, name, keep_alias=True, correct=False, dry_run=False): lines.append({"segment": sid, "recording": s["recording"], "idx": s["idx"], "before": s["text"], "after": after}) if dry_run: return {"name": new, "lines": len(lines), "recordings": len({x["recording"] for x in lines}), "preview": lines[:200]} - if keep_alias and e["key"] != new_key: - _add_alias(db, e["space"], e["key"], e["id"]) - db.q("UPDATE $r SET name = $n, key = $k, ekey = $ek", r=R("entity", e["id"]), n=new, k=new_key, ek=f"{e['space']}:{new_key}") - db.q("DELETE entity_alias WHERE space = $s AND key = $k", s=e["space"], k=new_key) + with graph_history.change(db, "entity.rename", entities=[e["id"]], aliases=[(e["space"], new_key)]): + if keep_alias and e["key"] != new_key: + _add_alias(db, e["space"], e["key"], e["id"]) + db.q("UPDATE $r SET name = $n, key = $k, ekey = $ek", r=R("entity", e["id"]), n=new, k=new_key, ek=f"{e['space']}:{new_key}") + db.q("DELETE entity_alias WHERE space = $s AND key = $k", s=e["space"], k=new_key) for x in lines: db.q("UPDATE $s SET text = $t", s=R("segment", x["segment"]), t=x["after"]) db.q( @@ -777,6 +785,13 @@ def merge(db, keep, others, user=None): ) if not snaps: raise ValueError("pick at least one other entity") + with graph_history.change(db, "entity.merge", entities=[k["id"]] + [sn["entity"]["id"] for sn in snaps]) as ch: + mid = _merge(db, k, snaps, user) + ch.add(merge=mid) + return mid + + +def _merge(db, k, snaps, user): for sn in snaps: o = sn["entity"] rows = [ @@ -816,6 +831,11 @@ def undo_merge(db, mid): m = db.one("SELECT * FROM $r", r=R("entity_merge", int(mid))) if not m or m.get("undone"): raise ValueError("nothing to undo") + with graph_history.change(db, "entity.unmerge", entities=[m["keep"]] + [sn["entity"]["id"] for sn in m["snapshots"]], merge=int(mid)): + _undo_merge(db, m, mid) + + +def _undo_merge(db, m, mid): keep = m["keep"] for sn in m["snapshots"]: o = sn["entity"] @@ -888,6 +908,14 @@ def move_mention(db, mention, target=None, new_name=None, new_type="TERM", remov m = db.one("SELECT record::id(in) AS segment, recording, space, speaker, text, entity FROM $r", r=R("mentions", mention)) if not m: raise KeyError(mention) + with graph_history.change(db, "mention.remove" if remove else "mention.move", entities=[m["entity"]]) as ch: + tid = _move_mention(db, mention, m, ch, target, new_name, new_type, remove) + ch.mark(m["entity"], tid) + ch.add(mention=store.clean({"segment": m["segment"], "text": m["text"], "from": m["entity"], "to": tid or None})) + return tid + + +def _move_mention(db, mention, m, ch, target, new_name, new_type, remove): key = analyze.ent_key(m["text"]) tid = 0 if not remove: @@ -898,10 +926,12 @@ def move_mention(db, mention, target=None, new_name=None, new_type="TERM", remov fixed = entity_setup.effective(db, m["space"], rec.get("collection"))["mode"] == "fixed" if row: tid = row["id"] + ch.touch([tid]) else: _check_type(db, m["space"], new_type) tid = db.next_id("entity") d = {"space": m["space"], "key": nk, "ekey": f"{m['space']}:{nk}", "name": new_name.strip(), "type": new_type} + ch.created(tid) db.q("CREATE $r CONTENT $d", r=R("entity", tid), d=d) if fixed and not (row or {}).get("builtin"): # a name given where the list is fixed joins the list db.q("UPDATE $r SET defined = true", r=R("entity", tid)) @@ -936,7 +966,8 @@ def remove(db, eid): raise ValueError("Only a defined entity can be deleted; hide this one instead.") if db.rows("SELECT id FROM mentions WHERE entity = $e LIMIT 1", e=e["id"]): raise ValueError(f"{e['name']} is mentioned: merge it into another entity, or hide it.") - db.run(["DELETE entity_alias WHERE entity = $e", "DELETE $r"], e=e["id"], r=R("entity", e["id"])) + with graph_history.change(db, "entity.delete", entities=[e["id"]]): + db.run(["DELETE entity_alias WHERE entity = $e", "DELETE $r"], e=e["id"], r=R("entity", e["id"])) def link(db, a, b, check_spaces=True): @@ -944,9 +975,11 @@ def link(db, a, b, check_spaces=True): if check_spaces and x["space"] == y["space"]: raise ValueError("these are in the same namespace: merge them instead") lo, hi = sorted((x["id"], y["id"])) - db.q("UPSERT $r CONTENT $d", r=R("entity_link", f"{lo}-{hi}"), d={"a": lo, "b": hi}) + with graph_history.change(db, "link.add", entities=[lo, hi]): + db.q("UPSERT $r CONTENT $d", r=R("entity_link", f"{lo}-{hi}"), d={"a": lo, "b": hi}) def unlink(db, a, b): lo, hi = sorted((int(a), int(b))) - db.q("DELETE $r", r=R("entity_link", f"{lo}-{hi}")) + with graph_history.change(db, "link.remove", entities=[lo, hi]): + db.q("DELETE $r", r=R("entity_link", f"{lo}-{hi}")) diff --git a/fastapi_backend/app/domain/entity_map.py b/fastapi_backend/app/domain/entity_map.py index 4cc6016ce..32d582081 100644 --- a/fastapi_backend/app/domain/entity_map.py +++ b/fastapi_backend/app/domain/entity_map.py @@ -24,7 +24,7 @@ import logging -from . import analyze, entity_setup, llm, store +from . import analyze, entity_setup, graph_history, llm, store R = store.R log = logging.getLogger("lens") @@ -51,19 +51,21 @@ def builtins(db, sid): out[kind] = row["id"] continue eid = db.next_id("entity") - db.q( - "CREATE $r CONTENT $d", - r=R("entity", eid), - d={ - "space": int(sid), - "key": key, - "ekey": f"{sid}:{key}", - "name": name, - "type": "TERM", - "builtin": kind, - "description": BUILTIN_HELP[kind], - }, - ) + with graph_history.change(db, "entity.builtin") as ch: + ch.created(eid) + db.q( + "CREATE $r CONTENT $d", + r=R("entity", eid), + d={ + "space": int(sid), + "key": key, + "ekey": f"{sid}:{key}", + "name": name, + "type": "TERM", + "builtin": kind, + "description": BUILTIN_HELP[kind], + }, + ) out[kind] = eid return out @@ -160,11 +162,13 @@ def place(db, cfg, sid, cid, setup, found, seg_texts=()): out[key] = special[UNLABELED] else: out[key] = db.next_id("entity") - db.q( - "CREATE $r CONTENT $d", - r=R("entity", out[key]), - d={"space": int(sid), "key": key, "ekey": f"{sid}:{key}", "name": name, "type": typ}, - ) + with graph_history.change(db, "entity.found") as ch: + ch.created(out[key]) + db.q( + "CREATE $r CONTENT $d", + r=R("entity", out[key]), + d={"space": int(sid), "key": key, "ekey": f"{sid}:{key}", "name": name, "type": typ}, + ) return out @@ -187,6 +191,13 @@ def define(db, sid, name, typ, description=None, aliases=(), collection=None, us plan = _alias_plan(db, int(sid), key, row["id"] if row else None, aliases) # check everything before writing if description is not None and len(str(description).strip()) > 2000: raise ValueError("A description can have up to 2000 characters.") + with graph_history.change(db, "entity.define", entities=[row["id"]] if row else [], aliases=[(int(sid), k) for k in plan[0]]) as ch: + eid = _define(db, sid, row, key, name, typ, description, collection, user, plan, ch) + builtins(db, sid) + return eid + + +def _define(db, sid, row, key, name, typ, description, collection, user, plan, ch): if row: eid = row["id"] db.q( @@ -197,6 +208,7 @@ def define(db, sid, name, typ, description=None, aliases=(), collection=None, us ) else: eid = db.next_id("entity") + ch.created(eid) db.q( "CREATE $r CONTENT $d", r=R("entity", eid), @@ -216,8 +228,7 @@ def define(db, sid, name, typ, description=None, aliases=(), collection=None, us from . import entities entities.describe(db, eid, description) - set_aliases(db, eid, aliases, user=user, plan=plan) - builtins(db, sid) + set_aliases(db, eid, None, user=user, plan=plan) return eid @@ -243,13 +254,14 @@ def set_aliases(db, eid, aliases, user=None, plan=None): merged into it (the merge can be undone).""" e = db.one("SELECT space, key FROM $r", r=R("entity", int(eid))) keys, fold = plan or _alias_plan(db, e["space"], e["key"], int(eid), aliases) - if fold: - from . import entities + with graph_history.change(db, "entity.aliases", entities=[int(eid), *fold], aliases=[(e["space"], k) for k in keys]): + if fold: + from . import entities - entities.merge(db, int(eid), fold, user=user) - db.q("DELETE entity_alias WHERE entity = $e", e=int(eid)) - for k in keys: - db.q("UPSERT $r CONTENT $d", r=R("entity_alias", f"{e['space']}:{k}"), d={"space": e["space"], "key": k, "entity": int(eid)}) + entities.merge(db, int(eid), fold, user=user) + db.q("DELETE entity_alias WHERE entity = $e", e=int(eid)) + for k in keys: + db.q("UPSERT $r CONTENT $d", r=R("entity_alias", f"{e['space']}:{k}"), d={"space": e["space"], "key": k, "entity": int(eid)}) # ---------- matching by description (the model) ---------- @@ -356,5 +368,8 @@ def described(db, sid): def learn(db, sid, mapped): """Names the model placed become those entities' other names ({key: entity id}).""" - for key, eid in mapped.items(): - db.q("UPSERT $r CONTENT $d", r=R("entity_alias", f"{int(sid)}:{key}"), d={"space": int(sid), "key": key, "entity": int(eid)}) + if not mapped: + return + with graph_history.change(db, "entity.learned", aliases=[(int(sid), k) for k in mapped]): + for key, eid in mapped.items(): + db.q("UPSERT $r CONTENT $d", r=R("entity_alias", f"{int(sid)}:{key}"), d={"space": int(sid), "key": key, "entity": int(eid)}) diff --git a/fastapi_backend/app/domain/graph_history.py b/fastapi_backend/app/domain/graph_history.py new file mode 100644 index 000000000..1b574e928 --- /dev/null +++ b/fastapi_backend/app/domain/graph_history.py @@ -0,0 +1,479 @@ +"""The entity graph's history: every change is an append-only event, so the graph can be seen as of any version, +compared between two versions, and an entity's own history read (docs/graph-history.md). + +What is versioned is what people and agents curate: the rows of `entity` (name, type, description, hidden, defined), +`entity_alias` (the other ways a name is said), `entity_link` (the same thing in two namespaces) and `entity_distinct` +(two things someone said aren't one). Mentions are not: analysis makes them from transcripts, whose edits have their +own history (segment_edit). An as-of view uses today's mentions. + +A `graph_event:` row is one change, and `n` is the graph's version after it: +{at, op, actor, via, why, spaces, entities, ops: [{t: table, k: key, b: row before, a: row after, s: spaces}], origin}. +`op` names what happened (entity.rename, entity.merge, link.add, analysis, ...); `actor` who did it (an email, +`routine:`, `analysis`, `system`); `via` through what (web, token, oauth, assistant, mcp, routine, analysis, cli); +`origin` what it came from (a graph change, a merge, a routine run, a recording, an approval). Events are never +changed or deleted; undoing a change is a new event. + +Writers wrap their writes: `with graph_history.change(db, "entity.rename", entities=[eid]):` snapshots the rows of the +entities (and their aliases, links and distinct pairs) before and after the block, and records what differs. Changes +inside a change join it, so a define that folds others in by a merge is one event. Who and why come from +`graph_history.acting(actor=..., via=..., why=...)` around the work (the API sets the caller for each request). +""" + +from __future__ import annotations + +import contextlib +import contextvars +import re + +from . import store + +R = store.R +TABLES = ("entity", "entity_alias", "entity_link", "entity_distinct") +VIAS = ("web", "token", "oauth", "assistant", "mcp", "routine", "workflow", "analysis", "cli", "system") +SUMMARY = "record::id(id) AS version, at, op, actor, via, why, spaces, entities, origin, array::len(ops) AS changes" +TAG_NAME = re.compile(r"^[\w .:+-]{1,80}$") + +_acting: contextvars.ContextVar[dict | None] = contextvars.ContextVar("lens_graph_acting", default=None) +_open: contextvars.ContextVar[Change | None] = contextvars.ContextVar("lens_graph_change", default=None) + + +# ---------- who, through what, why ---------- +@contextlib.contextmanager +def acting(actor=None, via=None, why=None, **origin): + """Changes made inside are recorded as made by `actor`, through `via`, because of `why`, from `origin` (routine, + run, workflow, graph_change, recording, approval, ...). Nested ones add to (and override) the outer ones.""" + outer = _acting.get() or {} + mine = store.clean({"actor": actor, "via": via, "why": why}) + token = _acting.set({**outer, **mine, "origin": {**(outer.get("origin") or {}), **store.clean(origin)}}) + try: + yield + finally: + _acting.reset(token) + + +def note(**kw): + """Fill in the current `acting` block in place (the API learns who is calling after the request has started).""" + cur = _acting.get() + if cur is not None: + cur.update(store.clean(kw)) + + +def who(): + return dict(_acting.get() or {}) + + +# ---------- snapshots ---------- +def _key(table, k): + return int(k) if table == "entity" else str(k) + + +def _pairs(rows): + out = {} + for r in rows: + r = dict(r) + k = r.pop("id") + out[k] = r + return out + + +def _snap(db, eids=(), keys=()): + """{(table, key): row or None} for these entities, the aliases, links and distinct pairs that name them, and the + records named in `keys` [(table, key)].""" + out = {} + ids = sorted({int(e) for e in eids}) + if ids: + refs = [R("entity", i) for i in ids] + got = _pairs(db.rows("SELECT * FROM entity WHERE id IN $ids", ids=refs)) + for i in ids: + out[("entity", i)] = got.get(i) + for k, r in _pairs(db.rows("SELECT * FROM entity_alias WHERE entity IN $e", e=ids)).items(): + out[("entity_alias", str(k))] = r + for t in ("entity_link", "entity_distinct"): + for k, r in _pairs(db.rows(f"SELECT * FROM {t} WHERE a IN $e OR b IN $e", e=ids)).items(): + out[(t, str(k))] = r + by_table = {} + for t, k in keys: + if (t, _key(t, k)) not in out: + by_table.setdefault(t, set()).add(_key(t, k)) + for t, ks in by_table.items(): + got = _pairs(db.rows(f"SELECT * FROM {t} WHERE id IN $ids", ids=[R(t, k) for k in sorted(ks, key=str)])) + for k in ks: + out[(t, k)] = got.get(k) + return out + + +def _row_spaces(t, row, space_of): + if not row: + return set() + if t in ("entity", "entity_alias"): + return {row.get("space")} - {None} + return {space_of(row.get("a")), space_of(row.get("b"))} - {None} + + +class Change: + """One change being made: what it touches, what it was before.""" + + def __init__(self, db, op, why, origin): + self.db, self.op, self.why, self.origin = db, op, why, dict(origin) + self.before: dict = {} + self.ids: set[int] = set() + self.keys: set = set() + self.version: int | None = None + self.always = False + self.marked: set[int] = set() + + def touch(self, eids=(), aliases=(), keys=()): + """Watch these entities (before they change), alias keys [(space, key)] and records [(table, key)].""" + new_ids = {int(e) for e in eids if e is not None} - self.ids + new_keys = {("entity_alias", f"{int(s)}:{k}") for s, k in aliases} | {(t, _key(t, k)) for t, k in keys} + new_keys -= self.keys + if new_ids or new_keys: + for k, v in _snap(self.db, new_ids, new_keys).items(): + self.before.setdefault(k, v) + self.ids |= new_ids + self.keys |= new_keys + + def created(self, *eids): + """Entities this change made: they were nothing before.""" + for e in eids: + if e is not None: + self.before.setdefault(("entity", int(e)), None) + self.ids.add(int(e)) + + def add(self, **origin): + self.origin.update(store.clean(origin)) + + def mark(self, *eids): + """Record the event even when no versioned row changed (a moved mention), as about these entities.""" + self.always = True + self.marked |= {int(e) for e in eids if e} + + def ops(self): + after = _snap(self.db, self.ids, self.keys | set(self.before)) + out, ents = [], {} + for (t, k), row in {**self.before, **after}.items(): + if t == "entity": + ents[k] = after.get((t, k)) or self.before.get((t, k)) + for t, k in sorted(set(self.before) | set(after), key=lambda x: (TABLES.index(x[0]), str(x[1]))): + b, a = self.before.get((t, k)), after.get((t, k)) + if b != a: + out.append({"t": t, "k": k, "b": b, "a": a}) + missing = { + int(x) + for o in out + if o["t"] in ("entity_link", "entity_distinct") + for x in ((o.get("a") or o.get("b")).get("a"), (o.get("a") or o.get("b")).get("b")) + if x is not None and int(x) not in ents + } + if missing: + for r in self.db.rows("SELECT record::id(id) AS id, space FROM entity WHERE id IN $ids", ids=[R("entity", i) for i in missing]): + ents[r["id"]] = r + + def space_of(e): + return (ents.get(int(e)) or {}).get("space") if e is not None else None + + for o in out: + o["s"] = sorted(_row_spaces(o["t"], o.get("a"), space_of) | _row_spaces(o["t"], o.get("b"), space_of)) + return out + + +@contextlib.contextmanager +def change(db, op, entities=(), aliases=(), keys=(), why=None, **origin): + """Record what the block changes as one event (none when nothing changed). Inside another change, it joins it.""" + outer = _open.get() + if outer is not None: + outer.touch(entities, aliases, keys) + outer.add(**origin) + yield outer + return + ch = Change(db, op, why, origin) + ch.touch(entities, aliases, keys) + token = _open.set(ch) + try: + yield ch + finally: + _open.reset(token) + ops = ch.ops() + spaces = None + if ch.marked and not ops: + spaces = sorted(set(db.values("SELECT VALUE space FROM entity WHERE id IN $i", i=[R("entity", e) for e in ch.marked]))) + ch.version = record(db, ch.op, ops, why=ch.why, origin=ch.origin, entities=ch.marked, spaces=spaces, always=ch.always) + + +def record(db, op, ops, why=None, origin=None, actor=None, via=None, entities=(), spaces=None, always=False): + """Write one event for these ops; its version (None when there is nothing to record).""" + if not ops and not always: + return None + w = who() + n = db.next_id("graph_event") + ents = {int(o["k"]) for o in ops if o["t"] == "entity"} | {int(x) for o in ops if o["t"] != "entity" for x in _ends(o)} + ents = sorted(ents | {int(e) for e in entities}) + row = store.clean( + { + "at": store.now(), + "op": op, + "actor": actor or w.get("actor") or (w.get("via") if w.get("via") in ("analysis", "routine", "cli") else None) or "system", + "via": via or w.get("via") or "system", + "why": (str(why or w.get("why") or "")[:500]) or None, + "spaces": sorted({s for o in ops for s in o["s"]} | set(spaces or [])), + "entities": ents, + "ops": ops, + "origin": {**(w.get("origin") or {}), **store.clean(origin or {})} or None, + } + ) + db.q("CREATE $r CONTENT $d", r=R("graph_event", n), d=row) + return n + + +def _ends(o): + row = o.get("a") or o.get("b") or {} + if o["t"] == "entity_alias": + return [row["entity"]] if row.get("entity") is not None else [] + return [x for x in (row.get("a"), row.get("b")) if x is not None] + + +# ---------- reading the history ---------- +def head(db): + """The graph's current version (0 before anything was recorded).""" + got = db.values("SELECT VALUE n FROM $r", r=R("seq", "graph_event")) + return int(got[0]) if got and got[0] is not None else 0 + + +def _visible(ev, spaces): + return spaces is None or set(ev.get("spaces") or []) <= set(spaces) + + +def versions(db, spaces=None, entity=None, before=None, limit=50): + """Events, newest first, whose namespaces are all in `spaces` (None: all); with `entity`, the ones that touched it.""" + where, args = [], {"n": int(limit)} + if spaces is not None: + where.append("spaces ALLINSIDE $s") + args["s"] = sorted(spaces) + if entity is not None: + where.append("$e INSIDE entities") + args["e"] = int(entity) + if before is not None: + where.append("id < $b") + args["b"] = R("graph_event", int(before)) + sql = f"SELECT {SUMMARY} FROM graph_event" + (" WHERE " + " AND ".join(where) if where else "") + " ORDER BY version DESC LIMIT $n" + rows = db.rows(sql, **args) + names = _names(db, {e for r in rows for e in r.get("entities") or []}, rows_of=None) + tags = _tags_by_version(db) + for r in rows: + r["names"] = {str(e): names[e] for e in r.get("entities") or [] if e in names} + r["tags"] = tags.get(r["version"], []) + return rows + + +def event(db, version, spaces=None): + ev = db.one("SELECT *, record::id(id) AS version FROM $r", r=R("graph_event", int(version))) + if not ev or not _visible(ev, spaces): + raise KeyError(version) + ev.pop("id", None) + ev["tags"] = _tags_by_version(db).get(ev["version"], []) + return ev + + +def _events_after(db, version, upto=None): + sql = "SELECT record::id(id) AS version, ops FROM graph_event WHERE id > $v" + (" AND id <= $u" if upto is not None else "") + return db.rows(sql + " ORDER BY version", v=R("graph_event", int(version)), u=R("graph_event", int(upto or 0))) + + +def live(db): + """Today's graph: {table: {key: row}}.""" + return {t: {_key(t, k): r for k, r in _pairs(db.rows(f"SELECT * FROM {t}")).items()} for t in TABLES} + + +def _scoped(state, spaces): + if spaces is None: + return state + sp = set(spaces) + ents = {k: r for k, r in state["entity"].items() if r.get("space") in sp} + return { + "entity": ents, + "entity_alias": {k: r for k, r in state["entity_alias"].items() if r.get("space") in sp}, + "entity_link": {k: r for k, r in state["entity_link"].items() if r.get("a") in ents and r.get("b") in ents}, + "entity_distinct": {k: r for k, r in state["entity_distinct"].items() if r.get("a") in ents and r.get("b") in ents}, + } + + +def state_at(db, version, spaces=None): + """The graph as it was at `version`: today's rows with every later event walked back.""" + version = int(version) + if version < 0 or version > head(db): + raise ValueError(f"the graph has versions 0 to {head(db)}") + s = live(db) + for ev in reversed(_events_after(db, version)): + for o in reversed(ev["ops"]): + k = _key(o["t"], o["k"]) + if o.get("b") is None: + s[o["t"]].pop(k, None) + else: + s[o["t"]][k] = o.get("b") + return _scoped(s, spaces) + + +def as_of(db, version, spaces=None): + """The graph at a version, for people: entities with their other names, links and distinct pairs.""" + s = state_at(db, version, spaces) + al = {} + for r in s["entity_alias"].values(): + al.setdefault(r["entity"], []).append(r["key"]) + return { + "version": int(version), + "entities": [ + store.clean({"id": k, **{f: r.get(f) for f in ENTITY_FIELDS}, "aliases": sorted(al.get(k, [])) or None}) + for k, r in sorted(s["entity"].items()) + ], + "links": [[r["a"], r["b"]] for _, r in sorted(s["entity_link"].items())], + "distinct": [[r["a"], r["b"]] for _, r in sorted(s["entity_distinct"].items())], + } + + +ENTITY_FIELDS = ("space", "key", "name", "type", "description", "hidden", "hidden_reason", "defined", "builtin", "collection") + + +def diff(db, a, b, spaces=None): + """What changed between versions a and b (a < b): entities added, removed and changed (field by field), and the + aliases, links and distinct pairs added and removed.""" + a, b = int(a), int(b) + if a > b: + a, b = b, a + top = head(db) + if b > top: + raise ValueError(f"the graph has versions 0 to {top}") + first, last, events = {}, {}, 0 + for ev in _events_after(db, a, b): + events += 1 + for o in ev["ops"]: + k = (o["t"], _key(o["t"], o["k"])) + first.setdefault(k, o.get("b")) + last[k] = o.get("a") + sp = set(spaces) if spaces is not None else None + ent_space = {} + for (t, k), row in {**first, **last}.items(): + if t == "entity" and row: + ent_space[k] = row.get("space") + need = { + int(x) + for (t, _), row in {**first, **last}.items() + if t in ("entity_link", "entity_distinct") and row + for x in (row.get("a"), row.get("b")) + if int(x) not in ent_space + } + if need: + for r in db.rows("SELECT record::id(id) AS id, space FROM entity WHERE id IN $i", i=[R("entity", x) for x in need]): + ent_space[r["id"]] = r["space"] + + def seen(t, row): + if sp is None or not row: + return True + if t in ("entity", "entity_alias"): + return row.get("space") in sp + return ent_space.get(row.get("a")) in sp and ent_space.get(row.get("b")) in sp + + out = { + "from": a, + "to": b, + "events": events, + "entities": {"added": [], "removed": [], "changed": []}, + "aliases": {"added": [], "removed": []}, + "links": {"added": [], "removed": []}, + "distinct": {"added": [], "removed": []}, + } + names = _names(db, set(ent_space), rows_of={k: (last.get(("entity", k)) or first.get(("entity", k))) for k in ent_space}) + for t, k in sorted(first, key=lambda x: (TABLES.index(x[0]), str(x[1]))): + before, after = first[(t, k)], last[(t, k)] + if before == after or not (seen(t, before) and seen(t, after)): + continue + if t == "entity": + if before is None: + out["entities"]["added"].append(store.clean({"id": k, **{f: after.get(f) for f in ENTITY_FIELDS}})) + elif after is None: + out["entities"]["removed"].append(store.clean({"id": k, **{f: before.get(f) for f in ENTITY_FIELDS}})) + else: + fields = { + f: [before.get(f), after.get(f)] + for f in sorted(set(before) | set(after)) + if f != "ekey" and before.get(f) != after.get(f) + } + out["entities"]["changed"].append({"id": k, "name": after.get("name"), "fields": fields}) + elif t == "entity_alias": + for row, kind in ((before, "removed"), (after, "added")): + if row: + out["aliases"][kind].append( + {"key": row["key"], "entity": row["entity"], "name": names.get(row["entity"]), "space": row["space"]} + ) + else: + kind = {"entity_link": "links", "entity_distinct": "distinct"}[t] + row = after or before + pair = {"a": row["a"], "b": row["b"], "names": [names.get(row["a"]), names.get(row["b"])]} + out[kind]["added" if after else "removed"].append(pair) + return out + + +def _names(db, eids, rows_of=None): + out = {k: r.get("name") for k, r in (rows_of or {}).items() if r} + rest = [e for e in eids if e not in out] + if rest: + for r in db.rows("SELECT record::id(id) AS id, name FROM entity WHERE id IN $i", i=[R("entity", int(e)) for e in rest]): + out[r["id"]] = r["name"] + left = [e for e in rest if e not in out] + if left: # gone since: the name from the last event that had it + for ev in db.rows( + "SELECT record::id(id) AS version, ops FROM graph_event WHERE entities ANYINSIDE $e ORDER BY version DESC", e=left + ): + for o in ev["ops"]: + row = o.get("b") or o.get("a") + if o["t"] == "entity" and row and int(o["k"]) in left and int(o["k"]) not in out: + out[int(o["k"])] = row.get("name") + return out + + +# ---------- named versions ---------- +def _tags_by_version(db): + out = {} + for r in db.rows("SELECT name, version FROM graph_tag ORDER BY version"): + out.setdefault(r["version"], []).append(r["name"]) + return out + + +def tags(db): + return db.rows("SELECT name, version, note, by, at FROM graph_tag ORDER BY version DESC") + + +def tag(db, name, version=None, note=None, by=None): + """Name a version (default: the current one). A name is used once; naming again moves it.""" + name = " ".join(str(name or "").split()) + if not TAG_NAME.match(name): + raise ValueError("a name is 1 to 80 letters, digits, spaces or . : + -") + top = head(db) + v = top if version is None else int(version) + if not 0 <= v <= top: + raise ValueError(f"the graph has versions 0 to {top}") + db.q( + "UPSERT $r CONTENT $d", + r=R("graph_tag", name.lower()), + d=store.clean({"name": name, "version": v, "note": (str(note)[:300] if note else None), "by": by, "at": store.now()}), + ) + return {"name": name, "version": v} + + +def untag(db, name): + key = " ".join(str(name or "").split()).lower() + if not db.one("SELECT id FROM $r", r=R("graph_tag", key)): + raise KeyError(name) + db.q("DELETE $r", r=R("graph_tag", key)) + + +def resolve(db, ref): + """A version from a number, a name, or `head`.""" + if ref is None or str(ref).strip().lower() in ("", "head", "now", "latest"): + return head(db) + s = str(ref).strip() + if s.lstrip("v").isdigit(): + return int(s.lstrip("v")) + got = db.one("SELECT version FROM $r", r=R("graph_tag", " ".join(s.split()).lower())) + if not got: + raise KeyError(ref) + return int(got["version"]) diff --git a/fastapi_backend/app/domain/organize.py b/fastapi_backend/app/domain/organize.py index 47add8dae..25d5d4c18 100644 --- a/fastapi_backend/app/domain/organize.py +++ b/fastapi_backend/app/domain/organize.py @@ -22,7 +22,7 @@ from collections import Counter, defaultdict -from . import entities, llm, store +from . import entities, graph_history, llm, store R = store.R GRAPH_NODES = ("input", "pick", "condition", "merge", "candidates", "llm_judge", "filter", "apply_changes") @@ -276,11 +276,14 @@ def apply_changes(db, items, c, origin=None, propose_only=False, say=print): continue sure = v.get("confidence", it.get("confidence") or 0) if not propose_only and above is not None and sure >= above and stats["applied"] < cap: - done = _make(db, {**it, "keep": _keep(it) if it["kind"] == "merge" else None}, user) + why = v.get("why") or it.get("reason") + with graph_history.change(db, f"{it['kind']}.apply", entities=[it["a"]["id"], it["b"]["id"]], why=why) as ch: + done = _make(db, {**it, "keep": _keep(it) if it["kind"] == "merge" else None}, user) + if done is not None: + ch.add(graph_change=_record(db, it, "applied", origin, {**done, "decided_at": store.now(), "decided_by": user})) if done is None: stats["skipped"] += 1 continue - _record(db, it, "applied", origin, {**done, "decided_at": store.now(), "decided_by": user}) stats["applied"] += 1 else: if db.one("SELECT id FROM graph_change WHERE pair = $p AND status = 'proposed' LIMIT 1", p=it["pair"]): @@ -319,16 +322,24 @@ def propose(db, kind, a, b, reason=None, origin=None, apply=False, user=None): } if not apply: return _record(db, item, "proposed", origin), "proposed" - done = _make(db, {**item, "keep": x["id"] if kind == "merge" else None}, user) - if held: - db.q("UPDATE $r SET status = 'dismissed', decided_at = $t, decided_by = $u", r=R("graph_change", held["id"]), t=store.now(), u=user) - return _record( - db, - {**item}, - "applied", - origin, - {**(done or {}), "keep": x["id"] if kind == "merge" else None, "decided_at": store.now(), "decided_by": user}, - ), "applied" + with graph_history.change(db, f"{kind}.apply", entities=[x["id"], y["id"]], why=item["reason"]) as ch: + done = _make(db, {**item, "keep": x["id"] if kind == "merge" else None}, user) + if held: + db.q( + "UPDATE $r SET status = 'dismissed', decided_at = $t, decided_by = $u", + r=R("graph_change", held["id"]), + t=store.now(), + u=user, + ) + cid = _record( + db, + {**item}, + "applied", + origin, + {**(done or {}), "keep": x["id"] if kind == "merge" else None, "decided_at": store.now(), "decided_by": user}, + ) + ch.add(graph_change=cid) + return cid, "applied" def get_change(db, cid): @@ -347,7 +358,10 @@ def accept(db, cid, user=None, keep=None): if int(keep) not in (ch["a"]["id"], ch["b"]["id"]): raise ValueError("keep one of the two entities") ch["keep"] = int(keep) - done = _make(db, ch, user) + with graph_history.change( + db, f"{ch['kind']}.accept", entities=[ch["a"]["id"], ch["b"]["id"]], why=ch.get("reason"), graph_change=ch["id"] + ): + done = _make(db, ch, user) if done is None: db.q( "UPDATE $r SET status = 'dismissed', decided_at = $t, decided_by = $u, note = 'an entity is gone'", @@ -369,7 +383,8 @@ def dismiss(db, cid, user=None): if ch["status"] != "proposed": raise ValueError(f"this change is {ch['status']}, not proposed") if _alive(db, ch["a"]["id"]) and _alive(db, ch["b"]["id"]): - entities.not_same(db, ch["a"]["id"], ch["b"]["id"]) + with graph_history.change(db, f"{ch['kind']}.dismiss", graph_change=ch["id"]): + entities.not_same(db, ch["a"]["id"], ch["b"]["id"]) db.q("UPDATE $r SET status = 'dismissed', decided_at = $t, decided_by = $u", r=R("graph_change", ch["id"]), t=store.now(), u=user) @@ -378,12 +393,13 @@ def undo(db, cid, user=None): ch = get_change(db, cid) if ch["status"] != "applied": raise ValueError(f"this change is {ch['status']}, not applied") - if ch["kind"] == "merge": - m = db.one("SELECT undone FROM $r", r=R("entity_merge", int(ch["merge"]))) - if m and not m.get("undone"): # someone may have undone it from the entity page already - entities.undo_merge(db, ch["merge"]) - else: - entities.unlink(db, ch["a"]["id"], ch["b"]["id"]) + with graph_history.change(db, f"{ch['kind']}.undo", graph_change=ch["id"]): + if ch["kind"] == "merge": + m = db.one("SELECT undone FROM $r", r=R("entity_merge", int(ch["merge"]))) + if m and not m.get("undone"): # someone may have undone it from the entity page already + entities.undo_merge(db, ch["merge"]) + else: + entities.unlink(db, ch["a"]["id"], ch["b"]["id"]) db.q("UPDATE $r SET status = 'undone', undone_at = $t, undone_by = $u", r=R("graph_change", ch["id"]), t=store.now(), u=user) @@ -433,7 +449,10 @@ def run(db, cfg, wid, spaces, version=None, say=print, origin=None, propose_only origin = {**(origin or {}), "workflow": w["id"], "workflow_version": w["version"]} r = flow.Run(db, cfg, workflows.KITS["graph"], ctx, say, origin=origin, spaces=set(spaces), propose_only=propose_only) r.totals = {"applied": 0, "proposed": 0, "skipped": 0} - flow.run_graph(r, w["graph"]) + routine = origin.get("routine") + keep = {k: origin[k] for k in ("routine", "run", "workflow", "workflow_version") if origin.get(k) is not None} + with graph_history.acting(actor=f"routine:{routine}" if routine else None, via="routine" if routine else "workflow", **keep): + flow.run_graph(r, w["graph"]) done = {n["id"]: "done" if r.trace.get(n["id"], {}).get("status") == "done" else "skipped" for n in w["graph"]["nodes"]} return done, r.totals diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 2916aa33e..9dcef7807 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -834,6 +834,9 @@ def close(self): "DEFINE INDEX IF NOT EXISTS graph_change_run ON graph_change FIELDS run", "DEFINE INDEX IF NOT EXISTS graph_change_status ON graph_change FIELDS status", "DEFINE INDEX IF NOT EXISTS graph_change_pair ON graph_change FIELDS pair", + # the entity graph's history (graph_history.py): one append-only event per change, and named versions + "DEFINE TABLE IF NOT EXISTS graph_event SCHEMALESS", + "DEFINE TABLE IF NOT EXISTS graph_tag SCHEMALESS", "DEFINE TABLE IF NOT EXISTS output SCHEMALESS", "DEFINE INDEX IF NOT EXISTS output_rec ON output FIELDS recording", "DEFINE TABLE IF NOT EXISTS chat SCHEMALESS", diff --git a/fastapi_backend/tests/api/test_graph_history.py b/fastapi_backend/tests/api/test_graph_history.py new file mode 100644 index 000000000..d22cdc814 --- /dev/null +++ b/fastapi_backend/tests/api/test_graph_history.py @@ -0,0 +1,56 @@ +"""The graph's history over the API: who changed what through what, the graph as of a version, diffs and named +versions, with roles.""" + +from __future__ import annotations + +import pytest + +from tests.api.test_entities import Env +from tests.api.test_graph import token + + +@pytest.fixture +def env(db, cfg, folder, client): + return Env(db, cfg, folder, client) + + +def test_changes_record_the_caller(env): + c, ed, vi = env.c, env.h["editor"], env.h["viewer"] + assert c.get("/api/v1/graph/history").status_code == 401 + start = c.get("/api/v1/graph/history", headers=vi).json()["head"] + dyno = env.eid("Dyno Therapeutics") + assert c.post(f"/api/v1/entities/{dyno}/rename", headers=ed, json={"name": "Dyno"}).status_code == 200 + t = token(c, ed, "write") + assert c.patch(f"/api/v1/entities/{dyno}", headers=t, json={"description": "Gene therapy"}).status_code == 200 + got = c.get("/api/v1/graph/history", headers=vi, params={"entity": dyno}).json() + top = got["versions"][:2] + assert [(v["op"], v["actor"], v["via"]) for v in top] == [ + ("entity.describe", "ed@x.io", "token"), + ("entity.rename", "ed@x.io", "web"), + ] + assert got["head"] == start + 2 and top[0]["names"][str(dyno)] == "Dyno" + full = c.get(f"/api/v1/graph/history/{top[1]['version']}", headers=vi).json() + assert [(o["t"], o["b"]["name"], o["a"]["name"]) for o in full["ops"] if o["t"] == "entity"] == [ + ("entity", "Dyno Therapeutics", "Dyno") + ] + # the graph as it was, and what changed since + was = c.get(f"/api/v1/graph/as-of/{start}", headers=vi).json() + assert "Dyno Therapeutics" in {e["name"] for e in was["entities"]} + assert {e["space"] for e in was["entities"]} == {env.db.values("SELECT VALUE record::id(id) FROM space WHERE name = 'pods'")[0]} + d = c.get("/api/v1/graph/diff", headers=vi, params={"from": start}).json() + assert {f for x in d["entities"]["changed"] for f in x["fields"]} == {"name", "key", "description"} + assert c.get("/api/v1/graph/diff", headers=vi, params={"from": start, "to": 10**6}).status_code == 400 + assert c.get("/api/v1/graph/as-of/nope", headers=vi).status_code == 404 + assert c.get("/api/v1/graph/history", headers=vi, params={"namespace": "calls"}).status_code in (403, 404) + + +def test_named_versions(env): + c, ed, vi = env.c, env.h["editor"], env.h["viewer"] + assert c.post("/api/v1/graph/tags", headers=vi, json={"name": "before"}).status_code == 403 + v = c.post("/api/v1/graph/tags", headers=ed, json={"name": "before", "note": "clean-up"}).json()["version"] + c.post(f"/api/v1/entities/{env.eid('Dyno Therapeutics')}/rename", headers=ed, json={"name": "Dyno"}) + assert c.get("/api/v1/graph/tags", headers=vi).json()[0]["name"] == "before" + d = c.get("/api/v1/graph/diff", headers=vi, params={"from": "before"}).json() + assert (d["from"], d["to"], d["events"]) == (v, v + 1, 1) + assert c.delete("/api/v1/graph/tags/before", headers=ed).status_code == 200 + assert c.get("/api/v1/graph/diff", headers=vi, params={"from": "before"}).status_code == 404 diff --git a/fastapi_backend/tests/domain/test_graph_history.py b/fastapi_backend/tests/domain/test_graph_history.py new file mode 100644 index 000000000..99f891030 --- /dev/null +++ b/fastapi_backend/tests/domain/test_graph_history.py @@ -0,0 +1,156 @@ +"""The entity graph's history: every change is an event with who, through what and why; the graph as of any version +matches what it was then; diffs, an entity's history and named versions.""" + +from __future__ import annotations + +import pytest + +from app.domain import analyze, entities, entity_map, ingest, organize, store +from app.domain import graph_history as gh +from tests.helpers import quiet, seed + +EXTRA = ( + "Alice|N|We met the Northwind Labs team and the North Wind Labs lawyers.\n" + "Bob|N|Amazon Web Services hosts it; AWS bills monthly.\nAlice|N|Northwind Labs again, with Dyno Therapeutics." +) + + +@pytest.fixture +def spaces(db, cfg, folder): + seed(db, cfg, folder) + p = folder / "extra.txt" + p.write_text(EXTRA) + ingest.import_transcript(db, cfg, "pods", p, log=quiet) + analyze.analyze_pending(db, cfg, log=quiet) + return {store.ns_id(db, "pods"), store.ns_id(db, "calls")} + + +def eid(db, spaces, name, ns="pods"): + return [x["id"] for x in entities.list_entities(db, spaces, q=name)["items"] if x["name"] == name and x["namespace"] == ns][0] + + +def now(db): + return gh.state_at(db, gh.head(db)) + + +def test_analysis_is_recorded_and_the_start_is_empty(db, spaces): + found = gh.versions(db) + assert found and {v["op"] for v in found} == {"analysis"} + assert {v["via"] for v in found} == {"analysis"} and {v["actor"] for v in found} == {"analysis"} + assert all(v["origin"]["recording"] for v in found) + assert gh.state_at(db, 0)["entity"] == {} # before the first analysis there was nothing + assert now(db) == gh.live(db) + + +def test_analysing_again_changes_nothing(db, cfg, spaces): + again = gh.head(db) + analyze.analyze_pending(db, cfg, force=True, log=quiet) + assert gh.head(db) == again # nothing new was found, so nothing was recorded + + +def test_every_curation_is_an_event_and_every_version_replays(db, cfg, spaces): + v0 = gh.head(db) + dyno, calls_dyno = eid(db, spaces, "Dyno Therapeutics"), eid(db, spaces, "Dyno Therapeutics", "calls") + keep, other = eid(db, spaces, "Northwind Labs"), eid(db, spaces, "North Wind Labs") + aws, amazon = eid(db, spaces, "AWS"), eid(db, spaces, "Amazon Web Services") + snaps = {v0: now(db)} + with gh.acting(actor="ed@x.io", via="web", why="tidying up"): + entities.rename(db, dyno, "Dyno Therapeutics Inc") + snaps[gh.head(db)] = now(db) + entities.describe(db, dyno, "A gene therapy company") + entities.retype(db, [dyno], "PRODUCT") + entities.hide(db, aws, True, "noise") + snaps[gh.head(db)] = now(db) + entities.link(db, dyno, calls_dyno) + entities.not_same(db, aws, amazon) + mid = entities.merge(db, keep, [other], "ed@x.io") + snaps[gh.head(db)] = now(db) + entities.undo_merge(db, mid) + entities.unlink(db, dyno, calls_dyno) + snaps[gh.head(db)] = now(db) + assert [v["op"] for v in reversed(gh.versions(db, limit=9))] == [ + "entity.rename", + "entity.describe", + "entity.retype", + "entity.hide", + "link.add", + "distinct.add", + "entity.merge", + "entity.unmerge", + "link.remove", + ] + top = gh.versions(db, limit=1)[0] + assert (top["actor"], top["via"], top["why"]) == ("ed@x.io", "web", "tidying up") + for v, state in snaps.items(): # walking back from today gives exactly what the graph was then + assert gh.state_at(db, v) == state, v + merged = gh.event(db, [v for v in gh.versions(db) if v["op"] == "entity.merge"][0]["version"]) + assert merged["origin"]["merge"] == mid + assert ("entity", other) in {(o["t"], o["k"]) for o in merged["ops"] if o.get("a") is None} # the other one went + assert "north wind labs" in {o["a"]["key"] for o in merged["ops"] if o["t"] == "entity_alias" and o.get("a")} + + +def test_diff_history_and_scopes(db, cfg, spaces): + pods = store.ns_id(db, "pods") + v0 = gh.head(db) + dyno, calls_dyno = eid(db, spaces, "Dyno Therapeutics"), eid(db, spaces, "Dyno Therapeutics", "calls") + entities.rename(db, dyno, "Dyno") + entities.link(db, dyno, calls_dyno) + entities.hide(db, eid(db, spaces, "AWS"), True) + d = gh.diff(db, v0, gh.head(db)) + changed = {c["id"]: c["fields"] for c in d["entities"]["changed"]} + assert changed[dyno]["name"] == ["Dyno Therapeutics", "Dyno"] + assert d["links"]["added"] == [{"a": min(dyno, calls_dyno), "b": max(dyno, calls_dyno), "names": d["links"]["added"][0]["names"]}] + assert d["events"] == 3 + only_pods = gh.diff(db, v0, gh.head(db), {pods}) + assert only_pods["links"]["added"] == [] # one end isn't readable + assert dyno in {c["id"] for c in only_pods["entities"]["changed"]} + # the link touched both namespaces: only someone who reads both sees that event + assert "link.add" in [v["op"] for v in gh.versions(db, spaces)] + assert "link.add" not in [v["op"] for v in gh.versions(db, {pods})] + with pytest.raises(KeyError): + gh.event(db, [v for v in gh.versions(db) if v["op"] == "link.add"][0]["version"], {pods}) + mine = gh.versions(db, entity=dyno) + assert [v["op"] for v in mine][:2] == ["link.add", "entity.rename"] + assert mine[0]["names"][str(dyno)] == "Dyno" + assert gh.as_of(db, v0, {pods})["entities"][0]["space"] == pods + assert "Dyno Therapeutics" in {e["name"] for e in gh.as_of(db, v0, {pods})["entities"]} + with pytest.raises(ValueError): + gh.state_at(db, gh.head(db) + 1) + + +def test_moved_mentions_defined_entities_and_graph_changes(db, cfg, spaces): + aws = eid(db, spaces, "AWS") + mention = entities.mentions(db, aws, spaces)["items"][0]["mention"] + tid = entities.move_mention(db, mention, new_name="Amazon Web Services") + ev = gh.versions(db, limit=1)[0] + assert ev["op"] == "mention.move" and set(ev["entities"]) >= {aws, tid} + assert ev["origin"]["mention"]["from"] == aws + pods = store.ns_id(db, "pods") + new = entity_map.define(db, pods, "Lens", "PRODUCT", "Our archive", ["the lens app"]) + row = [v for v in gh.versions(db, limit=3) if v["op"] == "entity.define"][0] + full = gh.event(db, row["version"]) + assert {(o["t"], o["k"]) for o in full["ops"]} >= {("entity", new), ("entity_alias", f"{pods}:the lens app")} + a, b = eid(db, spaces, "Northwind Labs"), eid(db, spaces, "North Wind Labs") + cid, status = organize.propose(db, "merge", a, b, "same company", apply=True, user="ed@x.io") + applied = gh.versions(db, limit=1)[0] + assert (applied["op"], applied["origin"]["graph_change"], applied["why"]) == ("merge.apply", cid, "same company") + organize.undo(db, cid, "ed@x.io") + undone = gh.versions(db, limit=1)[0] + assert undone["op"] == "merge.undo" and undone["origin"]["graph_change"] == cid + + +def test_named_versions(db, cfg, spaces): + v = gh.head(db) + assert gh.tag(db, "before the cleanup", by="ed@x.io") == {"name": "before the cleanup", "version": v} + entities.rename(db, eid(db, spaces, "Dyno Therapeutics"), "Dyno") + assert gh.resolve(db, "Before the cleanup") == v + assert gh.resolve(db, "head") == gh.head(db) == v + 1 + assert gh.resolve(db, "v3") == 3 + assert gh.versions(db, limit=2)[1]["tags"] == ["before the cleanup"] + with pytest.raises(ValueError): + gh.tag(db, "x" * 81) + with pytest.raises(ValueError): + gh.tag(db, "later", gh.head(db) + 5) + gh.untag(db, "before the cleanup") + with pytest.raises(KeyError): + gh.resolve(db, "before the cleanup") diff --git a/mkdocs.yml b/mkdocs.yml index ebf3509dd..694fe1f42 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,6 +77,7 @@ nav: - Telemetry: telemetry.md - IIIF: iiif.md - Graph and Cypher: graph.md + - Graph history: graph-history.md - Linked data (RDF): rdf.md - Fedora repository: fedora.md - API: api.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index d43175085..4d51793df 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -975,8 +975,28 @@ export type { GetWorkflowResponses, Gpu, Graph, + GraphAsOfData, + GraphAsOfError, + GraphAsOfErrors, + GraphAsOfResponse, + GraphAsOfResponses, GraphChange, GraphChangeAccept, + GraphDiffData, + GraphDiffError, + GraphDiffErrors, + GraphDiffResponse, + GraphDiffResponses, + GraphHistoryEventData, + GraphHistoryEventError, + GraphHistoryEventErrors, + GraphHistoryEventResponse, + GraphHistoryEventResponses, + GraphHistoryListData, + GraphHistoryListError, + GraphHistoryListErrors, + GraphHistoryListResponse, + GraphHistoryListResponses, GraphPathsData, GraphPathsError, GraphPathsErrors, @@ -998,6 +1018,19 @@ export type { GraphSchemaErrors, GraphSchemaResponse, GraphSchemaResponses, + GraphTagData, + GraphTagError, + GraphTagErrors, + GraphTagResponse, + GraphTagResponses, + GraphTagsData, + GraphTagsResponse, + GraphTagsResponses, + GraphUntagData, + GraphUntagError, + GraphUntagErrors, + GraphUntagResponse, + GraphUntagResponses, Handling, HandlingChange, Health, @@ -2040,6 +2073,7 @@ export type { SyncFedoraData, SyncFedoraResponse, SyncFedoraResponses, + TagAsk, TagCount, TagsChanged, TelemetryExport, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 83b2dc969..9f23952b9 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -489,6 +489,18 @@ import type { GetWorkflowData, GetWorkflowErrors, GetWorkflowResponses, + GraphAsOfData, + GraphAsOfErrors, + GraphAsOfResponses, + GraphDiffData, + GraphDiffErrors, + GraphDiffResponses, + GraphHistoryEventData, + GraphHistoryEventErrors, + GraphHistoryEventResponses, + GraphHistoryListData, + GraphHistoryListErrors, + GraphHistoryListResponses, GraphPathsData, GraphPathsErrors, GraphPathsResponses, @@ -501,6 +513,14 @@ import type { GraphSchemaData, GraphSchemaErrors, GraphSchemaResponses, + GraphTagData, + GraphTagErrors, + GraphTagResponses, + GraphTagsData, + GraphTagsResponses, + GraphUntagData, + GraphUntagErrors, + GraphUntagResponses, HideEntityData, HideEntityErrors, HideEntityResponses, @@ -4804,6 +4824,110 @@ export class Graph { }, }); } + + /** + * Graph History List + * + * The graph's versions, newest first: what changed, who changed it, through what (web, token, oauth, assistant, + * mcp, routine, workflow, analysis, cli, system) and why, with the entities each touched. + */ + public static graphHistoryList( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/graph/history", + ...options, + }); + } + + /** + * Graph History Event + * + * One version in full: every record it changed, as it was before and after. + */ + public static graphHistoryEvent( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/graph/history/{version}", + ...options, + }); + } + + /** + * Graph As Of + * + * The graph as it was at a version: entities (with their other names), cross-namespace links and the pairs someone + * said are different. Mentions aren't versioned: counts elsewhere are today's. + */ + public static graphAsOf( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/graph/as-of/{version}", + ...options, + }); + } + + /** + * Graph Diff + * + * What changed between two versions: entities added, removed and changed (field by field), and other names, + * links and distinct pairs added and removed. + */ + public static graphDiff( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/graph/diff", + ...options, + }); + } + + /** + * Graph Tags + * + * Named versions, newest first. + */ + public static graphTags( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/graph/tags", + ...options, + }); + } + + /** + * Graph Tag + * + * Name a version ("before the cleanup") to come back to it; naming again moves the name. Editors of a namespace + * (or admins) can. + */ + public static graphTag( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/graph/tags", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Graph Untag + */ + public static graphUntag( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/graph/tags/{name}", + ...options, + }); + } } export class Metadata { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 217fdbd3d..3fb1ddd63 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -12123,6 +12123,26 @@ export type Suggestion = { [key: string]: unknown; }; +/** + * TagAsk + */ +export type TagAsk = { + /** + * Name + */ + name: string; + /** + * Version + * + * a version or a version's name (default: head) + */ + version?: string | null; + /** + * Note + */ + note?: string | null; +}; + /** * TagCount */ @@ -20511,6 +20531,264 @@ export type ProposeGraphChangeResponses = { export type ProposeGraphChangeResponse = ProposeGraphChangeResponses[keyof ProposeGraphChangeResponses]; +export type GraphHistoryListData = { + body?: never; + path?: never; + query?: { + /** + * Namespace + * + * one namespace (default: every namespace you can read) + */ + namespace?: string | null; + /** + * Entity + * + * only the changes that touched this entity + */ + entity?: number | null; + /** + * Before + * + * older than this version (the next page) + */ + before?: number | null; + /** + * Limit + */ + limit?: number; + }; + url: "/api/v1/graph/history"; +}; + +export type GraphHistoryListErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphHistoryListError = GraphHistoryListErrors[keyof GraphHistoryListErrors]; + +export type GraphHistoryListResponses = { + /** + * Response Graph-Graph History List + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphHistoryListResponse = GraphHistoryListResponses[keyof GraphHistoryListResponses]; + +export type GraphHistoryEventData = { + body?: never; + path: { + /** + * Version + */ + version: string; + }; + query?: never; + url: "/api/v1/graph/history/{version}"; +}; + +export type GraphHistoryEventErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphHistoryEventError = GraphHistoryEventErrors[keyof GraphHistoryEventErrors]; + +export type GraphHistoryEventResponses = { + /** + * Response Graph-Graph History Event + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphHistoryEventResponse = GraphHistoryEventResponses[keyof GraphHistoryEventResponses]; + +export type GraphAsOfData = { + body?: never; + path: { + /** + * Version + */ + version: string; + }; + query?: { + /** + * Namespace + * + * one namespace (default: every namespace you can read) + */ + namespace?: string | null; + }; + url: "/api/v1/graph/as-of/{version}"; +}; + +export type GraphAsOfErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphAsOfError = GraphAsOfErrors[keyof GraphAsOfErrors]; + +export type GraphAsOfResponses = { + /** + * Response Graph-Graph As Of + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphAsOfResponse = GraphAsOfResponses[keyof GraphAsOfResponses]; + +export type GraphDiffData = { + body?: never; + path?: never; + query: { + /** + * From + * + * a version, a version's name, or head + */ + from: string; + /** + * To + * + * a version, a version's name, or head + */ + to?: string; + /** + * Namespace + * + * one namespace (default: every namespace you can read) + */ + namespace?: string | null; + }; + url: "/api/v1/graph/diff"; +}; + +export type GraphDiffErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphDiffError = GraphDiffErrors[keyof GraphDiffErrors]; + +export type GraphDiffResponses = { + /** + * Response Graph-Graph Diff + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphDiffResponse = GraphDiffResponses[keyof GraphDiffResponses]; + +export type GraphTagsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/graph/tags"; +}; + +export type GraphTagsResponses = { + /** + * Response Graph-Graph Tags + * + * Successful Response + */ + 200: Array<{ + [key: string]: unknown; + }>; +}; + +export type GraphTagsResponse = GraphTagsResponses[keyof GraphTagsResponses]; + +export type GraphTagData = { + body: TagAsk; + path?: never; + query?: never; + url: "/api/v1/graph/tags"; +}; + +export type GraphTagErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphTagError = GraphTagErrors[keyof GraphTagErrors]; + +export type GraphTagResponses = { + /** + * Response Graph-Graph Tag + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphTagResponse = GraphTagResponses[keyof GraphTagResponses]; + +export type GraphUntagData = { + body?: never; + path: { + /** + * Name + */ + name: string; + }; + query?: never; + url: "/api/v1/graph/tags/{name}"; +}; + +export type GraphUntagErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphUntagError = GraphUntagErrors[keyof GraphUntagErrors]; + +export type GraphUntagResponses = { + /** + * Response Graph-Graph Untag + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphUntagResponse = GraphUntagResponses[keyof GraphUntagResponses]; + export type GetEntitySetupData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 5cb83de2b..7ba57514d 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -10779,6 +10779,407 @@ } } }, + "/api/v1/graph/history": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph History List", + "description": "The graph's versions, newest first: what changed, who changed it, through what (web, token, oauth, assistant,\nmcp, routine, workflow, analysis, cli, system) and why, with the entities each touched.", + "operationId": "graph_history_list", + "parameters": [ + { + "name": "namespace", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "one namespace (default: every namespace you can read)", + "title": "Namespace" + }, + "description": "one namespace (default: every namespace you can read)" + }, + { + "name": "entity", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "only the changes that touched this entity", + "title": "Entity" + }, + "description": "only the changes that touched this entity" + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "description": "older than this version (the next page)", + "title": "Before" + }, + "description": "older than this version (the next page)" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 500, + "minimum": 1, + "default": 50, + "title": "Limit" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Graph-Graph History List" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph/history/{version}": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph History Event", + "description": "One version in full: every record it changed, as it was before and after.", + "operationId": "graph_history_event", + "parameters": [ + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Version" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Graph-Graph History Event" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph/as-of/{version}": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph As Of", + "description": "The graph as it was at a version: entities (with their other names), cross-namespace links and the pairs someone\nsaid are different. Mentions aren't versioned: counts elsewhere are today's.", + "operationId": "graph_as_of", + "parameters": [ + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Version" + } + }, + { + "name": "namespace", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "one namespace (default: every namespace you can read)", + "title": "Namespace" + }, + "description": "one namespace (default: every namespace you can read)" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Graph-Graph As Of" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph/diff": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph Diff", + "description": "What changed between two versions: entities added, removed and changed (field by field), and other names,\nlinks and distinct pairs added and removed.", + "operationId": "graph_diff", + "parameters": [ + { + "name": "from", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "a version, a version's name, or head", + "title": "From" + }, + "description": "a version, a version's name, or head" + }, + { + "name": "to", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "a version, a version's name, or head", + "default": "head", + "title": "To" + }, + "description": "a version, a version's name, or head" + }, + { + "name": "namespace", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "one namespace (default: every namespace you can read)", + "title": "Namespace" + }, + "description": "one namespace (default: every namespace you can read)" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Graph-Graph Diff" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph/tags": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph Tags", + "description": "Named versions, newest first.", + "operationId": "graph_tags", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", + "title": "Response Graph-Graph Tags" + } + } + } + } + } + }, + "post": { + "tags": [ + "graph" + ], + "summary": "Graph Tag", + "description": "Name a version (\"before the cleanup\") to come back to it; naming again moves the name. Editors of a namespace\n(or admins) can.", + "operationId": "graph_tag", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TagAsk" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Graph-Graph Tag" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/graph/tags/{name}": { + "delete": { + "tags": [ + "graph" + ], + "summary": "Graph Untag", + "operationId": "graph_untag", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Name" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Graph-Graph Untag" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/entity-setup": { "get": { "tags": [ @@ -40519,6 +40920,45 @@ ], "title": "Suggestion" }, + "TagAsk": { + "properties": { + "name": { + "type": "string", + "maxLength": 80, + "minLength": 1, + "title": "Name" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version", + "description": "a version or a version's name (default: head)" + }, + "note": { + "anyOf": [ + { + "type": "string", + "maxLength": 300 + }, + { + "type": "null" + } + ], + "title": "Note" + } + }, + "type": "object", + "required": [ + "name" + ], + "title": "TagAsk" + }, "TagCount": { "properties": { "tag": { From 0222f54627c82df51b654d2922e072cb2d7f174c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 09:57:06 +0000 Subject: [PATCH 3/5] feat(graph): roll the graph back to a version, with a preview Every change since a version is undone newest first as one new event: merges with their mentions, graph changes marked undone, moved mentions back, then every record as it was. What analysis found stays. A rollback can itself be rolled back. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCB28xsaiCpiDKKk2JjRpe --- CHANGELOG.md | 3 +- DEVLOG.md | 6 +- docs/graph-history.md | 17 +- .../app/api/v1/routes/graph_history.py | 32 +++- fastapi_backend/app/domain/entity_map.py | 20 +-- fastapi_backend/app/domain/graph_history.py | 151 +++++++++++++++++- .../tests/api/test_graph_history.py | 23 +++ .../tests/domain/test_graph_history.py | 69 ++++++++ nextjs-frontend/app/openapi-client/index.ts | 6 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 24 +++ .../app/openapi-client/types.gen.ts | 53 ++++++ nextjs-frontend/openapi.json | 76 +++++++++ 12 files changed, 460 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74f4a1a4e..499324861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version "not the same" pairs is recorded as a numbered version: what changed, who (a person, a routine, the assistant, analysis), through what (the web app, a token, an app, MCP) and why. `GET /api/v1/graph/history` lists versions (one entity's with `entity`), `/graph/as-of/{version}` shows the graph as it was, `/graph/diff` what changed - between two versions, and `/graph/tags` names versions to come back to. See docs/graph-history.md. + between two versions, and `/graph/tags` names versions to come back to. `POST /graph/rollback` takes the graph back to a version, after a + preview, undoing merges with their mentions; a rollback can be rolled back too. See docs/graph-history.md. - **Query the graph in Cypher, and walk it.** The archive is now a property graph of namespaces, collections, recordings, speakers and entities. `POST /api/v1/graph/query` runs read-only Cypher (the language of Neo4j and ISO GQL) over the namespaces you can read; `/graph/related` gives a node's parents, children, ancestors, descendants or diff --git a/DEVLOG.md b/DEVLOG.md index 8eb34d74a..c2048e6cd 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -25,10 +25,10 @@ Model (fastapi_backend/app/domain/graph_history.py, docs/graph-history.md): Todo: -- [ ] Event log with who, through what and why; every curation path records (entity page, assistant, MCP, routines +- [x] Event log with who, through what and why; every curation path records (entity page, assistant, MCP, routines and graph changes, analysis, the entity map) -- [ ] Versions API: list, the graph as of a version, diff, an entity's history, named versions -- [ ] Rollback to a version, with a preview +- [x] Versions API: list, the graph as of a version, diff, an entity's history, named versions +- [x] Rollback to a version, with a preview - [ ] Checkpoints, replay and verify (`lens graph verify`, nightly checkpoint) - [ ] The explorer and Cypher as of a version (with the graph explorer thread) - [ ] Web app: a history timeline with diffs and rollback diff --git a/docs/graph-history.md b/docs/graph-history.md index 95ca29da5..18b3888e4 100644 --- a/docs/graph-history.md +++ b/docs/graph-history.md @@ -4,7 +4,7 @@ Every change to the entity graph is kept, in order, as an event that is never ch graph's version. So the graph can be seen as it was at any version, two versions compared, and one entity's history read: who renamed it, who merged what into it, which routine linked it, and why. -Built: the event log, versions, as-of, diffs, entity history, named versions. Planned: rollback to a version, +Built: the event log, versions, as-of, diffs, entity history, named versions, rollback to a version. Planned: checkpoints with replay and verify, the explorer and Cypher as of a version, a history timeline in the web app (DEVLOG.md). @@ -54,6 +54,21 @@ name, or `head`. | `GET /api/v1/graph/tags` | Named versions. | | `POST /api/v1/graph/tags` `{name, version, note}` | Name a version (default: the current one), such as "before the cleanup". Editors and admins. | | `DELETE /api/v1/graph/tags/{name}` | Drop a name; the version stays. | +| `POST /api/v1/graph/rollback` `{to, namespace, dry_run}` | Take the graph back to a version. A preview by default: the changes it would undo, newest first, and the entities, names, links and pairs that would change. `dry_run: false` does it. | + +## Rolling back + +A rollback undoes every change made since a version, in one namespace or in all the namespaces you can edit, newest +first, and records the whole of it as one new version (`graph.rollback`, with `rollback_to`). It is a change like +any other: rolling back to the version just before it brings everything back, merges included. + +- Merges come undone with their mentions; a merge undone since is merged again. Graph changes it takes back are + marked undone. Moved mentions go back to the entity they were on, and removed ones are said again. +- Then every record is set to how it was at that version. +- What analysis found since stays: it comes from the transcripts, and mentions point at it. Changes people made to + those entities are still taken back. +- An entity made since that is mentioned now stays (hide it instead); the reply lists it under `skipped`. +- It needs editor access to every namespace the changes touched: a link to a namespace you can't edit stops it. ## How it works diff --git a/fastapi_backend/app/api/v1/routes/graph_history.py b/fastapi_backend/app/api/v1/routes/graph_history.py index d7b1ba0be..9c09c4d61 100644 --- a/fastapi_backend/app/api/v1/routes/graph_history.py +++ b/fastapi_backend/app/api/v1/routes/graph_history.py @@ -8,7 +8,7 @@ from typing import Any -from fastapi import APIRouter, HTTPException, Query +from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field from app.api.deps import Acl, CurrentUser, Db, Writer, domain_errors @@ -114,3 +114,33 @@ def graph_untag(name: str, user: Writer, acl: Acl, db: Db) -> dict[str, Any]: raise HTTPException(404, "not found") from None auth.audit(db, user.as_audit(), "graph.untag", f"graph_tag:{name}") return {"ok": True} + + +class RollbackAsk(BaseModel): + to: str = Field(description="the version to go back to: a number, a version's name") + namespace: str | None = Field(None, description="only this namespace (default: every namespace you can edit)") + dry_run: bool = Field(True, description="only say what would change (the default); false to roll back") + + +@router.post("/graph/rollback") +def graph_rollback(request: Request, body: RollbackAsk, user: Writer, acl: Acl, db: Db) -> dict[str, Any]: + """Take the graph back to a version: every change since then in these namespaces is undone, newest first, as one + new version (so it can be rolled back too). Merges come undone with their mentions and moved mentions go back; + what analysis found since stays. Preview first (`dry_run`, the default). Needs editor access to every namespace the + changes touched.""" + editable = None if user.admin else set(acl.editable()) + if editable is not None and not editable: + raise HTTPException(403, "rolling back needs editor access to a namespace") + if body.namespace: + spaces: set[int] | None = {acl.namespace(body.namespace, "editor")} + else: + spaces = editable + try: + with domain_errors(): + out = graph_history.rollback(db, _version(db, body.to), spaces, editable, body.dry_run, user.email) + except PermissionError as e: + raise HTTPException(403, str(e)) from None + if out["done"]: + auth.audit(db, user.as_audit(), "graph.rollback", f"graph_event:{out['version']}", {"to": out["to"], "namespace": body.namespace}) + request.app.state.graph_cache.clear() + return out diff --git a/fastapi_backend/app/domain/entity_map.py b/fastapi_backend/app/domain/entity_map.py index 32d582081..ee7b84a5b 100644 --- a/fastapi_backend/app/domain/entity_map.py +++ b/fastapi_backend/app/domain/entity_map.py @@ -44,14 +44,14 @@ def _bkey(kind): def builtins(db, sid): """The namespace's Unknown and Unlabeled entities, made when missing: {kind: id}.""" out = {} - for kind, name in BUILTIN_NAMES.items(): - key = _bkey(kind) - row = db.one("SELECT record::id(id) AS id FROM entity WHERE ekey = $k", k=f"{sid}:{key}") - if row: - out[kind] = row["id"] - continue - eid = db.next_id("entity") - with graph_history.change(db, "entity.builtin") as ch: + with graph_history.change(db, "entity.builtin") as ch: + for kind, name in BUILTIN_NAMES.items(): + key = _bkey(kind) + row = db.one("SELECT record::id(id) AS id FROM entity WHERE ekey = $k", k=f"{sid}:{key}") + if row: + out[kind] = row["id"] + continue + eid = db.next_id("entity") ch.created(eid) db.q( "CREATE $r CONTENT $d", @@ -66,7 +66,7 @@ def builtins(db, sid): "description": BUILTIN_HELP[kind], }, ) - out[kind] = eid + out[kind] = eid return out @@ -193,7 +193,7 @@ def define(db, sid, name, typ, description=None, aliases=(), collection=None, us raise ValueError("A description can have up to 2000 characters.") with graph_history.change(db, "entity.define", entities=[row["id"]] if row else [], aliases=[(int(sid), k) for k in plan[0]]) as ch: eid = _define(db, sid, row, key, name, typ, description, collection, user, plan, ch) - builtins(db, sid) + builtins(db, sid) return eid diff --git a/fastapi_backend/app/domain/graph_history.py b/fastapi_backend/app/domain/graph_history.py index 1b574e928..0d8d74d9c 100644 --- a/fastapi_backend/app/domain/graph_history.py +++ b/fastapi_backend/app/domain/graph_history.py @@ -349,6 +349,11 @@ def diff(db, a, b, spaces=None): k = (o["t"], _key(o["t"], o["k"])) first.setdefault(k, o.get("b")) last[k] = o.get("a") + return {"from": a, "to": b, "events": events, **_describe(db, first, last, spaces)} + + +def _describe(db, first, last, spaces): + """{entities, aliases, links, distinct} that differ between rows `first` and `last` ({(table, key): row}).""" sp = set(spaces) if spaces is not None else None ent_space = {} for (t, k), row in {**first, **last}.items(): @@ -373,9 +378,6 @@ def seen(t, row): return ent_space.get(row.get("a")) in sp and ent_space.get(row.get("b")) in sp out = { - "from": a, - "to": b, - "events": events, "entities": {"added": [], "removed": [], "changed": []}, "aliases": {"added": [], "removed": []}, "links": {"added": [], "removed": []}, @@ -397,7 +399,8 @@ def seen(t, row): for f in sorted(set(before) | set(after)) if f != "ekey" and before.get(f) != after.get(f) } - out["entities"]["changed"].append({"id": k, "name": after.get("name"), "fields": fields}) + if fields: + out["entities"]["changed"].append({"id": k, "name": after.get("name"), "fields": fields}) elif t == "entity_alias": for row, kind in ((before, "removed"), (after, "added")): if row: @@ -477,3 +480,143 @@ def resolve(db, ref): if not got: raise KeyError(ref) return int(got["version"]) + + +# ---------- rolling back ---------- +MERGES = ("entity.merge", "merge.apply", "merge.accept") +APPLIED = ("merge.apply", "link.apply", "merge.accept", "link.accept") +KEPT = ("analysis",) # what analysis found stays: it comes from the transcripts, and mentions point at it + + +def _plan(db, to, spaces): + """The events a rollback to `to` in namespaces `spaces` takes back, oldest first, and the ones it leaves.""" + to = int(to) + top = head(db) + if not 0 <= to <= top: + raise ValueError(f"the graph has versions 0 to {top}") + rows = db.rows("SELECT *, record::id(id) AS version FROM graph_event WHERE id > $v ORDER BY version", v=R("graph_event", to)) + sp = set(spaces) if spaces is not None else None + mine = [ev for ev in rows if sp is None or set(ev.get("spaces") or []) & sp] + return [ev for ev in mine if ev["op"] not in KEPT], [ev for ev in mine if ev["op"] in KEPT] + + +def rollback(db, to, spaces=None, editable=None, dry_run=True, by=None): + """Take back every change made after version `to` in these namespaces (None: all), newest first, as one new event + (so a rollback can be rolled back too). What analysis found stays. `dry_run` only says what would change. + + Merges are undone with their mentions, graph changes are marked undone, moved mentions go back; then every record + is set to how it was at `to`. An entity made since that is mentioned now is kept (hide it instead).""" + events, kept = _plan(db, to, spaces) + if editable is not None: + blocked = sorted({s for ev in events for s in ev.get("spaces") or []} - set(editable)) + if blocked: + raise PermissionError("changes since then also touched namespaces you can't edit") + target = {} + for ev in events: + for o in ev["ops"]: + target.setdefault((o["t"], _key(o["t"], o["k"])), o.get("b")) + now = _snap(db, (), list(target)) + out = { + "to": int(to), + "head": head(db), + "undo": [{k: ev.get(k) for k in ("version", "op", "actor", "via", "at")} for ev in reversed(events)], + "kept": len(kept), + **_describe(db, now, target, None), + } + if dry_run or not events: + return {**out, "done": False} + with acting(actor=by, why=f"rolled back to version {int(to)}"), change(db, "graph.rollback", rollback_to=int(to)) as ch: + ch.touch(keys=list(target)) + skipped = _take_back(db, events, by) + skipped += _restore(db, target) + return {**out, "done": True, "version": ch.version, "skipped": skipped} + + +def _take_back(db, events, by): + """What has more to it than rows: merges (mentions), graph changes (their status) and moved mentions.""" + from . import entities, organize + + skipped = [] + merged = {ev["origin"]["merge"] for ev in events if ev["op"] in MERGES and (ev.get("origin") or {}).get("merge")} + unmerged = {ev["origin"]["merge"] for ev in events if ev["op"] == "entity.unmerge" and (ev.get("origin") or {}).get("merge")} + for ev in reversed(events): + o = ev.get("origin") or {} + try: + if ev["op"] in APPLIED and o.get("graph_change"): + ch = db.one("SELECT status FROM $r", r=R("graph_change", int(o["graph_change"]))) + if ch and ch["status"] == "applied": + organize.undo(db, o["graph_change"], by) + continue + if ev["op"] in MERGES and o.get("merge") and o["merge"] not in unmerged: + m = db.one("SELECT undone FROM $r", r=R("entity_merge", int(o["merge"]))) + if m and not m.get("undone"): + entities.undo_merge(db, o["merge"]) + elif ev["op"] == "entity.unmerge" and o.get("merge") and o["merge"] not in merged: + m = db.one("SELECT keep, snapshots FROM $r", r=R("entity_merge", int(o["merge"]))) + others = [sn["entity"]["id"] for sn in (m or {}).get("snapshots") or []] + if m and others: + entities.merge(db, m["keep"], others, by) + elif ev["op"] in ("mention.move", "mention.remove") and o.get("mention"): + _mention_back(db, o["mention"]) + except (ValueError, KeyError) as e: + skipped.append({"version": ev["version"], "op": ev["op"], "why": str(e)[:200]}) + return skipped + + +def _mention_back(db, mn): + """Point a moved mention back at the entity it was on (or say it again, when it was removed).""" + from . import analyze, entities + + src = mn.get("from") + if not src or not db.one("SELECT id FROM $r", r=R("entity", int(src))): + raise ValueError("the entity the mention was on is gone") + seg = R("segment", mn["segment"]) + if mn.get("to"): + row = db.one( + "SELECT record::id(id) AS id FROM mentions WHERE in = $s AND text = $t AND entity = $e LIMIT 1", s=seg, t=mn["text"], e=mn["to"] + ) + if row: + entities.move_mention(db, row["id"], target=int(src)) + return + rec = db.one("SELECT recording, space, speaker FROM $r", r=seg) + if not rec: + raise ValueError("the line the mention was on is gone") + db.q( + "RELATE $a->mentions->$b CONTENT $d", + a=seg, + b=R("entity", int(src)), + d=store.clean( + {"recording": rec["recording"], "space": rec["space"], "entity": int(src), "speaker": rec.get("speaker"), "text": mn["text"]} + ), + ) + db.q("DELETE $r", r=R("entity_override", f"{mn['segment']}:{analyze.ent_key(mn['text'])}")) + + +def _alive(db, eid): + return bool(db.one("SELECT id FROM $r", r=R("entity", int(eid)))) + + +def _restore(db, target): + """Set every record to its row in `target`: deletions first (an entity a name moves back to may need its key).""" + skipped = [] + now = _snap(db, (), list(target)) + gone = [(t, k) for (t, k), row in target.items() if row is None and now.get((t, k)) is not None] + back = [(t, k) for (t, k), row in target.items() if row is not None and now.get((t, k)) != row] + order = {t: n for n, t in enumerate(("entity_link", "entity_distinct", "entity_alias", "entity"))} + for t, k in sorted(gone, key=lambda x: order[x[0]]): + if t == "entity" and db.rows("SELECT id FROM mentions WHERE entity = $e LIMIT 1", e=int(k)): + row = now[(t, k)] + into = (target.get(("entity_alias", f"{row['space']}:{row['key']}")) or {}).get("entity") + stays = target[("entity", int(into))] is not None if ("entity", int(into or 0)) in target else bool(into and _alive(db, into)) + if into and into != int(k) and stays: + from . import entities # its name was another entity's other name then: it was merged into that one + + entities.merge(db, int(into), [int(k)]) + else: + skipped.append({"entity": int(k), "why": "it is mentioned now: hide it instead"}) + continue + if db.one("SELECT id FROM $r", r=R(t, k)): + db.q("DELETE $r", r=R(t, k)) + for t, k in sorted(back, key=lambda x: -order[x[0]]): + db.q("UPSERT $r CONTENT $d", r=R(t, k), d=target[(t, k)]) + return skipped diff --git a/fastapi_backend/tests/api/test_graph_history.py b/fastapi_backend/tests/api/test_graph_history.py index d22cdc814..28b8e307e 100644 --- a/fastapi_backend/tests/api/test_graph_history.py +++ b/fastapi_backend/tests/api/test_graph_history.py @@ -54,3 +54,26 @@ def test_named_versions(env): assert (d["from"], d["to"], d["events"]) == (v, v + 1, 1) assert c.delete("/api/v1/graph/tags/before", headers=ed).status_code == 200 assert c.get("/api/v1/graph/diff", headers=vi, params={"from": "before"}).status_code == 404 + + +def test_rollback_previews_then_takes_back(env): + c, ed, vi = env.c, env.h["editor"], env.h["viewer"] + start = c.get("/api/v1/graph/history", headers=vi).json()["head"] + dyno = env.eid("Dyno Therapeutics") + keep, other = env.eid("Northwind Labs"), env.eid("North Wind Labs") + c.post(f"/api/v1/entities/{dyno}/rename", headers=ed, json={"name": "Dyno"}) + c.post("/api/v1/entities/merge", headers=ed, json={"keep": keep, "others": [other]}) + assert c.post("/api/v1/graph/rollback", headers=vi, json={"to": str(start)}).status_code == 403 + pv = c.post("/api/v1/graph/rollback", headers=ed, json={"to": str(start)}).json() + assert pv["done"] is False and [u["op"] for u in pv["undo"]] == ["entity.merge", "entity.rename"] + assert c.get("/api/v1/graph/history", headers=vi).json()["head"] == start + 2 # nothing changed yet + done = c.post("/api/v1/graph/rollback", headers=ed, json={"to": str(start), "dry_run": False}).json() + assert done["done"] and done["version"] == start + 3 + assert c.get(f"/api/v1/entities/{dyno}", headers=vi).json()["name"] == "Dyno Therapeutics" + assert c.get(f"/api/v1/entities/{other}", headers=vi).status_code == 200 # the merge came undone + top = c.get("/api/v1/graph/history", headers=vi).json()["versions"][0] + assert (top["op"], top["actor"], top["via"]) == ("graph.rollback", "ed@x.io", "web") + # a link to a namespace the editor can't edit blocks a rollback past it + c.post(f"/api/v1/entities/{dyno}/link", headers=env.h["admin"], json={"with": env.eid("Dyno Therapeutics", "calls")}) + assert c.post("/api/v1/graph/rollback", headers=ed, json={"to": str(start)}).status_code == 403 + assert c.post("/api/v1/graph/rollback", headers=env.h["admin"], json={"to": str(start), "dry_run": False}).json()["done"] diff --git a/fastapi_backend/tests/domain/test_graph_history.py b/fastapi_backend/tests/domain/test_graph_history.py index 99f891030..7e4ae8c33 100644 --- a/fastapi_backend/tests/domain/test_graph_history.py +++ b/fastapi_backend/tests/domain/test_graph_history.py @@ -154,3 +154,72 @@ def test_named_versions(db, cfg, spaces): gh.untag(db, "before the cleanup") with pytest.raises(KeyError): gh.resolve(db, "before the cleanup") + + +def test_rollback_takes_everything_back_and_can_itself_be_rolled_back(db, cfg, spaces): + pods = store.ns_id(db, "pods") + v0 = gh.head(db) + start = now(db) + counts = {e["id"]: e["mentions"] for e in entities.list_entities(db, spaces)["items"]} + dyno, aws = eid(db, spaces, "Dyno Therapeutics"), eid(db, spaces, "AWS") + keep, other = eid(db, spaces, "Northwind Labs"), eid(db, spaces, "North Wind Labs") + entities.rename(db, dyno, "Dyno") + entities.hide(db, aws, True, "noise") + entities.merge(db, keep, [other], "ed@x.io") + cid, _ = organize.propose(db, "merge", aws, eid(db, spaces, "Amazon Web Services"), "acronym", apply=True, user="ed@x.io") + mention = entities.mentions(db, dyno, spaces)["items"][0]["mention"] + moved_to = entities.move_mention(db, mention, new_name="Dyno Labs") + entity_map.define(db, pods, "Lens", "PRODUCT", "Our archive") + after = now(db) + top = gh.head(db) + + pv = gh.rollback(db, v0, {pods}) + assert pv["done"] is False and gh.head(db) == top # a preview changes nothing + assert [u["op"] for u in pv["undo"]][:2] == ["entity.define", "mention.move"] + assert {e["name"] for e in pv["entities"]["added"]} >= {"North Wind Labs", "Amazon Web Services"} + assert {e["name"] for e in pv["entities"]["removed"]} >= {"Lens", "Dyno Labs"} + assert any(c["fields"].get("name") == ["Dyno", "Dyno Therapeutics"] for c in pv["entities"]["changed"]) + + done = gh.rollback(db, v0, {pods}, dry_run=False, by="ed@x.io") + assert done["done"] and done["skipped"] == [] and gh.head(db) == done["version"] + assert now(db) == start + assert {e["id"]: e["mentions"] for e in entities.list_entities(db, spaces)["items"]} == counts # mentions are back too + assert organize.get_change(db, cid)["status"] == "undone" + assert not db.one("SELECT id FROM $r", r=store.R("entity", moved_to)) + ev = gh.versions(db, limit=1)[0] + assert (ev["op"], ev["actor"], ev["why"], ev["origin"]["rollback_to"]) == ( + "graph.rollback", + "ed@x.io", + f"rolled back to version {v0}", + v0, + ) + + # the rollback is a change like any other: rolling back to just before it brings the later state back + again = gh.rollback(db, top, {pods}) + assert again["done"] is False + redo = gh.rollback(db, top, {pods}, dry_run=False) + assert redo["skipped"] == [] + assert now(db) == after # merges it undid are merged again + + +def test_rollback_needs_editor_rights_on_every_namespace_touched(db, cfg, spaces): + pods, calls = store.ns_id(db, "pods"), store.ns_id(db, "calls") + v0 = gh.head(db) + entities.link(db, eid(db, spaces, "Dyno Therapeutics"), eid(db, spaces, "Dyno Therapeutics", "calls")) + with pytest.raises(PermissionError): + gh.rollback(db, v0, {pods}, editable=[pods]) + assert gh.rollback(db, v0, {pods}, editable=[pods, calls], dry_run=False)["done"] + assert db.rows("SELECT * FROM entity_link") == [] + + +def test_rollback_keeps_what_analysis_found(db, cfg, spaces, folder): + v0 = gh.head(db) + p = folder / "more.txt" + p.write_text("Alice|N|Zebra Robotics called again about Zebra Robotics.") + ingest.import_transcript(db, cfg, "pods", p, log=quiet) + analyze.analyze_pending(db, cfg, log=quiet) + zebra = eid(db, spaces, "Zebra Robotics") + entities.rename(db, zebra, "Zebra") + out = gh.rollback(db, v0, dry_run=False) + assert out["kept"] == 1 + assert entities.detail(db, zebra, spaces)["name"] == "Zebra Robotics" # found since: kept, its rename taken back diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 4d51793df..fbfcec8ad 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1013,6 +1013,11 @@ export type { GraphRelatedErrors, GraphRelatedResponse, GraphRelatedResponses, + GraphRollbackData, + GraphRollbackError, + GraphRollbackErrors, + GraphRollbackResponse, + GraphRollbackResponses, GraphSchemaData, GraphSchemaError, GraphSchemaErrors, @@ -1814,6 +1819,7 @@ export type { RevokeTokenErrors, RevokeTokenResponse, RevokeTokenResponses, + RollbackAsk, RotateNotifySecretData, RotateNotifySecretError, RotateNotifySecretErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 9f23952b9..504addb9a 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -510,6 +510,9 @@ import type { GraphRelatedData, GraphRelatedErrors, GraphRelatedResponses, + GraphRollbackData, + GraphRollbackErrors, + GraphRollbackResponses, GraphSchemaData, GraphSchemaErrors, GraphSchemaResponses, @@ -4928,6 +4931,27 @@ export class Graph { ...options, }); } + + /** + * Graph Rollback + * + * Take the graph back to a version: every change since then in these namespaces is undone, newest first, as one + * new version (so it can be rolled back too). Merges come undone with their mentions and moved mentions go back; + * what analysis found since stays. Preview first (`dry_run`, the default). Needs editor access to every namespace the + * changes touched. + */ + public static graphRollback( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/graph/rollback", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } } export class Metadata { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 3fb1ddd63..9f8df754e 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -9607,6 +9607,30 @@ export type Reviewed = { [key: string]: unknown; }; +/** + * RollbackAsk + */ +export type RollbackAsk = { + /** + * To + * + * the version to go back to: a number, a version's name + */ + to: string; + /** + * Namespace + * + * only this namespace (default: every namespace you can edit) + */ + namespace?: string | null; + /** + * Dry Run + * + * only say what would change (the default); false to roll back + */ + dry_run?: boolean; +}; + /** * Routine */ @@ -20789,6 +20813,35 @@ export type GraphUntagResponses = { export type GraphUntagResponse = GraphUntagResponses[keyof GraphUntagResponses]; +export type GraphRollbackData = { + body: RollbackAsk; + path?: never; + query?: never; + url: "/api/v1/graph/rollback"; +}; + +export type GraphRollbackErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GraphRollbackError = GraphRollbackErrors[keyof GraphRollbackErrors]; + +export type GraphRollbackResponses = { + /** + * Response Graph-Graph Rollback + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphRollbackResponse = GraphRollbackResponses[keyof GraphRollbackResponses]; + export type GetEntitySetupData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 7ba57514d..852705dee 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -11180,6 +11180,50 @@ } } }, + "/api/v1/graph/rollback": { + "post": { + "tags": [ + "graph" + ], + "summary": "Graph Rollback", + "description": "Take the graph back to a version: every change since then in these namespaces is undone, newest first, as one\nnew version (so it can be rolled back too). Merges come undone with their mentions and moved mentions go back;\nwhat analysis found since stays. Preview first (`dry_run`, the default). Needs editor access to every namespace the\nchanges touched.", + "operationId": "graph_rollback", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RollbackAsk" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Graph-Graph Rollback" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/entity-setup": { "get": { "tags": [ @@ -36496,6 +36540,38 @@ ], "title": "Reviewed" }, + "RollbackAsk": { + "properties": { + "to": { + "type": "string", + "title": "To", + "description": "the version to go back to: a number, a version's name" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace", + "description": "only this namespace (default: every namespace you can edit)" + }, + "dry_run": { + "type": "boolean", + "title": "Dry Run", + "description": "only say what would change (the default); false to roll back", + "default": true + } + }, + "type": "object", + "required": [ + "to" + ], + "title": "RollbackAsk" + }, "Routine": { "properties": { "id": { From c186066306ecc9afb57a79199c7eb957bf591ac0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 10:00:54 +0000 Subject: [PATCH 4/5] feat(graph): checkpoints, replay and verify for the graph's history The graph is kept whole with its first recorded change and every 1000 after; replaying from the newest checkpoint rebuilds it from history alone. Verify compares the replay with today's graph and can record any unrecorded writes as one graph.drift change. lens history verify, checkpoint and list. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCB28xsaiCpiDKKk2JjRpe --- CHANGELOG.md | 4 +- DEVLOG.md | 2 +- docs/graph-history.md | 25 ++++- .../app/api/v1/routes/graph_history.py | 33 +++++- fastapi_backend/app/cli.py | 28 +++++ fastapi_backend/app/domain/graph_history.py | 103 ++++++++++++++++++ fastapi_backend/app/domain/store.py | 3 + .../tests/api/test_graph_history.py | 12 ++ fastapi_backend/tests/domain/test_cli.py | 4 + .../tests/domain/test_graph_history.py | 30 +++++ nextjs-frontend/app/openapi-client/index.ts | 12 ++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 65 +++++++++++ .../app/openapi-client/types.gen.ts | 80 ++++++++++++++ nextjs-frontend/openapi.json | 95 ++++++++++++++++ 14 files changed, 489 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 499324861..f54c0a36e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version analysis), through what (the web app, a token, an app, MCP) and why. `GET /api/v1/graph/history` lists versions (one entity's with `entity`), `/graph/as-of/{version}` shows the graph as it was, `/graph/diff` what changed between two versions, and `/graph/tags` names versions to come back to. `POST /graph/rollback` takes the graph back to a version, after a - preview, undoing merges with their mentions; a rollback can be rolled back too. See docs/graph-history.md. + preview, undoing merges with their mentions; a rollback can be rolled back too. Checkpoints let the graph be replayed + from its history, and `lens history verify` (or `/graph/verify`) checks the replay matches today's graph. See + docs/graph-history.md. - **Query the graph in Cypher, and walk it.** The archive is now a property graph of namespaces, collections, recordings, speakers and entities. `POST /api/v1/graph/query` runs read-only Cypher (the language of Neo4j and ISO GQL) over the namespaces you can read; `/graph/related` gives a node's parents, children, ancestors, descendants or diff --git a/DEVLOG.md b/DEVLOG.md index c2048e6cd..6a9e22d3e 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -29,7 +29,7 @@ Todo: and graph changes, analysis, the entity map) - [x] Versions API: list, the graph as of a version, diff, an entity's history, named versions - [x] Rollback to a version, with a preview -- [ ] Checkpoints, replay and verify (`lens graph verify`, nightly checkpoint) +- [x] Checkpoints, replay and verify (`lens graph verify`, nightly checkpoint) - [ ] The explorer and Cypher as of a version (with the graph explorer thread) - [ ] Web app: a history timeline with diffs and rollback - [ ] Graph events in each resource's activity history (with the activity history thread) diff --git a/docs/graph-history.md b/docs/graph-history.md index 18b3888e4..04d72775d 100644 --- a/docs/graph-history.md +++ b/docs/graph-history.md @@ -4,9 +4,8 @@ Every change to the entity graph is kept, in order, as an event that is never ch graph's version. So the graph can be seen as it was at any version, two versions compared, and one entity's history read: who renamed it, who merged what into it, which routine linked it, and why. -Built: the event log, versions, as-of, diffs, entity history, named versions, rollback to a version. Planned: -checkpoints with replay and verify, the explorer and Cypher as of a version, a history timeline in the web app -(DEVLOG.md). +Built: the event log, versions, as-of, diffs, entity history, named versions, rollback to a version, checkpoints with +replay and verify. Planned: the explorer and Cypher as of a version, a history timeline in the web app (DEVLOG.md). ## What is versioned @@ -70,6 +69,24 @@ any other: rolling back to the version just before it brings everything back, me - An entity made since that is mentioned now stays (hide it instead); the reply lists it under `skipped`. - It needs editor access to every namespace the changes touched: a link to a namespace you can't edit stops it. +## Replay and verify + +A checkpoint keeps the whole graph as it was at one version. The first is taken with the first recorded change (the +graph as it was before it, so an archive that had entities before this feature replays too), then one every 1000 +changes. Replaying starts from the newest checkpoint and applies the events after it, without reading today's rows. + +Verify replays the history and compares it with the graph. Anything that differs was written without being recorded +(by an older version of Lens, or by hand in the database). Fixing records it as one `graph.drift` change, so the +history matches again. + +| Call | What it does | +| --- | --- | +| `GET /api/v1/graph/verify` | `{same, differences, from, head}` and what differs. Admins. | +| `POST /api/v1/graph/verify` | Record what differs as one change. Admins. | +| `GET /api/v1/graph/checkpoints`, `POST /api/v1/graph/checkpoints` | List checkpoints; keep one now. Admins. | + +On the command line: `lens history verify [--fix]`, `lens history checkpoint`, `lens history list`. + ## How it works Code that changes the graph wraps its writes in `graph_history.change(db, op, entities=...)`. It reads the rows of @@ -85,4 +102,4 @@ events between two versions: for each record, how it was before the first and af - Speakers (same-person links, names) in the history. - A mention-level history. -- Checkpoints so very old versions don't walk back through every later event. +- As-of from the nearest checkpoint, so very old versions don't walk back through every later event. diff --git a/fastapi_backend/app/api/v1/routes/graph_history.py b/fastapi_backend/app/api/v1/routes/graph_history.py index 9c09c4d61..7b268b5a1 100644 --- a/fastapi_backend/app/api/v1/routes/graph_history.py +++ b/fastapi_backend/app/api/v1/routes/graph_history.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field -from app.api.deps import Acl, CurrentUser, Db, Writer, domain_errors +from app.api.deps import Acl, AdminReader, AdminWriter, CurrentUser, Db, Writer, domain_errors from app.domain import auth, graph_history router = APIRouter(tags=["graph"]) @@ -144,3 +144,34 @@ def graph_rollback(request: Request, body: RollbackAsk, user: Writer, acl: Acl, auth.audit(db, user.as_audit(), "graph.rollback", f"graph_event:{out['version']}", {"to": out["to"], "namespace": body.namespace}) request.app.state.graph_cache.clear() return out + + +@router.get("/graph/verify") +def graph_verify(user: AdminReader, db: Db) -> dict[str, Any]: + """Replay the history from its newest checkpoint and compare it with today's graph: what differs was written + without being recorded. Admins.""" + return graph_history.verify(db) + + +@router.post("/graph/verify") +def graph_verify_fix(request: Request, user: AdminWriter, db: Db) -> dict[str, Any]: + """Record what differs from the replayed history as one change (`graph.drift`), so they match again. Admins.""" + out = graph_history.verify(db, fix=True) + if out.get("version"): + auth.audit(db, user.as_audit(), "graph.drift", f"graph_event:{out['version']}", {"differences": out["differences"]}) + request.app.state.graph_cache.clear() + return out + + +@router.get("/graph/checkpoints") +def graph_checkpoints(user: AdminReader, db: Db) -> list[dict[str, Any]]: + """The versions the graph is kept whole at, to replay from (taken with the first change, then every 1000).""" + return graph_history.checkpoints(db) + + +@router.post("/graph/checkpoints") +def graph_checkpoint(user: AdminWriter, db: Db) -> dict[str, Any]: + """Keep the whole graph as it is now. Admins.""" + v = graph_history.checkpoint(db) + auth.audit(db, user.as_audit(), "graph.checkpoint", f"graph_checkpoint:{v}") + return {"version": v} diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 49d9ce3ab..bfdf2daf3 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -92,6 +92,13 @@ def _main_base(argv=None): p = sub.add_parser("graph") p.add_argument("--scope", default="global", help="global, or ns:") p.add_argument("--out") + p = sub.add_parser("history", help="the entity graph's history: verify it replays to today's graph, or keep a checkpoint") + h2 = p.add_subparsers(dest="action", required=True) + h2.add_parser("verify", help="replay the history and compare it with the graph").add_argument( + "--fix", action="store_true", help="record what differs as one change" + ) + h2.add_parser("checkpoint", help="keep the whole graph as it is now, to replay from") + h2.add_parser("list", help="the latest versions") p = sub.add_parser("serve") p.add_argument("--host") p.add_argument("--port", type=int) @@ -172,6 +179,27 @@ def _main_base(argv=None): if a.out: pathlib.Path(a.out).write_text(text, encoding="utf-8") print(f"{len(g['nodes'])} nodes, {len(g['edges'])} edges across {', '.join(g['namespaces']) or 'nothing'}") + elif a.cmd == "history": + from .domain import graph_history as gh + + if a.action == "checkpoint": + print(f"checkpoint at version {gh.checkpoint(conn)}") + elif a.action == "list": + for v in gh.versions(conn, limit=20): + print( + f" v{v['version']} {v['at']} {v['op']} by {v['actor']} via {v['via']}" + (f": {v['why']}" if v.get("why") else "") + ) + else: + with gh.acting(actor="cli", via="cli"): + got = gh.verify(conn, fix=a.fix) + if got["same"]: + print(f"the history replays to today's graph (version {got['head']}, from checkpoint {got['from']})") + else: + print( + f"{got['differences']} records differ from the replayed history" + + (f"; recorded as version {got['version']}" if got.get("version") else "; --fix records them") + ) + raise SystemExit(0 if got.get("version") else 1) elif a.cmd == "status": names = store.space_names(conn) agg: dict[tuple[str, str | None], tuple[int, int]] = {} diff --git a/fastapi_backend/app/domain/graph_history.py b/fastapi_backend/app/domain/graph_history.py index 0d8d74d9c..fb0a0bfb7 100644 --- a/fastapi_backend/app/domain/graph_history.py +++ b/fastapi_backend/app/domain/graph_history.py @@ -23,11 +23,13 @@ import contextlib import contextvars +import logging import re from . import store R = store.R +log = logging.getLogger("lens") TABLES = ("entity", "entity_alias", "entity_link", "entity_distinct") VIAS = ("web", "token", "oauth", "assistant", "mcp", "routine", "workflow", "analysis", "cli", "system") SUMMARY = "record::id(id) AS version, at, op, actor, via, why, spaces, entities, origin, array::len(ops) AS changes" @@ -222,6 +224,11 @@ def record(db, op, ops, why=None, origin=None, actor=None, via=None, entities=() } ) db.q("CREATE $r CONTENT $d", r=R("graph_event", n), d=row) + if n == 1 or n % CHECKPOINT_EVERY == 0: # the graph before its first change, then every so often, to replay from + try: + checkpoint(db, 0 if n == 1 else n) + except Exception: # noqa: BLE001 - a missed checkpoint is taken later; the change is still recorded + log.exception("graph checkpoint at %s failed", n) return n @@ -620,3 +627,99 @@ def _restore(db, target): for t, k in sorted(back, key=lambda x: -order[x[0]]): db.q("UPSERT $r CONTENT $d", r=R(t, k), d=target[(t, k)]) return skipped + + +# ---------- checkpoints, replay and verify ---------- +CHECKPOINT_EVERY = 1000 # events between automatic checkpoints +PART_ROWS = 2000 # records per stored part of a checkpoint + + +def _flat(state): + return [[t, k, row] for t in TABLES for k, row in sorted(state[t].items(), key=lambda x: str(x[0]))] + + +def checkpoint(db, version=None): + """Keep the whole graph as it was at `version` (default: now), so it can be replayed from there. Its version.""" + v = head(db) if version is None else int(version) + state = state_at(db, v) + rows = _flat(state) + db.q("DELETE graph_checkpoint_part WHERE checkpoint = $v", v=v) + parts = [rows[i : i + PART_ROWS] for i in range(0, len(rows), PART_ROWS)] + for n, part in enumerate(parts): + db.q("CREATE $r CONTENT $d", r=R("graph_checkpoint_part", f"{v}-{n}"), d={"checkpoint": v, "n": n, "rows": part}) + db.q( + "UPSERT $r CONTENT $d", + r=R("graph_checkpoint", v), + d={"version": v, "at": store.now(), "parts": len(parts), "counts": {t: len(state[t]) for t in TABLES}}, + ) + return v + + +def checkpoints(db): + return db.rows("SELECT version, at, counts FROM graph_checkpoint ORDER BY version DESC") + + +def _load(db, v): + s = {t: {} for t in TABLES} + for part in db.rows("SELECT n, rows FROM graph_checkpoint_part WHERE checkpoint = $v ORDER BY n", v=int(v)): + for t, k, row in part["rows"]: + s[t][_key(t, k)] = row + return s + + +def replay(db, upto=None): + """The graph rebuilt from the newest checkpoint at or before `upto` (default: now) and the events after it, without + reading today's rows. (state, the checkpoint it started from).""" + top = head(db) + upto = top if upto is None else int(upto) + base = db.one("SELECT version FROM graph_checkpoint WHERE version <= $u ORDER BY version DESC LIMIT 1", u=upto) + if not base: + ensure_checkpoint(db) + base = db.one("SELECT version FROM graph_checkpoint WHERE version <= $u ORDER BY version DESC LIMIT 1", u=upto) + s = _load(db, base["version"]) + for ev in _events_after(db, base["version"], upto): + for o in ev["ops"]: + k = _key(o["t"], o["k"]) + if o.get("a") is None: + s[o["t"]].pop(k, None) + else: + s[o["t"]][k] = o["a"] + return s, base["version"] + + +def verify(db, fix=False): + """Replay the history and compare it with today's rows. Anything that differs was written without being recorded; + `fix` records it as one `graph.drift` event, so the history matches again. {from, head, same, differences}.""" + replayed, base = replay(db) + live_rows = live(db) + first = {(t, k): row for t in TABLES for k, row in replayed[t].items()} + last = {(t, k): row for t in TABLES for k, row in live_rows[t].items()} + keys = set(first) | set(last) + first = {k: first.get(k) for k in keys} + last = {k: last.get(k) for k in keys} + differ = [k for k in keys if first[k] != last[k]] + out = {"from": base, "head": head(db), "same": not differ, "differences": len(differ), **_describe(db, first, last, None)} + if differ and fix: + ents = {k: r for (t, k), r in {**first, **last}.items() if t == "entity" and r} + + def space_of(e): + return (ents.get(int(e)) or {}).get("space") if e is not None else None + + ops = [ + store.clean({"t": t, "k": k, "b": first[(t, k)], "a": last[(t, k)]}) + | {"s": sorted(_row_spaces(t, first[(t, k)], space_of) | _row_spaces(t, last[(t, k)], space_of))} + for t, k in sorted(differ, key=lambda x: (TABLES.index(x[0]), str(x[1]))) + ] + out["version"] = record(db, "graph.drift", ops, why="written without being recorded; found by verify", actor="system", via="system") + return out + + +def ensure_checkpoint(db): + """A first checkpoint (the graph before its first recorded change) once there is none, and another every + CHECKPOINT_EVERY events. Cheap when nothing is due.""" + last = db.one("SELECT version FROM graph_checkpoint ORDER BY version DESC LIMIT 1") + if not last: + return checkpoint(db, 0) + if head(db) - last["version"] >= CHECKPOINT_EVERY: + return checkpoint(db) + return None diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 9dcef7807..39c030a23 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -837,6 +837,9 @@ def close(self): # the entity graph's history (graph_history.py): one append-only event per change, and named versions "DEFINE TABLE IF NOT EXISTS graph_event SCHEMALESS", "DEFINE TABLE IF NOT EXISTS graph_tag SCHEMALESS", + "DEFINE TABLE IF NOT EXISTS graph_checkpoint SCHEMALESS", + "DEFINE TABLE IF NOT EXISTS graph_checkpoint_part SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS graph_checkpoint_part_cp ON graph_checkpoint_part FIELDS checkpoint", "DEFINE TABLE IF NOT EXISTS output SCHEMALESS", "DEFINE INDEX IF NOT EXISTS output_rec ON output FIELDS recording", "DEFINE TABLE IF NOT EXISTS chat SCHEMALESS", diff --git a/fastapi_backend/tests/api/test_graph_history.py b/fastapi_backend/tests/api/test_graph_history.py index 28b8e307e..483c5d96b 100644 --- a/fastapi_backend/tests/api/test_graph_history.py +++ b/fastapi_backend/tests/api/test_graph_history.py @@ -77,3 +77,15 @@ def test_rollback_previews_then_takes_back(env): c.post(f"/api/v1/entities/{dyno}/link", headers=env.h["admin"], json={"with": env.eid("Dyno Therapeutics", "calls")}) assert c.post("/api/v1/graph/rollback", headers=ed, json={"to": str(start)}).status_code == 403 assert c.post("/api/v1/graph/rollback", headers=env.h["admin"], json={"to": str(start), "dry_run": False}).json()["done"] + + +def test_verify_and_checkpoints_are_for_admins(env): + c, ed, ad = env.c, env.h["editor"], env.h["admin"] + assert c.get("/api/v1/graph/verify", headers=ed).status_code == 403 + assert c.get("/api/v1/graph/verify", headers=ad).json()["same"] is True + v = c.post("/api/v1/graph/checkpoints", headers=ad).json()["version"] + assert v in [x["version"] for x in c.get("/api/v1/graph/checkpoints", headers=ad).json()] + env.db.q("UPDATE entity SET name = 'X' WHERE name = 'AWS'") + assert c.get("/api/v1/graph/verify", headers=ad).json()["differences"] == 1 + assert c.post("/api/v1/graph/verify", headers=ad).json()["version"] == v + 1 + assert c.get("/api/v1/graph/verify", headers=ad).json()["same"] is True diff --git a/fastapi_backend/tests/domain/test_cli.py b/fastapi_backend/tests/domain/test_cli.py index 1d5592359..2b7b1f31f 100644 --- a/fastapi_backend/tests/domain/test_cli.py +++ b/fastapi_backend/tests/domain/test_cli.py @@ -23,6 +23,9 @@ def run(*args): run("search", "capsid") run("speakers", "list", "pods") run("status") + run("history", "verify") + run("history", "checkpoint") + run("history", "list") monkeypatch.setattr("sys.stdin", io.StringIO("cli password 123\n")) run("users", "add", "cli@x.io", "--admin", "--password-stdin") run("users", "list") @@ -38,6 +41,7 @@ def run(*args): assert "match(es)" in text assert "Zed" in text assert "surrealkv://" in text + assert "the history replays to today's graph" in text and "analysis by analysis via analysis" in text assert (tmp_path / "data" / "reports" / "pods" / "index.html").exists() diff --git a/fastapi_backend/tests/domain/test_graph_history.py b/fastapi_backend/tests/domain/test_graph_history.py index 7e4ae8c33..601f0f366 100644 --- a/fastapi_backend/tests/domain/test_graph_history.py +++ b/fastapi_backend/tests/domain/test_graph_history.py @@ -223,3 +223,33 @@ def test_rollback_keeps_what_analysis_found(db, cfg, spaces, folder): out = gh.rollback(db, v0, dry_run=False) assert out["kept"] == 1 assert entities.detail(db, zebra, spaces)["name"] == "Zebra Robotics" # found since: kept, its rename taken back + + +def test_replay_from_checkpoints_and_verify(db, cfg, spaces, monkeypatch): + assert [c["version"] for c in gh.checkpoints(db)] == [0] # taken with the first change + assert gh.checkpoints(db)[0]["counts"]["entity"] == 0 + dyno = eid(db, spaces, "Dyno Therapeutics") + entities.rename(db, dyno, "Dyno") + state, base = gh.replay(db) + assert base == 0 and state == gh.live(db) # the whole graph, from nothing but its history + assert gh.verify(db)["same"] is True + monkeypatch.setattr(gh, "PART_ROWS", 3) + v = gh.checkpoint(db) + assert gh.replay(db)[1] == v and gh.replay(db)[0] == gh.live(db) + assert gh.replay(db, v - 1)[0] == gh.state_at(db, v - 1) # an older version replays from the older checkpoint + # a write that went around the history is found, and recorded when asked + db.q("UPDATE $r SET name = 'Sneaky'", r=store.R("entity", dyno)) + found = gh.verify(db) + assert found["same"] is False and found["entities"]["changed"][0]["fields"]["name"] == ["Dyno", "Sneaky"] + fixed = gh.verify(db, fix=True) + assert gh.event(db, fixed["version"])["op"] == "graph.drift" + assert gh.verify(db)["same"] is True + + +def test_checkpoints_every_so_often(db, cfg, spaces, monkeypatch): + monkeypatch.setattr(gh, "CHECKPOINT_EVERY", 2) + dyno = eid(db, spaces, "Dyno Therapeutics") + for n in range(4): + entities.rename(db, dyno, f"Dyno {n}") + assert len(gh.checkpoints(db)) >= 3 + assert gh.replay(db)[0] == gh.live(db) diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index fbfcec8ad..47ea60c05 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -982,6 +982,12 @@ export type { GraphAsOfResponses, GraphChange, GraphChangeAccept, + GraphCheckpointData, + GraphCheckpointResponse, + GraphCheckpointResponses, + GraphCheckpointsData, + GraphCheckpointsResponse, + GraphCheckpointsResponses, GraphDiffData, GraphDiffError, GraphDiffErrors, @@ -1036,6 +1042,12 @@ export type { GraphUntagErrors, GraphUntagResponse, GraphUntagResponses, + GraphVerifyData, + GraphVerifyFixData, + GraphVerifyFixResponse, + GraphVerifyFixResponses, + GraphVerifyResponse, + GraphVerifyResponses, Handling, HandlingChange, Health, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 504addb9a..f28a80d06 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -492,6 +492,10 @@ import type { GraphAsOfData, GraphAsOfErrors, GraphAsOfResponses, + GraphCheckpointData, + GraphCheckpointResponses, + GraphCheckpointsData, + GraphCheckpointsResponses, GraphDiffData, GraphDiffErrors, GraphDiffResponses, @@ -524,6 +528,10 @@ import type { GraphUntagData, GraphUntagErrors, GraphUntagResponses, + GraphVerifyData, + GraphVerifyFixData, + GraphVerifyFixResponses, + GraphVerifyResponses, HideEntityData, HideEntityErrors, HideEntityResponses, @@ -4952,6 +4960,63 @@ export class Graph { }, }); } + + /** + * Graph Verify + * + * Replay the history from its newest checkpoint and compare it with today's graph: what differs was written + * without being recorded. Admins. + */ + public static graphVerify( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/graph/verify", + ...options, + }); + } + + /** + * Graph Verify Fix + * + * Record what differs from the replayed history as one change (`graph.drift`), so they match again. Admins. + */ + public static graphVerifyFix( + options?: Options, + ): RequestResult { + return (options?.client ?? client).post({ + url: "/api/v1/graph/verify", + ...options, + }); + } + + /** + * Graph Checkpoints + * + * The versions the graph is kept whole at, to replay from (taken with the first change, then every 1000). + */ + public static graphCheckpoints( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/graph/checkpoints", + ...options, + }); + } + + /** + * Graph Checkpoint + * + * Keep the whole graph as it is now. Admins. + */ + public static graphCheckpoint( + options?: Options, + ): RequestResult { + return (options?.client ?? client).post({ + url: "/api/v1/graph/checkpoints", + ...options, + }); + } } export class Metadata { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 9f8df754e..598a55909 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -20842,6 +20842,86 @@ export type GraphRollbackResponses = { export type GraphRollbackResponse = GraphRollbackResponses[keyof GraphRollbackResponses]; +export type GraphVerifyData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/graph/verify"; +}; + +export type GraphVerifyResponses = { + /** + * Response Graph-Graph Verify + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphVerifyResponse = GraphVerifyResponses[keyof GraphVerifyResponses]; + +export type GraphVerifyFixData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/graph/verify"; +}; + +export type GraphVerifyFixResponses = { + /** + * Response Graph-Graph Verify Fix + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphVerifyFixResponse = GraphVerifyFixResponses[keyof GraphVerifyFixResponses]; + +export type GraphCheckpointsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/graph/checkpoints"; +}; + +export type GraphCheckpointsResponses = { + /** + * Response Graph-Graph Checkpoints + * + * Successful Response + */ + 200: Array<{ + [key: string]: unknown; + }>; +}; + +export type GraphCheckpointsResponse = GraphCheckpointsResponses[keyof GraphCheckpointsResponses]; + +export type GraphCheckpointData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/graph/checkpoints"; +}; + +export type GraphCheckpointResponses = { + /** + * Response Graph-Graph Checkpoint + * + * Successful Response + */ + 200: { + [key: string]: unknown; + }; +}; + +export type GraphCheckpointResponse = GraphCheckpointResponses[keyof GraphCheckpointResponses]; + export type GetEntitySetupData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 852705dee..f8d6ae272 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -11224,6 +11224,101 @@ } } }, + "/api/v1/graph/verify": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph Verify", + "description": "Replay the history from its newest checkpoint and compare it with today's graph: what differs was written\nwithout being recorded. Admins.", + "operationId": "graph_verify", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Graph-Graph Verify" + } + } + } + } + } + }, + "post": { + "tags": [ + "graph" + ], + "summary": "Graph Verify Fix", + "description": "Record what differs from the replayed history as one change (`graph.drift`), so they match again. Admins.", + "operationId": "graph_verify_fix", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Graph-Graph Verify Fix" + } + } + } + } + } + } + }, + "/api/v1/graph/checkpoints": { + "get": { + "tags": [ + "graph" + ], + "summary": "Graph Checkpoints", + "description": "The versions the graph is kept whole at, to replay from (taken with the first change, then every 1000).", + "operationId": "graph_checkpoints", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", + "title": "Response Graph-Graph Checkpoints" + } + } + } + } + } + }, + "post": { + "tags": [ + "graph" + ], + "summary": "Graph Checkpoint", + "description": "Keep the whole graph as it is now. Admins.", + "operationId": "graph_checkpoint", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Graph-Graph Checkpoint" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/entity-setup": { "get": { "tags": [ From d3e3bd9dd286795797c5cc768003554232244f50 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 4 Oct 2026 10:09:33 +0000 Subject: [PATCH 5/5] feat(graph): a History tab for the entity graph: every change, what it did, named versions and rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Routines › History lists every version with who, through what and why; opening one shows what changed and lets editors name it or roll back to it after a preview. An entity's History button shows its own changes. Diffs name the entities other names point at. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCB28xsaiCpiDKKk2JjRpe --- CHANGELOG.md | 2 + DEVLOG.md | 2 +- docs/graph-history.md | 5 +- fastapi_backend/app/domain/graph_history.py | 5 +- .../tests/domain/test_graph_history.py | 1 + .../__tests__/graph-history.test.ts | 36 ++ .../app/(app)/routines/history/page.tsx | 12 + .../components/entities/entity-drawer.tsx | 13 +- .../components/routines/graph-history.tsx | 330 ++++++++++++++++++ .../components/routines/history-model.ts | 109 ++++++ .../components/routines/routines-page.tsx | 5 +- 11 files changed, 511 insertions(+), 9 deletions(-) create mode 100644 nextjs-frontend/__tests__/graph-history.test.ts create mode 100644 nextjs-frontend/app/(app)/routines/history/page.tsx create mode 100644 nextjs-frontend/components/routines/graph-history.tsx create mode 100644 nextjs-frontend/components/routines/history-model.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c0a36e..f2cdeedae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version preview, undoing merges with their mentions; a rollback can be rolled back too. Checkpoints let the graph be replayed from its history, and `lens history verify` (or `/graph/verify`) checks the replay matches today's graph. See docs/graph-history.md. + In the web app, Routines › History shows every change and what it did, and names or rolls back to a version; an + entity's History button shows its own. - **Query the graph in Cypher, and walk it.** The archive is now a property graph of namespaces, collections, recordings, speakers and entities. `POST /api/v1/graph/query` runs read-only Cypher (the language of Neo4j and ISO GQL) over the namespaces you can read; `/graph/related` gives a node's parents, children, ancestors, descendants or diff --git a/DEVLOG.md b/DEVLOG.md index 6a9e22d3e..4fc4d41e0 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -31,7 +31,7 @@ Todo: - [x] Rollback to a version, with a preview - [x] Checkpoints, replay and verify (`lens graph verify`, nightly checkpoint) - [ ] The explorer and Cypher as of a version (with the graph explorer thread) -- [ ] Web app: a history timeline with diffs and rollback +- [x] Web app: a history timeline with diffs and rollback (Routines › History; an entity's History button) - [ ] Graph events in each resource's activity history (with the activity history thread) Refine later: speakers (same-person links, names) in the history; a mention-level history. diff --git a/docs/graph-history.md b/docs/graph-history.md index 04d72775d..9355d3447 100644 --- a/docs/graph-history.md +++ b/docs/graph-history.md @@ -5,7 +5,10 @@ graph's version. So the graph can be seen as it was at any version, two versions read: who renamed it, who merged what into it, which routine linked it, and why. Built: the event log, versions, as-of, diffs, entity history, named versions, rollback to a version, checkpoints with -replay and verify. Planned: the explorer and Cypher as of a version, a history timeline in the web app (DEVLOG.md). +replay and verify, the History tab. Planned: the explorer and Cypher as of a version (DEVLOG.md). + +In the web app, Routines › History lists every version; open one to see what changed, name it, or roll back to it +(with a preview). An entity's History button shows only its changes. ## What is versioned diff --git a/fastapi_backend/app/domain/graph_history.py b/fastapi_backend/app/domain/graph_history.py index fb0a0bfb7..10a892729 100644 --- a/fastapi_backend/app/domain/graph_history.py +++ b/fastapi_backend/app/domain/graph_history.py @@ -390,7 +390,10 @@ def seen(t, row): "links": {"added": [], "removed": []}, "distinct": {"added": [], "removed": []}, } - names = _names(db, set(ent_space), rows_of={k: (last.get(("entity", k)) or first.get(("entity", k))) for k in ent_space}) + named = set(ent_space) | { + int(row["entity"]) for (t, _), row in {**first, **last}.items() if t == "entity_alias" and row and row.get("entity") is not None + } + names = _names(db, named, rows_of={k: (last.get(("entity", k)) or first.get(("entity", k))) for k in ent_space}) for t, k in sorted(first, key=lambda x: (TABLES.index(x[0]), str(x[1]))): before, after = first[(t, k)], last[(t, k)] if before == after or not (seen(t, before) and seen(t, after)): diff --git a/fastapi_backend/tests/domain/test_graph_history.py b/fastapi_backend/tests/domain/test_graph_history.py index 601f0f366..516f8e819 100644 --- a/fastapi_backend/tests/domain/test_graph_history.py +++ b/fastapi_backend/tests/domain/test_graph_history.py @@ -101,6 +101,7 @@ def test_diff_history_and_scopes(db, cfg, spaces): assert changed[dyno]["name"] == ["Dyno Therapeutics", "Dyno"] assert d["links"]["added"] == [{"a": min(dyno, calls_dyno), "b": max(dyno, calls_dyno), "names": d["links"]["added"][0]["names"]}] assert d["events"] == 3 + assert d["aliases"]["added"] == [{"key": "dyno therapeutics", "entity": dyno, "name": "Dyno", "space": pods}] # the old name only_pods = gh.diff(db, v0, gh.head(db), {pods}) assert only_pods["links"]["added"] == [] # one end isn't readable assert dyno in {c["id"] for c in only_pods["entities"]["changed"]} diff --git a/nextjs-frontend/__tests__/graph-history.test.ts b/nextjs-frontend/__tests__/graph-history.test.ts new file mode 100644 index 000000000..9a72a6331 --- /dev/null +++ b/nextjs-frontend/__tests__/graph-history.test.ts @@ -0,0 +1,36 @@ +import { diffLines, type Diff } from "@/components/routines/history-model"; + +const empty: Diff = { + entities: { added: [], removed: [], changed: [] }, + aliases: { added: [], removed: [] }, + links: { added: [], removed: [] }, + distinct: { added: [], removed: [] }, +}; + +describe("graph history diffs", () => { + it("lists only what changed, in words", () => { + expect(diffLines(empty)).toEqual([]); + const d: Diff = { + ...empty, + entities: { + added: [{ id: 1, name: "Lens", type: "PRODUCT" }], + removed: [], + changed: [ + { + id: 2, + name: "Dyno", + fields: { name: ["Dyno Therapeutics", "Dyno"], key: ["a", "b"], hidden: [null, true] }, + }, + ], + }, + aliases: { added: [{ key: "dyno therapeutics", entity: 2, name: "Dyno" }], removed: [] }, + links: { added: [{ a: 2, b: 9, names: ["Dyno", "Dyno Therapeutics"] }], removed: [] }, + }; + expect(diffLines(d)).toEqual([ + { label: "Added", items: ["Lens (PRODUCT)"] }, + { label: "Changed", items: ["Dyno: name Dyno Therapeutics → Dyno, hidden — → true"] }, + { label: "Other names added", items: ["“dyno therapeutics” → Dyno"] }, + { label: "Linked", items: ["Dyno ↔ Dyno Therapeutics"] }, + ]); + }); +}); diff --git a/nextjs-frontend/app/(app)/routines/history/page.tsx b/nextjs-frontend/app/(app)/routines/history/page.tsx new file mode 100644 index 000000000..cfa2d3a7a --- /dev/null +++ b/nextjs-frontend/app/(app)/routines/history/page.tsx @@ -0,0 +1,12 @@ +import type { Metadata } from "next"; + +import { GraphHistoryPage } from "@/components/routines/graph-history"; + +export const metadata: Metadata = { title: "Graph history" }; + +/** Every change to the entity graph, to see what changed, name a version or roll back to one. */ +export default async function Page({ searchParams }: { searchParams: Promise<{ entity?: string }> }) { + const { entity } = await searchParams; + const n = Number(entity); + return 0 ? n : undefined} />; +} diff --git a/nextjs-frontend/components/entities/entity-drawer.tsx b/nextjs-frontend/components/entities/entity-drawer.tsx index ebdd03bd3..e5b7a8cc3 100644 --- a/nextjs-frontend/components/entities/entity-drawer.tsx +++ b/nextjs-frontend/components/entities/entity-drawer.tsx @@ -1,7 +1,7 @@ "use client"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Eye, EyeOff, Waypoints } from "lucide-react"; +import { Eye, EyeOff, History, Waypoints } from "lucide-react"; import Link from "next/link"; import { useEffect, useState } from "react"; @@ -129,9 +129,14 @@ export function EntityDrawer({ width={440} actions={ d && ( - + <> + + + ) } > diff --git a/nextjs-frontend/components/routines/graph-history.tsx b/nextjs-frontend/components/routines/graph-history.tsx new file mode 100644 index 000000000..e89532761 --- /dev/null +++ b/nextjs-frontend/components/routines/graph-history.tsx @@ -0,0 +1,330 @@ +"use client"; + +import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Bookmark, ChevronDown, ChevronRight, History, Undo2 } from "lucide-react"; +import Link from "next/link"; +import { useState } from "react"; + +import { Graph2 as Graph } from "@/app/openapi-client"; +import { + diffLines, + OPS, + VIA, + type Diff, + type Page, + type Rollback, + type Version, +} from "@/components/routines/history-model"; +import { RoutinesHeader } from "@/components/routines/routines-page"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Dialog } from "@/components/ui/dialog"; +import { Field, Input } from "@/components/ui/field"; +import { DateTime, EmptyState, SkeletonRows } from "@/components/ui/states"; +import { useToast } from "@/components/ui/toast"; +import { data, useApiClient } from "@/lib/api/browser"; +import { needRole, useArchive } from "@/lib/hooks/session"; + +function EntityLinks({ v }: { v: Version }) { + const ids = (v.entities ?? []).slice(0, 6); + const more = (v.entities?.length ?? 0) - ids.length; + return ( + <> + {ids.map((id, i) => ( + + {i > 0 && ", "} + + {v.names?.[String(id)] ?? `#${id}`} + + + ))} + {more > 0 && ` and ${more} more`} + + ); +} + +/** A diff as short lists: what was added, removed and changed. */ +export function DiffView({ d }: { d: Diff }) { + const rows = diffLines(d); + if (!rows.length) return

Nothing you can see changed.

; + return ( +
+ {rows.map((r) => ( +
+
{r.label}
+
+ {r.items.slice(0, 20).join("; ")} + {r.items.length > 20 && ` and ${r.items.length - 20} more`} +
+
+ ))} +
+ ); +} + +function VersionDiff({ v }: { v: number }) { + const client = useApiClient(); + const diff = useQuery({ + queryKey: ["graph-diff", v - 1, v], + queryFn: () => data(Graph.graphDiff({ client, query: { from: String(v - 1), to: String(v) } })) as Promise, + staleTime: Infinity, + }); + if (diff.isLoading) return ; + if (diff.error) return

{(diff.error as Error).message}

; + return diff.data ? : null; +} + +function NameDialog({ v, open, onOpenChange }: { v: number; open: boolean; onOpenChange: (o: boolean) => void }) { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const [name, setName] = useState(""); + const save = useMutation({ + mutationFn: () => data(Graph.graphTag({ client, body: { name, version: String(v) } })), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["graph-history"] }); + toast({ tone: "green", title: `Version ${v} is “${name}”` }); + onOpenChange(false); + }, + }); + return ( + + + + + } + > + + {({ id, invalid }) => ( + setName(e.target.value)} /> + )} + + + ); +} + +function RollbackDialog({ v, open, onOpenChange }: { v: number; open: boolean; onOpenChange: (o: boolean) => void }) { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const preview = useQuery({ + queryKey: ["graph-rollback", v], + queryFn: () => data(Graph.graphRollback({ client, body: { to: String(v) } })) as Promise, + enabled: open, + staleTime: 0, + }); + const run = useMutation({ + mutationFn: () => + data(Graph.graphRollback({ client, body: { to: String(v), dry_run: false } })) as Promise, + onSuccess: (r) => { + void qc.invalidateQueries({ queryKey: ["graph-history"] }); + void qc.invalidateQueries({ queryKey: ["graph"] }); + void qc.invalidateQueries({ queryKey: ["graph-changes"] }); + toast({ + tone: "green", + title: `Back to version ${v}`, + body: r.skipped?.length + ? `${r.skipped.length} couldn’t be taken back: ${r.skipped[0].why}` + : `This is version ${r.version}; roll back to ${(r.version ?? 1) - 1} to undo it.`, + }); + onOpenChange(false); + }, + }); + const p = preview.data; + return ( + + + + + } + > + {preview.isLoading ? ( + + ) : preview.error ? ( +

{(preview.error as Error).message}

+ ) : p && !p.undo.length ? ( +

Nothing you can edit changed since version {v}.

+ ) : p ? ( +
+ + {p.kept > 0 && ( +

+ {p.kept} analysis run{p.kept === 1 ? "" : "s"} since then stay{p.kept === 1 ? "s" : ""}. +

+ )} +
+ ) : null} + {run.isError &&

{run.error.message}

} +
+ ); +} + +function VersionRow({ v, head }: { v: Version; head: number }) { + const { can } = useArchive(); + const [open, setOpen] = useState(false); + const [dialog, setDialog] = useState<"name" | "rollback" | null>(null); + const editor = can("editor"); + return ( +
  • +
    + + v{v.version} + + {OPS[v.op] ?? v.op} + + {v.tags?.map((t) => ( + + {t} + + ))} + + + {v.actor} · {VIA[v.via] ?? v.via} · + +
    + {v.why &&

    {v.why}

    } + {open && ( +
    + +
    + + {v.version < head && ( + + )} +
    +
    + )} + {dialog === "name" && !o && setDialog(null)} />} + {dialog === "rollback" && !o && setDialog(null)} />} +
  • + ); +} + +export function useGraphHistory(entity?: number) { + const client = useApiClient(); + return useInfiniteQuery({ + queryKey: ["graph-history", entity ?? null], + initialPageParam: null as number | null, + queryFn: ({ pageParam }) => + data( + Graph.graphHistoryList({ client, query: { entity: entity ?? null, before: pageParam, limit: 50 } }), + ) as Promise, + getNextPageParam: (last) => last.next, + staleTime: 10_000, + }); +} + +/** Every change to the entity graph, newest first: who, through what, why and what changed, with naming a version + * and rolling back to one. */ +export function GraphHistoryPage({ entity }: { entity?: number }) { + const history = useGraphHistory(entity); + const pages = history.data?.pages ?? []; + const head = pages[0]?.head ?? 0; + const list = pages.flatMap((p) => p.versions); + return ( +
    + +

    + Every change to entities, their other names, links and “not the same” pairs is a version. The graph is at + version {head}. + {entity != null && ( + <> + {" "} + Showing the changes to one entity ·{" "} + + Show all + + + )} +

    + {history.isLoading ? ( + + ) : history.error ? ( + } + title="Couldn’t load the history" + actions={} + > + {(history.error as Error).message} + + ) : list.length ? ( + <> +
      + {list.map((v) => ( + + ))} +
    + {history.hasNextPage && ( + + )} + + ) : ( + } title="No changes yet"> + When analysis finds entities, or someone renames, merges or links them, each change shows here. + + )} +
    + ); +} diff --git a/nextjs-frontend/components/routines/history-model.ts b/nextjs-frontend/components/routines/history-model.ts new file mode 100644 index 000000000..58574bddd --- /dev/null +++ b/nextjs-frontend/components/routines/history-model.ts @@ -0,0 +1,109 @@ +/** The entity graph's history, as the History tab shows it (docs/graph-history.md). */ + +export type Version = { + version: number; + at: string; + op: string; + actor: string; + via: string; + why?: string | null; + entities?: number[]; + names?: Record; + changes?: number; + tags?: string[]; +}; +export type Pair = { a: number; b: number; names: (string | null)[] }; +export type Alias = { key: string; entity: number; name?: string | null }; +export type Ent = { id: number; name?: string; type?: string }; +export type Diff = { + entities: { + added: Ent[]; + removed: Ent[]; + changed: { id: number; name?: string; fields: Record }[]; + }; + aliases: { added: Alias[]; removed: Alias[] }; + links: { added: Pair[]; removed: Pair[] }; + distinct: { added: Pair[]; removed: Pair[] }; +}; +export type Page = { head: number; versions: Version[]; next: number | null }; +export type Rollback = Diff & { + to: number; + undo: Pick[]; + kept: number; + done: boolean; + version?: number; + skipped?: { why: string }[]; +}; + +/** What an event did, in words. */ +export const OPS: Record = { + "entity.rename": "Renamed", + "entity.describe": "Described", + "entity.retype": "Changed the type of", + "entity.hide": "Hid", + "entity.show": "Showed", + "entity.define": "Put on the list", + "entity.undefine": "Took off the list", + "entity.aliases": "Set other names of", + "entity.learned": "Learned other names for", + "entity.delete": "Deleted", + "entity.merge": "Merged", + "entity.unmerge": "Undid a merge of", + "entity.builtin": "Made", + "entity.found": "Found", + "link.add": "Linked", + "link.remove": "Unlinked", + "distinct.add": "Marked as different", + "mention.move": "Moved a mention between", + "mention.remove": "Removed a mention from", + "merge.apply": "Merged (graph change)", + "link.apply": "Linked (graph change)", + "merge.accept": "Accepted a merge of", + "link.accept": "Accepted a link of", + "merge.dismiss": "Dismissed a merge of", + "link.dismiss": "Dismissed a link of", + "merge.undo": "Undid a merge of", + "link.undo": "Undid a link of", + analysis: "Analysis found", + "graph.rollback": "Rolled back", + "graph.drift": "Recorded unrecorded changes to", +}; +export const VIA: Record = { + web: "web app", + token: "API token", + oauth: "app", + assistant: "assistant", + mcp: "MCP", + routine: "routine", + workflow: "workflow", + analysis: "analysis", + cli: "command line", + system: "Lens", +}; + +export const show = (v: unknown) => + v == null || v === "" ? "—" : typeof v === "object" ? JSON.stringify(v) : String(v); + +/** A diff as short lines: what was added, removed and changed, each with its label. */ +export function diffLines(d: Diff): { label: string; items: string[] }[] { + return [ + { label: "Added", items: d.entities.added.map((e) => `${e.name} (${e.type})`) }, + { label: "Removed", items: d.entities.removed.map((e) => `${e.name} (${e.type})`) }, + { + label: "Changed", + items: d.entities.changed.map( + (c) => + `${c.name}: ${Object.entries(c.fields) + .filter(([f]) => f !== "key") + .map(([f, [a, b]]) => `${f} ${show(a)} → ${show(b)}`) + .join(", ")}`, + ), + }, + { label: "Other names added", items: d.aliases.added.map((a) => `“${a.key}” → ${a.name ?? a.entity}`) }, + { label: "Other names removed", items: d.aliases.removed.map((a) => `“${a.key}” from ${a.name ?? a.entity}`) }, + { label: "Linked", items: d.links.added.map((p) => p.names.join(" ↔ ")) }, + { label: "Unlinked", items: d.links.removed.map((p) => p.names.join(" ↔ ")) }, + { label: "Marked different", items: d.distinct.added.map((p) => p.names.join(" ≠ ")) }, + { label: "No longer marked different", items: d.distinct.removed.map((p) => p.names.join(" ≠ ")) }, + ].filter((r) => r.items.length); +} diff --git a/nextjs-frontend/components/routines/routines-page.tsx b/nextjs-frontend/components/routines/routines-page.tsx index 14ad8d672..1ebb0fc94 100644 --- a/nextjs-frontend/components/routines/routines-page.tsx +++ b/nextjs-frontend/components/routines/routines-page.tsx @@ -22,8 +22,8 @@ import { useToast } from "@/components/ui/toast"; import { data, useApiClient } from "@/lib/api/browser"; import { useArchive } from "@/lib/hooks/session"; -/** "Routines" with the Routines / Proposed changes tabs; people who aren't admins see only the changes. */ -export function RoutinesHeader({ tab }: { tab: "routines" | "changes" }) { +/** "Routines" with the Routines / Proposed changes / History tabs; people who aren't admins see only the graph tabs. */ +export function RoutinesHeader({ tab }: { tab: "routines" | "changes" | "history" }) { const { admin } = useArchive(); const routines = useRoutines(); const proposed = useGraphChanges("proposed"); @@ -49,6 +49,7 @@ export function RoutinesHeader({ tab }: { tab: "routines" | "changes" }) { ? [{ value: "routines", label: "Routines", count: routines.data?.routines.length, href: "/routines" }] : []), { value: "changes", label: "Proposed changes", count: proposed.data?.length, href: "/routines/changes" }, + { value: "history", label: "History", href: "/routines/history" }, ]} />