From ee327cf7e78c0df8a748dcf27404c8216b8293f0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Fri, 16 Oct 2026 23:26:19 +0000 Subject: [PATCH 1/3] fix(assistant): let the assistant's tools be imported first Importing app.domain.ai_tools on its own failed: it imported automations, which loads the actions, and the agent's action subclasses Toolbox before ai_tools had defined it. Running one test file (tests/api/test_chats.py) hit it. ai_tools now imports automations last. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/app/domain/ai_tools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 843562243..9ca9f43fd 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -12,7 +12,6 @@ from . import ( auth, - automations as automationsmod, batches, cypher, entities, @@ -1861,3 +1860,7 @@ def author_extension(db, tool, args, user, admin, readable, editable): n = extensions.save_version(db, me, same["id"], manifest, args.get("notes"), origin="chat") return {"extension": same["id"], "version": n} return {"extension": extensions.create(db, me, manifest, origin="chat"), "version": 1} + + +# last, because automations imports the actions, and the agent's action (actions_ai.Box) builds on Toolbox above +from . import automations as automationsmod # noqa: E402 From defb487a21bc621dd7e835bfab4538441706b2a4 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:01:11 +0000 Subject: [PATCH 2/3] feat(chat): each answer records what produced it Every answer keeps `meta`: the model and the host name of its server, the tools offered and asked for, the scope it answered in (after auto-scope), its limits and what it used, how it ended (answered, direct, limit, fallback, stopped, error), tool results that read like instructions to an AI, how long it took, Lens's version, and tokens and cost from the ledger rows that name it. Questions keep how they were sent (web, api, voice, room, automation) and their options. The answer's id is taken once the question is saved and the answer runs in activity.scope(chat:, chat_message:), in the web chat, chat rooms and the automations' agent; the streamed answer holds the scope across its steps (activity.scoped). App completions keep no messages, so their record comes back in lens.record. The stream sends `meta` before `done`; GET /chats/{cid}/messages/{mid} returns one message with its record and ledger rows. Each tool step keeps its time and whether it failed. The web app shows it all from an i in the answer's header (a sheet on a phone); old answers say they weren't recorded. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- CHANGELOG.md | 5 + docs/api.md | 8 +- docs/assistant.md | 36 +- docs/database.md | 5 +- fastapi_backend/app/api/v1/routes/activity.py | 4 + fastapi_backend/app/api/v1/routes/chats.py | 65 +- .../app/api/v1/routes/completions.py | 8 +- fastapi_backend/app/domain/actions_ai.py | 32 +- fastapi_backend/app/domain/activity.py | 21 + fastapi_backend/app/domain/ai_tools.py | 4 + fastapi_backend/app/domain/bridge.py | 103 +-- fastapi_backend/app/domain/chat.py | 84 ++- fastapi_backend/app/domain/completions.py | 21 +- fastapi_backend/app/schemas/chats.py | 70 ++ fastapi_backend/tests/api/test_actions_ai.py | 6 +- fastapi_backend/tests/api/test_bridge.py | 6 +- .../tests/api/test_chat_records.py | 178 +++++ fastapi_backend/tests/api/test_completions.py | 7 + .../tests/api/test_prompt_injection.py | 1 + fastapi_backend/tests/fake_llm.py | 5 +- .../__tests__/chat-details.test.tsx | 168 +++++ nextjs-frontend/app/openapi-client/index.ts | 12 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 22 +- .../app/openapi-client/types.gen.ts | 325 +++++++++ nextjs-frontend/components/chat/answer.tsx | 8 +- nextjs-frontend/components/chat/chat-app.tsx | 8 +- nextjs-frontend/components/chat/details.tsx | 201 ++++++ nextjs-frontend/components/chat/steps.tsx | 9 +- nextjs-frontend/components/chat/stream.ts | 30 +- nextjs-frontend/components/costs/costs.tsx | 2 +- nextjs-frontend/openapi.json | 628 +++++++++++++++++- threat-model.md | 1 + threat-model.yaml | 2 +- 33 files changed, 1974 insertions(+), 111 deletions(-) create mode 100644 fastapi_backend/tests/api/test_chat_records.py create mode 100644 nextjs-frontend/__tests__/chat-details.test.tsx create mode 100644 nextjs-frontend/components/chat/details.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bbfb731..184fa7b55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now; Settings → Automations switches them all off and sets how far they may go ([Running automations](docs/automations.md#running-automations)). +- **Each answer says what produced it.** The *i* in an answer's header shows the model and the host it ran on, the + tools it was offered and used, the scope, its limits and what it used, tokens, cost (an estimate from your prices), + how long it took, how it ended and which tool results read like instructions to an AI, with the calls made for it. + Agents read the same record over the API (`GET /api/v1/chats//messages/`), and each tool step shows how long + it took. Answers from before have no record ([What each answer records](docs/assistant.md#what-each-answer-records)). - **Automations for agents.** Agents list, save, try and run automations through the MCP tools `automations`, `automation_actions`, `save_automation`, `test_automation` and `run_automation`, with problems in plain words. The assistant saves them switched off and asks before switching one on or running it diff --git a/docs/api.md b/docs/api.md index 6e8f3bec8..beed20cb5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -477,7 +477,8 @@ read there: What it cites comes back three ways, so any app can show it: `[n]` in the text with a **Sources** list of links after it (left out when the request has a `response_format`), `citations` (the link for `[n]` is the n-th), and `lens.sources` (`n`, `recording_id`, `title`, `time`, `speaker`, `url`, `cited`). `lens.steps` lists the tools the -model used. With `"stream": true` the answer comes as OpenAI's `chat.completion.chunk` events, ending with +model used, and `lens.record` what produced the answer, as a chat answer's record +([What each answer records](assistant.md#what-each-answer-records)) without tokens and cost. With `"stream": true` the answer comes as OpenAI's `chat.completion.chunk` events, ending with `data: [DONE]`; the last chunk before it carries `finish_reason: "stop"`, `citations` and `lens`. While the tools run, the stream sends SSE comments, so nothing in between closes it. @@ -1448,6 +1449,7 @@ GET /api/v1/chats/{cid} PATCH /api/v1/chats/{cid} DELETE /api/v1/chats/{cid} POST /api/v1/chats/{cid}/messages +GET /api/v1/chats/{cid}/messages/{mid} POST /api/v1/chats/{cid}/stop POST /api/v1/chats/{cid}/messages/{mid}/check GET /api/v1/approvals @@ -1469,7 +1471,9 @@ whether it uses tools (`tools`, `max_steps`), whether answers can be checked (`c (the configured one first; `llm.chat_models`, else the server's list); never the model server's address or key. A conversation's `model` (`POST /chats`, `PATCH /chats/{cid}`; null for the configured one) answers in it, and a question can name another (`POST /chats/{cid}/messages {content, model}`, e.g. to retry); a model that isn't offered is -a 400. Each answer records the `model` that wrote it. `POST /chats/{cid}/stop` stops the answer being written in your conversation after the piece or tool +a 400. Each answer records the `model` that wrote it, and its `meta` what produced it; `GET +/chats/{cid}/messages/{mid}` gives one message with its record and its rows in the activity ledger +([What each answer records](assistant.md#what-each-answer-records)). `POST /chats/{cid}/stop` stops the answer being written in your conversation after the piece or tool step it's on: the stream sends `stopped`, then `done` with the saved message, whose `stopped` is true and whose `content` is what came before (`(stopped)` when nothing had). `{stopping: false}` when nothing was being written. `POST /chats/{cid}/messages {content, edit}` edits one of your earlier questions (`edit` is its message id): it and diff --git a/docs/assistant.md b/docs/assistant.md index bacf0168f..02dc202ba 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -13,11 +13,12 @@ then. numbered. - **Answers:** the model answers only from those excerpts and cites them as [n]. Each citation carries the asset and timestamp. -- **Streaming:** answers arrive over server-sent events: `passages`, then `token`s, then `done`. +- **Streaming:** answers arrive over server-sent events: `passages`, then `token`s, then `meta` (the answer's + record, [below](#what-each-answer-records)) and `done`. - **Stopping:** Stop asks the server to end the answer after the piece or tool step it's on (whichever server process is writing it); what came before is saved, marked stopped. A model call already under way finishes first. - **Reopening:** a conversation's answers keep the tools the assistant used (and with what), any notice (the model - couldn't use tools), the error when there was no answer, and their latest source check. + couldn't use tools), the error when there was no answer, their latest source check and what produced them. - **Editing a question:** Edit on a question you asked asks it again as edited; the answer and everything after it are replaced (`POST /api/v1/chats//messages` with `edit`). - **Long conversations:** the model reads the latest six messages word for word. Once eight more have built up @@ -51,6 +52,37 @@ then. - **No model configured:** chat returns the best-matching passages instead. Anyone signed in can see whether a model is set up, and which (`GET /chats/capabilities`), so the app says so before the first question. +### What each answer records + +Every answer keeps a record of what produced it (`meta` on the message), so a person can trust and debug it and an +agent can read it. The **i** in an answer's header (Details) shows it, with the calls made for it and what the +conversation has cost this month; on a phone it opens as a sheet. + +| Field | What it says | +| --- | --- | +| `model`, `provider` | the model that wrote it, and the host name of the model server it ran on (never the server's address, path or key) | +| `tools` | `offered`: every tool the model was offered; `picked`: the tools the question asked for (null: all it may use) | +| `scope` | the scope it answered in, after a conversation over everything was narrowed to a namespace | +| `limits`, `used` | the rounds of tool calls it had (and the tool calls, when they're limited), and how many it used | +| `outcome` | `answered` (after looking things up), `direct` (without looking anything up), `limit` (its rounds ran out, so it answered from what it found), `fallback` (from a search: the model can't use tools, or none is set up), `stopped` or `error` | +| `tokens`, `cost_usd`, `cost_estimated` | the tokens of the model calls made for it, and what they cost from the prices in Settings ([Costs](activity.md#costs)); `cost_estimated` when some had a price but no token counts, so it cost at least this | +| `ms` | how long it took, from the question | +| `flagged` | tools whose results read like instructions to an AI ([below](#third-party-content-and-prompt-injection)) | +| `version` | the version of Lens that wrote it | + +Each tool step also keeps how long it took (`ms`) and whether it failed (`failed`). A question keeps how it was sent +(`via`: `web`, `api`, `voice`, `room` or `automation`) and the options it was asked with (`options`, such as another +`model`). + +An answer is written inside the activity ledger's scope `chat_message:` (with `chat:`), so every model and +tool call made for it names it ([Activity](activity.md)), and its record's tokens and cost are theirs: one ledger +total per answer, and no model call. `GET /api/v1/chats/` returns the record on each message, and +`GET /api/v1/chats//messages/` returns one message with its record and those ledger rows (`ledger`). +Answers in chat rooms ([Chat rooms](chat-rooms.md)) and the automations' agent record theirs the same way. An app's +chat completions keep no messages, so their record comes back in the reply's `lens.record`, without tokens and cost +([Chat completions](api.md#chat-completions-for-any-app)). Answers from before Lens kept records have none; Details +shows what they have and says the rest wasn't recorded. + ### Picking the namespace for a conversation over everything When a conversation has no scope (the assistant home starts it over everything), its first question goes to the diff --git a/docs/database.md b/docs/database.md index 7473ad304..d6a8d1c59 100644 --- a/docs/database.md +++ b/docs/database.md @@ -87,7 +87,10 @@ optional, and older resources simply don't have them. ## Upgrades Installs keep their data across updates (running `install.sh` again, a new image, `git pull`). New tables, fields and -indexes arrive through `SCHEMA` on the next start. Anything else, such as rewriting stored values, moving data between +indexes arrive through `SCHEMA` on the next start. A field that is only added needs no step when its readers treat it +as missing on older rows: a chat message's record (`meta`, +[What each answer records](assistant.md#what-each-answer-records)) is one, and messages from before it simply have +none. Anything else, such as rewriting stored values, moving data between tables, or changing or removing an index that already exists, is a named step in `app/domain/migrations.py`: * Every process that opens the database (the API, workers, `lens watch`, `lens` commands) runs the steps the database diff --git a/fastapi_backend/app/api/v1/routes/activity.py b/fastapi_backend/app/api/v1/routes/activity.py index ede0375f6..9c6f68399 100644 --- a/fastapi_backend/app/api/v1/routes/activity.py +++ b/fastapi_backend/app/api/v1/routes/activity.py @@ -48,6 +48,10 @@ def _may_see(db: Db, user: CurrentUser, acl: Acl, resource: str) -> None: return if table == "chat" and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", int(key)), a=user.id): return + if table == "chat_message": # a message in one of your conversations + m = db.one("SELECT chat FROM $r", r=activity.R("chat_message", int(key))) + if m and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", m["chat"]), a=user.id): + return raise HTTPException(403, "only admins see this resource's activity") diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py index aadedbd19..d0791f482 100644 --- a/fastapi_backend/app/api/v1/routes/chats.py +++ b/fastapi_backend/app/api/v1/routes/chats.py @@ -1,7 +1,7 @@ """Conversations with the archive, the assistant's approvals, and checking an answer against its sources. A question streams back server-sent events: ``scoped`` (the namespace a conversation over everything was narrowed -to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error`` and ``done``. +to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error``, ``meta`` (the answer's record) and ``done``. The assistant only ever sees, and cites, what the asker can read. """ @@ -19,7 +19,7 @@ from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Principal, Writer from app.api.media import sign_urls from app.api.v1.routes.collections import own_collection -from app.domain import ai_tools, approvals, auth, auto_scope, browser, chat, llm, store, uploads +from app.domain import activity, ai_tools, approvals, auth, auto_scope, browser, chat, llm, store, uploads from app.domain.store import DB from app.schemas.chats import ( AnswerCheck, @@ -34,6 +34,7 @@ ChatSummary, ChatUpdate, MessageCreate, + MessageDetail, StopResult, ) from app.schemas.common import Created, Ok @@ -129,6 +130,17 @@ def get_chat(cid: int, user: CurrentUser, acl: Acl, db: Db) -> Chat: return Chat.model_validate(sign_urls({**c, "messages": msgs}, full=True)) +@router.get("/chats/{cid}/messages/{mid}") +def get_message(cid: int, mid: int, user: CurrentUser, acl: Acl, db: Db) -> MessageDetail: + """One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and + used, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`: + the calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}.""" + m = next((x for x in get_chat(cid, user, acl, db).messages if x.id == mid), None) + if m is None: + raise HTTPException(404, "not found") + return MessageDetail.model_validate({**m.model_dump(by_alias=True), "ledger": activity.history(db, f"chat_message:{mid}", 200)}) + + @router.patch("/chats/{cid}") def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Ok: """Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one).""" @@ -177,7 +189,8 @@ def _ev(name: str, data: Any) -> str: async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> StreamingResponse: """Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST - /chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id). + /chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced + it), done (the saved message id). With `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and its new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked @@ -211,6 +224,8 @@ def run() -> Iterator[str]: model = await run_in_threadpool(_model, cfg, body.model) if body.model else c.get("model") if model and not body.model and model not in await run_in_threadpool(chat.model_choices, cfg): model = None + record = chat.Record(cfg, model) # what produced the answer, kept with it (from here: it times the answer) + mid = 0 # the answer's id, taken once the question is saved title = c["title"] page = body.context.model_dump() if body.context else None kept = chat.shared_context(page) # what the question keeps of the page @@ -229,14 +244,21 @@ def run() -> Iterator[str]: summary: dict[str, Any] | None = None # what compaction keeps of the earlier messages (chat.compact) def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: - nonlocal summary + nonlocal summary, mid try: # a conversation that fills the model's context is folded into its summary before it's answered chat.compact(db, cfg, cid, full_only=True) except Exception: # noqa: BLE001 - answered with what fits; tried again after the answer log.warning("chat %s: couldn't update the summary", cid, exc_info=True) past = chat.history(db, cid) summary = chat.memory(db, cfg, cid) - chat.add(db, cid, "user", q, attachments=files, context=kept) + via = "voice" if body.via == "voice" else "web" if user.via == "access" else "api" + chat.add(db, cid, "user", q, attachments=files, context=kept, meta=chat.question_record(via, model=body.model)) + mid = db.next_id("chat_message") + # what's worked out for the answer from here (the excerpts, the namespace it's about) counts for it in the ledger + with activity.scope(db, f"chat:{cid}", f"chat_message:{mid}", cfg=cfg): + return draw_on(past) + + def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: if not past and title == "New conversation": db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=q[:80]) # A conversation over everything gets the namespace its first question is about, when that's clear. @@ -251,6 +273,7 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: return past, chat.retrieve(db, q, readable, c["scope"], cfg=cfg), placed past, passages, placed = await run_in_threadpool(prepare) + record.scope = c.get("scope") or {} # after a conversation over everything was narrowed to what it's about picked = placed[1] if placed and placed[0] == "scoped" else None suggested = placed[1] if placed and placed[0] == "suggest" else None @@ -265,13 +288,17 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: ) notice: str | None = None - def save(text: str, cited: list[dict[str, Any]], **kw: Any) -> int: + def save(text: str, cited: list[dict[str, Any]], outcome: str = "answered", **kw: Any) -> Iterator[str]: + """The answer saved with its record: `meta`, then `done` (its id).""" wrote = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None - return int(chat.add(db, cid, "assistant", text, cited, steps=steps, notice=notice, model=wrote, **kw)) + meta = record.meta(db, outcome, f"chat_message:{mid}") + chat.add(db, cid, "assistant", text, cited, steps=steps, notice=notice, model=wrote, mid=mid, meta=meta, **kw) + yield _ev("meta", meta) + yield _ev("done", {"message": mid}) def stopped(text: str, cited: list[dict[str, Any]]) -> Iterator[str]: yield _ev("stopped", {"message": "Stopped"}) - yield _ev("done", {"message": save(text or "(stopped)", cited, stopped=True)}) + yield from save(text or "(stopped)", cited, "stopped", stopped=True) def answer(on: chat.Answering) -> Iterator[str]: nonlocal notice @@ -294,10 +321,10 @@ def answer(on: chat.Answering) -> Iterator[str]: browser=body.browser, ) try: - answer = "" + answer, how = "", "answered" # setting the server up, or working in the person's browser, takes more steps than looking things up steps_max = max(cfg["ai"].get("max_steps") or 6, 12 if setup else 0, browser.steps(cfg) if box.browser else 0) - for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary): + for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record): if kind == "step": steps.append(data) yield _ev("step", data) @@ -311,7 +338,7 @@ def answer(on: chat.Answering) -> Iterator[str]: elif kind == "direct" and passages and not setup and not (page and page.get("text")) and not box.cited(data): break else: - answer = data + answer, how = data, "direct" if kind == "direct" else "answered" else: box.after_answer(answer) for appr in box.approvals: @@ -319,23 +346,25 @@ def answer(on: chat.Answering) -> Iterator[str]: cited = box.cited(answer) yield _ev("passages", cited) yield _ev("token", {"text": answer}) - yield _ev("done", {"message": save(answer or "(no answer)", cited)}) + yield from save(answer or "(no answer)", cited, how) return except llm.ToolsUnsupported: notice = "This model can't use tools, so the answer comes from a search instead." yield _ev("notice", {"message": notice}) except llm.LLMError as e: yield _ev("error", {"message": str(e)}) - yield _ev("done", {"message": save("(no answer)", [], error=str(e))}) + yield from save("(no answer)", [], "error", error=str(e)) return yield _ev("passages", passages) text, error = "", None try: if llm.configured(cfg): - for piece in llm.stream_chat(cfg, chat.messages_for(asked, passages, past, summary, cfg), model=model): + pieces = llm.stream_chat(cfg, chat.messages_for(asked, passages, past, summary, cfg), model=model) + for piece in pieces: text += piece yield _ev("token", {"text": piece}) if on.stop_requested(): + pieces.close() # the call ends here, so its ledger row is in the answer's record yield from stopped(text, chat.cited(text, passages)) return else: @@ -344,7 +373,8 @@ def answer(on: chat.Answering) -> Iterator[str]: except llm.LLMError as e: error = str(e) yield _ev("error", {"message": error}) - yield _ev("done", {"message": save(text or "(no answer)", chat.cited(text, passages), error=error)}) + how = "error" if error else "fallback" if notice or not llm.configured(cfg) else "answered" + yield from save(text or "(no answer)", chat.cited(text, passages), how, error=error) def gen() -> Iterator[str]: on = chat.Answering(db, cid) @@ -355,7 +385,8 @@ def gen() -> Iterator[str]: yield _ev("step", steps[0]) if suggested: yield _ev("suggested", {"namespaces": suggested}) - for ev in answer(on): + # every call made for the answer counts for it in the ledger (the scope holds across the stream's steps) + for ev in activity.scoped(answer(on), db, f"chat:{cid}", f"chat_message:{mid}", cfg=cfg): saved = saved or ev.startswith("event: done") yield ev except Exception: # an answer that breaks off still ends with an error and is saved, so the question isn't left unanswered @@ -363,7 +394,7 @@ def gen() -> Iterator[str]: message = "Something went wrong while answering. Try again." yield _ev("error", {"message": message}) if not saved: - yield _ev("done", {"message": save("(no answer)", [], error=message)}) + yield from save("(no answer)", [], "error", error=message) finally: on.end() try: # once enough has built up, older messages are folded into the summary (after the answer, so it waits on nothing) diff --git a/fastapi_backend/app/api/v1/routes/completions.py b/fastapi_backend/app/api/v1/routes/completions.py index ef9a2b49c..0b7137daf 100644 --- a/fastapi_backend/app/api/v1/routes/completions.py +++ b/fastapi_backend/app/api/v1/routes/completions.py @@ -123,7 +123,13 @@ def create_chat_completion(name: str, body: ChatCompletionRequest, request: Requ def extras(done: dict[str, Any], steps: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]: found = completions.sources(web, done["text"], done["refs"]) - lens = {"namespace": name, "sources": found, "steps": steps, **({"notice": done["notice"]} if done.get("notice") else {})} + lens = { + "namespace": name, + "sources": found, + "steps": steps, + "record": done.get("record"), + **({"notice": done["notice"]} if done.get("notice") else {}), + } return found, {"citations": [s["url"] for s in found], "lens": lens} if not body.stream: diff --git a/fastapi_backend/app/domain/actions_ai.py b/fastapi_backend/app/domain/actions_ai.py index 05b9c0691..dc37e1cb5 100644 --- a/fastapi_backend/app/domain/actions_ai.py +++ b/fastapi_backend/app/domain/actions_ai.py @@ -257,7 +257,10 @@ def _ask_agent(db, cfg, intent, user, dry): if about: url = (about.get("object") or {}).get("url") if isinstance(about.get("object"), dict) else None page = {"url": str(url or f"/timeline?activity={about.get('id') or ''}"), "title": _short(happened, 300)} - chat.add(db, cid, "user", instructions, context=page, origin=box.origin) + chat.add(db, cid, "user", instructions, context=page, origin=box.origin, meta=chat.question_record("automation", tools=tools or None)) + # the answer's id up front: every call made for it names it in the ledger, and its record sums them + mid = db.next_id("chat_message") + record, spent = chat.Record(cfg, scope=box.scope, picked=tools or None), f"chat_message:{mid}" # reading the activity counts as reading outside text for this conversation from now on (ai_tools.read_before) steps = ( [{"tool": "automation_event", "args": {"id": about.get("id")}, "summary": f"Read what happened: {_short(happened, 200)}"}] @@ -265,21 +268,28 @@ def _ask_agent(db, cfg, intent, user, dry): else [] ) question = instructions + (_context(about, cfg) if about else "") - text = None + text, how = None, "answered" + + def said(words, cited, outcome, **kw): + """The answer, saved with its record.""" + chat.add(db, cid, "assistant", words or "(no answer)", cited, steps=steps, mid=mid, meta=record.meta(db, outcome, spent), **kw) + try: - for kind, data in chat.tool_answer(cfg, box, question, (), steps_max): - if kind == "step": - steps.append(data) - else: - text = data + with activity.scope(db, f"chat:{cid}", spent, cfg=cfg): + for kind, data in chat.tool_answer(cfg, box, question, (), steps_max, record=record): + if kind == "step": + steps.append(data) + else: + text, how = data, "direct" if kind == "direct" else "answered" except llm.ToolsUnsupported: - chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error="this model can't use tools") + said("", [], "error", error="this model can't use tools") raise ValueError("this model can't use tools, so the agent can't act: pick one that can in Settings › Model") from None except (llm.LLMError, activity.OverBudget) as e: - chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error=str(e)) + said("", [], "error", error=str(e)) raise ValueError(f"the agent couldn't answer: {e}") from None - box.after_answer(text or "") - chat.add(db, cid, "assistant", text or "(no answer)", box.cited(text), steps=steps, model=cfg["llm"].get("model")) + with activity.scope(db, f"chat:{cid}", spent, cfg=cfg): + box.after_answer(text or "") + said(text, box.cited(text), how, model=cfg["llm"].get("model")) return { "answer": text or "", "chat": cid, diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index c09c9a053..89f91c460 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -47,6 +47,7 @@ "workflows": "workflow", "custom-nodes": "custom_node", "chats": "chat", + "messages": "chat_message", # a conversation's messages/{mid} "jobs": "job", "namespaces": "space", "spaces": "space", @@ -113,6 +114,26 @@ def scope(db, *refs, cfg=None): _scope.reset(token) +def scoped(items, db, *refs, cfg=None): + """`items` (a generator) with each of its steps run inside scope(db, *refs). A `with scope(...)` inside a generator + doesn't hold across its yields when a streamed response iterates it from the threadpool: each step runs in a fresh + copy of the request's context.""" + it = iter(items) + try: + while True: + with scope(db, *refs, cfg=cfg): + try: + item = next(it) + except StopIteration: + return + yield item + finally: + close = getattr(it, "close", None) + if close: + with scope(db, *refs, cfg=cfg): + close() + + def current(): """The resources calls made here count for, in order, without repeats.""" s = _scope.get() or {} diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 9ca9f43fd..63151f162 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -532,6 +532,7 @@ def __init__( self.readable = {s for s in self.readable if names.get(s) in self.scope["namespaces"]} self.allowed = recsets.within(db, self.readable, self.scope.get("recordings"), self.scope.get("collections")) self.refs, self.reads, self.approvals = [], 0, [] + self.flagged, self.failed = [], False # tools whose results read like instructions to an AI; the last call failed # the extensions this person switched on or was given (extensions.py); hooks don't run inside hooks roles = {s: "editor" if s in self.editable else "viewer" for s in self.readable} self.me = extensions.who(self.account, user.get("email"), admin, roles) @@ -688,6 +689,7 @@ def call(self, name, args, model=False): """A tool's result (JSON) and the step's summary. `model`: the result as the model reads it, third-party text marked as data (untrusted), with what hooks add after it, unmarked.""" offered = {s["function"]["name"] for s in self.specs()} + self.failed = True # until it runs: a tool that's unknown or blocked failed try: if name not in offered: raise AttributeError(name) @@ -703,6 +705,7 @@ def call(self, name, args, model=False): extra += self._hook(h, {"said": self.said, "tool": name}) asked = len(self.approvals) result, summary = self._run(name, fn, args) + self.failed = result.startswith('{"error"') self.links |= untrusted.links(result) if self.untrusted(name) and len(self.approvals) == asked: # it read something, rather than asking for a yes self.read_outside(f"tool {name}") @@ -714,6 +717,7 @@ def call(self, name, args, model=False): why = untrusted.flags(result, self.cfg, model=True) if why: # the person sees it on the step, the model on the marker, admins in the audit log summary = f"{summary} (reads like instructions to an AI: {'; '.join(why)})" + self.flagged.append(name) auth.audit(self.db, self.user, "assistant.injection_flagged", f"chat:{self.chat}", {"tool": name, "why": why}) result = untrusted.wrap(result, f"tool {name}", self.cfg, why=why) return result + (f"\n\n{extra.strip()}" if extra else ""), summary diff --git a/fastapi_backend/app/domain/bridge.py b/fastapi_backend/app/domain/bridge.py index 491b2f4f7..cf995fba3 100644 --- a/fastapi_backend/app/domain/bridge.py +++ b/fastapi_backend/app/domain/bridge.py @@ -38,7 +38,7 @@ import urllib.error import urllib.request -from . import ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted +from . import activity, ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted from .store import R log = logging.getLogger(__name__) @@ -330,53 +330,64 @@ def _answer(db, cfg, base, account, cid, q, files, act_on, outside): past = chat.history(db, cid) summary = chat.memory(db, cfg, cid) scope = (db.one("SELECT scope FROM $r", r=R("chat", cid)) or {}).get("scope") or None - chat.add(db, cid, "user", q, attachments=list(files) or None) - asked = q + chat.attached_note(list(files)) - passages = chat.retrieve(db, q, readable, scope, cfg=cfg) - # the web addresses written by someone other than the model: the person, now and before, and the archive - keep = untrusted.links("\n".join([q] + [m["content"] or "" for m in past if m["role"] == "user"] + [p["text"] for p in passages])) - steps, approvals, notice = [], [], None - wrote = cfg["llm"].get("model") if llm.configured(cfg) else None - if llm.configured(cfg) and cfg["ai"].get("tools"): - box = ai_tools.Toolbox( - db, - cfg, - {"id": account["id"], "email": account["email"]}, - readable, - editable, - scope, - cid, - base, - account["admin"], - said=q, - outside=list(outside) + ai_tools.read_before(past, summary), - act_on=act_on, - ) + chat.add(db, cid, "user", q, attachments=list(files) or None, meta=chat.question_record("room")) + # the answer's id up front: every call made for it names it in the ledger, and its record sums them + mid = db.next_id("chat_message") + record, spent = chat.Record(cfg, scope=scope), f"chat_message:{mid}" + with activity.scope(db, f"chat:{cid}", spent, cfg=cfg): + asked = q + chat.attached_note(list(files)) + passages = chat.retrieve(db, q, readable, scope, cfg=cfg) + # the web addresses written by someone other than the model: the person, now and before, and the archive + keep = untrusted.links("\n".join([q] + [m["content"] or "" for m in past if m["role"] == "user"] + [p["text"] for p in passages])) + steps, approvals, notice = [], [], None + wrote = cfg["llm"].get("model") if llm.configured(cfg) else None + + def said(words, cited, outcome, **kw): + """The answer, saved with its record.""" + meta = record.meta(db, outcome, spent) + chat.add(db, cid, "assistant", words or "(no answer)", cited, steps=steps, model=wrote, mid=mid, meta=meta, **kw) + + if llm.configured(cfg) and cfg["ai"].get("tools"): + box = ai_tools.Toolbox( + db, + cfg, + {"id": account["id"], "email": account["email"]}, + readable, + editable, + scope, + cid, + base, + account["admin"], + said=q, + outside=list(outside) + ai_tools.read_before(past, summary), + act_on=act_on, + ) + try: + text, how = None, "answered" + for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary, record=record): + if kind == "step": + steps.append(data) + elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers + break + else: + text, how = data, "direct" if kind == "direct" else "answered" + if text is not None: + box.after_answer(text) + said(text, box.cited(text), how) + return text, box.approvals, keep | box.links + except llm.ToolsUnsupported: + notice = "This model can't use tools, so the answer comes from a search instead." + except llm.LLMError as e: + said("", [], "error", error=str(e)) + return "I couldn't answer that: " + str(e), box.approvals, keep | box.links + error = None try: - text = None - for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary): - if kind == "step": - steps.append(data) - elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers - break - else: - text = data - if text is not None: - box.after_answer(text) - chat.add(db, cid, "assistant", text or "(no answer)", box.cited(text), steps=steps, model=wrote) - return text, box.approvals, keep | box.links - except llm.ToolsUnsupported: - notice = "This model can't use tools, so the answer comes from a search instead." + text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary, cfg)) if llm.configured(cfg) else chat.fallback(passages) except llm.LLMError as e: - chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error=str(e), model=wrote) - return "I couldn't answer that: " + str(e), box.approvals, keep | box.links - error = None - try: - text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary, cfg)) if llm.configured(cfg) else chat.fallback(passages) - except llm.LLMError as e: - text, error = "I couldn't answer that: " + str(e), str(e) - chat.add(db, cid, "assistant", text or "(no answer)", chat.cited(text, passages), steps=steps, notice=notice, error=error, model=wrote) - return text, approvals, keep + text, error = "I couldn't answer that: " + str(e), str(e) + how = "error" if error else "fallback" if notice or not llm.configured(cfg) else "answered" + said(text, chat.cited(text, passages), how, notice=notice, error=error) + return text, approvals, keep # ---------- approving in the room ---------- diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py index a3b3180e4..035bde9a2 100644 --- a/fastapi_backend/app/domain/chat.py +++ b/fastapi_backend/app/domain/chat.py @@ -12,8 +12,9 @@ import re import time from collections import Counter, defaultdict +from urllib.parse import urlsplit -from . import llm, recsets, render, semantic, store, textindex, untrusted +from . import __version__, activity, llm, recsets, render, semantic, store, textindex, untrusted R = store.R STOP = set( @@ -345,7 +346,7 @@ def get(db, cid, account): def history(db, cid): return db.rows( "SELECT record::id(id) AS id, role, content, passages, created_at, stopped ?? false AS stopped, steps ?? [] AS steps, " - "attachments ?? [] AS attachments, notice, error, check, model, context FROM chat_message WHERE chat = $c ORDER BY id", + "attachments ?? [] AS attachments, notice, error, check, model, context, meta FROM chat_message WHERE chat = $c ORDER BY id", c=cid, ) @@ -364,11 +365,15 @@ def add( attachments=None, context=None, origin=None, + mid=None, + meta=None, ): """Save a message; an answer keeps the tool steps it took, any notice (e.g. the model can't use tools) and error; a question asked from a page keeps what it shared of it (shared_context). A question an automation asked keeps its - `origin` ({automation, run, node}), so it isn't taken for a person's message (`chat.message`).""" - mid = db.next_id("chat_message") + `origin` ({automation, run, node}), so it isn't taken for a person's message (`chat.message`). `meta` is its record + (Record; a question's: how it was sent), `mid` its id when it was taken up front (an answer's, so the ledger rows + of the calls made for it name it).""" + mid = mid or db.next_id("chat_message") db.q( "CREATE $r CONTENT $d", r=R("chat_message", mid), @@ -387,6 +392,7 @@ def add( "model": model, "context": context, "origin": origin or None, + "meta": meta or None, } ), ) @@ -585,6 +591,61 @@ def request_stop(db, cid): return bool(db.rows("UPDATE $r SET stop_requested = true WHERE answering = true RETURN id", r=R("chat", cid))) +def provider(cfg): + """The model server's host name, never its address's path or a key: where an answer's model ran.""" + return (urlsplit(cfg["llm"].get("base_url") or "").hostname or None) if llm.configured(cfg) else None + + +class Record: + """What produced an answer, kept on it as `meta` (docs/assistant.md, "What each answer records"): the model and the + host it ran on, the tools it was offered and the ones asked for, the scope it answered in, its limits and what it + used, how it ended, the tools whose results read like instructions to an AI, how long it took, and its tokens and + cost, from the ledger rows that name it (the answer runs in activity.scope(db, "chat_message:")). Made as the + answer starts, which it times from; tool_answer fills in the tools and counts.""" + + def __init__(self, cfg, model=None, scope=None, picked=None, tool_calls=None, research=None): + self.cfg, self.t0 = cfg, time.monotonic() + self.model = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None + self.scope, self.picked, self.research = scope or {}, picked, research + self.limits = {"rounds": None, "tool_calls": tool_calls} + self.used = {"rounds": 0, "tool_calls": 0} + self.offered, self.box, self.limited = [], None, False + + def offer(self, toolbox, specs, rounds): + self.box, self.offered, self.limits["rounds"] = toolbox, [s["function"]["name"] for s in specs], rounds + + def meta(self, db, outcome, resource=None): + """The record, once the answer ended with `outcome` (answered, direct, fallback, stopped or error; an answer + given because the rounds ran out is `limit`), with the tokens and cost of the ledger rows that name `resource` + (none without one).""" + if outcome in ("answered", "direct") and self.limited: + outcome = "limit" + out = { + "model": self.model, + "provider": provider(self.cfg), + "tools": {"picked": self.picked, "offered": self.offered}, + "scope": self.scope, + "limits": dict(self.limits), + "used": dict(self.used), + "outcome": outcome, + "ms": int((time.monotonic() - self.t0) * 1000), + "flagged": list(dict.fromkeys(getattr(self.box, "flagged", None) or [])), + "research": self.research, + "version": __version__, + } + if resource: + spent = activity.totals(db, resource) + out["tokens"] = {"in": spent["tokens_in"], "out": spent["tokens_out"]} + out["cost_usd"], out["cost_estimated"] = spent["cost_usd"], spent["estimate"] + return out + + +def question_record(via, **options): + """A question's record: how it was sent (web, api, room, voice, automation or completions) and the options it was + asked with.""" + return {"via": via, "options": store.clean(options)} + + TOOL_SYSTEM = ( SYSTEM + " You can call tools to look things up in the archive. Cite moments with the [n] numbers the tools return, and " "only those. Tools that run work or change data need the person's approval: say what you proposed and that it's waiting for them. " @@ -679,11 +740,12 @@ def _finish(cfg, msgs, tools, model): return text -def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None): +def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None): """The tool loop: yields ("step", {...}) for each tool call, then ("answer", text), or ("direct", text) when the model answered without looking anything up (it never saw the archive, so the caller can answer from a search instead). Every turn ends in an answer: when the steps run out, or the model replies with nothing, it's asked to answer from - what it found, with the tools off. Raises llm.ToolsUnsupported.""" + what it found, with the tools off. Raises llm.ToolsUnsupported. `record` (a Record) learns the tools offered and + counts the rounds and tool calls.""" system = TOOL_SYSTEM + untrusted.rules(cfg) + ("\n\n" + SETUP_SYSTEM if setup else "") note = getattr(toolbox, "system_note", None) # skills, and context from hooks (extensions.py) system += note() if note else "" @@ -695,6 +757,8 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set msgs = [{"role": "system", "content": system}] + past_turns(history, summary=summary, cfg=cfg) msgs.append({"role": "user", "content": question}) tools, looked, made = toolbox.specs(), False, set() + record = record or Record(cfg) + record.offer(toolbox, tools, max_steps) for _ in range(max_steps): msg = llm.chat_message(cfg, msgs, tools=tools, model=model) if not msg["tool_calls"]: @@ -703,6 +767,8 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set return break # a reply with nothing in it looked = True + record.used["rounds"] += 1 + record.used["tool_calls"] += len(msg["tool_calls"]) msgs.append({"role": "assistant", "content": msg["content"], "tool_calls": msg["tool_calls"]}) for c in msg["tool_calls"]: fn = c.get("function") or {} @@ -722,9 +788,13 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set # anything that may change what a read returns (a note written, a setting changed) makes every read new again made = made | {key} if name in SAME_AGAIN else set() # as the model reads it: what the tool read from the archive, the web or another server marked as data + t0 = time.monotonic() result, summary = toolbox.call(name, args, model=True) - yield "step", {"tool": name, "args": args, "summary": summary} + step = {"tool": name, "args": args, "summary": summary, "ms": int((time.monotonic() - t0) * 1000)} + yield "step", {**step, "failed": True} if getattr(toolbox, "failed", False) else step msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": result}) + else: # the rounds ran out + record.limited = True text = _finish(cfg, msgs, tools, model) or "I couldn't put an answer together from what I found. Try again, or ask something narrower." yield ("answer" if looked else "direct"), text diff --git a/fastapi_backend/app/domain/completions.py b/fastapi_backend/app/domain/completions.py index 44c44f45d..e344162ed 100644 --- a/fastapi_backend/app/domain/completions.py +++ b/fastapi_backend/app/domain/completions.py @@ -115,32 +115,34 @@ def specs(self): def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools=(), base=None, temperature=None): """Answer one request. Yields ("step", {tool, args, summary}) for each tool call, ("token", text) for the - answer's pieces, then ("done", {refs, text, model, notice}): what it can cite (numbered, as [n]) and the whole - answer. Raises llm.LLMError when the model server fails, and activity.OverBudget at the hard limit.""" + answer's pieces, then ("done", {refs, text, model, notice, record}): what it can cite (numbered, as [n]), the whole + answer and what produced it (chat.Record, without tokens and cost: no message keeps it). Raises llm.LLMError when + the model server fails, and activity.OverBudget at the hard limit.""" scope = {"namespaces": [ns]} + record = chat.Record(cfg, model, scope, picked=list(tools) or None) passages = chat.retrieve(db, question, {sid}, scope, cfg=cfg) extra, notice = note(db, sid, ns, app), None used = model or cfg["llm"].get("model") if not llm.configured(cfg): text = chat.fallback(passages) yield "token", text - yield "done", {"refs": passages, "text": text, "model": MODEL, "notice": None} + yield "done", {"refs": passages, "text": text, "model": MODEL, "notice": None, "record": record.meta(db, "fallback")} return if tools and cfg["ai"].get("tools", True): box = Box(db, cfg, user, sid, ns, tools, question, base, extra) try: - text = "" - for kind, data in chat.tool_answer(cfg, box, question, history, cfg["ai"].get("max_steps") or 6, model): + text, how = "", "answered" + for kind, data in chat.tool_answer(cfg, box, question, history, cfg["ai"].get("max_steps") or 6, model, record=record): if kind == "step": yield "step", data # a model that skips the tools hasn't seen the namespace: it answers from the excerpts instead elif kind == "direct" and passages and not box.cited(data): break else: - text = data + text, how = data, "direct" if kind == "direct" else "answered" else: yield "token", text - yield "done", {"refs": box.refs, "text": text, "model": used, "notice": None} + yield "done", {"refs": box.refs, "text": text, "model": used, "notice": None, "record": record.meta(db, how)} return except llm.ToolsUnsupported: notice = "This model can't use tools, so the answer comes from a search instead." @@ -150,7 +152,10 @@ def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools= for piece in llm.stream_chat(cfg, msgs, model=model, temperature=0.2 if temperature is None else temperature): text += piece yield "token", piece - yield "done", {"refs": passages, "text": text, "model": used, "notice": notice} + yield ( + "done", + {"refs": passages, "text": text, "model": used, "notice": notice, "record": record.meta(db, "fallback" if notice else "answered")}, + ) def link(web, ref): diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py index 2cf82c334..1ce9738df 100644 --- a/fastapi_backend/app/schemas/chats.py +++ b/fastapi_backend/app/schemas/chats.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, Field +from app.schemas.activity import ActivityEntry from app.schemas.common import RequestModel, ResponseModel @@ -80,6 +81,67 @@ class ToolStep(ResponseModel): tool: str args: dict[str, Any] = {} summary: str = "" + ms: int | None = Field(None, description="how long the tool took") + failed: bool | None = Field(None, description="the tool failed (or was unknown or blocked); its summary says why") + + +class RecordTools(ResponseModel): + picked: list[str] | None = Field(None, description="the tools or groups the question asked for; null: Auto") + offered: list[str] = Field(default_factory=list, description="every tool the model was offered") + + +class RecordLimits(ResponseModel): + rounds: int | None = Field(None, description="rounds of tool calls it had (calls in one round run together)") + tool_calls: int | None = Field(None, description="tool calls it had; null: only the rounds limit them") + + +class RecordUse(ResponseModel): + rounds: int = 0 + tool_calls: int = 0 + + +class RecordTokens(ResponseModel): + model_config = ConfigDict(extra="allow", populate_by_name=True, serialize_by_alias=True) + + in_: int = Field(0, alias="in") + out: int = 0 + + +class RecordOptions(ResponseModel): + """What a question was asked with (only what it set).""" + + model: str | None = None + tools: list[str] | None = None + max_tool_calls: int | None = None + scope: dict[str, Any] | None = None + research: bool | None = None + + +class MessageRecord(ResponseModel): + """What produced a message, recorded as it was written (docs/assistant.md, "What each answer records"). A question's + has `via` and `options`; an answer's the rest. Messages from before it was recorded have none.""" + + via: Literal["web", "api", "room", "voice", "automation", "completions"] | None = Field(None, description="a question: how it was sent") + options: RecordOptions | None = Field(None, description="a question: the options it was asked with") + model: str | None = Field(None, description="the model that wrote the answer") + provider: str | None = Field(None, description="the host name of the model server it ran on (never its address or key)") + tools: RecordTools | None = None + scope: dict[str, Any] | None = Field(None, description="the scope it answered in, after a conversation over everything was narrowed") + limits: RecordLimits | None = None + used: RecordUse | None = None + outcome: Literal["answered", "direct", "limit", "fallback", "stopped", "error"] | None = Field( + None, + description="answered (after looking things up), direct (without looking anything up), limit (its rounds ran " + "out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set " + "up), stopped, or error", + ) + tokens: RecordTokens | None = Field(None, description="the tokens of the model calls made for it (the activity ledger)") + cost_usd: float | None = Field(None, description="what those calls cost, from the prices in Settings") + cost_estimated: bool | None = Field(None, description="some calls had a price but no token counts: the cost is at least this") + ms: int | None = Field(None, description="how long it took to write, from the question") + flagged: list[str] | None = Field(None, description="tools whose results read like instructions to an AI") + research: int | None = Field(None, description="the research it belongs to") + version: str | None = Field(None, description="the version of Lens that wrote it") class PageContext(RequestModel): @@ -127,6 +189,13 @@ class ChatMessage(ResponseModel): error: str | None = Field(None, description="why there's no answer") check: AnswerCheck | None = Field(None, description="its latest source check (POST .../messages/{mid}/check)") model: str | None = Field(None, description="the model that wrote the answer") + meta: MessageRecord | None = Field(None, description="what produced it; none on messages from before it was recorded") + + +class MessageDetail(ChatMessage): + ledger: list[ActivityEntry] = Field( + default_factory=list, description="its rows in the activity ledger: the model and tool calls made for it, newest first" + ) class ChatCapabilities(ResponseModel): @@ -172,6 +241,7 @@ class MessageCreate(RequestModel): description="asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the " "assistant can use that browser (docs/browser-assistant.md)", ) + via: Literal["voice"] | None = Field(None, description="voice: said in a voice conversation (the question's record keeps it)") class ApprovalOrigin(ResponseModel): diff --git a/fastapi_backend/tests/api/test_actions_ai.py b/fastapi_backend/tests/api/test_actions_ai.py index 9f018cb95..77cc325ae 100644 --- a/fastapi_backend/tests/api/test_actions_ai.py +++ b/fastapi_backend/tests/api/test_actions_ai.py @@ -9,7 +9,7 @@ import pytest -from app.domain import actions, actions_ai, ai_tools, budgets, chat, notebook, store, untrusted +from app.domain import actions, actions_ai, activity, ai_tools, budgets, chat, notebook, store, untrusted from tests import fake_llm from tests.api._assist import start_llm from tests.api.test_setup_assistant import call @@ -171,6 +171,10 @@ def test_ask_agent_works_on_what_happened_and_saves_the_conversation(db, cfg, ll q, a = chat.history(db, got["chat"]) assert q["content"] == "File this where it belongs." and q["context"]["url"] == "/resources/7" assert a["content"] == got["answer"] and [s["tool"] for s in a["steps"]] == [s["tool"] for s in got["steps"]] + # it records what produced it, like any answer: the question came from an automation + assert q["meta"] == {"via": "automation", "options": {}} and a["meta"]["used"] == {"rounds": 2, "tool_calls": 2} + assert (a["meta"]["outcome"], a["meta"]["scope"]) == ("answered", {"namespaces": ["pods"]}) + assert activity.totals(db, f"chat_message:{a['id']}")["calls"] == 3 # its model calls name it in the ledger first = next(b for b in llm.seen if b.get("tools")) system, asked = first["messages"][0]["content"], first["messages"][-1]["content"] assert "acting for an automation, “Automation 4”, that runs for me" in system diff --git a/fastapi_backend/tests/api/test_bridge.py b/fastapi_backend/tests/api/test_bridge.py index 5a3cc8c75..ea489756e 100644 --- a/fastapi_backend/tests/api/test_bridge.py +++ b/fastapi_backend/tests/api/test_bridge.py @@ -5,7 +5,7 @@ import pytest -from app.domain import bridge, settings +from app.domain import activity, bridge, settings from tests import fake_llm, fake_matterbridge from tests.api._assist import start_llm from tests.api.test_setup_assistant import call @@ -98,6 +98,10 @@ def test_it_answers_what_is_said_to_it_and_nothing_else(db, cfg, room, llm): rows = db.rows("SELECT title, kind, account FROM chat WHERE kind = 'bridge' ORDER BY title") assert [r["title"] for r in rows] == ["Matterbridge · general · alice"] * 2 + ["Matterbridge · general · bob"] assert {r["kind"] for r in rows} == {"bridge"} and {r["account"] for r in rows} == {rows[0]["account"]} + # each question says it was asked in a room; each answer what produced it, its model calls naming it in the ledger + q, a = db.rows("SELECT record::id(id) AS id, role, meta FROM chat_message ORDER BY id")[:2] + assert q["meta"] == {"via": "room", "options": {}} and (a["meta"]["outcome"], a["meta"]["model"]) == ("answered", "fake") + assert activity.totals(db, f"chat_message:{a['id']}")["calls"] >= 1 room.waiting = [said("Lens, and on Monday?")] run(db, cfg) assert len(db.rows("SELECT id FROM chat WHERE kind = 'bridge'")) == 3 # alice keeps hers diff --git a/fastapi_backend/tests/api/test_chat_records.py b/fastapi_backend/tests/api/test_chat_records.py new file mode 100644 index 000000000..8f8bba2fc --- /dev/null +++ b/fastapi_backend/tests/api/test_chat_records.py @@ -0,0 +1,178 @@ +"""Each answer records what produced it, and each question how it was sent (docs/assistant.md, "What each answer +records"): the model and its host, the tools offered and used, the scope, limits and use, tokens and cost from the +ledger rows that name the answer, the time, how it ended and which tool results read like instructions to an AI.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import __version__, chat +from app.domain import llm as llm_mod +from tests import fake_llm +from tests.api._assist import Assist, sse, start_llm + +USAGE = {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110} + + +@pytest.fixture +def llm(cfg): + srv = start_llm(cfg) + cfg["telemetry"]["prices"] = {"fake": {"input": 1.0, "output": 2.0}} # USD per million tokens + fake_llm.Handler.usage = USAGE + yield fake_llm.Handler + fake_llm.Handler.usage = None + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, llm): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def plain(cfg): + """The plain search-and-answer path (no tools).""" + cfg["ai"]["tools"] = False + + +def tool(i, name, args): + return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}} + + +def search(i): + return {"content": "", "tool_calls": [tool(i, "search_transcripts", {"query": f"capsid {i}"})]} + + +def kept(d): + """A record without the fields it doesn't have (the API lists them as null).""" + return {k: kept(v) if isinstance(v, dict) else v for k, v in d.items() if v is not None} if isinstance(d, dict) else d + + +def ask(c, h, q, **body): + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + raw = c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": q, **body}).text + return cid, sse(raw), raw + + +def test_an_answer_records_what_produced_it(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + # one round of two calls, the second outside what the editor can read, then the answer + llm.tool_script = [ + { + "content": "", + "tool_calls": [ + tool(1, "search_transcripts", {"query": "Dyno Therapeutics"}), + tool(2, "read_transcript", {"recording_id": s.call}), + ], + }, + {"content": "Dyno Therapeutics designs capsids with machine learning [1]."}, + ] + cid, ev, raw = ask(c, h, "What does Dyno do?") + meta = ev["meta"][0] + assert raw.index("event: meta") < raw.index("event: done") # the record comes just before the end + assert (meta["model"], meta["provider"], meta["outcome"], meta["version"]) == ("fake", "127.0.0.1", "answered", __version__) + assert meta["tools"]["picked"] is None and {"search_transcripts", "read_transcript"} <= set(meta["tools"]["offered"]) + assert meta["limits"] == {"rounds": 6, "tool_calls": None} and meta["used"] == {"rounds": 1, "tool_calls": 2} + assert meta["flagged"] == [] and meta["research"] is None and isinstance(meta["ms"], int) and meta["ms"] >= 0 + assert "base_url" not in json.dumps(meta) and ":" not in meta["provider"] # the host name only, never the address + # each step says how long it took, and whether it failed + assert [isinstance(x["ms"], int) for x in ev["step"]] == [True, True] + assert "failed" not in ev["step"][0] and ev["step"][1]["failed"] is True + + msgs = c.get(f"/api/v1/chats/{cid}", headers=h).json() + mid = ev["done"][0]["message"] + answer, question = msgs["messages"][-1], msgs["messages"][-2] + assert answer["id"] == mid and kept(answer["meta"]) == kept(meta) and answer["model"] == "fake" # kept, with `model` as before + assert meta["scope"] == msgs["scope"] # the scope it answered in, after the conversation was narrowed + assert kept(question["meta"]) == {"via": "web", "options": {}} and question["id"] == mid - 1 + + # every model call made for it names it in the ledger, and its tokens and cost are theirs + one = c.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=h).json() + assert kept(one["meta"]) == kept(meta) and one["content"] == answer["content"] + calls = [r for r in one["ledger"] if r["kind"] == "out"] + assert sum(r["action"] == "model.chat" for r in calls) >= 2 + assert all({f"chat_message:{mid}", f"chat:{cid}"} <= set(r["resources"]) for r in calls) + assert meta["tokens"] == {"in": sum(r.get("tokens_in") or 0 for r in calls), "out": sum(r.get("tokens_out") or 0 for r in calls)} + assert meta["tokens"]["in"] >= 200 and meta["cost_usd"] == pytest.approx(sum(r.get("cost_usd") or 0 for r in calls)) + assert meta["cost_usd"] > 0 and meta["cost_estimated"] is False + # yours alone: the message's activity too + assert c.get("/api/v1/activity", headers=h, params={"resource": f"chat_message:{mid}"}).status_code == 200 + cv, hv = s.cl["viewer"] + assert cv.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=hv).status_code == 404 + assert cv.get("/api/v1/activity", headers=hv, params={"resource": f"chat_message:{mid}"}).status_code == 403 + assert c.get(f"/api/v1/chats/{cid}/messages/999999", headers=h).status_code == 404 + + # a question said in a voice conversation, and one asked with another model, say so + llm.tool_script = [{"content": "Hello!"}] + cid, ev, _ = ask(c, h, "zzqx", via="voice", model="fake-large") + question = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-2] + assert kept(question["meta"]) == {"via": "voice", "options": {"model": "fake-large"}} and ev["meta"][0]["model"] == "fake-large" + # answered without looking anything up + assert (ev["meta"][0]["outcome"], ev["meta"][0]["used"]) == ("direct", {"rounds": 0, "tool_calls": 0}) + + +def test_how_an_answer_ended(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + # its rounds ran out, so it answered from what it found + llm.tool_script = [search(i) for i in range(6)] + [{"content": "The capsid model won [1]."}] + _, ev, _ = ask(c, h, "Would you hire them?") + assert (ev["meta"][0]["outcome"], ev["meta"][0]["used"]) == ("limit", {"rounds": 6, "tool_calls": 6}) + # the model can't use tools: from a search instead + llm.reject_tools = True + _, ev, _ = ask(c, h, "capsid") + meta = ev["meta"][0] + assert "notice" in ev and meta["outcome"] == "fallback" and meta["tokens"]["out"] >= USAGE["completion_tokens"] + llm.reject_tools = False + + +def test_stopped_and_failed_answers_are_recorded(plain, app, db, cfg, folder, new_client, llm, monkeypatch): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + monkeypatch.setattr(chat.Answering, "EVERY", 0) + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + stream = llm_mod.stream_chat + + def stopped(*a, **k): # the person presses Stop after the first piece + pieces = stream(*a, **k) + yield next(pieces) + chat.request_stop(db, cid) + yield from pieces + + monkeypatch.setattr(llm_mod, "stream_chat", stopped) + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "When does the shipment leave?"}).text) + meta = ev["meta"][0] + assert "stopped" in ev and meta["outcome"] == "stopped" and meta["tools"]["offered"] == [] and meta["limits"]["rounds"] is None + # the call that was cut off still counts for it (its row is written when the stream is closed) + one = c.get(f"/api/v1/chats/{cid}/messages/{ev['done'][0]['message']}", headers=h).json() + assert kept(one["meta"]) == kept(meta) and "model.chat" in [r["action"] for r in one["ledger"]] + + def broken(*_a, **_k): + raise llm_mod.LLMError("429 from the LLM server") + yield # a generator, like the real one + + monkeypatch.setattr(llm_mod, "stream_chat", broken) + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Once more?"}).text) + assert ev["meta"][0]["outcome"] == "error" and ev["error"][0]["message"] == "429 from the LLM server" + # with no model set up, the best excerpts come back on their own + assert c.put("/api/v1/settings/llm", headers=s.cl["admin"][1], json={"base_url": None}).status_code == 200 + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "capsid"}).text) + meta = ev["meta"][0] + assert (meta["outcome"], meta["model"], meta["provider"], meta["tokens"]) == ("fallback", None, None, {"in": 0, "out": 0}) + + +def test_old_messages_have_no_record(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + chat.add(db, cid, "user", "From before") + mid = chat.add(db, cid, "assistant", "An answer from before", [], steps=[{"tool": "search_transcripts", "args": {}, "summary": "x"}]) + got = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"] + assert [m["meta"] for m in got] == [None, None] and got[-1]["steps"][0].get("ms") is None + one = c.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=h).json() + assert one["meta"] is None and one["ledger"] == [] diff --git a/fastapi_backend/tests/api/test_completions.py b/fastapi_backend/tests/api/test_completions.py index dfea013cc..88a7f02bf 100644 --- a/fastapi_backend/tests/api/test_completions.py +++ b/fastapi_backend/tests/api/test_completions.py @@ -180,6 +180,13 @@ def tool(i, name, args): assert got["choices"][0]["message"]["content"].startswith("Dyno Therapeutics comes up on the pods [1].\n\nSources:") found = got["lens"]["sources"] assert found and {s["recording_id"] for s in found} <= {ep1, ep2} # the search never left pods + record = got["lens"]["record"] # what produced it, as a chat answer's record (no message keeps it: no tokens) + assert (record["outcome"], record["used"], record["tools"]) == ( + "answered", + {"rounds": 1, "tool_calls": 2}, + {"picked": ["search_transcripts"], "offered": ["search_transcripts"]}, + ) + assert record["scope"] == {"namespaces": ["pods"]} and "tokens" not in record result = next(m["content"] for m in llm.seen[1]["messages"] if m["role"] == "tool") assert "Dyno Therapeutics" in result and "leaves on Friday" not in result # calls is another namespace assert result.startswith("<") and 'source="tool search_transcripts"' in result # read as data (untrusted.wrap) diff --git a/fastapi_backend/tests/api/test_prompt_injection.py b/fastapi_backend/tests/api/test_prompt_injection.py index e4163caa4..c6e09dd59 100644 --- a/fastapi_backend/tests/api/test_prompt_injection.py +++ b/fastapi_backend/tests/api/test_prompt_injection.py @@ -266,6 +266,7 @@ def test_a_result_that_reads_like_instructions_is_flagged(app, db, cfg, folder, why = "reads like instructions to an AI: ignore previous instructions; new instructions or role" assert f"({why};" in ev["step"][0]["summary"] and "chat control tokens)" in ev["step"][0]["summary"] assert f'source="tool read_transcript" warning="{why};' in _tool_messages(llm)[0] + assert ev["meta"][0]["flagged"] == ["read_transcript"] # the answer's record names it too # and admins in the audit log: the reasons, never the text logged = [r for r in db.rows("SELECT * FROM audit_log") if r["action"] == "assistant.injection_flagged"] assert [(r["target"], r["detail"]["tool"]) for r in logged] == [(f"chat:{cid}", "read_transcript")] diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py index 284bfddb4..55ed02f90 100644 --- a/fastapi_backend/tests/fake_llm.py +++ b/fastapi_backend/tests/fake_llm.py @@ -49,7 +49,10 @@ def do_POST(self): if body.get("tool_choice") == "none" and not Handler.ignore_tool_choice: msg = {"content": msg.get("content")} return self._json( - {"choices": [{"message": {"role": "assistant", "content": msg.get("content"), "tool_calls": msg.get("tool_calls")}}]} + { + "choices": [{"message": {"role": "assistant", "content": msg.get("content"), "tool_calls": msg.get("tool_calls")}}], + **({"usage": Handler.usage} if Handler.usage else {}), + } ) if body.get("stream"): self.send_response(200) diff --git a/nextjs-frontend/__tests__/chat-details.test.tsx b/nextjs-frontend/__tests__/chat-details.test.tsx new file mode 100644 index 000000000..c280d72d1 --- /dev/null +++ b/nextjs-frontend/__tests__/chat-details.test.tsx @@ -0,0 +1,168 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Activity, Chats } from "@/app/openapi-client"; +import type { MessageRecord } from "@/app/openapi-client/types.gen"; +import { AnswerDetails, DetailsBody } from "@/components/chat/details"; +import { took } from "@/components/chat/steps"; +import { applyEvent, newTurn, savedSteps } from "@/components/chat/stream"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +jest.mock("@/app/openapi-client", () => ({ + Chats: { getMessage: jest.fn() }, + Activity: { resourceCosts: jest.fn() }, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); + +const META: MessageRecord = { + model: "qwen3:8b", + provider: "ollama.lan", + tools: { picked: null, offered: ["search_transcripts", "read_transcript", "web_read"] }, + scope: { namespaces: ["pods"] }, + limits: { rounds: 6, tool_calls: null }, + used: { rounds: 2, tool_calls: 3 }, + outcome: "limit", + tokens: { in: 5120, out: 410 }, + cost_usd: 0.0031, + cost_estimated: true, + ms: 8200, + flagged: ["web_read"], + version: "0.13.0", +}; + +function wide(matches: boolean) { + window.matchMedia = ((q: string) => ({ + matches: q.includes("max-width") ? !matches : matches, + addEventListener: () => {}, + removeEventListener: () => {}, + })) as unknown as typeof window.matchMedia; +} + +function show(ui: React.ReactNode) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + {ui} + , + ); +} + +beforeEach(() => { + jest.clearAllMocks(); + wide(true); + (Chats.getMessage as jest.Mock).mockImplementation(() => + ok({ + id: 8, + role: "assistant", + content: "x", + ledger: [ + { + id: "a1", + at: "2026-10-16T10:00:00Z", + kind: "out", + action: "model.chat", + model: "qwen3:8b", + tokens_in: 4000, + tokens_out: 300, + ms: 5100, + ok: true, + }, + { id: "a2", at: "2026-10-16T10:00:01Z", kind: "in", action: "POST /api/v1/chats/{cid}/messages", ok: true }, + ], + }), + ); + (Activity.resourceCosts as jest.Mock).mockImplementation(() => + ok({ costs: { "chat:3": { cost_usd: 0.02, tokens: 9000, calls: 4, unpriced: 0, estimate: false } } }), + ); +}); + +describe("an answer's record", () => { + it("says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended", async () => { + show( + , + ); + expect(screen.getByText("Its tool rounds ran out, so it answered from what it found")).toBeInTheDocument(); + expect(screen.getByText("qwen3:8b")).toBeInTheDocument(); + expect(screen.getByText(/on ollama\.lan/)).toBeInTheDocument(); + expect(screen.getByText("pods")).toBeInTheDocument(); + expect(screen.getByText("3 tools offered")).toBeInTheDocument(); + expect(screen.getByText(/2 of 6 rounds, 3 tool calls/)).toBeInTheDocument(); + expect(screen.getByText("search_transcripts, web_read")).toBeInTheDocument(); + expect(screen.getByText(/web_read: what it read looked like instructions to an AI/)).toBeInTheDocument(); + expect(screen.getByText("5.1k in · 410 out")).toBeInTheDocument(); + expect(screen.getByText("(estimated from the prices in Settings)")).toBeInTheDocument(); + expect(screen.getByText("≈")).toBeInTheDocument(); // some calls weren't priced: at least this + expect(screen.getByText("8.2 s")).toBeInTheDocument(); + // the calls made for it, from the ledger (calls out only) + expect(await screen.findByText(/Model call · qwen3:8b/)).toBeInTheDocument(); + expect(screen.queryByText(/POST \/api\/v1/)).not.toBeInTheDocument(); + expect(Chats.getMessage).toHaveBeenCalledWith(expect.objectContaining({ path: { cid: 3, mid: 8 } })); + expect(await screen.findByText("$0.02")).toBeInTheDocument(); // the conversation so far + }); + + it("says an old answer wasn't recorded, and asks the ledger nothing", () => { + show(); + expect(screen.getByText(/from before Lens recorded what produced each one/)).toBeInTheDocument(); + expect(Chats.getMessage).not.toHaveBeenCalled(); + }); + + it("an answer without tools or a price says so", () => { + const plain: MessageRecord = { + ...META, + tools: { picked: null, offered: [] }, + limits: { rounds: null, tool_calls: null }, + used: { rounds: 0, tool_calls: 0 }, + outcome: "answered", + cost_usd: 0, + cost_estimated: false, + flagged: [], + }; + show(); + expect(screen.getByText("None offered")).toBeInTheDocument(); + expect(screen.getByText("No tools")).toBeInTheDocument(); + expect(screen.getByText("Not costed: no price is set for its model")).toBeInTheDocument(); + expect(screen.queryByText("Flagged")).not.toBeInTheDocument(); + }); + + it("opens from the i in the answer's header, by keyboard too; on a phone in a sheet", async () => { + show(); + const button = screen.getByRole("button", { name: "Details: what produced this answer" }); + button.focus(); + fireEvent.click(button); + expect(await screen.findByText("What produced this answer")).toBeInTheDocument(); + expect(screen.getByText("qwen3:8b")).toBeInTheDocument(); + + wide(false); + show(); + fireEvent.click(screen.getAllByRole("button", { name: "Details: what produced this answer" }).at(-1)!); + expect(await screen.findByRole("dialog")).toHaveTextContent("What produced this answer"); + expect(screen.getByRole("dialog")).toHaveTextContent(/from before Lens recorded/); + }); +}); + +describe("the record in the stream", () => { + it("keeps the meta event, and each step's time and failure", () => { + let s = newTurn("q"); + s = applyEvent(s, { + event: "step", + data: '{"tool":"read_transcript","args":{},"summary":"read_transcript: not found","ms":40,"failed":true}', + }); + s = applyEvent(s, { event: "meta", data: JSON.stringify(META) }); + s = applyEvent(s, { event: "done", data: '{"message":8}' }); + expect(s.steps[0]).toMatchObject({ ms: 40, failed: true }); + expect(s.meta).toMatchObject({ outcome: "limit", provider: "ollama.lan" }); + expect(savedSteps([{ tool: "x" }])[0]).toEqual({ tool: "x", args: {}, summary: "" }); + expect([took(120), took(1400), took(12500)]).toEqual(["120 ms", "1.4 s", "13 s"]); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index b5db04b3b..c82ba439f 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1218,6 +1218,11 @@ export type { GetMediaError, GetMediaErrors, GetMediaResponses, + GetMessageData, + GetMessageError, + GetMessageErrors, + GetMessageResponse, + GetMessageResponses, GetMonitoringData, GetMonitoringResponse, GetMonitoringResponses, @@ -2147,6 +2152,8 @@ export type { MergeTopicsResponse, MergeTopicsResponses, MessageCreate, + MessageDetail, + MessageRecord, MetadataBulk, MetadataBulkResult, MetadataEdit, @@ -2515,6 +2522,11 @@ export type { RecordingTopicsResponse, RecordingTopicsResponses, RecordingUpdate, + RecordLimits, + RecordOptions, + RecordTokens, + RecordTools, + RecordUse, RedeemTicketData, RedeemTicketError, RedeemTicketErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 552691e6a..eb08ca52b 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -577,6 +577,9 @@ import type { GetMediaData, GetMediaErrors, GetMediaResponses, + GetMessageData, + GetMessageErrors, + GetMessageResponses, GetMonitoringData, GetMonitoringResponses, GetNamespaceAssistantData, @@ -9971,12 +9974,29 @@ export class Chats { }); } + /** + * Get Message + * + * One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and + * used, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`: + * the calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}. + */ + public static getMessage( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/chats/{cid}/messages/{mid}", + ...options, + }); + } + /** * Send Message * * Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model * reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST - * /chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id). + * /chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced + * it), done (the saved message id). * * With `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and * its new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 06ed72de5..01e1de85c 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -3618,6 +3618,10 @@ export type ChatMessage = { * the model that wrote the answer */ model?: string | null; + /** + * what produced it; none on messages from before it was recorded + */ + meta?: MessageRecord | null; [key: string]: unknown; }; @@ -10656,6 +10660,182 @@ export type MessageCreate = { * asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the assistant can use that browser (docs/browser-assistant.md) */ browser?: string | null; + /** + * Via + * + * voice: said in a voice conversation (the question's record keeps it) + */ + via?: "voice" | null; +}; + +/** + * MessageDetail + */ +export type MessageDetail = { + /** + * Id + */ + id: number; + /** + * Role + */ + role: "user" | "assistant"; + /** + * Content + */ + content: string; + /** + * a question asked from a page: the page, and any highlighted text + */ + context?: SharedContext | null; + /** + * Attachments + * + * files sent with it (POST /uploads with hold) + */ + attachments?: Array; + /** + * Passages + */ + passages?: Array | null; + /** + * Created At + */ + created_at?: string | null; + /** + * Stopped + * + * the answer was stopped (POST /chats/{cid}/stop): `content` is what came before + */ + stopped?: boolean; + /** + * Steps + * + * the tools the assistant used for this answer, in order + */ + steps?: Array; + /** + * Notice + * + * e.g. the model can't use tools, so the answer came from a search + */ + notice?: string | null; + /** + * Error + * + * why there's no answer + */ + error?: string | null; + /** + * its latest source check (POST .../messages/{mid}/check) + */ + check?: AnswerCheck | null; + /** + * Model + * + * the model that wrote the answer + */ + model?: string | null; + /** + * what produced it; none on messages from before it was recorded + */ + meta?: MessageRecord | null; + /** + * Ledger + * + * its rows in the activity ledger: the model and tool calls made for it, newest first + */ + ledger?: Array; + [key: string]: unknown; +}; + +/** + * MessageRecord + * + * What produced a message, recorded as it was written (docs/assistant.md, "What each answer records"). A question's + * has `via` and `options`; an answer's the rest. Messages from before it was recorded have none. + */ +export type MessageRecord = { + /** + * Via + * + * a question: how it was sent + */ + via?: "web" | "api" | "room" | "voice" | "automation" | "completions" | null; + /** + * a question: the options it was asked with + */ + options?: RecordOptions | null; + /** + * Model + * + * the model that wrote the answer + */ + model?: string | null; + /** + * Provider + * + * the host name of the model server it ran on (never its address or key) + */ + provider?: string | null; + tools?: RecordTools | null; + /** + * Scope + * + * the scope it answered in, after a conversation over everything was narrowed + */ + scope?: { + [key: string]: unknown; + } | null; + limits?: RecordLimits | null; + used?: RecordUse | null; + /** + * Outcome + * + * answered (after looking things up), direct (without looking anything up), limit (its rounds ran out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set up), stopped, or error + */ + outcome?: "answered" | "direct" | "limit" | "fallback" | "stopped" | "error" | null; + /** + * the tokens of the model calls made for it (the activity ledger) + */ + tokens?: RecordTokens | null; + /** + * Cost Usd + * + * what those calls cost, from the prices in Settings + */ + cost_usd?: number | null; + /** + * Cost Estimated + * + * some calls had a price but no token counts: the cost is at least this + */ + cost_estimated?: boolean | null; + /** + * Ms + * + * how long it took to write, from the question + */ + ms?: number | null; + /** + * Flagged + * + * tools whose results read like instructions to an AI + */ + flagged?: Array | null; + /** + * Research + * + * the research it belongs to + */ + research?: number | null; + /** + * Version + * + * the version of Lens that wrote it + */ + version?: string | null; + [key: string]: unknown; }; /** @@ -15424,6 +15604,105 @@ export type Reading = { [key: string]: unknown; }; +/** + * RecordLimits + */ +export type RecordLimits = { + /** + * Rounds + * + * rounds of tool calls it had (calls in one round run together) + */ + rounds?: number | null; + /** + * Tool Calls + * + * tool calls it had; null: only the rounds limit them + */ + tool_calls?: number | null; + [key: string]: unknown; +}; + +/** + * RecordOptions + * + * What a question was asked with (only what it set). + */ +export type RecordOptions = { + /** + * Model + */ + model?: string | null; + /** + * Tools + */ + tools?: Array | null; + /** + * Max Tool Calls + */ + max_tool_calls?: number | null; + /** + * Scope + */ + scope?: { + [key: string]: unknown; + } | null; + /** + * Research + */ + research?: boolean | null; + [key: string]: unknown; +}; + +/** + * RecordTokens + */ +export type RecordTokens = { + /** + * In + */ + in?: number; + /** + * Out + */ + out?: number; + [key: string]: unknown; +}; + +/** + * RecordTools + */ +export type RecordTools = { + /** + * Picked + * + * the tools or groups the question asked for; null: Auto + */ + picked?: Array | null; + /** + * Offered + * + * every tool the model was offered + */ + offered?: Array; + [key: string]: unknown; +}; + +/** + * RecordUse + */ +export type RecordUse = { + /** + * Rounds + */ + rounds?: number; + /** + * Tool Calls + */ + tool_calls?: number; + [key: string]: unknown; +}; + /** * Recording * @@ -20726,6 +21005,18 @@ export type ToolStep = { * Summary */ summary?: string; + /** + * Ms + * + * how long the tool took + */ + ms?: number | null; + /** + * Failed + * + * the tool failed (or was unknown or blocked); its summary says why + */ + failed?: boolean | null; [key: string]: unknown; }; @@ -38306,6 +38597,40 @@ export type UpdateChatResponses = { export type UpdateChatResponse = UpdateChatResponses[keyof UpdateChatResponses]; +export type GetMessageData = { + body?: never; + path: { + /** + * Cid + */ + cid: number; + /** + * Mid + */ + mid: number; + }; + query?: never; + url: "/api/v1/chats/{cid}/messages/{mid}"; +}; + +export type GetMessageErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetMessageError = GetMessageErrors[keyof GetMessageErrors]; + +export type GetMessageResponses = { + /** + * Successful Response + */ + 200: MessageDetail; +}; + +export type GetMessageResponse = GetMessageResponses[keyof GetMessageResponses]; + export type SendMessageData = { body: MessageCreate; path: { diff --git a/nextjs-frontend/components/chat/answer.tsx b/nextjs-frontend/components/chat/answer.tsx index 1775be1f4..130f52154 100644 --- a/nextjs-frontend/components/chat/answer.tsx +++ b/nextjs-frontend/components/chat/answer.tsx @@ -6,7 +6,7 @@ import Link from "next/link"; import { useMemo } from "react"; import { Chats, Search } from "@/app/openapi-client"; -import type { AnswerCheck, Passage } from "@/app/openapi-client/types.gen"; +import type { AnswerCheck, MessageRecord, Passage } from "@/app/openapi-client/types.gen"; import { ApprovalCard, type ApprovalView } from "@/components/chat/approval"; import { citedNumbers, @@ -17,6 +17,7 @@ import { saysNotCovered, } from "@/components/chat/cite"; import { CitationChip } from "@/components/chat/citation"; +import { AnswerDetails } from "@/components/chat/details"; import { keywords } from "@/components/chat/keywords"; import { RichText } from "@/components/chat/rich-text"; import { scopeWords, type Scope } from "@/components/chat/scope"; @@ -145,6 +146,8 @@ export type AnswerProps = { active?: boolean; check?: AnswerCheck | null; onChecked?: (c: AnswerCheck) => void; + /** What produced it (the answer's record); older answers have none. */ + meta?: MessageRecord | null; }; /** One answer: where it comes from, the tools used, approvals, the text with citations, and its error states. */ @@ -191,6 +194,9 @@ export function Answer(p: AnswerProps) { ? `From ${from}` : "Answer"} {p.model ? ` · ${p.model}` : ""} + {!streaming && p.messageId != null && ( + + )} {p.onShowSources && p.passages && p.passages.length > 0 && ( + ); + if (narrow) + return ( + <> + {trigger} + + + + + ); + return ( + + {trigger} + + What produced this answer + + + + ); +} diff --git a/nextjs-frontend/components/chat/steps.tsx b/nextjs-frontend/components/chat/steps.tsx index fd0af6992..95214da36 100644 --- a/nextjs-frontend/components/chat/steps.tsx +++ b/nextjs-frontend/components/chat/steps.tsx @@ -42,10 +42,16 @@ type Kind = "done" | "failed" | "waiting"; function kindOf(step: ToolStep): Kind { if (/^asked for approval/i.test(step.summary)) return "waiting"; - if (step.summary.startsWith(`${step.tool}:`) || /^unknown tool/i.test(step.summary)) return "failed"; + // answers record whether a step failed; older ones are read from the summary + if (step.failed || step.summary.startsWith(`${step.tool}:`) || /^unknown tool/i.test(step.summary)) return "failed"; return "done"; } +/** How long a step took: "120 ms", "1.4 s". */ +export function took(ms: number): string { + return ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(ms < 10_000 ? 1 : 0)} s`; +} + const GLYPH: Record = { done: { g: "✓", bg: "var(--aladdin-green)", label: "done" }, failed: { g: "✕", bg: "var(--aladdin-red)", label: "failed" }, @@ -83,6 +89,7 @@ function StepRow({ step, index }: { step: ToolStep; index: number }) { {g.label}: {title} + {step.ms != null && {took(step.ms)}} {open ? ( ) : ( diff --git a/nextjs-frontend/components/chat/stream.ts b/nextjs-frontend/components/chat/stream.ts index a4f6c837e..5d2cf420e 100644 --- a/nextjs-frontend/components/chat/stream.ts +++ b/nextjs-frontend/components/chat/stream.ts @@ -1,16 +1,20 @@ -import type { Estimate, Passage } from "@/app/openapi-client/types.gen"; +import type { Estimate, MessageRecord, Passage } from "@/app/openapi-client/types.gen"; /** * One question's answer as it streams from POST /chats/{id}/messages. The backend sends server-sent events: * `scoped` (a conversation over everything narrowed to the namespace it's about), `suggested` (namespaces to offer when that isn't clear), `step` (a tool the assistant used), `approval` (work waiting for the person), `notice`, `passages` (the numbered - * excerpts), `token` (answer text), `error`, `stopped` (Stop was pressed: what came is saved, marked stopped) and - * `done` (the saved message id). + * excerpts), `token` (answer text), `error`, `stopped` (Stop was pressed: what came is saved, marked stopped), `meta` + * (the answer's record: what produced it) and `done` (the saved message id). */ export type ToolStep = { tool: string; args: Record; summary: string; + /** How long the tool took. */ + ms?: number | null; + /** It failed (or was unknown or blocked); the summary says why. */ + failed?: boolean | null; }; export type PendingApproval = { id: number; @@ -37,6 +41,8 @@ export type TurnState = { text: string; error: string | null; messageId: number | null; + /** What produced the answer, once it's saved. */ + meta: MessageRecord | null; }; export function newTurn(question: string): TurnState { @@ -52,6 +58,7 @@ export function newTurn(question: string): TurnState { text: "", error: null, messageId: null, + meta: null, }; } @@ -79,6 +86,8 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T tool: str(o.tool, "tool"), args: (o.args as Record) ?? {}, summary: str(o.summary), + ms: typeof o.ms === "number" ? o.ms : null, + failed: o.failed === true || null, }, ], }; @@ -121,6 +130,8 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T }; case "stopped": return { ...s, status: "stopped" }; + case "meta": + return { ...s, meta: d && !Array.isArray(d) ? (o as MessageRecord) : null }; case "done": return { ...s, @@ -134,9 +145,18 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T /** Tool steps as an answer was saved with them (GET /chats/{id}), in the shape the live stream gives. */ export function savedSteps( - steps: { tool: string; args?: Record; summary?: string }[] | null | undefined, + steps: + | { tool: string; args?: Record; summary?: string; ms?: number | null; failed?: boolean | null }[] + | null + | undefined, ): ToolStep[] { - return (steps ?? []).map((s) => ({ tool: s.tool, args: s.args ?? {}, summary: s.summary ?? "" })); + return (steps ?? []).map((s) => ({ + tool: s.tool, + args: s.args ?? {}, + summary: s.summary ?? "", + ...(s.ms != null ? { ms: s.ms } : {}), + ...(s.failed ? { failed: true } : {}), + })); } /** A short plain-words line for a tool call, when the backend's summary is missing. */ diff --git a/nextjs-frontend/components/costs/costs.tsx b/nextjs-frontend/components/costs/costs.tsx index d4181c2ea..af318d8c9 100644 --- a/nextjs-frontend/components/costs/costs.tsx +++ b/nextjs-frontend/components/costs/costs.tsx @@ -124,7 +124,7 @@ const KIND: Record change: { label: "Change", icon: , tone: "gate" }, }; -function actionText(e: ActivityEntry): string { +export function actionText(e: ActivityEntry): string { const a = e.action; if (a.startsWith("model.")) return `${a.slice(6) === "chat" ? "Model call" : `Model ${a.slice(6)}`}${e.model ? ` · ${e.model}` : ""}`; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 285f91af5..cac3b647e 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -24952,13 +24952,65 @@ } } }, + "/api/v1/chats/{cid}/messages/{mid}": { + "get": { + "tags": [ + "chats" + ], + "summary": "Get Message", + "description": "One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and\nused, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`:\nthe calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}.", + "operationId": "get_message", + "parameters": [ + { + "name": "cid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Cid" + } + }, + { + "name": "mid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Mid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MessageDetail" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/chats/{cid}/messages": { "post": { "tags": [ "chats" ], "summary": "Send Message", - "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id).\n\nWith `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and\nits new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked\nfrom (where, and the highlighted text; not the page's text, which isn't kept).\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).", + "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nWith `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and\nits new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked\nfrom (where, and the highlighted text; not the page's text, which isn't kept).\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).", "operationId": "send_message", "parameters": [ { @@ -36383,6 +36435,17 @@ ], "title": "Model", "description": "the model that wrote the answer" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/components/schemas/MessageRecord" + }, + { + "type": "null" + } + ], + "description": "what produced it; none on messages from before it was recorded" } }, "additionalProperties": true, @@ -48396,12 +48459,389 @@ ], "title": "Browser", "description": "asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the assistant can use that browser (docs/browser-assistant.md)" + }, + "via": { + "anyOf": [ + { + "type": "string", + "const": "voice" + }, + { + "type": "null" + } + ], + "title": "Via", + "description": "voice: said in a voice conversation (the question's record keeps it)" } }, "additionalProperties": false, "type": "object", "title": "MessageCreate" }, + "MessageDetail": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "role": { + "type": "string", + "enum": [ + "user", + "assistant" + ], + "title": "Role" + }, + "content": { + "type": "string", + "title": "Content" + }, + "context": { + "anyOf": [ + { + "$ref": "#/components/schemas/SharedContext" + }, + { + "type": "null" + } + ], + "description": "a question asked from a page: the page, and any highlighted text" + }, + "attachments": { + "items": { + "$ref": "#/components/schemas/Attachment" + }, + "type": "array", + "title": "Attachments", + "description": "files sent with it (POST /uploads with hold)" + }, + "passages": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Passage" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Passages" + }, + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "stopped": { + "type": "boolean", + "title": "Stopped", + "description": "the answer was stopped (POST /chats/{cid}/stop): `content` is what came before", + "default": false + }, + "steps": { + "items": { + "$ref": "#/components/schemas/ToolStep" + }, + "type": "array", + "title": "Steps", + "description": "the tools the assistant used for this answer, in order" + }, + "notice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Notice", + "description": "e.g. the model can't use tools, so the answer came from a search" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "why there's no answer" + }, + "check": { + "anyOf": [ + { + "$ref": "#/components/schemas/AnswerCheck" + }, + { + "type": "null" + } + ], + "description": "its latest source check (POST .../messages/{mid}/check)" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model", + "description": "the model that wrote the answer" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/components/schemas/MessageRecord" + }, + { + "type": "null" + } + ], + "description": "what produced it; none on messages from before it was recorded" + }, + "ledger": { + "items": { + "$ref": "#/components/schemas/ActivityEntry" + }, + "type": "array", + "title": "Ledger", + "description": "its rows in the activity ledger: the model and tool calls made for it, newest first" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "role", + "content" + ], + "title": "MessageDetail" + }, + "MessageRecord": { + "properties": { + "via": { + "anyOf": [ + { + "type": "string", + "enum": [ + "web", + "api", + "room", + "voice", + "automation", + "completions" + ] + }, + { + "type": "null" + } + ], + "title": "Via", + "description": "a question: how it was sent" + }, + "options": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordOptions" + }, + { + "type": "null" + } + ], + "description": "a question: the options it was asked with" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model", + "description": "the model that wrote the answer" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Provider", + "description": "the host name of the model server it ran on (never its address or key)" + }, + "tools": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordTools" + }, + { + "type": "null" + } + ] + }, + "scope": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Scope", + "description": "the scope it answered in, after a conversation over everything was narrowed" + }, + "limits": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordLimits" + }, + { + "type": "null" + } + ] + }, + "used": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordUse" + }, + { + "type": "null" + } + ] + }, + "outcome": { + "anyOf": [ + { + "type": "string", + "enum": [ + "answered", + "direct", + "limit", + "fallback", + "stopped", + "error" + ] + }, + { + "type": "null" + } + ], + "title": "Outcome", + "description": "answered (after looking things up), direct (without looking anything up), limit (its rounds ran out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set up), stopped, or error" + }, + "tokens": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordTokens" + }, + { + "type": "null" + } + ], + "description": "the tokens of the model calls made for it (the activity ledger)" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what those calls cost, from the prices in Settings" + }, + "cost_estimated": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimated", + "description": "some calls had a price but no token counts: the cost is at least this" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms", + "description": "how long it took to write, from the question" + }, + "flagged": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Flagged", + "description": "tools whose results read like instructions to an AI" + }, + "research": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Research", + "description": "the research it belongs to" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version", + "description": "the version of Lens that wrote it" + } + }, + "additionalProperties": true, + "type": "object", + "title": "MessageRecord", + "description": "What produced a message, recorded as it was written (docs/assistant.md, \"What each answer records\"). A question's\nhas `via` and `options`; an answer's the rest. Messages from before it was recorded have none." + }, "MetadataBulk": { "properties": { "recordings": { @@ -56531,6 +56971,168 @@ ], "title": "Reading" }, + "RecordLimits": { + "properties": { + "rounds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rounds", + "description": "rounds of tool calls it had (calls in one round run together)" + }, + "tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tool Calls", + "description": "tool calls it had; null: only the rounds limit them" + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordLimits" + }, + "RecordOptions": { + "properties": { + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls" + }, + "scope": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Scope" + }, + "research": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Research" + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordOptions", + "description": "What a question was asked with (only what it set)." + }, + "RecordTokens": { + "properties": { + "in": { + "type": "integer", + "title": "In", + "default": 0 + }, + "out": { + "type": "integer", + "title": "Out", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordTokens" + }, + "RecordTools": { + "properties": { + "picked": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Picked", + "description": "the tools or groups the question asked for; null: Auto" + }, + "offered": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Offered", + "description": "every tool the model was offered" + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordTools" + }, + "RecordUse": { + "properties": { + "rounds": { + "type": "integer", + "title": "Rounds", + "default": 0 + }, + "tool_calls": { + "type": "integer", + "title": "Tool Calls", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordUse" + }, "Recording": { "properties": { "id": { @@ -65680,6 +66282,30 @@ "type": "string", "title": "Summary", "default": "" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms", + "description": "how long the tool took" + }, + "failed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Failed", + "description": "the tool failed (or was unknown or blocked); its summary says why" } }, "additionalProperties": true, diff --git a/threat-model.md b/threat-model.md index 11822785b..c263b536d 100644 --- a/threat-model.md +++ b/threat-model.md @@ -426,6 +426,7 @@ stored text for the consumer's format. | Embed player and `/s/` pages | `api/pages.py` | HTML built from DB values under a strict CSP; framable only per `server.embed_frame_ancestors` | that titles are inert outside this page | *(documented, docs/configuration.md "Content-Security-Policy")* | | Report pages | `api/pages.py` | links re-signed on read for every asset of the namespace in the page | that a report link opens only the report | *(inferred, Q10)* | | Assistant answers, summaries, LLM-written notes | Assistant | model output over untrusted content | that it is grounded or free of injected instructions | *(inferred, Q26)* | +| An answer's record (`meta` on a chat message, `GET /chats/{cid}/messages/{mid}`) | Assistant | the model, the model server's host name (never its address, path, port or key), tool names, scope, counts, tokens, cost and the answer's ledger rows, to the conversation's owner alone | that the model server's host name stays hidden from people who chat | *(documented, docs/assistant.md "never the server's address, path or key")* | | MCP tool results | `api/mcp_tools.py` | the person's readable data, as text, with chat-template control tokens broken up and a note when text reads like instructions to an AI | that an asset's text is safe to follow as instructions | *(documented, docs/mcp.md "sees exactly what that person sees")* | | Error text from remote servers | LLM, speech, rclone, IMAP callers | up to 300 characters of the remote reply and the URL as typed | that internal host names stay hidden | *(inferred, Q18)* | | Webhook bodies | `notify.py` | event data, HMAC-signed for the `webhook` kind only | that Slack or Discord messages are signed | *(documented, domain/notify.py docstring "signed the Standard Webhooks way")* | diff --git a/threat-model.yaml b/threat-model.yaml index 8e3afce75..df167370c 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:d8bda21af9290dd0c018869ca7e0f3b4ce7d78cfcd77d304cb369cd716e42ec5" +prose_version: "threat-model.md@sha256:097d71b310b2fd012ac29bc101532fabad87c9fdd5dacdb9b165a4855d1d50ef" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} From acf97d754568bf7de30a219294e30c5c1c20b71c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:04:31 +0000 Subject: [PATCH 3/3] test(notes): keep the daily notes test from failing on 2026-10-17 The last check asked for 2026-10-17's note after the call without a date had made today's note, so on that day it found the note the deleted root had left behind. It now asks for a day no earlier call makes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/tests/api/test_notebook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastapi_backend/tests/api/test_notebook.py b/fastapi_backend/tests/api/test_notebook.py index 8f3d79e31..40e065d03 100644 --- a/fastapi_backend/tests/api/test_notebook.py +++ b/fastapi_backend/tests/api/test_notebook.py @@ -587,7 +587,7 @@ def test_daily_notes(client, env, db): # editors only, and a date that is one assert client.post("/api/v1/notes/today", headers=hv, json={"ns": "pods"}).status_code == 403 assert client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "soon"}).status_code == 400 - # the daily notes' note, when deleted, is made again the next time + # the daily notes' note, when deleted, is made again the next time (for a day no earlier call made: not today) assert client.delete(f"/api/v1/notes/{root['id']}", headers=he).status_code == 200 - again = client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "2026-10-17"}).json() + again = client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "2020-02-29"}).json() assert again["parent"] not in (None, root["id"])