From ee327cf7e78c0df8a748dcf27404c8216b8293f0 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Fri, 16 Oct 2026 23:26:19 +0000
Subject: [PATCH 01/14] fix(assistant): let the assistant's tools be imported
first
Importing app.domain.ai_tools on its own failed: it imported automations,
which loads the actions, and the agent's action subclasses Toolbox before
ai_tools had defined it. Running one test file (tests/api/test_chats.py)
hit it. ai_tools now imports automations last.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
fastapi_backend/app/domain/ai_tools.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py
index 843562243..9ca9f43fd 100644
--- a/fastapi_backend/app/domain/ai_tools.py
+++ b/fastapi_backend/app/domain/ai_tools.py
@@ -12,7 +12,6 @@
from . import (
auth,
- automations as automationsmod,
batches,
cypher,
entities,
@@ -1861,3 +1860,7 @@ def author_extension(db, tool, args, user, admin, readable, editable):
n = extensions.save_version(db, me, same["id"], manifest, args.get("notes"), origin="chat")
return {"extension": same["id"], "version": n}
return {"extension": extensions.create(db, me, manifest, origin="chat"), "version": 1}
+
+
+# last, because automations imports the actions, and the agent's action (actions_ai.Box) builds on Toolbox above
+from . import automations as automationsmod # noqa: E402
From defb487a21bc621dd7e835bfab4538441706b2a4 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 00:01:11 +0000
Subject: [PATCH 02/14] feat(chat): each answer records what produced it
Every answer keeps `meta`: the model and the host name of its server,
the tools offered and asked for, the scope it answered in (after
auto-scope), its limits and what it used, how it ended (answered,
direct, limit, fallback, stopped, error), tool results that read like
instructions to an AI, how long it took, Lens's version, and tokens and
cost from the ledger rows that name it. Questions keep how they were
sent (web, api, voice, room, automation) and their options.
The answer's id is taken once the question is saved and the answer
runs in activity.scope(chat:, chat_message:), in the web
chat, chat rooms and the automations' agent; the streamed answer holds
the scope across its steps (activity.scoped). App completions keep no
messages, so their record comes back in lens.record. The stream sends
`meta` before `done`; GET /chats/{cid}/messages/{mid} returns one
message with its record and ledger rows. Each tool step keeps its time
and whether it failed. The web app shows it all from an i in the
answer's header (a sheet on a phone); old answers say they weren't
recorded.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
CHANGELOG.md | 5 +
docs/api.md | 8 +-
docs/assistant.md | 36 +-
docs/database.md | 5 +-
fastapi_backend/app/api/v1/routes/activity.py | 4 +
fastapi_backend/app/api/v1/routes/chats.py | 65 +-
.../app/api/v1/routes/completions.py | 8 +-
fastapi_backend/app/domain/actions_ai.py | 32 +-
fastapi_backend/app/domain/activity.py | 21 +
fastapi_backend/app/domain/ai_tools.py | 4 +
fastapi_backend/app/domain/bridge.py | 103 +--
fastapi_backend/app/domain/chat.py | 84 ++-
fastapi_backend/app/domain/completions.py | 21 +-
fastapi_backend/app/schemas/chats.py | 70 ++
fastapi_backend/tests/api/test_actions_ai.py | 6 +-
fastapi_backend/tests/api/test_bridge.py | 6 +-
.../tests/api/test_chat_records.py | 178 +++++
fastapi_backend/tests/api/test_completions.py | 7 +
.../tests/api/test_prompt_injection.py | 1 +
fastapi_backend/tests/fake_llm.py | 5 +-
.../__tests__/chat-details.test.tsx | 168 +++++
nextjs-frontend/app/openapi-client/index.ts | 12 +
nextjs-frontend/app/openapi-client/sdk.gen.ts | 22 +-
.../app/openapi-client/types.gen.ts | 325 +++++++++
nextjs-frontend/components/chat/answer.tsx | 8 +-
nextjs-frontend/components/chat/chat-app.tsx | 8 +-
nextjs-frontend/components/chat/details.tsx | 201 ++++++
nextjs-frontend/components/chat/steps.tsx | 9 +-
nextjs-frontend/components/chat/stream.ts | 30 +-
nextjs-frontend/components/costs/costs.tsx | 2 +-
nextjs-frontend/openapi.json | 628 +++++++++++++++++-
threat-model.md | 1 +
threat-model.yaml | 2 +-
33 files changed, 1974 insertions(+), 111 deletions(-)
create mode 100644 fastapi_backend/tests/api/test_chat_records.py
create mode 100644 nextjs-frontend/__tests__/chat-details.test.tsx
create mode 100644 nextjs-frontend/components/chat/details.tsx
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08bbfb731..184fa7b55 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now;
Settings → Automations switches them all off and sets how far they may go
([Running automations](docs/automations.md#running-automations)).
+- **Each answer says what produced it.** The *i* in an answer's header shows the model and the host it ran on, the
+ tools it was offered and used, the scope, its limits and what it used, tokens, cost (an estimate from your prices),
+ how long it took, how it ended and which tool results read like instructions to an AI, with the calls made for it.
+ Agents read the same record over the API (`GET /api/v1/chats//messages/`), and each tool step shows how long
+ it took. Answers from before have no record ([What each answer records](docs/assistant.md#what-each-answer-records)).
- **Automations for agents.** Agents list, save, try and run automations through the MCP tools `automations`,
`automation_actions`, `save_automation`, `test_automation` and `run_automation`, with problems in plain words. The
assistant saves them switched off and asks before switching one on or running it
diff --git a/docs/api.md b/docs/api.md
index 6e8f3bec8..beed20cb5 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -477,7 +477,8 @@ read there:
What it cites comes back three ways, so any app can show it: `[n]` in the text with a **Sources** list of links after
it (left out when the request has a `response_format`), `citations` (the link for `[n]` is the n-th), and
`lens.sources` (`n`, `recording_id`, `title`, `time`, `speaker`, `url`, `cited`). `lens.steps` lists the tools the
-model used. With `"stream": true` the answer comes as OpenAI's `chat.completion.chunk` events, ending with
+model used, and `lens.record` what produced the answer, as a chat answer's record
+([What each answer records](assistant.md#what-each-answer-records)) without tokens and cost. With `"stream": true` the answer comes as OpenAI's `chat.completion.chunk` events, ending with
`data: [DONE]`; the last chunk before it carries `finish_reason: "stop"`, `citations` and `lens`. While the tools run,
the stream sends SSE comments, so nothing in between closes it.
@@ -1448,6 +1449,7 @@ GET /api/v1/chats/{cid}
PATCH /api/v1/chats/{cid}
DELETE /api/v1/chats/{cid}
POST /api/v1/chats/{cid}/messages
+GET /api/v1/chats/{cid}/messages/{mid}
POST /api/v1/chats/{cid}/stop
POST /api/v1/chats/{cid}/messages/{mid}/check
GET /api/v1/approvals
@@ -1469,7 +1471,9 @@ whether it uses tools (`tools`, `max_steps`), whether answers can be checked (`c
(the configured one first; `llm.chat_models`, else the server's list); never the model server's address or key. A
conversation's `model` (`POST /chats`, `PATCH /chats/{cid}`; null for the configured one) answers in it, and a
question can name another (`POST /chats/{cid}/messages {content, model}`, e.g. to retry); a model that isn't offered is
-a 400. Each answer records the `model` that wrote it. `POST /chats/{cid}/stop` stops the answer being written in your conversation after the piece or tool
+a 400. Each answer records the `model` that wrote it, and its `meta` what produced it; `GET
+/chats/{cid}/messages/{mid}` gives one message with its record and its rows in the activity ledger
+([What each answer records](assistant.md#what-each-answer-records)). `POST /chats/{cid}/stop` stops the answer being written in your conversation after the piece or tool
step it's on: the stream sends `stopped`, then `done` with the saved message, whose `stopped` is true and whose
`content` is what came before (`(stopped)` when nothing had). `{stopping: false}` when nothing was being written.
`POST /chats/{cid}/messages {content, edit}` edits one of your earlier questions (`edit` is its message id): it and
diff --git a/docs/assistant.md b/docs/assistant.md
index bacf0168f..02dc202ba 100644
--- a/docs/assistant.md
+++ b/docs/assistant.md
@@ -13,11 +13,12 @@ then.
numbered.
- **Answers:** the model answers only from those excerpts and cites them as [n]. Each citation carries the asset
and timestamp.
-- **Streaming:** answers arrive over server-sent events: `passages`, then `token`s, then `done`.
+- **Streaming:** answers arrive over server-sent events: `passages`, then `token`s, then `meta` (the answer's
+ record, [below](#what-each-answer-records)) and `done`.
- **Stopping:** Stop asks the server to end the answer after the piece or tool step it's on (whichever server process
is writing it); what came before is saved, marked stopped. A model call already under way finishes first.
- **Reopening:** a conversation's answers keep the tools the assistant used (and with what), any notice (the model
- couldn't use tools), the error when there was no answer, and their latest source check.
+ couldn't use tools), the error when there was no answer, their latest source check and what produced them.
- **Editing a question:** Edit on a question you asked asks it again as edited; the answer and everything after it
are replaced (`POST /api/v1/chats//messages` with `edit`).
- **Long conversations:** the model reads the latest six messages word for word. Once eight more have built up
@@ -51,6 +52,37 @@ then.
- **No model configured:** chat returns the best-matching passages instead. Anyone signed in can see whether a model
is set up, and which (`GET /chats/capabilities`), so the app says so before the first question.
+### What each answer records
+
+Every answer keeps a record of what produced it (`meta` on the message), so a person can trust and debug it and an
+agent can read it. The **i** in an answer's header (Details) shows it, with the calls made for it and what the
+conversation has cost this month; on a phone it opens as a sheet.
+
+| Field | What it says |
+| --- | --- |
+| `model`, `provider` | the model that wrote it, and the host name of the model server it ran on (never the server's address, path or key) |
+| `tools` | `offered`: every tool the model was offered; `picked`: the tools the question asked for (null: all it may use) |
+| `scope` | the scope it answered in, after a conversation over everything was narrowed to a namespace |
+| `limits`, `used` | the rounds of tool calls it had (and the tool calls, when they're limited), and how many it used |
+| `outcome` | `answered` (after looking things up), `direct` (without looking anything up), `limit` (its rounds ran out, so it answered from what it found), `fallback` (from a search: the model can't use tools, or none is set up), `stopped` or `error` |
+| `tokens`, `cost_usd`, `cost_estimated` | the tokens of the model calls made for it, and what they cost from the prices in Settings ([Costs](activity.md#costs)); `cost_estimated` when some had a price but no token counts, so it cost at least this |
+| `ms` | how long it took, from the question |
+| `flagged` | tools whose results read like instructions to an AI ([below](#third-party-content-and-prompt-injection)) |
+| `version` | the version of Lens that wrote it |
+
+Each tool step also keeps how long it took (`ms`) and whether it failed (`failed`). A question keeps how it was sent
+(`via`: `web`, `api`, `voice`, `room` or `automation`) and the options it was asked with (`options`, such as another
+`model`).
+
+An answer is written inside the activity ledger's scope `chat_message:` (with `chat:`), so every model and
+tool call made for it names it ([Activity](activity.md)), and its record's tokens and cost are theirs: one ledger
+total per answer, and no model call. `GET /api/v1/chats/` returns the record on each message, and
+`GET /api/v1/chats//messages/` returns one message with its record and those ledger rows (`ledger`).
+Answers in chat rooms ([Chat rooms](chat-rooms.md)) and the automations' agent record theirs the same way. An app's
+chat completions keep no messages, so their record comes back in the reply's `lens.record`, without tokens and cost
+([Chat completions](api.md#chat-completions-for-any-app)). Answers from before Lens kept records have none; Details
+shows what they have and says the rest wasn't recorded.
+
### Picking the namespace for a conversation over everything
When a conversation has no scope (the assistant home starts it over everything), its first question goes to the
diff --git a/docs/database.md b/docs/database.md
index 7473ad304..d6a8d1c59 100644
--- a/docs/database.md
+++ b/docs/database.md
@@ -87,7 +87,10 @@ optional, and older resources simply don't have them.
## Upgrades
Installs keep their data across updates (running `install.sh` again, a new image, `git pull`). New tables, fields and
-indexes arrive through `SCHEMA` on the next start. Anything else, such as rewriting stored values, moving data between
+indexes arrive through `SCHEMA` on the next start. A field that is only added needs no step when its readers treat it
+as missing on older rows: a chat message's record (`meta`,
+[What each answer records](assistant.md#what-each-answer-records)) is one, and messages from before it simply have
+none. Anything else, such as rewriting stored values, moving data between
tables, or changing or removing an index that already exists, is a named step in `app/domain/migrations.py`:
* Every process that opens the database (the API, workers, `lens watch`, `lens` commands) runs the steps the database
diff --git a/fastapi_backend/app/api/v1/routes/activity.py b/fastapi_backend/app/api/v1/routes/activity.py
index ede0375f6..9c6f68399 100644
--- a/fastapi_backend/app/api/v1/routes/activity.py
+++ b/fastapi_backend/app/api/v1/routes/activity.py
@@ -48,6 +48,10 @@ def _may_see(db: Db, user: CurrentUser, acl: Acl, resource: str) -> None:
return
if table == "chat" and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", int(key)), a=user.id):
return
+ if table == "chat_message": # a message in one of your conversations
+ m = db.one("SELECT chat FROM $r", r=activity.R("chat_message", int(key)))
+ if m and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", m["chat"]), a=user.id):
+ return
raise HTTPException(403, "only admins see this resource's activity")
diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py
index aadedbd19..d0791f482 100644
--- a/fastapi_backend/app/api/v1/routes/chats.py
+++ b/fastapi_backend/app/api/v1/routes/chats.py
@@ -1,7 +1,7 @@
"""Conversations with the archive, the assistant's approvals, and checking an answer against its sources.
A question streams back server-sent events: ``scoped`` (the namespace a conversation over everything was narrowed
-to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error`` and ``done``.
+to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error``, ``meta`` (the answer's record) and ``done``.
The assistant only ever sees, and cites, what the asker can read.
"""
@@ -19,7 +19,7 @@
from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Principal, Writer
from app.api.media import sign_urls
from app.api.v1.routes.collections import own_collection
-from app.domain import ai_tools, approvals, auth, auto_scope, browser, chat, llm, store, uploads
+from app.domain import activity, ai_tools, approvals, auth, auto_scope, browser, chat, llm, store, uploads
from app.domain.store import DB
from app.schemas.chats import (
AnswerCheck,
@@ -34,6 +34,7 @@
ChatSummary,
ChatUpdate,
MessageCreate,
+ MessageDetail,
StopResult,
)
from app.schemas.common import Created, Ok
@@ -129,6 +130,17 @@ def get_chat(cid: int, user: CurrentUser, acl: Acl, db: Db) -> Chat:
return Chat.model_validate(sign_urls({**c, "messages": msgs}, full=True))
+@router.get("/chats/{cid}/messages/{mid}")
+def get_message(cid: int, mid: int, user: CurrentUser, acl: Acl, db: Db) -> MessageDetail:
+ """One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and
+ used, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`:
+ the calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}."""
+ m = next((x for x in get_chat(cid, user, acl, db).messages if x.id == mid), None)
+ if m is None:
+ raise HTTPException(404, "not found")
+ return MessageDetail.model_validate({**m.model_dump(by_alias=True), "ledger": activity.history(db, f"chat_message:{mid}", 200)})
+
+
@router.patch("/chats/{cid}")
def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Ok:
"""Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one)."""
@@ -177,7 +189,8 @@ def _ev(name: str, data: Any) -> str:
async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> StreamingResponse:
"""Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model
reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST
- /chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id).
+ /chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced
+ it), done (the saved message id).
With `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and
its new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked
@@ -211,6 +224,8 @@ def run() -> Iterator[str]:
model = await run_in_threadpool(_model, cfg, body.model) if body.model else c.get("model")
if model and not body.model and model not in await run_in_threadpool(chat.model_choices, cfg):
model = None
+ record = chat.Record(cfg, model) # what produced the answer, kept with it (from here: it times the answer)
+ mid = 0 # the answer's id, taken once the question is saved
title = c["title"]
page = body.context.model_dump() if body.context else None
kept = chat.shared_context(page) # what the question keeps of the page
@@ -229,14 +244,21 @@ def run() -> Iterator[str]:
summary: dict[str, Any] | None = None # what compaction keeps of the earlier messages (chat.compact)
def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
- nonlocal summary
+ nonlocal summary, mid
try: # a conversation that fills the model's context is folded into its summary before it's answered
chat.compact(db, cfg, cid, full_only=True)
except Exception: # noqa: BLE001 - answered with what fits; tried again after the answer
log.warning("chat %s: couldn't update the summary", cid, exc_info=True)
past = chat.history(db, cid)
summary = chat.memory(db, cfg, cid)
- chat.add(db, cid, "user", q, attachments=files, context=kept)
+ via = "voice" if body.via == "voice" else "web" if user.via == "access" else "api"
+ chat.add(db, cid, "user", q, attachments=files, context=kept, meta=chat.question_record(via, model=body.model))
+ mid = db.next_id("chat_message")
+ # what's worked out for the answer from here (the excerpts, the namespace it's about) counts for it in the ledger
+ with activity.scope(db, f"chat:{cid}", f"chat_message:{mid}", cfg=cfg):
+ return draw_on(past)
+
+ def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
if not past and title == "New conversation":
db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=q[:80])
# A conversation over everything gets the namespace its first question is about, when that's clear.
@@ -251,6 +273,7 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
return past, chat.retrieve(db, q, readable, c["scope"], cfg=cfg), placed
past, passages, placed = await run_in_threadpool(prepare)
+ record.scope = c.get("scope") or {} # after a conversation over everything was narrowed to what it's about
picked = placed[1] if placed and placed[0] == "scoped" else None
suggested = placed[1] if placed and placed[0] == "suggest" else None
@@ -265,13 +288,17 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
)
notice: str | None = None
- def save(text: str, cited: list[dict[str, Any]], **kw: Any) -> int:
+ def save(text: str, cited: list[dict[str, Any]], outcome: str = "answered", **kw: Any) -> Iterator[str]:
+ """The answer saved with its record: `meta`, then `done` (its id)."""
wrote = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None
- return int(chat.add(db, cid, "assistant", text, cited, steps=steps, notice=notice, model=wrote, **kw))
+ meta = record.meta(db, outcome, f"chat_message:{mid}")
+ chat.add(db, cid, "assistant", text, cited, steps=steps, notice=notice, model=wrote, mid=mid, meta=meta, **kw)
+ yield _ev("meta", meta)
+ yield _ev("done", {"message": mid})
def stopped(text: str, cited: list[dict[str, Any]]) -> Iterator[str]:
yield _ev("stopped", {"message": "Stopped"})
- yield _ev("done", {"message": save(text or "(stopped)", cited, stopped=True)})
+ yield from save(text or "(stopped)", cited, "stopped", stopped=True)
def answer(on: chat.Answering) -> Iterator[str]:
nonlocal notice
@@ -294,10 +321,10 @@ def answer(on: chat.Answering) -> Iterator[str]:
browser=body.browser,
)
try:
- answer = ""
+ answer, how = "", "answered"
# setting the server up, or working in the person's browser, takes more steps than looking things up
steps_max = max(cfg["ai"].get("max_steps") or 6, 12 if setup else 0, browser.steps(cfg) if box.browser else 0)
- for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary):
+ for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record):
if kind == "step":
steps.append(data)
yield _ev("step", data)
@@ -311,7 +338,7 @@ def answer(on: chat.Answering) -> Iterator[str]:
elif kind == "direct" and passages and not setup and not (page and page.get("text")) and not box.cited(data):
break
else:
- answer = data
+ answer, how = data, "direct" if kind == "direct" else "answered"
else:
box.after_answer(answer)
for appr in box.approvals:
@@ -319,23 +346,25 @@ def answer(on: chat.Answering) -> Iterator[str]:
cited = box.cited(answer)
yield _ev("passages", cited)
yield _ev("token", {"text": answer})
- yield _ev("done", {"message": save(answer or "(no answer)", cited)})
+ yield from save(answer or "(no answer)", cited, how)
return
except llm.ToolsUnsupported:
notice = "This model can't use tools, so the answer comes from a search instead."
yield _ev("notice", {"message": notice})
except llm.LLMError as e:
yield _ev("error", {"message": str(e)})
- yield _ev("done", {"message": save("(no answer)", [], error=str(e))})
+ yield from save("(no answer)", [], "error", error=str(e))
return
yield _ev("passages", passages)
text, error = "", None
try:
if llm.configured(cfg):
- for piece in llm.stream_chat(cfg, chat.messages_for(asked, passages, past, summary, cfg), model=model):
+ pieces = llm.stream_chat(cfg, chat.messages_for(asked, passages, past, summary, cfg), model=model)
+ for piece in pieces:
text += piece
yield _ev("token", {"text": piece})
if on.stop_requested():
+ pieces.close() # the call ends here, so its ledger row is in the answer's record
yield from stopped(text, chat.cited(text, passages))
return
else:
@@ -344,7 +373,8 @@ def answer(on: chat.Answering) -> Iterator[str]:
except llm.LLMError as e:
error = str(e)
yield _ev("error", {"message": error})
- yield _ev("done", {"message": save(text or "(no answer)", chat.cited(text, passages), error=error)})
+ how = "error" if error else "fallback" if notice or not llm.configured(cfg) else "answered"
+ yield from save(text or "(no answer)", chat.cited(text, passages), how, error=error)
def gen() -> Iterator[str]:
on = chat.Answering(db, cid)
@@ -355,7 +385,8 @@ def gen() -> Iterator[str]:
yield _ev("step", steps[0])
if suggested:
yield _ev("suggested", {"namespaces": suggested})
- for ev in answer(on):
+ # every call made for the answer counts for it in the ledger (the scope holds across the stream's steps)
+ for ev in activity.scoped(answer(on), db, f"chat:{cid}", f"chat_message:{mid}", cfg=cfg):
saved = saved or ev.startswith("event: done")
yield ev
except Exception: # an answer that breaks off still ends with an error and is saved, so the question isn't left unanswered
@@ -363,7 +394,7 @@ def gen() -> Iterator[str]:
message = "Something went wrong while answering. Try again."
yield _ev("error", {"message": message})
if not saved:
- yield _ev("done", {"message": save("(no answer)", [], error=message)})
+ yield from save("(no answer)", [], "error", error=message)
finally:
on.end()
try: # once enough has built up, older messages are folded into the summary (after the answer, so it waits on nothing)
diff --git a/fastapi_backend/app/api/v1/routes/completions.py b/fastapi_backend/app/api/v1/routes/completions.py
index ef9a2b49c..0b7137daf 100644
--- a/fastapi_backend/app/api/v1/routes/completions.py
+++ b/fastapi_backend/app/api/v1/routes/completions.py
@@ -123,7 +123,13 @@ def create_chat_completion(name: str, body: ChatCompletionRequest, request: Requ
def extras(done: dict[str, Any], steps: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
found = completions.sources(web, done["text"], done["refs"])
- lens = {"namespace": name, "sources": found, "steps": steps, **({"notice": done["notice"]} if done.get("notice") else {})}
+ lens = {
+ "namespace": name,
+ "sources": found,
+ "steps": steps,
+ "record": done.get("record"),
+ **({"notice": done["notice"]} if done.get("notice") else {}),
+ }
return found, {"citations": [s["url"] for s in found], "lens": lens}
if not body.stream:
diff --git a/fastapi_backend/app/domain/actions_ai.py b/fastapi_backend/app/domain/actions_ai.py
index 05b9c0691..dc37e1cb5 100644
--- a/fastapi_backend/app/domain/actions_ai.py
+++ b/fastapi_backend/app/domain/actions_ai.py
@@ -257,7 +257,10 @@ def _ask_agent(db, cfg, intent, user, dry):
if about:
url = (about.get("object") or {}).get("url") if isinstance(about.get("object"), dict) else None
page = {"url": str(url or f"/timeline?activity={about.get('id') or ''}"), "title": _short(happened, 300)}
- chat.add(db, cid, "user", instructions, context=page, origin=box.origin)
+ chat.add(db, cid, "user", instructions, context=page, origin=box.origin, meta=chat.question_record("automation", tools=tools or None))
+ # the answer's id up front: every call made for it names it in the ledger, and its record sums them
+ mid = db.next_id("chat_message")
+ record, spent = chat.Record(cfg, scope=box.scope, picked=tools or None), f"chat_message:{mid}"
# reading the activity counts as reading outside text for this conversation from now on (ai_tools.read_before)
steps = (
[{"tool": "automation_event", "args": {"id": about.get("id")}, "summary": f"Read what happened: {_short(happened, 200)}"}]
@@ -265,21 +268,28 @@ def _ask_agent(db, cfg, intent, user, dry):
else []
)
question = instructions + (_context(about, cfg) if about else "")
- text = None
+ text, how = None, "answered"
+
+ def said(words, cited, outcome, **kw):
+ """The answer, saved with its record."""
+ chat.add(db, cid, "assistant", words or "(no answer)", cited, steps=steps, mid=mid, meta=record.meta(db, outcome, spent), **kw)
+
try:
- for kind, data in chat.tool_answer(cfg, box, question, (), steps_max):
- if kind == "step":
- steps.append(data)
- else:
- text = data
+ with activity.scope(db, f"chat:{cid}", spent, cfg=cfg):
+ for kind, data in chat.tool_answer(cfg, box, question, (), steps_max, record=record):
+ if kind == "step":
+ steps.append(data)
+ else:
+ text, how = data, "direct" if kind == "direct" else "answered"
except llm.ToolsUnsupported:
- chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error="this model can't use tools")
+ said("", [], "error", error="this model can't use tools")
raise ValueError("this model can't use tools, so the agent can't act: pick one that can in Settings › Model") from None
except (llm.LLMError, activity.OverBudget) as e:
- chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error=str(e))
+ said("", [], "error", error=str(e))
raise ValueError(f"the agent couldn't answer: {e}") from None
- box.after_answer(text or "")
- chat.add(db, cid, "assistant", text or "(no answer)", box.cited(text), steps=steps, model=cfg["llm"].get("model"))
+ with activity.scope(db, f"chat:{cid}", spent, cfg=cfg):
+ box.after_answer(text or "")
+ said(text, box.cited(text), how, model=cfg["llm"].get("model"))
return {
"answer": text or "",
"chat": cid,
diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py
index c09c9a053..89f91c460 100644
--- a/fastapi_backend/app/domain/activity.py
+++ b/fastapi_backend/app/domain/activity.py
@@ -47,6 +47,7 @@
"workflows": "workflow",
"custom-nodes": "custom_node",
"chats": "chat",
+ "messages": "chat_message", # a conversation's messages/{mid}
"jobs": "job",
"namespaces": "space",
"spaces": "space",
@@ -113,6 +114,26 @@ def scope(db, *refs, cfg=None):
_scope.reset(token)
+def scoped(items, db, *refs, cfg=None):
+ """`items` (a generator) with each of its steps run inside scope(db, *refs). A `with scope(...)` inside a generator
+ doesn't hold across its yields when a streamed response iterates it from the threadpool: each step runs in a fresh
+ copy of the request's context."""
+ it = iter(items)
+ try:
+ while True:
+ with scope(db, *refs, cfg=cfg):
+ try:
+ item = next(it)
+ except StopIteration:
+ return
+ yield item
+ finally:
+ close = getattr(it, "close", None)
+ if close:
+ with scope(db, *refs, cfg=cfg):
+ close()
+
+
def current():
"""The resources calls made here count for, in order, without repeats."""
s = _scope.get() or {}
diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py
index 9ca9f43fd..63151f162 100644
--- a/fastapi_backend/app/domain/ai_tools.py
+++ b/fastapi_backend/app/domain/ai_tools.py
@@ -532,6 +532,7 @@ def __init__(
self.readable = {s for s in self.readable if names.get(s) in self.scope["namespaces"]}
self.allowed = recsets.within(db, self.readable, self.scope.get("recordings"), self.scope.get("collections"))
self.refs, self.reads, self.approvals = [], 0, []
+ self.flagged, self.failed = [], False # tools whose results read like instructions to an AI; the last call failed
# the extensions this person switched on or was given (extensions.py); hooks don't run inside hooks
roles = {s: "editor" if s in self.editable else "viewer" for s in self.readable}
self.me = extensions.who(self.account, user.get("email"), admin, roles)
@@ -688,6 +689,7 @@ def call(self, name, args, model=False):
"""A tool's result (JSON) and the step's summary. `model`: the result as the model reads it, third-party text
marked as data (untrusted), with what hooks add after it, unmarked."""
offered = {s["function"]["name"] for s in self.specs()}
+ self.failed = True # until it runs: a tool that's unknown or blocked failed
try:
if name not in offered:
raise AttributeError(name)
@@ -703,6 +705,7 @@ def call(self, name, args, model=False):
extra += self._hook(h, {"said": self.said, "tool": name})
asked = len(self.approvals)
result, summary = self._run(name, fn, args)
+ self.failed = result.startswith('{"error"')
self.links |= untrusted.links(result)
if self.untrusted(name) and len(self.approvals) == asked: # it read something, rather than asking for a yes
self.read_outside(f"tool {name}")
@@ -714,6 +717,7 @@ def call(self, name, args, model=False):
why = untrusted.flags(result, self.cfg, model=True)
if why: # the person sees it on the step, the model on the marker, admins in the audit log
summary = f"{summary} (reads like instructions to an AI: {'; '.join(why)})"
+ self.flagged.append(name)
auth.audit(self.db, self.user, "assistant.injection_flagged", f"chat:{self.chat}", {"tool": name, "why": why})
result = untrusted.wrap(result, f"tool {name}", self.cfg, why=why)
return result + (f"\n\n{extra.strip()}" if extra else ""), summary
diff --git a/fastapi_backend/app/domain/bridge.py b/fastapi_backend/app/domain/bridge.py
index 491b2f4f7..cf995fba3 100644
--- a/fastapi_backend/app/domain/bridge.py
+++ b/fastapi_backend/app/domain/bridge.py
@@ -38,7 +38,7 @@
import urllib.error
import urllib.request
-from . import ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted
+from . import activity, ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted
from .store import R
log = logging.getLogger(__name__)
@@ -330,53 +330,64 @@ def _answer(db, cfg, base, account, cid, q, files, act_on, outside):
past = chat.history(db, cid)
summary = chat.memory(db, cfg, cid)
scope = (db.one("SELECT scope FROM $r", r=R("chat", cid)) or {}).get("scope") or None
- chat.add(db, cid, "user", q, attachments=list(files) or None)
- asked = q + chat.attached_note(list(files))
- passages = chat.retrieve(db, q, readable, scope, cfg=cfg)
- # the web addresses written by someone other than the model: the person, now and before, and the archive
- keep = untrusted.links("\n".join([q] + [m["content"] or "" for m in past if m["role"] == "user"] + [p["text"] for p in passages]))
- steps, approvals, notice = [], [], None
- wrote = cfg["llm"].get("model") if llm.configured(cfg) else None
- if llm.configured(cfg) and cfg["ai"].get("tools"):
- box = ai_tools.Toolbox(
- db,
- cfg,
- {"id": account["id"], "email": account["email"]},
- readable,
- editable,
- scope,
- cid,
- base,
- account["admin"],
- said=q,
- outside=list(outside) + ai_tools.read_before(past, summary),
- act_on=act_on,
- )
+ chat.add(db, cid, "user", q, attachments=list(files) or None, meta=chat.question_record("room"))
+ # the answer's id up front: every call made for it names it in the ledger, and its record sums them
+ mid = db.next_id("chat_message")
+ record, spent = chat.Record(cfg, scope=scope), f"chat_message:{mid}"
+ with activity.scope(db, f"chat:{cid}", spent, cfg=cfg):
+ asked = q + chat.attached_note(list(files))
+ passages = chat.retrieve(db, q, readable, scope, cfg=cfg)
+ # the web addresses written by someone other than the model: the person, now and before, and the archive
+ keep = untrusted.links("\n".join([q] + [m["content"] or "" for m in past if m["role"] == "user"] + [p["text"] for p in passages]))
+ steps, approvals, notice = [], [], None
+ wrote = cfg["llm"].get("model") if llm.configured(cfg) else None
+
+ def said(words, cited, outcome, **kw):
+ """The answer, saved with its record."""
+ meta = record.meta(db, outcome, spent)
+ chat.add(db, cid, "assistant", words or "(no answer)", cited, steps=steps, model=wrote, mid=mid, meta=meta, **kw)
+
+ if llm.configured(cfg) and cfg["ai"].get("tools"):
+ box = ai_tools.Toolbox(
+ db,
+ cfg,
+ {"id": account["id"], "email": account["email"]},
+ readable,
+ editable,
+ scope,
+ cid,
+ base,
+ account["admin"],
+ said=q,
+ outside=list(outside) + ai_tools.read_before(past, summary),
+ act_on=act_on,
+ )
+ try:
+ text, how = None, "answered"
+ for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary, record=record):
+ if kind == "step":
+ steps.append(data)
+ elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers
+ break
+ else:
+ text, how = data, "direct" if kind == "direct" else "answered"
+ if text is not None:
+ box.after_answer(text)
+ said(text, box.cited(text), how)
+ return text, box.approvals, keep | box.links
+ except llm.ToolsUnsupported:
+ notice = "This model can't use tools, so the answer comes from a search instead."
+ except llm.LLMError as e:
+ said("", [], "error", error=str(e))
+ return "I couldn't answer that: " + str(e), box.approvals, keep | box.links
+ error = None
try:
- text = None
- for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary):
- if kind == "step":
- steps.append(data)
- elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers
- break
- else:
- text = data
- if text is not None:
- box.after_answer(text)
- chat.add(db, cid, "assistant", text or "(no answer)", box.cited(text), steps=steps, model=wrote)
- return text, box.approvals, keep | box.links
- except llm.ToolsUnsupported:
- notice = "This model can't use tools, so the answer comes from a search instead."
+ text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary, cfg)) if llm.configured(cfg) else chat.fallback(passages)
except llm.LLMError as e:
- chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error=str(e), model=wrote)
- return "I couldn't answer that: " + str(e), box.approvals, keep | box.links
- error = None
- try:
- text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary, cfg)) if llm.configured(cfg) else chat.fallback(passages)
- except llm.LLMError as e:
- text, error = "I couldn't answer that: " + str(e), str(e)
- chat.add(db, cid, "assistant", text or "(no answer)", chat.cited(text, passages), steps=steps, notice=notice, error=error, model=wrote)
- return text, approvals, keep
+ text, error = "I couldn't answer that: " + str(e), str(e)
+ how = "error" if error else "fallback" if notice or not llm.configured(cfg) else "answered"
+ said(text, chat.cited(text, passages), how, notice=notice, error=error)
+ return text, approvals, keep
# ---------- approving in the room ----------
diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py
index a3b3180e4..035bde9a2 100644
--- a/fastapi_backend/app/domain/chat.py
+++ b/fastapi_backend/app/domain/chat.py
@@ -12,8 +12,9 @@
import re
import time
from collections import Counter, defaultdict
+from urllib.parse import urlsplit
-from . import llm, recsets, render, semantic, store, textindex, untrusted
+from . import __version__, activity, llm, recsets, render, semantic, store, textindex, untrusted
R = store.R
STOP = set(
@@ -345,7 +346,7 @@ def get(db, cid, account):
def history(db, cid):
return db.rows(
"SELECT record::id(id) AS id, role, content, passages, created_at, stopped ?? false AS stopped, steps ?? [] AS steps, "
- "attachments ?? [] AS attachments, notice, error, check, model, context FROM chat_message WHERE chat = $c ORDER BY id",
+ "attachments ?? [] AS attachments, notice, error, check, model, context, meta FROM chat_message WHERE chat = $c ORDER BY id",
c=cid,
)
@@ -364,11 +365,15 @@ def add(
attachments=None,
context=None,
origin=None,
+ mid=None,
+ meta=None,
):
"""Save a message; an answer keeps the tool steps it took, any notice (e.g. the model can't use tools) and error; a
question asked from a page keeps what it shared of it (shared_context). A question an automation asked keeps its
- `origin` ({automation, run, node}), so it isn't taken for a person's message (`chat.message`)."""
- mid = db.next_id("chat_message")
+ `origin` ({automation, run, node}), so it isn't taken for a person's message (`chat.message`). `meta` is its record
+ (Record; a question's: how it was sent), `mid` its id when it was taken up front (an answer's, so the ledger rows
+ of the calls made for it name it)."""
+ mid = mid or db.next_id("chat_message")
db.q(
"CREATE $r CONTENT $d",
r=R("chat_message", mid),
@@ -387,6 +392,7 @@ def add(
"model": model,
"context": context,
"origin": origin or None,
+ "meta": meta or None,
}
),
)
@@ -585,6 +591,61 @@ def request_stop(db, cid):
return bool(db.rows("UPDATE $r SET stop_requested = true WHERE answering = true RETURN id", r=R("chat", cid)))
+def provider(cfg):
+ """The model server's host name, never its address's path or a key: where an answer's model ran."""
+ return (urlsplit(cfg["llm"].get("base_url") or "").hostname or None) if llm.configured(cfg) else None
+
+
+class Record:
+ """What produced an answer, kept on it as `meta` (docs/assistant.md, "What each answer records"): the model and the
+ host it ran on, the tools it was offered and the ones asked for, the scope it answered in, its limits and what it
+ used, how it ended, the tools whose results read like instructions to an AI, how long it took, and its tokens and
+ cost, from the ledger rows that name it (the answer runs in activity.scope(db, "chat_message:")). Made as the
+ answer starts, which it times from; tool_answer fills in the tools and counts."""
+
+ def __init__(self, cfg, model=None, scope=None, picked=None, tool_calls=None, research=None):
+ self.cfg, self.t0 = cfg, time.monotonic()
+ self.model = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None
+ self.scope, self.picked, self.research = scope or {}, picked, research
+ self.limits = {"rounds": None, "tool_calls": tool_calls}
+ self.used = {"rounds": 0, "tool_calls": 0}
+ self.offered, self.box, self.limited = [], None, False
+
+ def offer(self, toolbox, specs, rounds):
+ self.box, self.offered, self.limits["rounds"] = toolbox, [s["function"]["name"] for s in specs], rounds
+
+ def meta(self, db, outcome, resource=None):
+ """The record, once the answer ended with `outcome` (answered, direct, fallback, stopped or error; an answer
+ given because the rounds ran out is `limit`), with the tokens and cost of the ledger rows that name `resource`
+ (none without one)."""
+ if outcome in ("answered", "direct") and self.limited:
+ outcome = "limit"
+ out = {
+ "model": self.model,
+ "provider": provider(self.cfg),
+ "tools": {"picked": self.picked, "offered": self.offered},
+ "scope": self.scope,
+ "limits": dict(self.limits),
+ "used": dict(self.used),
+ "outcome": outcome,
+ "ms": int((time.monotonic() - self.t0) * 1000),
+ "flagged": list(dict.fromkeys(getattr(self.box, "flagged", None) or [])),
+ "research": self.research,
+ "version": __version__,
+ }
+ if resource:
+ spent = activity.totals(db, resource)
+ out["tokens"] = {"in": spent["tokens_in"], "out": spent["tokens_out"]}
+ out["cost_usd"], out["cost_estimated"] = spent["cost_usd"], spent["estimate"]
+ return out
+
+
+def question_record(via, **options):
+ """A question's record: how it was sent (web, api, room, voice, automation or completions) and the options it was
+ asked with."""
+ return {"via": via, "options": store.clean(options)}
+
+
TOOL_SYSTEM = (
SYSTEM + " You can call tools to look things up in the archive. Cite moments with the [n] numbers the tools return, and "
"only those. Tools that run work or change data need the person's approval: say what you proposed and that it's waiting for them. "
@@ -679,11 +740,12 @@ def _finish(cfg, msgs, tools, model):
return text
-def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None):
+def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None):
"""The tool loop: yields ("step", {...}) for each tool call, then ("answer", text), or ("direct", text) when the model
answered without looking anything up (it never saw the archive, so the caller can answer from a search instead).
Every turn ends in an answer: when the steps run out, or the model replies with nothing, it's asked to answer from
- what it found, with the tools off. Raises llm.ToolsUnsupported."""
+ what it found, with the tools off. Raises llm.ToolsUnsupported. `record` (a Record) learns the tools offered and
+ counts the rounds and tool calls."""
system = TOOL_SYSTEM + untrusted.rules(cfg) + ("\n\n" + SETUP_SYSTEM if setup else "")
note = getattr(toolbox, "system_note", None) # skills, and context from hooks (extensions.py)
system += note() if note else ""
@@ -695,6 +757,8 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set
msgs = [{"role": "system", "content": system}] + past_turns(history, summary=summary, cfg=cfg)
msgs.append({"role": "user", "content": question})
tools, looked, made = toolbox.specs(), False, set()
+ record = record or Record(cfg)
+ record.offer(toolbox, tools, max_steps)
for _ in range(max_steps):
msg = llm.chat_message(cfg, msgs, tools=tools, model=model)
if not msg["tool_calls"]:
@@ -703,6 +767,8 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set
return
break # a reply with nothing in it
looked = True
+ record.used["rounds"] += 1
+ record.used["tool_calls"] += len(msg["tool_calls"])
msgs.append({"role": "assistant", "content": msg["content"], "tool_calls": msg["tool_calls"]})
for c in msg["tool_calls"]:
fn = c.get("function") or {}
@@ -722,9 +788,13 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set
# anything that may change what a read returns (a note written, a setting changed) makes every read new again
made = made | {key} if name in SAME_AGAIN else set()
# as the model reads it: what the tool read from the archive, the web or another server marked as data
+ t0 = time.monotonic()
result, summary = toolbox.call(name, args, model=True)
- yield "step", {"tool": name, "args": args, "summary": summary}
+ step = {"tool": name, "args": args, "summary": summary, "ms": int((time.monotonic() - t0) * 1000)}
+ yield "step", {**step, "failed": True} if getattr(toolbox, "failed", False) else step
msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": result})
+ else: # the rounds ran out
+ record.limited = True
text = _finish(cfg, msgs, tools, model) or "I couldn't put an answer together from what I found. Try again, or ask something narrower."
yield ("answer" if looked else "direct"), text
diff --git a/fastapi_backend/app/domain/completions.py b/fastapi_backend/app/domain/completions.py
index 44c44f45d..e344162ed 100644
--- a/fastapi_backend/app/domain/completions.py
+++ b/fastapi_backend/app/domain/completions.py
@@ -115,32 +115,34 @@ def specs(self):
def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools=(), base=None, temperature=None):
"""Answer one request. Yields ("step", {tool, args, summary}) for each tool call, ("token", text) for the
- answer's pieces, then ("done", {refs, text, model, notice}): what it can cite (numbered, as [n]) and the whole
- answer. Raises llm.LLMError when the model server fails, and activity.OverBudget at the hard limit."""
+ answer's pieces, then ("done", {refs, text, model, notice, record}): what it can cite (numbered, as [n]), the whole
+ answer and what produced it (chat.Record, without tokens and cost: no message keeps it). Raises llm.LLMError when
+ the model server fails, and activity.OverBudget at the hard limit."""
scope = {"namespaces": [ns]}
+ record = chat.Record(cfg, model, scope, picked=list(tools) or None)
passages = chat.retrieve(db, question, {sid}, scope, cfg=cfg)
extra, notice = note(db, sid, ns, app), None
used = model or cfg["llm"].get("model")
if not llm.configured(cfg):
text = chat.fallback(passages)
yield "token", text
- yield "done", {"refs": passages, "text": text, "model": MODEL, "notice": None}
+ yield "done", {"refs": passages, "text": text, "model": MODEL, "notice": None, "record": record.meta(db, "fallback")}
return
if tools and cfg["ai"].get("tools", True):
box = Box(db, cfg, user, sid, ns, tools, question, base, extra)
try:
- text = ""
- for kind, data in chat.tool_answer(cfg, box, question, history, cfg["ai"].get("max_steps") or 6, model):
+ text, how = "", "answered"
+ for kind, data in chat.tool_answer(cfg, box, question, history, cfg["ai"].get("max_steps") or 6, model, record=record):
if kind == "step":
yield "step", data
# a model that skips the tools hasn't seen the namespace: it answers from the excerpts instead
elif kind == "direct" and passages and not box.cited(data):
break
else:
- text = data
+ text, how = data, "direct" if kind == "direct" else "answered"
else:
yield "token", text
- yield "done", {"refs": box.refs, "text": text, "model": used, "notice": None}
+ yield "done", {"refs": box.refs, "text": text, "model": used, "notice": None, "record": record.meta(db, how)}
return
except llm.ToolsUnsupported:
notice = "This model can't use tools, so the answer comes from a search instead."
@@ -150,7 +152,10 @@ def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools=
for piece in llm.stream_chat(cfg, msgs, model=model, temperature=0.2 if temperature is None else temperature):
text += piece
yield "token", piece
- yield "done", {"refs": passages, "text": text, "model": used, "notice": notice}
+ yield (
+ "done",
+ {"refs": passages, "text": text, "model": used, "notice": notice, "record": record.meta(db, "fallback" if notice else "answered")},
+ )
def link(web, ref):
diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py
index 2cf82c334..1ce9738df 100644
--- a/fastapi_backend/app/schemas/chats.py
+++ b/fastapi_backend/app/schemas/chats.py
@@ -6,6 +6,7 @@
from pydantic import ConfigDict, Field
+from app.schemas.activity import ActivityEntry
from app.schemas.common import RequestModel, ResponseModel
@@ -80,6 +81,67 @@ class ToolStep(ResponseModel):
tool: str
args: dict[str, Any] = {}
summary: str = ""
+ ms: int | None = Field(None, description="how long the tool took")
+ failed: bool | None = Field(None, description="the tool failed (or was unknown or blocked); its summary says why")
+
+
+class RecordTools(ResponseModel):
+ picked: list[str] | None = Field(None, description="the tools or groups the question asked for; null: Auto")
+ offered: list[str] = Field(default_factory=list, description="every tool the model was offered")
+
+
+class RecordLimits(ResponseModel):
+ rounds: int | None = Field(None, description="rounds of tool calls it had (calls in one round run together)")
+ tool_calls: int | None = Field(None, description="tool calls it had; null: only the rounds limit them")
+
+
+class RecordUse(ResponseModel):
+ rounds: int = 0
+ tool_calls: int = 0
+
+
+class RecordTokens(ResponseModel):
+ model_config = ConfigDict(extra="allow", populate_by_name=True, serialize_by_alias=True)
+
+ in_: int = Field(0, alias="in")
+ out: int = 0
+
+
+class RecordOptions(ResponseModel):
+ """What a question was asked with (only what it set)."""
+
+ model: str | None = None
+ tools: list[str] | None = None
+ max_tool_calls: int | None = None
+ scope: dict[str, Any] | None = None
+ research: bool | None = None
+
+
+class MessageRecord(ResponseModel):
+ """What produced a message, recorded as it was written (docs/assistant.md, "What each answer records"). A question's
+ has `via` and `options`; an answer's the rest. Messages from before it was recorded have none."""
+
+ via: Literal["web", "api", "room", "voice", "automation", "completions"] | None = Field(None, description="a question: how it was sent")
+ options: RecordOptions | None = Field(None, description="a question: the options it was asked with")
+ model: str | None = Field(None, description="the model that wrote the answer")
+ provider: str | None = Field(None, description="the host name of the model server it ran on (never its address or key)")
+ tools: RecordTools | None = None
+ scope: dict[str, Any] | None = Field(None, description="the scope it answered in, after a conversation over everything was narrowed")
+ limits: RecordLimits | None = None
+ used: RecordUse | None = None
+ outcome: Literal["answered", "direct", "limit", "fallback", "stopped", "error"] | None = Field(
+ None,
+ description="answered (after looking things up), direct (without looking anything up), limit (its rounds ran "
+ "out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set "
+ "up), stopped, or error",
+ )
+ tokens: RecordTokens | None = Field(None, description="the tokens of the model calls made for it (the activity ledger)")
+ cost_usd: float | None = Field(None, description="what those calls cost, from the prices in Settings")
+ cost_estimated: bool | None = Field(None, description="some calls had a price but no token counts: the cost is at least this")
+ ms: int | None = Field(None, description="how long it took to write, from the question")
+ flagged: list[str] | None = Field(None, description="tools whose results read like instructions to an AI")
+ research: int | None = Field(None, description="the research it belongs to")
+ version: str | None = Field(None, description="the version of Lens that wrote it")
class PageContext(RequestModel):
@@ -127,6 +189,13 @@ class ChatMessage(ResponseModel):
error: str | None = Field(None, description="why there's no answer")
check: AnswerCheck | None = Field(None, description="its latest source check (POST .../messages/{mid}/check)")
model: str | None = Field(None, description="the model that wrote the answer")
+ meta: MessageRecord | None = Field(None, description="what produced it; none on messages from before it was recorded")
+
+
+class MessageDetail(ChatMessage):
+ ledger: list[ActivityEntry] = Field(
+ default_factory=list, description="its rows in the activity ledger: the model and tool calls made for it, newest first"
+ )
class ChatCapabilities(ResponseModel):
@@ -172,6 +241,7 @@ class MessageCreate(RequestModel):
description="asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the "
"assistant can use that browser (docs/browser-assistant.md)",
)
+ via: Literal["voice"] | None = Field(None, description="voice: said in a voice conversation (the question's record keeps it)")
class ApprovalOrigin(ResponseModel):
diff --git a/fastapi_backend/tests/api/test_actions_ai.py b/fastapi_backend/tests/api/test_actions_ai.py
index 9f018cb95..77cc325ae 100644
--- a/fastapi_backend/tests/api/test_actions_ai.py
+++ b/fastapi_backend/tests/api/test_actions_ai.py
@@ -9,7 +9,7 @@
import pytest
-from app.domain import actions, actions_ai, ai_tools, budgets, chat, notebook, store, untrusted
+from app.domain import actions, actions_ai, activity, ai_tools, budgets, chat, notebook, store, untrusted
from tests import fake_llm
from tests.api._assist import start_llm
from tests.api.test_setup_assistant import call
@@ -171,6 +171,10 @@ def test_ask_agent_works_on_what_happened_and_saves_the_conversation(db, cfg, ll
q, a = chat.history(db, got["chat"])
assert q["content"] == "File this where it belongs." and q["context"]["url"] == "/resources/7"
assert a["content"] == got["answer"] and [s["tool"] for s in a["steps"]] == [s["tool"] for s in got["steps"]]
+ # it records what produced it, like any answer: the question came from an automation
+ assert q["meta"] == {"via": "automation", "options": {}} and a["meta"]["used"] == {"rounds": 2, "tool_calls": 2}
+ assert (a["meta"]["outcome"], a["meta"]["scope"]) == ("answered", {"namespaces": ["pods"]})
+ assert activity.totals(db, f"chat_message:{a['id']}")["calls"] == 3 # its model calls name it in the ledger
first = next(b for b in llm.seen if b.get("tools"))
system, asked = first["messages"][0]["content"], first["messages"][-1]["content"]
assert "acting for an automation, “Automation 4”, that runs for me" in system
diff --git a/fastapi_backend/tests/api/test_bridge.py b/fastapi_backend/tests/api/test_bridge.py
index 5a3cc8c75..ea489756e 100644
--- a/fastapi_backend/tests/api/test_bridge.py
+++ b/fastapi_backend/tests/api/test_bridge.py
@@ -5,7 +5,7 @@
import pytest
-from app.domain import bridge, settings
+from app.domain import activity, bridge, settings
from tests import fake_llm, fake_matterbridge
from tests.api._assist import start_llm
from tests.api.test_setup_assistant import call
@@ -98,6 +98,10 @@ def test_it_answers_what_is_said_to_it_and_nothing_else(db, cfg, room, llm):
rows = db.rows("SELECT title, kind, account FROM chat WHERE kind = 'bridge' ORDER BY title")
assert [r["title"] for r in rows] == ["Matterbridge · general · alice"] * 2 + ["Matterbridge · general · bob"]
assert {r["kind"] for r in rows} == {"bridge"} and {r["account"] for r in rows} == {rows[0]["account"]}
+ # each question says it was asked in a room; each answer what produced it, its model calls naming it in the ledger
+ q, a = db.rows("SELECT record::id(id) AS id, role, meta FROM chat_message ORDER BY id")[:2]
+ assert q["meta"] == {"via": "room", "options": {}} and (a["meta"]["outcome"], a["meta"]["model"]) == ("answered", "fake")
+ assert activity.totals(db, f"chat_message:{a['id']}")["calls"] >= 1
room.waiting = [said("Lens, and on Monday?")]
run(db, cfg)
assert len(db.rows("SELECT id FROM chat WHERE kind = 'bridge'")) == 3 # alice keeps hers
diff --git a/fastapi_backend/tests/api/test_chat_records.py b/fastapi_backend/tests/api/test_chat_records.py
new file mode 100644
index 000000000..8f8bba2fc
--- /dev/null
+++ b/fastapi_backend/tests/api/test_chat_records.py
@@ -0,0 +1,178 @@
+"""Each answer records what produced it, and each question how it was sent (docs/assistant.md, "What each answer
+records"): the model and its host, the tools offered and used, the scope, limits and use, tokens and cost from the
+ledger rows that name the answer, the time, how it ended and which tool results read like instructions to an AI."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from app.domain import __version__, chat
+from app.domain import llm as llm_mod
+from tests import fake_llm
+from tests.api._assist import Assist, sse, start_llm
+
+USAGE = {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}
+
+
+@pytest.fixture
+def llm(cfg):
+ srv = start_llm(cfg)
+ cfg["telemetry"]["prices"] = {"fake": {"input": 1.0, "output": 2.0}} # USD per million tokens
+ fake_llm.Handler.usage = USAGE
+ yield fake_llm.Handler
+ fake_llm.Handler.usage = None
+ srv.shutdown()
+
+
+@pytest.fixture
+def app(cfg, db, llm):
+ from app.main import create_app
+
+ return create_app(cfg, db, background=False)
+
+
+@pytest.fixture
+def plain(cfg):
+ """The plain search-and-answer path (no tools)."""
+ cfg["ai"]["tools"] = False
+
+
+def tool(i, name, args):
+ return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}
+
+
+def search(i):
+ return {"content": "", "tool_calls": [tool(i, "search_transcripts", {"query": f"capsid {i}"})]}
+
+
+def kept(d):
+ """A record without the fields it doesn't have (the API lists them as null)."""
+ return {k: kept(v) if isinstance(v, dict) else v for k, v in d.items() if v is not None} if isinstance(d, dict) else d
+
+
+def ask(c, h, q, **body):
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ raw = c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": q, **body}).text
+ return cid, sse(raw), raw
+
+
+def test_an_answer_records_what_produced_it(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ # one round of two calls, the second outside what the editor can read, then the answer
+ llm.tool_script = [
+ {
+ "content": "",
+ "tool_calls": [
+ tool(1, "search_transcripts", {"query": "Dyno Therapeutics"}),
+ tool(2, "read_transcript", {"recording_id": s.call}),
+ ],
+ },
+ {"content": "Dyno Therapeutics designs capsids with machine learning [1]."},
+ ]
+ cid, ev, raw = ask(c, h, "What does Dyno do?")
+ meta = ev["meta"][0]
+ assert raw.index("event: meta") < raw.index("event: done") # the record comes just before the end
+ assert (meta["model"], meta["provider"], meta["outcome"], meta["version"]) == ("fake", "127.0.0.1", "answered", __version__)
+ assert meta["tools"]["picked"] is None and {"search_transcripts", "read_transcript"} <= set(meta["tools"]["offered"])
+ assert meta["limits"] == {"rounds": 6, "tool_calls": None} and meta["used"] == {"rounds": 1, "tool_calls": 2}
+ assert meta["flagged"] == [] and meta["research"] is None and isinstance(meta["ms"], int) and meta["ms"] >= 0
+ assert "base_url" not in json.dumps(meta) and ":" not in meta["provider"] # the host name only, never the address
+ # each step says how long it took, and whether it failed
+ assert [isinstance(x["ms"], int) for x in ev["step"]] == [True, True]
+ assert "failed" not in ev["step"][0] and ev["step"][1]["failed"] is True
+
+ msgs = c.get(f"/api/v1/chats/{cid}", headers=h).json()
+ mid = ev["done"][0]["message"]
+ answer, question = msgs["messages"][-1], msgs["messages"][-2]
+ assert answer["id"] == mid and kept(answer["meta"]) == kept(meta) and answer["model"] == "fake" # kept, with `model` as before
+ assert meta["scope"] == msgs["scope"] # the scope it answered in, after the conversation was narrowed
+ assert kept(question["meta"]) == {"via": "web", "options": {}} and question["id"] == mid - 1
+
+ # every model call made for it names it in the ledger, and its tokens and cost are theirs
+ one = c.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=h).json()
+ assert kept(one["meta"]) == kept(meta) and one["content"] == answer["content"]
+ calls = [r for r in one["ledger"] if r["kind"] == "out"]
+ assert sum(r["action"] == "model.chat" for r in calls) >= 2
+ assert all({f"chat_message:{mid}", f"chat:{cid}"} <= set(r["resources"]) for r in calls)
+ assert meta["tokens"] == {"in": sum(r.get("tokens_in") or 0 for r in calls), "out": sum(r.get("tokens_out") or 0 for r in calls)}
+ assert meta["tokens"]["in"] >= 200 and meta["cost_usd"] == pytest.approx(sum(r.get("cost_usd") or 0 for r in calls))
+ assert meta["cost_usd"] > 0 and meta["cost_estimated"] is False
+ # yours alone: the message's activity too
+ assert c.get("/api/v1/activity", headers=h, params={"resource": f"chat_message:{mid}"}).status_code == 200
+ cv, hv = s.cl["viewer"]
+ assert cv.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=hv).status_code == 404
+ assert cv.get("/api/v1/activity", headers=hv, params={"resource": f"chat_message:{mid}"}).status_code == 403
+ assert c.get(f"/api/v1/chats/{cid}/messages/999999", headers=h).status_code == 404
+
+ # a question said in a voice conversation, and one asked with another model, say so
+ llm.tool_script = [{"content": "Hello!"}]
+ cid, ev, _ = ask(c, h, "zzqx", via="voice", model="fake-large")
+ question = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-2]
+ assert kept(question["meta"]) == {"via": "voice", "options": {"model": "fake-large"}} and ev["meta"][0]["model"] == "fake-large"
+ # answered without looking anything up
+ assert (ev["meta"][0]["outcome"], ev["meta"][0]["used"]) == ("direct", {"rounds": 0, "tool_calls": 0})
+
+
+def test_how_an_answer_ended(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ # its rounds ran out, so it answered from what it found
+ llm.tool_script = [search(i) for i in range(6)] + [{"content": "The capsid model won [1]."}]
+ _, ev, _ = ask(c, h, "Would you hire them?")
+ assert (ev["meta"][0]["outcome"], ev["meta"][0]["used"]) == ("limit", {"rounds": 6, "tool_calls": 6})
+ # the model can't use tools: from a search instead
+ llm.reject_tools = True
+ _, ev, _ = ask(c, h, "capsid")
+ meta = ev["meta"][0]
+ assert "notice" in ev and meta["outcome"] == "fallback" and meta["tokens"]["out"] >= USAGE["completion_tokens"]
+ llm.reject_tools = False
+
+
+def test_stopped_and_failed_answers_are_recorded(plain, app, db, cfg, folder, new_client, llm, monkeypatch):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ monkeypatch.setattr(chat.Answering, "EVERY", 0)
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ stream = llm_mod.stream_chat
+
+ def stopped(*a, **k): # the person presses Stop after the first piece
+ pieces = stream(*a, **k)
+ yield next(pieces)
+ chat.request_stop(db, cid)
+ yield from pieces
+
+ monkeypatch.setattr(llm_mod, "stream_chat", stopped)
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "When does the shipment leave?"}).text)
+ meta = ev["meta"][0]
+ assert "stopped" in ev and meta["outcome"] == "stopped" and meta["tools"]["offered"] == [] and meta["limits"]["rounds"] is None
+ # the call that was cut off still counts for it (its row is written when the stream is closed)
+ one = c.get(f"/api/v1/chats/{cid}/messages/{ev['done'][0]['message']}", headers=h).json()
+ assert kept(one["meta"]) == kept(meta) and "model.chat" in [r["action"] for r in one["ledger"]]
+
+ def broken(*_a, **_k):
+ raise llm_mod.LLMError("429 from the LLM server")
+ yield # a generator, like the real one
+
+ monkeypatch.setattr(llm_mod, "stream_chat", broken)
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Once more?"}).text)
+ assert ev["meta"][0]["outcome"] == "error" and ev["error"][0]["message"] == "429 from the LLM server"
+ # with no model set up, the best excerpts come back on their own
+ assert c.put("/api/v1/settings/llm", headers=s.cl["admin"][1], json={"base_url": None}).status_code == 200
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "capsid"}).text)
+ meta = ev["meta"][0]
+ assert (meta["outcome"], meta["model"], meta["provider"], meta["tokens"]) == ("fallback", None, None, {"in": 0, "out": 0})
+
+
+def test_old_messages_have_no_record(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ chat.add(db, cid, "user", "From before")
+ mid = chat.add(db, cid, "assistant", "An answer from before", [], steps=[{"tool": "search_transcripts", "args": {}, "summary": "x"}])
+ got = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"]
+ assert [m["meta"] for m in got] == [None, None] and got[-1]["steps"][0].get("ms") is None
+ one = c.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=h).json()
+ assert one["meta"] is None and one["ledger"] == []
diff --git a/fastapi_backend/tests/api/test_completions.py b/fastapi_backend/tests/api/test_completions.py
index dfea013cc..88a7f02bf 100644
--- a/fastapi_backend/tests/api/test_completions.py
+++ b/fastapi_backend/tests/api/test_completions.py
@@ -180,6 +180,13 @@ def tool(i, name, args):
assert got["choices"][0]["message"]["content"].startswith("Dyno Therapeutics comes up on the pods [1].\n\nSources:")
found = got["lens"]["sources"]
assert found and {s["recording_id"] for s in found} <= {ep1, ep2} # the search never left pods
+ record = got["lens"]["record"] # what produced it, as a chat answer's record (no message keeps it: no tokens)
+ assert (record["outcome"], record["used"], record["tools"]) == (
+ "answered",
+ {"rounds": 1, "tool_calls": 2},
+ {"picked": ["search_transcripts"], "offered": ["search_transcripts"]},
+ )
+ assert record["scope"] == {"namespaces": ["pods"]} and "tokens" not in record
result = next(m["content"] for m in llm.seen[1]["messages"] if m["role"] == "tool")
assert "Dyno Therapeutics" in result and "leaves on Friday" not in result # calls is another namespace
assert result.startswith("<") and 'source="tool search_transcripts"' in result # read as data (untrusted.wrap)
diff --git a/fastapi_backend/tests/api/test_prompt_injection.py b/fastapi_backend/tests/api/test_prompt_injection.py
index e4163caa4..c6e09dd59 100644
--- a/fastapi_backend/tests/api/test_prompt_injection.py
+++ b/fastapi_backend/tests/api/test_prompt_injection.py
@@ -266,6 +266,7 @@ def test_a_result_that_reads_like_instructions_is_flagged(app, db, cfg, folder,
why = "reads like instructions to an AI: ignore previous instructions; new instructions or role"
assert f"({why};" in ev["step"][0]["summary"] and "chat control tokens)" in ev["step"][0]["summary"]
assert f'source="tool read_transcript" warning="{why};' in _tool_messages(llm)[0]
+ assert ev["meta"][0]["flagged"] == ["read_transcript"] # the answer's record names it too
# and admins in the audit log: the reasons, never the text
logged = [r for r in db.rows("SELECT * FROM audit_log") if r["action"] == "assistant.injection_flagged"]
assert [(r["target"], r["detail"]["tool"]) for r in logged] == [(f"chat:{cid}", "read_transcript")]
diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py
index 284bfddb4..55ed02f90 100644
--- a/fastapi_backend/tests/fake_llm.py
+++ b/fastapi_backend/tests/fake_llm.py
@@ -49,7 +49,10 @@ def do_POST(self):
if body.get("tool_choice") == "none" and not Handler.ignore_tool_choice:
msg = {"content": msg.get("content")}
return self._json(
- {"choices": [{"message": {"role": "assistant", "content": msg.get("content"), "tool_calls": msg.get("tool_calls")}}]}
+ {
+ "choices": [{"message": {"role": "assistant", "content": msg.get("content"), "tool_calls": msg.get("tool_calls")}}],
+ **({"usage": Handler.usage} if Handler.usage else {}),
+ }
)
if body.get("stream"):
self.send_response(200)
diff --git a/nextjs-frontend/__tests__/chat-details.test.tsx b/nextjs-frontend/__tests__/chat-details.test.tsx
new file mode 100644
index 000000000..c280d72d1
--- /dev/null
+++ b/nextjs-frontend/__tests__/chat-details.test.tsx
@@ -0,0 +1,168 @@
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { fireEvent, render, screen } from "@testing-library/react";
+import "@testing-library/jest-dom";
+
+import { Activity, Chats } from "@/app/openapi-client";
+import type { MessageRecord } from "@/app/openapi-client/types.gen";
+import { AnswerDetails, DetailsBody } from "@/components/chat/details";
+import { took } from "@/components/chat/steps";
+import { applyEvent, newTurn, savedSteps } from "@/components/chat/stream";
+import { TooltipProvider } from "@/components/ui/tooltip";
+
+jest.mock("@/app/openapi-client", () => ({
+ Chats: { getMessage: jest.fn() },
+ Activity: { resourceCosts: jest.fn() },
+}));
+jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) }));
+
+const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } });
+
+const META: MessageRecord = {
+ model: "qwen3:8b",
+ provider: "ollama.lan",
+ tools: { picked: null, offered: ["search_transcripts", "read_transcript", "web_read"] },
+ scope: { namespaces: ["pods"] },
+ limits: { rounds: 6, tool_calls: null },
+ used: { rounds: 2, tool_calls: 3 },
+ outcome: "limit",
+ tokens: { in: 5120, out: 410 },
+ cost_usd: 0.0031,
+ cost_estimated: true,
+ ms: 8200,
+ flagged: ["web_read"],
+ version: "0.13.0",
+};
+
+function wide(matches: boolean) {
+ window.matchMedia = ((q: string) => ({
+ matches: q.includes("max-width") ? !matches : matches,
+ addEventListener: () => {},
+ removeEventListener: () => {},
+ })) as unknown as typeof window.matchMedia;
+}
+
+function show(ui: React.ReactNode) {
+ const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ return render(
+
+ {ui}
+ ,
+ );
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ wide(true);
+ (Chats.getMessage as jest.Mock).mockImplementation(() =>
+ ok({
+ id: 8,
+ role: "assistant",
+ content: "x",
+ ledger: [
+ {
+ id: "a1",
+ at: "2026-10-16T10:00:00Z",
+ kind: "out",
+ action: "model.chat",
+ model: "qwen3:8b",
+ tokens_in: 4000,
+ tokens_out: 300,
+ ms: 5100,
+ ok: true,
+ },
+ { id: "a2", at: "2026-10-16T10:00:01Z", kind: "in", action: "POST /api/v1/chats/{cid}/messages", ok: true },
+ ],
+ }),
+ );
+ (Activity.resourceCosts as jest.Mock).mockImplementation(() =>
+ ok({ costs: { "chat:3": { cost_usd: 0.02, tokens: 9000, calls: 4, unpriced: 0, estimate: false } } }),
+ );
+});
+
+describe("an answer's record", () => {
+ it("says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended", async () => {
+ show(
+ ,
+ );
+ expect(screen.getByText("Its tool rounds ran out, so it answered from what it found")).toBeInTheDocument();
+ expect(screen.getByText("qwen3:8b")).toBeInTheDocument();
+ expect(screen.getByText(/on ollama\.lan/)).toBeInTheDocument();
+ expect(screen.getByText("pods")).toBeInTheDocument();
+ expect(screen.getByText("3 tools offered")).toBeInTheDocument();
+ expect(screen.getByText(/2 of 6 rounds, 3 tool calls/)).toBeInTheDocument();
+ expect(screen.getByText("search_transcripts, web_read")).toBeInTheDocument();
+ expect(screen.getByText(/web_read: what it read looked like instructions to an AI/)).toBeInTheDocument();
+ expect(screen.getByText("5.1k in · 410 out")).toBeInTheDocument();
+ expect(screen.getByText("(estimated from the prices in Settings)")).toBeInTheDocument();
+ expect(screen.getByText("≈")).toBeInTheDocument(); // some calls weren't priced: at least this
+ expect(screen.getByText("8.2 s")).toBeInTheDocument();
+ // the calls made for it, from the ledger (calls out only)
+ expect(await screen.findByText(/Model call · qwen3:8b/)).toBeInTheDocument();
+ expect(screen.queryByText(/POST \/api\/v1/)).not.toBeInTheDocument();
+ expect(Chats.getMessage).toHaveBeenCalledWith(expect.objectContaining({ path: { cid: 3, mid: 8 } }));
+ expect(await screen.findByText("$0.02")).toBeInTheDocument(); // the conversation so far
+ });
+
+ it("says an old answer wasn't recorded, and asks the ledger nothing", () => {
+ show( );
+ expect(screen.getByText(/from before Lens recorded what produced each one/)).toBeInTheDocument();
+ expect(Chats.getMessage).not.toHaveBeenCalled();
+ });
+
+ it("an answer without tools or a price says so", () => {
+ const plain: MessageRecord = {
+ ...META,
+ tools: { picked: null, offered: [] },
+ limits: { rounds: null, tool_calls: null },
+ used: { rounds: 0, tool_calls: 0 },
+ outcome: "answered",
+ cost_usd: 0,
+ cost_estimated: false,
+ flagged: [],
+ };
+ show( );
+ expect(screen.getByText("None offered")).toBeInTheDocument();
+ expect(screen.getByText("No tools")).toBeInTheDocument();
+ expect(screen.getByText("Not costed: no price is set for its model")).toBeInTheDocument();
+ expect(screen.queryByText("Flagged")).not.toBeInTheDocument();
+ });
+
+ it("opens from the i in the answer's header, by keyboard too; on a phone in a sheet", async () => {
+ show( );
+ const button = screen.getByRole("button", { name: "Details: what produced this answer" });
+ button.focus();
+ fireEvent.click(button);
+ expect(await screen.findByText("What produced this answer")).toBeInTheDocument();
+ expect(screen.getByText("qwen3:8b")).toBeInTheDocument();
+
+ wide(false);
+ show( );
+ fireEvent.click(screen.getAllByRole("button", { name: "Details: what produced this answer" }).at(-1)!);
+ expect(await screen.findByRole("dialog")).toHaveTextContent("What produced this answer");
+ expect(screen.getByRole("dialog")).toHaveTextContent(/from before Lens recorded/);
+ });
+});
+
+describe("the record in the stream", () => {
+ it("keeps the meta event, and each step's time and failure", () => {
+ let s = newTurn("q");
+ s = applyEvent(s, {
+ event: "step",
+ data: '{"tool":"read_transcript","args":{},"summary":"read_transcript: not found","ms":40,"failed":true}',
+ });
+ s = applyEvent(s, { event: "meta", data: JSON.stringify(META) });
+ s = applyEvent(s, { event: "done", data: '{"message":8}' });
+ expect(s.steps[0]).toMatchObject({ ms: 40, failed: true });
+ expect(s.meta).toMatchObject({ outcome: "limit", provider: "ollama.lan" });
+ expect(savedSteps([{ tool: "x" }])[0]).toEqual({ tool: "x", args: {}, summary: "" });
+ expect([took(120), took(1400), took(12500)]).toEqual(["120 ms", "1.4 s", "13 s"]);
+ });
+});
diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts
index b5db04b3b..c82ba439f 100644
--- a/nextjs-frontend/app/openapi-client/index.ts
+++ b/nextjs-frontend/app/openapi-client/index.ts
@@ -1218,6 +1218,11 @@ export type {
GetMediaError,
GetMediaErrors,
GetMediaResponses,
+ GetMessageData,
+ GetMessageError,
+ GetMessageErrors,
+ GetMessageResponse,
+ GetMessageResponses,
GetMonitoringData,
GetMonitoringResponse,
GetMonitoringResponses,
@@ -2147,6 +2152,8 @@ export type {
MergeTopicsResponse,
MergeTopicsResponses,
MessageCreate,
+ MessageDetail,
+ MessageRecord,
MetadataBulk,
MetadataBulkResult,
MetadataEdit,
@@ -2515,6 +2522,11 @@ export type {
RecordingTopicsResponse,
RecordingTopicsResponses,
RecordingUpdate,
+ RecordLimits,
+ RecordOptions,
+ RecordTokens,
+ RecordTools,
+ RecordUse,
RedeemTicketData,
RedeemTicketError,
RedeemTicketErrors,
diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts
index 552691e6a..eb08ca52b 100644
--- a/nextjs-frontend/app/openapi-client/sdk.gen.ts
+++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts
@@ -577,6 +577,9 @@ import type {
GetMediaData,
GetMediaErrors,
GetMediaResponses,
+ GetMessageData,
+ GetMessageErrors,
+ GetMessageResponses,
GetMonitoringData,
GetMonitoringResponses,
GetNamespaceAssistantData,
@@ -9971,12 +9974,29 @@ export class Chats {
});
}
+ /**
+ * Get Message
+ *
+ * One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and
+ * used, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`:
+ * the calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}.
+ */
+ public static getMessage(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).get({
+ url: "/api/v1/chats/{cid}/messages/{mid}",
+ ...options,
+ });
+ }
+
/**
* Send Message
*
* Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model
* reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST
- * /chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id).
+ * /chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced
+ * it), done (the saved message id).
*
* With `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and
* its new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked
diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts
index 06ed72de5..01e1de85c 100644
--- a/nextjs-frontend/app/openapi-client/types.gen.ts
+++ b/nextjs-frontend/app/openapi-client/types.gen.ts
@@ -3618,6 +3618,10 @@ export type ChatMessage = {
* the model that wrote the answer
*/
model?: string | null;
+ /**
+ * what produced it; none on messages from before it was recorded
+ */
+ meta?: MessageRecord | null;
[key: string]: unknown;
};
@@ -10656,6 +10660,182 @@ export type MessageCreate = {
* asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the assistant can use that browser (docs/browser-assistant.md)
*/
browser?: string | null;
+ /**
+ * Via
+ *
+ * voice: said in a voice conversation (the question's record keeps it)
+ */
+ via?: "voice" | null;
+};
+
+/**
+ * MessageDetail
+ */
+export type MessageDetail = {
+ /**
+ * Id
+ */
+ id: number;
+ /**
+ * Role
+ */
+ role: "user" | "assistant";
+ /**
+ * Content
+ */
+ content: string;
+ /**
+ * a question asked from a page: the page, and any highlighted text
+ */
+ context?: SharedContext | null;
+ /**
+ * Attachments
+ *
+ * files sent with it (POST /uploads with hold)
+ */
+ attachments?: Array;
+ /**
+ * Passages
+ */
+ passages?: Array | null;
+ /**
+ * Created At
+ */
+ created_at?: string | null;
+ /**
+ * Stopped
+ *
+ * the answer was stopped (POST /chats/{cid}/stop): `content` is what came before
+ */
+ stopped?: boolean;
+ /**
+ * Steps
+ *
+ * the tools the assistant used for this answer, in order
+ */
+ steps?: Array;
+ /**
+ * Notice
+ *
+ * e.g. the model can't use tools, so the answer came from a search
+ */
+ notice?: string | null;
+ /**
+ * Error
+ *
+ * why there's no answer
+ */
+ error?: string | null;
+ /**
+ * its latest source check (POST .../messages/{mid}/check)
+ */
+ check?: AnswerCheck | null;
+ /**
+ * Model
+ *
+ * the model that wrote the answer
+ */
+ model?: string | null;
+ /**
+ * what produced it; none on messages from before it was recorded
+ */
+ meta?: MessageRecord | null;
+ /**
+ * Ledger
+ *
+ * its rows in the activity ledger: the model and tool calls made for it, newest first
+ */
+ ledger?: Array;
+ [key: string]: unknown;
+};
+
+/**
+ * MessageRecord
+ *
+ * What produced a message, recorded as it was written (docs/assistant.md, "What each answer records"). A question's
+ * has `via` and `options`; an answer's the rest. Messages from before it was recorded have none.
+ */
+export type MessageRecord = {
+ /**
+ * Via
+ *
+ * a question: how it was sent
+ */
+ via?: "web" | "api" | "room" | "voice" | "automation" | "completions" | null;
+ /**
+ * a question: the options it was asked with
+ */
+ options?: RecordOptions | null;
+ /**
+ * Model
+ *
+ * the model that wrote the answer
+ */
+ model?: string | null;
+ /**
+ * Provider
+ *
+ * the host name of the model server it ran on (never its address or key)
+ */
+ provider?: string | null;
+ tools?: RecordTools | null;
+ /**
+ * Scope
+ *
+ * the scope it answered in, after a conversation over everything was narrowed
+ */
+ scope?: {
+ [key: string]: unknown;
+ } | null;
+ limits?: RecordLimits | null;
+ used?: RecordUse | null;
+ /**
+ * Outcome
+ *
+ * answered (after looking things up), direct (without looking anything up), limit (its rounds ran out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set up), stopped, or error
+ */
+ outcome?: "answered" | "direct" | "limit" | "fallback" | "stopped" | "error" | null;
+ /**
+ * the tokens of the model calls made for it (the activity ledger)
+ */
+ tokens?: RecordTokens | null;
+ /**
+ * Cost Usd
+ *
+ * what those calls cost, from the prices in Settings
+ */
+ cost_usd?: number | null;
+ /**
+ * Cost Estimated
+ *
+ * some calls had a price but no token counts: the cost is at least this
+ */
+ cost_estimated?: boolean | null;
+ /**
+ * Ms
+ *
+ * how long it took to write, from the question
+ */
+ ms?: number | null;
+ /**
+ * Flagged
+ *
+ * tools whose results read like instructions to an AI
+ */
+ flagged?: Array | null;
+ /**
+ * Research
+ *
+ * the research it belongs to
+ */
+ research?: number | null;
+ /**
+ * Version
+ *
+ * the version of Lens that wrote it
+ */
+ version?: string | null;
+ [key: string]: unknown;
};
/**
@@ -15424,6 +15604,105 @@ export type Reading = {
[key: string]: unknown;
};
+/**
+ * RecordLimits
+ */
+export type RecordLimits = {
+ /**
+ * Rounds
+ *
+ * rounds of tool calls it had (calls in one round run together)
+ */
+ rounds?: number | null;
+ /**
+ * Tool Calls
+ *
+ * tool calls it had; null: only the rounds limit them
+ */
+ tool_calls?: number | null;
+ [key: string]: unknown;
+};
+
+/**
+ * RecordOptions
+ *
+ * What a question was asked with (only what it set).
+ */
+export type RecordOptions = {
+ /**
+ * Model
+ */
+ model?: string | null;
+ /**
+ * Tools
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ */
+ max_tool_calls?: number | null;
+ /**
+ * Scope
+ */
+ scope?: {
+ [key: string]: unknown;
+ } | null;
+ /**
+ * Research
+ */
+ research?: boolean | null;
+ [key: string]: unknown;
+};
+
+/**
+ * RecordTokens
+ */
+export type RecordTokens = {
+ /**
+ * In
+ */
+ in?: number;
+ /**
+ * Out
+ */
+ out?: number;
+ [key: string]: unknown;
+};
+
+/**
+ * RecordTools
+ */
+export type RecordTools = {
+ /**
+ * Picked
+ *
+ * the tools or groups the question asked for; null: Auto
+ */
+ picked?: Array | null;
+ /**
+ * Offered
+ *
+ * every tool the model was offered
+ */
+ offered?: Array;
+ [key: string]: unknown;
+};
+
+/**
+ * RecordUse
+ */
+export type RecordUse = {
+ /**
+ * Rounds
+ */
+ rounds?: number;
+ /**
+ * Tool Calls
+ */
+ tool_calls?: number;
+ [key: string]: unknown;
+};
+
/**
* Recording
*
@@ -20726,6 +21005,18 @@ export type ToolStep = {
* Summary
*/
summary?: string;
+ /**
+ * Ms
+ *
+ * how long the tool took
+ */
+ ms?: number | null;
+ /**
+ * Failed
+ *
+ * the tool failed (or was unknown or blocked); its summary says why
+ */
+ failed?: boolean | null;
[key: string]: unknown;
};
@@ -38306,6 +38597,40 @@ export type UpdateChatResponses = {
export type UpdateChatResponse = UpdateChatResponses[keyof UpdateChatResponses];
+export type GetMessageData = {
+ body?: never;
+ path: {
+ /**
+ * Cid
+ */
+ cid: number;
+ /**
+ * Mid
+ */
+ mid: number;
+ };
+ query?: never;
+ url: "/api/v1/chats/{cid}/messages/{mid}";
+};
+
+export type GetMessageErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type GetMessageError = GetMessageErrors[keyof GetMessageErrors];
+
+export type GetMessageResponses = {
+ /**
+ * Successful Response
+ */
+ 200: MessageDetail;
+};
+
+export type GetMessageResponse = GetMessageResponses[keyof GetMessageResponses];
+
export type SendMessageData = {
body: MessageCreate;
path: {
diff --git a/nextjs-frontend/components/chat/answer.tsx b/nextjs-frontend/components/chat/answer.tsx
index 1775be1f4..130f52154 100644
--- a/nextjs-frontend/components/chat/answer.tsx
+++ b/nextjs-frontend/components/chat/answer.tsx
@@ -6,7 +6,7 @@ import Link from "next/link";
import { useMemo } from "react";
import { Chats, Search } from "@/app/openapi-client";
-import type { AnswerCheck, Passage } from "@/app/openapi-client/types.gen";
+import type { AnswerCheck, MessageRecord, Passage } from "@/app/openapi-client/types.gen";
import { ApprovalCard, type ApprovalView } from "@/components/chat/approval";
import {
citedNumbers,
@@ -17,6 +17,7 @@ import {
saysNotCovered,
} from "@/components/chat/cite";
import { CitationChip } from "@/components/chat/citation";
+import { AnswerDetails } from "@/components/chat/details";
import { keywords } from "@/components/chat/keywords";
import { RichText } from "@/components/chat/rich-text";
import { scopeWords, type Scope } from "@/components/chat/scope";
@@ -145,6 +146,8 @@ export type AnswerProps = {
active?: boolean;
check?: AnswerCheck | null;
onChecked?: (c: AnswerCheck) => void;
+ /** What produced it (the answer's record); older answers have none. */
+ meta?: MessageRecord | null;
};
/** One answer: where it comes from, the tools used, approvals, the text with citations, and its error states. */
@@ -191,6 +194,9 @@ export function Answer(p: AnswerProps) {
? `From ${from}`
: "Answer"}
{p.model ? ` · ${p.model}` : ""}
+ {!streaming && p.messageId != null && (
+
+ )}
{p.onShowSources && p.passages && p.passages.length > 0 && (
(null);
+ const spoken = useRef<"voice" | undefined>(undefined); // the question being sent was said, in voice mode
const files = useAttachments();
const [queue, setQueue] = useState([]);
const threadRef = useRef(null);
@@ -242,7 +243,7 @@ export function ChatApp() {
try {
for await (const msg of streamSSE(`/api/v1/chats/${cid}/messages`, {
method: "POST",
- body: { content: question, model, attachments: attached.map((f) => f.id), edit },
+ body: { content: question, model, attachments: attached.map((f) => f.id), edit, via: spoken.current },
accessToken: session?.accessToken,
signal: ac.signal,
})) {
@@ -400,7 +401,8 @@ export function ChatApp() {
if (voiceRef.current) await speak(ok ? `Okay, using ${named.name}.` : `I couldn't use ${named.name}.`);
continue;
}
- const turn = await dispatchRef.current(said);
+ spoken.current = "voice"; // the question's record says it was said
+ const turn = await dispatchRef.current(said).finally(() => (spoken.current = undefined));
if (!voiceRef.current || !turn) break;
if (turn.status === "done" && turn.text) await speak(turn.text);
}
@@ -677,6 +679,7 @@ export function ChatApp() {
onRetryWith={it.q ? (m) => send(it.q!.content, m) : undefined}
onAddScope={addScope}
check={checks[it.a.id] ?? it.a.check}
+ meta={it.a.meta}
onChecked={(c) => setChecks((x) => ({ ...x, [it.a!.id]: c }))}
{...answerHandlers(aKey)}
/>
@@ -724,6 +727,7 @@ export function ChatApp() {
approvals={liveHere.approvals}
scope={scope}
model={model}
+ meta={liveHere.meta}
onStop={stop}
stopping={stopping != null && stopping === live?.chatId}
onRetry={() => send(liveHere.question)}
diff --git a/nextjs-frontend/components/chat/details.tsx b/nextjs-frontend/components/chat/details.tsx
new file mode 100644
index 000000000..55a37ae1b
--- /dev/null
+++ b/nextjs-frontend/components/chat/details.tsx
@@ -0,0 +1,201 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+import { Info } from "lucide-react";
+import { useState } from "react";
+
+import { Chats } from "@/app/openapi-client";
+import type { MessageRecord } from "@/app/openapi-client/types.gen";
+import { fromApiScope, scopeWords } from "@/components/chat/scope";
+import { took } from "@/components/chat/steps";
+import type { ToolStep } from "@/components/chat/stream";
+import { actionText, Cost, ResourceCost, tokens } from "@/components/costs/costs";
+import { useIsNarrow } from "@/components/library/use-media";
+import { Dialog } from "@/components/ui/dialog";
+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/menu";
+import { data, useApiClient } from "@/lib/api/browser";
+import { plural } from "@/lib/format";
+
+/** How an answer ended, in words. */
+export const OUTCOME: Record = {
+ answered: "Answered after looking things up",
+ direct: "Answered without looking anything up",
+ limit: "Its tool rounds ran out, so it answered from what it found",
+ fallback: "Answered from a search: the model can’t use tools, or none is set up",
+ stopped: "Stopped before it finished",
+ error: "Didn’t answer: something went wrong",
+};
+
+const ESTIMATE =
+ "An estimate from the prices in Settings › Telemetry; some calls had a price but no token counts, so it’s at least this.";
+
+function Row({ label, children }: { label: string; children: React.ReactNode }) {
+ return (
+ <>
+ {label}
+ {children}
+ >
+ );
+}
+
+/** What produced an answer (its record), with the calls made for it from the activity ledger. */
+export function DetailsBody({
+ meta,
+ steps,
+ chatId,
+ messageId,
+}: {
+ meta: MessageRecord | null | undefined;
+ steps: ToolStep[];
+ chatId: number | null;
+ messageId: number | null;
+}) {
+ const client = useApiClient();
+ const ledger = useQuery({
+ queryKey: ["chat-message", chatId, messageId],
+ queryFn: () =>
+ data(Chats.getMessage({ client, path: { cid: chatId as number, mid: messageId as number } })).then(
+ (m) => m.ledger ?? [],
+ ),
+ enabled: chatId != null && messageId != null && Boolean(meta),
+ staleTime: 30_000,
+ });
+ if (!meta)
+ return (
+
+ This answer is from before Lens recorded what produced each one, so only its model and steps are known.
+
+ );
+ const offered = meta.tools?.offered ?? [];
+ const used = [...new Set(steps.map((s) => s.tool))];
+ const rounds = meta.limits?.rounds;
+ const calls = (ledger.data ?? []).filter((r) => r.kind === "out");
+ return (
+
+
+ {meta.outcome ? (OUTCOME[meta.outcome] ?? meta.outcome) : "Not recorded"}
+
+ {meta.model ? {meta.model} : "None set up"}
+ {meta.provider ? ` on ${meta.provider}` : ""}
+
+ {scopeWords(fromApiScope(meta.scope))}
+
+ {meta.tools?.picked?.length ? `${meta.tools.picked.join(", ")} · ` : offered.length ? "Auto · " : ""}
+ {offered.length ? (
+
+
+ {plural(offered.length, "tool")} offered
+
+ {offered.join(", ")}
+
+ ) : (
+ "None offered"
+ )}
+
+
+ {rounds != null
+ ? `${meta.used?.rounds ?? 0} of ${plural(rounds, "round")}, ${plural(meta.used?.tool_calls ?? 0, "tool call")}${
+ meta.limits?.tool_calls != null ? ` of ${meta.limits.tool_calls}` : ""
+ }`
+ : "No tools"}
+ {used.length > 0 && (
+ {used.join(", ")}
+ )}
+
+ {(meta.flagged ?? []).length > 0 && (
+
+
+ {meta.flagged!.join(", ")}: what {meta.flagged!.length === 1 ? "it" : "they"} read looked like
+ instructions to an AI, so it was marked as data
+
+
+ )}
+ {meta.tokens && (
+
+ {tokens(meta.tokens.in ?? 0)} in · {tokens(meta.tokens.out ?? 0)} out
+
+ )}
+ {meta.cost_usd != null && (
+
+ {meta.cost_usd > 0 ? (
+ <>
+ {" "}
+ (estimated from the prices in Settings)
+ >
+ ) : (
+ "Not costed: no price is set for its model"
+ )}
+
+ )}
+ {meta.ms != null && {took(meta.ms)}
}
+ {meta.version && {meta.version}
}
+
+ {calls.length > 0 && (
+
+
Calls made for it
+
+ {calls.map((r, i) => (
+
+
+ {actionText(r)}
+ {r.ok ? "" : ` · failed${r.error ? ` (${r.error})` : ""}`}
+
+
+ {r.tokens_in != null || r.tokens_out != null
+ ? `${tokens((r.tokens_in ?? 0) + (r.tokens_out ?? 0))} tokens`
+ : ""}
+ {r.ms != null ? ` · ${took(r.ms)}` : ""}
+
+
+ ))}
+
+
+ )}
+ {chatId != null && (
+
+ This conversation this month:
+
+ )}
+
+ );
+}
+
+/** The "i" in an answer's header: what produced it, in a popover (a sheet on a phone). */
+export function AnswerDetails(p: {
+ meta: MessageRecord | null | undefined;
+ steps: ToolStep[];
+ chatId: number | null;
+ messageId: number | null;
+}) {
+ const narrow = useIsNarrow();
+ const [open, setOpen] = useState(false);
+ const trigger = (
+ setOpen(true) : undefined}
+ className="grid size-6 place-items-center rounded-full text-fg-muted hover:bg-surface-neutral hover:text-fg"
+ >
+
+
+ );
+ if (narrow)
+ return (
+ <>
+ {trigger}
+
+
+
+ >
+ );
+ return (
+
+ {trigger}
+
+ What produced this answer
+
+
+
+ );
+}
diff --git a/nextjs-frontend/components/chat/steps.tsx b/nextjs-frontend/components/chat/steps.tsx
index fd0af6992..95214da36 100644
--- a/nextjs-frontend/components/chat/steps.tsx
+++ b/nextjs-frontend/components/chat/steps.tsx
@@ -42,10 +42,16 @@ type Kind = "done" | "failed" | "waiting";
function kindOf(step: ToolStep): Kind {
if (/^asked for approval/i.test(step.summary)) return "waiting";
- if (step.summary.startsWith(`${step.tool}:`) || /^unknown tool/i.test(step.summary)) return "failed";
+ // answers record whether a step failed; older ones are read from the summary
+ if (step.failed || step.summary.startsWith(`${step.tool}:`) || /^unknown tool/i.test(step.summary)) return "failed";
return "done";
}
+/** How long a step took: "120 ms", "1.4 s". */
+export function took(ms: number): string {
+ return ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(ms < 10_000 ? 1 : 0)} s`;
+}
+
const GLYPH: Record = {
done: { g: "✓", bg: "var(--aladdin-green)", label: "done" },
failed: { g: "✕", bg: "var(--aladdin-red)", label: "failed" },
@@ -83,6 +89,7 @@ function StepRow({ step, index }: { step: ToolStep; index: number }) {
{g.label}:
{title}
+ {step.ms != null && {took(step.ms)} }
{open ? (
) : (
diff --git a/nextjs-frontend/components/chat/stream.ts b/nextjs-frontend/components/chat/stream.ts
index a4f6c837e..5d2cf420e 100644
--- a/nextjs-frontend/components/chat/stream.ts
+++ b/nextjs-frontend/components/chat/stream.ts
@@ -1,16 +1,20 @@
-import type { Estimate, Passage } from "@/app/openapi-client/types.gen";
+import type { Estimate, MessageRecord, Passage } from "@/app/openapi-client/types.gen";
/**
* One question's answer as it streams from POST /chats/{id}/messages. The backend sends server-sent events:
* `scoped` (a conversation over everything narrowed to the namespace it's about), `suggested` (namespaces to offer when that isn't clear), `step` (a tool the assistant used), `approval` (work waiting for the person), `notice`, `passages` (the numbered
- * excerpts), `token` (answer text), `error`, `stopped` (Stop was pressed: what came is saved, marked stopped) and
- * `done` (the saved message id).
+ * excerpts), `token` (answer text), `error`, `stopped` (Stop was pressed: what came is saved, marked stopped), `meta`
+ * (the answer's record: what produced it) and `done` (the saved message id).
*/
export type ToolStep = {
tool: string;
args: Record;
summary: string;
+ /** How long the tool took. */
+ ms?: number | null;
+ /** It failed (or was unknown or blocked); the summary says why. */
+ failed?: boolean | null;
};
export type PendingApproval = {
id: number;
@@ -37,6 +41,8 @@ export type TurnState = {
text: string;
error: string | null;
messageId: number | null;
+ /** What produced the answer, once it's saved. */
+ meta: MessageRecord | null;
};
export function newTurn(question: string): TurnState {
@@ -52,6 +58,7 @@ export function newTurn(question: string): TurnState {
text: "",
error: null,
messageId: null,
+ meta: null,
};
}
@@ -79,6 +86,8 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T
tool: str(o.tool, "tool"),
args: (o.args as Record) ?? {},
summary: str(o.summary),
+ ms: typeof o.ms === "number" ? o.ms : null,
+ failed: o.failed === true || null,
},
],
};
@@ -121,6 +130,8 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T
};
case "stopped":
return { ...s, status: "stopped" };
+ case "meta":
+ return { ...s, meta: d && !Array.isArray(d) ? (o as MessageRecord) : null };
case "done":
return {
...s,
@@ -134,9 +145,18 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T
/** Tool steps as an answer was saved with them (GET /chats/{id}), in the shape the live stream gives. */
export function savedSteps(
- steps: { tool: string; args?: Record; summary?: string }[] | null | undefined,
+ steps:
+ | { tool: string; args?: Record; summary?: string; ms?: number | null; failed?: boolean | null }[]
+ | null
+ | undefined,
): ToolStep[] {
- return (steps ?? []).map((s) => ({ tool: s.tool, args: s.args ?? {}, summary: s.summary ?? "" }));
+ return (steps ?? []).map((s) => ({
+ tool: s.tool,
+ args: s.args ?? {},
+ summary: s.summary ?? "",
+ ...(s.ms != null ? { ms: s.ms } : {}),
+ ...(s.failed ? { failed: true } : {}),
+ }));
}
/** A short plain-words line for a tool call, when the backend's summary is missing. */
diff --git a/nextjs-frontend/components/costs/costs.tsx b/nextjs-frontend/components/costs/costs.tsx
index d4181c2ea..af318d8c9 100644
--- a/nextjs-frontend/components/costs/costs.tsx
+++ b/nextjs-frontend/components/costs/costs.tsx
@@ -124,7 +124,7 @@ const KIND: Record
change: { label: "Change", icon: , tone: "gate" },
};
-function actionText(e: ActivityEntry): string {
+export function actionText(e: ActivityEntry): string {
const a = e.action;
if (a.startsWith("model."))
return `${a.slice(6) === "chat" ? "Model call" : `Model ${a.slice(6)}`}${e.model ? ` · ${e.model}` : ""}`;
diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json
index 285f91af5..cac3b647e 100644
--- a/nextjs-frontend/openapi.json
+++ b/nextjs-frontend/openapi.json
@@ -24952,13 +24952,65 @@
}
}
},
+ "/api/v1/chats/{cid}/messages/{mid}": {
+ "get": {
+ "tags": [
+ "chats"
+ ],
+ "summary": "Get Message",
+ "description": "One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and\nused, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`:\nthe calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}.",
+ "operationId": "get_message",
+ "parameters": [
+ {
+ "name": "cid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Cid"
+ }
+ },
+ {
+ "name": "mid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Mid"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/MessageDetail"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/v1/chats/{cid}/messages": {
"post": {
"tags": [
"chats"
],
"summary": "Send Message",
- "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id).\n\nWith `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and\nits new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked\nfrom (where, and the highlighted text; not the page's text, which isn't kept).\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).",
+ "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nWith `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and\nits new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked\nfrom (where, and the highlighted text; not the page's text, which isn't kept).\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).",
"operationId": "send_message",
"parameters": [
{
@@ -36383,6 +36435,17 @@
],
"title": "Model",
"description": "the model that wrote the answer"
+ },
+ "meta": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/MessageRecord"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "what produced it; none on messages from before it was recorded"
}
},
"additionalProperties": true,
@@ -48396,12 +48459,389 @@
],
"title": "Browser",
"description": "asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the assistant can use that browser (docs/browser-assistant.md)"
+ },
+ "via": {
+ "anyOf": [
+ {
+ "type": "string",
+ "const": "voice"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Via",
+ "description": "voice: said in a voice conversation (the question's record keeps it)"
}
},
"additionalProperties": false,
"type": "object",
"title": "MessageCreate"
},
+ "MessageDetail": {
+ "properties": {
+ "id": {
+ "type": "integer",
+ "title": "Id"
+ },
+ "role": {
+ "type": "string",
+ "enum": [
+ "user",
+ "assistant"
+ ],
+ "title": "Role"
+ },
+ "content": {
+ "type": "string",
+ "title": "Content"
+ },
+ "context": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/SharedContext"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "a question asked from a page: the page, and any highlighted text"
+ },
+ "attachments": {
+ "items": {
+ "$ref": "#/components/schemas/Attachment"
+ },
+ "type": "array",
+ "title": "Attachments",
+ "description": "files sent with it (POST /uploads with hold)"
+ },
+ "passages": {
+ "anyOf": [
+ {
+ "items": {
+ "$ref": "#/components/schemas/Passage"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Passages"
+ },
+ "created_at": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Created At"
+ },
+ "stopped": {
+ "type": "boolean",
+ "title": "Stopped",
+ "description": "the answer was stopped (POST /chats/{cid}/stop): `content` is what came before",
+ "default": false
+ },
+ "steps": {
+ "items": {
+ "$ref": "#/components/schemas/ToolStep"
+ },
+ "type": "array",
+ "title": "Steps",
+ "description": "the tools the assistant used for this answer, in order"
+ },
+ "notice": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Notice",
+ "description": "e.g. the model can't use tools, so the answer came from a search"
+ },
+ "error": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Error",
+ "description": "why there's no answer"
+ },
+ "check": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/AnswerCheck"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "its latest source check (POST .../messages/{mid}/check)"
+ },
+ "model": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Model",
+ "description": "the model that wrote the answer"
+ },
+ "meta": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/MessageRecord"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "what produced it; none on messages from before it was recorded"
+ },
+ "ledger": {
+ "items": {
+ "$ref": "#/components/schemas/ActivityEntry"
+ },
+ "type": "array",
+ "title": "Ledger",
+ "description": "its rows in the activity ledger: the model and tool calls made for it, newest first"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "id",
+ "role",
+ "content"
+ ],
+ "title": "MessageDetail"
+ },
+ "MessageRecord": {
+ "properties": {
+ "via": {
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "web",
+ "api",
+ "room",
+ "voice",
+ "automation",
+ "completions"
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Via",
+ "description": "a question: how it was sent"
+ },
+ "options": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RecordOptions"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "a question: the options it was asked with"
+ },
+ "model": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Model",
+ "description": "the model that wrote the answer"
+ },
+ "provider": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Provider",
+ "description": "the host name of the model server it ran on (never its address or key)"
+ },
+ "tools": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RecordTools"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "scope": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Scope",
+ "description": "the scope it answered in, after a conversation over everything was narrowed"
+ },
+ "limits": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RecordLimits"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "used": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RecordUse"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "outcome": {
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "answered",
+ "direct",
+ "limit",
+ "fallback",
+ "stopped",
+ "error"
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Outcome",
+ "description": "answered (after looking things up), direct (without looking anything up), limit (its rounds ran out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set up), stopped, or error"
+ },
+ "tokens": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RecordTokens"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "the tokens of the model calls made for it (the activity ledger)"
+ },
+ "cost_usd": {
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Cost Usd",
+ "description": "what those calls cost, from the prices in Settings"
+ },
+ "cost_estimated": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Cost Estimated",
+ "description": "some calls had a price but no token counts: the cost is at least this"
+ },
+ "ms": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Ms",
+ "description": "how long it took to write, from the question"
+ },
+ "flagged": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Flagged",
+ "description": "tools whose results read like instructions to an AI"
+ },
+ "research": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Research",
+ "description": "the research it belongs to"
+ },
+ "version": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Version",
+ "description": "the version of Lens that wrote it"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "MessageRecord",
+ "description": "What produced a message, recorded as it was written (docs/assistant.md, \"What each answer records\"). A question's\nhas `via` and `options`; an answer's the rest. Messages from before it was recorded have none."
+ },
"MetadataBulk": {
"properties": {
"recordings": {
@@ -56531,6 +56971,168 @@
],
"title": "Reading"
},
+ "RecordLimits": {
+ "properties": {
+ "rounds": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Rounds",
+ "description": "rounds of tool calls it had (calls in one round run together)"
+ },
+ "tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tool Calls",
+ "description": "tool calls it had; null: only the rounds limit them"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "RecordLimits"
+ },
+ "RecordOptions": {
+ "properties": {
+ "model": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Model"
+ },
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls"
+ },
+ "scope": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Scope"
+ },
+ "research": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Research"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "RecordOptions",
+ "description": "What a question was asked with (only what it set)."
+ },
+ "RecordTokens": {
+ "properties": {
+ "in": {
+ "type": "integer",
+ "title": "In",
+ "default": 0
+ },
+ "out": {
+ "type": "integer",
+ "title": "Out",
+ "default": 0
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "RecordTokens"
+ },
+ "RecordTools": {
+ "properties": {
+ "picked": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Picked",
+ "description": "the tools or groups the question asked for; null: Auto"
+ },
+ "offered": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "title": "Offered",
+ "description": "every tool the model was offered"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "RecordTools"
+ },
+ "RecordUse": {
+ "properties": {
+ "rounds": {
+ "type": "integer",
+ "title": "Rounds",
+ "default": 0
+ },
+ "tool_calls": {
+ "type": "integer",
+ "title": "Tool Calls",
+ "default": 0
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "RecordUse"
+ },
"Recording": {
"properties": {
"id": {
@@ -65680,6 +66282,30 @@
"type": "string",
"title": "Summary",
"default": ""
+ },
+ "ms": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Ms",
+ "description": "how long the tool took"
+ },
+ "failed": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Failed",
+ "description": "the tool failed (or was unknown or blocked); its summary says why"
}
},
"additionalProperties": true,
diff --git a/threat-model.md b/threat-model.md
index 11822785b..c263b536d 100644
--- a/threat-model.md
+++ b/threat-model.md
@@ -426,6 +426,7 @@ stored text for the consumer's format.
| Embed player and `/s/` pages | `api/pages.py` | HTML built from DB values under a strict CSP; framable only per `server.embed_frame_ancestors` | that titles are inert outside this page | *(documented, docs/configuration.md "Content-Security-Policy")* |
| Report pages | `api/pages.py` | links re-signed on read for every asset of the namespace in the page | that a report link opens only the report | *(inferred, Q10)* |
| Assistant answers, summaries, LLM-written notes | Assistant | model output over untrusted content | that it is grounded or free of injected instructions | *(inferred, Q26)* |
+| An answer's record (`meta` on a chat message, `GET /chats/{cid}/messages/{mid}`) | Assistant | the model, the model server's host name (never its address, path, port or key), tool names, scope, counts, tokens, cost and the answer's ledger rows, to the conversation's owner alone | that the model server's host name stays hidden from people who chat | *(documented, docs/assistant.md "never the server's address, path or key")* |
| MCP tool results | `api/mcp_tools.py` | the person's readable data, as text, with chat-template control tokens broken up and a note when text reads like instructions to an AI | that an asset's text is safe to follow as instructions | *(documented, docs/mcp.md "sees exactly what that person sees")* |
| Error text from remote servers | LLM, speech, rclone, IMAP callers | up to 300 characters of the remote reply and the URL as typed | that internal host names stay hidden | *(inferred, Q18)* |
| Webhook bodies | `notify.py` | event data, HMAC-signed for the `webhook` kind only | that Slack or Discord messages are signed | *(documented, domain/notify.py docstring "signed the Standard Webhooks way")* |
diff --git a/threat-model.yaml b/threat-model.yaml
index 8e3afce75..df167370c 100644
--- a/threat-model.yaml
+++ b/threat-model.yaml
@@ -2,7 +2,7 @@
# Authority order: prose > yaml > json. Regenerate whenever the prose changes.
schema: threat-model-sidecar/v2
project: lens
-prose_version: "threat-model.md@sha256:d8bda21af9290dd0c018869ca7e0f3b4ce7d78cfcd77d304cb369cd716e42ec5"
+prose_version: "threat-model.md@sha256:097d71b310b2fd012ac29bc101532fabad87c9fdd5dacdb9b165a4855d1d50ef"
model_status: under-review
triage_policy: strict
confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8}
From acf97d754568bf7de30a219294e30c5c1c20b71c Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 00:04:31 +0000
Subject: [PATCH 03/14] test(notes): keep the daily notes test from failing on
2026-10-17
The last check asked for 2026-10-17's note after the call without a date had made today's note, so on that day it
found the note the deleted root had left behind. It now asks for a day no earlier call makes.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
fastapi_backend/tests/api/test_notebook.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fastapi_backend/tests/api/test_notebook.py b/fastapi_backend/tests/api/test_notebook.py
index 8f3d79e31..40e065d03 100644
--- a/fastapi_backend/tests/api/test_notebook.py
+++ b/fastapi_backend/tests/api/test_notebook.py
@@ -587,7 +587,7 @@ def test_daily_notes(client, env, db):
# editors only, and a date that is one
assert client.post("/api/v1/notes/today", headers=hv, json={"ns": "pods"}).status_code == 403
assert client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "soon"}).status_code == 400
- # the daily notes' note, when deleted, is made again the next time
+ # the daily notes' note, when deleted, is made again the next time (for a day no earlier call made: not today)
assert client.delete(f"/api/v1/notes/{root['id']}", headers=he).status_code == 200
- again = client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "2026-10-17"}).json()
+ again = client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "2020-02-29"}).json()
assert again["parent"] not in (None, root["id"])
From 33a2c19079b6af9909a1cb695c0d52a505f17839 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 00:36:03 +0000
Subject: [PATCH 04/14] feat(chat): choose tools, a tool-call limit and a scope
per answer
Group the assistant's tools (GET /chats/tools lists the groups and tools a
person has, what each does and why a group is missing), offer an answer only
the tools picked (an admin's switch still wins), stop at a tool-call limit up
to the new ai.max_tool_calls setting, and narrow one answer to its own
namespaces. A conversation keeps its choice and new ones start with the last.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
fastapi_backend/app/api/v1/routes/chats.py | 108 +++++++-
fastapi_backend/app/domain/actions_ai.py | 4 +-
fastapi_backend/app/domain/ai_tools.py | 134 +++++++++-
fastapi_backend/app/domain/chat.py | 67 ++++-
fastapi_backend/app/domain/completions.py | 7 +-
fastapi_backend/app/domain/extensions.py | 8 +-
fastapi_backend/app/domain/settings.py | 4 +
fastapi_backend/app/domain/store.py | 1 +
fastapi_backend/app/schemas/chats.py | 72 +++++-
fastapi_backend/tests/api/test_chat_tools.py | 244 ++++++++++++++++++
fastapi_backend/tests/api/test_chats.py | 3 +-
.../components/settings/bodies.tsx | 2 +
nextjs-frontend/components/settings/model.ts | 9 +
13 files changed, 626 insertions(+), 37 deletions(-)
create mode 100644 fastapi_backend/tests/api/test_chat_tools.py
diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py
index 7f00f41ee..17a679cff 100644
--- a/fastapi_backend/app/api/v1/routes/chats.py
+++ b/fastapi_backend/app/api/v1/routes/chats.py
@@ -33,6 +33,7 @@
ChatCreate,
ChatScope,
ChatSummary,
+ ChatTools,
ChatUpdate,
HeadUpdate,
MessageCreate,
@@ -75,8 +76,8 @@ def _own_chat(db: DB, cid: int, user: Principal) -> dict[str, Any]:
def list_chats(user: CurrentUser, db: Db) -> list[ChatSummary]:
"""Your conversations, most recent first."""
return db.rows(
- "SELECT record::id(id) AS id, title, scope, model, kind ?? 'chat' AS kind, created_at, updated_at FROM chat WHERE account = $a "
- "ORDER BY updated_at DESC LIMIT 200",
+ "SELECT record::id(id) AS id, title, scope, model, kind ?? 'chat' AS kind, tools, max_tool_calls, created_at, updated_at "
+ "FROM chat WHERE account = $a ORDER BY updated_at DESC LIMIT 200",
a=user.id,
)
@@ -92,11 +93,36 @@ def chat_capabilities(user: CurrentUser, cfg: Cfg) -> ChatCapabilities:
model=cfg["llm"].get("model") if on else None,
tools=on and bool(cfg["ai"].get("tools")),
max_steps=int(cfg["ai"].get("max_steps") or 6),
+ max_tool_calls=int(cfg["ai"].get("max_tool_calls") or 50),
check=on,
models=chat.model_choices(cfg),
)
+def _box(db: DB, cfg: dict[str, Any], user: Principal, acl: Access, scope: dict[str, Any] | None, request: Request) -> ai_tools.Toolbox:
+ """The assistant's tools as this person has them, in `scope`, before anything is asked."""
+ return ai_tools.Toolbox(
+ db, cfg, user.as_audit(), set(acl.roles), set(acl.editable()), scope, None, request.app.state.archive.base, user.admin
+ )
+
+
+@router.get("/chats/tools")
+def chat_tools(user: CurrentUser, acl: Acl, db: Db, cfg: Cfg, request: Request, chat_id: int | None = None) -> ChatTools:
+ """The tool groups and tools an answer can be given (`tools` on a question or a conversation), as you have them: each
+ tool with what it does (read, asks: it waits for your yes, out: it sends what it's given to another host) and
+ whether you have it now, each group with why you don't when you have none of it, and the setting that changes it.
+ Built-in groups first, then your extensions' (mcp:, ext:). With `chat_id`, as in that
+ conversation's scope (a namespace's own memory is there only in a conversation about that namespace). Also: the most
+ tool calls an answer may be given, and your latest choice, which new conversations start with."""
+ scope = _own_chat(db, chat_id, user).get("scope") if chat_id is not None else None
+ box = _box(db, cfg, user, acl, scope, request)
+ return ChatTools(
+ groups=ai_tools.catalog(box),
+ max_tool_calls=int(cfg["ai"].get("max_tool_calls") or 50),
+ remembered=chat.remembered_tools(db, user.id),
+ )
+
+
def _model(cfg: dict[str, Any], model: str | None) -> str | None:
try:
return chat.check_model(cfg, model)
@@ -104,12 +130,40 @@ def _model(cfg: dict[str, Any], model: str | None) -> str | None:
raise HTTPException(400, str(e)) from None
+def _tools(box: ai_tools.Toolbox, tools: list[str] | None) -> list[str] | None:
+ """A choice of tools, each a group or tool this person has (400 otherwise)."""
+ if tools is None:
+ return None
+ bad = box.unknown(tools)
+ if bad:
+ raise HTTPException(400, f"no tool or tool group {bad[0]!r} (GET /chats/tools lists them)")
+ return list(dict.fromkeys(tools))
+
+
+def _limit(cfg: dict[str, Any], n: int | None) -> int | None:
+ try:
+ return chat.tool_calls_allowed(cfg, n)
+ except ValueError as e:
+ raise HTTPException(400, str(e)) from None
+
+
@router.post("/chats")
-def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, body: ChatCreate | None = None) -> Created:
+def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body: ChatCreate | None = None) -> Created:
+ """A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH
+ /chats/{cid}), else Auto: everything you may use, limited by the rounds alone."""
body = body or ChatCreate()
if body.kind == "setup" and not user.admin:
raise HTTPException(403, "only admins set up the server")
- return Created(id=chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind))
+ last = chat.remembered_tools(db, user.id)
+ tools = _tools(_box(db, cfg, user, acl, None, request), body.tools) if "tools" in body.model_fields_set else last["tools"]
+ limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in body.model_fields_set else _most(cfg, last["max_tool_calls"])
+ cid = chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind, tools, limit)
+ return Created(id=cid)
+
+
+def _most(cfg: dict[str, Any], n: int | None) -> int | None:
+ """A limit kept from before, within ai.max_tool_calls as it is now."""
+ return None if n is None else max(1, min(int(n), int(cfg["ai"].get("max_tool_calls") or 50)))
@router.get("/chats/{cid}")
@@ -172,15 +226,23 @@ def get_message(cid: int, mid: int, user: CurrentUser, acl: Acl, db: Db) -> Mess
@router.patch("/chats/{cid}")
-def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Ok:
- """Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one)."""
- _own_chat(db, cid, user)
+def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> Ok:
+ """Rename a conversation, change what it draws on, the model that answers in it (null: the configured one), or
+ what its answers may use (`tools`, null: Auto) and at most how many tool calls each makes (`max_tool_calls`, null:
+ no limit but the rounds). The choice of tools and limit is also what your new conversations start with."""
+ c = _own_chat(db, cid, user)
if "model" in body.model_fields_set:
db.q("UPDATE $r SET model = $m", r=R("chat", cid), m=_model(cfg, body.model))
if body.title:
db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=body.title[:120])
if "scope" in body.model_fields_set: # replaces the scope as a whole, so narrowing can also be removed
db.q("UPDATE $r SET scope = $s", r=R("chat", cid), s=_scope(acl, db, body.scope))
+ chose = {"tools", "max_tool_calls"} & body.model_fields_set
+ if chose:
+ tools = _tools(_box(db, cfg, user, acl, c.get("scope"), request), body.tools) if "tools" in chose else c.get("tools")
+ limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in chose else c.get("max_tool_calls")
+ db.q("UPDATE $r SET tools = $t, max_tool_calls = $n", r=R("chat", cid), t=tools, n=limit)
+ chat.remember_tools(db, user.id, tools, limit)
return Ok()
@@ -231,6 +293,11 @@ async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db
text, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the
new version, the message it follows and its versions); so does a question beside others under `parent`.
+ What the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a
+ search) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's
+ choice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its
+ record keeps what it was given and used.
+
`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the
conversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.
@@ -279,7 +346,15 @@ def run() -> Iterator[str]:
model = await run_in_threadpool(_model, cfg, body.model) if body.model else c.get("model")
if model and not body.model and model not in await run_in_threadpool(chat.model_choices, cfg):
model = None
- record = chat.Record(cfg, model) # what produced the answer, kept with it (from here: it times the answer)
+ # what this answer may use: its tools and at most how many calls (the conversation's, unless the question says), and
+ # what it draws on, when the question gives it a scope of its own (the conversation keeps its scope)
+ if body.tools is not None:
+ tools = await run_in_threadpool(lambda: _tools(_box(db, cfg, user, acl, c.get("scope"), request), body.tools))
+ else:
+ tools = c.get("tools")
+ limit = _limit(cfg, body.max_tool_calls) if body.max_tool_calls is not None else _most(cfg, c.get("max_tool_calls"))
+ own_scope = await run_in_threadpool(_scope, acl, db, body.scope) if body.scope is not None else None
+ record = chat.Record(cfg, model, picked=tools, tool_calls=limit) # what produced the answer (from here: it times it)
title = c["title"]
page = body.context.model_dump() if body.context else None
kept = chat.shared_context(page) # what the question keeps of the page
@@ -316,7 +391,7 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
summary = chat.memory(db, cfg, cid, past)
if again is None:
via = "voice" if body.via == "voice" else "web" if user.via == "access" else "api"
- meta = chat.question_record(via, model=body.model)
+ meta = chat.question_record(via, model=body.model, tools=tools, max_tool_calls=limit, scope=own_scope)
question = chat.add(db, cid, "user", q, attachments=files, context=kept, parent=under, meta=meta)
answer_id = db.next_id("chat_message")
if again is not None:
@@ -331,6 +406,8 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]:
if not past and again is None and title == "New conversation":
db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=q[:80])
+ if own_scope is not None: # this answer's own scope, as asked: nothing is narrowed for it
+ return past, chat.retrieve(db, q, readable, own_scope, cfg=cfg), None
# A conversation over everything gets the namespace its first question is about, when that's clear.
if past or c.get("kind") == "setup" or c.get("scope"):
return past, chat.retrieve(db, q, readable, c.get("scope"), cfg=cfg), None
@@ -343,7 +420,8 @@ def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict
return past, chat.retrieve(db, q, readable, c["scope"], cfg=cfg), placed
past, passages, placed = await run_in_threadpool(prepare)
- record.scope = c.get("scope") or {} # after a conversation over everything was narrowed to what it's about
+ scope = own_scope if own_scope is not None else c.get("scope") # after one over everything was narrowed to what it's about
+ record.scope = scope or {}
picked = placed[1] if placed and placed[0] == "scoped" else None
suggested = placed[1] if placed and placed[0] == "suggest" else None
@@ -374,7 +452,7 @@ def stopped(text: str, cited: list[dict[str, Any]]) -> Iterator[str]:
def answer(on: chat.Answering) -> Iterator[str]:
nonlocal notice
- if llm.configured(cfg) and cfg["ai"].get("tools"):
+ if llm.configured(cfg) and cfg["ai"].get("tools") and tools != []: # given no tools, it answers from a search
setup = c.get("kind") == "setup"
box = ai_tools.Toolbox(
db,
@@ -382,7 +460,7 @@ def answer(on: chat.Answering) -> Iterator[str]:
user.as_audit(),
readable,
set(acl.editable()),
- c.get("scope"),
+ scope,
cid,
request.app.state.archive.base,
user.admin,
@@ -391,12 +469,16 @@ def answer(on: chat.Answering) -> Iterator[str]:
# what it read before any tool ran: the page it was asked from, and earlier answers' excerpts and tools
outside=(["page"] if page else []) + ai_tools.read_before(past, summary),
browser=body.browser,
+ only=tools,
)
try:
answer, how = "", "answered"
# setting the server up, or working in the person's browser, takes more steps than looking things up
steps_max = max(cfg["ai"].get("max_steps") or 6, 12 if setup else 0, browser.steps(cfg) if box.browser else 0)
- for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record):
+ answering = chat.tool_answer(
+ cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record, max_tool_calls=limit
+ )
+ for kind, data in answering:
if kind == "step":
steps.append(data)
yield _ev("step", data)
diff --git a/fastapi_backend/app/domain/actions_ai.py b/fastapi_backend/app/domain/actions_ai.py
index dc37e1cb5..0f7ba75fb 100644
--- a/fastapi_backend/app/domain/actions_ai.py
+++ b/fastapi_backend/app/domain/actions_ai.py
@@ -111,12 +111,10 @@ class Box(ai_tools.Toolbox):
would write become approvals (the chat assistant writes them at once), what could carry data out of Lens asks first
whatever the run read (wary), and it keeps no memories."""
- only: set | None = None
note = ""
def specs(self):
- got = [s for s in super().specs() if s["function"]["name"] not in LEFT_OUT]
- return got if self.only is None else [s for s in got if s["function"]["name"] in self.only]
+ return [s for s in super().specs() if s["function"]["name"] not in LEFT_OUT]
def system_note(self):
return super().system_note() + self.note
diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py
index 63151f162..cd8fb2385 100644
--- a/fastapi_backend/app/domain/ai_tools.py
+++ b/fastapi_backend/app/domain/ai_tools.py
@@ -475,6 +475,117 @@ def builtin_names():
"decline_approval",
} | {t[0] for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS + MEMORY_TOOLS + AUTOMATION_TOOLS if t[4]}
+# built-in tools that carry what the model gives them to another host; after an answer read text from outside Lens,
+# they ask first (Toolbox.wary). The side panel's browser tools have their own guard (browser.py): the ones that act
+# on a web site are labelled as sending data out (BROWSER_OUT), and ask as browser.py says.
+SENDS_OUT: set[str] = set()
+BROWSER_OUT = {"browser_open", "browser_click", "browser_type", "browser_press"}
+
+# The groups of tools a question can be given (POST /chats/{cid}/messages `tools`; docs/assistant.md, "Choosing tools,
+# namespaces and a limit"), in the order the web app shows them, each built-in tool in one. Extensions add their own:
+# mcp: for an MCP server's tools, ext: for another extension's (extensions.Active.groups).
+GROUPS = {
+ "archive": {
+ "label": "Archive",
+ "tools": ["search_transcripts", "find_similar", "list_recordings", "read_transcript", "recording_outputs"],
+ },
+ "graph": {
+ "label": "Graph",
+ "tools": [
+ "find_entities",
+ "entity_mentions",
+ "entity_timeline",
+ "find_topics",
+ "topic_recordings",
+ "graph_neighbours",
+ "speaker_stats",
+ "graph_schema",
+ "graph_query",
+ "graph_related",
+ "graph_paths",
+ ],
+ },
+ "notes": {"label": "Notes", "tools": ["find_notes", "read_note", "write_note", "update_note"]},
+ "memory": {"label": "Namespace memory", "tools": [t[0] for t in MEMORY_TOOLS]},
+ "activity": {
+ "label": "Timeline and automations",
+ "tools": ["timeline", "mark_activities", "approvals", "decline_approval", "automations", "test_automation"]
+ + [t[0] for t in AUTOMATION_TOOLS],
+ },
+ "changes": {
+ "label": "Proposed changes",
+ "tools": ["suggest_topic", "propose_entity_change", "propose_topic_change", "entity_setup", "run_template"],
+ },
+ "files": {"label": "Importing files", "tools": [t[0] for t in ops_tools.FILE_TOOLS]},
+ "extensions": {"label": "Extensions and skills", "tools": [t[0] for t in AUTHOR_TOOLS] + ["use_skill"]},
+ "server": {"label": "The server", "tools": [t[0] for t in ops_tools.ADMIN_TOOLS]},
+ "browser": {"label": "Your browser", "tools": [t[0] for t in browsermod.TOOLS]},
+}
+
+
+def _effect(changes, out):
+ """What a tool does, as the tool picker labels it: asks (it waits for the person's yes), out (it sends what it's
+ given to another host) or read."""
+ return "asks" if changes else "out" if out else "read"
+
+
+def _why_missing(group, cfg):
+ """Why none of a built-in group's tools is offered to this person: in words, and the setting that would change it
+ (for admins), if one would."""
+ off = set(cfg["ai"].get("disabled_tools") or [])
+ if group == "browser" and not browsermod.enabled(cfg):
+ return "switched off by an admin", "ai.browser"
+ if group == "extensions" and not cfg["ai"].get("extensions", True):
+ return "switched off by an admin", "ai.extensions"
+ if all(t in off for t in GROUPS[group]["tools"]):
+ return "switched off by an admin", "ai.disabled_tools"
+ if group == "server":
+ return "for admins", None
+ if group == "memory":
+ return "in a conversation about one namespace whose own assistant is on", None
+ return "needs editor access to a namespace", None
+
+
+def _group(name, label, tools, why=None, setting=None):
+ return {
+ "name": name,
+ "label": label,
+ "tools": tools,
+ "asks": any(t["effect"] == "asks" for t in tools),
+ "out": any(t["out"] for t in tools),
+ "available": any(t["available"] for t in tools),
+ "why": why,
+ "setting": setting,
+ }
+
+
+def catalog(box):
+ """The tool groups a question can be given, for the person `box` is for (GET /chats/tools): the built-in ones, then
+ their extensions'. Each has its label and its tools: what each does (read, asks first or out: sends data to
+ another host), whether it sends data out, and whether this person has it now. A group with none has why, and the
+ setting that changes it. The browser's tools count as there while an admin allows them: a question has them while
+ the browser's Lens side panel it's asked from is connected."""
+ offered = {s["function"]["name"] for s in box.specs()}
+ off = set(box.cfg["ai"].get("disabled_tools") or [])
+ if browsermod.enabled(box.cfg):
+ offered |= {t for t in GROUPS["browser"]["tools"] if t not in off}
+ built = {t[0]: t[4] for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS + MEMORY_TOOLS + AUTOMATION_TOOLS}
+ out = []
+ for name, g in GROUPS.items():
+ tools = []
+ for t in g["tools"]:
+ sends = t in SENDS_OUT or t in BROWSER_OUT
+ tools.append({"name": t, "effect": _effect(built.get(t, False), sends), "out": sends, "available": t in offered})
+ out.append(_group(name, g["label"], tools, *(() if any(t["available"] for t in tools) else _why_missing(name, box.cfg))))
+ for name, g in box.ext.groups.items() if box.ext else ():
+ tools = []
+ for t in g["tools"]:
+ spec = box.ext.tools[t]["spec"]
+ sends = extensions.sends_out(spec)
+ tools.append({"name": t, "effect": _effect(spec["effect"] == "change", sends), "out": sends, "available": t in offered})
+ out.append(_group(name, g["label"], tools, None if any(t["available"] for t in tools) else "needs editor access to a namespace"))
+ return out
+
def read_before(history, summary=None):
"""Whether earlier answers in a conversation read text from outside Lens (excerpts, or tools other than Lens's own),
@@ -496,7 +607,8 @@ class Toolbox(ops_tools.OpsTools):
channels.Channel.acts). `browser`: the person's connected side panel (browser.py), whose browser the assistant can
then use. `account`: the person an automation acts for, when `user` is the automation itself (automation:,
the by of what it writes): their approvals, extensions, view of the timeline and marks on it; `origin` ({automation, run,
- node}) is kept on the approvals it asks for."""
+ node}) is kept on the approvals it asks for. `only`: the groups (GROUPS, an extension's mcp: or ext:)
+ and tools the question was given (None: every tool this person may use; an admin's switch still wins)."""
def __init__(
self,
@@ -516,8 +628,10 @@ def __init__(
act_on=(),
account=None,
origin=None,
+ only=None,
):
self.db, self.cfg, self.user, self.chat, self.said = db, cfg, user, chat_id, said
+ self.only = None if only is None else set(only)
self.account, self.origin = user["id"] if account is None else account, origin
# where this answer has read text from outside Lens so far (`outside`: before any tool ran, such as the page it
# was asked from or a chat room's message): once it has, tools that send data out of Lens or change the server
@@ -622,15 +736,31 @@ def specs(self):
tools += browsermod.TOOLS
if self.ext:
tools += self.ext.tool_specs(can_act or self.admin)
+ keep = self.chosen()
return [
{
"type": "function",
"function": {"name": n, "description": d, "parameters": {"type": "object", "properties": p, "required": req}},
}
for n, d, p, req, _ in tools
- if n not in off
+ if n not in off and (keep is None or n in keep)
]
+ def chosen(self):
+ """The tool names `only` keeps (None: every tool): the tools it names, and those of the groups it names."""
+ if self.only is None:
+ return None
+ groups = self.ext.groups if self.ext else {}
+ out = set()
+ for n in self.only:
+ out |= set(GROUPS[n]["tools"] if n in GROUPS else groups[n]["tools"] if n in groups else [n])
+ return out
+
+ def unknown(self, names):
+ """The names in a choice of tools that are no group or tool this person has (built-in, or of their extensions)."""
+ known = builtin_names() | set(GROUPS) | set(self.ext.groups if self.ext else ()) | set(self.ext.tools if self.ext else ())
+ return [n for n in names if n not in known]
+
def ref(self, rid, t0, text, speaker=None, title=None, source="said", page=None):
self.refs.append(
store.clean(
diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py
index df0e9d9b3..3ddef703f 100644
--- a/fastapi_backend/app/domain/chat.py
+++ b/fastapi_backend/app/domain/chat.py
@@ -312,8 +312,10 @@ def cited(text, passages):
# ---------- conversations ----------
-def create(db, account, title=None, scope=None, model=None, kind="chat"):
- """A conversation. A `setup` one (admins) is the assistant setting the server up: it acts instead of asking."""
+def create(db, account, title=None, scope=None, model=None, kind="chat", tools=None, max_tool_calls=None):
+ """A conversation. A `setup` one (admins) is the assistant setting the server up: it acts instead of asking.
+ `tools` and `max_tool_calls`: what its answers may use (None: Auto, everything the person may use) and at most how
+ many tool calls each makes (None: the rounds limit them)."""
cid = db.next_id("chat")
db.q(
"CREATE $r CONTENT $d",
@@ -325,6 +327,8 @@ def create(db, account, title=None, scope=None, model=None, kind="chat"):
"kind": kind if kind != "chat" else None,
"scope": scope or {},
"model": model,
+ "tools": tools,
+ "max_tool_calls": max_tool_calls,
"created_at": store.now(),
"updated_at": store.now(),
}
@@ -333,9 +337,21 @@ def create(db, account, title=None, scope=None, model=None, kind="chat"):
return cid
+def remember_tools(db, account, tools, max_tool_calls):
+ """Someone's latest choice of tools and tool-call limit, which their new conversations start with."""
+ db.q("UPDATE $r SET chat_tools = $t", r=R("account", account), t={"tools": tools, "max_tool_calls": max_tool_calls})
+
+
+def remembered_tools(db, account):
+ """{tools, max_tool_calls}: the choice someone's new conversations start with (both None: Auto, no limit)."""
+ got = (db.one("SELECT chat_tools FROM $r", r=R("account", account)) or {}).get("chat_tools") or {}
+ return {"tools": got.get("tools"), "max_tool_calls": got.get("max_tool_calls")}
+
+
def get(db, cid, account):
c = db.one(
- "SELECT record::id(id) AS id, account, title, scope, model, kind ?? 'chat' AS kind, head, created_at, updated_at FROM $r",
+ "SELECT record::id(id) AS id, account, title, scope, model, kind ?? 'chat' AS kind, head, tools, max_tool_calls, created_at, "
+ "updated_at FROM $r",
r=R("chat", cid),
)
if not c or c["account"] != account:
@@ -830,6 +846,18 @@ def attached_note(files):
"Don't call any more tools now: answer the question from what you found above, citing it, and say in a sentence "
"what you didn't get to check."
)
+SPENT = "This answer's tool calls are used up, so this one wasn't made."
+
+
+def tool_calls_allowed(cfg, asked):
+ """At most how many tool calls one answer makes when `asked` (None: no limit but the rounds), never past
+ ai.max_tool_calls. ValueError for one that isn't a whole number from 1 to that."""
+ if asked is None:
+ return None
+ most = int(cfg["ai"].get("max_tool_calls") or 50)
+ if isinstance(asked, bool) or not isinstance(asked, int) or not 1 <= asked <= most:
+ raise ValueError(f"max_tool_calls is a whole number from 1 to {most} (ai.max_tool_calls)")
+ return asked
def _plain(msgs):
@@ -867,20 +895,30 @@ def _finish(cfg, msgs, tools, model):
return text
-def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None):
+def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None, max_tool_calls=None):
"""The tool loop: yields ("step", {...}) for each tool call, then ("answer", text), or ("direct", text) when the model
answered without looking anything up (it never saw the archive, so the caller can answer from a search instead).
Every turn ends in an answer: when the steps run out, or the model replies with nothing, it's asked to answer from
what it found, with the tools off. Raises llm.ToolsUnsupported. `record` (a Record) learns the tools offered and
- counts the rounds and tool calls."""
+ counts the rounds and tool calls. `max_tool_calls` (tool_calls_allowed): at most that many calls, in as many rounds
+ as it takes (a round makes one call or more), instead of `max_steps` rounds; a round that asks for more makes the
+ first of them, and the answer is written from what they found (its record's outcome is limit)."""
+ limit = tool_calls_allowed(cfg, max_tool_calls)
+ max_steps = limit or max_steps
system = TOOL_SYSTEM + untrusted.rules(cfg) + ("\n\n" + SETUP_SYSTEM if setup else "")
note = getattr(toolbox, "system_note", None) # skills, and context from hooks (extensions.py)
system += note() if note else ""
system += summary_note(summary)
- system += (
- f"\n\nYou have up to {max_steps} rounds of tool calls for this question (make calls that don't depend on each other "
- "in the same round), then you answer. Answer as soon as you have enough."
- )
+ if limit:
+ system += (
+ f"\n\nYou have up to {limit} tool call{'s' if limit > 1 else ''} for this question in all (make calls that don't depend "
+ "on each other in the same round), then you answer. Answer as soon as you have enough."
+ )
+ else:
+ system += (
+ f"\n\nYou have up to {max_steps} rounds of tool calls for this question (make calls that don't depend on each other "
+ "in the same round), then you answer. Answer as soon as you have enough."
+ )
msgs = [{"role": "system", "content": system}] + past_turns(history, summary=summary, cfg=cfg)
msgs.append({"role": "user", "content": question})
tools, looked, made = toolbox.specs(), False, set()
@@ -894,10 +932,14 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set
return
break # a reply with nothing in it
looked = True
+ left = len(msg["tool_calls"]) if limit is None else limit - record.used["tool_calls"]
record.used["rounds"] += 1
- record.used["tool_calls"] += len(msg["tool_calls"])
+ record.used["tool_calls"] += min(len(msg["tool_calls"]), left)
msgs.append({"role": "assistant", "content": msg["content"], "tool_calls": msg["tool_calls"]})
- for c in msg["tool_calls"]:
+ for i, c in enumerate(msg["tool_calls"]):
+ if i >= left: # past the limit: every call still gets a reply, as the model's API expects
+ msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": json.dumps({"note": SPENT})})
+ continue
fn = c.get("function") or {}
args = fn.get("arguments") or {}
if isinstance(args, str): # a JSON string, as OpenAI sends it; some servers send the object itself
@@ -920,6 +962,9 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set
step = {"tool": name, "args": args, "summary": summary, "ms": int((time.monotonic() - t0) * 1000)}
yield "step", {**step, "failed": True} if getattr(toolbox, "failed", False) else step
msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": result})
+ if limit is not None and record.used["tool_calls"] >= limit: # the calls are used up: it answers from what they found
+ record.limited = True
+ break
else: # the rounds ran out
record.limited = True
text = _finish(cfg, msgs, tools, model) or "I couldn't put an answer together from what I found. Try again, or ask something narrower."
diff --git a/fastapi_backend/app/domain/completions.py b/fastapi_backend/app/domain/completions.py
index e344162ed..ca8d0d602 100644
--- a/fastapi_backend/app/domain/completions.py
+++ b/fastapi_backend/app/domain/completions.py
@@ -102,16 +102,13 @@ class Box(ai_tools.Toolbox):
none of the person's extensions, skills or hooks, and nothing that changes data."""
def __init__(self, db, cfg, user, sid, ns, tools, said, base, note_text):
- super().__init__(db, cfg, user, {sid}, set(), {"namespaces": [ns]}, None, base, said=said)
- self.only, self.note_text = set(tools), note_text
+ super().__init__(db, cfg, user, {sid}, set(), {"namespaces": [ns]}, None, base, said=said, only=tools)
+ self.note_text = note_text
self.ext, self.home, self.browser = None, None, None
def system_note(self):
return self.note_text
- def specs(self):
- return [s for s in super().specs() if s["function"]["name"] in self.only]
-
def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools=(), base=None, temperature=None):
"""Answer one request. Yields ("step", {tool, args, summary}) for each tool call, ("token", text) for the
diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py
index eb0079509..717ce680e 100644
--- a/fastapi_backend/app/domain/extensions.py
+++ b/fastapi_backend/app/domain/extensions.py
@@ -691,10 +691,13 @@ def _owner_admin(db, g):
class Active:
"""The extensions switched on for one person, plugins opened up: tools and skills by name, hooks by event. Each
- item knows the extension (and version) it comes from."""
+ item knows the extension (and version) it comes from. `groups`: each extension's tools as a group a question can be
+ given ({mcp: or ext:: {label, tools}}; ai_tools.GROUPS has the built-in ones): an MCP server's tools
+ under mcp:, any other extension's under ext:."""
def __init__(self, db, me):
self.tools, self.skills, self.hooks = {}, {}, {e: [] for e in EVENTS}
+ self.groups: dict[str, dict] = {}
for d in db.rows("SELECT record::id(id) AS id FROM extension WHERE deleted_at = NONE AND enabled = true ORDER BY id"):
g = get(db, d["id"])
if not can_use(g, me):
@@ -722,6 +725,9 @@ def __init__(self, db, me):
"spec": spec,
}
if it["kind"] == "tool":
+ if it["name"] not in self.tools:
+ group = ("mcp:" if spec["run"]["type"] == "mcp" else "ext:") + g["name"]
+ self.groups.setdefault(group, {"label": g.get("title") or g["name"], "tools": []})["tools"].append(it["name"])
self.tools.setdefault(it["name"], item)
elif it["kind"] == "skill":
self.skills.setdefault(it["name"], item)
diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py
index 72573411b..f07a1fa52 100644
--- a/fastapi_backend/app/domain/settings.py
+++ b/fastapi_backend/app/domain/settings.py
@@ -445,6 +445,10 @@ def _check(section, key, value, default):
if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 100):
raise ValueError("ai.browser_steps is a whole number of steps from 1 to 100")
return value
+ if (section, key) == ("ai", "max_tool_calls"):
+ if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 500):
+ raise ValueError("ai.max_tool_calls is a whole number of tool calls from 1 to 500")
+ return value
if (section, key) == ("llm", "describe_max"):
if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 1000):
raise ValueError("llm.describe_max is a whole number from 1 to 1000")
diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
index 8608aa685..9752c9f31 100644
--- a/fastapi_backend/app/domain/store.py
+++ b/fastapi_backend/app/domain/store.py
@@ -291,6 +291,7 @@ def labels():
"injection_check": "patterns", # flag text that reads like instructions to an AI: patterns, classifier, model, off
"disabled_tools": [],
"max_steps": 6,
+ "max_tool_calls": 50, # the most tool calls a question may give one answer (its `max_tool_calls`; chat.tool_answer)
"max_transcript_reads": 20,
"confirm_over_recordings": 100,
"confirm_over_cost": None,
diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py
index 443d31b78..4c8a7043d 100644
--- a/fastapi_backend/app/schemas/chats.py
+++ b/fastapi_backend/app/schemas/chats.py
@@ -36,6 +36,15 @@ class ChatCreate(RequestModel):
model: str | None = Field(
None, description="the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one"
)
+ tools: list[str] | None = Field(
+ None,
+ max_length=100,
+ description="what its answers may use: tool groups and tools (GET /chats/tools); [] for none; left out: your latest "
+ "choice (PATCH /chats/{cid}), else Auto",
+ )
+ max_tool_calls: int | None = Field(
+ None, ge=1, description="at most this many tool calls in each answer (1 to ai.max_tool_calls); left out: your latest choice"
+ )
class ChatUpdate(RequestModel):
@@ -44,6 +53,15 @@ class ChatUpdate(RequestModel):
model: str | None = Field(
None, description="the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one"
)
+ tools: list[str] | None = Field(
+ None,
+ max_length=100,
+ description="what its answers may use: tool groups and tools (GET /chats/tools); [] for none; null: Auto (everything "
+ "you may use). Your new conversations start with it too",
+ )
+ max_tool_calls: int | None = Field(
+ None, ge=1, description="at most this many tool calls in each answer (1 to ai.max_tool_calls); null: no limit but the rounds"
+ )
class ChatSummary(ResponseModel):
@@ -52,6 +70,8 @@ class ChatSummary(ResponseModel):
kind: Literal["chat", "setup", "bridge"] = Field("chat", description="bridge: a conversation in a chat room, through Matterbridge")
scope: dict[str, Any] = {}
model: str | None = Field(None, description="the model chosen for this conversation; null: the configured one")
+ tools: list[str] | None = Field(None, description="the tool groups and tools its answers may use; null: Auto")
+ max_tool_calls: int | None = Field(None, description="at most this many tool calls in each answer; null: no limit but the rounds")
created_at: str | None = None
updated_at: str | None = None
@@ -221,7 +241,8 @@ class ChatCapabilities(ResponseModel):
configured: bool = Field(description="a language model is set up; without one, answers are the best-matching passages")
model: str | None = Field(None, description="the model answers come from")
tools: bool = Field(description="the assistant looks things up with tools (and may propose work for approval)")
- max_steps: int = Field(description="the most tool calls it makes for one answer")
+ max_steps: int = Field(description="the most rounds of tool calls it makes for one answer (a round makes one call or more)")
+ max_tool_calls: int = Field(50, description="the most tool calls a question may give one answer (`max_tool_calls`)")
check: bool = Field(description="answers can be checked against their sources")
models: list[str] = Field(
default_factory=list, description="the models people may pick, the configured one first (llm.chat_models, else the server's list)"
@@ -275,6 +296,55 @@ class MessageCreate(RequestModel):
"assistant can use that browser (docs/browser-assistant.md)",
)
via: Literal["voice"] | None = Field(None, description="voice: said in a voice conversation (the question's record keeps it)")
+ tools: list[str] | None = Field(
+ None,
+ max_length=100,
+ description="what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists "
+ "them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An "
+ "admin's switch still wins",
+ )
+ max_tool_calls: int | None = Field(
+ None, ge=1, description="at most this many tool calls in this answer (1 to ai.max_tool_calls); null: the conversation's"
+ )
+ scope: ChatScope | None = Field(
+ None,
+ description="what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read",
+ )
+
+
+class ToolInfo(ResponseModel):
+ name: str
+ effect: Literal["read", "asks", "out"] = Field(
+ description="read: it looks things up; asks: it waits for your yes; out: it sends what it's given to another host"
+ )
+ out: bool = Field(False, description="it sends what it's given to another host (and asks first after outside text)")
+ available: bool = Field(description="you have it now")
+
+
+class ToolGroup(ResponseModel):
+ name: str = Field(description="what `tools` names it by: a built-in group, mcp: or ext:")
+ label: str
+ tools: list[ToolInfo]
+ asks: bool = Field(description="one of its tools waits for your yes")
+ out: bool = Field(description="one of its tools sends data to another host")
+ available: bool = Field(description="you have one of its tools now")
+ why: str | None = Field(None, description="why you don't, when you have none")
+ setting: str | None = Field(None, description="the setting that would change that (section.key), for admins")
+
+
+class ToolChoice(ResponseModel):
+ tools: list[str] | None = None
+ max_tool_calls: int | None = None
+
+
+class ChatTools(ResponseModel):
+ groups: list[ToolGroup] = Field(description="the built-in groups, then your extensions' (mcp:, ext:)")
+ max_tool_calls: int = Field(description="the most tool calls one answer may be given (ai.max_tool_calls)")
+ remembered: ToolChoice = Field(description="your latest choice, which new conversations start with")
+ namespaces_note: str = Field(
+ "Web and MCP tools reach beyond your namespaces: the namespaces you pick narrow the archive's tools only.",
+ description="what the choice of namespaces doesn't narrow",
+ )
class ApprovalOrigin(ResponseModel):
diff --git a/fastapi_backend/tests/api/test_chat_tools.py b/fastapi_backend/tests/api/test_chat_tools.py
new file mode 100644
index 000000000..44ecb2a9b
--- /dev/null
+++ b/fastapi_backend/tests/api/test_chat_tools.py
@@ -0,0 +1,244 @@
+"""Choosing what an answer may use (docs/assistant.md, "Choosing tools, namespaces and a limit"): the tool groups and
+tools a person has (GET /chats/tools), only the ones picked offered (an admin's switch still wins), at most so many tool
+calls, a scope for one answer alone, and the choice kept with the conversation and for the next ones."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from app.domain import ai_tools, browser, ops_tools
+from tests import fake_llm
+from tests.api._assist import Assist, sse, start_llm
+
+TOOL = """---
+name: translate
+kind: tool
+description: Translate text into another language.
+effect: read
+params:
+ - {name: text, kind: text, required: true}
+---
+
+Translate into French: {{text}}
+"""
+
+
+@pytest.fixture
+def llm(cfg):
+ srv = start_llm(cfg)
+ yield fake_llm.Handler
+ srv.shutdown()
+
+
+@pytest.fixture
+def app(cfg, db, llm):
+ from app.main import create_app
+
+ return create_app(cfg, db, background=False)
+
+
+def call(i, name, args):
+ return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}
+
+
+def offered(llm):
+ """The tools the model was offered, the last time it was offered any."""
+ return {t["function"]["name"] for t in [b for b in llm.seen if b.get("tools")][-1]["tools"]}
+
+
+def ask(c, h, cid, **body):
+ return sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "What did Dyno say?", **body}).text)
+
+
+def test_every_built_in_tool_is_in_one_group():
+ every = [t for g in ai_tools.GROUPS.values() for t in g["tools"]]
+ assert sorted(every) == sorted(ai_tools.builtin_names()) and len(every) == len(set(every))
+ assert not set(ai_tools.GROUPS) & ai_tools.builtin_names() # a name is a group or a tool, never both
+ assert ai_tools.BROWSER_OUT <= {t[0] for t in browser.TOOLS}
+
+
+def test_the_catalog_says_what_each_person_has(app, db, cfg, folder, new_client):
+ s = Assist(app, db, cfg, folder, new_client)
+ cv, hv = s.cl["viewer"]
+ got = cv.get("/api/v1/chats/tools", headers=hv).json()
+ assert got["max_tool_calls"] == 50 and got["remembered"] == {"tools": None, "max_tool_calls": None}
+ groups = {g["name"]: g for g in got["groups"]}
+ assert list(groups)[: len(ai_tools.GROUPS)] == list(ai_tools.GROUPS)
+ tools = {t["name"]: t for g in got["groups"] for t in g["tools"]}
+ # a viewer reads the archive and the notes, but writing a note and proposing changes need an editor
+ assert groups["archive"]["available"] and not groups["archive"]["asks"] and not groups["archive"]["out"]
+ assert tools["find_notes"]["available"] and not tools["write_note"]["available"] and tools["write_note"]["effect"] == "asks"
+ assert groups["changes"]["asks"] and tools["entity_setup"]["available"] and not tools["propose_entity_change"]["available"]
+ assert (groups["server"]["why"], groups["server"]["setting"]) == ("for admins", None)
+ # the browser's tools are there while the side panel is allowed; the ones that act on a site send data out
+ assert groups["browser"]["available"] and groups["browser"]["out"] and tools["browser_type"]["effect"] == "out"
+ assert tools["browser_read"]["effect"] == "read"
+
+ ca, ha = s.cl["admin"]
+ groups = {g["name"]: g for g in ca.get("/api/v1/chats/tools", headers=ha).json()["groups"]}
+ assert groups["server"]["available"] and groups["changes"]["available"]
+ # what an admin switched off isn't offered, and says where it's switched back on
+ off = {"disabled_tools": [t[0] for t in ops_tools.ADMIN_TOOLS], "browser": False}
+ assert ca.put("/api/v1/settings/ai", headers=ha, json=off).status_code == 200
+ groups = {g["name"]: g for g in ca.get("/api/v1/chats/tools", headers=ha).json()["groups"]}
+ assert (groups["server"]["available"], groups["server"]["why"], groups["server"]["setting"]) == (
+ False,
+ "switched off by an admin",
+ "ai.disabled_tools",
+ )
+ assert (groups["browser"]["available"], groups["browser"]["setting"]) == (False, "ai.browser")
+ assert cv.get("/api/v1/chats/tools").status_code == 401
+
+
+def test_an_answer_is_offered_only_what_was_picked(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ llm.tool_script = [{"content": "", "tool_calls": [call(1, "find_notes", {"query": "Dyno"})]}, {"content": "Nothing noted."}]
+ ev = ask(c, h, cid, tools=["notes", "find_entities"])
+ assert offered(llm) == {"find_notes", "read_note", "write_note", "update_note", "find_entities"}
+ meta = ev["meta"][0]
+ assert meta["tools"]["picked"] == ["notes", "find_entities"] and set(meta["tools"]["offered"]) == offered(llm)
+ question = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-2]
+ assert question["meta"]["options"]["tools"] == ["notes", "find_entities"]
+ # a tool it wasn't given isn't run, even when the model calls it
+ llm.tool_script = [{"content": "", "tool_calls": [call(2, "search_transcripts", {"query": "Dyno"})]}, {"content": "Done."}]
+ ev = ask(c, h, cid, tools=["notes"])
+ assert ev["step"][0]["summary"] == "unknown tool search_transcripts" and ev["step"][0]["failed"] is True
+
+ # an admin's switch still wins
+ ca, ha = s.cl["admin"]
+ assert ca.put("/api/v1/settings/ai", headers=ha, json={"disabled_tools": ["write_note"]}).status_code == 200
+ llm.tool_script = [{"content": "Nothing noted."}]
+ ask(c, h, cid, tools=["notes"])
+ assert offered(llm) == {"find_notes", "read_note", "update_note"}
+ assert ca.put("/api/v1/settings/ai", headers=ha, json={"disabled_tools": []}).status_code == 200
+
+ # no tools: it answers from a search, and the model is offered none
+ llm.seen.clear()
+ ev = ask(c, h, cid, tools=[])
+ assert not [b for b in llm.seen if b.get("tools")] and ev["meta"][0]["tools"] == {"picked": [], "offered": []}
+ assert ev["passages"] and ev["meta"][0]["outcome"] == "answered"
+
+ # what isn't a group or a tool is told, not ignored
+ r = c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "x", "tools": ["archive", "nope"]})
+ assert r.status_code == 400 and "nope" in r.json()["detail"]
+
+
+def test_an_extensions_tools_are_a_group(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ assert c.post("/api/v1/extensions", headers=h, json={"text": TOOL}).status_code == 200
+ groups = {g["name"]: g for g in c.get("/api/v1/chats/tools", headers=h).json()["groups"]}
+ assert groups["ext:translate"]["tools"] == [{"name": "translate", "effect": "read", "out": False, "available": True}]
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ llm.tool_script = [{"content": "Bonjour."}]
+ ask(c, h, cid, tools=["ext:translate"])
+ assert offered(llm) == {"translate"}
+ # another person doesn't have it
+ cv, hv = s.cl["viewer"]
+ vid = cv.post("/api/v1/chats", headers=hv, json={}).json()["id"]
+ assert cv.post(f"/api/v1/chats/{vid}/messages", headers=hv, json={"content": "x", "tools": ["ext:translate"]}).status_code == 400
+
+
+def test_at_most_so_many_tool_calls(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ llm.seen.clear()
+ # two calls, then two more of which one is made: three in all, and the answer is written from what they found
+ llm.tool_script = [
+ {"content": "", "tool_calls": [call(1, "search_transcripts", {"query": "a"}), call(2, "find_entities", {"query": "b"})]},
+ {"content": "", "tool_calls": [call(3, "find_topics", {"query": "c"}), call(4, "list_recordings", {})]},
+ {"content": "", "tool_calls": [call(5, "find_notes", {"query": "d"})]},
+ {"content": "Dyno designs capsids."},
+ ]
+ ev = ask(c, h, cid, max_tool_calls=3)
+ assert [x["tool"] for x in ev["step"]] == ["search_transcripts", "find_entities", "find_topics"]
+ meta = ev["meta"][0]
+ assert (meta["outcome"], meta["limits"], meta["used"]) == ("limit", {"rounds": 3, "tool_calls": 3}, {"rounds": 2, "tool_calls": 3})
+ asked = [b for b in llm.seen if b.get("tools")]
+ assert "up to 3 tool calls for this question in all" in asked[0]["messages"][0]["content"]
+ # the call past the limit is answered too, as the model's API needs, and not made
+ replies = [m for m in asked[-1]["messages"] if m["role"] == "tool"]
+ assert json.loads(replies[-1]["content"])["note"].startswith("This answer's tool calls are used up")
+ assert asked[-1].get("tool_choice") == "none" # then it answers, with the tools off
+
+ # more rounds than ai.max_steps, when the limit allows: one call each
+ llm.tool_script = [{"content": "", "tool_calls": [call(i, "find_topics", {"query": str(i)})]} for i in range(8)] + [
+ {"content": "Found it."}
+ ]
+ meta = ask(c, h, cid, max_tool_calls=8)["meta"][0]
+ assert (meta["outcome"], meta["used"]) == ("limit", {"rounds": 8, "tool_calls": 8})
+ # an answer within its limit answered as usual
+ llm.tool_script = [{"content": "", "tool_calls": [call(1, "find_topics", {"query": "x"})]}, {"content": "Done."}]
+ meta = ask(c, h, cid, max_tool_calls=8)["meta"][0]
+ assert (meta["outcome"], meta["used"]["tool_calls"]) == ("answered", 1)
+
+ # within 1 and ai.max_tool_calls
+ for n, code in ((0, 422), (51, 400)):
+ assert c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "x", "max_tool_calls": n}).status_code == code
+ ca, ha = s.cl["admin"]
+ for bad in (0, 501, "5", True):
+ assert ca.put("/api/v1/settings/ai", headers=ha, json={"max_tool_calls": bad}).status_code == 400, bad
+ assert ca.put("/api/v1/settings/ai", headers=ha, json={"max_tool_calls": 2}).status_code == 200
+ assert c.get("/api/v1/chats/capabilities", headers=h).json()["max_tool_calls"] == 2
+ assert c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "x", "max_tool_calls": 3}).status_code == 400
+
+
+def test_a_conversation_keeps_its_choice_and_new_ones_start_with_it(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ assert c.get(f"/api/v1/chats/{cid}", headers=h).json()["tools"] is None # Auto
+ assert c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": ["archive"], "max_tool_calls": 5}).status_code == 200
+ assert c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": ["nope"]}).status_code == 400
+ got = c.get(f"/api/v1/chats/{cid}", headers=h).json()
+ assert (got["tools"], got["max_tool_calls"]) == (["archive"], 5)
+ assert [x for x in c.get("/api/v1/chats", headers=h).json() if x["id"] == cid][0]["tools"] == ["archive"]
+ # its answers use it, unless a question says otherwise
+ llm.tool_script = [{"content": "Done."}]
+ meta = ask(c, h, cid)["meta"][0]
+ assert (meta["tools"]["picked"], meta["limits"]["tool_calls"]) == (["archive"], 5)
+ assert set(meta["tools"]["offered"]) == set(ai_tools.GROUPS["archive"]["tools"])
+ llm.tool_script = [{"content": "Done."}]
+ meta = ask(c, h, cid, tools=["graph"], max_tool_calls=2)["meta"][0]
+ assert (meta["tools"]["picked"], meta["limits"]["tool_calls"]) == (["graph"], 2)
+ # the latest choice is what new conversations start with, and what the catalog says
+ assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": ["archive"], "max_tool_calls": 5}
+ new = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ got = c.get(f"/api/v1/chats/{new}", headers=h).json()
+ assert (got["tools"], got["max_tool_calls"]) == (["archive"], 5)
+ # unless it says otherwise; back to Auto
+ new = c.post("/api/v1/chats", headers=h, json={"tools": None, "max_tool_calls": None}).json()["id"]
+ got = c.get(f"/api/v1/chats/{new}", headers=h).json()
+ assert (got["tools"], got["max_tool_calls"]) == (None, None)
+ c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": None, "max_tool_calls": None})
+ assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": None, "max_tool_calls": None}
+ # someone else's
+ cv, hv = s.cl["viewer"]
+ assert cv.get("/api/v1/chats/tools", headers=hv).json()["remembered"] == {"tools": None, "max_tool_calls": None}
+ assert cv.get("/api/v1/chats/tools", headers=hv, params={"chat_id": cid}).status_code == 404
+
+
+def test_a_scope_for_one_answer(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["admin"]
+ cid = c.post("/api/v1/chats", headers=h, json={"scope": {"namespaces": ["pods"]}}).json()["id"]
+ llm.tool_script = [{"content": "", "tool_calls": [call(1, "search_transcripts", {"query": "shipment"})]}, {"content": "Friday [1]."}]
+ ev = ask(c, h, cid, scope={"namespaces": ["calls"]})
+ assert ev["meta"][0]["scope"] == {"namespaces": ["calls"]}
+ assert ev["passages"][0] and {p["recording_id"] for p in ev["passages"][0]} == {s.call}
+ got = c.get(f"/api/v1/chats/{cid}", headers=h).json()
+ assert got["scope"] == {"namespaces": ["pods"]} # the conversation keeps its own
+ assert got["messages"][-2]["meta"]["options"]["scope"] == {"namespaces": ["calls"]}
+ # everything one can read, for this answer
+ llm.tool_script = [{"content": "Done."}]
+ assert ask(c, h, cid, scope={})["meta"][0]["scope"] == {}
+ # and only what one can read
+ cv, hv = s.cl["viewer"]
+ vid = cv.post("/api/v1/chats", headers=hv, json={}).json()["id"]
+ r = cv.post(f"/api/v1/chats/{vid}/messages", headers=hv, json={"content": "x", "scope": {"namespaces": ["calls"]}})
+ assert r.status_code == 404
diff --git a/fastapi_backend/tests/api/test_chats.py b/fastapi_backend/tests/api/test_chats.py
index 04e09ba46..0de23bf09 100644
--- a/fastapi_backend/tests/api/test_chats.py
+++ b/fastapi_backend/tests/api/test_chats.py
@@ -159,7 +159,8 @@ def test_capabilities_for_everyone(client, new_client, db, cfg, folder, llm):
make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"})
hv = login(client, "vi@x.io", "viewer password 1")
caps = client.get("/api/v1/chats/capabilities", headers=hv).json()
- assert caps == {"configured": True, "model": "fake", "tools": True, "max_steps": 6, "check": True, "models": ["fake", "fake-large"]}
+ want = {"configured": True, "model": "fake", "tools": True, "max_steps": 6, "max_tool_calls": 50, "check": True}
+ assert caps == {**want, "models": ["fake", "fake-large"]}
assert "base_url" not in json.dumps(caps) and new_client().get("/api/v1/chats/capabilities").status_code == 401
admin = new_client()
assert (
diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx
index 9f2b7f1f9..c74d0ea68 100644
--- a/nextjs-frontend/components/settings/bodies.tsx
+++ b/nextjs-frontend/components/settings/bodies.tsx
@@ -2530,6 +2530,7 @@ const LIMITS = [
"ai.price_in",
"ai.price_out",
"ai.max_steps",
+ "ai.max_tool_calls",
"ai.browser_steps",
"ai.max_transcript_reads",
"ai.compact_model",
@@ -2596,6 +2597,7 @@ function AiBody({ ctx }: { ctx: BodyCtx }) {
+
diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts
index 05684aa54..2f6517d8a 100644
--- a/nextjs-frontend/components/settings/model.ts
+++ b/nextjs-frontend/components/settings/model.ts
@@ -905,6 +905,15 @@ export const FIELDS: FieldSpec[] = [
min: 1,
max: 50,
},
+ {
+ section: "ai",
+ key: "max_tool_calls",
+ label: "Most tool calls a question may ask for",
+ kind: "int",
+ hint: "The highest limit a question or a conversation can set on the tool calls of one answer.",
+ min: 1,
+ max: 500,
+ },
{
section: "ai",
key: "browser",
From 957c3f1bc854ab5cfc20201ca37962ecf5e339f1 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 00:41:22 +0000
Subject: [PATCH 05/14] docs(chat): choosing tools, namespaces and a limit;
regenerate the client
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
CHANGELOG.md | 7 +
docs/api.md | 10 +-
docs/assistant.md | 52 ++-
nextjs-frontend/app/openapi-client/index.ts | 9 +
nextjs-frontend/app/openapi-client/sdk.gen.ts | 34 +-
.../app/openapi-client/types.gen.ts | 226 +++++++++-
nextjs-frontend/openapi.json | 389 +++++++++++++++++-
7 files changed, 720 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f68b7a688..86017937a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
## Unreleased
+- **Choose the tools, a limit and the namespaces of an answer.** A conversation, or one question, can pick the tool
+ groups or tools its answers may use (`tools`), cap their tool calls (`max_tool_calls`, up to the new Settings → AI
+ assistant → Most tool calls a question may ask for, 50 by default) and narrow one answer to its own namespaces
+ (`scope`). `GET /api/v1/chats/tools` lists the groups a person has, what each tool does (reads, asks first, sends data
+ out) and why a group is missing. An admin's switches still win, and new conversations start with the person's last
+ choice. The web app's Tools chip is in progress
+ ([Choosing tools, namespaces and a limit](docs/assistant.md#choosing-tools-namespaces-and-a-limit)).
- **Automations run.** An automation saved over the API (`/api/v1/automations`) now reacts to new assets in its
namespaces: when its If passes, it runs its workflow as the person who made it, within its limits and budgets, and
each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now;
diff --git a/docs/api.md b/docs/api.md
index beed20cb5..33c9771bc 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -1444,6 +1444,7 @@ version.
```
GET /api/v1/chats
GET /api/v1/chats/capabilities
+GET /api/v1/chats/tools
POST /api/v1/chats
GET /api/v1/chats/{cid}
PATCH /api/v1/chats/{cid}
@@ -1467,7 +1468,7 @@ that node runs without asking from now on). Approving, editing and `always` on a
session (403 for an API key or an app); each approval's `can` says what you may do with it.
`GET /chats/capabilities` tells anyone signed in whether a language model is set up (`configured`), which (`model`),
-whether it uses tools (`tools`, `max_steps`), whether answers can be checked (`check`) and the `models` people may pick
+whether it uses tools (`tools`, `max_steps`, `max_tool_calls`), whether answers can be checked (`check`) and the `models` people may pick
(the configured one first; `llm.chat_models`, else the server's list); never the model server's address or key. A
conversation's `model` (`POST /chats`, `PATCH /chats/{cid}`; null for the configured one) answers in it, and a
question can name another (`POST /chats/{cid}/messages {content, model}`, e.g. to retry); a model that isn't offered is
@@ -1480,6 +1481,13 @@ step it's on: the stream sends `stopped`, then `done` with the saved message, wh
everything after it are replaced by this question and its new answer; 404 when `edit` isn't a question in that
conversation. A conversation titled after the first question is retitled when that question is edited.
+`GET /chats/tools` lists the tool groups and tools you have, each tool's `effect` (`read`, `asks` or `out`), why a
+group isn't available and the setting that switches it on, the most tool calls a question may ask for
+(`max_tool_calls`) and your latest choice (`remembered`); `chat_id` reads it for that conversation's scope. A
+conversation's `tools` and `max_tool_calls` (`POST /chats`, `PATCH /chats/{cid}`; null for Auto) are what its answers
+may use, and a question can set its own, with a `scope` for that answer alone (`POST /chats/{cid}/messages {content,
+tools, max_tool_calls, scope}`) ([Choosing tools, namespaces and a limit](assistant.md#choosing-tools-namespaces-and-a-limit)).
+
A conversation's `scope` narrows what it draws on: `namespaces`, `recordings`, `collections`, `speakers`, `from` and
`to`; every key narrows it further, and an empty scope is everything you can read. `collections` are ids of
collections you can see (yours or shared); the conversation draws on their assets as they are each time it
diff --git a/docs/assistant.md b/docs/assistant.md
index 7a5d336aa..39693560d 100644
--- a/docs/assistant.md
+++ b/docs/assistant.md
@@ -152,7 +152,8 @@ When the configured model supports function calling, chat becomes an agent.
(`save_automation` with `enabled`) and running one now wait for the person's approval; switching one off acts at once
([Automations](automations.md#agents)).
- **Limits:** everything is limited to what the person can read and to the conversation's scope, and capped by
- `ai.max_steps` (Settings → AI assistant, Tool steps per answer; 6 by default) and `ai.max_transcript_reads`.
+ `ai.max_steps` (Settings → AI assistant, Tool steps per answer; 6 by default) and `ai.max_transcript_reads`, or by
+ the tool calls a question allows ([below](#choosing-tools-namespaces-and-a-limit)).
- **Always an answer:** the model is told its step budget. When the steps run out, or it replies with nothing, it is
asked once more to answer from what it found, with the tools off, and to say what it didn't get to check. A server
that calls tools anyway gets the conversation again as plain text, without tools. That last ask is one more model
@@ -181,6 +182,55 @@ When the configured model supports function calling, chat becomes an agent.
and type in that browser and download files into the conversation, on sites you allow, asking before anything that
sends, deletes or buys. See [The assistant in your browser](browser-assistant.md).
+### Choosing tools, namespaces and a limit
+
+By default (Auto) an answer may use every tool the person has. A conversation, or one question, can instead pick the
+tools it may use, how many tool calls it may make, and which namespaces it reads.
+
+- **Tool groups.** `GET /api/v1/chats/tools` lists the groups and the tools in each that the person has, and why a
+ group isn't there for them:
+ - `archive` (search and read assets);
+ - `graph` (entities, topics, the knowledge graph);
+ - `notes`;
+ - `memory` (a namespace's own assistant);
+ - `activity` (the timeline, approvals and automations);
+ - `changes` (proposals that become approvals);
+ - `files` (importing files and web pages);
+ - `extensions`;
+ - `server` (admins);
+ - `browser`;
+ - `mcp:` and `ext:`, one for each of the person's extensions.
+
+ Each tool says its `effect`: `read`, `asks` (it waits for the person's approval) or `out` (it sends data out of
+ Lens, as the browser's open, click and type do). A group that isn't available says why (switched off by an admin,
+ for admins, or needs editor access to a namespace) and names the setting that switches it back on (`setting`,
+ such as `ai.browser`).
+- **Picking.** `tools` (on `POST /chats`, `PATCH /chats/` or one question, `POST /chats//messages`) is a list
+ of group and tool names. The model is then offered only those tools, and a tool it calls that it wasn't offered isn't
+ run. Null is Auto. An empty list means no tools: the answer is written from a search, as when the model can't use
+ tools. A name that is no group or tool the person has is a 400. An admin's switches (`ai.disabled_tools`,
+ `ai.browser`, `ai.extensions`) still win over any pick.
+- **A limit on tool calls.** `max_tool_calls` (in the same places) caps how many tool calls one answer may make in
+ all. The model is told the budget. Calls past it get a reply saying they weren't made, and then the answer is written
+ from what was found, with outcome `limit`. With a limit, the rounds of calls may go past `ai.max_steps`, up to the
+ limit. It is a whole number from 1 to `ai.max_tool_calls` (Settings → AI assistant, Most tool calls a question may
+ ask for; 50 by default, at most 500). `GET /chats/capabilities` gives it as `max_tool_calls`.
+- **A scope for one answer.** `scope` on `POST /chats//messages` narrows that answer alone, with the same keys as
+ a conversation's scope. `{}` means everything the person can read, and a namespace they can't read is a 404. The
+ conversation keeps its own scope, and a question with a scope isn't narrowed by the
+ [namespace picker](#picking-the-namespace-for-a-conversation-over-everything).
+- **Remembered.** Each answer records what it was offered and its limits (`tools`, `limits`, `scope`), and each
+ question records the options it was asked with ([What each answer records](#what-each-answer-records)). A
+ conversation keeps its `tools` and `max_tool_calls`. The person's latest choice is kept on their account, so a new
+ conversation starts with it (`remembered` in `GET /chats/tools`) unless `POST /chats` says otherwise.
+- **Web and MCP tools reach beyond your namespaces.** The namespaces picked narrow the archive's tools only. A web
+ search, an MCP server or the browser reads and sends wherever it reaches.
+
+In the web app, a Tools chip beside the composer's scope picks these (in progress).
+
+Refine later: tool choice in chat rooms and voice mode, a default choice per namespace, and the model suggesting the
+tools a question needs.
+
## Third-party content and prompt injection
The assistant reads text nobody vouched for: what was said in recordings, documents, text read off screens, web pages
diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts
index 9219f0cff..de6d5e827 100644
--- a/nextjs-frontend/app/openapi-client/index.ts
+++ b/nextjs-frontend/app/openapi-client/index.ts
@@ -313,6 +313,12 @@ export type {
ChatNode,
ChatScope,
ChatSummary,
+ ChatTools,
+ ChatToolsData,
+ ChatToolsError,
+ ChatToolsErrors,
+ ChatToolsResponse,
+ ChatToolsResponses,
ChatUpdate,
CheckComponentsData,
CheckComponentsResponse,
@@ -3276,6 +3282,9 @@ export type {
TokenPair,
TokenResponse,
TokenResponses,
+ ToolChoice,
+ ToolGroup,
+ ToolInfo,
ToolStep,
TopicAbout,
TopicCandidate,
diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts
index 91c43fc31..8acb7e7d5 100644
--- a/nextjs-frontend/app/openapi-client/sdk.gen.ts
+++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts
@@ -102,6 +102,9 @@ import type {
ChangePasswordResponses,
ChatCapabilitiesData,
ChatCapabilitiesResponses,
+ ChatToolsData,
+ ChatToolsErrors,
+ ChatToolsResponses,
CheckComponentsData,
CheckComponentsResponses,
CheckManifestData,
@@ -9968,6 +9971,9 @@ export class Chats {
/**
* Create Chat
+ *
+ * A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH
+ * /chats/{cid}), else Auto: everything you may use, limited by the rounds alone.
*/
public static createChat(
options?: Options,
@@ -9998,6 +10004,25 @@ export class Chats {
});
}
+ /**
+ * Chat Tools
+ *
+ * The tool groups and tools an answer can be given (`tools` on a question or a conversation), as you have them: each
+ * tool with what it does (read, asks: it waits for your yes, out: it sends what it's given to another host) and
+ * whether you have it now, each group with why you don't when you have none of it, and the setting that changes it.
+ * Built-in groups first, then your extensions' (mcp:, ext:). With `chat_id`, as in that
+ * conversation's scope (a namespace's own memory is there only in a conversation about that namespace). Also: the most
+ * tool calls an answer may be given, and your latest choice, which new conversations start with.
+ */
+ public static chatTools(
+ options?: Options,
+ ): RequestResult {
+ return (options?.client ?? client).get({
+ url: "/api/v1/chats/tools",
+ ...options,
+ });
+ }
+
/**
* Delete Chat
*/
@@ -10028,7 +10053,9 @@ export class Chats {
/**
* Update Chat
*
- * Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one).
+ * Rename a conversation, change what it draws on, the model that answers in it (null: the configured one), or
+ * what its answers may use (`tools`, null: Auto) and at most how many tool calls each makes (`max_tool_calls`, null:
+ * no limit but the rounds). The choice of tools and limit is also what your new conversations start with.
*/
public static updateChat(
options: Options,
@@ -10095,6 +10122,11 @@ export class Chats {
* text, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the
* new version, the message it follows and its versions); so does a question beside others under `parent`.
*
+ * What the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a
+ * search) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's
+ * choice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its
+ * record keeps what it was given and used.
+ *
* `/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the
* conversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.
*
diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts
index 9a4c4cfe5..e12ad2a8d 100644
--- a/nextjs-frontend/app/openapi-client/types.gen.ts
+++ b/nextjs-frontend/app/openapi-client/types.gen.ts
@@ -3383,6 +3383,18 @@ export type Chat = {
* the model chosen for this conversation; null: the configured one
*/
model?: string | null;
+ /**
+ * Tools
+ *
+ * the tool groups and tools its answers may use; null: Auto
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ *
+ * at most this many tool calls in each answer; null: no limit but the rounds
+ */
+ max_tool_calls?: number | null;
/**
* Created At
*/
@@ -3441,9 +3453,15 @@ export type ChatCapabilities = {
/**
* Max Steps
*
- * the most tool calls it makes for one answer
+ * the most rounds of tool calls it makes for one answer (a round makes one call or more)
*/
max_steps: number;
+ /**
+ * Max Tool Calls
+ *
+ * the most tool calls a question may give one answer (`max_tool_calls`)
+ */
+ max_tool_calls?: number;
/**
* Check
*
@@ -3562,6 +3580,18 @@ export type ChatCreate = {
* the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one
*/
model?: string | null;
+ /**
+ * Tools
+ *
+ * what its answers may use: tool groups and tools (GET /chats/tools); [] for none; left out: your latest choice (PATCH /chats/{cid}), else Auto
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ *
+ * at most this many tool calls in each answer (1 to ai.max_tool_calls); left out: your latest choice
+ */
+ max_tool_calls?: number | null;
};
/**
@@ -3742,6 +3772,18 @@ export type ChatSummary = {
* the model chosen for this conversation; null: the configured one
*/
model?: string | null;
+ /**
+ * Tools
+ *
+ * the tool groups and tools its answers may use; null: Auto
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ *
+ * at most this many tool calls in each answer; null: no limit but the rounds
+ */
+ max_tool_calls?: number | null;
/**
* Created At
*/
@@ -3753,6 +3795,35 @@ export type ChatSummary = {
[key: string]: unknown;
};
+/**
+ * ChatTools
+ */
+export type ChatTools = {
+ /**
+ * Groups
+ *
+ * the built-in groups, then your extensions' (mcp:, ext:)
+ */
+ groups: Array;
+ /**
+ * Max Tool Calls
+ *
+ * the most tool calls one answer may be given (ai.max_tool_calls)
+ */
+ max_tool_calls: number;
+ /**
+ * your latest choice, which new conversations start with
+ */
+ remembered: ToolChoice;
+ /**
+ * Namespaces Note
+ *
+ * what the choice of namespaces doesn't narrow
+ */
+ namespaces_note?: string;
+ [key: string]: unknown;
+};
+
/**
* ChatUpdate
*/
@@ -3768,6 +3839,18 @@ export type ChatUpdate = {
* the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one
*/
model?: string | null;
+ /**
+ * Tools
+ *
+ * what its answers may use: tool groups and tools (GET /chats/tools); [] for none; null: Auto (everything you may use). Your new conversations start with it too
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ *
+ * at most this many tool calls in each answer (1 to ai.max_tool_calls); null: no limit but the rounds
+ */
+ max_tool_calls?: number | null;
};
/**
@@ -10871,6 +10954,22 @@ export type MessageCreate = {
* voice: said in a voice conversation (the question's record keeps it)
*/
via?: "voice" | null;
+ /**
+ * Tools
+ *
+ * what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ *
+ * at most this many tool calls in this answer (1 to ai.max_tool_calls); null: the conversation's
+ */
+ max_tool_calls?: number | null;
+ /**
+ * what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read
+ */
+ scope?: ChatScope | null;
};
/**
@@ -21202,6 +21301,101 @@ export type TokenPair = {
[key: string]: unknown;
};
+/**
+ * ToolChoice
+ */
+export type ToolChoice = {
+ /**
+ * Tools
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ */
+ max_tool_calls?: number | null;
+ [key: string]: unknown;
+};
+
+/**
+ * ToolGroup
+ */
+export type ToolGroup = {
+ /**
+ * Name
+ *
+ * what `tools` names it by: a built-in group, mcp: or ext:
+ */
+ name: string;
+ /**
+ * Label
+ */
+ label: string;
+ /**
+ * Tools
+ */
+ tools: Array;
+ /**
+ * Asks
+ *
+ * one of its tools waits for your yes
+ */
+ asks: boolean;
+ /**
+ * Out
+ *
+ * one of its tools sends data to another host
+ */
+ out: boolean;
+ /**
+ * Available
+ *
+ * you have one of its tools now
+ */
+ available: boolean;
+ /**
+ * Why
+ *
+ * why you don't, when you have none
+ */
+ why?: string | null;
+ /**
+ * Setting
+ *
+ * the setting that would change that (section.key), for admins
+ */
+ setting?: string | null;
+ [key: string]: unknown;
+};
+
+/**
+ * ToolInfo
+ */
+export type ToolInfo = {
+ /**
+ * Name
+ */
+ name: string;
+ /**
+ * Effect
+ *
+ * read: it looks things up; asks: it waits for your yes; out: it sends what it's given to another host
+ */
+ effect: "read" | "asks" | "out";
+ /**
+ * Out
+ *
+ * it sends what it's given to another host (and asks first after outside text)
+ */
+ out?: boolean;
+ /**
+ * Available
+ *
+ * you have it now
+ */
+ available: boolean;
+ [key: string]: unknown;
+};
+
/**
* ToolStep
*/
@@ -38825,6 +39019,36 @@ export type ChatCapabilitiesResponses = {
export type ChatCapabilitiesResponse = ChatCapabilitiesResponses[keyof ChatCapabilitiesResponses];
+export type ChatToolsData = {
+ body?: never;
+ path?: never;
+ query?: {
+ /**
+ * Chat Id
+ */
+ chat_id?: number | null;
+ };
+ url: "/api/v1/chats/tools";
+};
+
+export type ChatToolsErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ChatToolsError = ChatToolsErrors[keyof ChatToolsErrors];
+
+export type ChatToolsResponses = {
+ /**
+ * Successful Response
+ */
+ 200: ChatTools;
+};
+
+export type ChatToolsResponse = ChatToolsResponses[keyof ChatToolsResponses];
+
export type DeleteChatData = {
body?: never;
path: {
diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json
index c9f6b8a70..a1f398067 100644
--- a/nextjs-frontend/openapi.json
+++ b/nextjs-frontend/openapi.json
@@ -24891,6 +24891,7 @@
"chats"
],
"summary": "Create Chat",
+ "description": "A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH\n/chats/{cid}), else Auto: everything you may use, limited by the rounds alone.",
"operationId": "create_chat",
"requestBody": {
"content": {
@@ -24955,6 +24956,56 @@
}
}
},
+ "/api/v1/chats/tools": {
+ "get": {
+ "tags": [
+ "chats"
+ ],
+ "summary": "Chat Tools",
+ "description": "The tool groups and tools an answer can be given (`tools` on a question or a conversation), as you have them: each\ntool with what it does (read, asks: it waits for your yes, out: it sends what it's given to another host) and\nwhether you have it now, each group with why you don't when you have none of it, and the setting that changes it.\nBuilt-in groups first, then your extensions' (mcp:, ext:). With `chat_id`, as in that\nconversation's scope (a namespace's own memory is there only in a conversation about that namespace). Also: the most\ntool calls an answer may be given, and your latest choice, which new conversations start with.",
+ "operationId": "chat_tools",
+ "parameters": [
+ {
+ "name": "chat_id",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Chat Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ChatTools"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/v1/chats/{cid}": {
"get": {
"tags": [
@@ -25002,7 +25053,7 @@
"chats"
],
"summary": "Update Chat",
- "description": "Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one).",
+ "description": "Rename a conversation, change what it draws on, the model that answers in it (null: the configured one), or\nwhat its answers may use (`tools`, null: Auto) and at most how many tool calls each makes (`max_tool_calls`, null:\nno limit but the rounds). The choice of tools and limit is also what your new conversations start with.",
"operationId": "update_chat",
"parameters": [
{
@@ -25200,7 +25251,7 @@
"chats"
],
"summary": "Send Message",
- "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nA conversation is a tree of versions (GET /chats/{cid}: `tree`, and each message's `versions`); the question goes\non from `parent` when given (any message of the conversation), else from the version showing (`head`). With\n`regenerate` (an answer's id) its question is answered again, as a new version beside that answer (`content` isn't\nneeded; `model` may pick another). With `edit`, one of your earlier questions is edited: this question becomes a new\nversion beside it and is answered, and the old one stays with what followed it (404 if it isn't a question in this\nconversation). Either keeps the page the question was asked from (where, and the highlighted text; not the page's\ntext, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the\nnew version, the message it follows and its versions); so does a question beside others under `parent`.\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).",
+ "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nA conversation is a tree of versions (GET /chats/{cid}: `tree`, and each message's `versions`); the question goes\non from `parent` when given (any message of the conversation), else from the version showing (`head`). With\n`regenerate` (an answer's id) its question is answered again, as a new version beside that answer (`content` isn't\nneeded; `model` may pick another). With `edit`, one of your earlier questions is edited: this question becomes a new\nversion beside it and is answered, and the old one stays with what followed it (404 if it isn't a question in this\nconversation). Either keeps the page the question was asked from (where, and the highlighted text; not the page's\ntext, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the\nnew version, the message it follows and its versions); so does a question beside others under `parent`.\n\nWhat the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a\nsearch) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's\nchoice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its\nrecord keeps what it was given and used.\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).",
"operationId": "send_message",
"parameters": [
{
@@ -36229,6 +36280,33 @@
"title": "Model",
"description": "the model chosen for this conversation; null: the configured one"
},
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools",
+ "description": "the tool groups and tools its answers may use; null: Auto"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls",
+ "description": "at most this many tool calls in each answer; null: no limit but the rounds"
+ },
"created_at": {
"anyOf": [
{
@@ -36320,7 +36398,13 @@
"max_steps": {
"type": "integer",
"title": "Max Steps",
- "description": "the most tool calls it makes for one answer"
+ "description": "the most rounds of tool calls it makes for one answer (a round makes one call or more)"
+ },
+ "max_tool_calls": {
+ "type": "integer",
+ "title": "Max Tool Calls",
+ "description": "the most tool calls a question may give one answer (`max_tool_calls`)",
+ "default": 50
},
"check": {
"type": "boolean",
@@ -36517,6 +36601,35 @@
],
"title": "Model",
"description": "the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one"
+ },
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "maxItems": 100
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools",
+ "description": "what its answers may use: tool groups and tools (GET /chats/tools); [] for none; left out: your latest choice (PATCH /chats/{cid}), else Auto"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer",
+ "minimum": 1.0
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls",
+ "description": "at most this many tool calls in each answer (1 to ai.max_tool_calls); left out: your latest choice"
}
},
"additionalProperties": false,
@@ -36864,6 +36977,33 @@
"title": "Model",
"description": "the model chosen for this conversation; null: the configured one"
},
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools",
+ "description": "the tool groups and tools its answers may use; null: Auto"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls",
+ "description": "at most this many tool calls in each answer; null: no limit but the rounds"
+ },
"created_at": {
"anyOf": [
{
@@ -36895,6 +37035,41 @@
],
"title": "ChatSummary"
},
+ "ChatTools": {
+ "properties": {
+ "groups": {
+ "items": {
+ "$ref": "#/components/schemas/ToolGroup"
+ },
+ "type": "array",
+ "title": "Groups",
+ "description": "the built-in groups, then your extensions' (mcp:, ext:)"
+ },
+ "max_tool_calls": {
+ "type": "integer",
+ "title": "Max Tool Calls",
+ "description": "the most tool calls one answer may be given (ai.max_tool_calls)"
+ },
+ "remembered": {
+ "$ref": "#/components/schemas/ToolChoice",
+ "description": "your latest choice, which new conversations start with"
+ },
+ "namespaces_note": {
+ "type": "string",
+ "title": "Namespaces Note",
+ "description": "what the choice of namespaces doesn't narrow",
+ "default": "Web and MCP tools reach beyond your namespaces: the namespaces you pick narrow the archive's tools only."
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "groups",
+ "max_tool_calls",
+ "remembered"
+ ],
+ "title": "ChatTools"
+ },
"ChatUpdate": {
"properties": {
"title": {
@@ -36929,6 +37104,35 @@
],
"title": "Model",
"description": "the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one"
+ },
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "maxItems": 100
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools",
+ "description": "what its answers may use: tool groups and tools (GET /chats/tools); [] for none; null: Auto (everything you may use). Your new conversations start with it too"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer",
+ "minimum": 1.0
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls",
+ "description": "at most this many tool calls in each answer (1 to ai.max_tool_calls); null: no limit but the rounds"
}
},
"additionalProperties": false,
@@ -48974,6 +49178,46 @@
],
"title": "Via",
"description": "voice: said in a voice conversation (the question's record keeps it)"
+ },
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "maxItems": 100
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools",
+ "description": "what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer",
+ "minimum": 1.0
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls",
+ "description": "at most this many tool calls in this answer (1 to ai.max_tool_calls); null: the conversation's"
+ },
+ "scope": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ChatScope"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read"
}
},
"additionalProperties": false,
@@ -66791,6 +67035,145 @@
],
"title": "TokenPair"
},
+ "ToolChoice": {
+ "properties": {
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "ToolChoice"
+ },
+ "ToolGroup": {
+ "properties": {
+ "name": {
+ "type": "string",
+ "title": "Name",
+ "description": "what `tools` names it by: a built-in group, mcp: or ext:"
+ },
+ "label": {
+ "type": "string",
+ "title": "Label"
+ },
+ "tools": {
+ "items": {
+ "$ref": "#/components/schemas/ToolInfo"
+ },
+ "type": "array",
+ "title": "Tools"
+ },
+ "asks": {
+ "type": "boolean",
+ "title": "Asks",
+ "description": "one of its tools waits for your yes"
+ },
+ "out": {
+ "type": "boolean",
+ "title": "Out",
+ "description": "one of its tools sends data to another host"
+ },
+ "available": {
+ "type": "boolean",
+ "title": "Available",
+ "description": "you have one of its tools now"
+ },
+ "why": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Why",
+ "description": "why you don't, when you have none"
+ },
+ "setting": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Setting",
+ "description": "the setting that would change that (section.key), for admins"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "name",
+ "label",
+ "tools",
+ "asks",
+ "out",
+ "available"
+ ],
+ "title": "ToolGroup"
+ },
+ "ToolInfo": {
+ "properties": {
+ "name": {
+ "type": "string",
+ "title": "Name"
+ },
+ "effect": {
+ "type": "string",
+ "enum": [
+ "read",
+ "asks",
+ "out"
+ ],
+ "title": "Effect",
+ "description": "read: it looks things up; asks: it waits for your yes; out: it sends what it's given to another host"
+ },
+ "out": {
+ "type": "boolean",
+ "title": "Out",
+ "description": "it sends what it's given to another host (and asks first after outside text)",
+ "default": false
+ },
+ "available": {
+ "type": "boolean",
+ "title": "Available",
+ "description": "you have it now"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "name",
+ "effect",
+ "available"
+ ],
+ "title": "ToolInfo"
+ },
"ToolStep": {
"properties": {
"tool": {
From 8759a60f561100bfa245b401bed942f38d9ac53f Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 00:43:06 +0000
Subject: [PATCH 06/14] feat(chat): a choice made as a conversation starts is
the latest choice
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
docs/assistant.md | 5 +++--
fastapi_backend/app/api/v1/routes/chats.py | 7 +++++--
fastapi_backend/tests/api/test_chat_tools.py | 3 +++
nextjs-frontend/app/openapi-client/sdk.gen.ts | 5 +++--
nextjs-frontend/openapi.json | 2 +-
5 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/docs/assistant.md b/docs/assistant.md
index 39693560d..eb127b4b1 100644
--- a/docs/assistant.md
+++ b/docs/assistant.md
@@ -221,8 +221,9 @@ tools it may use, how many tool calls it may make, and which namespaces it reads
[namespace picker](#picking-the-namespace-for-a-conversation-over-everything).
- **Remembered.** Each answer records what it was offered and its limits (`tools`, `limits`, `scope`), and each
question records the options it was asked with ([What each answer records](#what-each-answer-records)). A
- conversation keeps its `tools` and `max_tool_calls`. The person's latest choice is kept on their account, so a new
- conversation starts with it (`remembered` in `GET /chats/tools`) unless `POST /chats` says otherwise.
+ conversation keeps its `tools` and `max_tool_calls`. The person's latest choice (the last `POST /chats` or `PATCH
+ /chats/` that set them) is kept on their account, so a new conversation starts with it (`remembered` in `GET
+ /chats/tools`) unless `POST /chats` says otherwise.
- **Web and MCP tools reach beyond your namespaces.** The namespaces picked narrow the archive's tools only. A web
search, an MCP server or the browser reads and sends wherever it reaches.
diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py
index 17a679cff..3b649b82c 100644
--- a/fastapi_backend/app/api/v1/routes/chats.py
+++ b/fastapi_backend/app/api/v1/routes/chats.py
@@ -149,8 +149,9 @@ def _limit(cfg: dict[str, Any], n: int | None) -> int | None:
@router.post("/chats")
def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body: ChatCreate | None = None) -> Created:
- """A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH
- /chats/{cid}), else Auto: everything you may use, limited by the rounds alone."""
+ """A conversation. Its answers may use what `tools` and `max_tool_calls` say, which become your latest choice;
+ left out, your latest choice (here or PATCH /chats/{cid}), else Auto: everything you may use, limited by the rounds
+ alone."""
body = body or ChatCreate()
if body.kind == "setup" and not user.admin:
raise HTTPException(403, "only admins set up the server")
@@ -158,6 +159,8 @@ def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body
tools = _tools(_box(db, cfg, user, acl, None, request), body.tools) if "tools" in body.model_fields_set else last["tools"]
limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in body.model_fields_set else _most(cfg, last["max_tool_calls"])
cid = chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind, tools, limit)
+ if body.kind == "chat" and {"tools", "max_tool_calls"} & body.model_fields_set:
+ chat.remember_tools(db, user.id, tools, limit)
return Created(id=cid)
diff --git a/fastapi_backend/tests/api/test_chat_tools.py b/fastapi_backend/tests/api/test_chat_tools.py
index 44ecb2a9b..b89e4f743 100644
--- a/fastapi_backend/tests/api/test_chat_tools.py
+++ b/fastapi_backend/tests/api/test_chat_tools.py
@@ -215,6 +215,9 @@ def test_a_conversation_keeps_its_choice_and_new_ones_start_with_it(app, db, cfg
new = c.post("/api/v1/chats", headers=h, json={"tools": None, "max_tool_calls": None}).json()["id"]
got = c.get(f"/api/v1/chats/{new}", headers=h).json()
assert (got["tools"], got["max_tool_calls"]) == (None, None)
+ # a choice made as one starts is the latest choice too
+ c.post("/api/v1/chats", headers=h, json={"tools": ["notes"], "max_tool_calls": 4})
+ assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": ["notes"], "max_tool_calls": 4}
c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": None, "max_tool_calls": None})
assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": None, "max_tool_calls": None}
# someone else's
diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts
index 8acb7e7d5..44ef4f17a 100644
--- a/nextjs-frontend/app/openapi-client/sdk.gen.ts
+++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts
@@ -9972,8 +9972,9 @@ export class Chats {
/**
* Create Chat
*
- * A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH
- * /chats/{cid}), else Auto: everything you may use, limited by the rounds alone.
+ * A conversation. Its answers may use what `tools` and `max_tool_calls` say, which become your latest choice;
+ * left out, your latest choice (here or PATCH /chats/{cid}), else Auto: everything you may use, limited by the rounds
+ * alone.
*/
public static createChat(
options?: Options,
diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json
index a1f398067..3e178c3a3 100644
--- a/nextjs-frontend/openapi.json
+++ b/nextjs-frontend/openapi.json
@@ -24891,7 +24891,7 @@
"chats"
],
"summary": "Create Chat",
- "description": "A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH\n/chats/{cid}), else Auto: everything you may use, limited by the rounds alone.",
+ "description": "A conversation. Its answers may use what `tools` and `max_tool_calls` say, which become your latest choice;\nleft out, your latest choice (here or PATCH /chats/{cid}), else Auto: everything you may use, limited by the rounds\nalone.",
"operationId": "create_chat",
"requestBody": {
"content": {
From 73d10e79edbc70d55ab48d7799a460238877fd2b Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 00:57:41 +0000
Subject: [PATCH 07/14] fix(chat): setting the server up keeps every tool,
whatever the latest choice
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
fastapi_backend/app/api/v1/routes/chats.py | 6 ++++--
fastapi_backend/app/schemas/chats.py | 2 +-
fastapi_backend/tests/api/test_chat_tools.py | 5 +++++
nextjs-frontend/app/openapi-client/types.gen.ts | 2 +-
nextjs-frontend/openapi.json | 2 +-
5 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py
index 3b649b82c..651a0a22a 100644
--- a/fastapi_backend/app/api/v1/routes/chats.py
+++ b/fastapi_backend/app/api/v1/routes/chats.py
@@ -155,7 +155,8 @@ def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body
body = body or ChatCreate()
if body.kind == "setup" and not user.admin:
raise HTTPException(403, "only admins set up the server")
- last = chat.remembered_tools(db, user.id)
+ # setting the server up keeps every tool: your latest choice is for conversations about the archive
+ last = chat.remembered_tools(db, user.id) if body.kind == "chat" else {"tools": None, "max_tool_calls": None}
tools = _tools(_box(db, cfg, user, acl, None, request), body.tools) if "tools" in body.model_fields_set else last["tools"]
limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in body.model_fields_set else _most(cfg, last["max_tool_calls"])
cid = chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind, tools, limit)
@@ -245,7 +246,8 @@ def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg:
tools = _tools(_box(db, cfg, user, acl, c.get("scope"), request), body.tools) if "tools" in chose else c.get("tools")
limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in chose else c.get("max_tool_calls")
db.q("UPDATE $r SET tools = $t, max_tool_calls = $n", r=R("chat", cid), t=tools, n=limit)
- chat.remember_tools(db, user.id, tools, limit)
+ if c.get("kind") == "chat":
+ chat.remember_tools(db, user.id, tools, limit)
return Ok()
diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py
index 4c8a7043d..9a2ce6dd1 100644
--- a/fastapi_backend/app/schemas/chats.py
+++ b/fastapi_backend/app/schemas/chats.py
@@ -299,7 +299,7 @@ class MessageCreate(RequestModel):
tools: list[str] | None = Field(
None,
max_length=100,
- description="what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists "
+ description="what this answer may use: tool groups (archive, notes, mcp:tracker) and tools, as GET /chats/tools lists "
"them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An "
"admin's switch still wins",
)
diff --git a/fastapi_backend/tests/api/test_chat_tools.py b/fastapi_backend/tests/api/test_chat_tools.py
index b89e4f743..39f41151a 100644
--- a/fastapi_backend/tests/api/test_chat_tools.py
+++ b/fastapi_backend/tests/api/test_chat_tools.py
@@ -220,6 +220,11 @@ def test_a_conversation_keeps_its_choice_and_new_ones_start_with_it(app, db, cfg
assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": ["notes"], "max_tool_calls": 4}
c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": None, "max_tool_calls": None})
assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": None, "max_tool_calls": None}
+ # setting the server up keeps every tool
+ ca, ha = s.cl["admin"]
+ ca.patch(f"/api/v1/chats/{ca.post('/api/v1/chats', headers=ha, json={}).json()['id']}", headers=ha, json={"tools": ["archive"]})
+ setup = ca.post("/api/v1/chats", headers=ha, json={"kind": "setup"}).json()["id"]
+ assert ca.get(f"/api/v1/chats/{setup}", headers=ha).json()["tools"] is None
# someone else's
cv, hv = s.cl["viewer"]
assert cv.get("/api/v1/chats/tools", headers=hv).json()["remembered"] == {"tools": None, "max_tool_calls": None}
diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts
index e12ad2a8d..5dac559b2 100644
--- a/nextjs-frontend/app/openapi-client/types.gen.ts
+++ b/nextjs-frontend/app/openapi-client/types.gen.ts
@@ -10957,7 +10957,7 @@ export type MessageCreate = {
/**
* Tools
*
- * what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins
+ * what this answer may use: tool groups (archive, notes, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins
*/
tools?: Array | null;
/**
diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json
index 3e178c3a3..3cc88e66a 100644
--- a/nextjs-frontend/openapi.json
+++ b/nextjs-frontend/openapi.json
@@ -49193,7 +49193,7 @@
}
],
"title": "Tools",
- "description": "what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins"
+ "description": "what this answer may use: tool groups (archive, notes, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins"
},
"max_tool_calls": {
"anyOf": [
From 3f0b9de311e8e838007a51f32d2915f77a914aad Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 02:09:35 +0000
Subject: [PATCH 08/14] test(entities): wait for the typed search before
counting merge options
The find box empties while the list for what was typed loads, so on a busy
runner the first list could pass the wait and then vanish.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
nextjs-frontend/__tests__/entities-curate.test.tsx | 2 ++
1 file changed, 2 insertions(+)
diff --git a/nextjs-frontend/__tests__/entities-curate.test.tsx b/nextjs-frontend/__tests__/entities-curate.test.tsx
index e7c7e7473..65c89e21e 100644
--- a/nextjs-frontend/__tests__/entities-curate.test.tsx
+++ b/nextjs-frontend/__tests__/entities-curate.test.tsx
@@ -79,6 +79,8 @@ describe("merging into an entity", () => {
const box = screen.getByRole("combobox", { name: "Entity to merge into Northwind Labs" });
expect(box).toHaveFocus();
fireEvent.change(box, { target: { value: "north" } });
+ // the list for what was typed, not the first one (which empties while the typed one loads, on a slow runner)
+ await waitFor(() => expect(m(Entities.listEntities).mock.calls.some(([o]) => o.query.q === "north")).toBe(true));
// itself and the entities that are always there can't be merged
await waitFor(() => expect(screen.getAllByRole("option")).toHaveLength(1));
expect(screen.getByRole("option")).toHaveTextContent("North Wind Labs");
From e6b5cfa32e51c1693a65eef8a151ec93e7ba19a0 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 03:19:45 +0000
Subject: [PATCH 09/14] test(settings): the AI section's server values include
ai.max_tool_calls
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
nextjs-frontend/__tests__/settings-web-search.test.tsx | 1 +
1 file changed, 1 insertion(+)
diff --git a/nextjs-frontend/__tests__/settings-web-search.test.tsx b/nextjs-frontend/__tests__/settings-web-search.test.tsx
index 3081ea8a1..9e164a653 100644
--- a/nextjs-frontend/__tests__/settings-web-search.test.tsx
+++ b/nextjs-frontend/__tests__/settings-web-search.test.tsx
@@ -32,6 +32,7 @@ const web = (values: Record) =>
injection_check: "patterns",
disabled_tools: [],
max_steps: 6,
+ max_tool_calls: 50,
browser_steps: 25,
max_transcript_reads: 20,
confirm_over_recordings: 100,
From a9e21e57fb5290098b7d85dcdd8ac292b3417830 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 04:13:27 +0000
Subject: [PATCH 10/14] fix(chat): find_resources belongs to the archive's
tools
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
fastapi_backend/app/domain/ai_tools.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py
index 554c9306c..7fde22b22 100644
--- a/fastapi_backend/app/domain/ai_tools.py
+++ b/fastapi_backend/app/domain/ai_tools.py
@@ -537,7 +537,7 @@ def builtin_names():
GROUPS = {
"archive": {
"label": "Archive",
- "tools": ["search_transcripts", "find_similar", "list_recordings", "read_transcript", "recording_outputs"],
+ "tools": ["search_transcripts", "find_resources", "find_similar", "list_recordings", "read_transcript", "recording_outputs"],
},
"graph": {
"label": "Graph",
From 98b7a8bb29d76bacd00565938a2e4f1de3b3b607 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 04:18:03 +0000
Subject: [PATCH 11/14] docs(chat): a new built-in tool joins a tool group
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
fastapi_backend/app/domain/ai_tools.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py
index 1248ab17e..6e9375bb2 100644
--- a/fastapi_backend/app/domain/ai_tools.py
+++ b/fastapi_backend/app/domain/ai_tools.py
@@ -551,8 +551,9 @@ def builtin_names():
BROWSER_OUT = {"browser_open", "browser_click", "browser_type", "browser_press"}
# The groups of tools a question can be given (POST /chats/{cid}/messages `tools`; docs/assistant.md, "Choosing tools,
-# namespaces and a limit"), in the order the web app shows them, each built-in tool in one. Extensions add their own:
-# mcp: for an MCP server's tools, ext: for another extension's (extensions.Active.groups).
+# namespaces and a limit"), in the order the web app shows them, each built-in tool in one: a new tool joins one here
+# (test_chat_tools checks). Extensions add their own: mcp: for an MCP server's tools, ext: for another
+# extension's (extensions.Active.groups).
GROUPS = {
"archive": {
"label": "Archive",
From 1b55d2b692f45ee8d40eb9a12f2f3d724c5c8444 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 05:13:18 +0000
Subject: [PATCH 12/14] fix(ci): the upgrade check counts entities of every
type
Ported from #512 so this PR's upgrade check passes while that lands: the
entity-types step re-types weekday Terms as Dates, which the default entity
list leaves out, so the check read them as gone.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
ci/upgrade/lens_upgrade.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/ci/upgrade/lens_upgrade.py b/ci/upgrade/lens_upgrade.py
index a0b3c75b2..8593e2fd6 100755
--- a/ci/upgrade/lens_upgrade.py
+++ b/ci/upgrade/lens_upgrade.py
@@ -192,7 +192,10 @@ def facts(api, token):
c["text"] for c in api.get(f"/resources/{rid}/comments")
)
entities = api.get("/entities", namespaces=NS, limit=200)["items"]
- out["entities"] = sorted(e["name"] for e in entities)
+ # every entity by name, whatever its type: an upgrade step may type a Term (Monday becomes a Date, which the list
+ # leaves out unless asked for), but none may go
+ quiet = api.get("/entities", namespaces=NS, limit=200, types="DATE,NUMBER")["items"]
+ out["entities"] = sorted(e["name"] for e in entities + quiet)
for e in entities:
if e["name"] == ENTITY["name"]:
out[f"entity {e['name']}"] = {
From 3256021cf08b50b34eace7682525b89ce06b0f22 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 05:52:23 +0000
Subject: [PATCH 13/14] feat(chat): deep research, a plan to approve and a
cited report
A goal (/research, research: true, POST /chats/{cid}/research, the assistant's and MCP's start_research) gets one planning call: follow-up questions and a plan with a labelled estimate, waiting as an approval of the assistant's (tool research). Approved in the web app, the inbox or with a yes in a chat room (an API key or an app only declines, unless research.agents_skip_review), it runs in the background: claimed by one process with a heartbeat, resumed from the last finished stage, one run per person, within its tool calls, research.max_minutes and budgets, stoppable. Gather uses only the plan's sources' read tools; write and check renumber citations and rewrite or mark unsupported claims. The report saves as a note (chat links) or makes a podcast.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
CHANGELOG.md | 9 +
docs/assistant.md | 67 +
docs/database.md | 4 +-
fastapi_backend/app/api/mcp_tools.py | 108 ++
fastapi_backend/app/api/v1/router.py | 2 +
fastapi_backend/app/api/v1/routes/chats.py | 87 +-
fastapi_backend/app/api/v1/routes/notebook.py | 5 +-
fastapi_backend/app/api/v1/routes/research.py | 196 +++
fastapi_backend/app/domain/ai_tools.py | 53 +-
fastapi_backend/app/domain/approvals.py | 14 +-
fastapi_backend/app/domain/bridge.py | 41 +-
fastapi_backend/app/domain/chat.py | 7 +-
fastapi_backend/app/domain/notebook.py | 15 +-
fastapi_backend/app/domain/research.py | 1197 +++++++++++++++++
fastapi_backend/app/domain/routines.py | 4 +-
fastapi_backend/app/domain/settings.py | 21 +
fastapi_backend/app/domain/store.py | 10 +
fastapi_backend/app/schemas/chats.py | 8 +
fastapi_backend/app/schemas/research.py | 138 ++
fastapi_backend/tests/api/_assist.py | 1 +
fastapi_backend/tests/api/test_mcp.py | 1 +
fastapi_backend/tests/api/test_research.py | 583 ++++++++
fastapi_backend/tests/fake_llm.py | 31 +
nextjs-frontend/app/openapi-client/index.ts | 55 +
nextjs-frontend/app/openapi-client/sdk.gen.ts | 170 +++
.../app/openapi-client/types.gen.ts | 706 ++++++++++
nextjs-frontend/openapi.json | 1183 +++++++++++++++-
threat-model.json | 19 +-
threat-model.md | 4 +-
threat-model.yaml | 21 +-
30 files changed, 4729 insertions(+), 31 deletions(-)
create mode 100644 fastapi_backend/app/api/v1/routes/research.py
create mode 100644 fastapi_backend/app/domain/research.py
create mode 100644 fastapi_backend/app/schemas/research.py
create mode 100644 fastapi_backend/tests/api/test_research.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 27fcd57a7..7d076581e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,15 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
## Unreleased
+- **Deep research.** Give the assistant a goal (`/research `, `research: true` on a question, `POST
+ /api/v1/chats//research`, its `start_research` tool, or MCP) and it plans the research: up to three follow-up
+ questions, the sub-questions, where it looks (the archive, the graph, notes, the web, the code sandbox), the tool
+ calls it may make, the report's outline, and a labelled cost estimate. Approve the plan as it is or edited, revise
+ it, or say "yes" in a chat room; an API key or an app only declines it unless `research.agents_skip_review` is on.
+ It runs in the background, one run per person at a time, within its tool calls, `research.max_minutes` and your
+ budgets, survives a restart, and stops when asked; the cited report comes back in the conversation, its claims
+ checked against their sources, and saves as a note or makes a podcast episode. The web app's plan and progress
+ cards are in progress ([Deep research](docs/assistant.md#deep-research)).
- **Choose the tools, a limit and the namespaces of an answer.** A conversation, or one question, can pick the tool
groups or tools its answers may use (`tools`), cap their tool calls (`max_tool_calls`, up to the new Settings → AI
assistant → Most tool calls a question may ask for, 50 by default) and narrow one answer to its own namespaces
diff --git a/docs/assistant.md b/docs/assistant.md
index bb1a27587..f46a50289 100644
--- a/docs/assistant.md
+++ b/docs/assistant.md
@@ -256,6 +256,7 @@ tools it may use, how many tool calls it may make, and which namespaces it reads
- `changes` (proposals that become approvals);
- `files` (importing files and web pages);
- `code` (the [code sandbox](#code-sandbox): Python and bash, once an admin turns it on);
+ - `research` ([deep research](#deep-research): `start_research` plans research that waits for your approval);
- `extensions`;
- `server` (admins);
- `browser`;
@@ -340,6 +341,72 @@ gateway on the host, or OpenShell's MicroVM driver); the Podman driver for hosts
API for agents (today an agent asks the assistant); one real run against OpenShell in CI (the tests use a fake
gateway with OpenShell's own client).
+## Deep research
+
+Built in the API, MCP, chat rooms and the assistant's tools; the web app's plan and progress cards are in progress.
+Give the assistant a goal that needs many look-ups, and it comes back with a plan; once you approve it, the research
+runs in the background and a cited report comes back in the conversation.
+
+- **Asking.** Start a question with `/research` (`/research what changed in the capsid work this year?`), send it with
+ `research: true` (`POST /api/v1/chats//messages`), call `POST /api/v1/chats//research {goal}`, or ask the
+ assistant for research: its `start_research` tool (the `research` [tool group](#choosing-tools-namespaces-and-a-limit))
+ works from chat, voice and [chat rooms](chat-rooms.md). It needs a language model that uses tools.
+- **The plan.** One model call reads the goal and writes up to three follow-up questions, each with answers to pick
+ from, and a plan: a title, the sub-questions it will answer, where it will look (`archive`, `graph`, `notes`, `web`,
+ `code`: only the sources this person has, and of those only what the conversation's choice of tools keeps), the
+ most tool calls it may make, and the report's outline. With it comes an estimate of what it will cost, always
+ labelled an estimate: from what each tool call of past research runs cost in the activity ledger, else its likely
+ tokens at the model's price in Settings (no figure when the model has no price). The plan is the answer to the
+ question, and it waits as an approval of the assistant's (tool `research`).
+- **Approve, edit or revise.** Approve it as it is, or with your edits (`POST /api/v1/approvals/` with
+ `decision: approve` and `params`: any of `title`, `sub_questions`, `sources`, `max_tool_calls`, `report_outline`), in
+ the conversation or the Approvals page; or send your answers and edits back for one more planning call (`POST
+ /api/v1/research//revise {answers, plan}`), which changes the plan and its approval in place. In a chat room,
+ "yes" approves the plan as it is. Decline it and nothing runs. `review: false` on `POST /chats//research` starts
+ it at once, as your own pick.
+- **Agents only decline.** An API key, an app or an agent (MCP, the assistant) can decline a plan but not approve it,
+ as for every approval, unless an admin turns on `research.agents_skip_review` (off by default): then an agent's plan
+ starts without review. The `research` settings are saved with `PUT /api/v1/settings/research`; their place in
+ Settings → AI assistant is in progress.
+- **The run.** Once approved, one process runs it at a time (a claim it keeps with a heartbeat every 10 seconds;
+ another process takes it over after two minutes of silence and goes on from the last finished step), in stages:
+ - *gather*: each sub-question in turn, with the assistant's tool loop, its share of the tool calls that are left,
+ and the plan's sources' tools alone (read tools only). Searches and page reads on the web, and code runs that
+ reach the web, don't ask each time: approving the plan approved them. Any other tool that changes something
+ still asks.
+ - *write*: one call writes the report in Markdown, following the outline, citing [n] across every finding,
+ numbered as one list.
+ - *check*: each cited claim is checked against what it cites ([check the sources](#chat)); one that isn't
+ supported is rewritten once, or marked "(not supported by its sources)".
+ - *done*: the report is an answer in the conversation (`kind: research`), after its last message, with its
+ [record](#what-each-answer-records) (`research` is its id; its tokens and cost are the whole run's).
+- **Limits.** A person has one run going at a time; the next waits its turn. `research.max_tool_calls` (40) caps a
+ plan's tool calls, and `research.max_minutes` (20) a run's time: past it, the report is written from what it found
+ and says which sub-questions it didn't get to (outcome `limit`). `research.model` picks another model for research
+ (empty: the conversation's).
+- **Budgets.** Before each stage the [budgets](budgets.md) of the conversation's namespaces and `all` are checked. One
+ that is over holds the run (`held`, saying why) until the budget allows it, or until you let it go on anyway (`POST
+ /api/v1/research//resume`); a budget that skips stops it with what it found.
+- **Progress and stopping.** `GET /api/v1/research/` gives its status (planned, declined, queued, running, held,
+ done, stopped, failed), stage, and progress: sub-questions done, sources found, tool calls, minutes, and tokens and
+ cost so far. `GET /api/v1/research?chat_id=` lists a conversation's. `POST /api/v1/research//stop` stops it
+ after the step it's on (at once while it waits), and what it found so far is saved in the conversation as its
+ answer, marked stopped.
+- **After.** `POST /api/v1/research//note` saves the report as a [note](notes.md) in the namespace it looked in
+ (`namespace` when there are several), its archive citations as mention links to what they cite, web ones as links,
+ and a link back to the conversation (`@[Research: title](chat:12)`), which only you see named. `POST
+ /api/v1/research//podcast` makes a [podcast episode](podcasts.md) from the assets the report cites, about its
+ goal. The report reads aloud and exports like any answer.
+- **Agents.** MCP has `start_research` (a new conversation unless `chat_id` is given; `namespaces`, `tools`,
+ `max_tool_calls`) and `research_status` (the plan and progress, and once it's done the report with its sources and
+ their links).
+- **Cost.** Planning is one call; gathering makes at most the plan's tool calls, each round a call; writing and
+ checking are two or three calls. Every call is in the activity ledger for the conversation and `research:`.
+
+Refine later: sub-questions in parallel, research on a schedule, sharing a report with a namespace's members, research
+as an automation action, a report template people can edit, and the report posted back into the chat room it was
+asked in (today the room gets the plan, and the report is in the conversation in Lens).
+
## Third-party content and prompt injection
The assistant reads text nobody vouched for: what was said in recordings, documents, text read off screens, web pages
diff --git a/docs/database.md b/docs/database.md
index ef27b36fc..de85f27c8 100644
--- a/docs/database.md
+++ b/docs/database.md
@@ -90,7 +90,9 @@ Installs keep their data across updates (running `install.sh` again, a new image
indexes arrive through `SCHEMA` on the next start. A field that is only added needs no step when its readers treat it
as missing on older rows: a chat message's record (`meta`,
[What each answer records](assistant.md#what-each-answer-records)) is one, and messages from before it simply have
-none. Anything else, such as rewriting stored values, moving data between
+none. [Deep research](assistant.md#deep-research) is another: its tables (`research`, and `research_run` for the claim
+of the process running one) are new, a message's `kind` is missing on older ones, and notes accept `chat` links
+without rewriting any. Anything else, such as rewriting stored values, moving data between
tables, or changing or removing an index that already exists, is a named step in `app/domain/migrations.py`:
* Every process that opens the database (the API, workers, `lens watch`, `lens` commands) runs the steps the database
diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py
index 39bab5979..a3fbbe4f7 100644
--- a/fastapi_backend/app/api/mcp_tools.py
+++ b/fastapi_backend/app/api/mcp_tools.py
@@ -29,11 +29,13 @@
from app.api.v1.routes import activities as activity_routes
from app.api.v1.routes import automation_vars as var_routes
from app.api.v1.routes import automations as automation_routes
+from app.api.v1.routes import chats as chat_routes
from app.api.v1.routes import entities as entity_routes
from app.api.v1.routes import graph as graph_routes
from app.api.v1.routes import namespaces as namespace_routes
from app.api.v1.routes import notebook as note_routes
from app.api.v1.routes import recordings as recording_routes
+from app.api.v1.routes import research as research_routes
from app.api.v1.routes import routines as routine_routes
from app.api.v1.routes import search as search_routes
from app.api.v1.routes import topics as topic_routes
@@ -43,6 +45,7 @@
approvals,
auth,
blobs,
+ chat,
decide,
events,
feeds,
@@ -55,6 +58,7 @@
nsfw,
rdf,
render,
+ research,
settings,
sources,
store,
@@ -78,6 +82,7 @@
AutomationTry,
AutomationUpdate,
)
+from app.schemas.chats import ChatScope
from app.schemas.entities import EntityHide, EntityList, EntityRename, EntityRetype, EntityUpdate
from app.schemas.notebook import NotePage
from app.schemas.search import SearchResults
@@ -2340,6 +2345,109 @@ def decline_approval(ctx: Context, id: int) -> dict[str, Any]:
return {"approval": id, "status": out.get("status")}
+# ---------- deep research (docs/assistant.md#deep-research) ----------
+@tool(
+ "start_research",
+ "Start deep research",
+ "Research a goal for the person: one planning call makes a plan (the sub-questions, where it looks: the archive, the "
+ "graph, notes, the web or the code sandbox, the tool calls it may make, the report's outline) and up to three "
+ "follow-up questions. The plan waits for the person's approval in Lens (url), unless the admin let agents start "
+ "research without review (research.agents_skip_review); approved, it runs in the background and its cited report "
+ "comes back in the conversation: follow it with research_status. Needs a token with the write scope.",
+ Arg("goal", "string", "what to research, in the person's words", required=True, max_length=4000),
+ Arg("chat_id", "integer", "the conversation to research in; a new one when left out", minimum=1),
+ Arg("namespaces", "string[]", "only these namespaces (a new conversation's scope)"),
+ Arg("tools", "string[]", "what it may use: tool groups (archive, graph, notes, web, code) and tools; else the conversation's choice"),
+ Arg("max_tool_calls", "integer", "the most tool calls it may make (research.max_tool_calls at most)", minimum=1, maximum=500),
+ writes=True,
+)
+def start_research(
+ ctx: Context, goal: str, chat_id: int | None, namespaces: list[str] | None, tools: list[str] | None, max_tool_calls: int | None
+) -> dict[str, Any]:
+ _writer(ctx, "start research")
+ if not research.available(ctx.cfg):
+ raise ToolError("research needs a language model that uses tools: an admin sets one up in Settings › AI assistant")
+ asked = [ctx.ns] if ctx.ns else namespaces
+ scope = chat_routes._scope(ctx.acl, ctx.db, ChatScope(namespaces=asked)) if asked else None
+ if chat_id is not None:
+ c = chat_routes._own_chat(ctx.db, chat_id, ctx.user)
+ scope = scope if scope is not None else c.get("scope")
+ else:
+ chat_id = chat.create(ctx.db, ctx.user.id, goal[:80], scope)
+ c = chat.get(ctx.db, chat_id, ctx.user.id)
+ box = research_routes.toolbox(ctx.db, ctx.cfg, ctx.user, ctx.acl, chat_id, scope, None, ctx.request)
+ if tools is not None and box.unknown(tools):
+ raise ToolError(f"no tool or tool group {box.unknown(tools)[0]!r}")
+ box = research_routes.toolbox(
+ ctx.db, ctx.cfg, ctx.user, ctx.acl, chat_id, scope, tools if tools is not None else c.get("tools"), ctx.request
+ )
+ skip = research.settings(ctx.cfg)["agents_skip_review"]
+ out = research.start(
+ ctx.db,
+ ctx.cfg,
+ box,
+ " ".join(goal.split()),
+ "api",
+ model=research.model_for(ctx.cfg),
+ max_tool_calls=max_tool_calls,
+ options={"tools": tools, "max_tool_calls": max_tool_calls},
+ review=not skip,
+ )
+ if out.get("error"):
+ raise ToolError(out["error"])
+ r = research.view(ctx.db, out["research"])
+ return {
+ "research": r["id"],
+ "chat_id": chat_id,
+ "status": r["status"],
+ "plan": r["plan"],
+ "questions": r["questions"],
+ "estimate": r["estimate"],
+ "approval": r["approval"],
+ "url": f"{ctx.web}/chat/{chat_id}",
+ "next": "running: follow it with research_status"
+ if r["status"] in ("queued", "running")
+ else "waiting for the person to approve the plan in Lens (url); follow it with research_status",
+ }
+
+
+@tool(
+ "research_status",
+ "Research status",
+ "Where a deep research run stands: its plan and status (planned: waiting for the person's approval; queued, "
+ "running, held by a budget, done, stopped, failed), its stage (gather, write, check, done) and progress "
+ "(sub-questions done, sources found, tool calls and cost so far). Once it's done (or stopped), the report in "
+ "Markdown with its numbered sources, each with a link.",
+ Arg("id", "integer", "the research (start_research's)", required=True, minimum=1),
+)
+def research_status(ctx: Context, id: int) -> dict[str, Any]: # noqa: A002 - the argument's name
+ try:
+ r = research.mine(ctx.db, id, ctx.user.id)
+ except KeyError:
+ raise ToolError(f"no research {id} of yours") from None
+ out = research.view(ctx.db, r)
+ out["url"] = f"{ctx.web}/chat/{r['chat']}"
+ if r.get("answer"):
+ m = ctx.db.one("SELECT content, passages FROM $r", r=R("chat_message", r["answer"])) or {}
+ ok = None if ctx.user.admin else set(ctx.acl.spaces())
+ spaces = {
+ x["id"]: x["space"]
+ for x in ctx.db.rows(
+ "SELECT record::id(id) AS id, space FROM recording WHERE id IN $ids",
+ ids=[R("recording", p["recording_id"]) for p in m.get("passages") or [] if p.get("recording_id") is not None],
+ )
+ }
+ sources = []
+ for p in m.get("passages") or []:
+ if p.get("source") == "web":
+ sources.append({"n": p["n"], "title": p.get("title"), "url": p.get("url"), "site": p.get("site")})
+ elif p.get("recording_id") in spaces and (ok is None or spaces[p["recording_id"]] in ok):
+ sources.append({"n": p["n"], "title": p.get("title"), "at": p.get("time"), "url": ctx.link(p["recording_id"], p.get("t0"))})
+ out["report"] = m.get("content")
+ out["sources"] = sources
+ return out
+
+
@tool(
"event_kinds",
"Kinds of activity",
diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py
index 8243c6574..88f5cae60 100644
--- a/fastapi_backend/app/api/v1/router.py
+++ b/fastapi_backend/app/api/v1/router.py
@@ -54,6 +54,7 @@
rdf,
recordings,
requests,
+ research,
routines,
search,
searches,
@@ -133,6 +134,7 @@
extensions,
marketplaces,
chats,
+ research,
browser,
collections,
batches,
diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py
index 4fd24084e..86597bc33 100644
--- a/fastapi_backend/app/api/v1/routes/chats.py
+++ b/fastapi_backend/app/api/v1/routes/chats.py
@@ -2,7 +2,7 @@
A question streams back server-sent events: ``branch`` (a new version: regenerated, edited, or beside others),
``scoped`` (the namespace a conversation over everything was narrowed
-to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error``, ``meta`` (the answer's record) and ``done``.
+to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``research`` (a research goal's plan, research.py), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error``, ``meta`` (the answer's record) and ``done``.
The assistant only ever sees, and cites, what the asker can read.
"""
@@ -20,7 +20,7 @@
from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Principal, Writer
from app.api.media import sign_urls
from app.api.v1.routes.collections import own_collection
-from app.domain import activity, ai_tools, approvals, auth, auto_scope, browser, chat, llm, sandbox, store, uploads
+from app.domain import activity, ai_tools, approvals, auth, auto_scope, browser, chat, llm, research, sandbox, store, uploads
from app.domain.store import DB
from app.schemas.chats import (
AnswerCheck,
@@ -281,6 +281,71 @@ def _ev(name: str, data: Any) -> str:
return f"event: {name}\ndata: {json.dumps(sign_urls(data, full=True), default=str)}\n\n"
+def _research_reply(
+ cid: int,
+ c: dict[str, Any],
+ goal: str,
+ said: str,
+ under: Any,
+ model: str | None,
+ tools: list[str] | None,
+ body: MessageCreate,
+ own_scope: dict[str, Any] | None,
+ user: Principal,
+ acl: Access,
+ db: DB,
+ cfg: dict[str, Any],
+ request: Request,
+) -> StreamingResponse:
+ """A question that is a research goal (`research: true`, or /research ): its answer is a plan to approve
+ (research.start). Streams research (the plan, as GET /research/{id}), approval, passages, token (the plan as
+ text), meta and done; error when the plan couldn't be made."""
+ if not research.available(cfg):
+ raise HTTPException(400, "research needs a language model that uses tools (Settings › AI assistant)")
+ scope = own_scope if own_scope is not None else c.get("scope")
+
+ def run() -> Iterator[str]:
+ box = ai_tools.Toolbox(
+ db,
+ cfg,
+ user.as_audit(),
+ set(acl.roles),
+ set(acl.editable()),
+ scope,
+ cid,
+ request.app.state.archive.base,
+ user.admin,
+ only=tools,
+ )
+ if c["title"] == "New conversation":
+ db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=goal[:80])
+ via = "voice" if body.via == "voice" else "web" if user.via == "access" else "api"
+ options = {"model": body.model, "tools": body.tools, "max_tool_calls": body.max_tool_calls, "scope": own_scope}
+ out = research.start(
+ db,
+ cfg,
+ box,
+ goal,
+ via,
+ said=said,
+ parent=under,
+ model=research.model_for(cfg, model),
+ max_tool_calls=body.max_tool_calls,
+ options=options,
+ )
+ if out.get("error"):
+ yield _ev("error", {"message": out["error"]})
+ else:
+ yield _ev("research", research.view(db, out["research"]))
+ yield _ev("approval", out["approval"])
+ yield _ev("passages", [])
+ yield _ev("token", {"text": out["text"]})
+ yield _ev("meta", out["meta"])
+ yield _ev("done", {"message": out["message"]})
+
+ return StreamingResponse(run(), media_type="text/event-stream", headers=SSE_HEADERS)
+
+
@router.post(
"/chats/{cid}/messages",
response_class=StreamingResponse,
@@ -309,6 +374,10 @@ async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db
`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the
conversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.
+ With `research` (or /research ) the content is a research goal: the answer is a plan to approve (streamed as
+ `research`, then `approval`), which runs in the background once approved (POST /chats/{cid}/research, GET
+ /research/{id}).
+
With `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that
browser while the side panel is connected: open pages, read them, click, type and download files into the
conversation (docs/browser-assistant.md)."""
@@ -362,6 +431,9 @@ def run() -> Iterator[str]:
tools = c.get("tools")
limit = _limit(cfg, body.max_tool_calls) if body.max_tool_calls is not None else _most(cfg, c.get("max_tool_calls"))
own_scope = await run_in_threadpool(_scope, acl, db, body.scope) if body.scope is not None else None
+ goal = (q if body.research else research.command(q)) if not files and body.edit is None and again is None else None
+ if goal: # a research goal: the answer is a plan to approve (research.py)
+ return _research_reply(cid, c, goal, q, under, model, tools, body, own_scope, user, acl, db, cfg, request)
record = chat.Record(cfg, model, picked=tools, tool_calls=limit) # what produced the answer (from here: it times it)
title = c["title"]
page = body.context.model_dump() if body.context else None
@@ -598,15 +670,16 @@ def _via(user: Principal) -> str:
return "web" if user.via == "access" else user.via
-def approval_view(a: dict, via: str = "web") -> Any:
+def approval_view(a: dict, via: str = "web", cfg: dict[str, Any] | None = None) -> Any:
"""An approval as the API shows it: an automation's intent with credentials hidden, and what `via` can do."""
- return {**a, "intent": approvals.shown(a.get("intent")), "told": a.get("told") or [], "can": approvals.can(a, via)}
+ return {**a, "intent": approvals.shown(a.get("intent")), "told": a.get("told") or [], "can": approvals.can(a, via, cfg)}
@router.get("/approvals")
def list_approvals(
user: CurrentUser,
db: Db,
+ cfg: Cfg,
chat_id: int | None = None,
status: Literal["pending", "running", "done", "failed", "declined", "expired"] | None = None,
origin: Literal["automation", "chat"] | None = None,
@@ -618,7 +691,7 @@ def list_approvals(
actions. Filter by `status` (pending is waiting), `origin` (automation or chat), one `automation` or one chat
(`chat_id`); page with `before`."""
rows = approvals.listing(db, user.id, status, origin, automation, chat_id, before, limit)
- return [approval_view(a, _via(user)) for a in rows]
+ return [approval_view(a, _via(user), cfg) for a in rows]
@router.get("/approvals/count")
@@ -628,12 +701,12 @@ def count_approvals(user: CurrentUser, db: Db) -> ApprovalCount:
@router.get("/approvals/{aid}")
-def get_approval(aid: int, user: CurrentUser, db: Db) -> Approval:
+def get_approval(aid: int, user: CurrentUser, db: Db, cfg: Cfg) -> Approval:
"""One of your approvals."""
a = approvals.get(db, aid)
if not a or a["account"] != user.id:
raise HTTPException(404, "not found")
- return approval_view(a, _via(user))
+ return approval_view(a, _via(user), cfg)
@router.post("/approvals/{aid}")
diff --git a/fastapi_backend/app/api/v1/routes/notebook.py b/fastapi_backend/app/api/v1/routes/notebook.py
index 43218d06c..1670f7453 100644
--- a/fastapi_backend/app/api/v1/routes/notebook.py
+++ b/fastapi_backend/app/api/v1/routes/notebook.py
@@ -74,8 +74,9 @@ def _out(acl: Access, db: DB, p: dict[str, Any], user: Principal) -> NotePage:
out_links = []
for x in links:
t = found.get(x["target"])
- seen = t is not None and t.get("space") in readable
- out_links.append({**x, "name": t["name"] if seen else None, "namespace": names.get(t["space"]) if seen else None})
+ # a conversation (chat:12) is named only for the person whose conversation it is
+ seen = t is not None and (t.get("account") == user.id if x["target"].startswith("chat:") else t.get("space") in readable)
+ out_links.append({**x, "name": t["name"] if seen else None, "namespace": names.get(t.get("space")) if seen else None})
writer = db.one("SELECT email FROM $r", r=store.R("account", p["created_by"])) if p.get("created_by") else None
targets = [f"page:{p['id']}"] + ([p["about"]] if p.get("about") else [])
return NotePage(
diff --git a/fastapi_backend/app/api/v1/routes/research.py b/fastapi_backend/app/api/v1/routes/research.py
new file mode 100644
index 000000000..02e382393
--- /dev/null
+++ b/fastapi_backend/app/api/v1/routes/research.py
@@ -0,0 +1,196 @@
+"""Deep research (app/domain/research.py, docs/assistant.md#deep-research): a goal gets a plan to approve; approved,
+it runs in the background and its cited report comes back in the conversation. Its plan's approval is decided like
+any other (POST /approvals/{id}); these routes start one, follow it, stop it and use its report."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Request
+
+from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Principal, Writer
+from app.api.v1.routes import podcasts as podcast_routes
+from app.api.v1.routes.chats import _own_chat, _scope, _tools, _via
+from app.domain import ai_tools, auth, chat, llm, research, store
+from app.domain.store import DB
+from app.schemas.podcasts import PodcastCreate, PodcastSelection, PodcastStarted
+from app.schemas.research import (
+ Research,
+ ResearchNote,
+ ResearchNoteMade,
+ ResearchPodcast,
+ ResearchRevise,
+ ResearchStart,
+ ResearchStarted,
+ ResearchStop,
+)
+
+router = APIRouter(tags=["research"])
+
+
+def toolbox(
+ db: DB,
+ cfg: dict[str, Any],
+ user: Principal,
+ acl: Access,
+ cid: int,
+ scope: dict[str, Any] | None,
+ tools: list[str] | None,
+ request: Request,
+) -> ai_tools.Toolbox:
+ """The person's tools in the conversation `cid`, as research plans with them."""
+ return ai_tools.Toolbox(
+ db,
+ cfg,
+ user.as_audit(),
+ set(acl.roles),
+ set(acl.editable()),
+ scope,
+ cid,
+ request.app.state.archive.base,
+ user.admin,
+ only=tools,
+ )
+
+
+def _mine(db: DB, rid: int, user: Principal) -> dict[str, Any]:
+ try:
+ return research.mine(db, rid, user.id)
+ except KeyError:
+ raise HTTPException(404, "not found") from None
+
+
+def view(db: DB, r: dict[str, Any]) -> Research:
+ return Research.model_validate(research.view(db, r))
+
+
+@router.post("/chats/{cid}/research")
+def start_research(cid: int, body: ResearchStart, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> ResearchStarted:
+ """Start research on a goal in a conversation: the goal is saved as a question and one model call plans it (up to
+ three follow-up questions with answers to pick, the sub-questions, where it looks, the tool calls it may make, the
+ report's outline, and an estimate of the cost). The plan waits as an approval (POST /approvals/{id}: approve it as
+ it is, or with `params` its edits; or decline), or revise it (POST /research/{id}/revise). With `review` false it
+ starts at once: your own pick when signed in; an API key or an app needs research.agents_skip_review. The same as a
+ question with `research: true`, or /research ."""
+ c = _own_chat(db, cid, user)
+ if not research.available(cfg):
+ raise HTTPException(400, "research needs a language model that uses tools (Settings › AI assistant)")
+ via = _via(user)
+ if not body.review and not research.agents_may_approve(cfg, via):
+ raise HTTPException(403, "starting research without review is the person's own pick (research.agents_skip_review)")
+ scope = _scope(acl, db, body.scope) if body.scope is not None else c.get("scope")
+ pre = toolbox(db, cfg, user, acl, cid, scope, None, request)
+ tools = _tools(pre, body.tools) if body.tools is not None else c.get("tools")
+ if body.max_tool_calls is not None and body.max_tool_calls > research.settings(cfg)["max_tool_calls"]:
+ raise HTTPException(
+ 400, f"max_tool_calls is a whole number from 1 to {research.settings(cfg)['max_tool_calls']} (research.max_tool_calls)"
+ )
+ box = toolbox(db, cfg, user, acl, cid, scope, tools, request)
+ if not research.sources_of(box):
+ raise HTTPException(400, "none of the tools you picked can look things up for research")
+ kept = c.get("model") # the conversation's model, while it's still offered
+ model = research.model_for(cfg, kept if kept and kept in chat.model_choices(cfg) else None)
+ if c["title"] == "New conversation":
+ db.q("UPDATE $r SET title = $t", r=store.R("chat", cid), t=body.goal[:80])
+ out = research.start(
+ db,
+ cfg,
+ box,
+ " ".join(body.goal.split()),
+ "web" if via == "web" else "api",
+ model=model,
+ max_tool_calls=body.max_tool_calls,
+ options={"tools": body.tools, "max_tool_calls": body.max_tool_calls, "scope": scope if body.scope is not None else None},
+ review=body.review,
+ )
+ if out.get("error"):
+ raise HTTPException(400, out["error"])
+ return ResearchStarted(
+ research=view(db, out["research"]), question=out["question"], message=out["message"], text=out["text"], meta=out["meta"]
+ )
+
+
+@router.get("/research")
+def list_research(user: CurrentUser, db: Db, chat_id: int | None = None, limit: int = 50) -> list[Research]:
+ """Your research, newest first; with `chat_id`, that conversation's (to show each plan's or run's card)."""
+ where = "account = $a" + (" AND chat = $c" if chat_id is not None else "")
+ rows = db.rows(
+ f"SELECT *, record::id(id) AS id FROM research WHERE {where} ORDER BY id DESC LIMIT $n",
+ a=user.id,
+ c=chat_id,
+ n=max(1, min(limit, 200)),
+ )
+ return [view(db, r) for r in rows]
+
+
+@router.get("/research/{rid}")
+def get_research(rid: int, user: CurrentUser, db: Db) -> Research:
+ """A research run: its plan, status and stage (gather, write, check, done), progress (sub-questions done, sources
+ found, tool calls and cost so far, from the activity ledger) and, once it's done, the report's message
+ (`answer`, in the conversation). What it waits for when a budget holds it (`held`), and why it ended early."""
+ return view(db, _mine(db, rid, user))
+
+
+@router.post("/research/{rid}/revise")
+def revise_research(rid: int, body: ResearchRevise, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> Research:
+ """Send your answers to the follow-up questions and your edits to the plan back for one more planning call; the
+ plan and its approval change in place. Only while the plan waits for approval."""
+ r = _mine(db, rid, user)
+ box = toolbox(db, cfg, user, acl, r["chat"], r.get("scope"), r.get("tools"), request)
+ try:
+ return view(db, research.revise(db, cfg, box, r, body.answers, body.plan))
+ except (ValueError, llm.LLMError) as e:
+ raise HTTPException(400, str(e)) from None
+
+
+@router.post("/research/{rid}/stop")
+def stop_research(rid: int, user: Writer, db: Db, cfg: Cfg) -> ResearchStop:
+ """Stop a run: it stops after the step it's on and keeps what it found, saved in the conversation as its answer,
+ marked stopped (one that hasn't started, or waits on a budget, stops now). Works whichever process runs it."""
+ _mine(db, rid, user)
+ return ResearchStop(stopping=research.request_stop(db, cfg, rid))
+
+
+@router.post("/research/{rid}/resume")
+def resume_research(rid: int, user: Writer, db: Db, cfg: Cfg) -> Research:
+ """Let a run that a budget holds go on anyway (your pick, as "run it once" is for a held routine run)."""
+ _mine(db, rid, user)
+ try:
+ return view(db, research.go_on(db, cfg, rid, user.as_audit()))
+ except ValueError as e:
+ raise HTTPException(409, str(e)) from None
+
+
+@router.post("/research/{rid}/note")
+def research_note(rid: int, user: Writer, acl: Acl, db: Db, body: ResearchNote | None = None) -> ResearchNoteMade:
+ """Save the report as a note: in the namespace it looked in (`namespace` when there are several), its archive
+ citations as links to the assets they cite, web citations as links, and a link back to the conversation (shown to
+ you alone). Needs editor access there."""
+ r = _mine(db, rid, user)
+ try:
+ out = research.to_note(db, r, user.id, set(acl.editable()), (body or ResearchNote()).namespace)
+ except PermissionError as e:
+ raise HTTPException(403, str(e)) from None
+ except ValueError as e:
+ raise HTTPException(400, str(e)) from None
+ auth.audit(db, user.as_audit(), "research.note", f"note_page:{out['page']}", {"research": rid})
+ return ResearchNoteMade(**out)
+
+
+@router.post("/research/{rid}/podcast", status_code=202)
+def research_podcast(rid: int, user: Writer, acl: Acl, db: Db, cfg: Cfg, body: ResearchPodcast | None = None) -> PodcastStarted:
+ """Make a podcast episode from the assets the report cites (POST /podcasts with them picked): about the research
+ goal unless `prompt` says otherwise, titled after the report."""
+ r = _mine(db, rid, user)
+ body = body or ResearchPodcast()
+ picked = research.podcast_sources(db, r, acl.readable())
+ if not picked:
+ raise HTTPException(400, "the report cites no assets from the archive to make an episode from")
+ ask = PodcastCreate(
+ selection=PodcastSelection(recordings=picked[:100]),
+ prompt=body.prompt or (r.get("goal") or "")[:2000] or None,
+ length=body.length,
+ style=body.style,
+ title=body.title or (r.get("plan") or {}).get("title"),
+ )
+ return podcast_routes.start(acl, user, db, cfg, ask)
diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py
index 6e9375bb2..99eeb44d2 100644
--- a/fastapi_backend/app/domain/ai_tools.py
+++ b/fastapi_backend/app/domain/ai_tools.py
@@ -31,6 +31,7 @@
ops_tools,
recsets,
render,
+ research as researchmod,
sandbox as sandboxmod,
search as searchmod,
browser as browsermod,
@@ -519,6 +520,7 @@ def builtin_names():
+ AUTOMATION_TOOLS
+ browsermod.TOOLS
+ sandboxmod.TOOLS
+ + researchmod.TOOLS
)
return {t[0] for t in every + webmod.TOOLS} | {"use_skill"}
@@ -542,7 +544,14 @@ def builtin_names():
"decline_approval",
} | {
t[0]
- for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS + MEMORY_TOOLS + AUTOMATION_TOOLS + sandboxmod.TOOLS
+ for t in TOOLS
+ + ops_tools.ADMIN_TOOLS
+ + ops_tools.FILE_TOOLS
+ + AUTHOR_TOOLS
+ + MEMORY_TOOLS
+ + AUTOMATION_TOOLS
+ + sandboxmod.TOOLS
+ + researchmod.TOOLS
if t[4]
}
@@ -597,6 +606,7 @@ def builtin_names():
},
"files": {"label": "Importing files", "tools": [t[0] for t in ops_tools.FILE_TOOLS]},
"code": {"label": "Code sandbox", "tools": [t[0] for t in sandboxmod.TOOLS]},
+ "research": {"label": "Deep research", "tools": [t[0] for t in researchmod.TOOLS]},
"extensions": {"label": "Extensions and skills", "tools": [t[0] for t in AUTHOR_TOOLS] + ["use_skill"]},
"server": {"label": "The server", "tools": [t[0] for t in ops_tools.ADMIN_TOOLS]},
"browser": {"label": "Your browser", "tools": [t[0] for t in browsermod.TOOLS]},
@@ -621,6 +631,8 @@ def _why_missing(group, cfg):
return "the code sandbox is off", "sandbox.enabled"
if group == "code" and sandboxmod.settings(cfg)["who"] == "admins":
return "for admins", "sandbox.who"
+ if group == "research" and researchmod.unavailable(cfg):
+ return researchmod.unavailable(cfg)
if group == "extensions" and not cfg["ai"].get("extensions", True):
return "switched off by an admin", "ai.extensions"
if all(t in off for t in GROUPS[group]["tools"]):
@@ -657,6 +669,8 @@ def catalog(box):
offered |= {t for t in GROUPS["browser"]["tools"] if t not in off}
if sandboxmod.offered(box.cfg, box.admin, bool(box.editable)): # every conversation of theirs has its sandbox
offered |= {t for t in GROUPS["code"]["tools"] if t not in off}
+ if researchmod.available(box.cfg): # every conversation of theirs can start research
+ offered |= {t for t in GROUPS["research"]["tools"] if t not in off}
# code that can reach another host can carry out what it's given
runs_out = {"run_python", "run_bash"} if sandboxmod.sends_out(box.cfg) else set()
built = {
@@ -669,6 +683,7 @@ def catalog(box):
+ AUTOMATION_TOOLS
+ sandboxmod.TOOLS
+ webmod.TOOLS
+ + researchmod.TOOLS
}
out = []
for name, g in GROUPS.items():
@@ -709,7 +724,9 @@ class Toolbox(ops_tools.OpsTools):
`account`: the person an automation acts for, when `user` is the automation itself (automation:,
the by of what it writes): their approvals, extensions, view of the timeline and marks on it; `origin` ({automation, run,
node}) is kept on the approvals it asks for. `only`: the groups (GROUPS, an extension's mcp: or ext:)
- and tools the question was given (None: every tool this person may use; an admin's switch still wins)."""
+ and tools the question was given (None: every tool this person may use; an admin's switch still wins).
+ `allowed_out`: tools that send data out of Lens and don't ask first even after the answer read outside text,
+ because the person already approved them for this work (an approved research plan's web search, research.py)."""
def __init__(
self,
@@ -730,9 +747,11 @@ def __init__(
account=None,
origin=None,
only=None,
+ allowed_out=(),
):
self.db, self.cfg, self.user, self.chat, self.said = db, cfg, user, chat_id, said
self.only = None if only is None else set(only)
+ self.allowed_out = set(allowed_out or ())
self.account, self.origin = user["id"] if account is None else account, origin
# where this answer has read text from outside Lens so far (`outside`: before any tool ran, such as the page it
# was asked from or a chat room's message): once it has, tools that send data out of Lens or change the server
@@ -846,6 +865,8 @@ def specs(self):
tools += sandboxmod.TOOLS
if self.web:
tools += webmod.TOOLS
+ if self.chat is not None and researchmod.available(self.cfg):
+ tools += researchmod.TOOLS
if self.ext:
tools += self.ext.tool_specs(can_act or self.admin)
keep = self.chosen()
@@ -1072,7 +1093,7 @@ def _sandbox(self, fn, *args):
def _sandbox_run(self, kind, source, files):
source = str(source or "")
files = [str(f) for f in ([files] if isinstance(files, str) else files or [])]
- if sandboxmod.sends_out(self.cfg) and self.wary():
+ if sandboxmod.sends_out(self.cfg) and self.wary() and f"run_{kind}" not in self.allowed_out:
# code that reaches another host can carry out what the answer read: after text from outside Lens, which
# may have put it there, a run asks first
first = next((x.strip() for x in source.splitlines() if x.strip()), "")[:80]
@@ -1103,8 +1124,9 @@ def t_import_sandbox_file(self, path, namespace=None):
def _web(self, tool, args, what, own):
"""A web search or page read, as (what it found, None), or (None, the approval it waits for): the words or the
address leave the machine, so once this answer has read text from outside Lens, which may have put them there,
- it asks first, unless the person wrote them (`own`) or already approved the same call in this conversation."""
- if own or not self.wary():
+ it asks first, unless the person wrote them (`own`), already approved the same call in this conversation, or
+ approved this tool for the work it's doing (allowed_out: an approved research plan's)."""
+ if own or not self.wary() or tool in self.allowed_out:
return webmod.run(self.cfg, tool, args), None
rows = (
self.db.rows(
@@ -1162,6 +1184,25 @@ def t_web_read(self, url, find=None):
"web_read", args, out
)
+ # ---- deep research (research.py): a plan to approve, then a run in the background ----
+ def t_start_research(self, goal):
+ goal = " ".join(str(goal or "").split())
+ if not goal:
+ raise ValueError("give the goal to research")
+ if self.chat is None:
+ raise ValueError("research runs in a conversation")
+ model = researchmod.model_for(self.cfg, (self.db.one("SELECT model FROM $r", r=R("chat", self.chat)) or {}).get("model"))
+ r = researchmod.propose(self.db, self.cfg, self, goal, model)
+ out = {
+ "status": "waiting for the person's approval",
+ "research": r["id"],
+ "approval": r["approval"],
+ "plan": r["plan"],
+ "questions": r["questions"],
+ "estimate": r["estimate"],
+ }
+ return out, f"Planned {researchmod.summary(r['plan'])}; it waits for the person's approval"
+
# ---- extensions, from chat ----
def _ext_named(self, name):
for g in extensions.visible(self.db, self.me):
@@ -2078,6 +2119,8 @@ def approve(db, cfg, aid, user, editable, decision="approve", base=None, admin=F
a = approvals.get(db, aid)
if not a or a["status"] != "pending":
raise ValueError("nothing to approve")
+ if a["tool"] == "research": # a research plan: approved (as it is, or with `params` its edits) or declined
+ return researchmod.decide(db, cfg, a, user, decision, params, via)
if via == "agent" and decision != "decline":
raise PermissionError("approving is the person's own pick: an agent can only decline")
if a.get("intent"):
diff --git a/fastapi_backend/app/domain/approvals.py b/fastapi_backend/app/domain/approvals.py
index fc3cd61a1..be98ac046 100644
--- a/fastapi_backend/app/domain/approvals.py
+++ b/fastapi_backend/app/domain/approvals.py
@@ -260,11 +260,21 @@ def edited(intent, new):
return out
-def can(row, via="web"):
+def can(row, via="web", cfg=None):
"""What the person can do with a waiting approval from where they are (`via`: web for a signed-in session, chat for
- their own chat app, token or oauth for an API key or an app, agent for MCP and the assistant, which only decline)."""
+ their own chat app, token or oauth for an API key or an app, agent for MCP and the assistant, which only decline).
+ A research plan is approved by the person only (edited in the web app), unless research.agents_skip_review is on
+ in `cfg`."""
pending = row.get("status") == "pending"
intent = row.get("intent")
+ if row.get("tool") == "research":
+ skip = bool(((cfg or {}).get("research") or {}).get("agents_skip_review"))
+ return {
+ "approve": pending and (via in ("web", "chat") or skip),
+ "decline": pending,
+ "edit": pending and via == "web",
+ "always": False,
+ }
if not intent:
return {"approve": pending and via != "agent", "decline": pending, "edit": False, "always": False}
person = via in ("web", "chat")
diff --git a/fastapi_backend/app/domain/bridge.py b/fastapi_backend/app/domain/bridge.py
index a2c829c07..f845846ab 100644
--- a/fastapi_backend/app/domain/bridge.py
+++ b/fastapi_backend/app/domain/bridge.py
@@ -38,7 +38,21 @@
import urllib.error
import urllib.request
-from . import activity, ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted
+from . import (
+ activity,
+ ai_tools,
+ auth,
+ auto_scope,
+ channels,
+ chat,
+ graph_history,
+ llm,
+ matterbridge,
+ ns_assistant,
+ research,
+ store,
+ untrusted,
+)
from .store import R
log = logging.getLogger(__name__)
@@ -330,6 +344,25 @@ def _answer(db, cfg, base, account, cid, q, files, act_on, outside):
past = chat.path(db, cid) # a room goes on from the head (the version showing, should it be switched in the app)
summary = chat.memory(db, cfg, cid, past)
scope = (db.one("SELECT scope FROM $r", r=R("chat", cid)) or {}).get("scope") or None
+ goal = research.command(q) if not files else None
+ if goal and research.available(cfg): # /research : a plan, approved with a "yes" here
+ box = ai_tools.Toolbox(
+ db,
+ cfg,
+ {"id": account["id"], "email": account["email"]},
+ readable,
+ editable,
+ scope,
+ cid,
+ base,
+ account["admin"],
+ said=q,
+ outside=list(outside) + ai_tools.read_before(past, summary),
+ )
+ out = research.start(db, cfg, box, goal, "room", said=q)
+ if out.get("error"):
+ return "I couldn't plan that: " + out["error"], [], set()
+ return out["text"], [out["approval"]], set()
chat.add(db, cid, "user", q, attachments=list(files) or None, meta=chat.question_record("room"))
# the answer's id up front: every call made for it names it in the ledger, and its record sums them
mid = db.next_id("chat_message")
@@ -392,9 +425,9 @@ def said(words, cited, outcome, **kw):
# ---------- approving in the room ----------
# what can be approved by a reply in the room: changes to the entities and topics (history keeps them, and merges and
-# deletes can be undone) and new namespaces; settings, imports, batch runs, extensions and forgetting are approved in
-# the web app
-ROOM_SAFE = {"propose_entity_change", "propose_topic_change", "create_namespace"}
+# deletes can be undone), new namespaces and a research plan as it is (it only reads, within the plan, and can be
+# stopped); settings, imports, batch runs, extensions and forgetting are approved in the web app
+ROOM_SAFE = {"propose_entity_change", "propose_topic_change", "create_namespace", "research"}
YES_RX = re.compile(r"^(?:yes|y|yep|approve|approved|ok|okay|do it|go ahead)(?:\s+#?(\d+|all))?\s*[.!]*$", re.I)
NO_RX = re.compile(r"^(?:no|n|nope|decline|cancel|don't)(?:\s+#?(\d+|all))?\s*[.!]*$", re.I)
diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py
index 4be4be36d..f03c0af3b 100644
--- a/fastapi_backend/app/domain/chat.py
+++ b/fastapi_backend/app/domain/chat.py
@@ -369,7 +369,7 @@ def history(db, cid, ids=None):
"""A conversation's messages, oldest first: every version's, or only those with `ids` (a path)."""
return db.rows(
"SELECT record::id(id) AS id, role, content, passages, created_at, stopped ?? false AS stopped, steps ?? [] AS steps, "
- "attachments ?? [] AS attachments, notice, error, check, model, context, parent, meta FROM chat_message WHERE chat = $c"
+ "attachments ?? [] AS attachments, notice, error, check, model, context, parent, meta, kind FROM chat_message WHERE chat = $c"
+ (" AND id IN $ids" if ids is not None else "")
+ " ORDER BY id",
c=cid,
@@ -504,13 +504,15 @@ def add(
parent=HEAD,
mid=None,
meta=None,
+ kind=None,
):
"""Save a message under `parent` (by default the head; None: a first message) and make it the head; an answer
keeps the tool steps it took, any notice (e.g. the model can't use tools) and error; a question asked from a page
keeps what it shared of it (shared_context). A question an automation asked keeps its `origin` ({automation, run,
node}), so it isn't taken for a person's message (`chat.message`). `mid`: an id taken earlier with
db.next_id("chat_message"), after the parent was saved (an answer's, so the ledger rows of the calls made for it
- name it). `meta` is its record (Record; a question's: how it was sent)."""
+ name it). `meta` is its record (Record; a question's: how it was sent). `kind`: research for a research report,
+ research_plan for the plan it waits on (research.py)."""
parent = head(db, cid) if parent == HEAD else parent
mid = mid or db.next_id("chat_message")
db.q(
@@ -533,6 +535,7 @@ def add(
"origin": origin or None,
"parent": parent,
"meta": meta or None,
+ "kind": kind,
}
),
)
diff --git a/fastapi_backend/app/domain/notebook.py b/fastapi_backend/app/domain/notebook.py
index ba85995e8..cea8c54fe 100644
--- a/fastapi_backend/app/domain/notebook.py
+++ b/fastapi_backend/app/domain/notebook.py
@@ -10,6 +10,8 @@
@[Weekly call](recording:12) @[Ada Lovelace](entity:5) @[Plans](page:3) #[Capsid design](topic:9)
@ is for resources, people and other pages; # is for topics, the namespace's vocabulary (topics.py). Links are kept as note_link rows, so a page shows its backlinks and the graph sees them.
+@[Research: capsids](chat:12) links a conversation (a research report's note links back to the one it came from),
+which is named and linked only for the person whose conversation it is.
The editor may also keep its own document state (`doc`, opaque: a BlockSuite/Yjs snapshot) next to the Markdown, which
holds what Markdown can't, like drawings on the edgeless canvas. A change to the Markdown alone (the assistant's, say)
@@ -38,7 +40,9 @@
PLACES = ("project", "area", "resource", "archive")
AUTHORS = ("person", "assistant")
VIEWS = ("page", "edgeless") # how the editor shows it: a document, or the endless canvas (drawings, diagrams)
-KINDS = ("recording", "entity", "topic", "collection", "speaker", "page") # what a page can link to
+# what a page can link to; a conversation (chat, as a research report's note links back to it) shows as a link only
+# to the person whose conversation it is
+KINDS = ("recording", "entity", "topic", "collection", "speaker", "page", "chat")
ABOUT = ("recording", "entity", "topic", "collection", "speaker") # what can have a page of its own
TITLE_MAX = 200
SUMMARY_MAX = 300
@@ -795,7 +799,8 @@ def backlinks(db, sid, targets):
def labels(db, targets):
- """Current names for link targets: {"recording:12": "Weekly call"}; the ones that are gone are left out."""
+ """Current names for link targets: {"recording:12": "Weekly call"}; the ones that are gone are left out. A
+ conversation's (chat:12) has its `account` instead of a `space`: only that person sees its name."""
by_kind = {}
for t in targets:
kind, _, key = t.partition(":")
@@ -805,6 +810,12 @@ def labels(db, targets):
for kind, keys in by_kind.items():
table = "note_page" if kind == "page" else kind
name = {"recording": "title ?? path", "page": "title", "speaker": "name ?? label", "topic": "label"}.get(kind, "name")
+ if kind == "chat":
+ for r in db.rows(
+ "SELECT record::id(id) AS id, account, title AS name FROM chat WHERE id IN $ids", ids=[R("chat", k) for k in keys]
+ ):
+ out[f"chat:{r['id']}"] = {"name": r.get("name"), "account": r.get("account")}
+ continue
for r in db.rows(
f"SELECT record::id(id) AS id, space, {name} AS name FROM {table} WHERE id IN $ids", ids=[R(table, k) for k in keys]
):
diff --git a/fastapi_backend/app/domain/research.py b/fastapi_backend/app/domain/research.py
new file mode 100644
index 000000000..e1f052b53
--- /dev/null
+++ b/fastapi_backend/app/domain/research.py
@@ -0,0 +1,1197 @@
+"""Deep research (docs/assistant.md, "Deep research"): give a goal, approve a plan, get a cited report back.
+
+A goal (`research: true` on a question, `/research `, POST /chats/{cid}/research, the MCP tool or the
+assistant's own `start_research`) gets one planning call: up to three follow-up questions, each with answers to pick
+from, and a plan (the sub-questions it will answer, the sources it will look in, the tool calls it may make, the
+report's outline). The plan is saved as a `research` row and waits as an approval of the assistant's (tool
+`research`), with an estimate of what it will cost. The person approves it as it is or edited (Enter in the chat,
+the Approvals page, "yes" in a chat app), or revises it (their answers and edits go back for one more planning call).
+An API key or an app only declines it, unless research.agents_skip_review lets agents start runs without review.
+
+Approved, it runs in the background, claimed by one process (CREATE-as-claim on `research_run:`, a heartbeat
+every HEARTBEAT seconds, taken over after STALE seconds of silence and resumed from the last finished stage), in
+stages:
+
+ gather each sub-question answered with the tool loop (chat.tool_answer), its share of the tool calls, and a
+ toolbox of the plan's sources only, whose outbound tools (web search, a sandbox that reaches the web)
+ don't ask again: the person approved them with the plan (Toolbox.allowed_out). Other change tools still
+ ask.
+ write one call: the report in Markdown, following the outline, citing [n] across every finding, renumbered
+ check chat.check_sources on the report; unsupported claims are rewritten once, or marked
+ done the report saved in the conversation as an answer of kind `research`, with its record (chat.Record)
+
+Every call counts in the ledger for the conversation and `research:`. Before each stage the budgets of the
+scope's namespaces and `all` are checked: one over holds the run (on_over ask) until the budget allows or the person
+lets it go on, or stops it with what it has (skip). research.max_minutes stops a run with what it found; Stop
+(POST /research/{id}/stop) stops it after the step it's on, like stopping an answer. A person has one run going at a
+time; the next waits its turn.
+
+After: save the report as a note (archive citations become mention links, web ones links, and it links back to the
+conversation), read it aloud, make a podcast episode from its archive sources, or export the conversation.
+
+Refine later: sub-questions in parallel, research on a schedule, sharing a report with namespace members, research as
+an automation action, a report template people can edit, the report posted back into the chat room it was asked in.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+import threading
+import time
+import uuid
+
+from . import activity, auth, budgets, chat, llm, notebook, store, telemetry, untrusted
+
+log = logging.getLogger("lens")
+R = store.R
+
+STATUSES = ("planned", "declined", "queued", "running", "held", "done", "stopped", "failed")
+ACTIVE = ("queued", "running", "held") # approved and not finished
+STAGES = ("gather", "write", "check", "done")
+SOURCES = ("archive", "graph", "notes", "web", "code")
+# the tools a run may use for each source: reading only (code that runs in the sandbox reads and works things out)
+SOURCE_TOOLS = {
+ "archive": ["search_transcripts", "find_resources", "find_similar", "list_recordings", "read_transcript", "recording_outputs"],
+ "graph": [
+ "find_entities",
+ "entity_mentions",
+ "entity_timeline",
+ "find_topics",
+ "topic_recordings",
+ "graph_neighbours",
+ "speaker_stats",
+ "graph_schema",
+ "graph_query",
+ "graph_related",
+ "graph_paths",
+ ],
+ "notes": ["find_notes", "read_note"],
+ "web": ["web_search", "web_read"],
+ "code": ["run_python", "run_bash", "sandbox_files"],
+}
+SOURCE_LABELS = {"archive": "the archive", "graph": "the graph", "notes": "notes", "web": "the web", "code": "the code sandbox"}
+QUESTIONS_MAX, ANSWERS_MAX, SUBS_MAX, OUTLINE_MAX = 3, 4, 8, 10
+HEARTBEAT = 10 # seconds between a run's heartbeats
+STALE = 120 # seconds of silence after which another process takes a run over
+SWEEP_SECONDS = 30 # how often the scheduler looks for runs no process is on
+LOOKBACK = 20 # past runs an estimate is drawn from
+
+# the start_research tool (ai_tools.GROUPS "research"): it plans, and the plan waits for the person's approval
+TOOLS = [
+ (
+ "start_research",
+ "Plan deep research on a goal that needs many look-ups (several sub-questions, more than one source): you get "
+ "a plan with follow-up questions, which waits for the person's approval; once approved it runs in the "
+ "background and the cited report comes back in this conversation. Use it when the person asks for research or "
+ "a report, not for a quick question.",
+ {"goal": {"type": "string", "description": "what to research, in the person's words"}},
+ ["goal"],
+ True,
+ )
+]
+
+PLAN_SCHEMA = {
+ "type": "object",
+ "required": ["questions", "plan"],
+ "properties": {
+ "questions": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["question", "answers"],
+ "properties": {"question": {"type": "string"}, "answers": {"type": "array", "items": {"type": "string"}}},
+ },
+ },
+ "plan": {
+ "type": "object",
+ "required": ["title", "goal", "sub_questions", "sources", "max_tool_calls", "report_outline"],
+ "properties": {
+ "title": {"type": "string"},
+ "goal": {"type": "string"},
+ "sub_questions": {"type": "array", "items": {"type": "string"}},
+ "sources": {"type": "array", "items": {"type": "string", "enum": list(SOURCES)}},
+ "max_tool_calls": {"type": "integer"},
+ "report_outline": {"type": "array", "items": {"type": "string"}},
+ },
+ },
+ },
+}
+PLAN_SYSTEM = (
+ "You plan deep research for a person. From their goal, write a plan: a short title, the goal in one sentence, "
+ "the sub-questions that together answer it (3 to {subs}, each answerable by looking things up), the sources to "
+ "look in (only from: {sources}), how many tool calls it may make in all (at most {calls}; about 3 to 5 per "
+ "sub-question) and the report's outline (its section headings). Ask follow-up questions (at most 3, each with 2 "
+ "to 4 short suggested answers) only where the answer would change the plan; none when the goal is clear. When "
+ "you're given answers and edits to an earlier plan, follow them."
+)
+FIND = (
+ "Research goal: {goal}\n\nAnswer this part of it: {sub}\n\nLook it up with the tools, then answer in a short "
+ "paragraph or a few bullets, citing what you found as [n]. Say plainly what you couldn't find."
+)
+WRITE_SYSTEM = (
+ "You write research reports. From the findings below, write the report in Markdown: a title (# heading), a short "
+ "summary, then the outline's sections (## headings), then what is still open. Cite every claim with the [n] numbers "
+ "of the excerpts it rests on, and only those; say so where the findings don't cover a section. Don't make up "
+ "anything the findings don't say."
+)
+REWRITE_SCHEMA = {
+ "type": "object",
+ "required": ["rewrites"],
+ "properties": {
+ "rewrites": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["claim", "text"],
+ "properties": {"claim": {"type": "string"}, "text": {"type": "string"}},
+ },
+ }
+ },
+}
+REWRITE_SYSTEM = (
+ "These sentences of a report aren't supported by the excerpts they cite. Rewrite each so that it says only what "
+ "its excerpts support, keeping its [n] citations; give an empty text for one the excerpts don't support at all."
+)
+UNSUPPORTED = " *(not supported by its sources)*"
+
+
+class Stopped(Exception):
+ """The run stops here: asked to, out of time, or another process took it over."""
+
+ def __init__(self, why):
+ super().__init__(why)
+ self.why = why
+
+
+# ---------- settings ----------
+def settings(cfg):
+ r = cfg.get("research") or {}
+ return {
+ "max_tool_calls": int(r.get("max_tool_calls") or 40),
+ "max_minutes": int(r.get("max_minutes") or 20),
+ "model": r.get("model") or None,
+ "agents_skip_review": bool(r.get("agents_skip_review")),
+ }
+
+
+def unavailable(cfg):
+ """Why research can't run, and the setting that would change it, or None: it needs a language model that uses
+ tools."""
+ if not llm.configured(cfg):
+ return "no language model is set up", "llm.model"
+ if not cfg["ai"].get("tools"):
+ return "the assistant's tools are switched off", "ai.tools"
+ return None
+
+
+def available(cfg):
+ """Whether research can run: a language model that uses tools."""
+ return unavailable(cfg) is None
+
+
+def agents_may_approve(cfg, via):
+ """Whether a pick made `via` (web, chat, token, oauth or agent) may approve a plan: the person's own, signed in or
+ in their chat app; an API key, an app or an agent only with research.agents_skip_review."""
+ return via in ("web", "chat") or settings(cfg)["agents_skip_review"]
+
+
+# ---------- planning ----------
+def sources_of(box):
+ """The sources this person's toolbox (as the conversation and their choice of tools have it) can look in."""
+ offered = {s["function"]["name"] for s in box.specs()}
+ return [s for s in SOURCES if offered & set(SOURCE_TOOLS[s])]
+
+
+def _text(v, n):
+ return " ".join(str(v or "").split())[:n]
+
+
+def clean_plan(cfg, plan, sources, goal=None):
+ """A plan as a run takes it: its sub-questions, sources (of those this person has), tool calls (1 to
+ research.max_tool_calls) and outline, trimmed. ValueError for one with no sub-question or no source."""
+ plan = plan or {}
+ subs = [_text(s, 300) for s in plan.get("sub_questions") or [] if _text(s, 300)][:SUBS_MAX]
+ if not subs:
+ raise ValueError("a plan has at least one sub-question")
+ picked = [s for s in dict.fromkeys(plan.get("sources") or []) if s in sources]
+ if not picked:
+ if plan.get("sources"):
+ raise ValueError(f"a plan looks in {', '.join(sources) or 'nothing you have'}")
+ picked = sources[:1]
+ if not picked:
+ raise ValueError("you have no tools research can look things up with")
+ most = settings(cfg)["max_tool_calls"]
+ try:
+ calls = int(plan.get("max_tool_calls") or 0)
+ except (TypeError, ValueError):
+ raise ValueError("max_tool_calls is a whole number") from None
+ calls = max(len(subs), min(most, calls or 4 * len(subs)))
+ calls = min(calls, most)
+ goal = _text(plan.get("goal") or goal, 1000)
+ return {
+ "title": _text(plan.get("title") or goal, 120) or "Research",
+ "goal": goal,
+ "sub_questions": subs,
+ "sources": picked,
+ "max_tool_calls": calls,
+ "report_outline": [_text(s, 200) for s in plan.get("report_outline") or [] if _text(s, 200)][:OUTLINE_MAX],
+ }
+
+
+def _questions(raw):
+ out = []
+ for q in raw or []:
+ text = _text((q or {}).get("question"), 300)
+ if text:
+ out.append({"question": text, "answers": [_text(a, 120) for a in (q.get("answers") or []) if _text(a, 120)][:ANSWERS_MAX]})
+ return out[:QUESTIONS_MAX]
+
+
+def model_for(cfg, chat_model=None):
+ """The model research uses: research.model, else the conversation's, else the configured one (None)."""
+ return settings(cfg)["model"] or chat_model or None
+
+
+def plan(cfg, goal, sources, scope=None, model=None, previous=None, answers=None):
+ """One planning call: {questions, plan}. `previous` (a plan) with `answers` ({question: answer}) revises it."""
+ if not llm.configured(cfg):
+ raise llm.LLMError("research needs a language model: set one up in Settings › AI assistant")
+ s = settings(cfg)
+ system = PLAN_SYSTEM.format(subs=SUBS_MAX, sources=", ".join(sources), calls=s["max_tool_calls"]) + untrusted.rules(cfg)
+ parts = [f"Goal: {goal}"]
+ if (scope or {}).get("namespaces"):
+ parts.append("It looks only in these namespaces: " + ", ".join(scope["namespaces"]))
+ if previous:
+ lines = "\n".join(f"- {s}" for s in previous.get("sub_questions") or [])
+ parts.append(
+ f"The earlier plan, as the person edited it:\nTitle: {previous.get('title')}\nSub-questions:\n{lines}\n"
+ f"Sources: {', '.join(previous.get('sources') or [])}\nTool calls: {previous.get('max_tool_calls')}\n"
+ f"Outline: {'; '.join(previous.get('report_outline') or [])}"
+ )
+ if answers:
+ parts.append("Their answers:\n" + "\n".join(f"- {q}: {a}" for q, a in answers.items() if a))
+ out = llm.json_out(cfg, system, "\n\n".join(parts), PLAN_SCHEMA, model=model)
+ return {"questions": [] if answers else _questions(out.get("questions")), "plan": clean_plan(cfg, out.get("plan"), sources, goal)}
+
+
+def estimate(db, cfg, p, model=None):
+ """What a plan will likely cost, always labelled an estimate: from the ledger's past research runs (what each tool
+ call cost them, times this plan's), else its tokens at the model's price in Settings. {usd, tokens, tool_calls,
+ basis, estimate}."""
+ calls = int(p["max_tool_calls"])
+ past = db.rows(
+ "SELECT record::id(id) AS id, used, finished_at FROM research WHERE status = 'done' ORDER BY finished_at DESC LIMIT $n", n=LOOKBACK
+ )
+ usd = tokens = used = 0.0
+ runs = 0
+ for r in past:
+ n = (r.get("used") or {}).get("tool_calls") or 0
+ if n:
+ t = activity.totals(db, f"research:{r['id']}")
+ usd, tokens, used, runs = usd + t["cost_usd"], tokens + t["tokens_in"] + t["tokens_out"], used + n, runs + 1
+ if runs:
+ return {
+ "usd": round(usd / used * calls, 4),
+ "tokens": int(tokens / used * calls),
+ "tool_calls": calls,
+ "basis": f"what each tool call of {runs} past research run{'s' if runs != 1 else ''} cost",
+ "estimate": True,
+ }
+ subs = len(p["sub_questions"])
+ tin, tout = 3000 + calls * 2500 + 8000 + 4000, subs * 500 + 2500 + 800 # planning, gathering, writing, checking
+ name = model or cfg["llm"].get("model")
+ cost = telemetry.cost(name, tin, tout, telemetry.prices(cfg))
+ return {
+ "usd": round(cost, 4) if cost is not None else None,
+ "tokens": tin + tout,
+ "tool_calls": calls,
+ "basis": f"about {tin + tout:,} tokens at {name}'s price in Settings"
+ if cost is not None
+ else f"about {tin + tout:,} tokens ({name} has no price in Settings)",
+ "estimate": True,
+ }
+
+
+def summary(p):
+ n = len(p["sub_questions"])
+ where = ", ".join(SOURCE_LABELS[s] for s in p["sources"])
+ return f"Research: {p['title']} ({n} sub-question{'s' if n != 1 else ''}, in {where}, up to {p['max_tool_calls']} tool calls)"
+
+
+def create(db, cfg, box, goal, made, message=None, model=None, rid=None):
+ """Save a plan ({questions, plan}) as a `research` row of the conversation `box` is in (`rid`: its id, taken
+ before planning so the planning call counts for it), and the approval it waits for (box._approval). Returns the
+ row."""
+ rid = rid or db.next_id("research")
+ p = made["plan"]
+ est = estimate(db, cfg, p, model)
+ t = store.now()
+ db.q(
+ "CREATE $r CONTENT $d",
+ r=R("research", rid),
+ d=store.clean(
+ {
+ "chat": box.chat,
+ "message": message,
+ "account": box.account,
+ "goal": _text(goal, 2000),
+ "status": "planned",
+ "plan": p,
+ "questions": made["questions"],
+ "scope": box.scope or None,
+ "tools": sorted(box.only) if box.only is not None else None,
+ "model": model,
+ "estimate": est,
+ "revisions": 0,
+ "created_at": t,
+ "updated_at": t,
+ }
+ ),
+ )
+ box._approval("research", {"research": rid, "plan": p}, summary(p), est)
+ aid = box.approvals[-1]["id"]
+ db.q("UPDATE $r SET approval = $a", r=R("research", rid), a=aid)
+ return get(db, rid)
+
+
+def propose(db, cfg, box, goal, model=None):
+ """The assistant's start_research: a plan for `goal` in the conversation `box` is in, waiting for approval (the
+ planning call counts for the research too). Returns the row."""
+ rid = db.next_id("research")
+ with activity.scope(db, f"research:{rid}", cfg=cfg):
+ made = plan(cfg, goal, sources_of(box), box.scope, model)
+ return create(db, cfg, box, goal, made, model=model, rid=rid)
+
+
+COMMAND = re.compile(r"^/research\s+(.+)$", re.S | re.I)
+
+
+def command(text):
+ """The goal of `/research `, else None."""
+ m = COMMAND.match((text or "").strip())
+ return " ".join(m.group(1).split()) if m else None
+
+
+def _money(usd):
+ return f"${usd:,.4f}".rstrip("0").rstrip(".") if usd < 1 else f"${usd:,.2f}"
+
+
+def plan_text(r):
+ """The plan as the conversation shows it, for people reading it in a chat app or without the plan card."""
+ p, est = r["plan"], r.get("estimate") or {}
+ where = ", ".join(SOURCE_LABELS[s] for s in p["sources"])
+ cost = f", about {_money(est['usd'])} (an estimate)" if est.get("usd") is not None else ""
+ lines = [f"Here's a plan to research **{p['title']}**:", ""]
+ lines += [f"{i}. {q}" for i, q in enumerate(p["sub_questions"], 1)]
+ lines += ["", f"It looks in {where}, with up to {p['max_tool_calls']} tool calls{cost}."]
+ if p.get("report_outline"):
+ lines.append("The report: " + "; ".join(p["report_outline"]) + ".")
+ if r.get("questions"):
+ lines += ["", "A few questions, so it fits what you need:"]
+ lines += [f"- {q['question']}" + (f" ({' / '.join(q['answers'])})" if q.get("answers") else "") for q in r["questions"]]
+ lines += ["", "Approve it to start, or answer the questions or edit it and revise it."]
+ return "\n".join(lines)
+
+
+def start(db, cfg, box, goal, via="web", said=None, parent=chat.HEAD, model=None, max_tool_calls=None, options=None, review=True):
+ """A research goal asked in the conversation `box` is in: the question saved (`said`: as typed, e.g. with
+ /research), one planning call, the plan saved and its approval, and the plan as the answer (kind research_plan).
+ With review False the person's own pick (via web or chat; an agent's only with research.agents_skip_review)
+ approves it at once. Returns {research, approval, question, message, text, meta}, with `error` when the plan
+ couldn't be made (saved as the answer's error)."""
+ cid = box.chat
+ question = chat.add(db, cid, "user", said or goal, parent=parent, meta=chat.question_record(via, research=True, **(options or {})))
+ rid, mid = db.next_id("research"), db.next_id("chat_message")
+ record = chat.Record(cfg, model, box.scope, picked=sorted(box.only) if box.only is not None else None, research=rid)
+ wrote = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None
+ try:
+ with activity.scope(db, f"chat:{cid}", f"chat_message:{mid}", f"research:{rid}", cfg=cfg):
+ made = plan(cfg, goal, sources_of(box), box.scope, model)
+ if max_tool_calls:
+ made["plan"]["max_tool_calls"] = max(1, min(int(max_tool_calls), settings(cfg)["max_tool_calls"]))
+ except (llm.LLMError, ValueError) as e:
+ meta = record.meta(db, "error", f"chat_message:{mid}")
+ chat.add(db, cid, "assistant", "(no plan)", [], error=str(e), model=wrote, parent=question, mid=mid, meta=meta)
+ return {"error": str(e), "question": question, "message": mid, "meta": meta}
+ r = create(db, cfg, box, goal, made, message=question, model=model, rid=rid)
+ text = plan_text(r)
+ meta = record.meta(db, "answered", f"chat_message:{mid}")
+ chat.add(db, cid, "assistant", text, [], model=wrote, parent=question, mid=mid, meta=meta, kind="research_plan")
+ db.q("UPDATE $r SET plan_message = $m", r=R("research", rid), m=mid)
+ approval = box.approvals[-1]
+ if not review:
+ a = db.one("SELECT *, record::id(id) AS id FROM $r", r=R("approval", approval["id"]))
+ decide(db, cfg, a, box.user, "approve", None, via if via in ("web", "chat") else "agent")
+ return {"research": get(db, rid), "approval": approval, "question": question, "message": mid, "text": text, "meta": meta}
+
+
+def get(db, rid):
+ r = db.one("SELECT *, record::id(id) AS id FROM $r", r=R("research", int(rid)))
+ if not r:
+ raise KeyError(rid)
+ return r
+
+
+def mine(db, rid, account):
+ """A research row of this person's, or KeyError."""
+ r = get(db, rid)
+ if r.get("account") != account:
+ raise KeyError(rid)
+ return r
+
+
+def view(db, r):
+ """A research row as the API shows it, with what it has spent so far (the ledger rows that name it)."""
+ spent = activity.totals(db, f"research:{r['id']}")
+ findings = r.get("findings") or []
+ p = r.get("plan") or {}
+ sources = {(x.get("url") or "", x.get("recording_id"), x.get("t0"), x.get("page")) for f in findings for x in f.get("passages") or []}
+ return {
+ "id": r["id"],
+ "chat": r.get("chat"),
+ "message": r.get("message"),
+ "plan_message": r.get("plan_message"),
+ "answer": r.get("answer"),
+ "approval": r.get("approval"),
+ "status": r.get("status"),
+ "stage": r.get("stage"),
+ "goal": r.get("goal"),
+ "plan": p,
+ "questions": r.get("questions") or [],
+ "answers": r.get("answers") or {},
+ "scope": r.get("scope") or {},
+ "estimate": r.get("estimate"),
+ "progress": {
+ "sub_questions": len(p.get("sub_questions") or []),
+ "done": len(findings),
+ "sources": len(sources) + int(r.get("sources_now") or 0),
+ "tool_calls": int((r.get("used") or {}).get("tool_calls") or 0),
+ "cost_usd": spent["cost_usd"],
+ "tokens": spent["tokens_in"] + spent["tokens_out"],
+ "cost_estimated": spent["estimate"],
+ "minutes": round(float(r.get("ran_s") or 0) / 60, 1),
+ },
+ "findings": [{"question": f["question"], "tool_calls": f.get("tool_calls") or 0} for f in findings],
+ "held": r.get("held"),
+ "error": r.get("error"),
+ "ended": r.get("ended"),
+ "revisions": r.get("revisions") or 0,
+ "notes": r.get("notes") or [],
+ "created_at": r.get("created_at"),
+ "started_at": r.get("started_at"),
+ "finished_at": r.get("finished_at"),
+ }
+
+
+def revise(db, cfg, box, r, answers=None, edited=None):
+ """One more planning call with the person's answers and edits; the plan and its approval change in place."""
+ if r["status"] != "planned":
+ raise ValueError(f"this research is {r['status']}: only a plan waiting for approval can be revised")
+ sources = sources_of(box)
+ previous = clean_plan(cfg, {**(r.get("plan") or {}), **(edited or {})}, sources, r.get("goal"))
+ answers = {_text(k, 300): _text(v, 300) for k, v in (answers or {}).items() if _text(k, 300)}
+ with activity.scope(db, f"chat:{r['chat']}", f"research:{r['id']}", cfg=cfg):
+ made = plan(cfg, r.get("goal") or previous["goal"], sources, r.get("scope"), r.get("model"), previous, answers)
+ p = made["plan"]
+ est = estimate(db, cfg, p, r.get("model"))
+ db.q(
+ "UPDATE $r SET plan = $p, questions = $q, answers = $a, estimate = $e, revisions += 1, updated_at = $t",
+ r=R("research", r["id"]),
+ p=p,
+ q=made["questions"],
+ a=answers or None,
+ e=est,
+ t=store.now(),
+ )
+ db.q(
+ "UPDATE $r SET args = $args, summary = $s, estimate = $e WHERE status = 'pending'",
+ r=R("approval", r["approval"]),
+ args={"research": r["id"], "plan": p},
+ s=summary(p),
+ e=est,
+ )
+ return get(db, r["id"])
+
+
+# ---------- deciding ----------
+def decide(db, cfg, a, user, decision, params=None, via="web"):
+ """The person's pick on a plan's approval: approve (as it is, or with `params`: the plan edited; `edit` is the
+ same) or decline. An approved plan queues its run, which starts at once unless the person has one going."""
+ decision = "approve" if decision == "edit" else decision
+ if decision not in ("approve", "decline"):
+ raise ValueError("a research plan is approved or declined")
+ r = get(db, a["args"]["research"])
+ if decision == "approve" and not agents_may_approve(cfg, via):
+ raise PermissionError("approving a research plan is the person's own pick: an agent can only decline (research.agents_skip_review)")
+ p = r["plan"]
+ if decision == "approve" and params: # the plan as the person edited it, within the sources they have
+ box = toolbox(db, cfg, {**r, "plan": {**p, "sources": list(SOURCES)}}, account_of(db, r["account"]))
+ allowed = sources_of(box) if box else p["sources"]
+ p = clean_plan(cfg, {**p, **{k: v for k, v in params.items() if v is not None}}, allowed, r.get("goal"))
+ if not approvals_take(db, a["id"], "done" if decision == "approve" else "declined", user):
+ raise ValueError("nothing to approve")
+ if decision == "decline":
+ db.q("UPDATE $r SET status = 'declined', updated_at = $t", r=R("research", r["id"]), t=store.now())
+ return {"status": "declined", "result": {"research": r["id"], "status": "declined"}}
+ queue(db, cfg, r["id"], p, user)
+ result = {"research": r["id"], "status": "queued"}
+ db.q("UPDATE $r SET result = $res", r=R("approval", a["id"]), res=result)
+ return {"status": "done", "result": result}
+
+
+def approvals_take(db, aid, status, user):
+ """Claim the plan's approval for one decision: True when this one won (approvals._take)."""
+ from . import approvals
+
+ return approvals._take(db, aid, status, user)
+
+
+def queue(db, cfg, rid, p, user=None):
+ """An approved plan, waiting for its run: started now, unless the person already has one going."""
+ db.q(
+ "UPDATE $r SET status = 'queued', plan = $p, stage = 'gather', approved_by = $u, approved_at = $t, updated_at = $t",
+ r=R("research", rid),
+ p=p,
+ u=(user or {}).get("email"),
+ t=store.now(),
+ )
+ r = get(db, rid)
+ if not _busy(db, r["account"], rid):
+ launch(db, cfg, rid)
+
+
+def _busy(db, account, rid=None):
+ """Whether the person has another run going (one at a time each)."""
+ return bool(
+ db.rows(
+ "SELECT id FROM research WHERE account = $a AND status IN ['running', 'held'] AND id != $r LIMIT 1",
+ a=account,
+ r=R("research", int(rid or 0)),
+ )
+ )
+
+
+def launch(db, cfg, rid):
+ """Run it in the background, in this process, if it claims it."""
+ threading.Thread(target=run, args=(db, cfg, rid), daemon=True, name=f"research {rid}").start()
+
+
+def request_stop(db, cfg, rid):
+ """Ask a run to stop after the step it's on; a run that hasn't started (or waits on a budget) stops now. False when
+ it's already over."""
+ r = get(db, rid)
+ if r["status"] in ("queued", "held"):
+ took = db.rows(
+ "UPDATE $r SET status = 'stopped', ended = 'stopped', finished_at = $t, updated_at = $t WHERE status IN ['queued', 'held'] RETURN id",
+ r=R("research", rid),
+ t=store.now(),
+ )
+ if took and (r.get("findings") or []):
+ # what it found before it was held stays: saved as a report of its findings, without another model call
+ _save_found(db, cfg, get(db, rid))
+ return bool(took)
+ if r["status"] == "planned":
+ return False
+ return bool(db.rows("UPDATE $r SET stop_requested = true WHERE status = 'running' RETURN id", r=R("research", rid)))
+
+
+def go_on(db, cfg, rid, user=None):
+ """Let a run held by a budget go on this once (the person's pick, as budgets.py's "run it once")."""
+ took = db.rows(
+ "UPDATE $r SET status = 'queued', over_ok = true, held = NONE, updated_at = $t WHERE status = 'held' RETURN id",
+ r=R("research", rid),
+ t=store.now(),
+ )
+ if not took:
+ raise ValueError("only a run held by a budget can go on")
+ launch(db, cfg, rid)
+ return get(db, rid)
+
+
+# ---------- claiming a run ----------
+def _ago(seconds):
+ return (dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=seconds)).isoformat(timespec="seconds")
+
+
+def claim(db, rid, owner):
+ """Take a run for this process (CREATE-as-claim on research_run:): True when it's ours. A claim whose
+ heartbeat stopped STALE seconds ago is another process gone, so it's taken over."""
+ rec = R("research_run", int(rid))
+ for _ in range(2):
+ try:
+ db.q(
+ "CREATE $r CONTENT $d",
+ r=rec,
+ d={"research": int(rid), "owner": owner, "heartbeat_at": store.now(), "claimed_at": store.now()},
+ )
+ return True
+ except Exception as e: # noqa: BLE001 - only a claim that's already there means someone else has it
+ if "already exists" not in str(e):
+ raise
+ if not db.rows("DELETE $r WHERE heartbeat_at < $s RETURN BEFORE", r=rec, s=_ago(STALE)):
+ return False
+ return False
+
+
+def beat(db, rid, owner):
+ """The run's heartbeat: False when the claim isn't ours any more (another process took it over)."""
+ return bool(
+ db.rows("UPDATE $r SET heartbeat_at = $t WHERE owner = $o RETURN id", r=R("research_run", int(rid)), t=store.now(), o=owner)
+ )
+
+
+def release(db, rid, owner):
+ db.q("DELETE $r WHERE owner = $o", r=R("research_run", int(rid)), o=owner)
+
+
+def sweep(db, cfg, log=None):
+ """Runs no process is on: an approved one waiting its turn, one whose process went away (its claim gone stale) and
+ one a budget held, once the budget allows it. Each is started here, in the background."""
+ rows = db.rows("SELECT record::id(id) AS id, account, status, created_at FROM research WHERE status IN $s ORDER BY id", s=list(ACTIVE))
+ claimed = {
+ c["research"]: c
+ for c in db.rows("SELECT research, heartbeat_at FROM research_run WHERE research IN $ids", ids=[r["id"] for r in rows])
+ }
+ going = {r["account"] for r in rows if r["status"] == "running" and r["id"] in claimed}
+ started = []
+ for r in rows:
+ c = claimed.get(r["id"])
+ if c and (c.get("heartbeat_at") or "") >= _ago(STALE):
+ continue # a process is on it
+ if r["status"] == "held":
+ full = get(db, r["id"])
+ if _over(db, cfg, full):
+ continue
+ db.q("UPDATE $r SET status = 'queued', held = NONE WHERE status = 'held'", r=R("research", r["id"]))
+ elif r["account"] in going and r["status"] == "queued":
+ continue # their other run first
+ going.add(r["account"])
+ started.append(r["id"])
+ launch(db, cfg, r["id"])
+ if started and log:
+ log(f"research: started {', '.join(map(str, started))}")
+ return started
+
+
+# ---------- running ----------
+def account_of(db, aid):
+ a = auth.get_account(db, aid)
+ if not a or a.get("disabled") or a.get("removed_at"):
+ return None
+ return {"id": a["id"], "email": a["email"], "admin": bool(a.get("admin"))}
+
+
+def toolbox(db, cfg, r, account, sub=None):
+ """The person's tools for the run: only the plan's sources' (and of those, what the conversation's choice of tools
+ keeps), and its outbound tools allowed without asking again (the plan approved them)."""
+ from . import ai_tools, sandbox
+
+ if not account:
+ return None
+ roles = auth.roles(db, account)
+ readable, editable = set(roles), {s for s in roles if auth.allows(roles, s, "editor")}
+ user = {"id": account["id"], "email": account["email"]}
+ p = r.get("plan") or {}
+ box = ai_tools.Toolbox(
+ db,
+ cfg,
+ user,
+ readable,
+ editable,
+ r.get("scope"),
+ r.get("chat"),
+ None,
+ account["admin"],
+ said=f"{r.get('goal') or ''} {sub or ''}",
+ only=r.get("tools"),
+ )
+ keep = box.chosen()
+ names = {t for s in p.get("sources") or [] for t in SOURCE_TOOLS.get(s, [])}
+ box.only = names if keep is None else names & keep
+ out = set()
+ if "web" in (p.get("sources") or []):
+ out |= set(SOURCE_TOOLS["web"])
+ if "code" in (p.get("sources") or []) and sandbox.sends_out(cfg):
+ out |= {"run_python", "run_bash"}
+ box.allowed_out = out
+ return box
+
+
+def _over(db, cfg, r):
+ """Why the run must wait for a budget, or None: the budgets of the scope's namespaces and `all` (budgets.check)."""
+ if r.get("over_ok"):
+ return None
+ names = {v: k for k, v in store.space_names(db).items()}
+ spaces = [f"space:{names[n]}" for n in (r.get("scope") or {}).get("namespaces") or [] if n in names]
+ got = budgets.check(db, cfg, spaces)
+ return None if got["go"] else got
+
+
+class Run:
+ """One process's turn at a run: its claim, kept by a heartbeat every HEARTBEAT seconds (from a thread of its own,
+ so a long model call doesn't let the claim go stale), the time it has run, and what stops it."""
+
+ def __init__(self, db, cfg, rid, owner):
+ self.db, self.cfg, self.rid, self.owner = db, cfg, rid, owner
+ self.r = get(db, rid)
+ self.t0, self.ran = time.monotonic(), float(self.r.get("ran_s") or 0)
+ self.lost, self.done = False, threading.Event()
+ self.limit = settings(cfg)["max_minutes"] * 60
+ threading.Thread(target=self._beat, daemon=True, name=f"research {rid} heartbeat").start()
+
+ def _beat(self):
+ while not self.done.wait(HEARTBEAT):
+ try:
+ if not beat(self.db, self.rid, self.owner):
+ self.lost = True
+ return
+ self.db.q("UPDATE $r SET ran_s = $s", r=R("research", self.rid), s=round(self.elapsed(), 1))
+ except Exception: # noqa: BLE001 - the next beat tries again; a claim that goes stale is taken over
+ log.warning("research %s: no heartbeat", self.rid, exc_info=True)
+
+ def end(self):
+ self.done.set()
+ try:
+ self.db.q("UPDATE $r SET ran_s = $s", r=R("research", self.rid), s=round(self.elapsed(), 1))
+ except Exception: # noqa: BLE001 - only the time it ran is lost
+ log.warning("research %s: couldn't save the time it ran", self.rid, exc_info=True)
+
+ def elapsed(self):
+ return self.ran + time.monotonic() - self.t0
+
+ def tick(self):
+ """Between steps: whether another process took the run over, or the person asked it to stop."""
+ if self.lost:
+ raise Stopped("lost")
+ row = self.db.one("SELECT stop_requested FROM $r", r=R("research", self.rid)) or {}
+ if row.get("stop_requested"):
+ raise Stopped("stopped")
+
+ def out_of_time(self):
+ return self.elapsed() >= self.limit
+
+
+def run(db, cfg, rid, owner=None):
+ """Claim the run and carry it out, from its last finished stage, until it's done, held, stopped or failed. False
+ when another process has it (or it isn't waiting for one)."""
+ owner = owner or uuid.uuid4().hex
+ r = get(db, rid)
+ if r["status"] not in ("queued", "running") or not claim(db, rid, owner):
+ return False
+ try:
+ r = get(db, rid) # as it is now we hold it
+ if r["status"] not in ("queued", "running"):
+ return False
+ if r["status"] == "queued" and _busy(db, r["account"], rid):
+ return False # their other run first; the sweep starts this one after it
+ db.q(
+ "UPDATE $r SET status = 'running', started_at = started_at ?? $t, updated_at = $t",
+ r=R("research", rid),
+ t=store.now(),
+ )
+ turn = Run(db, cfg, rid, owner)
+ try:
+ with activity.scope(db, f"chat:{r['chat']}", f"research:{rid}", cfg=cfg):
+ _stages(db, cfg, turn)
+ finally:
+ turn.end()
+ return True
+ except Exception as e: # noqa: BLE001 - a run that breaks says why, keeps what it found, and frees the person's turn
+ log.exception("research %s failed", rid)
+ why = str(e) if isinstance(e, llm.LLMError) else f"{type(e).__name__}: {e}"
+ took = db.rows(
+ "UPDATE $r SET status = 'failed', error = $e, finished_at = $t, updated_at = $t WHERE status = 'running' RETURN id",
+ r=R("research", rid),
+ e=why[:500],
+ t=store.now(),
+ )
+ if took:
+ try:
+ _save_found(db, cfg, get(db, rid), "error")
+ except Exception: # noqa: BLE001 - the run's row keeps its findings and the error
+ log.exception("research %s: couldn't save what it found", rid)
+ return True
+ finally:
+ release(db, rid, owner)
+ _next(db, cfg, rid)
+
+
+def _next(db, cfg, rid):
+ """The person's next approved run, once this one is over."""
+ r = db.one("SELECT account, status FROM $r", r=R("research", rid)) or {}
+ if r.get("status") in ("running", "held", "queued") or not r.get("account"):
+ return
+ nxt = db.one("SELECT record::id(id) AS id FROM research WHERE account = $a AND status = 'queued' ORDER BY id LIMIT 1", a=r["account"])
+ if nxt:
+ launch(db, cfg, nxt["id"])
+
+
+def _hold(db, rid, got):
+ """A budget is over: the run waits (on_over ask), saying why, or stops with what it has (skip)."""
+ if got["action"] == "skip":
+ raise Stopped("budget")
+ db.q(
+ "UPDATE $r SET status = 'held', held = $h, updated_at = $t",
+ r=R("research", rid),
+ h={"why": got["why"], "budget": got["resource"]},
+ t=store.now(),
+ )
+
+
+def _stages(db, cfg, run):
+ rid = run.rid
+ account = account_of(db, run.r["account"])
+ if not account:
+ raise RuntimeError("the person who asked for it can't sign in any more")
+ try:
+ while True:
+ r = get(db, rid)
+ stage = r.get("stage") or "gather"
+ if stage == "done":
+ return
+ got = _over(db, cfg, r)
+ if got:
+ _hold(db, rid, got)
+ return
+ if stage == "gather":
+ _gather(db, cfg, run, r, account)
+ db.q("UPDATE $r SET stage = 'write'", r=R("research", rid))
+ elif stage == "write":
+ _write(db, cfg, run, r)
+ db.q("UPDATE $r SET stage = 'check'", r=R("research", rid))
+ elif stage == "check":
+ _check(db, cfg, run, r)
+ db.q("UPDATE $r SET stage = 'done'", r=R("research", rid))
+ _finish(db, cfg, run, get(db, rid), "timeout" if r.get("ended") == "timeout" else None)
+ return
+ except activity.OverBudget as e: # the overall budget's hard limit, reached mid-stage: it waits, as before a stage
+ _hold(db, rid, {"action": "ask", "why": str(e), "resource": budgets.ALL})
+ except Stopped as s:
+ if s.why == "lost": # another process has it now, and goes on from the last finished stage
+ return
+ db.q(
+ "UPDATE $r SET status = 'stopped', ended = $w, stop_requested = false, finished_at = $t, updated_at = $t",
+ r=R("research", rid),
+ w=s.why,
+ t=store.now(),
+ )
+ _save_found(db, cfg, get(db, rid))
+
+
+def _model(db, cfg, r):
+ """The model the run uses: research.model, else the conversation's (while it's still offered), else the configured one."""
+ if settings(cfg)["model"]:
+ return settings(cfg)["model"]
+ m = r.get("model") or (db.one("SELECT model FROM $r", r=R("chat", r["chat"])) or {}).get("model")
+ return m if m and m in chat.model_choices(cfg) else None
+
+
+def _gather(db, cfg, run, r, account):
+ """Each sub-question not yet answered, with its share of the tool calls that are left."""
+ p = r["plan"]
+ findings = list(r.get("findings") or [])
+ used = int((r.get("used") or {}).get("tool_calls") or 0)
+ model = _model(db, cfg, r)
+ most = int(cfg["ai"].get("max_tool_calls") or 50)
+ for i, sub in enumerate(p["sub_questions"]):
+ if i < len(findings):
+ continue
+ if run.out_of_time():
+ db.q("UPDATE $r SET ended = 'timeout'", r=R("research", run.rid))
+ return
+ run.tick()
+ left = max(0, p["max_tool_calls"] - used)
+ if left <= 0:
+ break
+ share = max(1, min(most, left // (len(p["sub_questions"]) - i)))
+ box = toolbox(db, cfg, r, account, sub)
+ q = FIND.format(goal=p["goal"], sub=sub)
+ record = chat.Record(cfg, model, r.get("scope"), picked=p["sources"], tool_calls=share, research=run.rid)
+ text, steps = "", []
+ try:
+ for kind, data in chat.tool_answer(cfg, box, q, (), share, model, record=record, max_tool_calls=share):
+ if kind == "step":
+ steps.append(data)
+ db.q(
+ "UPDATE $r SET used = $u, sources_now = $s",
+ r=R("research", run.rid),
+ u={"tool_calls": used + record.used["tool_calls"]},
+ s=len(box.refs),
+ )
+ run.tick()
+ if run.out_of_time():
+ break
+ else:
+ text = data
+ cited = box.cited(text)
+ except llm.ToolsUnsupported: # a model that can't call tools: answered from a search of the archive
+ passages = chat.retrieve(db, sub, box.readable, r.get("scope"), cfg=cfg)
+ text = llm.chat(cfg, chat.messages_for(q, passages, cfg=cfg), model=model)
+ cited = [x for x in chat.cited(text, passages) if x["used"]]
+ used += record.used["tool_calls"]
+ findings.append(
+ {"question": sub, "text": text or "(nothing found)", "passages": cited, "tool_calls": record.used["tool_calls"], "steps": steps}
+ )
+ db.q(
+ "UPDATE $r SET findings = $f, used = $u, sources_now = 0, updated_at = $t",
+ r=R("research", run.rid),
+ f=findings,
+ u={"tool_calls": used},
+ t=store.now(),
+ )
+ if run.out_of_time():
+ db.q("UPDATE $r SET ended = 'timeout'", r=R("research", run.rid))
+ return
+
+
+def _key(p):
+ return (p.get("url") or "", p.get("recording_id"), p.get("t0"), p.get("page"))
+
+
+def renumber(findings):
+ """Every finding's citations as one numbered list: (the passages, each finding's text cited by their new numbers).
+ The same moment or page cited in two findings keeps one number."""
+ out, at, texts = [], {}, []
+ for f in findings:
+ local = {}
+ for x in f.get("passages") or []:
+ k = _key(x)
+ if k not in at:
+ at[k] = len(out) + 1
+ out.append({**{k2: v for k2, v in x.items() if k2 != "used"}, "n": at[k]})
+ local[x["n"]] = at[k]
+ texts.append(re.sub(r"\[(\d+)\]", lambda m: f"[{local[int(m.group(1))]}]" if int(m.group(1)) in local else "", f.get("text") or ""))
+ return out, texts
+
+
+def _write(db, cfg, run, r):
+ """The report: one call, from every finding, following the outline."""
+ p = r["plan"]
+ passages, texts = renumber(r.get("findings") or [])
+ run.tick()
+ if not passages and not any(t.strip() for t in texts):
+ report = f"# {p['title']}\n\nNothing turned up for this goal in {', '.join(SOURCE_LABELS[s] for s in p['sources'])}."
+ else:
+ found = "\n\n".join(
+ f"## {f['question']}\n" + untrusted.wrap(t, "research finding", cfg) for f, t in zip(r.get("findings") or [], texts)
+ )
+ outline = "\n".join(f"- {s}" for s in p.get("report_outline") or []) or "- (your choice)"
+ user = f"Goal: {p['goal']}\nTitle: {p['title']}\nOutline:\n{outline}\n\nFindings:\n\n{found}\n\nExcerpts:\n\n" + (
+ chat.excerpts(passages, cfg) or "(none)"
+ )
+ if r.get("ended") == "timeout":
+ user += "\n\nThe research ran out of time before every sub-question was looked into: say which weren't."
+ report = llm.chat(
+ cfg,
+ [{"role": "system", "content": WRITE_SYSTEM + untrusted.rules(cfg)}, {"role": "user", "content": user}],
+ model=_model(db, cfg, r),
+ )
+ db.q("UPDATE $r SET report = $t, passages = $p, updated_at = $n", r=R("research", run.rid), t=report, p=passages, n=store.now())
+
+
+def _check(db, cfg, run, r):
+ """Each cited claim of the report against its excerpts; an unsupported one is rewritten once, or marked."""
+ report, passages = r.get("report") or "", r.get("passages") or []
+ run.tick()
+ if not passages:
+ db.q("UPDATE $r SET check = $c", r=R("research", run.rid), c={"claims": 0, "supported": 0, "verdicts": [], "uncited": []})
+ return
+ got = chat.check_sources(cfg, report, passages)
+ bad = [v["claim"] for v in got["verdicts"] if not v.get("supported") and v.get("claim") in report]
+ if bad:
+ run.tick()
+ try:
+ fixes = llm.json_out(
+ cfg,
+ REWRITE_SYSTEM + untrusted.rules(cfg),
+ f"Excerpts:\n\n{chat.excerpts(passages, cfg)}\n\nSentences:\n" + "\n".join(f"- {c}" for c in bad),
+ REWRITE_SCHEMA,
+ model=_model(db, cfg, r),
+ )["rewrites"]
+ except llm.LLMError:
+ fixes = []
+ by = {f.get("claim"): (f.get("text") or "").strip() for f in fixes if isinstance(f, dict)}
+ for c in bad:
+ new = by.get(c)
+ report = report.replace(c, new if new and new != c else c + UNSUPPORTED, 1)
+ db.q("UPDATE $r SET report = $t, check = $c", r=R("research", run.rid), t=report, c={**got, "rewritten": len(bad)})
+
+
+def _record(db, cfg, r, outcome):
+ """The report's record (chat.Record): the model, the plan's sources and tool calls, what it used, how it ended,
+ and the tokens and cost of every call the run made (research:)."""
+ model = _model(db, cfg, r)
+ rec = chat.Record(
+ cfg,
+ model,
+ r.get("scope"),
+ picked=(r.get("plan") or {}).get("sources"),
+ tool_calls=(r.get("plan") or {}).get("max_tool_calls"),
+ research=r["id"],
+ )
+ rec.offered = sorted({t for s in (r.get("plan") or {}).get("sources") or [] for t in SOURCE_TOOLS.get(s, [])})
+ rec.used = {"rounds": len(r.get("findings") or []), "tool_calls": int((r.get("used") or {}).get("tool_calls") or 0)}
+ meta = rec.meta(db, outcome, f"research:{r['id']}")
+ meta["ms"] = int(float(r.get("ran_s") or 0) * 1000)
+ return meta
+
+
+def _steps(r):
+ return [s for f in r.get("findings") or [] for s in f.get("steps") or []]
+
+
+ANSWERING_WAIT = 120 # seconds a report waits for an answer being written in the conversation to end
+
+
+def _where(db, cid):
+ """Where the report goes in the conversation: after its last message, once an answer being written there ends (a
+ question waiting for its answer isn't followed by the report)."""
+ end = time.monotonic() + ANSWERING_WAIT
+ while time.monotonic() < end and (db.one("SELECT answering FROM $r", r=R("chat", cid)) or {}).get("answering"):
+ time.sleep(1)
+ return chat.head(db, cid)
+
+
+def _answer(db, cfg, r, text, passages, outcome, stopped=False):
+ """The report saved in the conversation, where it is now, as an answer of kind research with its record."""
+ parent = _where(db, r["chat"])
+ mid = db.next_id("chat_message")
+ meta = _record(db, cfg, r, outcome)
+ wrote = _model(db, cfg, r) or (cfg["llm"].get("model") if llm.configured(cfg) else None)
+ chat.add(
+ db,
+ r["chat"],
+ "assistant",
+ text,
+ chat.cited(text, passages),
+ stopped=stopped,
+ steps=_steps(r),
+ model=wrote,
+ parent=parent,
+ mid=mid,
+ meta=meta,
+ kind="research",
+ )
+ db.q("UPDATE $r SET answer = $m, updated_at = $t", r=R("research", r["id"]), m=mid, t=store.now())
+ return mid
+
+
+def _finish(db, cfg, run, r, ended):
+ _answer(db, cfg, r, r.get("report") or "(no report)", r.get("passages") or [], "limit" if ended else "answered")
+ db.q("UPDATE $r SET status = 'done', finished_at = $t, updated_at = $t", r=R("research", run.rid), t=store.now())
+
+
+def _save_found(db, cfg, r, outcome="stopped"):
+ """A run that stopped (or broke off: `outcome` error) keeps what it found: its report when it got that far, else its
+ findings, saved as the answer without another model call."""
+ if r.get("answer"):
+ return
+ passages, texts = renumber(r.get("findings") or [])
+ said = "Stopped" if outcome == "stopped" else f"It broke off ({r.get('error') or 'an error'})"
+ if r.get("report"):
+ text, passages = r["report"], r.get("passages") or passages
+ elif any(t.strip() for t in texts):
+ text = f"# {r['plan']['title']}\n\n{said}. What it found so far:\n\n" + "\n\n".join(
+ f"## {f['question']}\n{t}" for f, t in zip(r.get("findings") or [], texts)
+ )
+ else:
+ text = f"# {r['plan']['title']}\n\n{said} before it found anything."
+ _answer(db, cfg, r, text, passages, outcome, stopped=outcome == "stopped")
+
+
+# ---------- after: a note, a podcast ----------
+def mention(p):
+ """A citation as a note links it: an asset's moment as a mention, a web page as a link."""
+ if p.get("source") == "web" and p.get("url"):
+ return f"[{_label(p.get('title') or p.get('site') or p['url'])}]({p['url']})"
+ if p.get("recording_id") is not None:
+ return f"@[{_label(p.get('title') or 'Asset')}](recording:{p['recording_id']})"
+ return ""
+
+
+def _label(text):
+ return re.sub(r"[\[\]\n]", " ", str(text or "")).strip()[:200] or "source"
+
+
+def note_body(r, report, passages, title):
+ """The report as a note: each [n] becomes the link to what it cites, and the note links back to the conversation."""
+ by = {p["n"]: p for p in passages}
+
+ def cite(m):
+ p = by.get(int(m.group(1)))
+ return f" ({mention(p)})" if p and mention(p) else ""
+
+ body = re.sub(r"\s*\[(\d+)\]", cite, report or "")
+ return body.rstrip() + f"\n\n@[Research: {_label(title)}](chat:{r['chat']})\n"
+
+
+def note_spaces(db, r, editable):
+ """The namespaces the note can go in: the scope's (or, when it's over everything, those of the archive it cites),
+ of those the person can edit."""
+ names = store.space_names(db)
+ scoped = (r.get("scope") or {}).get("namespaces") or []
+ if scoped:
+ ids = [sid for sid, n in names.items() if n in scoped]
+ else:
+ recs = [p["recording_id"] for p in r.get("passages") or [] if p.get("recording_id") is not None]
+ ids = sorted(
+ {x["space"] for x in db.rows("SELECT space FROM recording WHERE id IN $ids", ids=[R("recording", i) for i in recs])}
+ if recs
+ else set()
+ )
+ if not ids:
+ ids = sorted(editable)
+ return [sid for sid in ids if sid in editable]
+
+
+def to_note(db, r, account, editable, namespace=None):
+ """Save the report as a notebook page; its id and namespace. ValueError when it has none yet, or the namespace
+ must be picked (several are possible) or can't be written."""
+ if not r.get("answer") and not r.get("report"):
+ raise ValueError("the report isn't ready yet")
+ report = _report_text(db, r)
+ choices = note_spaces(db, r, editable)
+ names = store.space_names(db)
+ if namespace:
+ sid = next((s for s, n in names.items() if n == namespace), None)
+ if sid is None or sid not in editable:
+ raise PermissionError(f"you can't write notes in {namespace}")
+ elif len(choices) == 1:
+ sid = choices[0]
+ elif not choices:
+ raise PermissionError("you can't write notes in any namespace this research looked in")
+ else:
+ raise ValueError("pick a namespace: " + ", ".join(names[s] for s in choices))
+ title = (r.get("plan") or {}).get("title") or "Research"
+ pid = notebook.create(db, sid, account, title, note_body(r, report, r.get("passages") or [], title), author="assistant")
+ db.q("UPDATE $r SET notes = array::union(notes ?? [], [$n])", r=R("research", r["id"]), n=pid)
+ return {"page": pid, "namespace": names[sid]}
+
+
+def _report_text(db, r):
+ """The report as it was saved in the conversation (the person may have stopped the run)."""
+ if r.get("answer"):
+ m = db.one("SELECT content FROM $r", r=R("chat_message", r["answer"])) or {}
+ if m.get("content"):
+ return m["content"]
+ return r.get("report") or ""
+
+
+def podcast_sources(db, r, readable):
+ """The assets the report cites, that the person can still read: an episode's selection."""
+ used = {p["n"] for p in chat.cited(_report_text(db, r), r.get("passages") or []) if p["used"]}
+ ids = list(
+ dict.fromkeys(
+ p["recording_id"] for p in r.get("passages") or [] if p.get("recording_id") is not None and (p["n"] in used or not used)
+ )
+ )
+ if not ids:
+ return []
+ ok = {
+ x["id"]
+ for x in db.rows("SELECT record::id(id) AS id, space FROM recording WHERE id IN $ids", ids=[R("recording", i) for i in ids])
+ if readable is None or x["space"] in readable
+ }
+ return [i for i in ids if i in ok]
diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py
index 7f1291dea..c79253fb3 100644
--- a/fastapi_backend/app/domain/routines.py
+++ b/fastapi_backend/app/domain/routines.py
@@ -675,7 +675,8 @@ def start(db, cfg_fn, stop, log=None):
"""The scheduling threads, until `stop` is set: watched folders scanned when due (every sources.check_seconds),
routines run when due (every CHECK_SECONDS), the scheduled backup taken when due, and automations' timers fired
when due (timers.py). The API runs them with its background work; `lens worker` runs them too, since Docker and
- the packages run the API without it."""
+ the packages run the API without it. Research runs no process is on (research.py) are picked up here too."""
+ from . import research # it loads the chat, which loads what routines run
def every(seconds, fn, what):
def loop():
@@ -696,6 +697,7 @@ def loop():
every(lambda: cfg_fn()["fedora"]["sync_seconds"], fedora.sync_due, "fedora"),
every(lambda: notebook.QUIET_SECONDS / 2, notebook.organise_due, "notes"),
every(lambda: CHECK_SECONDS, backups.run_due, "backups"),
+ every(lambda: research.SWEEP_SECONDS, research.sweep, "research"),
timers.start(db, cfg_fn, stop, log=log),
]
diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py
index 5f9dc13ab..7d502a895 100644
--- a/fastapi_backend/app/domain/settings.py
+++ b/fastapi_backend/app/domain/settings.py
@@ -37,6 +37,7 @@
"ai": None,
"sandbox": None,
"web": None,
+ "research": None,
"decisions": None,
"components": None,
"voice": None,
@@ -507,6 +508,8 @@ def _check(section, key, value, default):
return _sandbox_setting(key, value)
if section == "web" and key != "search":
return _web_setting(key, value)
+ if section == "research":
+ return _research_setting(key, value)
if section == "components":
return _component_setting(key, value)
if section == "nsfw":
@@ -1393,6 +1396,24 @@ def _web_setting(key, value):
return value
+RESEARCH_RANGES = {"max_tool_calls": (1, 500), "max_minutes": (1, 240)}
+
+
+def _research_setting(key, value):
+ if key == "agents_skip_review":
+ if not isinstance(value, bool):
+ raise ValueError("research.agents_skip_review is true or false")
+ return value
+ if key == "model":
+ if value is not None and not (isinstance(value, str) and len(value.strip()) <= 200):
+ raise ValueError("research.model is a model's name")
+ return (value or "").strip() or None
+ lo, hi = RESEARCH_RANGES[key]
+ if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi):
+ raise ValueError(f"research.{key} is a whole number from {lo} to {hi}")
+ return value
+
+
def _share_setting(key, value):
if key in ("enabled", "s3", "webdav", "remote"):
if not isinstance(value, bool):
diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
index 15050600a..2b272b99a 100644
--- a/fastapi_backend/app/domain/store.py
+++ b/fastapi_backend/app/domain/store.py
@@ -314,6 +314,10 @@ def labels():
# searxng_url; brave and tavily need api_key. results: per search; read_chars: of a page's text the model reads;
# render: a page that comes back nearly empty is loaded with Chromium (scripts run), when Chromium is installed
"web": {"search": "off", "searxng_url": None, "api_key": None, "results": 5, "read_chars": 20000, "render": False},
+ # deep research (research.py, docs/assistant.md#deep-research): the most tool calls a plan may make, the minutes
+ # a run may take before it writes its report with what it found, a model for it (null: the conversation's), and
+ # whether an agent (an API key, an app, MCP) may start a run without the person reviewing its plan
+ "research": {"max_tool_calls": 40, "max_minutes": 20, "model": None, "agents_skip_review": False},
# talking to Lens (voice.py): input auto uses the server's speech-to-text engine when it has one, else the
# browser's; spoken answers come from tts_model (an OpenAI-compatible /audio/speech), else the browser reads them
# stt: the engine that hears voice chat ("same": the transcription engine); tts_provider: who reads answers aloud
@@ -1463,6 +1467,12 @@ def close(self):
"DEFINE INDEX IF NOT EXISTS browser_action_session ON browser_action FIELDS session, state",
# each conversation's code sandbox on the OpenShell gateway (sandbox.py): its name, settings and last use
"DEFINE TABLE IF NOT EXISTS code_sandbox SCHEMALESS",
+ # deep research (research.py): a goal's plan, its run's progress and report; research_run: is the claim of
+ # the process carrying a run out, with its heartbeat
+ "DEFINE TABLE IF NOT EXISTS research SCHEMALESS",
+ "DEFINE INDEX IF NOT EXISTS research_account ON research FIELDS account, status",
+ "DEFINE INDEX IF NOT EXISTS research_chat ON research FIELDS chat",
+ "DEFINE TABLE IF NOT EXISTS research_run SCHEMALESS",
"DEFINE TABLE IF NOT EXISTS segment_edit SCHEMALESS",
"DEFINE INDEX IF NOT EXISTS segment_edit_rec ON segment_edit FIELDS recording",
# descriptive metadata history, IIIF change discovery and IIIF authorization
diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py
index b512cf257..84470c449 100644
--- a/fastapi_backend/app/schemas/chats.py
+++ b/fastapi_backend/app/schemas/chats.py
@@ -228,6 +228,9 @@ class ChatMessage(ResponseModel):
parent: int | None = Field(None, description="the message it follows; null for a first question")
versions: Versions | None = Field(None, description="it and the others said in its place, from regenerating or editing")
meta: MessageRecord | None = Field(None, description="what produced it; none on messages from before it was recorded")
+ kind: Literal["research", "research_plan"] | None = Field(
+ None, description="research: a deep research report; research_plan: the plan it waits on (GET /research/{meta.research})"
+ )
class MessageDetail(ChatMessage):
@@ -318,6 +321,11 @@ class MessageCreate(RequestModel):
None,
description="what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read",
)
+ research: bool | None = Field(
+ None,
+ description="the content is a research goal: the reply is a plan to approve (streams `research`, then `approval`), "
+ "as /research and POST /chats/{cid}/research do",
+ )
class ToolInfo(ResponseModel):
diff --git a/fastapi_backend/app/schemas/research.py b/fastapi_backend/app/schemas/research.py
new file mode 100644
index 000000000..233c61763
--- /dev/null
+++ b/fastapi_backend/app/schemas/research.py
@@ -0,0 +1,138 @@
+"""Deep research: a goal's plan, its run in the background and its report (app/domain/research.py)."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import Field
+
+from app.schemas.chats import ChatScope, MessageRecord
+from app.schemas.common import RequestModel, ResponseModel
+from app.schemas.podcasts import Style
+
+Source = Literal["archive", "graph", "notes", "web", "code"]
+
+
+class ResearchQuestion(ResponseModel):
+ question: str
+ answers: list[str] = Field(default_factory=list, description="suggested answers to pick from")
+
+
+class ResearchPlan(ResponseModel):
+ title: str
+ goal: str
+ sub_questions: list[str] = Field(description="what it answers, one look-up at a time, in order")
+ sources: list[Source] = Field(description="where it looks: the archive, the graph, notes, the web, the code sandbox")
+ max_tool_calls: int = Field(description="the most tool calls the run makes in all (research.max_tool_calls at most)")
+ report_outline: list[str] = Field(default_factory=list, description="the report's sections")
+
+
+class ResearchEstimate(ResponseModel):
+ usd: float | None = Field(None, description="what it will likely cost, from the prices in Settings; null without a price")
+ tokens: int | None = None
+ tool_calls: int | None = None
+ basis: str | None = Field(None, description="what the estimate is drawn from")
+ estimate: bool = True
+
+
+class ResearchProgress(ResponseModel):
+ sub_questions: int = 0
+ done: int = Field(0, description="sub-questions answered")
+ sources: int = Field(0, description="sources found so far")
+ tool_calls: int = 0
+ cost_usd: float = Field(0.0, description="spent so far (the activity ledger rows of research:)")
+ tokens: int = 0
+ cost_estimated: bool = False
+ minutes: float = Field(0.0, description="how long it has run")
+
+
+class ResearchFinding(ResponseModel):
+ question: str
+ tool_calls: int = 0
+
+
+class ResearchHeld(ResponseModel):
+ why: str
+ budget: str | None = Field(None, description="the budget it waits for (all, space:)")
+
+
+class Research(ResponseModel):
+ id: int
+ chat: int
+ message: int | None = Field(None, description="the question that asked for it")
+ plan_message: int | None = Field(None, description="the answer that shows the plan")
+ answer: int | None = Field(None, description="the report, once saved in the conversation")
+ approval: int | None = Field(
+ None, description="the plan's approval (POST /approvals/{id}: approve, with `params` its edits, or decline)"
+ )
+ status: Literal["planned", "declined", "queued", "running", "held", "done", "stopped", "failed"] = Field(
+ description="planned (waiting for approval), declined, queued (approved, waiting its turn), running, held (by a "
+ "budget), done, stopped or failed"
+ )
+ stage: Literal["gather", "write", "check", "done"] | None = None
+ goal: str | None = None
+ plan: ResearchPlan
+ questions: list[ResearchQuestion] = Field(default_factory=list, description="follow-up questions; answer them with revise")
+ answers: dict[str, str] = Field(default_factory=dict, description="the answers given to them")
+ scope: dict[str, Any] = Field(default_factory=dict)
+ estimate: ResearchEstimate | None = None
+ progress: ResearchProgress
+ findings: list[ResearchFinding] = Field(default_factory=list, description="the sub-questions answered so far")
+ held: ResearchHeld | None = None
+ error: str | None = None
+ ended: Literal["stopped", "timeout", "budget"] | None = Field(
+ None, description="why it ended early: stopped, out of time (research.max_minutes), or a budget"
+ )
+ revisions: int = 0
+ notes: list[int] = Field(default_factory=list, description="notes made from the report")
+ created_at: str | None = None
+ started_at: str | None = None
+ finished_at: str | None = None
+
+
+class ResearchStart(RequestModel):
+ goal: str = Field(min_length=1, max_length=4000, description="what to research")
+ tools: list[str] | None = Field(None, max_length=100, description="what it may use (GET /chats/tools); null: the conversation's choice")
+ max_tool_calls: int | None = Field(None, ge=1, description="the most tool calls the plan may make (research.max_tool_calls at most)")
+ scope: ChatScope | None = Field(None, description="what it draws on, instead of the conversation's scope")
+ review: bool = Field(
+ True,
+ description="false: start the run at once, without the plan waiting for review (signed in; an API key or an app "
+ "only with research.agents_skip_review)",
+ )
+
+
+class ResearchStarted(ResponseModel):
+ research: Research
+ question: int = Field(description="the question saved in the conversation")
+ message: int = Field(description="the answer that shows the plan")
+ text: str = Field(description="the plan as the conversation shows it")
+ meta: MessageRecord | None = None
+
+
+class ResearchRevise(RequestModel):
+ answers: dict[str, str] = Field(default_factory=dict, description="the follow-up questions' answers, by question")
+ plan: dict[str, Any] | None = Field(
+ None, description="the plan as edited: any of title, sub_questions, sources, max_tool_calls, report_outline"
+ )
+
+
+class ResearchNote(RequestModel):
+ namespace: str | None = Field(None, description="where the note goes; needed when the research looked in several")
+
+
+class ResearchNoteMade(ResponseModel):
+ page: int
+ namespace: str
+
+
+class ResearchPodcast(RequestModel):
+ prompt: str | None = Field(None, max_length=2000, description="an angle; else the research goal")
+ length: int = Field(10, ge=1, le=120, description="minutes (up to podcasts.max_minutes)")
+ style: Style = "deep-dive"
+ title: str | None = Field(None, max_length=200, description="else the report's title")
+
+
+class ResearchStop(ResponseModel):
+ ok: bool = True
+ stopping: bool = Field(description="it stops after the step it's on (or now, when it hadn't started or waited on a budget)")
diff --git a/fastapi_backend/tests/api/_assist.py b/fastapi_backend/tests/api/_assist.py
index 0eacfb47f..3d86306f2 100644
--- a/fastapi_backend/tests/api/_assist.py
+++ b/fastapi_backend/tests/api/_assist.py
@@ -27,6 +27,7 @@ def start_llm(cfg):
fake_llm.Handler.tool_script, fake_llm.Handler.reject_tools, fake_llm.Handler.decision = [], False, None
fake_llm.Handler.ignore_tool_choice = False
fake_llm.Handler.names, fake_llm.Handler.cypher_script = None, []
+ fake_llm.Handler.research_plan = fake_llm.Handler.research_report = None
fake_llm.Handler.seen = [] # what earlier tests on this worker sent isn't this test's
srv, url = fake_llm.start()
cfg["llm"].update(base_url=url, model="fake")
diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py
index 6e720b9df..2843f9c7e 100644
--- a/fastapi_backend/tests/api/test_mcp.py
+++ b/fastapi_backend/tests/api/test_mcp.py
@@ -91,6 +91,7 @@ def test_clients_are_sent_to_sign_in(client, app, env):
"set_install_settings",
"mark_activities",
"decline_approval",
+ "start_research",
"set_automation_vars",
"save_automation",
"run_automation",
diff --git a/fastapi_backend/tests/api/test_research.py b/fastapi_backend/tests/api/test_research.py
new file mode 100644
index 000000000..f8e7bc7a4
--- /dev/null
+++ b/fastapi_backend/tests/api/test_research.py
@@ -0,0 +1,583 @@
+"""Deep research (research.py, docs/assistant.md#deep-research): a goal gets a plan with follow-up questions, which
+waits for the person's approval (in the web app, the inbox or a chat room; an API key or an app only declines it unless
+research.agents_skip_review); approved, it runs in stages, claimed by one process at a time, within its tool calls,
+minutes and budgets, and stops when asked; its report comes back checked against its sources, and becomes a note or a
+podcast episode."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from app.api.v1.routes import podcasts as podcast_routes
+from app.domain import activity, ai_tools, bridge, budgets, chat, research, store
+from tests import fake_llm
+from tests.api._assist import Assist, sse, start_llm
+from tests.api.test_web_search import _Site, site # noqa: F401 - the fake web site, for a plan that looks on the web
+
+GOAL = "What happened with the capsid work?"
+REPORT = "# Capsid research\n\nThe capsid model beat the human designs [1]. Samples ship on Friday [2]. Mars has capsids too [2]."
+
+
+@pytest.fixture
+def llm(cfg):
+ srv = start_llm(cfg)
+ yield fake_llm.Handler
+ srv.shutdown()
+
+
+@pytest.fixture
+def app(cfg, db, llm):
+ from app.main import create_app
+
+ return create_app(cfg, db, background=False)
+
+
+@pytest.fixture
+def now(monkeypatch):
+ """A run carried out where it's launched, not in a thread of its own, so the test sees it end."""
+ monkeypatch.setattr(research, "launch", lambda db, cfg, rid: research.run(db, cfg, rid))
+
+
+@pytest.fixture
+def launched(monkeypatch):
+ """The runs launched, none of them carried out."""
+ got: list[int] = []
+ monkeypatch.setattr(research, "launch", lambda db, cfg, rid: got.append(rid))
+ return got
+
+
+def call(i, name, args):
+ return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}
+
+
+def archive():
+ """The model's turns for the default plan's two sub-questions: one search of the archive each, then a finding."""
+ return [
+ {"content": "", "tool_calls": [call(1, "search_transcripts", {"query": "capsid model"})]},
+ {"content": "The capsid model beat the human designs [1]."},
+ {"content": "", "tool_calls": [call(2, "search_transcripts", {"query": "exploit benchmark"})]},
+ {"content": "The exploit benchmark measures working exploits [1]."},
+ ]
+
+
+def new_chat(c, h):
+ return c.post("/api/v1/chats", headers=h, json={}).json()["id"]
+
+
+def plan(c, h, cid, goal=GOAL, **body):
+ r = c.post(f"/api/v1/chats/{cid}/research", headers=h, json={"goal": goal, **body})
+ assert r.status_code == 200, r.text
+ return r.json()["research"]
+
+
+def approve(c, h, res, **body):
+ return c.post(f"/api/v1/approvals/{res['approval']}", headers=h, json={"decision": "approve", **body})
+
+
+def status(c, h, res):
+ return c.get(f"/api/v1/research/{res['id']}", headers=h).json()
+
+
+def token(c, h):
+ return {"Authorization": f"Bearer {c.post('/api/v1/tokens', headers=h, json={'name': 'agent', 'scope': 'write'}).json()['token']}"}
+
+
+def mcp(c, h, name, **args):
+ r = c.post("/mcp", headers=h, json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": name, "arguments": args}})
+ assert r.status_code == 200, r.text
+ res = r.json()["result"]
+ assert res["isError"] is False, res
+ return res["structuredContent"]
+
+
+def live(app):
+ """The settings the app's requests read (a copy of cfg, kept until a setting is saved)."""
+ return app.state.settings.current()
+
+
+def set_up(s, section, **values):
+ """An admin saves settings, as in the web app."""
+ ca, ha = s.cl["admin"]
+ r = ca.put(f"/api/v1/settings/{section}", headers=ha, json=values)
+ assert r.status_code == 200, r.text
+
+
+def schema(body):
+ return ((body.get("response_format") or {}).get("json_schema") or {}).get("schema") or {}
+
+
+def me(db, email="ed@x.io"):
+ return {**db.one("SELECT record::id(id) AS id, email FROM account WHERE email = $e", e=email), "admin": False}
+
+
+def test_a_goal_gets_a_plan_that_waits_for_approval_and_can_be_revised(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ r = c.post(f"/api/v1/chats/{cid}/research", headers=h, json={"goal": GOAL})
+ assert r.status_code == 200, r.text
+ got = r.json()
+ res = got["research"]
+ assert (res["status"], res["stage"], res["goal"]) == ("planned", None, GOAL)
+ assert res["plan"] == {
+ "title": "Capsid research",
+ "goal": GOAL,
+ "sub_questions": ["What did the capsid model do?", "When do the samples ship?"],
+ "sources": ["archive"],
+ "max_tool_calls": 6,
+ "report_outline": ["Findings", "Open questions"],
+ }
+ assert res["questions"] == [{"question": "Which period?", "answers": ["This year", "All time"]}]
+ # what it will likely cost, labelled an estimate (no past runs yet: its tokens, and the fake model has no price)
+ est = res["estimate"]
+ assert est["estimate"] is True and est["tool_calls"] == 6 and est["usd"] is None and "fake has no price" in est["basis"]
+ assert res["progress"] == {
+ "sub_questions": 2,
+ "done": 0,
+ "sources": 0,
+ "tool_calls": 0,
+ "cost_usd": 0.0,
+ "tokens": 0,
+ "cost_estimated": False,
+ "minutes": 0.0,
+ }
+ # the planner was told the sources this person has, and the most tool calls a plan may make
+ asked = next(b for b in llm.seen if {"questions", "plan"} <= set(schema(b).get("properties", {})))
+ assert "only from: archive, graph, notes" in asked["messages"][0]["content"] and "at most 40" in asked["messages"][0]["content"]
+ assert asked["messages"][-1]["content"] == f"Goal: {GOAL}"
+ # the conversation: the goal as a question asked for research, then the plan as the answer it waits on
+ view = c.get(f"/api/v1/chats/{cid}", headers=h).json()
+ assert view["title"] == GOAL
+ q, a = view["messages"]
+ assert q["content"] == GOAL and q["meta"]["via"] == "web" and q["meta"]["options"]["research"] is True
+ assert a["id"] == got["message"] and a["kind"] == "research_plan" and a["content"] == got["text"]
+ assert a["meta"]["research"] == res["id"] and a["meta"]["outcome"] == "answered"
+ assert "1. What did the capsid model do?" in got["text"] and "- Which period? (This year / All time)" in got["text"]
+ # it waits as an approval of the assistant's: the person approves it (as it is or edited) or declines it
+ ap = c.get(f"/api/v1/approvals/{res['approval']}", headers=h).json()
+ assert ap["tool"] == "research" and ap["status"] == "pending" and ap["chat"] == cid
+ assert ap["summary"] == "Research: Capsid research (2 sub-questions, in the archive, up to 6 tool calls)"
+ assert ap["can"] == {"approve": True, "decline": True, "edit": True, "always": False}
+ assert ap["estimate"]["tool_calls"] == 6
+ assert [x["id"] for x in c.get(f"/api/v1/research?chat_id={cid}", headers=h).json()] == [res["id"]]
+ # revised: the answers and the edits go back for one more planning call, and the plan and its approval change
+ before = len(llm.seen)
+ edits = {"sub_questions": ["What did the capsid model do?"], "max_tool_calls": 3}
+ r = c.post(f"/api/v1/research/{res['id']}/revise", headers=h, json={"answers": {"Which period?": "This year"}, "plan": edits})
+ assert r.status_code == 200, r.text
+ rev = r.json()
+ sent = llm.seen[before]["messages"][-1]["content"]
+ assert "Sub-questions:\n- What did the capsid model do?\nSources: archive\nTool calls: 3" in sent
+ assert sent.endswith("Their answers:\n- Which period?: This year")
+ assert rev["plan"]["title"] == "Capsid research (revised)" and rev["questions"] == [] and rev["revisions"] == 1
+ assert rev["answers"] == {"Which period?": "This year"} and rev["status"] == "planned"
+ a = db.one("SELECT args, summary FROM $r", r=store.R("approval", res["approval"]))
+ assert a["args"]["plan"]["title"] == "Capsid research (revised)" and a["summary"].startswith("Research: Capsid research (revised)")
+ # both planning calls count for the research, and the conversation
+ assert activity.totals(db, f"research:{res['id']}")["by_kind"]["out"]["calls"] == 2
+ assert activity.totals(db, f"chat:{cid}")["by_kind"]["out"]["calls"] == 2
+ # another person's research isn't there for them
+ cv, hv = s.cl["viewer"]
+ assert cv.get(f"/api/v1/research/{res['id']}", headers=hv).status_code == 404
+ assert cv.post(f"/api/v1/research/{res['id']}/revise", headers=hv, json={}).status_code in (403, 404)
+
+
+def test_a_question_marked_research_or_slash_research_streams_the_plan(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": f"/research {GOAL}"}).text)
+ assert ev["research"][0]["status"] == "planned" and ev["research"][0]["goal"] == GOAL
+ assert ev["approval"][0]["tool"] == "research" and ev["approval"][0]["id"] == ev["research"][0]["approval"]
+ assert ev["token"][0]["text"].startswith("Here's a plan to research **Capsid research**")
+ assert ev["meta"][0]["research"] == ev["research"][0]["id"] and "done" in ev
+ msgs = chat.history(db, cid)
+ assert msgs[0]["content"] == f"/research {GOAL}" and msgs[-1]["kind"] == "research_plan"
+ # research: true does the same with the question as it is
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Capsid shipments", "research": True}).text)
+ assert ev["research"][0]["goal"] == "Capsid shipments"
+ # an ordinary question is answered as before
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "What did Dyno say?"}).text)
+ assert "research" not in ev and ev["token"]
+ # with no model set up, research isn't offered
+ live(app)["llm"]["base_url"] = None
+ assert c.post(f"/api/v1/chats/{cid}/research", headers=h, json={"goal": GOAL}).status_code == 400
+
+
+def test_an_approved_plan_runs_and_its_report_comes_back_checked(app, db, cfg, folder, new_client, llm, now):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ res = plan(c, h, cid)
+ llm.tool_script = archive()
+ r = approve(c, h, res)
+ assert r.status_code == 200, r.text
+ assert r.json()["status"] == "done" and r.json()["result"] == {"research": res["id"], "status": "queued"}
+ got = status(c, h, res)
+ assert (got["status"], got["stage"], got["ended"], got["error"]) == ("done", "done", None, None)
+ assert got["progress"]["done"] == 2 and got["progress"]["tool_calls"] == 2 and got["progress"]["sources"] >= 2
+ assert got["findings"] == [
+ {"question": "What did the capsid model do?", "tool_calls": 1},
+ {"question": "When do the samples ship?", "tool_calls": 1},
+ ]
+ assert got["started_at"] and got["finished_at"] and llm.tool_script == []
+ # each sub-question was looked up with the archive's tools alone, within its share of the plan's tool calls
+ rounds = [b for b in llm.seen if b.get("tools")]
+ assert len(rounds) == 4
+ assert {t["function"]["name"] for b in rounds for t in b["tools"]} <= set(research.SOURCE_TOOLS["archive"])
+ assert "up to 3 tool calls" in rounds[0]["messages"][0]["content"]
+ assert "Answer this part of it: What did the capsid model do?" in rounds[0]["messages"][-1]["content"]
+ # the report was written from both findings, their citations numbered as one list
+ writer = next(b for b in llm.seen if b["messages"][0]["content"].startswith("You write research reports"))
+ told = writer["messages"][-1]["content"]
+ assert "The capsid model beat the human designs [1]." in told and "The exploit benchmark measures working exploits [2]." in told
+ assert "- Findings\n- Open questions" in told
+ # checked: the first unsupported claim rewritten, the one the excerpts don't support at all marked
+ view = c.get(f"/api/v1/chats/{cid}", headers=h).json()
+ report = view["messages"][-1]
+ assert report["id"] == got["answer"] and report["kind"] == "research" and report["parent"] == res["plan_message"]
+ assert report["content"] == REPORT + research.UNSUPPORTED
+ assert [p["n"] for p in report["passages"]] == [1, 2]
+ assert {p["recording_id"] for p in report["passages"]} <= {s.a, s.b}
+ assert [x["tool"] for x in report["steps"]] == ["search_transcripts", "search_transcripts"]
+ meta = report["meta"]
+ assert meta["research"] == res["id"] and meta["outcome"] == "answered" and meta["tools"]["picked"] == ["archive"]
+ assert meta["used"] == {"rounds": 2, "tool_calls": 2} and meta["limits"]["tool_calls"] == 6
+ # every call it made counts for research:: planning, four rounds, writing, checking, rewriting
+ assert activity.totals(db, f"research:{res['id']}")["by_kind"]["out"]["calls"] == 8
+ assert db.rows("SELECT * FROM research_run") == [] # its claim is let go
+ # an agent follows it: the report, and its sources with links
+ st = mcp(c, token(c, h), "research_status", id=res["id"])
+ assert st["status"] == "done" and st["report"] == report["content"] and st["url"].endswith(f"/chat/{cid}")
+ assert [x["n"] for x in st["sources"]] == [1, 2] and all(x["url"] for x in st["sources"])
+ # the next plan's estimate is drawn from what this run cost per tool call
+ nxt = plan(c, h, cid, "More about capsids")
+ assert nxt["estimate"]["basis"] == "what each tool call of 1 past research run cost" and nxt["estimate"]["tool_calls"] == 6
+
+
+def test_a_plan_that_looks_on_the_web_searches_without_asking_again(site, app, db, cfg, folder, new_client, llm, now): # noqa: F811 - the site first: it widens the networks the app reads
+ s = Assist(app, db, cfg, folder, new_client)
+ set_up(s, "web", search="searxng", searxng_url=f"{site}/searx")
+ cfg["web"].update(search="searxng", searxng_url=f"{site}/searx")
+ llm.research_plan = {
+ "title": "Harbour research",
+ "goal": "When do boats come in?",
+ "sub_questions": ["When do boats come in?"],
+ "sources": ["archive", "web"],
+ "max_tool_calls": 4,
+ "report_outline": [],
+ }
+ llm.research_report = "# Harbour research\n\nBoats come in at dawn [1]."
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ res = plan(c, h, cid, "When do boats come in at the harbour?")
+ assert (
+ res["plan"]["sources"] == ["archive", "web"]
+ and "the web" in c.get(f"/api/v1/approvals/{res['approval']}", headers=h).json()["summary"]
+ )
+ # the second search's words are the model's own, after it read the first's results: the plan approved the web,
+ # so it searches without asking again
+ llm.tool_script = [
+ {"content": "", "tool_calls": [call(1, "web_search", {"query": "harbour boats"})]},
+ {"content": "", "tool_calls": [call(2, "web_search", {"query": "secret budget figures"})]},
+ {"content": "Boats come in at dawn [1]."},
+ ]
+ assert approve(c, h, res).status_code == 200
+ assert status(c, h, res)["status"] == "done"
+ assert [g["query"]["q"] for g in _Site.got if g["path"] == "/searx/search"] == ["harbour boats", "secret budget figures"]
+ assert db.rows("SELECT id FROM approval WHERE tool != 'research'") == []
+ report = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-1]
+ assert report["kind"] == "research" and report["passages"][0]["source"] == "web"
+ assert report["passages"][0]["url"] == "https://www.harbour.example/news"
+ # the web tools allowed for the run are the plan's alone
+ box = research.toolbox(db, cfg, research.get(db, res["id"]), research.account_of(db, me(db)["id"]))
+ assert box.allowed_out == {"web_search", "web_read"} and box.only == set(
+ research.SOURCE_TOOLS["archive"] + research.SOURCE_TOOLS["web"]
+ )
+
+
+def test_who_may_approve_a_plan(app, db, cfg, folder, new_client, llm, launched):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ ht = token(c, h)
+ cid = new_chat(c, h)
+ res = plan(c, h, cid)
+ # an API key can only decline it, and can't start research without review
+ assert c.get(f"/api/v1/approvals/{res['approval']}", headers=ht).json()["can"] == {
+ "approve": False,
+ "decline": True,
+ "edit": False,
+ "always": False,
+ }
+ r = approve(c, ht, res)
+ assert r.status_code == 403 and "the person's own pick" in r.json()["detail"]
+ assert c.post(f"/api/v1/chats/{cid}/research", headers=ht, json={"goal": GOAL, "review": False}).status_code == 403
+ assert status(c, h, res)["status"] == "planned" and launched == []
+ r = c.post(f"/api/v1/approvals/{res['approval']}", headers=ht, json={"decision": "decline"})
+ assert r.status_code == 200 and r.json()["status"] == "declined"
+ assert status(c, h, res)["status"] == "declined" and approve(c, h, res).status_code == 400 # decided once
+ # the person approves it edited, within the sources they have
+ res = plan(c, h, cid)
+ r = approve(c, h, res, params={"sources": ["web"]})
+ assert r.status_code == 400 and r.json()["detail"] == "a plan looks in archive, graph, notes"
+ assert status(c, h, res)["status"] == "planned" # a bad edit doesn't decide it
+ r = approve(c, h, res, params={"sub_questions": ["What did the capsid model do?"], "max_tool_calls": 2})
+ assert r.status_code == 200, r.text
+ got = status(c, h, res)
+ assert got["status"] == "queued" and got["plan"]["sub_questions"] == ["What did the capsid model do?"]
+ assert got["plan"]["max_tool_calls"] == 2 and launched == [res["id"]]
+ # signed in, the person may skip the review; so may an agent once research.agents_skip_review lets it
+ r = c.post(f"/api/v1/chats/{cid}/research", headers=h, json={"goal": GOAL, "review": False})
+ assert r.status_code == 200 and r.json()["research"]["status"] == "queued"
+ set_up(s, "research", agents_skip_review=True)
+ res = plan(c, ht, cid)
+ assert c.get(f"/api/v1/approvals/{res['approval']}", headers=ht).json()["can"]["approve"] is True
+ assert approve(c, ht, res).status_code == 200 and status(c, h, res)["status"] == "queued"
+ # a person has one run going at a time: the others wait their turn (none was running here, so each launched)
+ assert len(launched) == 3
+
+
+def test_a_plan_is_approved_with_a_yes_in_a_chat_room(app, db, cfg, folder, new_client, llm, launched):
+ Assist(app, db, cfg, folder, new_client)
+ account = me(db)
+ cfg["bridge"] = {**cfg["bridge"], "users": ["alice"], "approve": "low_risk"}
+ cid = chat.create(db, account["id"], kind="bridge")
+ text, waiting = bridge.answer(db, cfg, cfg, account, cid, f"/research {GOAL}")
+ assert text.startswith("Here's a plan to research **Capsid research**") and [a["tool"] for a in waiting] == ["research"]
+ rid = db.one("SELECT record::id(id) AS id FROM research WHERE approval = $a", a=waiting[0]["id"])["id"]
+ q = chat.history(db, cid)[0]
+ assert q["content"] == f"/research {GOAL}" and q["meta"] == {"via": "room", "options": {"research": True}}
+ # someone not let in to approve isn't heard; alice's yes approves it as it is
+ assert bridge.decide_in_room(db, cfg, cfg, account, cid, {"username": "eve"}, "yes") is None
+ assert bridge.decide_in_room(db, cfg, cfg, account, cid, {"username": "alice"}, "yes") == f"Done: {waiting[0]['summary']}"
+ assert research.get(db, rid)["status"] == "queued" and launched == [rid]
+
+
+def test_agents_start_research_over_mcp(app, db, cfg, folder, new_client, llm, launched):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ ht = token(c, h)
+ got = mcp(c, ht, "start_research", goal=GOAL)
+ assert got["status"] == "planned" and got["approval"] and got["next"].startswith("waiting for the person to approve")
+ assert got["url"].endswith(f"/chat/{got['chat_id']}") and got["questions"][0]["question"] == "Which period?"
+ assert chat.get(db, got["chat_id"], me(db)["id"])["title"] == GOAL
+ st = mcp(c, ht, "research_status", id=got["research"])
+ assert st["status"] == "planned" and "report" not in st
+ # declining is an agent's to do; approving isn't
+ with pytest.raises(PermissionError):
+ ai_tools.approve(db, cfg, got["approval"], me(db), {s.pods}, via="agent")
+ # with research.agents_skip_review, it starts at once
+ set_up(s, "research", agents_skip_review=True)
+ again = mcp(c, ht, "start_research", goal="Capsid shipments", chat_id=got["chat_id"], tools=["archive"], max_tool_calls=3)
+ assert again["status"] == "queued" and again["next"].startswith("running") and launched == [again["research"]]
+ r = research.get(db, again["research"])
+ assert r["plan"]["max_tool_calls"] == 3 and r["tools"] == ["archive"]
+
+
+def test_the_assistant_plans_research_with_its_own_tool(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ groups = {g["name"]: g for g in c.get("/api/v1/chats/tools", headers=h).json()["groups"]}
+ assert groups["research"]["available"] and [t["name"] for t in groups["research"]["tools"]] == ["start_research"]
+ llm.tool_script = [
+ {"content": "", "tool_calls": [call(1, "start_research", {"goal": GOAL})]},
+ {"content": "I planned it: approve it to start."},
+ ]
+ ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Research the capsid work for me"}).text)
+ assert ev["step"][0]["summary"].startswith("Planned Research: Capsid research (2 sub-questions")
+ assert ev["approval"][0]["tool"] == "research"
+ r = db.one("SELECT record::id(id) AS id, status, chat FROM research WHERE approval = $a", a=ev["approval"][0]["id"])
+ assert r["status"] == "planned" and r["chat"] == cid
+ assert activity.totals(db, f"research:{r['id']}")["by_kind"]["out"]["calls"] == 1 # its planning call
+ # outside a conversation there's nothing to research in
+ box = ai_tools.Toolbox(db, cfg, me(db), {s.pods}, {s.pods}, {}, None)
+ assert "start_research" not in {t["function"]["name"] for t in box.specs()}
+ # without a model that uses tools, the group says why
+ live(app)["ai"]["tools"] = False
+ groups = {g["name"]: g for g in c.get("/api/v1/chats/tools", headers=h).json()["groups"]}
+ assert not groups["research"]["available"] and groups["research"]["setting"] == "ai.tools"
+ cfg["llm"]["base_url"] = None
+ assert research.unavailable(cfg) == ("no language model is set up", "llm.model")
+
+
+def test_a_budget_holds_a_run_until_it_allows_it(app, db, cfg, folder, new_client, llm, now):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ res = plan(c, h, cid)
+ budgets.put(db, "all", usd=0.5, on_over="ask")
+ activity.record("out", "test.spend", db=db, cost_usd=1.0)
+ llm.tool_script = archive()
+ assert approve(c, h, res).status_code == 200
+ got = status(c, h, res)
+ assert got["status"] == "held" and got["held"]["budget"] == "all" and "spent" in got["held"]["why"]
+ assert got["progress"]["done"] == 0 and len(llm.tool_script) == 4 # it asked nothing
+ # the sweep leaves it while the budget is over, and starts it once the budget allows it
+ assert research.sweep(db, cfg) == []
+ budgets.put(db, "all", usd=100, on_over="ask")
+ assert research.sweep(db, cfg) == [res["id"]]
+ assert status(c, h, res)["status"] == "done" and status(c, h, res)["held"] is None
+ # the person can let a held run go on anyway; only a held one
+ budgets.put(db, "all", usd=0.5, on_over="ask")
+ res = plan(c, h, cid)
+ llm.tool_script = archive()
+ approve(c, h, res)
+ assert status(c, h, res)["status"] == "held"
+ r = c.post(f"/api/v1/research/{res['id']}/resume", headers=h)
+ assert r.status_code == 200, r.text
+ assert status(c, h, res)["status"] == "done"
+ assert c.post(f"/api/v1/research/{res['id']}/resume", headers=h).status_code == 409
+ # a budget that skips stops the run with what it has
+ budgets.put(db, "all", usd=0.5, on_over="skip")
+ res = plan(c, h, cid)
+ approve(c, h, res)
+ got = status(c, h, res)
+ assert (got["status"], got["ended"]) == ("stopped", "budget")
+ last = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-1]
+ assert last["id"] == got["answer"] and last["content"] == "# Capsid research\n\nStopped before it found anything."
+
+
+def test_stop_ends_a_run_and_keeps_what_it_found(app, db, cfg, folder, new_client, llm, monkeypatch):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ # one waiting its turn stops at once; a plan that isn't approved is declined, not stopped
+ monkeypatch.setattr(research, "launch", lambda db, cfg, rid: None)
+ res = plan(c, h, cid)
+ approve(c, h, res)
+ assert c.post(f"/api/v1/research/{res['id']}/stop", headers=h).json() == {"ok": True, "stopping": True}
+ got = status(c, h, res)
+ assert (got["status"], got["ended"], got["answer"]) == ("stopped", "stopped", None)
+ assert c.post(f"/api/v1/research/{plan(c, h, cid)['id']}/stop", headers=h).json()["stopping"] is False
+ # a running one stops after the step it's on: here, as the second sub-question's search comes back
+ monkeypatch.setattr(research, "launch", lambda db, cfg, rid: research.run(db, cfg, rid))
+ res = plan(c, h, cid)
+ first, found, second, _ = archive()
+
+ def stop_now(body):
+ research.request_stop(db, cfg, res["id"])
+ return second
+
+ llm.tool_script = [first, found, stop_now, {"content": "never asked"}]
+ approve(c, h, res)
+ got = status(c, h, res)
+ assert (got["status"], got["ended"], got["progress"]["done"]) == ("stopped", "stopped", 1)
+ assert llm.tool_script == [{"content": "never asked"}]
+ last = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-1]
+ assert last["id"] == got["answer"] and last["kind"] == "research" and last["stopped"] is True
+ assert last["content"] == (
+ "# Capsid research\n\nStopped. What it found so far:\n\n## What did the capsid model do?\nThe capsid model beat the human designs [1]."
+ )
+ assert last["meta"]["outcome"] == "stopped" and [p["n"] for p in last["passages"]] == [1]
+ assert c.post(f"/api/v1/research/{res['id']}/stop", headers=h).json()["stopping"] is False # it's over
+
+
+def test_a_run_whose_process_went_away_is_taken_over_where_it_left_off(app, db, cfg, folder, new_client, llm, monkeypatch):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ monkeypatch.setattr(research, "launch", lambda db, cfg, rid: None)
+ res = plan(c, h, cid)
+ approve(c, h, res)
+ rid = res["id"]
+ # one process holds a run at a time, kept by its heartbeat
+ assert research.claim(db, rid, "a") and not research.claim(db, rid, "b")
+ assert research.beat(db, rid, "a") and not research.beat(db, rid, "b")
+ # process a answered the first sub-question, then went away
+ box = ai_tools.Toolbox(db, cfg, me(db), {s.pods}, {s.pods}, {}, cid)
+ box.call("search_transcripts", {"query": "capsid model"})
+ finding = {
+ "question": "What did the capsid model do?",
+ "text": "The capsid model beat the human designs [1].",
+ "passages": box.cited("[1]"),
+ "tool_calls": 1,
+ "steps": [],
+ }
+ db.q("UPDATE $r SET status = 'running', findings = [$f], used = {tool_calls: 1}", r=store.R("research", rid), f=finding)
+ assert research.sweep(db, cfg) == [] # its claim is fresh: a process is on it
+ db.q("UPDATE $r SET heartbeat_at = $t", r=store.R("research_run", rid), t=research._ago(research.STALE + 5))
+ monkeypatch.setattr(research, "launch", lambda db, cfg, rid: research.run(db, cfg, rid))
+ llm.tool_script = archive()[2:]
+ assert research.sweep(db, cfg) == [rid]
+ got = status(c, h, res)
+ assert got["status"] == "done" and got["progress"]["done"] == 2 and got["progress"]["tool_calls"] == 2
+ # only the second sub-question was looked up again
+ rounds = [b for b in llm.seen if b.get("tools")]
+ assert len(rounds) == 2 and "When do the samples ship?" in rounds[0]["messages"][-1]["content"]
+ assert not research.beat(db, rid, "a") and research.run(db, cfg, rid) is False # done: nothing to take
+
+
+def test_a_report_becomes_a_note_and_a_podcast(app, db, cfg, folder, new_client, llm, now, monkeypatch):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ res = plan(c, h, cid)
+ assert c.post(f"/api/v1/research/{res['id']}/note", headers=h).status_code == 400 # no report yet
+ llm.tool_script = archive()
+ approve(c, h, res)
+ r = c.post(f"/api/v1/research/{res['id']}/note", headers=h)
+ assert r.status_code == 200, r.text
+ made = r.json()
+ assert made["namespace"] == "pods"
+ page = c.get(f"/api/v1/notes/{made['page']}", headers=h).json()
+ passages = research.get(db, res["id"])["passages"]
+ first = passages[0]
+ assert page["title"] == "Capsid research" and page["author"] == "assistant"
+ assert research.mention(first) == f"@[{first.get('title') or 'Asset'}](recording:{first['recording_id']})"
+ assert f"The capsid model beat the human designs ({research.mention(first)})." in page["body"]
+ assert page["body"].rstrip().endswith(f"@[Research: Capsid research](chat:{cid})")
+ links = {x["target"]: x for x in page["links"]}
+ assert links[f"chat:{cid}"]["name"] == GOAL and links[f"recording:{first['recording_id']}"]["name"]
+ assert status(c, h, res)["notes"] == [made["page"]]
+ # someone else who reads the note sees the conversation's link, but not its name
+ cv, hv = s.cl["viewer"]
+ seen = {x["target"]: x for x in cv.get(f"/api/v1/notes/{made['page']}", headers=hv).json()["links"]}
+ assert seen[f"chat:{cid}"]["name"] is None and seen[f"recording:{first['recording_id']}"]["name"]
+ # a podcast episode from the assets the report cites, about the goal, titled after the report
+ asked = []
+
+ def start(acl, user, db, cfg, body):
+ asked.append(body)
+ return {"episode": 99, "job": 1, "namespace": "podcasts", "placed": "podcasts"}
+
+ monkeypatch.setattr(podcast_routes, "start", start)
+ r = c.post(f"/api/v1/research/{res['id']}/podcast", headers=h, json={"length": 5})
+ assert r.status_code == 202, r.text
+ assert asked[0].selection.recordings == list(dict.fromkeys(p["recording_id"] for p in passages))
+ assert (asked[0].prompt, asked[0].title, asked[0].length) == (GOAL, "Capsid research", 5)
+
+
+def test_research_settings(app, db, cfg, folder, new_client, llm):
+ s = Assist(app, db, cfg, folder, new_client)
+ ca, ha = s.cl["admin"]
+ for bad in ({"max_tool_calls": 0}, {"max_minutes": 241}, {"agents_skip_review": "yes"}, {"model": 3}, {"max_tool_calls": True}):
+ assert ca.put("/api/v1/settings/research", headers=ha, json=bad).status_code == 400, bad
+ r = ca.put("/api/v1/settings/research", headers=ha, json={"max_tool_calls": 12, "max_minutes": 5, "model": " fake-large "})
+ assert r.status_code == 200, r.text
+ got = ca.get("/api/v1/settings", headers=ha).json()["research"]["values"]
+ assert got == {"max_tool_calls": 12, "max_minutes": 5, "model": "fake-large", "agents_skip_review": False}
+ # a plan's tool calls stay within research.max_tool_calls
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ assert c.post(f"/api/v1/chats/{cid}/research", headers=h, json={"goal": GOAL, "max_tool_calls": 13}).status_code == 400
+ assert research.clean_plan(cfg | {"research": got}, {"sub_questions": ["a"], "max_tool_calls": 99}, ["archive"])["max_tool_calls"] == 12
+ assert ca.put("/api/v1/settings/research", headers=s.cl["editor"][1], json={"max_minutes": 9}).status_code == 403
+
+
+def test_a_run_out_of_time_writes_its_report_with_what_it_found(app, db, cfg, folder, new_client, llm, now, monkeypatch):
+ s = Assist(app, db, cfg, folder, new_client)
+ c, h = s.cl["editor"]
+ cid = new_chat(c, h)
+ res = plan(c, h, cid)
+ # out of time once the first sub-question is answered
+ monkeypatch.setattr(research.Run, "out_of_time", lambda self: bool(research.get(self.db, self.rid).get("findings") or []))
+ llm.tool_script = archive()
+ approve(c, h, res)
+ got = status(c, h, res)
+ assert (got["status"], got["ended"], got["progress"]["done"]) == ("done", "timeout", 1)
+ writer = next(b for b in llm.seen if b["messages"][0]["content"].startswith("You write research reports"))
+ assert "ran out of time before every sub-question was looked into" in writer["messages"][-1]["content"]
+ assert c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-1]["meta"]["outcome"] == "limit"
diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py
index 55ed02f90..4c93d24e5 100644
--- a/fastapi_backend/tests/fake_llm.py
+++ b/fastapi_backend/tests/fake_llm.py
@@ -18,6 +18,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
cypher_script = [] # Cypher to answer graph questions with (graph_ask.py), in order
podcast_script = None # the lines a podcast script comes back with (podcasts.write), else a built-in one
usage = None # token counts to report with each answer (and as a streamed answer's last chunk), like OpenAI
+ research_plan = None # a research plan (research.plan) to come back with, else a built-in one about the capsid
+ research_report = None # a research report (research._write) to come back with, else one citing [1] and [2]
def _json(self, obj):
data = json.dumps(obj).encode()
@@ -165,6 +167,30 @@ def do_POST(self):
else:
checks.append({"line": int(k), "verdict": "supported"})
content = json.dumps({"checks": checks})
+ elif {"questions", "plan"} <= set(schema.get("properties", {})):
+ # a research plan (research.plan): Handler.research_plan, else two sub-questions about the capsid in the
+ # archive, and a follow-up question unless it's a revision (answers given)
+ text = body["messages"][-1]["content"]
+ goal = re.search(r"^Goal: (.*)$", text, re.M).group(1)
+ plan = Handler.research_plan or {
+ "title": "Capsid research",
+ "goal": goal,
+ "sub_questions": ["What did the capsid model do?", "When do the samples ship?"],
+ "sources": ["archive"],
+ "max_tool_calls": 6,
+ "report_outline": ["Findings", "Open questions"],
+ }
+ revised = "Their answers:" in text or "The earlier plan" in text
+ questions = [] if revised else [{"question": "Which period?", "answers": ["This year", "All time"]}]
+ if revised:
+ plan = {**plan, "title": plan["title"] + " (revised)"}
+ content = json.dumps({"questions": questions, "plan": plan})
+ elif "rewrites" in schema.get("properties", {}):
+ # a research report's unsupported claims (research._check): the first one rewritten, the rest given up on
+ claims = [l[2:] for l in body["messages"][-1]["content"].splitlines() if l.startswith("- ")]
+ content = json.dumps(
+ {"rewrites": [{"claim": c, "text": "Samples ship on Friday [2]." if i == 0 else ""} for i, c in enumerate(claims)]}
+ )
elif "cypher" in schema.get("properties", {}):
# graph questions: the scripted queries in order, else one that finds organisations
q = Handler.cypher_script.pop(0) if Handler.cypher_script else "MATCH (e:Organisation) RETURN e, e.name LIMIT 50"
@@ -217,6 +243,11 @@ def do_POST(self):
content = json.dumps({"title": "Capsid plan" if title == "Untitled" else title, "summary": f"About {first}"})
elif set(schema.get("properties", {})) == {"choice", "confidence"}:
content = json.dumps(Handler.decision or {"choice": schema["properties"]["choice"]["enum"][0], "confidence": 0.9})
+ elif str(body["messages"][0].get("content") or "").startswith("You write research reports"):
+ content = Handler.research_report or (
+ "# Capsid research\n\nThe capsid model beat the human designs [1]. The shipment leaves on Friday [2]. "
+ "Mars has capsids too [2]."
+ )
elif body.get("response_format"):
content = json.dumps(
{
diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts
index cc6a059bb..d8ad47d51 100644
--- a/nextjs-frontend/app/openapi-client/index.ts
+++ b/nextjs-frontend/app/openapi-client/index.ts
@@ -35,6 +35,7 @@ export {
Podcasts,
Public,
Rdf,
+ Research as Research2,
Resources,
Routines,
Search,
@@ -1380,6 +1381,11 @@ export type {
GetRecordingWordcloudErrors,
GetRecordingWordcloudResponse,
GetRecordingWordcloudResponses,
+ GetResearchData,
+ GetResearchError,
+ GetResearchErrors,
+ GetResearchResponse,
+ GetResearchResponses,
GetResourceFieldsData,
GetResourceFieldsError,
GetResourceFieldsErrors,
@@ -1974,6 +1980,11 @@ export type {
ListRecordingsErrors,
ListRecordingsResponse,
ListRecordingsResponses,
+ ListResearchData,
+ ListResearchError,
+ ListResearchErrors,
+ ListResearchResponse,
+ ListResearchResponses,
ListRolesData,
ListRolesResponse,
ListRolesResponses,
@@ -2692,6 +2703,30 @@ export type {
RequestAccessErrors,
RequestAccessResponse,
RequestAccessResponses,
+ Research,
+ ResearchEstimate,
+ ResearchFinding,
+ ResearchHeld,
+ ResearchNote,
+ ResearchNoteData,
+ ResearchNoteError,
+ ResearchNoteErrors,
+ ResearchNoteMade,
+ ResearchNoteResponse,
+ ResearchNoteResponses,
+ ResearchPlan,
+ ResearchPodcast,
+ ResearchPodcastData,
+ ResearchPodcastError,
+ ResearchPodcastErrors,
+ ResearchPodcastResponse,
+ ResearchPodcastResponses,
+ ResearchProgress,
+ ResearchQuestion,
+ ResearchRevise,
+ ResearchStart,
+ ResearchStarted,
+ ResearchStop,
ResetPasswordData,
ResetPasswordError,
ResetPasswordErrors,
@@ -2739,6 +2774,11 @@ export type {
RestoreVersionResponse,
RestoreVersionResponses,
RestoreWork,
+ ResumeResearchData,
+ ResumeResearchError,
+ ResumeResearchErrors,
+ ResumeResearchResponse,
+ ResumeResearchResponses,
RetagRecordingsData,
RetagRecordingsError,
RetagRecordingsErrors,
@@ -2763,6 +2803,11 @@ export type {
ReviewNewData,
ReviewNewResponse,
ReviewNewResponses,
+ ReviseResearchData,
+ ReviseResearchError,
+ ReviseResearchErrors,
+ ReviseResearchResponse,
+ ReviseResearchResponses,
RevokeAnyTokenData,
RevokeAnyTokenError,
RevokeAnyTokenErrors,
@@ -3135,6 +3180,11 @@ export type {
StartRcloneSigninErrors,
StartRcloneSigninResponse,
StartRcloneSigninResponses,
+ StartResearchData,
+ StartResearchError,
+ StartResearchErrors,
+ StartResearchResponse,
+ StartResearchResponses,
StartUpgradeData,
StartUpgradeError,
StartUpgradeErrors,
@@ -3156,6 +3206,11 @@ export type {
StopAnswerErrors,
StopAnswerResponse,
StopAnswerResponses,
+ StopResearchData,
+ StopResearchError,
+ StopResearchErrors,
+ StopResearchResponse,
+ StopResearchResponses,
StopResult,
Stream,
StreamActivity,
diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts
index 3e3aaf2ec..0a606f38c 100644
--- a/nextjs-frontend/app/openapi-client/sdk.gen.ts
+++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts
@@ -668,6 +668,9 @@ import type {
GetRecordingWordcloudData,
GetRecordingWordcloudErrors,
GetRecordingWordcloudResponses,
+ GetResearchData,
+ GetResearchErrors,
+ GetResearchResponses,
GetResourceFieldsData,
GetResourceFieldsErrors,
GetResourceFieldsResponses,
@@ -986,6 +989,9 @@ import type {
ListRecordingsData,
ListRecordingsErrors,
ListRecordingsResponses,
+ ListResearchData,
+ ListResearchErrors,
+ ListResearchResponses,
ListRolesData,
ListRolesResponses,
ListRoutinesData,
@@ -1293,6 +1299,12 @@ import type {
RequestAccessData,
RequestAccessErrors,
RequestAccessResponses,
+ ResearchNoteData,
+ ResearchNoteErrors,
+ ResearchNoteResponses,
+ ResearchPodcastData,
+ ResearchPodcastErrors,
+ ResearchPodcastResponses,
ResetPasswordData,
ResetPasswordErrors,
ResetPasswordResponses,
@@ -1317,6 +1329,9 @@ import type {
RestoreVersionData,
RestoreVersionErrors,
RestoreVersionResponses,
+ ResumeResearchData,
+ ResumeResearchErrors,
+ ResumeResearchResponses,
RetagRecordingsData,
RetagRecordingsErrors,
RetagRecordingsResponses,
@@ -1331,6 +1346,9 @@ import type {
RevertMetadataEditResponses,
ReviewNewData,
ReviewNewResponses,
+ ReviseResearchData,
+ ReviseResearchErrors,
+ ReviseResearchResponses,
RevokeAnyTokenData,
RevokeAnyTokenErrors,
RevokeAnyTokenResponses,
@@ -1492,6 +1510,9 @@ import type {
StartRcloneSigninData,
StartRcloneSigninErrors,
StartRcloneSigninResponses,
+ StartResearchData,
+ StartResearchErrors,
+ StartResearchResponses,
StartUpgradeData,
StartUpgradeErrors,
StartUpgradeResponses,
@@ -1503,6 +1524,9 @@ import type {
StopAnswerData,
StopAnswerErrors,
StopAnswerResponses,
+ StopResearchData,
+ StopResearchErrors,
+ StopResearchResponses,
StreamEventsData,
StreamEventsErrors,
StreamEventsResponses,
@@ -10223,6 +10247,10 @@ export class Chats {
* `/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the
* conversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.
*
+ * With `research` (or /research ) the content is a research goal: the answer is a plan to approve (streamed as
+ * `research`, then `approval`), which runs in the background once approved (POST /chats/{cid}/research, GET
+ * /research/{id}).
+ *
* With `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that
* browser while the side panel is connected: open pages, read them, click, type and download files into the
* conversation (docs/browser-assistant.md).
@@ -10335,6 +10363,148 @@ export class Chats {
}
}
+export class Research {
+ /**
+ * Start Research
+ *
+ * Start research on a goal in a conversation: the goal is saved as a question and one model call plans it (up to
+ * three follow-up questions with answers to pick, the sub-questions, where it looks, the tool calls it may make, the
+ * report's outline, and an estimate of the cost). The plan waits as an approval (POST /approvals/{id}: approve it as
+ * it is, or with `params` its edits; or decline), or revise it (POST /research/{id}/revise). With `review` false it
+ * starts at once: your own pick when signed in; an API key or an app needs research.agents_skip_review. The same as a
+ * question with `research: true`, or /research .
+ */
+ public static startResearch(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).post({
+ url: "/api/v1/chats/{cid}/research",
+ ...options,
+ headers: {
+ "Content-Type": "application/json",
+ ...options.headers,
+ },
+ });
+ }
+
+ /**
+ * List Research
+ *
+ * Your research, newest first; with `chat_id`, that conversation's (to show each plan's or run's card).
+ */
+ public static listResearch(
+ options?: Options,
+ ): RequestResult {
+ return (options?.client ?? client).get({
+ url: "/api/v1/research",
+ ...options,
+ });
+ }
+
+ /**
+ * Get Research
+ *
+ * A research run: its plan, status and stage (gather, write, check, done), progress (sub-questions done, sources
+ * found, tool calls and cost so far, from the activity ledger) and, once it's done, the report's message
+ * (`answer`, in the conversation). What it waits for when a budget holds it (`held`), and why it ended early.
+ */
+ public static getResearch(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).get({
+ url: "/api/v1/research/{rid}",
+ ...options,
+ });
+ }
+
+ /**
+ * Revise Research
+ *
+ * Send your answers to the follow-up questions and your edits to the plan back for one more planning call; the
+ * plan and its approval change in place. Only while the plan waits for approval.
+ */
+ public static reviseResearch(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).post({
+ url: "/api/v1/research/{rid}/revise",
+ ...options,
+ headers: {
+ "Content-Type": "application/json",
+ ...options.headers,
+ },
+ });
+ }
+
+ /**
+ * Stop Research
+ *
+ * Stop a run: it stops after the step it's on and keeps what it found, saved in the conversation as its answer,
+ * marked stopped (one that hasn't started, or waits on a budget, stops now). Works whichever process runs it.
+ */
+ public static stopResearch(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).post({
+ url: "/api/v1/research/{rid}/stop",
+ ...options,
+ });
+ }
+
+ /**
+ * Resume Research
+ *
+ * Let a run that a budget holds go on anyway (your pick, as "run it once" is for a held routine run).
+ */
+ public static resumeResearch(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).post({
+ url: "/api/v1/research/{rid}/resume",
+ ...options,
+ });
+ }
+
+ /**
+ * Research Note
+ *
+ * Save the report as a note: in the namespace it looked in (`namespace` when there are several), its archive
+ * citations as links to the assets they cite, web citations as links, and a link back to the conversation (shown to
+ * you alone). Needs editor access there.
+ */
+ public static researchNote(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).post({
+ url: "/api/v1/research/{rid}/note",
+ ...options,
+ headers: {
+ "Content-Type": "application/json",
+ ...options.headers,
+ },
+ });
+ }
+
+ /**
+ * Research Podcast
+ *
+ * Make a podcast episode from the assets the report cites (POST /podcasts with them picked): about the research
+ * goal unless `prompt` says otherwise, titled after the report.
+ */
+ public static researchPodcast(
+ options: Options,
+ ): RequestResult {
+ return (options.client ?? client).post({
+ url: "/api/v1/research/{rid}/podcast",
+ ...options,
+ headers: {
+ "Content-Type": "application/json",
+ ...options.headers,
+ },
+ });
+ }
+}
+
export class Browser {
/**
* List Sessions
diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts
index a7e125871..61dcbbd22 100644
--- a/nextjs-frontend/app/openapi-client/types.gen.ts
+++ b/nextjs-frontend/app/openapi-client/types.gen.ts
@@ -3676,6 +3676,12 @@ export type ChatMessage = {
* what produced it; none on messages from before it was recorded
*/
meta?: MessageRecord | null;
+ /**
+ * Kind
+ *
+ * research: a deep research report; research_plan: the plan it waits on (GET /research/{meta.research})
+ */
+ kind?: "research" | "research_plan" | null;
[key: string]: unknown;
};
@@ -11587,6 +11593,12 @@ export type MessageCreate = {
* what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read
*/
scope?: ChatScope | null;
+ /**
+ * Research
+ *
+ * the content is a research goal: the reply is a plan to approve (streams `research`, then `approval`), as /research and POST /chats/{cid}/research do
+ */
+ research?: boolean | null;
};
/**
@@ -11671,6 +11683,12 @@ export type MessageDetail = {
* what produced it; none on messages from before it was recorded
*/
meta?: MessageRecord | null;
+ /**
+ * Kind
+ *
+ * research: a deep research report; research_plan: the plan it waits on (GET /research/{meta.research})
+ */
+ kind?: "research" | "research_plan" | null;
/**
* Ledger
*
@@ -17472,6 +17490,442 @@ export type ReprocessRequest = {
pipeline?: number | null;
};
+/**
+ * Research
+ */
+export type Research = {
+ /**
+ * Id
+ */
+ id: number;
+ /**
+ * Chat
+ */
+ chat: number;
+ /**
+ * Message
+ *
+ * the question that asked for it
+ */
+ message?: number | null;
+ /**
+ * Plan Message
+ *
+ * the answer that shows the plan
+ */
+ plan_message?: number | null;
+ /**
+ * Answer
+ *
+ * the report, once saved in the conversation
+ */
+ answer?: number | null;
+ /**
+ * Approval
+ *
+ * the plan's approval (POST /approvals/{id}: approve, with `params` its edits, or decline)
+ */
+ approval?: number | null;
+ /**
+ * Status
+ *
+ * planned (waiting for approval), declined, queued (approved, waiting its turn), running, held (by a budget), done, stopped or failed
+ */
+ status: "planned" | "declined" | "queued" | "running" | "held" | "done" | "stopped" | "failed";
+ /**
+ * Stage
+ */
+ stage?: "gather" | "write" | "check" | "done" | null;
+ /**
+ * Goal
+ */
+ goal?: string | null;
+ plan: ResearchPlan;
+ /**
+ * Questions
+ *
+ * follow-up questions; answer them with revise
+ */
+ questions?: Array;
+ /**
+ * Answers
+ *
+ * the answers given to them
+ */
+ answers?: {
+ [key: string]: string;
+ };
+ /**
+ * Scope
+ */
+ scope?: {
+ [key: string]: unknown;
+ };
+ estimate?: ResearchEstimate | null;
+ progress: ResearchProgress;
+ /**
+ * Findings
+ *
+ * the sub-questions answered so far
+ */
+ findings?: Array;
+ held?: ResearchHeld | null;
+ /**
+ * Error
+ */
+ error?: string | null;
+ /**
+ * Ended
+ *
+ * why it ended early: stopped, out of time (research.max_minutes), or a budget
+ */
+ ended?: "stopped" | "timeout" | "budget" | null;
+ /**
+ * Revisions
+ */
+ revisions?: number;
+ /**
+ * Notes
+ *
+ * notes made from the report
+ */
+ notes?: Array;
+ /**
+ * Created At
+ */
+ created_at?: string | null;
+ /**
+ * Started At
+ */
+ started_at?: string | null;
+ /**
+ * Finished At
+ */
+ finished_at?: string | null;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchEstimate
+ */
+export type ResearchEstimate = {
+ /**
+ * Usd
+ *
+ * what it will likely cost, from the prices in Settings; null without a price
+ */
+ usd?: number | null;
+ /**
+ * Tokens
+ */
+ tokens?: number | null;
+ /**
+ * Tool Calls
+ */
+ tool_calls?: number | null;
+ /**
+ * Basis
+ *
+ * what the estimate is drawn from
+ */
+ basis?: string | null;
+ /**
+ * Estimate
+ */
+ estimate?: boolean;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchFinding
+ */
+export type ResearchFinding = {
+ /**
+ * Question
+ */
+ question: string;
+ /**
+ * Tool Calls
+ */
+ tool_calls?: number;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchHeld
+ */
+export type ResearchHeld = {
+ /**
+ * Why
+ */
+ why: string;
+ /**
+ * Budget
+ *
+ * the budget it waits for (all, space:)
+ */
+ budget?: string | null;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchNote
+ */
+export type ResearchNote = {
+ /**
+ * Namespace
+ *
+ * where the note goes; needed when the research looked in several
+ */
+ namespace?: string | null;
+};
+
+/**
+ * ResearchNoteMade
+ */
+export type ResearchNoteMade = {
+ /**
+ * Page
+ */
+ page: number;
+ /**
+ * Namespace
+ */
+ namespace: string;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchPlan
+ */
+export type ResearchPlan = {
+ /**
+ * Title
+ */
+ title: string;
+ /**
+ * Goal
+ */
+ goal: string;
+ /**
+ * Sub Questions
+ *
+ * what it answers, one look-up at a time, in order
+ */
+ sub_questions: Array;
+ /**
+ * Sources
+ *
+ * where it looks: the archive, the graph, notes, the web, the code sandbox
+ */
+ sources: Array<"archive" | "graph" | "notes" | "web" | "code">;
+ /**
+ * Max Tool Calls
+ *
+ * the most tool calls the run makes in all (research.max_tool_calls at most)
+ */
+ max_tool_calls: number;
+ /**
+ * Report Outline
+ *
+ * the report's sections
+ */
+ report_outline?: Array;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchPodcast
+ */
+export type ResearchPodcast = {
+ /**
+ * Prompt
+ *
+ * an angle; else the research goal
+ */
+ prompt?: string | null;
+ /**
+ * Length
+ *
+ * minutes (up to podcasts.max_minutes)
+ */
+ length?: number;
+ /**
+ * Style
+ */
+ style?: "deep-dive" | "recap" | "debate" | "beginner";
+ /**
+ * Title
+ *
+ * else the report's title
+ */
+ title?: string | null;
+};
+
+/**
+ * ResearchProgress
+ */
+export type ResearchProgress = {
+ /**
+ * Sub Questions
+ */
+ sub_questions?: number;
+ /**
+ * Done
+ *
+ * sub-questions answered
+ */
+ done?: number;
+ /**
+ * Sources
+ *
+ * sources found so far
+ */
+ sources?: number;
+ /**
+ * Tool Calls
+ */
+ tool_calls?: number;
+ /**
+ * Cost Usd
+ *
+ * spent so far (the activity ledger rows of research:)
+ */
+ cost_usd?: number;
+ /**
+ * Tokens
+ */
+ tokens?: number;
+ /**
+ * Cost Estimated
+ */
+ cost_estimated?: boolean;
+ /**
+ * Minutes
+ *
+ * how long it has run
+ */
+ minutes?: number;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchQuestion
+ */
+export type ResearchQuestion = {
+ /**
+ * Question
+ */
+ question: string;
+ /**
+ * Answers
+ *
+ * suggested answers to pick from
+ */
+ answers?: Array;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchRevise
+ */
+export type ResearchRevise = {
+ /**
+ * Answers
+ *
+ * the follow-up questions' answers, by question
+ */
+ answers?: {
+ [key: string]: string;
+ };
+ /**
+ * Plan
+ *
+ * the plan as edited: any of title, sub_questions, sources, max_tool_calls, report_outline
+ */
+ plan?: {
+ [key: string]: unknown;
+ } | null;
+};
+
+/**
+ * ResearchStart
+ */
+export type ResearchStart = {
+ /**
+ * Goal
+ *
+ * what to research
+ */
+ goal: string;
+ /**
+ * Tools
+ *
+ * what it may use (GET /chats/tools); null: the conversation's choice
+ */
+ tools?: Array | null;
+ /**
+ * Max Tool Calls
+ *
+ * the most tool calls the plan may make (research.max_tool_calls at most)
+ */
+ max_tool_calls?: number | null;
+ /**
+ * what it draws on, instead of the conversation's scope
+ */
+ scope?: ChatScope | null;
+ /**
+ * Review
+ *
+ * false: start the run at once, without the plan waiting for review (signed in; an API key or an app only with research.agents_skip_review)
+ */
+ review?: boolean;
+};
+
+/**
+ * ResearchStarted
+ */
+export type ResearchStarted = {
+ research: Research;
+ /**
+ * Question
+ *
+ * the question saved in the conversation
+ */
+ question: number;
+ /**
+ * Message
+ *
+ * the answer that shows the plan
+ */
+ message: number;
+ /**
+ * Text
+ *
+ * the plan as the conversation shows it
+ */
+ text: string;
+ meta?: MessageRecord | null;
+ [key: string]: unknown;
+};
+
+/**
+ * ResearchStop
+ */
+export type ResearchStop = {
+ /**
+ * Ok
+ */
+ ok?: boolean;
+ /**
+ * Stopping
+ *
+ * it stops after the step it's on (or now, when it hadn't started or waited on a budget)
+ */
+ stopping: boolean;
+ [key: string]: unknown;
+};
+
/**
* ResetPasswordRequest
*/
@@ -40243,6 +40697,258 @@ export type DecideApprovalResponses = {
export type DecideApprovalResponse = DecideApprovalResponses[keyof DecideApprovalResponses];
+export type StartResearchData = {
+ body: ResearchStart;
+ path: {
+ /**
+ * Cid
+ */
+ cid: number;
+ };
+ query?: never;
+ url: "/api/v1/chats/{cid}/research";
+};
+
+export type StartResearchErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type StartResearchError = StartResearchErrors[keyof StartResearchErrors];
+
+export type StartResearchResponses = {
+ /**
+ * Successful Response
+ */
+ 200: ResearchStarted;
+};
+
+export type StartResearchResponse = StartResearchResponses[keyof StartResearchResponses];
+
+export type ListResearchData = {
+ body?: never;
+ path?: never;
+ query?: {
+ /**
+ * Chat Id
+ */
+ chat_id?: number | null;
+ /**
+ * Limit
+ */
+ limit?: number;
+ };
+ url: "/api/v1/research";
+};
+
+export type ListResearchErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ListResearchError = ListResearchErrors[keyof ListResearchErrors];
+
+export type ListResearchResponses = {
+ /**
+ * Response Research-List Research
+ *
+ * Successful Response
+ */
+ 200: Array;
+};
+
+export type ListResearchResponse = ListResearchResponses[keyof ListResearchResponses];
+
+export type GetResearchData = {
+ body?: never;
+ path: {
+ /**
+ * Rid
+ */
+ rid: number;
+ };
+ query?: never;
+ url: "/api/v1/research/{rid}";
+};
+
+export type GetResearchErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type GetResearchError = GetResearchErrors[keyof GetResearchErrors];
+
+export type GetResearchResponses = {
+ /**
+ * Successful Response
+ */
+ 200: Research;
+};
+
+export type GetResearchResponse = GetResearchResponses[keyof GetResearchResponses];
+
+export type ReviseResearchData = {
+ body: ResearchRevise;
+ path: {
+ /**
+ * Rid
+ */
+ rid: number;
+ };
+ query?: never;
+ url: "/api/v1/research/{rid}/revise";
+};
+
+export type ReviseResearchErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ReviseResearchError = ReviseResearchErrors[keyof ReviseResearchErrors];
+
+export type ReviseResearchResponses = {
+ /**
+ * Successful Response
+ */
+ 200: Research;
+};
+
+export type ReviseResearchResponse = ReviseResearchResponses[keyof ReviseResearchResponses];
+
+export type StopResearchData = {
+ body?: never;
+ path: {
+ /**
+ * Rid
+ */
+ rid: number;
+ };
+ query?: never;
+ url: "/api/v1/research/{rid}/stop";
+};
+
+export type StopResearchErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type StopResearchError = StopResearchErrors[keyof StopResearchErrors];
+
+export type StopResearchResponses = {
+ /**
+ * Successful Response
+ */
+ 200: ResearchStop;
+};
+
+export type StopResearchResponse = StopResearchResponses[keyof StopResearchResponses];
+
+export type ResumeResearchData = {
+ body?: never;
+ path: {
+ /**
+ * Rid
+ */
+ rid: number;
+ };
+ query?: never;
+ url: "/api/v1/research/{rid}/resume";
+};
+
+export type ResumeResearchErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ResumeResearchError = ResumeResearchErrors[keyof ResumeResearchErrors];
+
+export type ResumeResearchResponses = {
+ /**
+ * Successful Response
+ */
+ 200: Research;
+};
+
+export type ResumeResearchResponse = ResumeResearchResponses[keyof ResumeResearchResponses];
+
+export type ResearchNoteData = {
+ /**
+ * Body
+ */
+ body?: ResearchNote | null;
+ path: {
+ /**
+ * Rid
+ */
+ rid: number;
+ };
+ query?: never;
+ url: "/api/v1/research/{rid}/note";
+};
+
+export type ResearchNoteErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ResearchNoteError = ResearchNoteErrors[keyof ResearchNoteErrors];
+
+export type ResearchNoteResponses = {
+ /**
+ * Successful Response
+ */
+ 200: ResearchNoteMade;
+};
+
+export type ResearchNoteResponse = ResearchNoteResponses[keyof ResearchNoteResponses];
+
+export type ResearchPodcastData = {
+ /**
+ * Body
+ */
+ body?: ResearchPodcast | null;
+ path: {
+ /**
+ * Rid
+ */
+ rid: number;
+ };
+ query?: never;
+ url: "/api/v1/research/{rid}/podcast";
+};
+
+export type ResearchPodcastErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ResearchPodcastError = ResearchPodcastErrors[keyof ResearchPodcastErrors];
+
+export type ResearchPodcastResponses = {
+ /**
+ * Successful Response
+ */
+ 202: PodcastStarted;
+};
+
+export type ResearchPodcastResponse = ResearchPodcastResponses[keyof ResearchPodcastResponses];
+
export type ListSessionsData = {
body?: never;
path?: never;
diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json
index 072ce006e..80dc1d514 100644
--- a/nextjs-frontend/openapi.json
+++ b/nextjs-frontend/openapi.json
@@ -25519,7 +25519,7 @@
"chats"
],
"summary": "Send Message",
- "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nA conversation is a tree of versions (GET /chats/{cid}: `tree`, and each message's `versions`); the question goes\non from `parent` when given (any message of the conversation), else from the version showing (`head`). With\n`regenerate` (an answer's id) its question is answered again, as a new version beside that answer (`content` isn't\nneeded; `model` may pick another). With `edit`, one of your earlier questions is edited: this question becomes a new\nversion beside it and is answered, and the old one stays with what followed it (404 if it isn't a question in this\nconversation). Either keeps the page the question was asked from (where, and the highlighted text; not the page's\ntext, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the\nnew version, the message it follows and its versions); so does a question beside others under `parent`.\n\nWhat the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a\nsearch) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's\nchoice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its\nrecord keeps what it was given and used.\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).",
+ "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nA conversation is a tree of versions (GET /chats/{cid}: `tree`, and each message's `versions`); the question goes\non from `parent` when given (any message of the conversation), else from the version showing (`head`). With\n`regenerate` (an answer's id) its question is answered again, as a new version beside that answer (`content` isn't\nneeded; `model` may pick another). With `edit`, one of your earlier questions is edited: this question becomes a new\nversion beside it and is answered, and the old one stays with what followed it (404 if it isn't a question in this\nconversation). Either keeps the page the question was asked from (where, and the highlighted text; not the page's\ntext, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the\nnew version, the message it follows and its versions); so does a question beside others under `parent`.\n\nWhat the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a\nsearch) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's\nchoice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its\nrecord keeps what it was given and used.\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `research` (or /research ) the content is a research goal: the answer is a plan to approve (streamed as\n`research`, then `approval`), which runs in the background once approved (POST /chats/{cid}/research, GET\n/research/{id}).\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).",
"operationId": "send_message",
"parameters": [
{
@@ -25917,6 +25917,425 @@
}
}
},
+ "/api/v1/chats/{cid}/research": {
+ "post": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Start Research",
+ "description": "Start research on a goal in a conversation: the goal is saved as a question and one model call plans it (up to\nthree follow-up questions with answers to pick, the sub-questions, where it looks, the tool calls it may make, the\nreport's outline, and an estimate of the cost). The plan waits as an approval (POST /approvals/{id}: approve it as\nit is, or with `params` its edits; or decline), or revise it (POST /research/{id}/revise). With `review` false it\nstarts at once: your own pick when signed in; an API key or an app needs research.agents_skip_review. The same as a\nquestion with `research: true`, or /research .",
+ "operationId": "start_research",
+ "parameters": [
+ {
+ "name": "cid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Cid"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ResearchStart"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ResearchStarted"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research": {
+ "get": {
+ "tags": [
+ "research"
+ ],
+ "summary": "List Research",
+ "description": "Your research, newest first; with `chat_id`, that conversation's (to show each plan's or run's card).",
+ "operationId": "list_research",
+ "parameters": [
+ {
+ "name": "chat_id",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Chat Id"
+ }
+ },
+ {
+ "name": "limit",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 50,
+ "title": "Limit"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/Research"
+ },
+ "title": "Response Research-List Research"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research/{rid}": {
+ "get": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Get Research",
+ "description": "A research run: its plan, status and stage (gather, write, check, done), progress (sub-questions done, sources\nfound, tool calls and cost so far, from the activity ledger) and, once it's done, the report's message\n(`answer`, in the conversation). What it waits for when a budget holds it (`held`), and why it ended early.",
+ "operationId": "get_research",
+ "parameters": [
+ {
+ "name": "rid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Rid"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Research"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research/{rid}/revise": {
+ "post": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Revise Research",
+ "description": "Send your answers to the follow-up questions and your edits to the plan back for one more planning call; the\nplan and its approval change in place. Only while the plan waits for approval.",
+ "operationId": "revise_research",
+ "parameters": [
+ {
+ "name": "rid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Rid"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ResearchRevise"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Research"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research/{rid}/stop": {
+ "post": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Stop Research",
+ "description": "Stop a run: it stops after the step it's on and keeps what it found, saved in the conversation as its answer,\nmarked stopped (one that hasn't started, or waits on a budget, stops now). Works whichever process runs it.",
+ "operationId": "stop_research",
+ "parameters": [
+ {
+ "name": "rid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Rid"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ResearchStop"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research/{rid}/resume": {
+ "post": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Resume Research",
+ "description": "Let a run that a budget holds go on anyway (your pick, as \"run it once\" is for a held routine run).",
+ "operationId": "resume_research",
+ "parameters": [
+ {
+ "name": "rid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Rid"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Research"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research/{rid}/note": {
+ "post": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Research Note",
+ "description": "Save the report as a note: in the namespace it looked in (`namespace` when there are several), its archive\ncitations as links to the assets they cite, web citations as links, and a link back to the conversation (shown to\nyou alone). Needs editor access there.",
+ "operationId": "research_note",
+ "parameters": [
+ {
+ "name": "rid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Rid"
+ }
+ }
+ ],
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ResearchNote"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Body"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ResearchNoteMade"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/research/{rid}/podcast": {
+ "post": {
+ "tags": [
+ "research"
+ ],
+ "summary": "Research Podcast",
+ "description": "Make a podcast episode from the assets the report cites (POST /podcasts with them picked): about the research\ngoal unless `prompt` says otherwise, titled after the report.",
+ "operationId": "research_podcast",
+ "parameters": [
+ {
+ "name": "rid",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "title": "Rid"
+ }
+ }
+ ],
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ResearchPodcast"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Body"
+ }
+ }
+ }
+ },
+ "responses": {
+ "202": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/PodcastStarted"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/v1/browser/sessions": {
"get": {
"tags": [
@@ -37099,6 +37518,22 @@
}
],
"description": "what produced it; none on messages from before it was recorded"
+ },
+ "kind": {
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "research",
+ "research_plan"
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Kind",
+ "description": "research: a deep research report; research_plan: the plan it waits on (GET /research/{meta.research})"
}
},
"additionalProperties": true,
@@ -50763,6 +51198,18 @@
}
],
"description": "what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read"
+ },
+ "research": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Research",
+ "description": "the content is a research goal: the reply is a plan to approve (streams `research`, then `approval`), as /research and POST /chats/{cid}/research do"
}
},
"additionalProperties": false,
@@ -50926,6 +51373,22 @@
],
"description": "what produced it; none on messages from before it was recorded"
},
+ "kind": {
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "research",
+ "research_plan"
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Kind",
+ "description": "research: a deep research report; research_plan: the plan it waits on (GET /research/{meta.research})"
+ },
"ledger": {
"items": {
"$ref": "#/components/schemas/ActivityEntry"
@@ -60890,6 +61353,724 @@
"type": "object",
"title": "ReprocessRequest"
},
+ "Research": {
+ "properties": {
+ "id": {
+ "type": "integer",
+ "title": "Id"
+ },
+ "chat": {
+ "type": "integer",
+ "title": "Chat"
+ },
+ "message": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Message",
+ "description": "the question that asked for it"
+ },
+ "plan_message": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Plan Message",
+ "description": "the answer that shows the plan"
+ },
+ "answer": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Answer",
+ "description": "the report, once saved in the conversation"
+ },
+ "approval": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Approval",
+ "description": "the plan's approval (POST /approvals/{id}: approve, with `params` its edits, or decline)"
+ },
+ "status": {
+ "type": "string",
+ "enum": [
+ "planned",
+ "declined",
+ "queued",
+ "running",
+ "held",
+ "done",
+ "stopped",
+ "failed"
+ ],
+ "title": "Status",
+ "description": "planned (waiting for approval), declined, queued (approved, waiting its turn), running, held (by a budget), done, stopped or failed"
+ },
+ "stage": {
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "gather",
+ "write",
+ "check",
+ "done"
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Stage"
+ },
+ "goal": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Goal"
+ },
+ "plan": {
+ "$ref": "#/components/schemas/ResearchPlan"
+ },
+ "questions": {
+ "items": {
+ "$ref": "#/components/schemas/ResearchQuestion"
+ },
+ "type": "array",
+ "title": "Questions",
+ "description": "follow-up questions; answer them with revise"
+ },
+ "answers": {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object",
+ "title": "Answers",
+ "description": "the answers given to them"
+ },
+ "scope": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Scope"
+ },
+ "estimate": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ResearchEstimate"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "progress": {
+ "$ref": "#/components/schemas/ResearchProgress"
+ },
+ "findings": {
+ "items": {
+ "$ref": "#/components/schemas/ResearchFinding"
+ },
+ "type": "array",
+ "title": "Findings",
+ "description": "the sub-questions answered so far"
+ },
+ "held": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ResearchHeld"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "error": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Error"
+ },
+ "ended": {
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "stopped",
+ "timeout",
+ "budget"
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Ended",
+ "description": "why it ended early: stopped, out of time (research.max_minutes), or a budget"
+ },
+ "revisions": {
+ "type": "integer",
+ "title": "Revisions",
+ "default": 0
+ },
+ "notes": {
+ "items": {
+ "type": "integer"
+ },
+ "type": "array",
+ "title": "Notes",
+ "description": "notes made from the report"
+ },
+ "created_at": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Created At"
+ },
+ "started_at": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Started At"
+ },
+ "finished_at": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Finished At"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "id",
+ "chat",
+ "status",
+ "plan",
+ "progress"
+ ],
+ "title": "Research"
+ },
+ "ResearchEstimate": {
+ "properties": {
+ "usd": {
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Usd",
+ "description": "what it will likely cost, from the prices in Settings; null without a price"
+ },
+ "tokens": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tokens"
+ },
+ "tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tool Calls"
+ },
+ "basis": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Basis",
+ "description": "what the estimate is drawn from"
+ },
+ "estimate": {
+ "type": "boolean",
+ "title": "Estimate",
+ "default": true
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "ResearchEstimate"
+ },
+ "ResearchFinding": {
+ "properties": {
+ "question": {
+ "type": "string",
+ "title": "Question"
+ },
+ "tool_calls": {
+ "type": "integer",
+ "title": "Tool Calls",
+ "default": 0
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "question"
+ ],
+ "title": "ResearchFinding"
+ },
+ "ResearchHeld": {
+ "properties": {
+ "why": {
+ "type": "string",
+ "title": "Why"
+ },
+ "budget": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Budget",
+ "description": "the budget it waits for (all, space:)"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "why"
+ ],
+ "title": "ResearchHeld"
+ },
+ "ResearchNote": {
+ "properties": {
+ "namespace": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Namespace",
+ "description": "where the note goes; needed when the research looked in several"
+ }
+ },
+ "additionalProperties": false,
+ "type": "object",
+ "title": "ResearchNote"
+ },
+ "ResearchNoteMade": {
+ "properties": {
+ "page": {
+ "type": "integer",
+ "title": "Page"
+ },
+ "namespace": {
+ "type": "string",
+ "title": "Namespace"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "page",
+ "namespace"
+ ],
+ "title": "ResearchNoteMade"
+ },
+ "ResearchPlan": {
+ "properties": {
+ "title": {
+ "type": "string",
+ "title": "Title"
+ },
+ "goal": {
+ "type": "string",
+ "title": "Goal"
+ },
+ "sub_questions": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "title": "Sub Questions",
+ "description": "what it answers, one look-up at a time, in order"
+ },
+ "sources": {
+ "items": {
+ "type": "string",
+ "enum": [
+ "archive",
+ "graph",
+ "notes",
+ "web",
+ "code"
+ ]
+ },
+ "type": "array",
+ "title": "Sources",
+ "description": "where it looks: the archive, the graph, notes, the web, the code sandbox"
+ },
+ "max_tool_calls": {
+ "type": "integer",
+ "title": "Max Tool Calls",
+ "description": "the most tool calls the run makes in all (research.max_tool_calls at most)"
+ },
+ "report_outline": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "title": "Report Outline",
+ "description": "the report's sections"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "title",
+ "goal",
+ "sub_questions",
+ "sources",
+ "max_tool_calls"
+ ],
+ "title": "ResearchPlan"
+ },
+ "ResearchPodcast": {
+ "properties": {
+ "prompt": {
+ "anyOf": [
+ {
+ "type": "string",
+ "maxLength": 2000
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Prompt",
+ "description": "an angle; else the research goal"
+ },
+ "length": {
+ "type": "integer",
+ "maximum": 120.0,
+ "minimum": 1.0,
+ "title": "Length",
+ "description": "minutes (up to podcasts.max_minutes)",
+ "default": 10
+ },
+ "style": {
+ "type": "string",
+ "enum": [
+ "deep-dive",
+ "recap",
+ "debate",
+ "beginner"
+ ],
+ "title": "Style",
+ "default": "deep-dive"
+ },
+ "title": {
+ "anyOf": [
+ {
+ "type": "string",
+ "maxLength": 200
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Title",
+ "description": "else the report's title"
+ }
+ },
+ "additionalProperties": false,
+ "type": "object",
+ "title": "ResearchPodcast"
+ },
+ "ResearchProgress": {
+ "properties": {
+ "sub_questions": {
+ "type": "integer",
+ "title": "Sub Questions",
+ "default": 0
+ },
+ "done": {
+ "type": "integer",
+ "title": "Done",
+ "description": "sub-questions answered",
+ "default": 0
+ },
+ "sources": {
+ "type": "integer",
+ "title": "Sources",
+ "description": "sources found so far",
+ "default": 0
+ },
+ "tool_calls": {
+ "type": "integer",
+ "title": "Tool Calls",
+ "default": 0
+ },
+ "cost_usd": {
+ "type": "number",
+ "title": "Cost Usd",
+ "description": "spent so far (the activity ledger rows of research:)",
+ "default": 0.0
+ },
+ "tokens": {
+ "type": "integer",
+ "title": "Tokens",
+ "default": 0
+ },
+ "cost_estimated": {
+ "type": "boolean",
+ "title": "Cost Estimated",
+ "default": false
+ },
+ "minutes": {
+ "type": "number",
+ "title": "Minutes",
+ "description": "how long it has run",
+ "default": 0.0
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "title": "ResearchProgress"
+ },
+ "ResearchQuestion": {
+ "properties": {
+ "question": {
+ "type": "string",
+ "title": "Question"
+ },
+ "answers": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "title": "Answers",
+ "description": "suggested answers to pick from"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "question"
+ ],
+ "title": "ResearchQuestion"
+ },
+ "ResearchRevise": {
+ "properties": {
+ "answers": {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object",
+ "title": "Answers",
+ "description": "the follow-up questions' answers, by question"
+ },
+ "plan": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Plan",
+ "description": "the plan as edited: any of title, sub_questions, sources, max_tool_calls, report_outline"
+ }
+ },
+ "additionalProperties": false,
+ "type": "object",
+ "title": "ResearchRevise"
+ },
+ "ResearchStart": {
+ "properties": {
+ "goal": {
+ "type": "string",
+ "maxLength": 4000,
+ "minLength": 1,
+ "title": "Goal",
+ "description": "what to research"
+ },
+ "tools": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array",
+ "maxItems": 100
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools",
+ "description": "what it may use (GET /chats/tools); null: the conversation's choice"
+ },
+ "max_tool_calls": {
+ "anyOf": [
+ {
+ "type": "integer",
+ "minimum": 1.0
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Tool Calls",
+ "description": "the most tool calls the plan may make (research.max_tool_calls at most)"
+ },
+ "scope": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ChatScope"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "what it draws on, instead of the conversation's scope"
+ },
+ "review": {
+ "type": "boolean",
+ "title": "Review",
+ "description": "false: start the run at once, without the plan waiting for review (signed in; an API key or an app only with research.agents_skip_review)",
+ "default": true
+ }
+ },
+ "additionalProperties": false,
+ "type": "object",
+ "required": [
+ "goal"
+ ],
+ "title": "ResearchStart"
+ },
+ "ResearchStarted": {
+ "properties": {
+ "research": {
+ "$ref": "#/components/schemas/Research"
+ },
+ "question": {
+ "type": "integer",
+ "title": "Question",
+ "description": "the question saved in the conversation"
+ },
+ "message": {
+ "type": "integer",
+ "title": "Message",
+ "description": "the answer that shows the plan"
+ },
+ "text": {
+ "type": "string",
+ "title": "Text",
+ "description": "the plan as the conversation shows it"
+ },
+ "meta": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/MessageRecord"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "research",
+ "question",
+ "message",
+ "text"
+ ],
+ "title": "ResearchStarted"
+ },
+ "ResearchStop": {
+ "properties": {
+ "ok": {
+ "type": "boolean",
+ "title": "Ok",
+ "default": true
+ },
+ "stopping": {
+ "type": "boolean",
+ "title": "Stopping",
+ "description": "it stops after the step it's on (or now, when it hadn't started or waited on a budget)"
+ }
+ },
+ "additionalProperties": true,
+ "type": "object",
+ "required": [
+ "stopping"
+ ],
+ "title": "ResearchStop"
+ },
"ResetPasswordRequest": {
"properties": {
"token": {
diff --git a/threat-model.json b/threat-model.json
index 4b358c50f..daa9a4d44 100644
--- a/threat-model.json
+++ b/threat-model.json
@@ -637,6 +637,15 @@
"provenance": "documented",
"source": "docs/assistant.md \"Code sandbox\""
},
+ {
+ "entry_point": "deep research",
+ "parameter": "/research , research: true, POST /chats/{cid}/research, /research/{id} (revise, stop, resume, note, podcast), start_research (assistant and MCP), research_status (MCP): a goal, answers to follow-up questions, plan edits; research.* settings (admins)",
+ "attacker_controllable": "yes",
+ "component": "assistant",
+ "caller_must_enforce": "none — the plan waits for the person's own approval (P-RESEARCH); the run uses only its sources' read tools, as the person, within its tool calls, minutes and budgets; its web searches and page reads don't ask again, bounded by the plan and the ledger; one run per person at a time; the report's claims are checked against what they cite",
+ "provenance": "documented",
+ "source": "docs/assistant.md \"Deep research\""
+ },
{
"entry_point": "microsandbox program requests",
"parameter": "ask/NAME.want in /data/.lens/programs: a program's name",
@@ -1123,7 +1132,7 @@
"property": "P-INJECTION-EGRESS",
"violation_symptom": "info-leak",
"severity_tier": "correctness",
- "conditions": "Once an assistant answer has read text from outside Lens (a tool result other than Lens's own, the page it was asked from, a namespace assistant's memories, a message in a chat room with no bridge.users, or an earlier answer in the conversation that read any of these), tools that could carry it out of Lens (the person's own read web tools, a web search or page read neither the person nor a tool gave it, other servers' MCP tools, Python tools with the network) and an admin setup conversation's own changes wait for the person's approval. A chat room's answer keeps a web address linkable only when the person, the archive, a tool or Lens wrote it. A bypass is VALID-HARDENING (Q26). An automation's model and agent actions (ask_model, call_tool, ask_agent) run with nobody reading along: the activity and earlier steps reach the model marked as data, every change they would make (notes included) is an approval of the automation's person, and anything that could carry data out of Lens asks first whatever the run read (docs/automations.md \"The model and the agent\"). A saved variable a run set from outside text stays marked as data in later runs' prompts, and an action whose params read it counts as reading outside text (docs/automations.md \"Saved variables\").",
+ "conditions": "Once an assistant answer has read text from outside Lens (a tool result other than Lens's own, the page it was asked from, a namespace assistant's memories, a message in a chat room with no bridge.users, or an earlier answer in the conversation that read any of these), tools that could carry it out of Lens (the person's own read web tools, a web search or page read neither the person nor a tool gave it, other servers' MCP tools, Python tools with the network) and an admin setup conversation's own changes wait for the person's approval. A chat room's answer keeps a web address linkable only when the person, the archive, a tool or Lens wrote it. A bypass is VALID-HARDENING (Q26). An automation's model and agent actions (ask_model, call_tool, ask_agent) run with nobody reading along: the activity and earlier steps reach the model marked as data, every change they would make (notes included) is an approval of the automation's person, and anything that could carry data out of Lens asks first whatever the run read (docs/automations.md \"The model and the agent\"). A saved variable a run set from outside text stays marked as data in later runs' prompts, and an action whose params read it counts as reading outside text (docs/automations.md \"Saved variables\"). Within a research run the person approved, the plan's web searches and page reads, and code with the network when the plan names the code sandbox, don't ask again (P-RESEARCH).",
"provenance": "documented",
"source": "docs/assistant.md \"it read out of Lens waits for your approval\""
},
@@ -1327,6 +1336,14 @@
"provenance": "documented",
"source": "docs/assistant.md \"Code sandbox\""
},
+ {
+ "property": "P-RESEARCH",
+ "violation_symptom": "integrity-bypass",
+ "severity_tier": "correctness",
+ "conditions": "a research plan runs only once its person approves it, signed in or with a yes in their chat room from someone in bridge.users; an API key, an app or an agent only declines it unless research.agents_skip_review; the run acts as the person with their access at run time, offered only its sources' read tools, within the plan's tool calls, research.max_minutes and the budgets of its namespaces and all, checked before each stage; its web searches and page reads (and code with the network, when the plan names the sandbox) don't ask again; one run per person at a time; one process at a time; every call in the ledger under research:",
+ "provenance": "documented",
+ "source": "docs/assistant.md \"Deep research\""
+ },
{
"property": "MX-AUTHZ-LIST-CAPS",
"violation_symptom": "x-cap-exceeded",
diff --git a/threat-model.md b/threat-model.md
index 1dbf7142a..80bc7a39a 100644
--- a/threat-model.md
+++ b/threat-model.md
@@ -312,6 +312,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34.
| MCP `edit_entity`, `edit_topic`, `merge_topics`, `delete_topic`, `undo_topic_change`, `undo_graph_change`; assistant `propose_topic_change` | entity and topic ids, names, labels, definitions | yes, an agent with an editor's write-scope token; indirectly through content the model reads | data, x-prompt-injection | the curate permission in each namespace touched and a write-capable principal (the same route functions as the web app); the assistant's changes wait for the person's approval; merges and deletes are kept to undo, and audited | *(documented, docs/mcp.md "curate entities and topics as an editor does")* |
| `/api/v1/automations` (save, try, run now, decide, draft from a sentence), the MCP and assistant automation tools; the activities an automation runs on | triggers, `where` and If match trees, the event workflow graph (action params are Jinja templates), `policy`, `limits`, an activity id; the text of the activities themselves | yes: any editor, for automations in namespaces they edit; indirectly through activity text (mail, titles, chat) that reaches action params | callback/code (declarative), data, x-prompt-injection | none; it runs as its maker with their permissions at run time (P-AUTOMATION-ACTOR), actions reaching outside Lens ask first, outside text marks intents untrusted and breaks up links (P-INJECTION-EGRESS), and chains and bursts are bounded (P-AUTOMATION-LOOP) | *(documented, docs/automations.md "Running automations")* |
| Assistant code sandbox tools (`run_python`, `run_bash`, `sandbox_files`, `import_sandbox_file`) and the `sandbox.*` settings | code and scripts the model writes, attachment ids, a path in the sandbox; the gateway address and image (admins) | yes: the people `sandbox.who` names, and indirectly through content the model reads | callback/code, data, x-prompt-injection | none; off by default; the code runs in the conversation's own OpenShell sandbox, which holds none of Lens's data or keys and reaches no network unless `sandbox.network` allows it (P-SANDBOX); with the network on, a run after outside text asks first (P-INJECTION-EGRESS); files go in only from the person's own attachments to that conversation, and come out only as an import the person approves | *(documented, docs/assistant.md "Code sandbox")* |
+| Deep research: `/research `, `research: true` on a question, `POST /chats/{cid}/research`, `/research/{id}` (revise, stop, resume, note, podcast), the assistant's and MCP's `start_research`, MCP's `research_status`, and the `research.*` settings | a goal, answers to follow-up questions, plan edits (sub-questions, sources, tool calls, outline); settings (admins) | yes: anyone who may use the assistant, in their own conversations; indirectly through what the run reads, through the model | data, x-prompt-injection, size | none; the plan waits for the person's own approval (P-RESEARCH); the run uses only the read tools of its plan's sources, as the person with their current access, within its tool calls, minutes and budgets; its web searches and page reads don't ask again, bounded by the plan and kept in the ledger; one run per person at a time; the report's claims are checked against what they cite and marked when unsupported | *(documented, docs/assistant.md "Deep research")* |
| `ask/NAME.want` in a microsandbox's `/data/.lens/programs` (LibreOffice and Chromium on first use) | a program's name | writable by the Lens user inside the sandbox | x-env-config | the start script, as root, installs only `libreoffice` or `chromium`, each a fixed list of signed Debian packages, and answers in `got/`, which only root writes; it reads nothing from the file but its name, and only deletes any other name or link | *(documented, docs/microsandbox.md "LibreOffice and Chromium")* |
**Stateful postconditions.** A refused or failed one-time secret check consumes nothing; a used one is deleted before
@@ -529,7 +530,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c
| P-RETRIEVAL | Chat, assistant tools and MCP return only what the person can read, within the conversation's scope, and the tool loop stops at `ai.max_steps` and `ai.max_transcript_reads`. A namespace's chat completions answer from that namespace alone, with only the read tools its owners offered. | info-leak (scope); hang (caps) | security-critical (scope); correctness-only (caps) | *(documented, docs/assistant.md "limited to what the person can read")* | Voided by: Admins read everything; an admin setup conversation adds server tools and the `act` flag — `domain/ai_tools.py:287-289`; `ai.extensions: false` turns extensions off. |
| P-APPROVAL | Assistant tools that change data wait for the person's approval, except notes and an admin's setup conversation. Inviting people and changing their access always wait, even there, and the assistant cannot remove an account. | integrity-bypass | security-critical | *(documented, docs/assistant.md "Approvals")* | Voided by: Admins read everything; an admin setup conversation adds server tools and the `act` flag — `domain/ai_tools.py:287-289`; `ai.extensions: false` turns extensions off. |
| P-AUTOMATION-APPROVAL | An automation's action that asks first (effect `out`, or `lens` beyond the event's own object) waits as an approval of the automation's person, and nothing runs until they pick. Approving, editing or allowing a node from now on (`always`) needs them signed in to Lens, or (approving an action that stays inside Lens, effect `lens`) a yes in the private chat Lens told them in, from someone on that channel's `users` list with approving on, while it is the only thing waiting there or named by its number; an action that reaches outside Lens is approved on its card. API keys, apps, MCP and the assistant can only list and decline. A decision is claimed once (two at once can't both carry it out). At most 100 wait per automation and each tells the person once, so a flood of activities can't bury the inbox or their chat; undecided ones expire after `automations.approval_days` and nothing runs. | integrity-bypass | security-critical | *(documented, docs/automations.md "Approvals")* | Voided by: the person setting a node to `auto` (`always`, or the builder's toggle) — that node then runs without asking; an admin raising `automations.approval_days`. The summary and the message the person is told hold text the run read from outside Lens: links in it are broken up (P-INJECTION-EGRESS), and what runs is the intent the card shows. |
-| P-INJECTION-EGRESS | Once an assistant answer has read text from outside Lens (a tool result other than Lens's own, the page it was asked from, a namespace assistant's memories, a message in a chat room with no `bridge.users`, or an earlier answer in the conversation that read any of these), tools that could carry it out of Lens (the person's own `read` web tools, a web search or page read neither the person nor a tool gave it, other servers' MCP tools, Python tools with the network, code in the code sandbox with its network on) and an admin setup conversation's own changes wait for the person's approval. A chat room's answer keeps a web address linkable only when the person, the archive, a tool or Lens wrote it. A bypass is `VALID-HARDENING` (Q26). An automation's model and agent actions (`ask_model`, `call_tool`, `ask_agent`) run with nobody reading along: the activity and earlier steps reach the model marked as data, every change they would make (notes included) is an approval of the automation's person, and anything that could carry data out of Lens asks first whatever the run read (docs/automations.md "The model and the agent"). A saved variable a run set from outside text stays marked as data in later runs' prompts, and an action whose params read it counts as reading outside text (docs/automations.md "Saved variables"). | info-leak | correctness-only | *(documented, docs/assistant.md "it read out of Lens waits for your approval")* | Voided by: `ai.untrusted_asks: false` — `domain/ai_tools.py:519`; `bridge.safe_links: false` — `domain/bridge.py:298`. Notes and memories are written at once: they stay in Lens. Hooks run as their owners set them (P-EXT-SCOPE). An address the read text wrote itself stays linkable in a room. The person's browser asks first as P-BROWSER-GUARD says. |
+| P-INJECTION-EGRESS | Once an assistant answer has read text from outside Lens (a tool result other than Lens's own, the page it was asked from, a namespace assistant's memories, a message in a chat room with no `bridge.users`, or an earlier answer in the conversation that read any of these), tools that could carry it out of Lens (the person's own `read` web tools, a web search or page read neither the person nor a tool gave it, other servers' MCP tools, Python tools with the network, code in the code sandbox with its network on) and an admin setup conversation's own changes wait for the person's approval. A chat room's answer keeps a web address linkable only when the person, the archive, a tool or Lens wrote it. A bypass is `VALID-HARDENING` (Q26). An automation's model and agent actions (`ask_model`, `call_tool`, `ask_agent`) run with nobody reading along: the activity and earlier steps reach the model marked as data, every change they would make (notes included) is an approval of the automation's person, and anything that could carry data out of Lens asks first whatever the run read (docs/automations.md "The model and the agent"). A saved variable a run set from outside text stays marked as data in later runs' prompts, and an action whose params read it counts as reading outside text (docs/automations.md "Saved variables"). Within a research run the person approved, the plan's web searches and page reads, and code runs with the network when the plan names the code sandbox, don't ask again: approving the plan approved them (P-RESEARCH). | info-leak | correctness-only | *(documented, docs/assistant.md "it read out of Lens waits for your approval")* | Voided by: `ai.untrusted_asks: false` — `domain/ai_tools.py:519`; `bridge.safe_links: false` — `domain/bridge.py:298`. Notes and memories are written at once: they stay in Lens. Hooks run as their owners set them (P-EXT-SCOPE). An address the read text wrote itself stays linkable in a room. The person's browser asks first as P-BROWSER-GUARD says. |
| P-EXT-SCOPE | Your own hooks run in your conversations; another person's hooks run only when an admin shared them. | integrity-bypass | security-critical | *(documented, docs/assistant.md "other people's run only when an admin shared them")* | Voided by: An admin sharing the extension — `domain/extensions.py:595-598`. |
| P-NO-CODE | No non-admin can run code on the host. This covers owners, editors, viewers, API tokens, MCP agents, OAuth apps and model tool calls acting for one. Python tools are written by admins only and run only while an admin owns them. | integrity-bypass | security-critical | *(maintainer, 2026-10)* | Voided by: Making someone admin; authoring gate — `domain/extensions.py:129-130`; run gate — `domain/extensions.py:600-601`. `grep -rnE "eval\(\|exec\(\|pickle\|yaml\.load\(\|__import__" fastapi_backend/app` — one `exec`, inside the code-tool child. |
| P-UPLOAD | Uploads accept only the types in `uploads.extensions` and are size-capped. Nothing uploaded is executed. Files are served as audio, video or a download, and names carry no folders. | bad-data-accepted | security-critical | *(documented, docs/configuration.md "Uploads accept only the types")* | Voided by: Admin-added types; check — `domain/uploads.py:157-158`. The `attachments` role takes any type (never executed). Bytes are not checked against the extension. |
@@ -555,6 +556,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c
| P-BROWSER | The assistant uses a person's browser only from that person's own conversations and API calls, while their own Lens side panel is connected (seen in the last 20 seconds, not closed). Only that person sees the session's actions and answers them, once each. `ai.browser` off refuses new sessions and offers no browser tools. | integrity-bypass | security-critical | *(documented, docs/browser-assistant.md "Only you")* | Voided by: The side panel carries out what it's asked: which sites it acts on, and what it asks the person first, is the extension's (docs/browser-assistant.md); an agent holding the person's write token can ask too — `api/v1/routes/browser.py`. |
| P-BROWSER-GUARD | The Lens extension's side panel reads and acts only on sites the person allowed in it (and the browser granted), and opens an address only on such a site. It asks the person before a click, submit or Enter that the model marks consequential or whose control's words or posting form look so, and, once the answer has read text from outside Lens (`ai.untrusted_asks`), before opening or downloading an address or typing text the person's own message doesn't contain. It never types into password, card or file fields, never reports their values, and shows each action as it happens. | integrity-bypass | security-critical | *(documented, docs/browser-assistant.md "Letting it use your browser")* | Voided by: The consequential check is the model's flag plus a word list (`extensions/browser/src/core/act.js`): a control with no telling words that a misled model doesn't mark runs without asking; synthetic events reach only the page's own handlers. Removing a site from the options stops the side panel acting there, but leaves the browser's access to it for the clipper. Lens sends the ask (`confirm`) and the extension honours it: an extension older than this doesn't ask. |
| P-SANDBOX | Code the assistant runs (`run_python`, `run_bash`; `sandbox.enabled`, off by default) runs only for people `sandbox.who` allows (editors and admins, or admins; never viewers), in a sandbox of that conversation's own on the OpenShell gateway, never in Lens's processes. The sandbox gets none of Lens's data, keys or database address: only the code and the files a run copies in, which must be attached to that conversation by that person (25 MB each, 100 MB a run). Its policy allows no egress with `sandbox.network: off`, only pypi.org, files.pythonhosted.org and registry.npmjs.org on 443 with `packages`, and only public addresses on 80 and 443 with `web` (no private, loopback, link-local or reserved range). A file leaves it only as one of the person's held uploads (25 MB) placed in a namespace by an `import_files` approval. A sandbox stops after `sandbox.idle_minutes` idle and is deleted after `sandbox.keep_hours` unused or with its conversation. | integrity-bypass | security-critical | *(documented, docs/assistant.md "Code sandbox")* | Voided by: isolation inside the sandbox is OpenShell's (§1.9); `landlock.compatibility: best_effort` runs without Landlock on a kernel that lacks it — `domain/sandbox.py`; whoever reaches the gateway's address can run sandboxes of their own; what the code prints reaches the model as third-party text (N-PROMPT). |
+| P-RESEARCH | A research plan (`domain/research.py`) runs only once the person who asked for it approves it: signed in, or with a "yes" in their chat room from someone in `bridge.users`; an API key, an app or an agent (MCP, the assistant) can only decline it, unless an admin turns on `research.agents_skip_review`. The run acts as that person with their access at run time and stops if their account is disabled. It is offered only the read tools of the sources its plan names (never a tool that changes anything), makes at most the plan's tool calls (`research.max_tool_calls`, 40 by default), stops looking things up after `research.max_minutes` and writes up what it found, and checks the budgets of its namespaces and `all` before each stage, waiting or stopping as they say. Its web searches and page reads, and code runs with the network when the plan names the code sandbox, don't ask again after outside text, since the plan approved them. A person has one run going at a time. One process carries a run out at a time (a claim on `research_run:` with a heartbeat every 10 seconds, taken over only after two minutes of silence). Every call it makes is in the activity ledger for the conversation and `research:`. | integrity-bypass | correctness-only | *(documented, docs/assistant.md "Deep research")* | Voided by: `research.agents_skip_review: true` lets an agent's plan run without review — `domain/research.py` `agents_may_approve`; a run's web queries can carry archive text it read, within its plan's tool calls (the misled model of N-PROMPT); a "yes" in a room is the room's people's (N-BRIDGE). |
### Worked routing examples
diff --git a/threat-model.yaml b/threat-model.yaml
index 69a9a1a19..71ed13f37 100644
--- a/threat-model.yaml
+++ b/threat-model.yaml
@@ -2,7 +2,7 @@
# Authority order: prose > yaml > json. Regenerate whenever the prose changes.
schema: threat-model-sidecar/v2
project: lens
-prose_version: "threat-model.md@sha256:44475d4514f343e65a581b45ce853b957961abe7f49de464f7da9972569857cb"
+prose_version: "threat-model.md@sha256:f2b4194a60682fd237dec7f3030e4d4e366a51294ab37f0c5ff74029f7887bf2"
model_status: under-review
triage_policy: strict
confidence: {documented: 204, maintainer: 46, inferred: 91, assumption: 8}
@@ -578,6 +578,15 @@ entry_points:
control_kinds: [callback/code, data, x-prompt-injection]
caller_must_enforce: "none — off by default; the code runs in the conversation's own OpenShell sandbox with none of Lens's data or keys and no network unless sandbox.network allows it (P-SANDBOX); with the network on, a run after outside text asks first (P-INJECTION-EGRESS); files in only from the person's attachments, out only through an approved import"
provenance: {kind: documented, source: 'docs/assistant.md "Code sandbox"'}
+ - id: "deep research"
+ component: assistant
+ parameters:
+ - name: "/research , research: true, POST /chats/{cid}/research, /research/{id} (revise, stop, resume, note, podcast), start_research (assistant and MCP), research_status (MCP): a goal, answers to follow-up questions, plan edits; research.* settings (admins)"
+ attacker_controllable: true
+ x-condition: "anyone who may use the assistant, in their own conversations; indirectly through what the run reads, through the model"
+ control_kinds: [data, x-prompt-injection, size]
+ caller_must_enforce: "none — the plan waits for the person's own approval (P-RESEARCH); the run uses only its sources' read tools, as the person, within its tool calls, minutes and budgets; its web searches and page reads don't ask again, bounded by the plan and the ledger; one run per person at a time; the report's claims are checked against what they cite"
+ provenance: {kind: documented, source: 'docs/assistant.md "Deep research"'}
- id: "microsandbox program requests"
component: authz
parameters:
@@ -1551,7 +1560,7 @@ properties_claimed:
kind: confidentiality
components: [assistant]
tier: correctness-only
- conditions: "Once an assistant answer has read text from outside Lens (a tool result other than Lens's own, the page it was asked from, a namespace assistant's memories, a message in a chat room with no bridge.users, or an earlier answer in the conversation that read any of these), tools that could carry it out of Lens (the person's own read web tools, a web search or page read neither the person nor a tool gave it, other servers' MCP tools, Python tools with the network) and an admin setup conversation's own changes wait for the person's approval. A chat room's answer keeps a web address linkable only when the person, the archive, a tool or Lens wrote it. A bypass is VALID-HARDENING (Q26). An automation's model and agent actions (ask_model, call_tool, ask_agent) run with nobody reading along: the activity and earlier steps reach the model marked as data, every change they would make (notes included) is an approval of the automation's person, and anything that could carry data out of Lens asks first whatever the run read (docs/automations.md \"The model and the agent\"). A saved variable a run set from outside text stays marked as data in later runs' prompts, and an action whose params read it counts as reading outside text (docs/automations.md \"Saved variables\")."
+ conditions: "Once an assistant answer has read text from outside Lens (a tool result other than Lens's own, the page it was asked from, a namespace assistant's memories, a message in a chat room with no bridge.users, or an earlier answer in the conversation that read any of these), tools that could carry it out of Lens (the person's own read web tools, a web search or page read neither the person nor a tool gave it, other servers' MCP tools, Python tools with the network) and an admin setup conversation's own changes wait for the person's approval. A chat room's answer keeps a web address linkable only when the person, the archive, a tool or Lens wrote it. A bypass is VALID-HARDENING (Q26). An automation's model and agent actions (ask_model, call_tool, ask_agent) run with nobody reading along: the activity and earlier steps reach the model marked as data, every change they would make (notes included) is an approval of the automation's person, and anything that could carry data out of Lens asks first whatever the run read (docs/automations.md \"The model and the agent\"). A saved variable a run set from outside text stays marked as data in later runs' prompts, and an action whose params read it counts as reading outside text (docs/automations.md \"Saved variables\"). Within a research run the person approved, the plan's web searches and page reads, and code with the network when the plan names the code sandbox, don't ask again (P-RESEARCH)."
violation_symptoms: [info-leak]
x-voided-by: "ai.untrusted_asks: false (domain/ai_tools.py:519); bridge.safe_links: false (domain/bridge.py:298); notes and memories are written at once; hooks run as their owners set them (P-EXT-SCOPE); an address the read text wrote itself stays linkable in a room; the person's browser asks first as P-BROWSER-GUARD says"
provenance: {kind: documented, source: 'docs/assistant.md "it read out of Lens waits for your approval"'}
@@ -1758,6 +1767,14 @@ properties_claimed:
violation_symptoms: [integrity-bypass]
x-voided-by: "isolation inside the sandbox is OpenShell's; landlock best_effort runs without Landlock where the kernel lacks it (domain/sandbox.py); whoever reaches the gateway can run sandboxes of their own; what the code prints is third-party text (N-PROMPT)"
provenance: {kind: documented, source: 'docs/assistant.md "Code sandbox"'}
+ - id: P-RESEARCH
+ kind: integrity
+ components: [assistant]
+ tier: correctness-only
+ conditions: "a research plan runs only once its person approves it, signed in or with a yes in their chat room from someone in bridge.users; an API key, an app or an agent only declines it unless research.agents_skip_review; the run acts as the person with their access at run time, offered only its sources' read tools, within the plan's tool calls (research.max_tool_calls), research.max_minutes and the budgets of its namespaces and all, checked before each stage; its web searches and page reads (and code with the network, when the plan names the sandbox) don't ask again; one run per person at a time; one process at a time (research_run:, a heartbeat every 10 s, taken over after 2 minutes of silence); every call in the ledger under research:"
+ violation_symptoms: [integrity-bypass]
+ x-voided-by: "research.agents_skip_review lets an agent's plan run without review (domain/research.py agents_may_approve); a run's web queries can carry archive text it read, within its tool calls (N-PROMPT); a yes in a room is the room's people's (N-BRIDGE)"
+ provenance: {kind: documented, source: 'docs/assistant.md "Deep research"'}
- id: MX-AUTHZ-LIST-CAPS
kind: resource-bound
components: [authz]
From 95e897d3c464df804d7cf0a9a45ebbc5645fd893 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Sat, 17 Oct 2026 06:27:35 +0000
Subject: [PATCH 14/14] feat(chat): deep research in the web app, a plan card
to approve and a live progress card
The composer's Deep research switch (and /research) sends a goal; its answer is a plan card: the follow-up questions with answers to tap, the sub-questions to reword, add or remove, where it looks, a tool-call stepper with the estimate, and Enter approves it (or revises it once a question is answered), with Decline. Approved, it becomes a progress card read from GET /research/{id}, so it survives a reload: stage, sub-questions ticked off, sources, tool calls, cost and minutes, Go on anyway when a budget holds it, and Stop. The report saves as a note (picking the namespace when there are several), reads aloud, makes a podcast or exports as Markdown. Notes open chat: links, and Settings > AI assistant > Deep research holds the research limits.
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B
---
CHANGELOG.md | 6 +-
docs/assistant.md | 23 +-
.../__tests__/chat-research.test.tsx | 321 ++++++++
.../__tests__/settings-web-search.test.tsx | 1 +
nextjs-frontend/components/chat/answer.tsx | 3 +
nextjs-frontend/components/chat/chat-app.tsx | 121 ++-
nextjs-frontend/components/chat/composer.tsx | 44 +-
nextjs-frontend/components/chat/research.tsx | 763 ++++++++++++++++++
nextjs-frontend/components/chat/stream.ts | 10 +-
.../components/notes/block-editor.tsx | 2 +-
nextjs-frontend/components/notes/links.ts | 5 +-
.../components/settings/bodies.tsx | 16 +
nextjs-frontend/components/settings/model.ts | 40 +-
13 files changed, 1301 insertions(+), 54 deletions(-)
create mode 100644 nextjs-frontend/__tests__/chat-research.test.tsx
create mode 100644 nextjs-frontend/components/chat/research.tsx
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c00d9c1bd..ade6bbf60 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,8 +11,10 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
it, or say "yes" in a chat room; an API key or an app only declines it unless `research.agents_skip_review` is on.
It runs in the background, one run per person at a time, within its tool calls, `research.max_minutes` and your
budgets, survives a restart, and stops when asked; the cited report comes back in the conversation, its claims
- checked against their sources, and saves as a note or makes a podcast episode. The web app's plan and progress
- cards are in progress ([Deep research](docs/assistant.md#deep-research)).
+ checked against their sources, and saves as a note or makes a podcast episode. In the web app, the composer's Deep
+ research switch asks for it; the plan is a card to edit and approve with Enter, then a progress card that survives a
+ reload; and the report saves as a note, reads aloud, makes a podcast or exports as Markdown. Settings → AI assistant
+ → Deep research holds its limits ([Deep research](docs/assistant.md#deep-research)).
- **Choose the tools, a limit and the namespaces of an answer.** A conversation, or one question, can pick the tool
groups or tools its answers may use (`tools`), cap their tool calls (`max_tool_calls`, up to the new Settings → AI
assistant → Most tool calls a question may ask for, 50 by default) and narrow one answer to its own namespaces
diff --git a/docs/assistant.md b/docs/assistant.md
index 775b3fd93..b0f50f1f5 100644
--- a/docs/assistant.md
+++ b/docs/assistant.md
@@ -376,11 +376,12 @@ assistant); one real run against OpenShell in CI (the tests use a fake gateway w
## Deep research
-Built in the API, MCP, chat rooms and the assistant's tools; the web app's plan and progress cards are in progress.
+Built: the web app, the API, MCP, chat rooms and the assistant's tools.
Give the assistant a goal that needs many look-ups, and it comes back with a plan; once you approve it, the research
runs in the background and a cited report comes back in the conversation.
-- **Asking.** Start a question with `/research` (`/research what changed in the capsid work this year?`), send it with
+- **Asking.** Turn on the composer's Deep research switch (the telescope) and type the goal, start a question with
+ `/research` (`/research what changed in the capsid work this year?`), send it with
`research: true` (`POST /api/v1/chats//messages`), call `POST /api/v1/chats//research {goal}`, or ask the
assistant for research: its `start_research` tool (the `research` [tool group](#choosing-tools-namespaces-and-a-limit))
works from chat, voice and [chat rooms](chat-rooms.md). It needs a language model that uses tools.
@@ -391,6 +392,11 @@ runs in the background and a cited report comes back in the conversation.
labelled an estimate: from what each tool call of past research runs cost in the activity ledger, else its likely
tokens at the model's price in Settings (no figure when the model has no price). The plan is the answer to the
question, and it waits as an approval of the assistant's (tool `research`).
+- **The plan card.** In the conversation the plan is a card: the follow-up questions with their answers to tap (or
+ type your own), the sub-questions as a list to reword, add to or remove from, where it looks as ticks (only the
+ sources you have), a stepper for its tool calls with the estimate beside it, and the report's outline. Enter
+ approves it, with your edits; once you've answered a question, Enter revises it with your answers instead (Approve
+ as it is stays one click away). Decline drops it.
- **Approve, edit or revise.** Approve it as it is, or with your edits (`POST /api/v1/approvals/` with
`decision: approve` and `params`: any of `title`, `sub_questions`, `sources`, `max_tool_calls`, `report_outline`), in
the conversation or the Approvals page; or send your answers and edits back for one more planning call (`POST
@@ -399,8 +405,8 @@ runs in the background and a cited report comes back in the conversation.
it at once, as your own pick.
- **Agents only decline.** An API key, an app or an agent (MCP, the assistant) can decline a plan but not approve it,
as for every approval, unless an admin turns on `research.agents_skip_review` (off by default): then an agent's plan
- starts without review. The `research` settings are saved with `PUT /api/v1/settings/research`; their place in
- Settings → AI assistant is in progress.
+ starts without review. The `research` settings are in Settings → AI assistant → Deep research, and
+ `PUT /api/v1/settings/research`.
- **The run.** Once approved, one process runs it at a time (a claim it keeps with a heartbeat every 10 seconds;
another process takes it over after two minutes of silence and goes on from the last finished step), in stages:
- *gather*: each sub-question in turn, with the assistant's tool loop, its share of the tool calls that are left,
@@ -420,7 +426,10 @@ runs in the background and a cited report comes back in the conversation.
- **Budgets.** Before each stage the [budgets](budgets.md) of the conversation's namespaces and `all` are checked. One
that is over holds the run (`held`, saying why) until the budget allows it, or until you let it go on anyway (`POST
/api/v1/research//resume`); a budget that skips stops it with what it found.
-- **Progress and stopping.** `GET /api/v1/research/` gives its status (planned, declined, queued, running, held,
+- **Progress and stopping.** Once approved, the plan card becomes a progress card that reads the run again every few
+ seconds, so it carries on after a reload and on a phone: its stage, a bar of the sub-questions done with each one
+ ticked off, sources found, tool calls, cost and minutes, why a budget holds it with Go on anyway, and Stop. When the
+ report is saved, the conversation shows it. `GET /api/v1/research/` gives its status (planned, declined, queued, running, held,
done, stopped, failed), stage, and progress: sub-questions done, sources found, tool calls, minutes, and tokens and
cost so far. `GET /api/v1/research?chat_id=` lists a conversation's. `POST /api/v1/research//stop` stops it
after the step it's on (at once while it waits), and what it found so far is saved in the conversation as its
@@ -429,7 +438,9 @@ runs in the background and a cited report comes back in the conversation.
(`namespace` when there are several), its archive citations as mention links to what they cite, web ones as links,
and a link back to the conversation (`@[Research: title](chat:12)`), which only you see named. `POST
/api/v1/research//podcast` makes a [podcast episode](podcasts.md) from the assets the report cites, about its
- goal. The report reads aloud and exports like any answer.
+ goal. In the web app the report has these as Save as note (asking which namespace when it could go in several),
+ Read aloud (with the voice in Settings → AI assistant), Make a podcast, and Export (a Markdown file with the sources
+ it cites). Follow-up questions go on from the report.
- **Agents.** MCP has `start_research` (a new conversation unless `chat_id` is given; `namespaces`, `tools`,
`max_tool_calls`) and `research_status` (the plan and progress, and once it's done the report with its sources and
their links).
diff --git a/nextjs-frontend/__tests__/chat-research.test.tsx b/nextjs-frontend/__tests__/chat-research.test.tsx
new file mode 100644
index 000000000..ce5cf6cfd
--- /dev/null
+++ b/nextjs-frontend/__tests__/chat-research.test.tsx
@@ -0,0 +1,321 @@
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import "@testing-library/jest-dom";
+
+import { Chats, Research2 } from "@/app/openapi-client";
+import type { Research } from "@/app/openapi-client/types.gen";
+import { Composer } from "@/components/chat/composer";
+import {
+ estimateLine,
+ newAnswers,
+ planEdits,
+ ReportActions,
+ reportMarkdown,
+ ResearchCard,
+ usd,
+} from "@/components/chat/research";
+import { applyEvent, newTurn } from "@/components/chat/stream";
+import { hrefFor } from "@/components/notes/links";
+import { TooltipProvider } from "@/components/ui/tooltip";
+
+jest.mock("@/app/openapi-client", () => ({
+ Chats: { decideApproval: jest.fn(), chatTools: jest.fn() },
+ Research2: {
+ getResearch: jest.fn(),
+ reviseResearch: jest.fn(),
+ stopResearch: jest.fn(),
+ resumeResearch: jest.fn(),
+ researchNote: jest.fn(),
+ researchPodcast: jest.fn(),
+ },
+}));
+jest.mock("@/lib/api/browser", () => ({ useApiClient: () => ({}), data: (p: Promise) => p }));
+jest.mock("next/navigation", () => ({ useRouter: () => ({ push: jest.fn() }) }));
+jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) }));
+jest.mock("@/lib/hooks/session", () => ({ useArchive: () => ({}) }));
+jest.mock("@/components/import/pending", () => ({ chooseFiles: jest.fn(() => Promise.resolve([])) }));
+const speak = jest.fn(() => Promise.resolve());
+jest.mock("@/lib/voice", () => ({ useVoice: () => ({ speak, stop: jest.fn() }) }));
+
+const PLAN: Research = {
+ id: 4,
+ chat: 9,
+ approval: 31,
+ status: "planned",
+ stage: null,
+ goal: "What did customers say about pricing?",
+ plan: {
+ title: "Pricing feedback",
+ goal: "What did customers say about pricing?",
+ sub_questions: ["Which calls mention pricing?", "What do they object to?"],
+ sources: ["archive"],
+ max_tool_calls: 8,
+ report_outline: ["Summary", "Objections"],
+ },
+ questions: [{ question: "Which period?", answers: ["This quarter", "All time"] }],
+ answers: {},
+ estimate: { usd: 0.02, tokens: 30000, tool_calls: 8, basis: "about 30,000 tokens", estimate: true },
+ progress: { sub_questions: 2, done: 0, sources: 0, tool_calls: 0, cost_usd: 0.001, minutes: 0 },
+ findings: [],
+ revisions: 0,
+};
+
+// Radix's checkbox measures itself
+globalThis.ResizeObserver ??= class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+} as unknown as typeof ResizeObserver;
+
+const ok = (v: unknown) => Promise.resolve(v);
+let qc: QueryClient;
+
+function show(ui: React.ReactNode) {
+ qc = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ return render(
+
+ {ui}
+ ,
+ );
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ (Chats.chatTools as jest.Mock).mockImplementation(() =>
+ ok({
+ groups: [
+ { name: "archive", available: true },
+ { name: "web", available: true },
+ { name: "code", available: false },
+ ],
+ }),
+ );
+ (Chats.decideApproval as jest.Mock).mockImplementation(() => ok({ status: "done", result: { status: "queued" } }));
+ (Research2.getResearch as jest.Mock).mockImplementation(() => ok(PLAN));
+});
+afterEach(() => qc?.clear());
+
+describe("research helpers", () => {
+ it("reads the research event into the turn", () => {
+ const s = applyEvent(newTurn("/research pricing"), { event: "research", data: JSON.stringify(PLAN) });
+ expect(s.research).toMatchObject({ id: 4, status: "planned" });
+ expect(applyEvent(newTurn("x"), { event: "research", data: "{}" }).research).toBeNull();
+ });
+
+ it("formats small costs and scales the estimate with the tool calls, labelled", () => {
+ expect(usd(0.0042)).toBe("$0.0042");
+ expect(usd(2.5)).toBe("$2.50");
+ expect(usd(null)).toBeNull();
+ expect(estimateLine(PLAN, 16)).toBe("about $0.04 (an estimate)");
+ expect(estimateLine({ ...PLAN, estimate: { usd: null, tool_calls: 8 } }, 8)).toMatch(/no estimate/);
+ });
+
+ it("sends only what changed, and only answers not sent yet", () => {
+ const d = { subs: [...PLAN.plan.sub_questions], sources: PLAN.plan.sources, calls: 8, answers: {} };
+ expect(planEdits(PLAN, d)).toBeNull();
+ expect(planEdits(PLAN, { ...d, subs: ["Which calls mention pricing?", " "], calls: 12 })).toEqual({
+ sub_questions: ["Which calls mention pricing?"],
+ max_tool_calls: 12,
+ });
+ expect(
+ newAnswers(
+ { ...PLAN, answers: { "Which period?": "All time" } },
+ { ...d, answers: { "Which period?": "All time", x: " " } },
+ ),
+ ).toEqual({});
+ expect(newAnswers(PLAN, { ...d, answers: { "Which period?": "This quarter" } })).toEqual({
+ "Which period?": "This quarter",
+ });
+ });
+
+ it("exports the report with the sources it cites", () => {
+ const md = reportMarkdown("Pricing", "Too dear [1]. See the post [3].", [
+ { n: 1, text: "", title: "Renewal call", time: "4:05", speaker: "Ada", recording_id: 2 },
+ { n: 2, text: "", title: "Unused" },
+ { n: 3, text: "", title: "Blog", url: "https://example.com/p", source: "web" },
+ ]);
+ expect(md).toBe(
+ "# Pricing\n\nToo dear [1]. See the post [3].\n\n## Sources\n\n- [1] Renewal call, at 4:05, Ada\n- [3] Blog \n",
+ );
+ });
+
+ it("opens a note's link back to its conversation", () => {
+ expect(hrefFor("chat:12")).toBe("/chat/12");
+ });
+});
+
+describe("the plan card", () => {
+ it("approves the plan with Enter, sending the edits", async () => {
+ show( );
+ const first = await screen.findByRole("textbox", { name: "Sub-question 1" });
+ expect(screen.getByText("Pricing feedback")).toBeInTheDocument();
+ expect(screen.getByText("about $0.02 (an estimate)")).toBeInTheDocument();
+ expect(await screen.findByRole("checkbox", { name: "The web" })).toBeInTheDocument();
+ expect(screen.queryByRole("checkbox", { name: "The code sandbox" })).not.toBeInTheDocument();
+ fireEvent.change(first, { target: { value: "Which calls mention price?" } });
+ fireEvent.click(screen.getByRole("button", { name: "More tool calls" }));
+ fireEvent.submit(first.closest("form")!);
+ await waitFor(() => expect(Chats.decideApproval).toHaveBeenCalled());
+ expect((Chats.decideApproval as jest.Mock).mock.calls[0][0]).toMatchObject({
+ path: { aid: 31 },
+ body: {
+ decision: "approve",
+ params: { sub_questions: ["Which calls mention price?", "What do they object to?"], max_tool_calls: 9 },
+ },
+ });
+ });
+
+ it("revises with the answers picked, and declines", async () => {
+ (Research2.reviseResearch as jest.Mock).mockImplementation(() => ok({ ...PLAN, revisions: 1 }));
+ show( );
+ fireEvent.click(await screen.findByRole("radio", { name: "This quarter" }));
+ expect(screen.getByRole("button", { name: "Revise with my answers" })).toBeInTheDocument();
+ fireEvent.submit(screen.getByRole("form", { name: "Research plan" }));
+ await waitFor(() => expect(Research2.reviseResearch).toHaveBeenCalled());
+ expect((Research2.reviseResearch as jest.Mock).mock.calls[0][0]).toMatchObject({
+ path: { rid: 4 },
+ body: { answers: { "Which period?": "This quarter" } },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Decline" }));
+ await waitFor(() =>
+ expect((Chats.decideApproval as jest.Mock).mock.calls[0][0]).toMatchObject({ body: { decision: "decline" } }),
+ );
+ });
+
+ it("adds and removes sub-questions, keeping one", async () => {
+ show( );
+ fireEvent.click(await screen.findByRole("button", { name: "Add a sub-question" }));
+ expect(screen.getByRole("textbox", { name: "Sub-question 3" })).toHaveFocus();
+ fireEvent.click(screen.getByRole("button", { name: "Remove sub-question 1" }));
+ fireEvent.click(screen.getByRole("button", { name: "Remove sub-question 1" }));
+ expect(screen.getByRole("button", { name: "Remove sub-question 1" })).toBeDisabled();
+ });
+});
+
+describe("the progress card", () => {
+ const running: Research = {
+ ...PLAN,
+ status: "running",
+ stage: "gather",
+ findings: [{ question: "Which calls mention pricing?", tool_calls: 3 }],
+ progress: {
+ sub_questions: 2,
+ done: 1,
+ sources: 4,
+ tool_calls: 3,
+ cost_usd: 0.0042,
+ cost_estimated: true,
+ minutes: 1.5,
+ },
+ };
+
+ it("shows the stage, what's done and spent, and stops the run", async () => {
+ (Research2.getResearch as jest.Mock).mockImplementation(() => ok(running));
+ (Research2.stopResearch as jest.Mock).mockImplementation(() => ok({ stopping: true }));
+ show( );
+ expect(await screen.findByText("Looking things up…")).toBeInTheDocument();
+ expect(screen.getByRole("progressbar", { name: "Sub-questions answered" })).toHaveAttribute("aria-valuenow", "1");
+ expect(
+ screen.getByText(/1 of 2 sub-questions · 4 sources · 3 of 8 tool calls · \$0.0042 \(an estimate\) · 1.5 min/),
+ ).toBeInTheDocument();
+ fireEvent.click(screen.getByRole("button", { name: "Stop" }));
+ await waitFor(() =>
+ expect(Research2.stopResearch).toHaveBeenCalledWith(expect.objectContaining({ path: { rid: 4 } })),
+ );
+ });
+
+ it("says why a budget holds it, and goes on when asked", async () => {
+ (Research2.getResearch as jest.Mock).mockImplementation(() =>
+ ok({ ...running, status: "held", held: { why: "the monthly budget is spent", budget: "all" } }),
+ );
+ (Research2.resumeResearch as jest.Mock).mockImplementation(() => ok({ ...running }));
+ show( );
+ expect(await screen.findByText("Paused: the monthly budget is spent")).toBeInTheDocument();
+ fireEvent.click(screen.getByRole("button", { name: "Go on anyway" }));
+ await waitFor(() => expect(Research2.resumeResearch).toHaveBeenCalled());
+ });
+
+ it("reads the conversation again when the report is saved", async () => {
+ (Research2.getResearch as jest.Mock).mockImplementation(() =>
+ ok({ ...running, status: "done", stage: "done", answer: 77 }),
+ );
+ show( );
+ qc.setQueryData(["chat", 9], { id: 9, messages: [{ id: 5, role: "assistant", content: "plan" }] });
+ const spy = jest.spyOn(qc, "invalidateQueries");
+ expect(await screen.findByText("Done: the report is below")).toBeInTheDocument();
+ await waitFor(() => expect(spy).toHaveBeenCalledWith({ queryKey: ["chat", 9] }));
+ });
+});
+
+describe("the report's actions", () => {
+ it("saves it as a note, asking where when it could go in several namespaces", async () => {
+ (Research2.researchNote as jest.Mock)
+ .mockImplementationOnce(() => Promise.reject(new Error("pick a namespace: pods, sales")))
+ .mockImplementationOnce(() => ok({ page: 12, namespace: "sales" }));
+ show( );
+ fireEvent.click(screen.getByRole("button", { name: "Save as note" }));
+ fireEvent.click(await screen.findByRole("button", { name: "sales" }));
+ await waitFor(() =>
+ expect((Research2.researchNote as jest.Mock).mock.calls[1][0]).toMatchObject({ body: { namespace: "sales" } }),
+ );
+ await waitFor(() => expect(screen.queryByRole("group", { name: "Save the note in" })).not.toBeInTheDocument());
+ });
+
+ it("reads it aloud, makes a podcast and exports Markdown", async () => {
+ (Research2.researchPodcast as jest.Mock).mockImplementation(() =>
+ ok({ episode: 5, job: 1, namespace: "podcasts" }),
+ );
+ const url = jest.fn(() => "blob:x");
+ Object.assign(URL, { createObjectURL: url, revokeObjectURL: jest.fn() });
+ show( );
+ fireEvent.click(screen.getByRole("button", { name: "Read aloud" }));
+ expect(speak).toHaveBeenCalledWith("Too dear [1].");
+ fireEvent.click(screen.getByRole("button", { name: "Make a podcast" }));
+ await waitFor(() =>
+ expect(Research2.researchPodcast).toHaveBeenCalledWith(expect.objectContaining({ path: { rid: 4 } })),
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Export" }));
+ expect(url).toHaveBeenCalled();
+ });
+});
+
+describe("the Deep research switch", () => {
+ it("turns the next message into a research goal, and says when it can't", () => {
+ const onToggle = jest.fn();
+ const { rerender } = render(
+ {}}
+ onSend={() => {}}
+ busy={false}
+ placeholder="Ask"
+ research={{ on: false, onToggle }}
+ />,
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Deep research" }));
+ expect(onToggle).toHaveBeenCalled();
+ rerender(
+ {}}
+ onSend={() => {}}
+ busy={false}
+ placeholder="Ask"
+ research={{ on: true, onToggle }}
+ />,
+ );
+ expect(screen.getByRole("button", { name: "Deep research" })).toHaveAttribute("aria-pressed", "true");
+ expect(screen.getByText("Deep research: you approve a plan first")).toBeInTheDocument();
+ rerender(
+ {}}
+ onSend={() => {}}
+ busy={false}
+ placeholder="Ask"
+ research={{ on: false, onToggle, off: "Deep research takes a goal, not files" }}
+ />,
+ );
+ expect(screen.getByRole("button", { name: "Deep research" })).toBeDisabled();
+ });
+});
diff --git a/nextjs-frontend/__tests__/settings-web-search.test.tsx b/nextjs-frontend/__tests__/settings-web-search.test.tsx
index 9e164a653..1c0965f47 100644
--- a/nextjs-frontend/__tests__/settings-web-search.test.tsx
+++ b/nextjs-frontend/__tests__/settings-web-search.test.tsx
@@ -56,6 +56,7 @@ const web = (values: Record) =>
laya_url: null,
},
},
+ research: { values: { max_tool_calls: 40, max_minutes: 20, model: null, agents_skip_review: false } },
voice: {
values: {
input: "auto",
diff --git a/nextjs-frontend/components/chat/answer.tsx b/nextjs-frontend/components/chat/answer.tsx
index 1d47c82aa..ca414121d 100644
--- a/nextjs-frontend/components/chat/answer.tsx
+++ b/nextjs-frontend/components/chat/answer.tsx
@@ -157,6 +157,8 @@ export type AnswerProps = {
onChecked?: (c: AnswerCheck) => void;
/** What produced it (the answer's record); older answers have none. */
meta?: MessageRecord | null;
+ /** More of what can be done with it, under it (a research report's: save as note, read aloud...). */
+ actions?: React.ReactNode;
};
/** One answer: where it comes from, the tools used, approvals, the text with citations, and its error states. */
@@ -365,6 +367,7 @@ export function Answer(p: AnswerProps) {
)}
{result && }
+ {!streaming && p.actions}
);
}
diff --git a/nextjs-frontend/components/chat/chat-app.tsx b/nextjs-frontend/components/chat/chat-app.tsx
index f8ed6d229..ea7903d94 100644
--- a/nextjs-frontend/components/chat/chat-app.tsx
+++ b/nextjs-frontend/components/chat/chat-app.tsx
@@ -17,6 +17,7 @@ import { ConversationList } from "@/components/chat/conversations";
import { useApprovals, useChat, useChats, useLlmStatus, useStopAnswer } from "@/components/chat/data";
import { EmptyChat } from "@/components/chat/empty";
import { NamespaceOffer, spokenPick } from "@/components/chat/namespace-offer";
+import { ReportActions, ResearchCard } from "@/components/chat/research";
import { fromApiScope, scopeFromParams, toApiScope, type Scope } from "@/components/chat/scope";
import { CitationSheet, SourcesPanel, SourcesSheet } from "@/components/chat/sources";
import {
@@ -44,10 +45,11 @@ import { useVoice, VoiceError } from "@/lib/voice";
type Sent = { id: string; filename: string; size: number };
/** An answer being written; `beside`: the message it makes a version of (an edited question, a regenerated answer). */
type Live = { chatId: number; turn: TurnState; files: Sent[]; beside?: number };
-type Queued = { key: number; text: string; files: Sent[] };
+type Queued = { key: number; text: string; files: Sent[]; research?: boolean };
-/** /clear and /compact, which the model doesn't answer, so there's nothing to regenerate. */
-const COMMAND = /^\s*\/(clear|compact)(\s|$)/i;
+/** /clear and /compact, which the model doesn't answer, and /research, answered with a plan to approve: there's nothing
+ * to regenerate. */
+const COMMAND = /^\s*\/(clear|compact|research)(\s|$)/i;
/** What a message with only files says (the server says the same when it gets none). */
type Extra = { steps: ToolStep[]; notice: string | null; error: string | null };
@@ -241,6 +243,7 @@ export function ChatApp() {
attached: Sent[] = [],
edit?: number,
regenerate?: number,
+ research?: boolean,
): Promise => {
// a new question goes on from the version showing; an edit or a regenerate is a version beside its message
const shown = qc.getQueryData(["chat", cid])?.messages ?? [];
@@ -265,6 +268,7 @@ export function ChatApp() {
edit,
parent,
via: spoken.current,
+ research: research || undefined,
},
accessToken: session?.accessToken,
signal: ac.signal,
@@ -274,6 +278,8 @@ export function ChatApp() {
show(turn);
if (turn.scoped && !was) narrowed(cid, turn.scoped);
if (msg.event === "suggested" && turn.suggested?.length) setOffer({ cid, items: turn.suggested });
+ // the plan's card reads the research from here on
+ if (msg.event === "research" && turn.research) qc.setQueryData(["research", turn.research.id], turn.research);
}
if (turn.status === "streaming")
turn = {
@@ -313,6 +319,7 @@ export function ChatApp() {
attached: Sent[] = [],
kind: "chat" | "setup" = "chat",
edit?: number,
+ research?: boolean,
): Promise => {
let cid = kind === "setup" ? null : activeId;
if (cid == null) {
@@ -337,11 +344,14 @@ export function ChatApp() {
return null;
}
}
- return run(cid, question, model, attached, edit);
+ return run(cid, question, model, attached, edit, undefined, research);
},
[activeId, client, draftScope, draftModel, qc, router, run, toast],
);
+ // The Deep research switch: what's typed next is a research goal, answered with a plan to approve.
+ const [researchOn, setResearchOn] = useState(false);
+
// What you send while an answer is being written waits its turn, so you never have to wait to type.
const send = useCallback(
(text?: string, model?: string) => {
@@ -349,20 +359,22 @@ export function ChatApp() {
text == null ? files.take().map((x) => ({ id: x.upload!.id, filename: x.file.name, size: x.file.size })) : [];
const question = (text ?? draft).trim() || (attached.length ? onlyFiles(attached.length) : "");
if (!question) return;
+ const research = text == null && researchOn && !attached.length;
if (text == null) setDraft("");
+ if (research) setResearchOn(false);
if (live) {
- setQueue((q) => [...q, { key: Date.now() + q.length, text: question, files: attached }]);
+ setQueue((q) => [...q, { key: Date.now() + q.length, text: question, files: attached, research }]);
return;
}
- void dispatch(question, model, attached);
+ void dispatch(question, model, attached, "chat", undefined, research);
},
- [dispatch, draft, files, live],
+ [dispatch, draft, files, live, researchOn],
);
useEffect(() => {
if (live || !queue.length) return;
const [next, ...rest] = queue;
setQueue(rest);
- void dispatch(next.text, undefined, next.files);
+ void dispatch(next.text, undefined, next.files, "chat", undefined, next.research);
}, [live, queue, dispatch]);
// Regenerating writes a new answer to the same question, as a version beside the old one, streaming in its place.
@@ -728,7 +740,9 @@ export function ChatApp() {
/>
)}
- {it.a ? (
+ {it.a?.kind === "research_plan" && it.a.meta?.research != null ? (
+
+ ) : it.a ? (
!liveApprovalIds.has(a.id))
+ .filter((a) => !liveApprovalIds.has(a.id) && a.tool !== "research")
.map((a) => ({
...a,
estimate: a.estimate as Estimate | null,
@@ -748,11 +762,15 @@ export function ChatApp() {
scope={scope}
model={it.a.model ?? model}
onRetry={
- it.q && !live && !COMMAND.test(it.q.content) ? () => regenerate(it.q!, it.a!) : undefined
+ it.q && !live && !it.a.kind && !COMMAND.test(it.q.content)
+ ? () => regenerate(it.q!, it.a!)
+ : undefined
}
models={llm.models}
onRegenerate={
- it.q && !live && !COMMAND.test(it.q.content) ? (m) => regenerate(it.q!, it.a!, m) : undefined
+ it.q && !live && !it.a.kind && !COMMAND.test(it.q.content)
+ ? (m) => regenerate(it.q!, it.a!, m)
+ : undefined
}
versions={
setChecks((x) => ({ ...x, [it.a!.id]: c }))}
+ actions={
+ it.a.kind === "research" && it.a.meta?.research != null ? (
+
+ ) : undefined
+ }
{...answerHandlers(aKey)}
/>
) : partial ? (
@@ -798,26 +821,30 @@ export function ChatApp() {
{liveHere && (
-
send(liveHere.question)}
- onAddScope={addScope}
- {...answerHandlers("live")}
- />
+ {liveHere.research ? (
+
+ ) : (
+ a.tool !== "research")}
+ scope={scope}
+ model={model}
+ meta={liveHere.meta}
+ onStop={stop}
+ stopping={stopping != null && stopping === live?.chatId}
+ onRetry={() => send(liveHere.question)}
+ onAddScope={addScope}
+ {...answerHandlers("live")}
+ />
+ )}
)}
@@ -884,15 +911,31 @@ export function ChatApp() {
// a new conversation (nothing open, or one asked for): typing goes straight into it
autoFocus={composing || !showThread}
placeholder={
- voice.listening
- ? "Listening…"
- : voice.speaking
- ? "Reading the answer aloud…"
- : msgs.length || liveHere
- ? "Ask a follow-up…"
- : "Ask across your archive…"
+ researchOn
+ ? "What should it research?"
+ : voice.listening
+ ? "Listening…"
+ : voice.speaking
+ ? "Reading the answer aloud…"
+ : msgs.length || liveHere
+ ? "Ask a follow-up…"
+ : "Ask across your archive…"
}
voice={voice.supported ? { on: voiceOn, onToggle: () => setVoiceOn((v) => !v) } : undefined}
+ research={
+ setupChat
+ ? undefined
+ : {
+ on: researchOn,
+ onToggle: () => setResearchOn((v) => !v),
+ off:
+ llm.known && (!llm.configured || !llm.tools)
+ ? "Deep research needs a language model that uses tools (Settings › AI assistant)"
+ : files.items.length
+ ? "Deep research takes a goal, not files"
+ : null,
+ }
+ }
/>
diff --git a/nextjs-frontend/components/chat/composer.tsx b/nextjs-frontend/components/chat/composer.tsx
index a23b49b54..c46f2c5fb 100644
--- a/nextjs-frontend/components/chat/composer.tsx
+++ b/nextjs-frontend/components/chat/composer.tsx
@@ -15,6 +15,7 @@ import {
Paperclip,
Plus,
Sparkles,
+ Telescope,
Wrench,
X,
} from "lucide-react";
@@ -436,7 +437,8 @@ export function ScopeBar({
/**
* The question box: Enter sends, Shift+Enter makes a new line. Typing goes on while an answer is written: what you
* send then waits its turn (`busy`). With `onFiles`, files can be attached (the clip, dropping them on the box, or
- * pasting them), shown in `files`; a message can be only files.
+ * pasting them), shown in `files`; a message can be only files. With `research`, a Deep research switch: what's sent
+ * next is a research goal, answered with a plan to approve (as /research is).
*/
export function Composer({
value,
@@ -451,6 +453,7 @@ export function Composer({
hasFiles = false,
uploading = false,
voice,
+ research,
}: {
value: string;
onChange: (v: string) => void;
@@ -467,6 +470,8 @@ export function Composer({
uploading?: boolean;
/** Voice mode: a mic that talks the conversation (hear, send, read the answer aloud) until tapped again. */
voice?: { on: boolean; onToggle: () => void };
+ /** Deep research: the next message is a research goal. `off`: why it can't be, when it can't. */
+ research?: { on: boolean; onToggle: () => void; off?: string | null };
}) {
const ref = useRef(null);
const [over, setOver] = useState(false);
@@ -503,6 +508,20 @@ export function Composer({
)}
>
{files}
+ {research?.on && (
+
+
+ Deep research: you approve a plan first
+
+
+
+
+ )}
{onFiles && (
)}
+ {research && (
+
+
+
+ )}