From ee327cf7e78c0df8a748dcf27404c8216b8293f0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Fri, 16 Oct 2026 23:26:19 +0000 Subject: [PATCH 01/12] fix(assistant): let the assistant's tools be imported first Importing app.domain.ai_tools on its own failed: it imported automations, which loads the actions, and the agent's action subclasses Toolbox before ai_tools had defined it. Running one test file (tests/api/test_chats.py) hit it. ai_tools now imports automations last. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/app/domain/ai_tools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 843562243..9ca9f43fd 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -12,7 +12,6 @@ from . import ( auth, - automations as automationsmod, batches, cypher, entities, @@ -1861,3 +1860,7 @@ def author_extension(db, tool, args, user, admin, readable, editable): n = extensions.save_version(db, me, same["id"], manifest, args.get("notes"), origin="chat") return {"extension": same["id"], "version": n} return {"extension": extensions.create(db, me, manifest, origin="chat"), "version": 1} + + +# last, because automations imports the actions, and the agent's action (actions_ai.Box) builds on Toolbox above +from . import automations as automationsmod # noqa: E402 From defb487a21bc621dd7e835bfab4538441706b2a4 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:01:11 +0000 Subject: [PATCH 02/12] feat(chat): each answer records what produced it Every answer keeps `meta`: the model and the host name of its server, the tools offered and asked for, the scope it answered in (after auto-scope), its limits and what it used, how it ended (answered, direct, limit, fallback, stopped, error), tool results that read like instructions to an AI, how long it took, Lens's version, and tokens and cost from the ledger rows that name it. Questions keep how they were sent (web, api, voice, room, automation) and their options. The answer's id is taken once the question is saved and the answer runs in activity.scope(chat:, chat_message:), in the web chat, chat rooms and the automations' agent; the streamed answer holds the scope across its steps (activity.scoped). App completions keep no messages, so their record comes back in lens.record. The stream sends `meta` before `done`; GET /chats/{cid}/messages/{mid} returns one message with its record and ledger rows. Each tool step keeps its time and whether it failed. The web app shows it all from an i in the answer's header (a sheet on a phone); old answers say they weren't recorded. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- CHANGELOG.md | 5 + docs/api.md | 8 +- docs/assistant.md | 36 +- docs/database.md | 5 +- fastapi_backend/app/api/v1/routes/activity.py | 4 + fastapi_backend/app/api/v1/routes/chats.py | 65 +- .../app/api/v1/routes/completions.py | 8 +- fastapi_backend/app/domain/actions_ai.py | 32 +- fastapi_backend/app/domain/activity.py | 21 + fastapi_backend/app/domain/ai_tools.py | 4 + fastapi_backend/app/domain/bridge.py | 103 +-- fastapi_backend/app/domain/chat.py | 84 ++- fastapi_backend/app/domain/completions.py | 21 +- fastapi_backend/app/schemas/chats.py | 70 ++ fastapi_backend/tests/api/test_actions_ai.py | 6 +- fastapi_backend/tests/api/test_bridge.py | 6 +- .../tests/api/test_chat_records.py | 178 +++++ fastapi_backend/tests/api/test_completions.py | 7 + .../tests/api/test_prompt_injection.py | 1 + fastapi_backend/tests/fake_llm.py | 5 +- .../__tests__/chat-details.test.tsx | 168 +++++ nextjs-frontend/app/openapi-client/index.ts | 12 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 22 +- .../app/openapi-client/types.gen.ts | 325 +++++++++ nextjs-frontend/components/chat/answer.tsx | 8 +- nextjs-frontend/components/chat/chat-app.tsx | 8 +- nextjs-frontend/components/chat/details.tsx | 201 ++++++ nextjs-frontend/components/chat/steps.tsx | 9 +- nextjs-frontend/components/chat/stream.ts | 30 +- nextjs-frontend/components/costs/costs.tsx | 2 +- nextjs-frontend/openapi.json | 628 +++++++++++++++++- threat-model.md | 1 + threat-model.yaml | 2 +- 33 files changed, 1974 insertions(+), 111 deletions(-) create mode 100644 fastapi_backend/tests/api/test_chat_records.py create mode 100644 nextjs-frontend/__tests__/chat-details.test.tsx create mode 100644 nextjs-frontend/components/chat/details.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bbfb731..184fa7b55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now; Settings → Automations switches them all off and sets how far they may go ([Running automations](docs/automations.md#running-automations)). +- **Each answer says what produced it.** The *i* in an answer's header shows the model and the host it ran on, the + tools it was offered and used, the scope, its limits and what it used, tokens, cost (an estimate from your prices), + how long it took, how it ended and which tool results read like instructions to an AI, with the calls made for it. + Agents read the same record over the API (`GET /api/v1/chats//messages/`), and each tool step shows how long + it took. Answers from before have no record ([What each answer records](docs/assistant.md#what-each-answer-records)). - **Automations for agents.** Agents list, save, try and run automations through the MCP tools `automations`, `automation_actions`, `save_automation`, `test_automation` and `run_automation`, with problems in plain words. The assistant saves them switched off and asks before switching one on or running it diff --git a/docs/api.md b/docs/api.md index 6e8f3bec8..beed20cb5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -477,7 +477,8 @@ read there: What it cites comes back three ways, so any app can show it: `[n]` in the text with a **Sources** list of links after it (left out when the request has a `response_format`), `citations` (the link for `[n]` is the n-th), and `lens.sources` (`n`, `recording_id`, `title`, `time`, `speaker`, `url`, `cited`). `lens.steps` lists the tools the -model used. With `"stream": true` the answer comes as OpenAI's `chat.completion.chunk` events, ending with +model used, and `lens.record` what produced the answer, as a chat answer's record +([What each answer records](assistant.md#what-each-answer-records)) without tokens and cost. With `"stream": true` the answer comes as OpenAI's `chat.completion.chunk` events, ending with `data: [DONE]`; the last chunk before it carries `finish_reason: "stop"`, `citations` and `lens`. While the tools run, the stream sends SSE comments, so nothing in between closes it. @@ -1448,6 +1449,7 @@ GET /api/v1/chats/{cid} PATCH /api/v1/chats/{cid} DELETE /api/v1/chats/{cid} POST /api/v1/chats/{cid}/messages +GET /api/v1/chats/{cid}/messages/{mid} POST /api/v1/chats/{cid}/stop POST /api/v1/chats/{cid}/messages/{mid}/check GET /api/v1/approvals @@ -1469,7 +1471,9 @@ whether it uses tools (`tools`, `max_steps`), whether answers can be checked (`c (the configured one first; `llm.chat_models`, else the server's list); never the model server's address or key. A conversation's `model` (`POST /chats`, `PATCH /chats/{cid}`; null for the configured one) answers in it, and a question can name another (`POST /chats/{cid}/messages {content, model}`, e.g. to retry); a model that isn't offered is -a 400. Each answer records the `model` that wrote it. `POST /chats/{cid}/stop` stops the answer being written in your conversation after the piece or tool +a 400. Each answer records the `model` that wrote it, and its `meta` what produced it; `GET +/chats/{cid}/messages/{mid}` gives one message with its record and its rows in the activity ledger +([What each answer records](assistant.md#what-each-answer-records)). `POST /chats/{cid}/stop` stops the answer being written in your conversation after the piece or tool step it's on: the stream sends `stopped`, then `done` with the saved message, whose `stopped` is true and whose `content` is what came before (`(stopped)` when nothing had). `{stopping: false}` when nothing was being written. `POST /chats/{cid}/messages {content, edit}` edits one of your earlier questions (`edit` is its message id): it and diff --git a/docs/assistant.md b/docs/assistant.md index bacf0168f..02dc202ba 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -13,11 +13,12 @@ then. numbered. - **Answers:** the model answers only from those excerpts and cites them as [n]. Each citation carries the asset and timestamp. -- **Streaming:** answers arrive over server-sent events: `passages`, then `token`s, then `done`. +- **Streaming:** answers arrive over server-sent events: `passages`, then `token`s, then `meta` (the answer's + record, [below](#what-each-answer-records)) and `done`. - **Stopping:** Stop asks the server to end the answer after the piece or tool step it's on (whichever server process is writing it); what came before is saved, marked stopped. A model call already under way finishes first. - **Reopening:** a conversation's answers keep the tools the assistant used (and with what), any notice (the model - couldn't use tools), the error when there was no answer, and their latest source check. + couldn't use tools), the error when there was no answer, their latest source check and what produced them. - **Editing a question:** Edit on a question you asked asks it again as edited; the answer and everything after it are replaced (`POST /api/v1/chats//messages` with `edit`). - **Long conversations:** the model reads the latest six messages word for word. Once eight more have built up @@ -51,6 +52,37 @@ then. - **No model configured:** chat returns the best-matching passages instead. Anyone signed in can see whether a model is set up, and which (`GET /chats/capabilities`), so the app says so before the first question. +### What each answer records + +Every answer keeps a record of what produced it (`meta` on the message), so a person can trust and debug it and an +agent can read it. The **i** in an answer's header (Details) shows it, with the calls made for it and what the +conversation has cost this month; on a phone it opens as a sheet. + +| Field | What it says | +| --- | --- | +| `model`, `provider` | the model that wrote it, and the host name of the model server it ran on (never the server's address, path or key) | +| `tools` | `offered`: every tool the model was offered; `picked`: the tools the question asked for (null: all it may use) | +| `scope` | the scope it answered in, after a conversation over everything was narrowed to a namespace | +| `limits`, `used` | the rounds of tool calls it had (and the tool calls, when they're limited), and how many it used | +| `outcome` | `answered` (after looking things up), `direct` (without looking anything up), `limit` (its rounds ran out, so it answered from what it found), `fallback` (from a search: the model can't use tools, or none is set up), `stopped` or `error` | +| `tokens`, `cost_usd`, `cost_estimated` | the tokens of the model calls made for it, and what they cost from the prices in Settings ([Costs](activity.md#costs)); `cost_estimated` when some had a price but no token counts, so it cost at least this | +| `ms` | how long it took, from the question | +| `flagged` | tools whose results read like instructions to an AI ([below](#third-party-content-and-prompt-injection)) | +| `version` | the version of Lens that wrote it | + +Each tool step also keeps how long it took (`ms`) and whether it failed (`failed`). A question keeps how it was sent +(`via`: `web`, `api`, `voice`, `room` or `automation`) and the options it was asked with (`options`, such as another +`model`). + +An answer is written inside the activity ledger's scope `chat_message:` (with `chat:`), so every model and +tool call made for it names it ([Activity](activity.md)), and its record's tokens and cost are theirs: one ledger +total per answer, and no model call. `GET /api/v1/chats/` returns the record on each message, and +`GET /api/v1/chats//messages/` returns one message with its record and those ledger rows (`ledger`). +Answers in chat rooms ([Chat rooms](chat-rooms.md)) and the automations' agent record theirs the same way. An app's +chat completions keep no messages, so their record comes back in the reply's `lens.record`, without tokens and cost +([Chat completions](api.md#chat-completions-for-any-app)). Answers from before Lens kept records have none; Details +shows what they have and says the rest wasn't recorded. + ### Picking the namespace for a conversation over everything When a conversation has no scope (the assistant home starts it over everything), its first question goes to the diff --git a/docs/database.md b/docs/database.md index 7473ad304..d6a8d1c59 100644 --- a/docs/database.md +++ b/docs/database.md @@ -87,7 +87,10 @@ optional, and older resources simply don't have them. ## Upgrades Installs keep their data across updates (running `install.sh` again, a new image, `git pull`). New tables, fields and -indexes arrive through `SCHEMA` on the next start. Anything else, such as rewriting stored values, moving data between +indexes arrive through `SCHEMA` on the next start. A field that is only added needs no step when its readers treat it +as missing on older rows: a chat message's record (`meta`, +[What each answer records](assistant.md#what-each-answer-records)) is one, and messages from before it simply have +none. Anything else, such as rewriting stored values, moving data between tables, or changing or removing an index that already exists, is a named step in `app/domain/migrations.py`: * Every process that opens the database (the API, workers, `lens watch`, `lens` commands) runs the steps the database diff --git a/fastapi_backend/app/api/v1/routes/activity.py b/fastapi_backend/app/api/v1/routes/activity.py index ede0375f6..9c6f68399 100644 --- a/fastapi_backend/app/api/v1/routes/activity.py +++ b/fastapi_backend/app/api/v1/routes/activity.py @@ -48,6 +48,10 @@ def _may_see(db: Db, user: CurrentUser, acl: Acl, resource: str) -> None: return if table == "chat" and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", int(key)), a=user.id): return + if table == "chat_message": # a message in one of your conversations + m = db.one("SELECT chat FROM $r", r=activity.R("chat_message", int(key))) + if m and db.one("SELECT id FROM $r WHERE account = $a", r=activity.R("chat", m["chat"]), a=user.id): + return raise HTTPException(403, "only admins see this resource's activity") diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py index aadedbd19..d0791f482 100644 --- a/fastapi_backend/app/api/v1/routes/chats.py +++ b/fastapi_backend/app/api/v1/routes/chats.py @@ -1,7 +1,7 @@ """Conversations with the archive, the assistant's approvals, and checking an answer against its sources. A question streams back server-sent events: ``scoped`` (the namespace a conversation over everything was narrowed -to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error`` and ``done``. +to) or ``suggested`` (namespaces to offer when it isn't clear, new ones included), ``step`` (a tool the assistant used), ``approval`` (work it proposed that waits for you), ``notice``, ``passages`` (the numbered excerpts), ``token`` (answer text), ``error``, ``meta`` (the answer's record) and ``done``. The assistant only ever sees, and cites, what the asker can read. """ @@ -19,7 +19,7 @@ from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Principal, Writer from app.api.media import sign_urls from app.api.v1.routes.collections import own_collection -from app.domain import ai_tools, approvals, auth, auto_scope, browser, chat, llm, store, uploads +from app.domain import activity, ai_tools, approvals, auth, auto_scope, browser, chat, llm, store, uploads from app.domain.store import DB from app.schemas.chats import ( AnswerCheck, @@ -34,6 +34,7 @@ ChatSummary, ChatUpdate, MessageCreate, + MessageDetail, StopResult, ) from app.schemas.common import Created, Ok @@ -129,6 +130,17 @@ def get_chat(cid: int, user: CurrentUser, acl: Acl, db: Db) -> Chat: return Chat.model_validate(sign_urls({**c, "messages": msgs}, full=True)) +@router.get("/chats/{cid}/messages/{mid}") +def get_message(cid: int, mid: int, user: CurrentUser, acl: Acl, db: Db) -> MessageDetail: + """One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and + used, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`: + the calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}.""" + m = next((x for x in get_chat(cid, user, acl, db).messages if x.id == mid), None) + if m is None: + raise HTTPException(404, "not found") + return MessageDetail.model_validate({**m.model_dump(by_alias=True), "ledger": activity.history(db, f"chat_message:{mid}", 200)}) + + @router.patch("/chats/{cid}") def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Ok: """Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one).""" @@ -177,7 +189,8 @@ def _ev(name: str, data: Any) -> str: async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> StreamingResponse: """Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST - /chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id). + /chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced + it), done (the saved message id). With `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and its new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked @@ -211,6 +224,8 @@ def run() -> Iterator[str]: model = await run_in_threadpool(_model, cfg, body.model) if body.model else c.get("model") if model and not body.model and model not in await run_in_threadpool(chat.model_choices, cfg): model = None + record = chat.Record(cfg, model) # what produced the answer, kept with it (from here: it times the answer) + mid = 0 # the answer's id, taken once the question is saved title = c["title"] page = body.context.model_dump() if body.context else None kept = chat.shared_context(page) # what the question keeps of the page @@ -229,14 +244,21 @@ def run() -> Iterator[str]: summary: dict[str, Any] | None = None # what compaction keeps of the earlier messages (chat.compact) def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: - nonlocal summary + nonlocal summary, mid try: # a conversation that fills the model's context is folded into its summary before it's answered chat.compact(db, cfg, cid, full_only=True) except Exception: # noqa: BLE001 - answered with what fits; tried again after the answer log.warning("chat %s: couldn't update the summary", cid, exc_info=True) past = chat.history(db, cid) summary = chat.memory(db, cfg, cid) - chat.add(db, cid, "user", q, attachments=files, context=kept) + via = "voice" if body.via == "voice" else "web" if user.via == "access" else "api" + chat.add(db, cid, "user", q, attachments=files, context=kept, meta=chat.question_record(via, model=body.model)) + mid = db.next_id("chat_message") + # what's worked out for the answer from here (the excerpts, the namespace it's about) counts for it in the ledger + with activity.scope(db, f"chat:{cid}", f"chat_message:{mid}", cfg=cfg): + return draw_on(past) + + def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: if not past and title == "New conversation": db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=q[:80]) # A conversation over everything gets the namespace its first question is about, when that's clear. @@ -251,6 +273,7 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: return past, chat.retrieve(db, q, readable, c["scope"], cfg=cfg), placed past, passages, placed = await run_in_threadpool(prepare) + record.scope = c.get("scope") or {} # after a conversation over everything was narrowed to what it's about picked = placed[1] if placed and placed[0] == "scoped" else None suggested = placed[1] if placed and placed[0] == "suggest" else None @@ -265,13 +288,17 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: ) notice: str | None = None - def save(text: str, cited: list[dict[str, Any]], **kw: Any) -> int: + def save(text: str, cited: list[dict[str, Any]], outcome: str = "answered", **kw: Any) -> Iterator[str]: + """The answer saved with its record: `meta`, then `done` (its id).""" wrote = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None - return int(chat.add(db, cid, "assistant", text, cited, steps=steps, notice=notice, model=wrote, **kw)) + meta = record.meta(db, outcome, f"chat_message:{mid}") + chat.add(db, cid, "assistant", text, cited, steps=steps, notice=notice, model=wrote, mid=mid, meta=meta, **kw) + yield _ev("meta", meta) + yield _ev("done", {"message": mid}) def stopped(text: str, cited: list[dict[str, Any]]) -> Iterator[str]: yield _ev("stopped", {"message": "Stopped"}) - yield _ev("done", {"message": save(text or "(stopped)", cited, stopped=True)}) + yield from save(text or "(stopped)", cited, "stopped", stopped=True) def answer(on: chat.Answering) -> Iterator[str]: nonlocal notice @@ -294,10 +321,10 @@ def answer(on: chat.Answering) -> Iterator[str]: browser=body.browser, ) try: - answer = "" + answer, how = "", "answered" # setting the server up, or working in the person's browser, takes more steps than looking things up steps_max = max(cfg["ai"].get("max_steps") or 6, 12 if setup else 0, browser.steps(cfg) if box.browser else 0) - for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary): + for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record): if kind == "step": steps.append(data) yield _ev("step", data) @@ -311,7 +338,7 @@ def answer(on: chat.Answering) -> Iterator[str]: elif kind == "direct" and passages and not setup and not (page and page.get("text")) and not box.cited(data): break else: - answer = data + answer, how = data, "direct" if kind == "direct" else "answered" else: box.after_answer(answer) for appr in box.approvals: @@ -319,23 +346,25 @@ def answer(on: chat.Answering) -> Iterator[str]: cited = box.cited(answer) yield _ev("passages", cited) yield _ev("token", {"text": answer}) - yield _ev("done", {"message": save(answer or "(no answer)", cited)}) + yield from save(answer or "(no answer)", cited, how) return except llm.ToolsUnsupported: notice = "This model can't use tools, so the answer comes from a search instead." yield _ev("notice", {"message": notice}) except llm.LLMError as e: yield _ev("error", {"message": str(e)}) - yield _ev("done", {"message": save("(no answer)", [], error=str(e))}) + yield from save("(no answer)", [], "error", error=str(e)) return yield _ev("passages", passages) text, error = "", None try: if llm.configured(cfg): - for piece in llm.stream_chat(cfg, chat.messages_for(asked, passages, past, summary, cfg), model=model): + pieces = llm.stream_chat(cfg, chat.messages_for(asked, passages, past, summary, cfg), model=model) + for piece in pieces: text += piece yield _ev("token", {"text": piece}) if on.stop_requested(): + pieces.close() # the call ends here, so its ledger row is in the answer's record yield from stopped(text, chat.cited(text, passages)) return else: @@ -344,7 +373,8 @@ def answer(on: chat.Answering) -> Iterator[str]: except llm.LLMError as e: error = str(e) yield _ev("error", {"message": error}) - yield _ev("done", {"message": save(text or "(no answer)", chat.cited(text, passages), error=error)}) + how = "error" if error else "fallback" if notice or not llm.configured(cfg) else "answered" + yield from save(text or "(no answer)", chat.cited(text, passages), how, error=error) def gen() -> Iterator[str]: on = chat.Answering(db, cid) @@ -355,7 +385,8 @@ def gen() -> Iterator[str]: yield _ev("step", steps[0]) if suggested: yield _ev("suggested", {"namespaces": suggested}) - for ev in answer(on): + # every call made for the answer counts for it in the ledger (the scope holds across the stream's steps) + for ev in activity.scoped(answer(on), db, f"chat:{cid}", f"chat_message:{mid}", cfg=cfg): saved = saved or ev.startswith("event: done") yield ev except Exception: # an answer that breaks off still ends with an error and is saved, so the question isn't left unanswered @@ -363,7 +394,7 @@ def gen() -> Iterator[str]: message = "Something went wrong while answering. Try again." yield _ev("error", {"message": message}) if not saved: - yield _ev("done", {"message": save("(no answer)", [], error=message)}) + yield from save("(no answer)", [], "error", error=message) finally: on.end() try: # once enough has built up, older messages are folded into the summary (after the answer, so it waits on nothing) diff --git a/fastapi_backend/app/api/v1/routes/completions.py b/fastapi_backend/app/api/v1/routes/completions.py index ef9a2b49c..0b7137daf 100644 --- a/fastapi_backend/app/api/v1/routes/completions.py +++ b/fastapi_backend/app/api/v1/routes/completions.py @@ -123,7 +123,13 @@ def create_chat_completion(name: str, body: ChatCompletionRequest, request: Requ def extras(done: dict[str, Any], steps: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]: found = completions.sources(web, done["text"], done["refs"]) - lens = {"namespace": name, "sources": found, "steps": steps, **({"notice": done["notice"]} if done.get("notice") else {})} + lens = { + "namespace": name, + "sources": found, + "steps": steps, + "record": done.get("record"), + **({"notice": done["notice"]} if done.get("notice") else {}), + } return found, {"citations": [s["url"] for s in found], "lens": lens} if not body.stream: diff --git a/fastapi_backend/app/domain/actions_ai.py b/fastapi_backend/app/domain/actions_ai.py index 05b9c0691..dc37e1cb5 100644 --- a/fastapi_backend/app/domain/actions_ai.py +++ b/fastapi_backend/app/domain/actions_ai.py @@ -257,7 +257,10 @@ def _ask_agent(db, cfg, intent, user, dry): if about: url = (about.get("object") or {}).get("url") if isinstance(about.get("object"), dict) else None page = {"url": str(url or f"/timeline?activity={about.get('id') or ''}"), "title": _short(happened, 300)} - chat.add(db, cid, "user", instructions, context=page, origin=box.origin) + chat.add(db, cid, "user", instructions, context=page, origin=box.origin, meta=chat.question_record("automation", tools=tools or None)) + # the answer's id up front: every call made for it names it in the ledger, and its record sums them + mid = db.next_id("chat_message") + record, spent = chat.Record(cfg, scope=box.scope, picked=tools or None), f"chat_message:{mid}" # reading the activity counts as reading outside text for this conversation from now on (ai_tools.read_before) steps = ( [{"tool": "automation_event", "args": {"id": about.get("id")}, "summary": f"Read what happened: {_short(happened, 200)}"}] @@ -265,21 +268,28 @@ def _ask_agent(db, cfg, intent, user, dry): else [] ) question = instructions + (_context(about, cfg) if about else "") - text = None + text, how = None, "answered" + + def said(words, cited, outcome, **kw): + """The answer, saved with its record.""" + chat.add(db, cid, "assistant", words or "(no answer)", cited, steps=steps, mid=mid, meta=record.meta(db, outcome, spent), **kw) + try: - for kind, data in chat.tool_answer(cfg, box, question, (), steps_max): - if kind == "step": - steps.append(data) - else: - text = data + with activity.scope(db, f"chat:{cid}", spent, cfg=cfg): + for kind, data in chat.tool_answer(cfg, box, question, (), steps_max, record=record): + if kind == "step": + steps.append(data) + else: + text, how = data, "direct" if kind == "direct" else "answered" except llm.ToolsUnsupported: - chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error="this model can't use tools") + said("", [], "error", error="this model can't use tools") raise ValueError("this model can't use tools, so the agent can't act: pick one that can in Settings › Model") from None except (llm.LLMError, activity.OverBudget) as e: - chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error=str(e)) + said("", [], "error", error=str(e)) raise ValueError(f"the agent couldn't answer: {e}") from None - box.after_answer(text or "") - chat.add(db, cid, "assistant", text or "(no answer)", box.cited(text), steps=steps, model=cfg["llm"].get("model")) + with activity.scope(db, f"chat:{cid}", spent, cfg=cfg): + box.after_answer(text or "") + said(text, box.cited(text), how, model=cfg["llm"].get("model")) return { "answer": text or "", "chat": cid, diff --git a/fastapi_backend/app/domain/activity.py b/fastapi_backend/app/domain/activity.py index c09c9a053..89f91c460 100644 --- a/fastapi_backend/app/domain/activity.py +++ b/fastapi_backend/app/domain/activity.py @@ -47,6 +47,7 @@ "workflows": "workflow", "custom-nodes": "custom_node", "chats": "chat", + "messages": "chat_message", # a conversation's messages/{mid} "jobs": "job", "namespaces": "space", "spaces": "space", @@ -113,6 +114,26 @@ def scope(db, *refs, cfg=None): _scope.reset(token) +def scoped(items, db, *refs, cfg=None): + """`items` (a generator) with each of its steps run inside scope(db, *refs). A `with scope(...)` inside a generator + doesn't hold across its yields when a streamed response iterates it from the threadpool: each step runs in a fresh + copy of the request's context.""" + it = iter(items) + try: + while True: + with scope(db, *refs, cfg=cfg): + try: + item = next(it) + except StopIteration: + return + yield item + finally: + close = getattr(it, "close", None) + if close: + with scope(db, *refs, cfg=cfg): + close() + + def current(): """The resources calls made here count for, in order, without repeats.""" s = _scope.get() or {} diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 9ca9f43fd..63151f162 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -532,6 +532,7 @@ def __init__( self.readable = {s for s in self.readable if names.get(s) in self.scope["namespaces"]} self.allowed = recsets.within(db, self.readable, self.scope.get("recordings"), self.scope.get("collections")) self.refs, self.reads, self.approvals = [], 0, [] + self.flagged, self.failed = [], False # tools whose results read like instructions to an AI; the last call failed # the extensions this person switched on or was given (extensions.py); hooks don't run inside hooks roles = {s: "editor" if s in self.editable else "viewer" for s in self.readable} self.me = extensions.who(self.account, user.get("email"), admin, roles) @@ -688,6 +689,7 @@ def call(self, name, args, model=False): """A tool's result (JSON) and the step's summary. `model`: the result as the model reads it, third-party text marked as data (untrusted), with what hooks add after it, unmarked.""" offered = {s["function"]["name"] for s in self.specs()} + self.failed = True # until it runs: a tool that's unknown or blocked failed try: if name not in offered: raise AttributeError(name) @@ -703,6 +705,7 @@ def call(self, name, args, model=False): extra += self._hook(h, {"said": self.said, "tool": name}) asked = len(self.approvals) result, summary = self._run(name, fn, args) + self.failed = result.startswith('{"error"') self.links |= untrusted.links(result) if self.untrusted(name) and len(self.approvals) == asked: # it read something, rather than asking for a yes self.read_outside(f"tool {name}") @@ -714,6 +717,7 @@ def call(self, name, args, model=False): why = untrusted.flags(result, self.cfg, model=True) if why: # the person sees it on the step, the model on the marker, admins in the audit log summary = f"{summary} (reads like instructions to an AI: {'; '.join(why)})" + self.flagged.append(name) auth.audit(self.db, self.user, "assistant.injection_flagged", f"chat:{self.chat}", {"tool": name, "why": why}) result = untrusted.wrap(result, f"tool {name}", self.cfg, why=why) return result + (f"\n\n{extra.strip()}" if extra else ""), summary diff --git a/fastapi_backend/app/domain/bridge.py b/fastapi_backend/app/domain/bridge.py index 491b2f4f7..cf995fba3 100644 --- a/fastapi_backend/app/domain/bridge.py +++ b/fastapi_backend/app/domain/bridge.py @@ -38,7 +38,7 @@ import urllib.error import urllib.request -from . import ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted +from . import activity, ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store, untrusted from .store import R log = logging.getLogger(__name__) @@ -330,53 +330,64 @@ def _answer(db, cfg, base, account, cid, q, files, act_on, outside): past = chat.history(db, cid) summary = chat.memory(db, cfg, cid) scope = (db.one("SELECT scope FROM $r", r=R("chat", cid)) or {}).get("scope") or None - chat.add(db, cid, "user", q, attachments=list(files) or None) - asked = q + chat.attached_note(list(files)) - passages = chat.retrieve(db, q, readable, scope, cfg=cfg) - # the web addresses written by someone other than the model: the person, now and before, and the archive - keep = untrusted.links("\n".join([q] + [m["content"] or "" for m in past if m["role"] == "user"] + [p["text"] for p in passages])) - steps, approvals, notice = [], [], None - wrote = cfg["llm"].get("model") if llm.configured(cfg) else None - if llm.configured(cfg) and cfg["ai"].get("tools"): - box = ai_tools.Toolbox( - db, - cfg, - {"id": account["id"], "email": account["email"]}, - readable, - editable, - scope, - cid, - base, - account["admin"], - said=q, - outside=list(outside) + ai_tools.read_before(past, summary), - act_on=act_on, - ) + chat.add(db, cid, "user", q, attachments=list(files) or None, meta=chat.question_record("room")) + # the answer's id up front: every call made for it names it in the ledger, and its record sums them + mid = db.next_id("chat_message") + record, spent = chat.Record(cfg, scope=scope), f"chat_message:{mid}" + with activity.scope(db, f"chat:{cid}", spent, cfg=cfg): + asked = q + chat.attached_note(list(files)) + passages = chat.retrieve(db, q, readable, scope, cfg=cfg) + # the web addresses written by someone other than the model: the person, now and before, and the archive + keep = untrusted.links("\n".join([q] + [m["content"] or "" for m in past if m["role"] == "user"] + [p["text"] for p in passages])) + steps, approvals, notice = [], [], None + wrote = cfg["llm"].get("model") if llm.configured(cfg) else None + + def said(words, cited, outcome, **kw): + """The answer, saved with its record.""" + meta = record.meta(db, outcome, spent) + chat.add(db, cid, "assistant", words or "(no answer)", cited, steps=steps, model=wrote, mid=mid, meta=meta, **kw) + + if llm.configured(cfg) and cfg["ai"].get("tools"): + box = ai_tools.Toolbox( + db, + cfg, + {"id": account["id"], "email": account["email"]}, + readable, + editable, + scope, + cid, + base, + account["admin"], + said=q, + outside=list(outside) + ai_tools.read_before(past, summary), + act_on=act_on, + ) + try: + text, how = None, "answered" + for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary, record=record): + if kind == "step": + steps.append(data) + elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers + break + else: + text, how = data, "direct" if kind == "direct" else "answered" + if text is not None: + box.after_answer(text) + said(text, box.cited(text), how) + return text, box.approvals, keep | box.links + except llm.ToolsUnsupported: + notice = "This model can't use tools, so the answer comes from a search instead." + except llm.LLMError as e: + said("", [], "error", error=str(e)) + return "I couldn't answer that: " + str(e), box.approvals, keep | box.links + error = None try: - text = None - for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary): - if kind == "step": - steps.append(data) - elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers - break - else: - text = data - if text is not None: - box.after_answer(text) - chat.add(db, cid, "assistant", text or "(no answer)", box.cited(text), steps=steps, model=wrote) - return text, box.approvals, keep | box.links - except llm.ToolsUnsupported: - notice = "This model can't use tools, so the answer comes from a search instead." + text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary, cfg)) if llm.configured(cfg) else chat.fallback(passages) except llm.LLMError as e: - chat.add(db, cid, "assistant", "(no answer)", [], steps=steps, error=str(e), model=wrote) - return "I couldn't answer that: " + str(e), box.approvals, keep | box.links - error = None - try: - text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary, cfg)) if llm.configured(cfg) else chat.fallback(passages) - except llm.LLMError as e: - text, error = "I couldn't answer that: " + str(e), str(e) - chat.add(db, cid, "assistant", text or "(no answer)", chat.cited(text, passages), steps=steps, notice=notice, error=error, model=wrote) - return text, approvals, keep + text, error = "I couldn't answer that: " + str(e), str(e) + how = "error" if error else "fallback" if notice or not llm.configured(cfg) else "answered" + said(text, chat.cited(text, passages), how, notice=notice, error=error) + return text, approvals, keep # ---------- approving in the room ---------- diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py index a3b3180e4..035bde9a2 100644 --- a/fastapi_backend/app/domain/chat.py +++ b/fastapi_backend/app/domain/chat.py @@ -12,8 +12,9 @@ import re import time from collections import Counter, defaultdict +from urllib.parse import urlsplit -from . import llm, recsets, render, semantic, store, textindex, untrusted +from . import __version__, activity, llm, recsets, render, semantic, store, textindex, untrusted R = store.R STOP = set( @@ -345,7 +346,7 @@ def get(db, cid, account): def history(db, cid): return db.rows( "SELECT record::id(id) AS id, role, content, passages, created_at, stopped ?? false AS stopped, steps ?? [] AS steps, " - "attachments ?? [] AS attachments, notice, error, check, model, context FROM chat_message WHERE chat = $c ORDER BY id", + "attachments ?? [] AS attachments, notice, error, check, model, context, meta FROM chat_message WHERE chat = $c ORDER BY id", c=cid, ) @@ -364,11 +365,15 @@ def add( attachments=None, context=None, origin=None, + mid=None, + meta=None, ): """Save a message; an answer keeps the tool steps it took, any notice (e.g. the model can't use tools) and error; a question asked from a page keeps what it shared of it (shared_context). A question an automation asked keeps its - `origin` ({automation, run, node}), so it isn't taken for a person's message (`chat.message`).""" - mid = db.next_id("chat_message") + `origin` ({automation, run, node}), so it isn't taken for a person's message (`chat.message`). `meta` is its record + (Record; a question's: how it was sent), `mid` its id when it was taken up front (an answer's, so the ledger rows + of the calls made for it name it).""" + mid = mid or db.next_id("chat_message") db.q( "CREATE $r CONTENT $d", r=R("chat_message", mid), @@ -387,6 +392,7 @@ def add( "model": model, "context": context, "origin": origin or None, + "meta": meta or None, } ), ) @@ -585,6 +591,61 @@ def request_stop(db, cid): return bool(db.rows("UPDATE $r SET stop_requested = true WHERE answering = true RETURN id", r=R("chat", cid))) +def provider(cfg): + """The model server's host name, never its address's path or a key: where an answer's model ran.""" + return (urlsplit(cfg["llm"].get("base_url") or "").hostname or None) if llm.configured(cfg) else None + + +class Record: + """What produced an answer, kept on it as `meta` (docs/assistant.md, "What each answer records"): the model and the + host it ran on, the tools it was offered and the ones asked for, the scope it answered in, its limits and what it + used, how it ended, the tools whose results read like instructions to an AI, how long it took, and its tokens and + cost, from the ledger rows that name it (the answer runs in activity.scope(db, "chat_message:")). Made as the + answer starts, which it times from; tool_answer fills in the tools and counts.""" + + def __init__(self, cfg, model=None, scope=None, picked=None, tool_calls=None, research=None): + self.cfg, self.t0 = cfg, time.monotonic() + self.model = (model or cfg["llm"].get("model")) if llm.configured(cfg) else None + self.scope, self.picked, self.research = scope or {}, picked, research + self.limits = {"rounds": None, "tool_calls": tool_calls} + self.used = {"rounds": 0, "tool_calls": 0} + self.offered, self.box, self.limited = [], None, False + + def offer(self, toolbox, specs, rounds): + self.box, self.offered, self.limits["rounds"] = toolbox, [s["function"]["name"] for s in specs], rounds + + def meta(self, db, outcome, resource=None): + """The record, once the answer ended with `outcome` (answered, direct, fallback, stopped or error; an answer + given because the rounds ran out is `limit`), with the tokens and cost of the ledger rows that name `resource` + (none without one).""" + if outcome in ("answered", "direct") and self.limited: + outcome = "limit" + out = { + "model": self.model, + "provider": provider(self.cfg), + "tools": {"picked": self.picked, "offered": self.offered}, + "scope": self.scope, + "limits": dict(self.limits), + "used": dict(self.used), + "outcome": outcome, + "ms": int((time.monotonic() - self.t0) * 1000), + "flagged": list(dict.fromkeys(getattr(self.box, "flagged", None) or [])), + "research": self.research, + "version": __version__, + } + if resource: + spent = activity.totals(db, resource) + out["tokens"] = {"in": spent["tokens_in"], "out": spent["tokens_out"]} + out["cost_usd"], out["cost_estimated"] = spent["cost_usd"], spent["estimate"] + return out + + +def question_record(via, **options): + """A question's record: how it was sent (web, api, room, voice, automation or completions) and the options it was + asked with.""" + return {"via": via, "options": store.clean(options)} + + TOOL_SYSTEM = ( SYSTEM + " You can call tools to look things up in the archive. Cite moments with the [n] numbers the tools return, and " "only those. Tools that run work or change data need the person's approval: say what you proposed and that it's waiting for them. " @@ -679,11 +740,12 @@ def _finish(cfg, msgs, tools, model): return text -def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None): +def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None): """The tool loop: yields ("step", {...}) for each tool call, then ("answer", text), or ("direct", text) when the model answered without looking anything up (it never saw the archive, so the caller can answer from a search instead). Every turn ends in an answer: when the steps run out, or the model replies with nothing, it's asked to answer from - what it found, with the tools off. Raises llm.ToolsUnsupported.""" + what it found, with the tools off. Raises llm.ToolsUnsupported. `record` (a Record) learns the tools offered and + counts the rounds and tool calls.""" system = TOOL_SYSTEM + untrusted.rules(cfg) + ("\n\n" + SETUP_SYSTEM if setup else "") note = getattr(toolbox, "system_note", None) # skills, and context from hooks (extensions.py) system += note() if note else "" @@ -695,6 +757,8 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set msgs = [{"role": "system", "content": system}] + past_turns(history, summary=summary, cfg=cfg) msgs.append({"role": "user", "content": question}) tools, looked, made = toolbox.specs(), False, set() + record = record or Record(cfg) + record.offer(toolbox, tools, max_steps) for _ in range(max_steps): msg = llm.chat_message(cfg, msgs, tools=tools, model=model) if not msg["tool_calls"]: @@ -703,6 +767,8 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set return break # a reply with nothing in it looked = True + record.used["rounds"] += 1 + record.used["tool_calls"] += len(msg["tool_calls"]) msgs.append({"role": "assistant", "content": msg["content"], "tool_calls": msg["tool_calls"]}) for c in msg["tool_calls"]: fn = c.get("function") or {} @@ -722,9 +788,13 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set # anything that may change what a read returns (a note written, a setting changed) makes every read new again made = made | {key} if name in SAME_AGAIN else set() # as the model reads it: what the tool read from the archive, the web or another server marked as data + t0 = time.monotonic() result, summary = toolbox.call(name, args, model=True) - yield "step", {"tool": name, "args": args, "summary": summary} + step = {"tool": name, "args": args, "summary": summary, "ms": int((time.monotonic() - t0) * 1000)} + yield "step", {**step, "failed": True} if getattr(toolbox, "failed", False) else step msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": result}) + else: # the rounds ran out + record.limited = True text = _finish(cfg, msgs, tools, model) or "I couldn't put an answer together from what I found. Try again, or ask something narrower." yield ("answer" if looked else "direct"), text diff --git a/fastapi_backend/app/domain/completions.py b/fastapi_backend/app/domain/completions.py index 44c44f45d..e344162ed 100644 --- a/fastapi_backend/app/domain/completions.py +++ b/fastapi_backend/app/domain/completions.py @@ -115,32 +115,34 @@ def specs(self): def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools=(), base=None, temperature=None): """Answer one request. Yields ("step", {tool, args, summary}) for each tool call, ("token", text) for the - answer's pieces, then ("done", {refs, text, model, notice}): what it can cite (numbered, as [n]) and the whole - answer. Raises llm.LLMError when the model server fails, and activity.OverBudget at the hard limit.""" + answer's pieces, then ("done", {refs, text, model, notice, record}): what it can cite (numbered, as [n]), the whole + answer and what produced it (chat.Record, without tokens and cost: no message keeps it). Raises llm.LLMError when + the model server fails, and activity.OverBudget at the hard limit.""" scope = {"namespaces": [ns]} + record = chat.Record(cfg, model, scope, picked=list(tools) or None) passages = chat.retrieve(db, question, {sid}, scope, cfg=cfg) extra, notice = note(db, sid, ns, app), None used = model or cfg["llm"].get("model") if not llm.configured(cfg): text = chat.fallback(passages) yield "token", text - yield "done", {"refs": passages, "text": text, "model": MODEL, "notice": None} + yield "done", {"refs": passages, "text": text, "model": MODEL, "notice": None, "record": record.meta(db, "fallback")} return if tools and cfg["ai"].get("tools", True): box = Box(db, cfg, user, sid, ns, tools, question, base, extra) try: - text = "" - for kind, data in chat.tool_answer(cfg, box, question, history, cfg["ai"].get("max_steps") or 6, model): + text, how = "", "answered" + for kind, data in chat.tool_answer(cfg, box, question, history, cfg["ai"].get("max_steps") or 6, model, record=record): if kind == "step": yield "step", data # a model that skips the tools hasn't seen the namespace: it answers from the excerpts instead elif kind == "direct" and passages and not box.cited(data): break else: - text = data + text, how = data, "direct" if kind == "direct" else "answered" else: yield "token", text - yield "done", {"refs": box.refs, "text": text, "model": used, "notice": None} + yield "done", {"refs": box.refs, "text": text, "model": used, "notice": None, "record": record.meta(db, how)} return except llm.ToolsUnsupported: notice = "This model can't use tools, so the answer comes from a search instead." @@ -150,7 +152,10 @@ def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools= for piece in llm.stream_chat(cfg, msgs, model=model, temperature=0.2 if temperature is None else temperature): text += piece yield "token", piece - yield "done", {"refs": passages, "text": text, "model": used, "notice": notice} + yield ( + "done", + {"refs": passages, "text": text, "model": used, "notice": notice, "record": record.meta(db, "fallback" if notice else "answered")}, + ) def link(web, ref): diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py index 2cf82c334..1ce9738df 100644 --- a/fastapi_backend/app/schemas/chats.py +++ b/fastapi_backend/app/schemas/chats.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, Field +from app.schemas.activity import ActivityEntry from app.schemas.common import RequestModel, ResponseModel @@ -80,6 +81,67 @@ class ToolStep(ResponseModel): tool: str args: dict[str, Any] = {} summary: str = "" + ms: int | None = Field(None, description="how long the tool took") + failed: bool | None = Field(None, description="the tool failed (or was unknown or blocked); its summary says why") + + +class RecordTools(ResponseModel): + picked: list[str] | None = Field(None, description="the tools or groups the question asked for; null: Auto") + offered: list[str] = Field(default_factory=list, description="every tool the model was offered") + + +class RecordLimits(ResponseModel): + rounds: int | None = Field(None, description="rounds of tool calls it had (calls in one round run together)") + tool_calls: int | None = Field(None, description="tool calls it had; null: only the rounds limit them") + + +class RecordUse(ResponseModel): + rounds: int = 0 + tool_calls: int = 0 + + +class RecordTokens(ResponseModel): + model_config = ConfigDict(extra="allow", populate_by_name=True, serialize_by_alias=True) + + in_: int = Field(0, alias="in") + out: int = 0 + + +class RecordOptions(ResponseModel): + """What a question was asked with (only what it set).""" + + model: str | None = None + tools: list[str] | None = None + max_tool_calls: int | None = None + scope: dict[str, Any] | None = None + research: bool | None = None + + +class MessageRecord(ResponseModel): + """What produced a message, recorded as it was written (docs/assistant.md, "What each answer records"). A question's + has `via` and `options`; an answer's the rest. Messages from before it was recorded have none.""" + + via: Literal["web", "api", "room", "voice", "automation", "completions"] | None = Field(None, description="a question: how it was sent") + options: RecordOptions | None = Field(None, description="a question: the options it was asked with") + model: str | None = Field(None, description="the model that wrote the answer") + provider: str | None = Field(None, description="the host name of the model server it ran on (never its address or key)") + tools: RecordTools | None = None + scope: dict[str, Any] | None = Field(None, description="the scope it answered in, after a conversation over everything was narrowed") + limits: RecordLimits | None = None + used: RecordUse | None = None + outcome: Literal["answered", "direct", "limit", "fallback", "stopped", "error"] | None = Field( + None, + description="answered (after looking things up), direct (without looking anything up), limit (its rounds ran " + "out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set " + "up), stopped, or error", + ) + tokens: RecordTokens | None = Field(None, description="the tokens of the model calls made for it (the activity ledger)") + cost_usd: float | None = Field(None, description="what those calls cost, from the prices in Settings") + cost_estimated: bool | None = Field(None, description="some calls had a price but no token counts: the cost is at least this") + ms: int | None = Field(None, description="how long it took to write, from the question") + flagged: list[str] | None = Field(None, description="tools whose results read like instructions to an AI") + research: int | None = Field(None, description="the research it belongs to") + version: str | None = Field(None, description="the version of Lens that wrote it") class PageContext(RequestModel): @@ -127,6 +189,13 @@ class ChatMessage(ResponseModel): error: str | None = Field(None, description="why there's no answer") check: AnswerCheck | None = Field(None, description="its latest source check (POST .../messages/{mid}/check)") model: str | None = Field(None, description="the model that wrote the answer") + meta: MessageRecord | None = Field(None, description="what produced it; none on messages from before it was recorded") + + +class MessageDetail(ChatMessage): + ledger: list[ActivityEntry] = Field( + default_factory=list, description="its rows in the activity ledger: the model and tool calls made for it, newest first" + ) class ChatCapabilities(ResponseModel): @@ -172,6 +241,7 @@ class MessageCreate(RequestModel): description="asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the " "assistant can use that browser (docs/browser-assistant.md)", ) + via: Literal["voice"] | None = Field(None, description="voice: said in a voice conversation (the question's record keeps it)") class ApprovalOrigin(ResponseModel): diff --git a/fastapi_backend/tests/api/test_actions_ai.py b/fastapi_backend/tests/api/test_actions_ai.py index 9f018cb95..77cc325ae 100644 --- a/fastapi_backend/tests/api/test_actions_ai.py +++ b/fastapi_backend/tests/api/test_actions_ai.py @@ -9,7 +9,7 @@ import pytest -from app.domain import actions, actions_ai, ai_tools, budgets, chat, notebook, store, untrusted +from app.domain import actions, actions_ai, activity, ai_tools, budgets, chat, notebook, store, untrusted from tests import fake_llm from tests.api._assist import start_llm from tests.api.test_setup_assistant import call @@ -171,6 +171,10 @@ def test_ask_agent_works_on_what_happened_and_saves_the_conversation(db, cfg, ll q, a = chat.history(db, got["chat"]) assert q["content"] == "File this where it belongs." and q["context"]["url"] == "/resources/7" assert a["content"] == got["answer"] and [s["tool"] for s in a["steps"]] == [s["tool"] for s in got["steps"]] + # it records what produced it, like any answer: the question came from an automation + assert q["meta"] == {"via": "automation", "options": {}} and a["meta"]["used"] == {"rounds": 2, "tool_calls": 2} + assert (a["meta"]["outcome"], a["meta"]["scope"]) == ("answered", {"namespaces": ["pods"]}) + assert activity.totals(db, f"chat_message:{a['id']}")["calls"] == 3 # its model calls name it in the ledger first = next(b for b in llm.seen if b.get("tools")) system, asked = first["messages"][0]["content"], first["messages"][-1]["content"] assert "acting for an automation, “Automation 4”, that runs for me" in system diff --git a/fastapi_backend/tests/api/test_bridge.py b/fastapi_backend/tests/api/test_bridge.py index 5a3cc8c75..ea489756e 100644 --- a/fastapi_backend/tests/api/test_bridge.py +++ b/fastapi_backend/tests/api/test_bridge.py @@ -5,7 +5,7 @@ import pytest -from app.domain import bridge, settings +from app.domain import activity, bridge, settings from tests import fake_llm, fake_matterbridge from tests.api._assist import start_llm from tests.api.test_setup_assistant import call @@ -98,6 +98,10 @@ def test_it_answers_what_is_said_to_it_and_nothing_else(db, cfg, room, llm): rows = db.rows("SELECT title, kind, account FROM chat WHERE kind = 'bridge' ORDER BY title") assert [r["title"] for r in rows] == ["Matterbridge · general · alice"] * 2 + ["Matterbridge · general · bob"] assert {r["kind"] for r in rows} == {"bridge"} and {r["account"] for r in rows} == {rows[0]["account"]} + # each question says it was asked in a room; each answer what produced it, its model calls naming it in the ledger + q, a = db.rows("SELECT record::id(id) AS id, role, meta FROM chat_message ORDER BY id")[:2] + assert q["meta"] == {"via": "room", "options": {}} and (a["meta"]["outcome"], a["meta"]["model"]) == ("answered", "fake") + assert activity.totals(db, f"chat_message:{a['id']}")["calls"] >= 1 room.waiting = [said("Lens, and on Monday?")] run(db, cfg) assert len(db.rows("SELECT id FROM chat WHERE kind = 'bridge'")) == 3 # alice keeps hers diff --git a/fastapi_backend/tests/api/test_chat_records.py b/fastapi_backend/tests/api/test_chat_records.py new file mode 100644 index 000000000..8f8bba2fc --- /dev/null +++ b/fastapi_backend/tests/api/test_chat_records.py @@ -0,0 +1,178 @@ +"""Each answer records what produced it, and each question how it was sent (docs/assistant.md, "What each answer +records"): the model and its host, the tools offered and used, the scope, limits and use, tokens and cost from the +ledger rows that name the answer, the time, how it ended and which tool results read like instructions to an AI.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import __version__, chat +from app.domain import llm as llm_mod +from tests import fake_llm +from tests.api._assist import Assist, sse, start_llm + +USAGE = {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110} + + +@pytest.fixture +def llm(cfg): + srv = start_llm(cfg) + cfg["telemetry"]["prices"] = {"fake": {"input": 1.0, "output": 2.0}} # USD per million tokens + fake_llm.Handler.usage = USAGE + yield fake_llm.Handler + fake_llm.Handler.usage = None + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, llm): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def plain(cfg): + """The plain search-and-answer path (no tools).""" + cfg["ai"]["tools"] = False + + +def tool(i, name, args): + return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}} + + +def search(i): + return {"content": "", "tool_calls": [tool(i, "search_transcripts", {"query": f"capsid {i}"})]} + + +def kept(d): + """A record without the fields it doesn't have (the API lists them as null).""" + return {k: kept(v) if isinstance(v, dict) else v for k, v in d.items() if v is not None} if isinstance(d, dict) else d + + +def ask(c, h, q, **body): + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + raw = c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": q, **body}).text + return cid, sse(raw), raw + + +def test_an_answer_records_what_produced_it(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + # one round of two calls, the second outside what the editor can read, then the answer + llm.tool_script = [ + { + "content": "", + "tool_calls": [ + tool(1, "search_transcripts", {"query": "Dyno Therapeutics"}), + tool(2, "read_transcript", {"recording_id": s.call}), + ], + }, + {"content": "Dyno Therapeutics designs capsids with machine learning [1]."}, + ] + cid, ev, raw = ask(c, h, "What does Dyno do?") + meta = ev["meta"][0] + assert raw.index("event: meta") < raw.index("event: done") # the record comes just before the end + assert (meta["model"], meta["provider"], meta["outcome"], meta["version"]) == ("fake", "127.0.0.1", "answered", __version__) + assert meta["tools"]["picked"] is None and {"search_transcripts", "read_transcript"} <= set(meta["tools"]["offered"]) + assert meta["limits"] == {"rounds": 6, "tool_calls": None} and meta["used"] == {"rounds": 1, "tool_calls": 2} + assert meta["flagged"] == [] and meta["research"] is None and isinstance(meta["ms"], int) and meta["ms"] >= 0 + assert "base_url" not in json.dumps(meta) and ":" not in meta["provider"] # the host name only, never the address + # each step says how long it took, and whether it failed + assert [isinstance(x["ms"], int) for x in ev["step"]] == [True, True] + assert "failed" not in ev["step"][0] and ev["step"][1]["failed"] is True + + msgs = c.get(f"/api/v1/chats/{cid}", headers=h).json() + mid = ev["done"][0]["message"] + answer, question = msgs["messages"][-1], msgs["messages"][-2] + assert answer["id"] == mid and kept(answer["meta"]) == kept(meta) and answer["model"] == "fake" # kept, with `model` as before + assert meta["scope"] == msgs["scope"] # the scope it answered in, after the conversation was narrowed + assert kept(question["meta"]) == {"via": "web", "options": {}} and question["id"] == mid - 1 + + # every model call made for it names it in the ledger, and its tokens and cost are theirs + one = c.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=h).json() + assert kept(one["meta"]) == kept(meta) and one["content"] == answer["content"] + calls = [r for r in one["ledger"] if r["kind"] == "out"] + assert sum(r["action"] == "model.chat" for r in calls) >= 2 + assert all({f"chat_message:{mid}", f"chat:{cid}"} <= set(r["resources"]) for r in calls) + assert meta["tokens"] == {"in": sum(r.get("tokens_in") or 0 for r in calls), "out": sum(r.get("tokens_out") or 0 for r in calls)} + assert meta["tokens"]["in"] >= 200 and meta["cost_usd"] == pytest.approx(sum(r.get("cost_usd") or 0 for r in calls)) + assert meta["cost_usd"] > 0 and meta["cost_estimated"] is False + # yours alone: the message's activity too + assert c.get("/api/v1/activity", headers=h, params={"resource": f"chat_message:{mid}"}).status_code == 200 + cv, hv = s.cl["viewer"] + assert cv.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=hv).status_code == 404 + assert cv.get("/api/v1/activity", headers=hv, params={"resource": f"chat_message:{mid}"}).status_code == 403 + assert c.get(f"/api/v1/chats/{cid}/messages/999999", headers=h).status_code == 404 + + # a question said in a voice conversation, and one asked with another model, say so + llm.tool_script = [{"content": "Hello!"}] + cid, ev, _ = ask(c, h, "zzqx", via="voice", model="fake-large") + question = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-2] + assert kept(question["meta"]) == {"via": "voice", "options": {"model": "fake-large"}} and ev["meta"][0]["model"] == "fake-large" + # answered without looking anything up + assert (ev["meta"][0]["outcome"], ev["meta"][0]["used"]) == ("direct", {"rounds": 0, "tool_calls": 0}) + + +def test_how_an_answer_ended(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + # its rounds ran out, so it answered from what it found + llm.tool_script = [search(i) for i in range(6)] + [{"content": "The capsid model won [1]."}] + _, ev, _ = ask(c, h, "Would you hire them?") + assert (ev["meta"][0]["outcome"], ev["meta"][0]["used"]) == ("limit", {"rounds": 6, "tool_calls": 6}) + # the model can't use tools: from a search instead + llm.reject_tools = True + _, ev, _ = ask(c, h, "capsid") + meta = ev["meta"][0] + assert "notice" in ev and meta["outcome"] == "fallback" and meta["tokens"]["out"] >= USAGE["completion_tokens"] + llm.reject_tools = False + + +def test_stopped_and_failed_answers_are_recorded(plain, app, db, cfg, folder, new_client, llm, monkeypatch): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + monkeypatch.setattr(chat.Answering, "EVERY", 0) + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + stream = llm_mod.stream_chat + + def stopped(*a, **k): # the person presses Stop after the first piece + pieces = stream(*a, **k) + yield next(pieces) + chat.request_stop(db, cid) + yield from pieces + + monkeypatch.setattr(llm_mod, "stream_chat", stopped) + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "When does the shipment leave?"}).text) + meta = ev["meta"][0] + assert "stopped" in ev and meta["outcome"] == "stopped" and meta["tools"]["offered"] == [] and meta["limits"]["rounds"] is None + # the call that was cut off still counts for it (its row is written when the stream is closed) + one = c.get(f"/api/v1/chats/{cid}/messages/{ev['done'][0]['message']}", headers=h).json() + assert kept(one["meta"]) == kept(meta) and "model.chat" in [r["action"] for r in one["ledger"]] + + def broken(*_a, **_k): + raise llm_mod.LLMError("429 from the LLM server") + yield # a generator, like the real one + + monkeypatch.setattr(llm_mod, "stream_chat", broken) + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Once more?"}).text) + assert ev["meta"][0]["outcome"] == "error" and ev["error"][0]["message"] == "429 from the LLM server" + # with no model set up, the best excerpts come back on their own + assert c.put("/api/v1/settings/llm", headers=s.cl["admin"][1], json={"base_url": None}).status_code == 200 + ev = sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "capsid"}).text) + meta = ev["meta"][0] + assert (meta["outcome"], meta["model"], meta["provider"], meta["tokens"]) == ("fallback", None, None, {"in": 0, "out": 0}) + + +def test_old_messages_have_no_record(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + chat.add(db, cid, "user", "From before") + mid = chat.add(db, cid, "assistant", "An answer from before", [], steps=[{"tool": "search_transcripts", "args": {}, "summary": "x"}]) + got = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"] + assert [m["meta"] for m in got] == [None, None] and got[-1]["steps"][0].get("ms") is None + one = c.get(f"/api/v1/chats/{cid}/messages/{mid}", headers=h).json() + assert one["meta"] is None and one["ledger"] == [] diff --git a/fastapi_backend/tests/api/test_completions.py b/fastapi_backend/tests/api/test_completions.py index dfea013cc..88a7f02bf 100644 --- a/fastapi_backend/tests/api/test_completions.py +++ b/fastapi_backend/tests/api/test_completions.py @@ -180,6 +180,13 @@ def tool(i, name, args): assert got["choices"][0]["message"]["content"].startswith("Dyno Therapeutics comes up on the pods [1].\n\nSources:") found = got["lens"]["sources"] assert found and {s["recording_id"] for s in found} <= {ep1, ep2} # the search never left pods + record = got["lens"]["record"] # what produced it, as a chat answer's record (no message keeps it: no tokens) + assert (record["outcome"], record["used"], record["tools"]) == ( + "answered", + {"rounds": 1, "tool_calls": 2}, + {"picked": ["search_transcripts"], "offered": ["search_transcripts"]}, + ) + assert record["scope"] == {"namespaces": ["pods"]} and "tokens" not in record result = next(m["content"] for m in llm.seen[1]["messages"] if m["role"] == "tool") assert "Dyno Therapeutics" in result and "leaves on Friday" not in result # calls is another namespace assert result.startswith("<") and 'source="tool search_transcripts"' in result # read as data (untrusted.wrap) diff --git a/fastapi_backend/tests/api/test_prompt_injection.py b/fastapi_backend/tests/api/test_prompt_injection.py index e4163caa4..c6e09dd59 100644 --- a/fastapi_backend/tests/api/test_prompt_injection.py +++ b/fastapi_backend/tests/api/test_prompt_injection.py @@ -266,6 +266,7 @@ def test_a_result_that_reads_like_instructions_is_flagged(app, db, cfg, folder, why = "reads like instructions to an AI: ignore previous instructions; new instructions or role" assert f"({why};" in ev["step"][0]["summary"] and "chat control tokens)" in ev["step"][0]["summary"] assert f'source="tool read_transcript" warning="{why};' in _tool_messages(llm)[0] + assert ev["meta"][0]["flagged"] == ["read_transcript"] # the answer's record names it too # and admins in the audit log: the reasons, never the text logged = [r for r in db.rows("SELECT * FROM audit_log") if r["action"] == "assistant.injection_flagged"] assert [(r["target"], r["detail"]["tool"]) for r in logged] == [(f"chat:{cid}", "read_transcript")] diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py index 284bfddb4..55ed02f90 100644 --- a/fastapi_backend/tests/fake_llm.py +++ b/fastapi_backend/tests/fake_llm.py @@ -49,7 +49,10 @@ def do_POST(self): if body.get("tool_choice") == "none" and not Handler.ignore_tool_choice: msg = {"content": msg.get("content")} return self._json( - {"choices": [{"message": {"role": "assistant", "content": msg.get("content"), "tool_calls": msg.get("tool_calls")}}]} + { + "choices": [{"message": {"role": "assistant", "content": msg.get("content"), "tool_calls": msg.get("tool_calls")}}], + **({"usage": Handler.usage} if Handler.usage else {}), + } ) if body.get("stream"): self.send_response(200) diff --git a/nextjs-frontend/__tests__/chat-details.test.tsx b/nextjs-frontend/__tests__/chat-details.test.tsx new file mode 100644 index 000000000..c280d72d1 --- /dev/null +++ b/nextjs-frontend/__tests__/chat-details.test.tsx @@ -0,0 +1,168 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Activity, Chats } from "@/app/openapi-client"; +import type { MessageRecord } from "@/app/openapi-client/types.gen"; +import { AnswerDetails, DetailsBody } from "@/components/chat/details"; +import { took } from "@/components/chat/steps"; +import { applyEvent, newTurn, savedSteps } from "@/components/chat/stream"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +jest.mock("@/app/openapi-client", () => ({ + Chats: { getMessage: jest.fn() }, + Activity: { resourceCosts: jest.fn() }, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); + +const META: MessageRecord = { + model: "qwen3:8b", + provider: "ollama.lan", + tools: { picked: null, offered: ["search_transcripts", "read_transcript", "web_read"] }, + scope: { namespaces: ["pods"] }, + limits: { rounds: 6, tool_calls: null }, + used: { rounds: 2, tool_calls: 3 }, + outcome: "limit", + tokens: { in: 5120, out: 410 }, + cost_usd: 0.0031, + cost_estimated: true, + ms: 8200, + flagged: ["web_read"], + version: "0.13.0", +}; + +function wide(matches: boolean) { + window.matchMedia = ((q: string) => ({ + matches: q.includes("max-width") ? !matches : matches, + addEventListener: () => {}, + removeEventListener: () => {}, + })) as unknown as typeof window.matchMedia; +} + +function show(ui: React.ReactNode) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + {ui} + , + ); +} + +beforeEach(() => { + jest.clearAllMocks(); + wide(true); + (Chats.getMessage as jest.Mock).mockImplementation(() => + ok({ + id: 8, + role: "assistant", + content: "x", + ledger: [ + { + id: "a1", + at: "2026-10-16T10:00:00Z", + kind: "out", + action: "model.chat", + model: "qwen3:8b", + tokens_in: 4000, + tokens_out: 300, + ms: 5100, + ok: true, + }, + { id: "a2", at: "2026-10-16T10:00:01Z", kind: "in", action: "POST /api/v1/chats/{cid}/messages", ok: true }, + ], + }), + ); + (Activity.resourceCosts as jest.Mock).mockImplementation(() => + ok({ costs: { "chat:3": { cost_usd: 0.02, tokens: 9000, calls: 4, unpriced: 0, estimate: false } } }), + ); +}); + +describe("an answer's record", () => { + it("says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended", async () => { + show( + , + ); + expect(screen.getByText("Its tool rounds ran out, so it answered from what it found")).toBeInTheDocument(); + expect(screen.getByText("qwen3:8b")).toBeInTheDocument(); + expect(screen.getByText(/on ollama\.lan/)).toBeInTheDocument(); + expect(screen.getByText("pods")).toBeInTheDocument(); + expect(screen.getByText("3 tools offered")).toBeInTheDocument(); + expect(screen.getByText(/2 of 6 rounds, 3 tool calls/)).toBeInTheDocument(); + expect(screen.getByText("search_transcripts, web_read")).toBeInTheDocument(); + expect(screen.getByText(/web_read: what it read looked like instructions to an AI/)).toBeInTheDocument(); + expect(screen.getByText("5.1k in · 410 out")).toBeInTheDocument(); + expect(screen.getByText("(estimated from the prices in Settings)")).toBeInTheDocument(); + expect(screen.getByText("≈")).toBeInTheDocument(); // some calls weren't priced: at least this + expect(screen.getByText("8.2 s")).toBeInTheDocument(); + // the calls made for it, from the ledger (calls out only) + expect(await screen.findByText(/Model call · qwen3:8b/)).toBeInTheDocument(); + expect(screen.queryByText(/POST \/api\/v1/)).not.toBeInTheDocument(); + expect(Chats.getMessage).toHaveBeenCalledWith(expect.objectContaining({ path: { cid: 3, mid: 8 } })); + expect(await screen.findByText("$0.02")).toBeInTheDocument(); // the conversation so far + }); + + it("says an old answer wasn't recorded, and asks the ledger nothing", () => { + show(); + expect(screen.getByText(/from before Lens recorded what produced each one/)).toBeInTheDocument(); + expect(Chats.getMessage).not.toHaveBeenCalled(); + }); + + it("an answer without tools or a price says so", () => { + const plain: MessageRecord = { + ...META, + tools: { picked: null, offered: [] }, + limits: { rounds: null, tool_calls: null }, + used: { rounds: 0, tool_calls: 0 }, + outcome: "answered", + cost_usd: 0, + cost_estimated: false, + flagged: [], + }; + show(); + expect(screen.getByText("None offered")).toBeInTheDocument(); + expect(screen.getByText("No tools")).toBeInTheDocument(); + expect(screen.getByText("Not costed: no price is set for its model")).toBeInTheDocument(); + expect(screen.queryByText("Flagged")).not.toBeInTheDocument(); + }); + + it("opens from the i in the answer's header, by keyboard too; on a phone in a sheet", async () => { + show(); + const button = screen.getByRole("button", { name: "Details: what produced this answer" }); + button.focus(); + fireEvent.click(button); + expect(await screen.findByText("What produced this answer")).toBeInTheDocument(); + expect(screen.getByText("qwen3:8b")).toBeInTheDocument(); + + wide(false); + show(); + fireEvent.click(screen.getAllByRole("button", { name: "Details: what produced this answer" }).at(-1)!); + expect(await screen.findByRole("dialog")).toHaveTextContent("What produced this answer"); + expect(screen.getByRole("dialog")).toHaveTextContent(/from before Lens recorded/); + }); +}); + +describe("the record in the stream", () => { + it("keeps the meta event, and each step's time and failure", () => { + let s = newTurn("q"); + s = applyEvent(s, { + event: "step", + data: '{"tool":"read_transcript","args":{},"summary":"read_transcript: not found","ms":40,"failed":true}', + }); + s = applyEvent(s, { event: "meta", data: JSON.stringify(META) }); + s = applyEvent(s, { event: "done", data: '{"message":8}' }); + expect(s.steps[0]).toMatchObject({ ms: 40, failed: true }); + expect(s.meta).toMatchObject({ outcome: "limit", provider: "ollama.lan" }); + expect(savedSteps([{ tool: "x" }])[0]).toEqual({ tool: "x", args: {}, summary: "" }); + expect([took(120), took(1400), took(12500)]).toEqual(["120 ms", "1.4 s", "13 s"]); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index b5db04b3b..c82ba439f 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1218,6 +1218,11 @@ export type { GetMediaError, GetMediaErrors, GetMediaResponses, + GetMessageData, + GetMessageError, + GetMessageErrors, + GetMessageResponse, + GetMessageResponses, GetMonitoringData, GetMonitoringResponse, GetMonitoringResponses, @@ -2147,6 +2152,8 @@ export type { MergeTopicsResponse, MergeTopicsResponses, MessageCreate, + MessageDetail, + MessageRecord, MetadataBulk, MetadataBulkResult, MetadataEdit, @@ -2515,6 +2522,11 @@ export type { RecordingTopicsResponse, RecordingTopicsResponses, RecordingUpdate, + RecordLimits, + RecordOptions, + RecordTokens, + RecordTools, + RecordUse, RedeemTicketData, RedeemTicketError, RedeemTicketErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 552691e6a..eb08ca52b 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -577,6 +577,9 @@ import type { GetMediaData, GetMediaErrors, GetMediaResponses, + GetMessageData, + GetMessageErrors, + GetMessageResponses, GetMonitoringData, GetMonitoringResponses, GetNamespaceAssistantData, @@ -9971,12 +9974,29 @@ export class Chats { }); } + /** + * Get Message + * + * One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and + * used, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`: + * the calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}. + */ + public static getMessage( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/chats/{cid}/messages/{mid}", + ...options, + }); + } + /** * Send Message * * Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model * reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST - * /chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id). + * /chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced + * it), done (the saved message id). * * With `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and * its new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 06ed72de5..01e1de85c 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -3618,6 +3618,10 @@ export type ChatMessage = { * the model that wrote the answer */ model?: string | null; + /** + * what produced it; none on messages from before it was recorded + */ + meta?: MessageRecord | null; [key: string]: unknown; }; @@ -10656,6 +10660,182 @@ export type MessageCreate = { * asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the assistant can use that browser (docs/browser-assistant.md) */ browser?: string | null; + /** + * Via + * + * voice: said in a voice conversation (the question's record keeps it) + */ + via?: "voice" | null; +}; + +/** + * MessageDetail + */ +export type MessageDetail = { + /** + * Id + */ + id: number; + /** + * Role + */ + role: "user" | "assistant"; + /** + * Content + */ + content: string; + /** + * a question asked from a page: the page, and any highlighted text + */ + context?: SharedContext | null; + /** + * Attachments + * + * files sent with it (POST /uploads with hold) + */ + attachments?: Array; + /** + * Passages + */ + passages?: Array | null; + /** + * Created At + */ + created_at?: string | null; + /** + * Stopped + * + * the answer was stopped (POST /chats/{cid}/stop): `content` is what came before + */ + stopped?: boolean; + /** + * Steps + * + * the tools the assistant used for this answer, in order + */ + steps?: Array; + /** + * Notice + * + * e.g. the model can't use tools, so the answer came from a search + */ + notice?: string | null; + /** + * Error + * + * why there's no answer + */ + error?: string | null; + /** + * its latest source check (POST .../messages/{mid}/check) + */ + check?: AnswerCheck | null; + /** + * Model + * + * the model that wrote the answer + */ + model?: string | null; + /** + * what produced it; none on messages from before it was recorded + */ + meta?: MessageRecord | null; + /** + * Ledger + * + * its rows in the activity ledger: the model and tool calls made for it, newest first + */ + ledger?: Array; + [key: string]: unknown; +}; + +/** + * MessageRecord + * + * What produced a message, recorded as it was written (docs/assistant.md, "What each answer records"). A question's + * has `via` and `options`; an answer's the rest. Messages from before it was recorded have none. + */ +export type MessageRecord = { + /** + * Via + * + * a question: how it was sent + */ + via?: "web" | "api" | "room" | "voice" | "automation" | "completions" | null; + /** + * a question: the options it was asked with + */ + options?: RecordOptions | null; + /** + * Model + * + * the model that wrote the answer + */ + model?: string | null; + /** + * Provider + * + * the host name of the model server it ran on (never its address or key) + */ + provider?: string | null; + tools?: RecordTools | null; + /** + * Scope + * + * the scope it answered in, after a conversation over everything was narrowed + */ + scope?: { + [key: string]: unknown; + } | null; + limits?: RecordLimits | null; + used?: RecordUse | null; + /** + * Outcome + * + * answered (after looking things up), direct (without looking anything up), limit (its rounds ran out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set up), stopped, or error + */ + outcome?: "answered" | "direct" | "limit" | "fallback" | "stopped" | "error" | null; + /** + * the tokens of the model calls made for it (the activity ledger) + */ + tokens?: RecordTokens | null; + /** + * Cost Usd + * + * what those calls cost, from the prices in Settings + */ + cost_usd?: number | null; + /** + * Cost Estimated + * + * some calls had a price but no token counts: the cost is at least this + */ + cost_estimated?: boolean | null; + /** + * Ms + * + * how long it took to write, from the question + */ + ms?: number | null; + /** + * Flagged + * + * tools whose results read like instructions to an AI + */ + flagged?: Array | null; + /** + * Research + * + * the research it belongs to + */ + research?: number | null; + /** + * Version + * + * the version of Lens that wrote it + */ + version?: string | null; + [key: string]: unknown; }; /** @@ -15424,6 +15604,105 @@ export type Reading = { [key: string]: unknown; }; +/** + * RecordLimits + */ +export type RecordLimits = { + /** + * Rounds + * + * rounds of tool calls it had (calls in one round run together) + */ + rounds?: number | null; + /** + * Tool Calls + * + * tool calls it had; null: only the rounds limit them + */ + tool_calls?: number | null; + [key: string]: unknown; +}; + +/** + * RecordOptions + * + * What a question was asked with (only what it set). + */ +export type RecordOptions = { + /** + * Model + */ + model?: string | null; + /** + * Tools + */ + tools?: Array | null; + /** + * Max Tool Calls + */ + max_tool_calls?: number | null; + /** + * Scope + */ + scope?: { + [key: string]: unknown; + } | null; + /** + * Research + */ + research?: boolean | null; + [key: string]: unknown; +}; + +/** + * RecordTokens + */ +export type RecordTokens = { + /** + * In + */ + in?: number; + /** + * Out + */ + out?: number; + [key: string]: unknown; +}; + +/** + * RecordTools + */ +export type RecordTools = { + /** + * Picked + * + * the tools or groups the question asked for; null: Auto + */ + picked?: Array | null; + /** + * Offered + * + * every tool the model was offered + */ + offered?: Array; + [key: string]: unknown; +}; + +/** + * RecordUse + */ +export type RecordUse = { + /** + * Rounds + */ + rounds?: number; + /** + * Tool Calls + */ + tool_calls?: number; + [key: string]: unknown; +}; + /** * Recording * @@ -20726,6 +21005,18 @@ export type ToolStep = { * Summary */ summary?: string; + /** + * Ms + * + * how long the tool took + */ + ms?: number | null; + /** + * Failed + * + * the tool failed (or was unknown or blocked); its summary says why + */ + failed?: boolean | null; [key: string]: unknown; }; @@ -38306,6 +38597,40 @@ export type UpdateChatResponses = { export type UpdateChatResponse = UpdateChatResponses[keyof UpdateChatResponses]; +export type GetMessageData = { + body?: never; + path: { + /** + * Cid + */ + cid: number; + /** + * Mid + */ + mid: number; + }; + query?: never; + url: "/api/v1/chats/{cid}/messages/{mid}"; +}; + +export type GetMessageErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetMessageError = GetMessageErrors[keyof GetMessageErrors]; + +export type GetMessageResponses = { + /** + * Successful Response + */ + 200: MessageDetail; +}; + +export type GetMessageResponse = GetMessageResponses[keyof GetMessageResponses]; + export type SendMessageData = { body: MessageCreate; path: { diff --git a/nextjs-frontend/components/chat/answer.tsx b/nextjs-frontend/components/chat/answer.tsx index 1775be1f4..130f52154 100644 --- a/nextjs-frontend/components/chat/answer.tsx +++ b/nextjs-frontend/components/chat/answer.tsx @@ -6,7 +6,7 @@ import Link from "next/link"; import { useMemo } from "react"; import { Chats, Search } from "@/app/openapi-client"; -import type { AnswerCheck, Passage } from "@/app/openapi-client/types.gen"; +import type { AnswerCheck, MessageRecord, Passage } from "@/app/openapi-client/types.gen"; import { ApprovalCard, type ApprovalView } from "@/components/chat/approval"; import { citedNumbers, @@ -17,6 +17,7 @@ import { saysNotCovered, } from "@/components/chat/cite"; import { CitationChip } from "@/components/chat/citation"; +import { AnswerDetails } from "@/components/chat/details"; import { keywords } from "@/components/chat/keywords"; import { RichText } from "@/components/chat/rich-text"; import { scopeWords, type Scope } from "@/components/chat/scope"; @@ -145,6 +146,8 @@ export type AnswerProps = { active?: boolean; check?: AnswerCheck | null; onChecked?: (c: AnswerCheck) => void; + /** What produced it (the answer's record); older answers have none. */ + meta?: MessageRecord | null; }; /** One answer: where it comes from, the tools used, approvals, the text with citations, and its error states. */ @@ -191,6 +194,9 @@ export function Answer(p: AnswerProps) { ? `From ${from}` : "Answer"} {p.model ? ` · ${p.model}` : ""} + {!streaming && p.messageId != null && ( + + )} {p.onShowSources && p.passages && p.passages.length > 0 && ( + ); + if (narrow) + return ( + <> + {trigger} + + + + + ); + return ( + + {trigger} + + What produced this answer + + + + ); +} diff --git a/nextjs-frontend/components/chat/steps.tsx b/nextjs-frontend/components/chat/steps.tsx index fd0af6992..95214da36 100644 --- a/nextjs-frontend/components/chat/steps.tsx +++ b/nextjs-frontend/components/chat/steps.tsx @@ -42,10 +42,16 @@ type Kind = "done" | "failed" | "waiting"; function kindOf(step: ToolStep): Kind { if (/^asked for approval/i.test(step.summary)) return "waiting"; - if (step.summary.startsWith(`${step.tool}:`) || /^unknown tool/i.test(step.summary)) return "failed"; + // answers record whether a step failed; older ones are read from the summary + if (step.failed || step.summary.startsWith(`${step.tool}:`) || /^unknown tool/i.test(step.summary)) return "failed"; return "done"; } +/** How long a step took: "120 ms", "1.4 s". */ +export function took(ms: number): string { + return ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(ms < 10_000 ? 1 : 0)} s`; +} + const GLYPH: Record = { done: { g: "✓", bg: "var(--aladdin-green)", label: "done" }, failed: { g: "✕", bg: "var(--aladdin-red)", label: "failed" }, @@ -83,6 +89,7 @@ function StepRow({ step, index }: { step: ToolStep; index: number }) { {g.label}: {title} + {step.ms != null && {took(step.ms)}} {open ? ( ) : ( diff --git a/nextjs-frontend/components/chat/stream.ts b/nextjs-frontend/components/chat/stream.ts index a4f6c837e..5d2cf420e 100644 --- a/nextjs-frontend/components/chat/stream.ts +++ b/nextjs-frontend/components/chat/stream.ts @@ -1,16 +1,20 @@ -import type { Estimate, Passage } from "@/app/openapi-client/types.gen"; +import type { Estimate, MessageRecord, Passage } from "@/app/openapi-client/types.gen"; /** * One question's answer as it streams from POST /chats/{id}/messages. The backend sends server-sent events: * `scoped` (a conversation over everything narrowed to the namespace it's about), `suggested` (namespaces to offer when that isn't clear), `step` (a tool the assistant used), `approval` (work waiting for the person), `notice`, `passages` (the numbered - * excerpts), `token` (answer text), `error`, `stopped` (Stop was pressed: what came is saved, marked stopped) and - * `done` (the saved message id). + * excerpts), `token` (answer text), `error`, `stopped` (Stop was pressed: what came is saved, marked stopped), `meta` + * (the answer's record: what produced it) and `done` (the saved message id). */ export type ToolStep = { tool: string; args: Record; summary: string; + /** How long the tool took. */ + ms?: number | null; + /** It failed (or was unknown or blocked); the summary says why. */ + failed?: boolean | null; }; export type PendingApproval = { id: number; @@ -37,6 +41,8 @@ export type TurnState = { text: string; error: string | null; messageId: number | null; + /** What produced the answer, once it's saved. */ + meta: MessageRecord | null; }; export function newTurn(question: string): TurnState { @@ -52,6 +58,7 @@ export function newTurn(question: string): TurnState { text: "", error: null, messageId: null, + meta: null, }; } @@ -79,6 +86,8 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T tool: str(o.tool, "tool"), args: (o.args as Record) ?? {}, summary: str(o.summary), + ms: typeof o.ms === "number" ? o.ms : null, + failed: o.failed === true || null, }, ], }; @@ -121,6 +130,8 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T }; case "stopped": return { ...s, status: "stopped" }; + case "meta": + return { ...s, meta: d && !Array.isArray(d) ? (o as MessageRecord) : null }; case "done": return { ...s, @@ -134,9 +145,18 @@ export function applyEvent(s: TurnState, ev: { event: string; data: string }): T /** Tool steps as an answer was saved with them (GET /chats/{id}), in the shape the live stream gives. */ export function savedSteps( - steps: { tool: string; args?: Record; summary?: string }[] | null | undefined, + steps: + | { tool: string; args?: Record; summary?: string; ms?: number | null; failed?: boolean | null }[] + | null + | undefined, ): ToolStep[] { - return (steps ?? []).map((s) => ({ tool: s.tool, args: s.args ?? {}, summary: s.summary ?? "" })); + return (steps ?? []).map((s) => ({ + tool: s.tool, + args: s.args ?? {}, + summary: s.summary ?? "", + ...(s.ms != null ? { ms: s.ms } : {}), + ...(s.failed ? { failed: true } : {}), + })); } /** A short plain-words line for a tool call, when the backend's summary is missing. */ diff --git a/nextjs-frontend/components/costs/costs.tsx b/nextjs-frontend/components/costs/costs.tsx index d4181c2ea..af318d8c9 100644 --- a/nextjs-frontend/components/costs/costs.tsx +++ b/nextjs-frontend/components/costs/costs.tsx @@ -124,7 +124,7 @@ const KIND: Record change: { label: "Change", icon: , tone: "gate" }, }; -function actionText(e: ActivityEntry): string { +export function actionText(e: ActivityEntry): string { const a = e.action; if (a.startsWith("model.")) return `${a.slice(6) === "chat" ? "Model call" : `Model ${a.slice(6)}`}${e.model ? ` · ${e.model}` : ""}`; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 285f91af5..cac3b647e 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -24952,13 +24952,65 @@ } } }, + "/api/v1/chats/{cid}/messages/{mid}": { + "get": { + "tags": [ + "chats" + ], + "summary": "Get Message", + "description": "One message with its record (`meta`: for an answer, the model and the host it ran on, the tools offered and\nused, the scope, limits and use, tokens, cost, time and how it ended) and its rows in the activity ledger (`ledger`:\nthe calls made for it, newest first). Citations follow your current access, as in GET /chats/{cid}.", + "operationId": "get_message", + "parameters": [ + { + "name": "cid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Cid" + } + }, + { + "name": "mid", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Mid" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MessageDetail" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/chats/{cid}/messages": { "post": { "tags": [ "chats" ], "summary": "Send Message", - "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id).\n\nWith `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and\nits new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked\nfrom (where, and the highlighted text; not the page's text, which isn't kept).\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).", + "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nWith `edit`, one of your earlier questions is edited: it and everything after it are replaced by this question and\nits new answer (404 if it isn't a question in this conversation). Without `context`, it keeps the page it was asked\nfrom (where, and the highlighted text; not the page's text, which isn't kept).\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).", "operationId": "send_message", "parameters": [ { @@ -36383,6 +36435,17 @@ ], "title": "Model", "description": "the model that wrote the answer" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/components/schemas/MessageRecord" + }, + { + "type": "null" + } + ], + "description": "what produced it; none on messages from before it was recorded" } }, "additionalProperties": true, @@ -48396,12 +48459,389 @@ ], "title": "Browser", "description": "asked from a browser's Lens side panel: its session (POST /browser/sessions). While it's connected, the assistant can use that browser (docs/browser-assistant.md)" + }, + "via": { + "anyOf": [ + { + "type": "string", + "const": "voice" + }, + { + "type": "null" + } + ], + "title": "Via", + "description": "voice: said in a voice conversation (the question's record keeps it)" } }, "additionalProperties": false, "type": "object", "title": "MessageCreate" }, + "MessageDetail": { + "properties": { + "id": { + "type": "integer", + "title": "Id" + }, + "role": { + "type": "string", + "enum": [ + "user", + "assistant" + ], + "title": "Role" + }, + "content": { + "type": "string", + "title": "Content" + }, + "context": { + "anyOf": [ + { + "$ref": "#/components/schemas/SharedContext" + }, + { + "type": "null" + } + ], + "description": "a question asked from a page: the page, and any highlighted text" + }, + "attachments": { + "items": { + "$ref": "#/components/schemas/Attachment" + }, + "type": "array", + "title": "Attachments", + "description": "files sent with it (POST /uploads with hold)" + }, + "passages": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Passage" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Passages" + }, + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "stopped": { + "type": "boolean", + "title": "Stopped", + "description": "the answer was stopped (POST /chats/{cid}/stop): `content` is what came before", + "default": false + }, + "steps": { + "items": { + "$ref": "#/components/schemas/ToolStep" + }, + "type": "array", + "title": "Steps", + "description": "the tools the assistant used for this answer, in order" + }, + "notice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Notice", + "description": "e.g. the model can't use tools, so the answer came from a search" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "why there's no answer" + }, + "check": { + "anyOf": [ + { + "$ref": "#/components/schemas/AnswerCheck" + }, + { + "type": "null" + } + ], + "description": "its latest source check (POST .../messages/{mid}/check)" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model", + "description": "the model that wrote the answer" + }, + "meta": { + "anyOf": [ + { + "$ref": "#/components/schemas/MessageRecord" + }, + { + "type": "null" + } + ], + "description": "what produced it; none on messages from before it was recorded" + }, + "ledger": { + "items": { + "$ref": "#/components/schemas/ActivityEntry" + }, + "type": "array", + "title": "Ledger", + "description": "its rows in the activity ledger: the model and tool calls made for it, newest first" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "role", + "content" + ], + "title": "MessageDetail" + }, + "MessageRecord": { + "properties": { + "via": { + "anyOf": [ + { + "type": "string", + "enum": [ + "web", + "api", + "room", + "voice", + "automation", + "completions" + ] + }, + { + "type": "null" + } + ], + "title": "Via", + "description": "a question: how it was sent" + }, + "options": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordOptions" + }, + { + "type": "null" + } + ], + "description": "a question: the options it was asked with" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model", + "description": "the model that wrote the answer" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Provider", + "description": "the host name of the model server it ran on (never its address or key)" + }, + "tools": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordTools" + }, + { + "type": "null" + } + ] + }, + "scope": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Scope", + "description": "the scope it answered in, after a conversation over everything was narrowed" + }, + "limits": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordLimits" + }, + { + "type": "null" + } + ] + }, + "used": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordUse" + }, + { + "type": "null" + } + ] + }, + "outcome": { + "anyOf": [ + { + "type": "string", + "enum": [ + "answered", + "direct", + "limit", + "fallback", + "stopped", + "error" + ] + }, + { + "type": "null" + } + ], + "title": "Outcome", + "description": "answered (after looking things up), direct (without looking anything up), limit (its rounds ran out, so it answered from what it found), fallback (from a search: the model can't use tools, or none is set up), stopped, or error" + }, + "tokens": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecordTokens" + }, + { + "type": "null" + } + ], + "description": "the tokens of the model calls made for it (the activity ledger)" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd", + "description": "what those calls cost, from the prices in Settings" + }, + "cost_estimated": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Cost Estimated", + "description": "some calls had a price but no token counts: the cost is at least this" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms", + "description": "how long it took to write, from the question" + }, + "flagged": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Flagged", + "description": "tools whose results read like instructions to an AI" + }, + "research": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Research", + "description": "the research it belongs to" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version", + "description": "the version of Lens that wrote it" + } + }, + "additionalProperties": true, + "type": "object", + "title": "MessageRecord", + "description": "What produced a message, recorded as it was written (docs/assistant.md, \"What each answer records\"). A question's\nhas `via` and `options`; an answer's the rest. Messages from before it was recorded have none." + }, "MetadataBulk": { "properties": { "recordings": { @@ -56531,6 +56971,168 @@ ], "title": "Reading" }, + "RecordLimits": { + "properties": { + "rounds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rounds", + "description": "rounds of tool calls it had (calls in one round run together)" + }, + "tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tool Calls", + "description": "tool calls it had; null: only the rounds limit them" + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordLimits" + }, + "RecordOptions": { + "properties": { + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls" + }, + "scope": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Scope" + }, + "research": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Research" + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordOptions", + "description": "What a question was asked with (only what it set)." + }, + "RecordTokens": { + "properties": { + "in": { + "type": "integer", + "title": "In", + "default": 0 + }, + "out": { + "type": "integer", + "title": "Out", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordTokens" + }, + "RecordTools": { + "properties": { + "picked": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Picked", + "description": "the tools or groups the question asked for; null: Auto" + }, + "offered": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Offered", + "description": "every tool the model was offered" + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordTools" + }, + "RecordUse": { + "properties": { + "rounds": { + "type": "integer", + "title": "Rounds", + "default": 0 + }, + "tool_calls": { + "type": "integer", + "title": "Tool Calls", + "default": 0 + } + }, + "additionalProperties": true, + "type": "object", + "title": "RecordUse" + }, "Recording": { "properties": { "id": { @@ -65680,6 +66282,30 @@ "type": "string", "title": "Summary", "default": "" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms", + "description": "how long the tool took" + }, + "failed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Failed", + "description": "the tool failed (or was unknown or blocked); its summary says why" } }, "additionalProperties": true, diff --git a/threat-model.md b/threat-model.md index 11822785b..c263b536d 100644 --- a/threat-model.md +++ b/threat-model.md @@ -426,6 +426,7 @@ stored text for the consumer's format. | Embed player and `/s/` pages | `api/pages.py` | HTML built from DB values under a strict CSP; framable only per `server.embed_frame_ancestors` | that titles are inert outside this page | *(documented, docs/configuration.md "Content-Security-Policy")* | | Report pages | `api/pages.py` | links re-signed on read for every asset of the namespace in the page | that a report link opens only the report | *(inferred, Q10)* | | Assistant answers, summaries, LLM-written notes | Assistant | model output over untrusted content | that it is grounded or free of injected instructions | *(inferred, Q26)* | +| An answer's record (`meta` on a chat message, `GET /chats/{cid}/messages/{mid}`) | Assistant | the model, the model server's host name (never its address, path, port or key), tool names, scope, counts, tokens, cost and the answer's ledger rows, to the conversation's owner alone | that the model server's host name stays hidden from people who chat | *(documented, docs/assistant.md "never the server's address, path or key")* | | MCP tool results | `api/mcp_tools.py` | the person's readable data, as text, with chat-template control tokens broken up and a note when text reads like instructions to an AI | that an asset's text is safe to follow as instructions | *(documented, docs/mcp.md "sees exactly what that person sees")* | | Error text from remote servers | LLM, speech, rclone, IMAP callers | up to 300 characters of the remote reply and the URL as typed | that internal host names stay hidden | *(inferred, Q18)* | | Webhook bodies | `notify.py` | event data, HMAC-signed for the `webhook` kind only | that Slack or Discord messages are signed | *(documented, domain/notify.py docstring "signed the Standard Webhooks way")* | diff --git a/threat-model.yaml b/threat-model.yaml index 8e3afce75..df167370c 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:d8bda21af9290dd0c018869ca7e0f3b4ce7d78cfcd77d304cb369cd716e42ec5" +prose_version: "threat-model.md@sha256:097d71b310b2fd012ac29bc101532fabad87c9fdd5dacdb9b165a4855d1d50ef" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} From acf97d754568bf7de30a219294e30c5c1c20b71c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:04:31 +0000 Subject: [PATCH 03/12] test(notes): keep the daily notes test from failing on 2026-10-17 The last check asked for 2026-10-17's note after the call without a date had made today's note, so on that day it found the note the deleted root had left behind. It now asks for a day no earlier call makes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/tests/api/test_notebook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastapi_backend/tests/api/test_notebook.py b/fastapi_backend/tests/api/test_notebook.py index 8f3d79e31..40e065d03 100644 --- a/fastapi_backend/tests/api/test_notebook.py +++ b/fastapi_backend/tests/api/test_notebook.py @@ -587,7 +587,7 @@ def test_daily_notes(client, env, db): # editors only, and a date that is one assert client.post("/api/v1/notes/today", headers=hv, json={"ns": "pods"}).status_code == 403 assert client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "soon"}).status_code == 400 - # the daily notes' note, when deleted, is made again the next time + # the daily notes' note, when deleted, is made again the next time (for a day no earlier call made: not today) assert client.delete(f"/api/v1/notes/{root['id']}", headers=he).status_code == 200 - again = client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "2026-10-17"}).json() + again = client.post("/api/v1/notes/today", headers=he, json={"ns": "pods", "date": "2020-02-29"}).json() assert again["parent"] not in (None, root["id"]) From 33a2c19079b6af9909a1cb695c0d52a505f17839 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:36:03 +0000 Subject: [PATCH 04/12] feat(chat): choose tools, a tool-call limit and a scope per answer Group the assistant's tools (GET /chats/tools lists the groups and tools a person has, what each does and why a group is missing), offer an answer only the tools picked (an admin's switch still wins), stop at a tool-call limit up to the new ai.max_tool_calls setting, and narrow one answer to its own namespaces. A conversation keeps its choice and new ones start with the last. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/app/api/v1/routes/chats.py | 108 +++++++- fastapi_backend/app/domain/actions_ai.py | 4 +- fastapi_backend/app/domain/ai_tools.py | 134 +++++++++- fastapi_backend/app/domain/chat.py | 67 ++++- fastapi_backend/app/domain/completions.py | 7 +- fastapi_backend/app/domain/extensions.py | 8 +- fastapi_backend/app/domain/settings.py | 4 + fastapi_backend/app/domain/store.py | 1 + fastapi_backend/app/schemas/chats.py | 72 +++++- fastapi_backend/tests/api/test_chat_tools.py | 244 ++++++++++++++++++ fastapi_backend/tests/api/test_chats.py | 3 +- .../components/settings/bodies.tsx | 2 + nextjs-frontend/components/settings/model.ts | 9 + 13 files changed, 626 insertions(+), 37 deletions(-) create mode 100644 fastapi_backend/tests/api/test_chat_tools.py diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py index 7f00f41ee..17a679cff 100644 --- a/fastapi_backend/app/api/v1/routes/chats.py +++ b/fastapi_backend/app/api/v1/routes/chats.py @@ -33,6 +33,7 @@ ChatCreate, ChatScope, ChatSummary, + ChatTools, ChatUpdate, HeadUpdate, MessageCreate, @@ -75,8 +76,8 @@ def _own_chat(db: DB, cid: int, user: Principal) -> dict[str, Any]: def list_chats(user: CurrentUser, db: Db) -> list[ChatSummary]: """Your conversations, most recent first.""" return db.rows( - "SELECT record::id(id) AS id, title, scope, model, kind ?? 'chat' AS kind, created_at, updated_at FROM chat WHERE account = $a " - "ORDER BY updated_at DESC LIMIT 200", + "SELECT record::id(id) AS id, title, scope, model, kind ?? 'chat' AS kind, tools, max_tool_calls, created_at, updated_at " + "FROM chat WHERE account = $a ORDER BY updated_at DESC LIMIT 200", a=user.id, ) @@ -92,11 +93,36 @@ def chat_capabilities(user: CurrentUser, cfg: Cfg) -> ChatCapabilities: model=cfg["llm"].get("model") if on else None, tools=on and bool(cfg["ai"].get("tools")), max_steps=int(cfg["ai"].get("max_steps") or 6), + max_tool_calls=int(cfg["ai"].get("max_tool_calls") or 50), check=on, models=chat.model_choices(cfg), ) +def _box(db: DB, cfg: dict[str, Any], user: Principal, acl: Access, scope: dict[str, Any] | None, request: Request) -> ai_tools.Toolbox: + """The assistant's tools as this person has them, in `scope`, before anything is asked.""" + return ai_tools.Toolbox( + db, cfg, user.as_audit(), set(acl.roles), set(acl.editable()), scope, None, request.app.state.archive.base, user.admin + ) + + +@router.get("/chats/tools") +def chat_tools(user: CurrentUser, acl: Acl, db: Db, cfg: Cfg, request: Request, chat_id: int | None = None) -> ChatTools: + """The tool groups and tools an answer can be given (`tools` on a question or a conversation), as you have them: each + tool with what it does (read, asks: it waits for your yes, out: it sends what it's given to another host) and + whether you have it now, each group with why you don't when you have none of it, and the setting that changes it. + Built-in groups first, then your extensions' (mcp:, ext:). With `chat_id`, as in that + conversation's scope (a namespace's own memory is there only in a conversation about that namespace). Also: the most + tool calls an answer may be given, and your latest choice, which new conversations start with.""" + scope = _own_chat(db, chat_id, user).get("scope") if chat_id is not None else None + box = _box(db, cfg, user, acl, scope, request) + return ChatTools( + groups=ai_tools.catalog(box), + max_tool_calls=int(cfg["ai"].get("max_tool_calls") or 50), + remembered=chat.remembered_tools(db, user.id), + ) + + def _model(cfg: dict[str, Any], model: str | None) -> str | None: try: return chat.check_model(cfg, model) @@ -104,12 +130,40 @@ def _model(cfg: dict[str, Any], model: str | None) -> str | None: raise HTTPException(400, str(e)) from None +def _tools(box: ai_tools.Toolbox, tools: list[str] | None) -> list[str] | None: + """A choice of tools, each a group or tool this person has (400 otherwise).""" + if tools is None: + return None + bad = box.unknown(tools) + if bad: + raise HTTPException(400, f"no tool or tool group {bad[0]!r} (GET /chats/tools lists them)") + return list(dict.fromkeys(tools)) + + +def _limit(cfg: dict[str, Any], n: int | None) -> int | None: + try: + return chat.tool_calls_allowed(cfg, n) + except ValueError as e: + raise HTTPException(400, str(e)) from None + + @router.post("/chats") -def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, body: ChatCreate | None = None) -> Created: +def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body: ChatCreate | None = None) -> Created: + """A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH + /chats/{cid}), else Auto: everything you may use, limited by the rounds alone.""" body = body or ChatCreate() if body.kind == "setup" and not user.admin: raise HTTPException(403, "only admins set up the server") - return Created(id=chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind)) + last = chat.remembered_tools(db, user.id) + tools = _tools(_box(db, cfg, user, acl, None, request), body.tools) if "tools" in body.model_fields_set else last["tools"] + limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in body.model_fields_set else _most(cfg, last["max_tool_calls"]) + cid = chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind, tools, limit) + return Created(id=cid) + + +def _most(cfg: dict[str, Any], n: int | None) -> int | None: + """A limit kept from before, within ai.max_tool_calls as it is now.""" + return None if n is None else max(1, min(int(n), int(cfg["ai"].get("max_tool_calls") or 50))) @router.get("/chats/{cid}") @@ -172,15 +226,23 @@ def get_message(cid: int, mid: int, user: CurrentUser, acl: Acl, db: Db) -> Mess @router.patch("/chats/{cid}") -def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Ok: - """Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one).""" - _own_chat(db, cid, user) +def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> Ok: + """Rename a conversation, change what it draws on, the model that answers in it (null: the configured one), or + what its answers may use (`tools`, null: Auto) and at most how many tool calls each makes (`max_tool_calls`, null: + no limit but the rounds). The choice of tools and limit is also what your new conversations start with.""" + c = _own_chat(db, cid, user) if "model" in body.model_fields_set: db.q("UPDATE $r SET model = $m", r=R("chat", cid), m=_model(cfg, body.model)) if body.title: db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=body.title[:120]) if "scope" in body.model_fields_set: # replaces the scope as a whole, so narrowing can also be removed db.q("UPDATE $r SET scope = $s", r=R("chat", cid), s=_scope(acl, db, body.scope)) + chose = {"tools", "max_tool_calls"} & body.model_fields_set + if chose: + tools = _tools(_box(db, cfg, user, acl, c.get("scope"), request), body.tools) if "tools" in chose else c.get("tools") + limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in chose else c.get("max_tool_calls") + db.q("UPDATE $r SET tools = $t, max_tool_calls = $n", r=R("chat", cid), t=tools, n=limit) + chat.remember_tools(db, user.id, tools, limit) return Ok() @@ -231,6 +293,11 @@ async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db text, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the new version, the message it follows and its versions); so does a question beside others under `parent`. + What the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a + search) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's + choice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its + record keeps what it was given and used. + `/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the conversation so far into its summary now; each streams cleared or compacted, then its reply as an answer. @@ -279,7 +346,15 @@ def run() -> Iterator[str]: model = await run_in_threadpool(_model, cfg, body.model) if body.model else c.get("model") if model and not body.model and model not in await run_in_threadpool(chat.model_choices, cfg): model = None - record = chat.Record(cfg, model) # what produced the answer, kept with it (from here: it times the answer) + # what this answer may use: its tools and at most how many calls (the conversation's, unless the question says), and + # what it draws on, when the question gives it a scope of its own (the conversation keeps its scope) + if body.tools is not None: + tools = await run_in_threadpool(lambda: _tools(_box(db, cfg, user, acl, c.get("scope"), request), body.tools)) + else: + tools = c.get("tools") + limit = _limit(cfg, body.max_tool_calls) if body.max_tool_calls is not None else _most(cfg, c.get("max_tool_calls")) + own_scope = await run_in_threadpool(_scope, acl, db, body.scope) if body.scope is not None else None + record = chat.Record(cfg, model, picked=tools, tool_calls=limit) # what produced the answer (from here: it times it) title = c["title"] page = body.context.model_dump() if body.context else None kept = chat.shared_context(page) # what the question keeps of the page @@ -316,7 +391,7 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: summary = chat.memory(db, cfg, cid, past) if again is None: via = "voice" if body.via == "voice" else "web" if user.via == "access" else "api" - meta = chat.question_record(via, model=body.model) + meta = chat.question_record(via, model=body.model, tools=tools, max_tool_calls=limit, scope=own_scope) question = chat.add(db, cid, "user", q, attachments=files, context=kept, parent=under, meta=meta) answer_id = db.next_id("chat_message") if again is not None: @@ -331,6 +406,8 @@ def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Any]: if not past and again is None and title == "New conversation": db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=q[:80]) + if own_scope is not None: # this answer's own scope, as asked: nothing is narrowed for it + return past, chat.retrieve(db, q, readable, own_scope, cfg=cfg), None # A conversation over everything gets the namespace its first question is about, when that's clear. if past or c.get("kind") == "setup" or c.get("scope"): return past, chat.retrieve(db, q, readable, c.get("scope"), cfg=cfg), None @@ -343,7 +420,8 @@ def draw_on(past: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict return past, chat.retrieve(db, q, readable, c["scope"], cfg=cfg), placed past, passages, placed = await run_in_threadpool(prepare) - record.scope = c.get("scope") or {} # after a conversation over everything was narrowed to what it's about + scope = own_scope if own_scope is not None else c.get("scope") # after one over everything was narrowed to what it's about + record.scope = scope or {} picked = placed[1] if placed and placed[0] == "scoped" else None suggested = placed[1] if placed and placed[0] == "suggest" else None @@ -374,7 +452,7 @@ def stopped(text: str, cited: list[dict[str, Any]]) -> Iterator[str]: def answer(on: chat.Answering) -> Iterator[str]: nonlocal notice - if llm.configured(cfg) and cfg["ai"].get("tools"): + if llm.configured(cfg) and cfg["ai"].get("tools") and tools != []: # given no tools, it answers from a search setup = c.get("kind") == "setup" box = ai_tools.Toolbox( db, @@ -382,7 +460,7 @@ def answer(on: chat.Answering) -> Iterator[str]: user.as_audit(), readable, set(acl.editable()), - c.get("scope"), + scope, cid, request.app.state.archive.base, user.admin, @@ -391,12 +469,16 @@ def answer(on: chat.Answering) -> Iterator[str]: # what it read before any tool ran: the page it was asked from, and earlier answers' excerpts and tools outside=(["page"] if page else []) + ai_tools.read_before(past, summary), browser=body.browser, + only=tools, ) try: answer, how = "", "answered" # setting the server up, or working in the person's browser, takes more steps than looking things up steps_max = max(cfg["ai"].get("max_steps") or 6, 12 if setup else 0, browser.steps(cfg) if box.browser else 0) - for kind, data in chat.tool_answer(cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record): + answering = chat.tool_answer( + cfg, box, asked, past, steps_max, model, setup=setup, summary=summary, record=record, max_tool_calls=limit + ) + for kind, data in answering: if kind == "step": steps.append(data) yield _ev("step", data) diff --git a/fastapi_backend/app/domain/actions_ai.py b/fastapi_backend/app/domain/actions_ai.py index dc37e1cb5..0f7ba75fb 100644 --- a/fastapi_backend/app/domain/actions_ai.py +++ b/fastapi_backend/app/domain/actions_ai.py @@ -111,12 +111,10 @@ class Box(ai_tools.Toolbox): would write become approvals (the chat assistant writes them at once), what could carry data out of Lens asks first whatever the run read (wary), and it keeps no memories.""" - only: set | None = None note = "" def specs(self): - got = [s for s in super().specs() if s["function"]["name"] not in LEFT_OUT] - return got if self.only is None else [s for s in got if s["function"]["name"] in self.only] + return [s for s in super().specs() if s["function"]["name"] not in LEFT_OUT] def system_note(self): return super().system_note() + self.note diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 63151f162..cd8fb2385 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -475,6 +475,117 @@ def builtin_names(): "decline_approval", } | {t[0] for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS + MEMORY_TOOLS + AUTOMATION_TOOLS if t[4]} +# built-in tools that carry what the model gives them to another host; after an answer read text from outside Lens, +# they ask first (Toolbox.wary). The side panel's browser tools have their own guard (browser.py): the ones that act +# on a web site are labelled as sending data out (BROWSER_OUT), and ask as browser.py says. +SENDS_OUT: set[str] = set() +BROWSER_OUT = {"browser_open", "browser_click", "browser_type", "browser_press"} + +# The groups of tools a question can be given (POST /chats/{cid}/messages `tools`; docs/assistant.md, "Choosing tools, +# namespaces and a limit"), in the order the web app shows them, each built-in tool in one. Extensions add their own: +# mcp: for an MCP server's tools, ext: for another extension's (extensions.Active.groups). +GROUPS = { + "archive": { + "label": "Archive", + "tools": ["search_transcripts", "find_similar", "list_recordings", "read_transcript", "recording_outputs"], + }, + "graph": { + "label": "Graph", + "tools": [ + "find_entities", + "entity_mentions", + "entity_timeline", + "find_topics", + "topic_recordings", + "graph_neighbours", + "speaker_stats", + "graph_schema", + "graph_query", + "graph_related", + "graph_paths", + ], + }, + "notes": {"label": "Notes", "tools": ["find_notes", "read_note", "write_note", "update_note"]}, + "memory": {"label": "Namespace memory", "tools": [t[0] for t in MEMORY_TOOLS]}, + "activity": { + "label": "Timeline and automations", + "tools": ["timeline", "mark_activities", "approvals", "decline_approval", "automations", "test_automation"] + + [t[0] for t in AUTOMATION_TOOLS], + }, + "changes": { + "label": "Proposed changes", + "tools": ["suggest_topic", "propose_entity_change", "propose_topic_change", "entity_setup", "run_template"], + }, + "files": {"label": "Importing files", "tools": [t[0] for t in ops_tools.FILE_TOOLS]}, + "extensions": {"label": "Extensions and skills", "tools": [t[0] for t in AUTHOR_TOOLS] + ["use_skill"]}, + "server": {"label": "The server", "tools": [t[0] for t in ops_tools.ADMIN_TOOLS]}, + "browser": {"label": "Your browser", "tools": [t[0] for t in browsermod.TOOLS]}, +} + + +def _effect(changes, out): + """What a tool does, as the tool picker labels it: asks (it waits for the person's yes), out (it sends what it's + given to another host) or read.""" + return "asks" if changes else "out" if out else "read" + + +def _why_missing(group, cfg): + """Why none of a built-in group's tools is offered to this person: in words, and the setting that would change it + (for admins), if one would.""" + off = set(cfg["ai"].get("disabled_tools") or []) + if group == "browser" and not browsermod.enabled(cfg): + return "switched off by an admin", "ai.browser" + if group == "extensions" and not cfg["ai"].get("extensions", True): + return "switched off by an admin", "ai.extensions" + if all(t in off for t in GROUPS[group]["tools"]): + return "switched off by an admin", "ai.disabled_tools" + if group == "server": + return "for admins", None + if group == "memory": + return "in a conversation about one namespace whose own assistant is on", None + return "needs editor access to a namespace", None + + +def _group(name, label, tools, why=None, setting=None): + return { + "name": name, + "label": label, + "tools": tools, + "asks": any(t["effect"] == "asks" for t in tools), + "out": any(t["out"] for t in tools), + "available": any(t["available"] for t in tools), + "why": why, + "setting": setting, + } + + +def catalog(box): + """The tool groups a question can be given, for the person `box` is for (GET /chats/tools): the built-in ones, then + their extensions'. Each has its label and its tools: what each does (read, asks first or out: sends data to + another host), whether it sends data out, and whether this person has it now. A group with none has why, and the + setting that changes it. The browser's tools count as there while an admin allows them: a question has them while + the browser's Lens side panel it's asked from is connected.""" + offered = {s["function"]["name"] for s in box.specs()} + off = set(box.cfg["ai"].get("disabled_tools") or []) + if browsermod.enabled(box.cfg): + offered |= {t for t in GROUPS["browser"]["tools"] if t not in off} + built = {t[0]: t[4] for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS + MEMORY_TOOLS + AUTOMATION_TOOLS} + out = [] + for name, g in GROUPS.items(): + tools = [] + for t in g["tools"]: + sends = t in SENDS_OUT or t in BROWSER_OUT + tools.append({"name": t, "effect": _effect(built.get(t, False), sends), "out": sends, "available": t in offered}) + out.append(_group(name, g["label"], tools, *(() if any(t["available"] for t in tools) else _why_missing(name, box.cfg)))) + for name, g in box.ext.groups.items() if box.ext else (): + tools = [] + for t in g["tools"]: + spec = box.ext.tools[t]["spec"] + sends = extensions.sends_out(spec) + tools.append({"name": t, "effect": _effect(spec["effect"] == "change", sends), "out": sends, "available": t in offered}) + out.append(_group(name, g["label"], tools, None if any(t["available"] for t in tools) else "needs editor access to a namespace")) + return out + def read_before(history, summary=None): """Whether earlier answers in a conversation read text from outside Lens (excerpts, or tools other than Lens's own), @@ -496,7 +607,8 @@ class Toolbox(ops_tools.OpsTools): channels.Channel.acts). `browser`: the person's connected side panel (browser.py), whose browser the assistant can then use. `account`: the person an automation acts for, when `user` is the automation itself (automation:, the by of what it writes): their approvals, extensions, view of the timeline and marks on it; `origin` ({automation, run, - node}) is kept on the approvals it asks for.""" + node}) is kept on the approvals it asks for. `only`: the groups (GROUPS, an extension's mcp: or ext:) + and tools the question was given (None: every tool this person may use; an admin's switch still wins).""" def __init__( self, @@ -516,8 +628,10 @@ def __init__( act_on=(), account=None, origin=None, + only=None, ): self.db, self.cfg, self.user, self.chat, self.said = db, cfg, user, chat_id, said + self.only = None if only is None else set(only) self.account, self.origin = user["id"] if account is None else account, origin # where this answer has read text from outside Lens so far (`outside`: before any tool ran, such as the page it # was asked from or a chat room's message): once it has, tools that send data out of Lens or change the server @@ -622,15 +736,31 @@ def specs(self): tools += browsermod.TOOLS if self.ext: tools += self.ext.tool_specs(can_act or self.admin) + keep = self.chosen() return [ { "type": "function", "function": {"name": n, "description": d, "parameters": {"type": "object", "properties": p, "required": req}}, } for n, d, p, req, _ in tools - if n not in off + if n not in off and (keep is None or n in keep) ] + def chosen(self): + """The tool names `only` keeps (None: every tool): the tools it names, and those of the groups it names.""" + if self.only is None: + return None + groups = self.ext.groups if self.ext else {} + out = set() + for n in self.only: + out |= set(GROUPS[n]["tools"] if n in GROUPS else groups[n]["tools"] if n in groups else [n]) + return out + + def unknown(self, names): + """The names in a choice of tools that are no group or tool this person has (built-in, or of their extensions).""" + known = builtin_names() | set(GROUPS) | set(self.ext.groups if self.ext else ()) | set(self.ext.tools if self.ext else ()) + return [n for n in names if n not in known] + def ref(self, rid, t0, text, speaker=None, title=None, source="said", page=None): self.refs.append( store.clean( diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py index df0e9d9b3..3ddef703f 100644 --- a/fastapi_backend/app/domain/chat.py +++ b/fastapi_backend/app/domain/chat.py @@ -312,8 +312,10 @@ def cited(text, passages): # ---------- conversations ---------- -def create(db, account, title=None, scope=None, model=None, kind="chat"): - """A conversation. A `setup` one (admins) is the assistant setting the server up: it acts instead of asking.""" +def create(db, account, title=None, scope=None, model=None, kind="chat", tools=None, max_tool_calls=None): + """A conversation. A `setup` one (admins) is the assistant setting the server up: it acts instead of asking. + `tools` and `max_tool_calls`: what its answers may use (None: Auto, everything the person may use) and at most how + many tool calls each makes (None: the rounds limit them).""" cid = db.next_id("chat") db.q( "CREATE $r CONTENT $d", @@ -325,6 +327,8 @@ def create(db, account, title=None, scope=None, model=None, kind="chat"): "kind": kind if kind != "chat" else None, "scope": scope or {}, "model": model, + "tools": tools, + "max_tool_calls": max_tool_calls, "created_at": store.now(), "updated_at": store.now(), } @@ -333,9 +337,21 @@ def create(db, account, title=None, scope=None, model=None, kind="chat"): return cid +def remember_tools(db, account, tools, max_tool_calls): + """Someone's latest choice of tools and tool-call limit, which their new conversations start with.""" + db.q("UPDATE $r SET chat_tools = $t", r=R("account", account), t={"tools": tools, "max_tool_calls": max_tool_calls}) + + +def remembered_tools(db, account): + """{tools, max_tool_calls}: the choice someone's new conversations start with (both None: Auto, no limit).""" + got = (db.one("SELECT chat_tools FROM $r", r=R("account", account)) or {}).get("chat_tools") or {} + return {"tools": got.get("tools"), "max_tool_calls": got.get("max_tool_calls")} + + def get(db, cid, account): c = db.one( - "SELECT record::id(id) AS id, account, title, scope, model, kind ?? 'chat' AS kind, head, created_at, updated_at FROM $r", + "SELECT record::id(id) AS id, account, title, scope, model, kind ?? 'chat' AS kind, head, tools, max_tool_calls, created_at, " + "updated_at FROM $r", r=R("chat", cid), ) if not c or c["account"] != account: @@ -830,6 +846,18 @@ def attached_note(files): "Don't call any more tools now: answer the question from what you found above, citing it, and say in a sentence " "what you didn't get to check." ) +SPENT = "This answer's tool calls are used up, so this one wasn't made." + + +def tool_calls_allowed(cfg, asked): + """At most how many tool calls one answer makes when `asked` (None: no limit but the rounds), never past + ai.max_tool_calls. ValueError for one that isn't a whole number from 1 to that.""" + if asked is None: + return None + most = int(cfg["ai"].get("max_tool_calls") or 50) + if isinstance(asked, bool) or not isinstance(asked, int) or not 1 <= asked <= most: + raise ValueError(f"max_tool_calls is a whole number from 1 to {most} (ai.max_tool_calls)") + return asked def _plain(msgs): @@ -867,20 +895,30 @@ def _finish(cfg, msgs, tools, model): return text -def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None): +def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, setup=False, summary=None, record=None, max_tool_calls=None): """The tool loop: yields ("step", {...}) for each tool call, then ("answer", text), or ("direct", text) when the model answered without looking anything up (it never saw the archive, so the caller can answer from a search instead). Every turn ends in an answer: when the steps run out, or the model replies with nothing, it's asked to answer from what it found, with the tools off. Raises llm.ToolsUnsupported. `record` (a Record) learns the tools offered and - counts the rounds and tool calls.""" + counts the rounds and tool calls. `max_tool_calls` (tool_calls_allowed): at most that many calls, in as many rounds + as it takes (a round makes one call or more), instead of `max_steps` rounds; a round that asks for more makes the + first of them, and the answer is written from what they found (its record's outcome is limit).""" + limit = tool_calls_allowed(cfg, max_tool_calls) + max_steps = limit or max_steps system = TOOL_SYSTEM + untrusted.rules(cfg) + ("\n\n" + SETUP_SYSTEM if setup else "") note = getattr(toolbox, "system_note", None) # skills, and context from hooks (extensions.py) system += note() if note else "" system += summary_note(summary) - system += ( - f"\n\nYou have up to {max_steps} rounds of tool calls for this question (make calls that don't depend on each other " - "in the same round), then you answer. Answer as soon as you have enough." - ) + if limit: + system += ( + f"\n\nYou have up to {limit} tool call{'s' if limit > 1 else ''} for this question in all (make calls that don't depend " + "on each other in the same round), then you answer. Answer as soon as you have enough." + ) + else: + system += ( + f"\n\nYou have up to {max_steps} rounds of tool calls for this question (make calls that don't depend on each other " + "in the same round), then you answer. Answer as soon as you have enough." + ) msgs = [{"role": "system", "content": system}] + past_turns(history, summary=summary, cfg=cfg) msgs.append({"role": "user", "content": question}) tools, looked, made = toolbox.specs(), False, set() @@ -894,10 +932,14 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set return break # a reply with nothing in it looked = True + left = len(msg["tool_calls"]) if limit is None else limit - record.used["tool_calls"] record.used["rounds"] += 1 - record.used["tool_calls"] += len(msg["tool_calls"]) + record.used["tool_calls"] += min(len(msg["tool_calls"]), left) msgs.append({"role": "assistant", "content": msg["content"], "tool_calls": msg["tool_calls"]}) - for c in msg["tool_calls"]: + for i, c in enumerate(msg["tool_calls"]): + if i >= left: # past the limit: every call still gets a reply, as the model's API expects + msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": json.dumps({"note": SPENT})}) + continue fn = c.get("function") or {} args = fn.get("arguments") or {} if isinstance(args, str): # a JSON string, as OpenAI sends it; some servers send the object itself @@ -920,6 +962,9 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set step = {"tool": name, "args": args, "summary": summary, "ms": int((time.monotonic() - t0) * 1000)} yield "step", {**step, "failed": True} if getattr(toolbox, "failed", False) else step msgs.append({"role": "tool", "tool_call_id": c.get("id"), "content": result}) + if limit is not None and record.used["tool_calls"] >= limit: # the calls are used up: it answers from what they found + record.limited = True + break else: # the rounds ran out record.limited = True text = _finish(cfg, msgs, tools, model) or "I couldn't put an answer together from what I found. Try again, or ask something narrower." diff --git a/fastapi_backend/app/domain/completions.py b/fastapi_backend/app/domain/completions.py index e344162ed..ca8d0d602 100644 --- a/fastapi_backend/app/domain/completions.py +++ b/fastapi_backend/app/domain/completions.py @@ -102,16 +102,13 @@ class Box(ai_tools.Toolbox): none of the person's extensions, skills or hooks, and nothing that changes data.""" def __init__(self, db, cfg, user, sid, ns, tools, said, base, note_text): - super().__init__(db, cfg, user, {sid}, set(), {"namespaces": [ns]}, None, base, said=said) - self.only, self.note_text = set(tools), note_text + super().__init__(db, cfg, user, {sid}, set(), {"namespaces": [ns]}, None, base, said=said, only=tools) + self.note_text = note_text self.ext, self.home, self.browser = None, None, None def system_note(self): return self.note_text - def specs(self): - return [s for s in super().specs() if s["function"]["name"] in self.only] - def answer(db, cfg, sid, ns, user, question, history, app="", model=None, tools=(), base=None, temperature=None): """Answer one request. Yields ("step", {tool, args, summary}) for each tool call, ("token", text) for the diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index eb0079509..717ce680e 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -691,10 +691,13 @@ def _owner_admin(db, g): class Active: """The extensions switched on for one person, plugins opened up: tools and skills by name, hooks by event. Each - item knows the extension (and version) it comes from.""" + item knows the extension (and version) it comes from. `groups`: each extension's tools as a group a question can be + given ({mcp: or ext:: {label, tools}}; ai_tools.GROUPS has the built-in ones): an MCP server's tools + under mcp:, any other extension's under ext:.""" def __init__(self, db, me): self.tools, self.skills, self.hooks = {}, {}, {e: [] for e in EVENTS} + self.groups: dict[str, dict] = {} for d in db.rows("SELECT record::id(id) AS id FROM extension WHERE deleted_at = NONE AND enabled = true ORDER BY id"): g = get(db, d["id"]) if not can_use(g, me): @@ -722,6 +725,9 @@ def __init__(self, db, me): "spec": spec, } if it["kind"] == "tool": + if it["name"] not in self.tools: + group = ("mcp:" if spec["run"]["type"] == "mcp" else "ext:") + g["name"] + self.groups.setdefault(group, {"label": g.get("title") or g["name"], "tools": []})["tools"].append(it["name"]) self.tools.setdefault(it["name"], item) elif it["kind"] == "skill": self.skills.setdefault(it["name"], item) diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 72573411b..f07a1fa52 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -445,6 +445,10 @@ def _check(section, key, value, default): if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 100): raise ValueError("ai.browser_steps is a whole number of steps from 1 to 100") return value + if (section, key) == ("ai", "max_tool_calls"): + if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 500): + raise ValueError("ai.max_tool_calls is a whole number of tool calls from 1 to 500") + return value if (section, key) == ("llm", "describe_max"): if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 1000): raise ValueError("llm.describe_max is a whole number from 1 to 1000") diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 8608aa685..9752c9f31 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -291,6 +291,7 @@ def labels(): "injection_check": "patterns", # flag text that reads like instructions to an AI: patterns, classifier, model, off "disabled_tools": [], "max_steps": 6, + "max_tool_calls": 50, # the most tool calls a question may give one answer (its `max_tool_calls`; chat.tool_answer) "max_transcript_reads": 20, "confirm_over_recordings": 100, "confirm_over_cost": None, diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py index 443d31b78..4c8a7043d 100644 --- a/fastapi_backend/app/schemas/chats.py +++ b/fastapi_backend/app/schemas/chats.py @@ -36,6 +36,15 @@ class ChatCreate(RequestModel): model: str | None = Field( None, description="the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one" ) + tools: list[str] | None = Field( + None, + max_length=100, + description="what its answers may use: tool groups and tools (GET /chats/tools); [] for none; left out: your latest " + "choice (PATCH /chats/{cid}), else Auto", + ) + max_tool_calls: int | None = Field( + None, ge=1, description="at most this many tool calls in each answer (1 to ai.max_tool_calls); left out: your latest choice" + ) class ChatUpdate(RequestModel): @@ -44,6 +53,15 @@ class ChatUpdate(RequestModel): model: str | None = Field( None, description="the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one" ) + tools: list[str] | None = Field( + None, + max_length=100, + description="what its answers may use: tool groups and tools (GET /chats/tools); [] for none; null: Auto (everything " + "you may use). Your new conversations start with it too", + ) + max_tool_calls: int | None = Field( + None, ge=1, description="at most this many tool calls in each answer (1 to ai.max_tool_calls); null: no limit but the rounds" + ) class ChatSummary(ResponseModel): @@ -52,6 +70,8 @@ class ChatSummary(ResponseModel): kind: Literal["chat", "setup", "bridge"] = Field("chat", description="bridge: a conversation in a chat room, through Matterbridge") scope: dict[str, Any] = {} model: str | None = Field(None, description="the model chosen for this conversation; null: the configured one") + tools: list[str] | None = Field(None, description="the tool groups and tools its answers may use; null: Auto") + max_tool_calls: int | None = Field(None, description="at most this many tool calls in each answer; null: no limit but the rounds") created_at: str | None = None updated_at: str | None = None @@ -221,7 +241,8 @@ class ChatCapabilities(ResponseModel): configured: bool = Field(description="a language model is set up; without one, answers are the best-matching passages") model: str | None = Field(None, description="the model answers come from") tools: bool = Field(description="the assistant looks things up with tools (and may propose work for approval)") - max_steps: int = Field(description="the most tool calls it makes for one answer") + max_steps: int = Field(description="the most rounds of tool calls it makes for one answer (a round makes one call or more)") + max_tool_calls: int = Field(50, description="the most tool calls a question may give one answer (`max_tool_calls`)") check: bool = Field(description="answers can be checked against their sources") models: list[str] = Field( default_factory=list, description="the models people may pick, the configured one first (llm.chat_models, else the server's list)" @@ -275,6 +296,55 @@ class MessageCreate(RequestModel): "assistant can use that browser (docs/browser-assistant.md)", ) via: Literal["voice"] | None = Field(None, description="voice: said in a voice conversation (the question's record keeps it)") + tools: list[str] | None = Field( + None, + max_length=100, + description="what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists " + "them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An " + "admin's switch still wins", + ) + max_tool_calls: int | None = Field( + None, ge=1, description="at most this many tool calls in this answer (1 to ai.max_tool_calls); null: the conversation's" + ) + scope: ChatScope | None = Field( + None, + description="what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read", + ) + + +class ToolInfo(ResponseModel): + name: str + effect: Literal["read", "asks", "out"] = Field( + description="read: it looks things up; asks: it waits for your yes; out: it sends what it's given to another host" + ) + out: bool = Field(False, description="it sends what it's given to another host (and asks first after outside text)") + available: bool = Field(description="you have it now") + + +class ToolGroup(ResponseModel): + name: str = Field(description="what `tools` names it by: a built-in group, mcp: or ext:") + label: str + tools: list[ToolInfo] + asks: bool = Field(description="one of its tools waits for your yes") + out: bool = Field(description="one of its tools sends data to another host") + available: bool = Field(description="you have one of its tools now") + why: str | None = Field(None, description="why you don't, when you have none") + setting: str | None = Field(None, description="the setting that would change that (section.key), for admins") + + +class ToolChoice(ResponseModel): + tools: list[str] | None = None + max_tool_calls: int | None = None + + +class ChatTools(ResponseModel): + groups: list[ToolGroup] = Field(description="the built-in groups, then your extensions' (mcp:, ext:)") + max_tool_calls: int = Field(description="the most tool calls one answer may be given (ai.max_tool_calls)") + remembered: ToolChoice = Field(description="your latest choice, which new conversations start with") + namespaces_note: str = Field( + "Web and MCP tools reach beyond your namespaces: the namespaces you pick narrow the archive's tools only.", + description="what the choice of namespaces doesn't narrow", + ) class ApprovalOrigin(ResponseModel): diff --git a/fastapi_backend/tests/api/test_chat_tools.py b/fastapi_backend/tests/api/test_chat_tools.py new file mode 100644 index 000000000..44ecb2a9b --- /dev/null +++ b/fastapi_backend/tests/api/test_chat_tools.py @@ -0,0 +1,244 @@ +"""Choosing what an answer may use (docs/assistant.md, "Choosing tools, namespaces and a limit"): the tool groups and +tools a person has (GET /chats/tools), only the ones picked offered (an admin's switch still wins), at most so many tool +calls, a scope for one answer alone, and the choice kept with the conversation and for the next ones.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import ai_tools, browser, ops_tools +from tests import fake_llm +from tests.api._assist import Assist, sse, start_llm + +TOOL = """--- +name: translate +kind: tool +description: Translate text into another language. +effect: read +params: + - {name: text, kind: text, required: true} +--- + +Translate into French: {{text}} +""" + + +@pytest.fixture +def llm(cfg): + srv = start_llm(cfg) + yield fake_llm.Handler + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, llm): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +def call(i, name, args): + return {"id": f"c{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}} + + +def offered(llm): + """The tools the model was offered, the last time it was offered any.""" + return {t["function"]["name"] for t in [b for b in llm.seen if b.get("tools")][-1]["tools"]} + + +def ask(c, h, cid, **body): + return sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "What did Dyno say?", **body}).text) + + +def test_every_built_in_tool_is_in_one_group(): + every = [t for g in ai_tools.GROUPS.values() for t in g["tools"]] + assert sorted(every) == sorted(ai_tools.builtin_names()) and len(every) == len(set(every)) + assert not set(ai_tools.GROUPS) & ai_tools.builtin_names() # a name is a group or a tool, never both + assert ai_tools.BROWSER_OUT <= {t[0] for t in browser.TOOLS} + + +def test_the_catalog_says_what_each_person_has(app, db, cfg, folder, new_client): + s = Assist(app, db, cfg, folder, new_client) + cv, hv = s.cl["viewer"] + got = cv.get("/api/v1/chats/tools", headers=hv).json() + assert got["max_tool_calls"] == 50 and got["remembered"] == {"tools": None, "max_tool_calls": None} + groups = {g["name"]: g for g in got["groups"]} + assert list(groups)[: len(ai_tools.GROUPS)] == list(ai_tools.GROUPS) + tools = {t["name"]: t for g in got["groups"] for t in g["tools"]} + # a viewer reads the archive and the notes, but writing a note and proposing changes need an editor + assert groups["archive"]["available"] and not groups["archive"]["asks"] and not groups["archive"]["out"] + assert tools["find_notes"]["available"] and not tools["write_note"]["available"] and tools["write_note"]["effect"] == "asks" + assert groups["changes"]["asks"] and tools["entity_setup"]["available"] and not tools["propose_entity_change"]["available"] + assert (groups["server"]["why"], groups["server"]["setting"]) == ("for admins", None) + # the browser's tools are there while the side panel is allowed; the ones that act on a site send data out + assert groups["browser"]["available"] and groups["browser"]["out"] and tools["browser_type"]["effect"] == "out" + assert tools["browser_read"]["effect"] == "read" + + ca, ha = s.cl["admin"] + groups = {g["name"]: g for g in ca.get("/api/v1/chats/tools", headers=ha).json()["groups"]} + assert groups["server"]["available"] and groups["changes"]["available"] + # what an admin switched off isn't offered, and says where it's switched back on + off = {"disabled_tools": [t[0] for t in ops_tools.ADMIN_TOOLS], "browser": False} + assert ca.put("/api/v1/settings/ai", headers=ha, json=off).status_code == 200 + groups = {g["name"]: g for g in ca.get("/api/v1/chats/tools", headers=ha).json()["groups"]} + assert (groups["server"]["available"], groups["server"]["why"], groups["server"]["setting"]) == ( + False, + "switched off by an admin", + "ai.disabled_tools", + ) + assert (groups["browser"]["available"], groups["browser"]["setting"]) == (False, "ai.browser") + assert cv.get("/api/v1/chats/tools").status_code == 401 + + +def test_an_answer_is_offered_only_what_was_picked(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + llm.tool_script = [{"content": "", "tool_calls": [call(1, "find_notes", {"query": "Dyno"})]}, {"content": "Nothing noted."}] + ev = ask(c, h, cid, tools=["notes", "find_entities"]) + assert offered(llm) == {"find_notes", "read_note", "write_note", "update_note", "find_entities"} + meta = ev["meta"][0] + assert meta["tools"]["picked"] == ["notes", "find_entities"] and set(meta["tools"]["offered"]) == offered(llm) + question = c.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"][-2] + assert question["meta"]["options"]["tools"] == ["notes", "find_entities"] + # a tool it wasn't given isn't run, even when the model calls it + llm.tool_script = [{"content": "", "tool_calls": [call(2, "search_transcripts", {"query": "Dyno"})]}, {"content": "Done."}] + ev = ask(c, h, cid, tools=["notes"]) + assert ev["step"][0]["summary"] == "unknown tool search_transcripts" and ev["step"][0]["failed"] is True + + # an admin's switch still wins + ca, ha = s.cl["admin"] + assert ca.put("/api/v1/settings/ai", headers=ha, json={"disabled_tools": ["write_note"]}).status_code == 200 + llm.tool_script = [{"content": "Nothing noted."}] + ask(c, h, cid, tools=["notes"]) + assert offered(llm) == {"find_notes", "read_note", "update_note"} + assert ca.put("/api/v1/settings/ai", headers=ha, json={"disabled_tools": []}).status_code == 200 + + # no tools: it answers from a search, and the model is offered none + llm.seen.clear() + ev = ask(c, h, cid, tools=[]) + assert not [b for b in llm.seen if b.get("tools")] and ev["meta"][0]["tools"] == {"picked": [], "offered": []} + assert ev["passages"] and ev["meta"][0]["outcome"] == "answered" + + # what isn't a group or a tool is told, not ignored + r = c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "x", "tools": ["archive", "nope"]}) + assert r.status_code == 400 and "nope" in r.json()["detail"] + + +def test_an_extensions_tools_are_a_group(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + assert c.post("/api/v1/extensions", headers=h, json={"text": TOOL}).status_code == 200 + groups = {g["name"]: g for g in c.get("/api/v1/chats/tools", headers=h).json()["groups"]} + assert groups["ext:translate"]["tools"] == [{"name": "translate", "effect": "read", "out": False, "available": True}] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + llm.tool_script = [{"content": "Bonjour."}] + ask(c, h, cid, tools=["ext:translate"]) + assert offered(llm) == {"translate"} + # another person doesn't have it + cv, hv = s.cl["viewer"] + vid = cv.post("/api/v1/chats", headers=hv, json={}).json()["id"] + assert cv.post(f"/api/v1/chats/{vid}/messages", headers=hv, json={"content": "x", "tools": ["ext:translate"]}).status_code == 400 + + +def test_at_most_so_many_tool_calls(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + llm.seen.clear() + # two calls, then two more of which one is made: three in all, and the answer is written from what they found + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "search_transcripts", {"query": "a"}), call(2, "find_entities", {"query": "b"})]}, + {"content": "", "tool_calls": [call(3, "find_topics", {"query": "c"}), call(4, "list_recordings", {})]}, + {"content": "", "tool_calls": [call(5, "find_notes", {"query": "d"})]}, + {"content": "Dyno designs capsids."}, + ] + ev = ask(c, h, cid, max_tool_calls=3) + assert [x["tool"] for x in ev["step"]] == ["search_transcripts", "find_entities", "find_topics"] + meta = ev["meta"][0] + assert (meta["outcome"], meta["limits"], meta["used"]) == ("limit", {"rounds": 3, "tool_calls": 3}, {"rounds": 2, "tool_calls": 3}) + asked = [b for b in llm.seen if b.get("tools")] + assert "up to 3 tool calls for this question in all" in asked[0]["messages"][0]["content"] + # the call past the limit is answered too, as the model's API needs, and not made + replies = [m for m in asked[-1]["messages"] if m["role"] == "tool"] + assert json.loads(replies[-1]["content"])["note"].startswith("This answer's tool calls are used up") + assert asked[-1].get("tool_choice") == "none" # then it answers, with the tools off + + # more rounds than ai.max_steps, when the limit allows: one call each + llm.tool_script = [{"content": "", "tool_calls": [call(i, "find_topics", {"query": str(i)})]} for i in range(8)] + [ + {"content": "Found it."} + ] + meta = ask(c, h, cid, max_tool_calls=8)["meta"][0] + assert (meta["outcome"], meta["used"]) == ("limit", {"rounds": 8, "tool_calls": 8}) + # an answer within its limit answered as usual + llm.tool_script = [{"content": "", "tool_calls": [call(1, "find_topics", {"query": "x"})]}, {"content": "Done."}] + meta = ask(c, h, cid, max_tool_calls=8)["meta"][0] + assert (meta["outcome"], meta["used"]["tool_calls"]) == ("answered", 1) + + # within 1 and ai.max_tool_calls + for n, code in ((0, 422), (51, 400)): + assert c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "x", "max_tool_calls": n}).status_code == code + ca, ha = s.cl["admin"] + for bad in (0, 501, "5", True): + assert ca.put("/api/v1/settings/ai", headers=ha, json={"max_tool_calls": bad}).status_code == 400, bad + assert ca.put("/api/v1/settings/ai", headers=ha, json={"max_tool_calls": 2}).status_code == 200 + assert c.get("/api/v1/chats/capabilities", headers=h).json()["max_tool_calls"] == 2 + assert c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "x", "max_tool_calls": 3}).status_code == 400 + + +def test_a_conversation_keeps_its_choice_and_new_ones_start_with_it(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + assert c.get(f"/api/v1/chats/{cid}", headers=h).json()["tools"] is None # Auto + assert c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": ["archive"], "max_tool_calls": 5}).status_code == 200 + assert c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": ["nope"]}).status_code == 400 + got = c.get(f"/api/v1/chats/{cid}", headers=h).json() + assert (got["tools"], got["max_tool_calls"]) == (["archive"], 5) + assert [x for x in c.get("/api/v1/chats", headers=h).json() if x["id"] == cid][0]["tools"] == ["archive"] + # its answers use it, unless a question says otherwise + llm.tool_script = [{"content": "Done."}] + meta = ask(c, h, cid)["meta"][0] + assert (meta["tools"]["picked"], meta["limits"]["tool_calls"]) == (["archive"], 5) + assert set(meta["tools"]["offered"]) == set(ai_tools.GROUPS["archive"]["tools"]) + llm.tool_script = [{"content": "Done."}] + meta = ask(c, h, cid, tools=["graph"], max_tool_calls=2)["meta"][0] + assert (meta["tools"]["picked"], meta["limits"]["tool_calls"]) == (["graph"], 2) + # the latest choice is what new conversations start with, and what the catalog says + assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": ["archive"], "max_tool_calls": 5} + new = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + got = c.get(f"/api/v1/chats/{new}", headers=h).json() + assert (got["tools"], got["max_tool_calls"]) == (["archive"], 5) + # unless it says otherwise; back to Auto + new = c.post("/api/v1/chats", headers=h, json={"tools": None, "max_tool_calls": None}).json()["id"] + got = c.get(f"/api/v1/chats/{new}", headers=h).json() + assert (got["tools"], got["max_tool_calls"]) == (None, None) + c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": None, "max_tool_calls": None}) + assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": None, "max_tool_calls": None} + # someone else's + cv, hv = s.cl["viewer"] + assert cv.get("/api/v1/chats/tools", headers=hv).json()["remembered"] == {"tools": None, "max_tool_calls": None} + assert cv.get("/api/v1/chats/tools", headers=hv, params={"chat_id": cid}).status_code == 404 + + +def test_a_scope_for_one_answer(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["admin"] + cid = c.post("/api/v1/chats", headers=h, json={"scope": {"namespaces": ["pods"]}}).json()["id"] + llm.tool_script = [{"content": "", "tool_calls": [call(1, "search_transcripts", {"query": "shipment"})]}, {"content": "Friday [1]."}] + ev = ask(c, h, cid, scope={"namespaces": ["calls"]}) + assert ev["meta"][0]["scope"] == {"namespaces": ["calls"]} + assert ev["passages"][0] and {p["recording_id"] for p in ev["passages"][0]} == {s.call} + got = c.get(f"/api/v1/chats/{cid}", headers=h).json() + assert got["scope"] == {"namespaces": ["pods"]} # the conversation keeps its own + assert got["messages"][-2]["meta"]["options"]["scope"] == {"namespaces": ["calls"]} + # everything one can read, for this answer + llm.tool_script = [{"content": "Done."}] + assert ask(c, h, cid, scope={})["meta"][0]["scope"] == {} + # and only what one can read + cv, hv = s.cl["viewer"] + vid = cv.post("/api/v1/chats", headers=hv, json={}).json()["id"] + r = cv.post(f"/api/v1/chats/{vid}/messages", headers=hv, json={"content": "x", "scope": {"namespaces": ["calls"]}}) + assert r.status_code == 404 diff --git a/fastapi_backend/tests/api/test_chats.py b/fastapi_backend/tests/api/test_chats.py index 04e09ba46..0de23bf09 100644 --- a/fastapi_backend/tests/api/test_chats.py +++ b/fastapi_backend/tests/api/test_chats.py @@ -159,7 +159,8 @@ def test_capabilities_for_everyone(client, new_client, db, cfg, folder, llm): make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) hv = login(client, "vi@x.io", "viewer password 1") caps = client.get("/api/v1/chats/capabilities", headers=hv).json() - assert caps == {"configured": True, "model": "fake", "tools": True, "max_steps": 6, "check": True, "models": ["fake", "fake-large"]} + want = {"configured": True, "model": "fake", "tools": True, "max_steps": 6, "max_tool_calls": 50, "check": True} + assert caps == {**want, "models": ["fake", "fake-large"]} assert "base_url" not in json.dumps(caps) and new_client().get("/api/v1/chats/capabilities").status_code == 401 admin = new_client() assert ( diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 9f2b7f1f9..c74d0ea68 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -2530,6 +2530,7 @@ const LIMITS = [ "ai.price_in", "ai.price_out", "ai.max_steps", + "ai.max_tool_calls", "ai.browser_steps", "ai.max_transcript_reads", "ai.compact_model", @@ -2596,6 +2597,7 @@ function AiBody({ ctx }: { ctx: BodyCtx }) { + diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 05684aa54..2f6517d8a 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -905,6 +905,15 @@ export const FIELDS: FieldSpec[] = [ min: 1, max: 50, }, + { + section: "ai", + key: "max_tool_calls", + label: "Most tool calls a question may ask for", + kind: "int", + hint: "The highest limit a question or a conversation can set on the tool calls of one answer.", + min: 1, + max: 500, + }, { section: "ai", key: "browser", From 957c3f1bc854ab5cfc20201ca37962ecf5e339f1 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:41:22 +0000 Subject: [PATCH 05/12] docs(chat): choosing tools, namespaces and a limit; regenerate the client Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- CHANGELOG.md | 7 + docs/api.md | 10 +- docs/assistant.md | 52 ++- nextjs-frontend/app/openapi-client/index.ts | 9 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 34 +- .../app/openapi-client/types.gen.ts | 226 +++++++++- nextjs-frontend/openapi.json | 389 +++++++++++++++++- 7 files changed, 720 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f68b7a688..86017937a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Choose the tools, a limit and the namespaces of an answer.** A conversation, or one question, can pick the tool + groups or tools its answers may use (`tools`), cap their tool calls (`max_tool_calls`, up to the new Settings → AI + assistant → Most tool calls a question may ask for, 50 by default) and narrow one answer to its own namespaces + (`scope`). `GET /api/v1/chats/tools` lists the groups a person has, what each tool does (reads, asks first, sends data + out) and why a group is missing. An admin's switches still win, and new conversations start with the person's last + choice. The web app's Tools chip is in progress + ([Choosing tools, namespaces and a limit](docs/assistant.md#choosing-tools-namespaces-and-a-limit)). - **Automations run.** An automation saved over the API (`/api/v1/automations`) now reacts to new assets in its namespaces: when its If passes, it runs its workflow as the person who made it, within its limits and budgets, and each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now; diff --git a/docs/api.md b/docs/api.md index beed20cb5..33c9771bc 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1444,6 +1444,7 @@ version. ``` GET /api/v1/chats GET /api/v1/chats/capabilities +GET /api/v1/chats/tools POST /api/v1/chats GET /api/v1/chats/{cid} PATCH /api/v1/chats/{cid} @@ -1467,7 +1468,7 @@ that node runs without asking from now on). Approving, editing and `always` on a session (403 for an API key or an app); each approval's `can` says what you may do with it. `GET /chats/capabilities` tells anyone signed in whether a language model is set up (`configured`), which (`model`), -whether it uses tools (`tools`, `max_steps`), whether answers can be checked (`check`) and the `models` people may pick +whether it uses tools (`tools`, `max_steps`, `max_tool_calls`), whether answers can be checked (`check`) and the `models` people may pick (the configured one first; `llm.chat_models`, else the server's list); never the model server's address or key. A conversation's `model` (`POST /chats`, `PATCH /chats/{cid}`; null for the configured one) answers in it, and a question can name another (`POST /chats/{cid}/messages {content, model}`, e.g. to retry); a model that isn't offered is @@ -1480,6 +1481,13 @@ step it's on: the stream sends `stopped`, then `done` with the saved message, wh everything after it are replaced by this question and its new answer; 404 when `edit` isn't a question in that conversation. A conversation titled after the first question is retitled when that question is edited. +`GET /chats/tools` lists the tool groups and tools you have, each tool's `effect` (`read`, `asks` or `out`), why a +group isn't available and the setting that switches it on, the most tool calls a question may ask for +(`max_tool_calls`) and your latest choice (`remembered`); `chat_id` reads it for that conversation's scope. A +conversation's `tools` and `max_tool_calls` (`POST /chats`, `PATCH /chats/{cid}`; null for Auto) are what its answers +may use, and a question can set its own, with a `scope` for that answer alone (`POST /chats/{cid}/messages {content, +tools, max_tool_calls, scope}`) ([Choosing tools, namespaces and a limit](assistant.md#choosing-tools-namespaces-and-a-limit)). + A conversation's `scope` narrows what it draws on: `namespaces`, `recordings`, `collections`, `speakers`, `from` and `to`; every key narrows it further, and an empty scope is everything you can read. `collections` are ids of collections you can see (yours or shared); the conversation draws on their assets as they are each time it diff --git a/docs/assistant.md b/docs/assistant.md index 7a5d336aa..39693560d 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -152,7 +152,8 @@ When the configured model supports function calling, chat becomes an agent. (`save_automation` with `enabled`) and running one now wait for the person's approval; switching one off acts at once ([Automations](automations.md#agents)). - **Limits:** everything is limited to what the person can read and to the conversation's scope, and capped by - `ai.max_steps` (Settings → AI assistant, Tool steps per answer; 6 by default) and `ai.max_transcript_reads`. + `ai.max_steps` (Settings → AI assistant, Tool steps per answer; 6 by default) and `ai.max_transcript_reads`, or by + the tool calls a question allows ([below](#choosing-tools-namespaces-and-a-limit)). - **Always an answer:** the model is told its step budget. When the steps run out, or it replies with nothing, it is asked once more to answer from what it found, with the tools off, and to say what it didn't get to check. A server that calls tools anyway gets the conversation again as plain text, without tools. That last ask is one more model @@ -181,6 +182,55 @@ When the configured model supports function calling, chat becomes an agent. and type in that browser and download files into the conversation, on sites you allow, asking before anything that sends, deletes or buys. See [The assistant in your browser](browser-assistant.md). +### Choosing tools, namespaces and a limit + +By default (Auto) an answer may use every tool the person has. A conversation, or one question, can instead pick the +tools it may use, how many tool calls it may make, and which namespaces it reads. + +- **Tool groups.** `GET /api/v1/chats/tools` lists the groups and the tools in each that the person has, and why a + group isn't there for them: + - `archive` (search and read assets); + - `graph` (entities, topics, the knowledge graph); + - `notes`; + - `memory` (a namespace's own assistant); + - `activity` (the timeline, approvals and automations); + - `changes` (proposals that become approvals); + - `files` (importing files and web pages); + - `extensions`; + - `server` (admins); + - `browser`; + - `mcp:` and `ext:`, one for each of the person's extensions. + + Each tool says its `effect`: `read`, `asks` (it waits for the person's approval) or `out` (it sends data out of + Lens, as the browser's open, click and type do). A group that isn't available says why (switched off by an admin, + for admins, or needs editor access to a namespace) and names the setting that switches it back on (`setting`, + such as `ai.browser`). +- **Picking.** `tools` (on `POST /chats`, `PATCH /chats/` or one question, `POST /chats//messages`) is a list + of group and tool names. The model is then offered only those tools, and a tool it calls that it wasn't offered isn't + run. Null is Auto. An empty list means no tools: the answer is written from a search, as when the model can't use + tools. A name that is no group or tool the person has is a 400. An admin's switches (`ai.disabled_tools`, + `ai.browser`, `ai.extensions`) still win over any pick. +- **A limit on tool calls.** `max_tool_calls` (in the same places) caps how many tool calls one answer may make in + all. The model is told the budget. Calls past it get a reply saying they weren't made, and then the answer is written + from what was found, with outcome `limit`. With a limit, the rounds of calls may go past `ai.max_steps`, up to the + limit. It is a whole number from 1 to `ai.max_tool_calls` (Settings → AI assistant, Most tool calls a question may + ask for; 50 by default, at most 500). `GET /chats/capabilities` gives it as `max_tool_calls`. +- **A scope for one answer.** `scope` on `POST /chats//messages` narrows that answer alone, with the same keys as + a conversation's scope. `{}` means everything the person can read, and a namespace they can't read is a 404. The + conversation keeps its own scope, and a question with a scope isn't narrowed by the + [namespace picker](#picking-the-namespace-for-a-conversation-over-everything). +- **Remembered.** Each answer records what it was offered and its limits (`tools`, `limits`, `scope`), and each + question records the options it was asked with ([What each answer records](#what-each-answer-records)). A + conversation keeps its `tools` and `max_tool_calls`. The person's latest choice is kept on their account, so a new + conversation starts with it (`remembered` in `GET /chats/tools`) unless `POST /chats` says otherwise. +- **Web and MCP tools reach beyond your namespaces.** The namespaces picked narrow the archive's tools only. A web + search, an MCP server or the browser reads and sends wherever it reaches. + +In the web app, a Tools chip beside the composer's scope picks these (in progress). + +Refine later: tool choice in chat rooms and voice mode, a default choice per namespace, and the model suggesting the +tools a question needs. + ## Third-party content and prompt injection The assistant reads text nobody vouched for: what was said in recordings, documents, text read off screens, web pages diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 9219f0cff..de6d5e827 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -313,6 +313,12 @@ export type { ChatNode, ChatScope, ChatSummary, + ChatTools, + ChatToolsData, + ChatToolsError, + ChatToolsErrors, + ChatToolsResponse, + ChatToolsResponses, ChatUpdate, CheckComponentsData, CheckComponentsResponse, @@ -3276,6 +3282,9 @@ export type { TokenPair, TokenResponse, TokenResponses, + ToolChoice, + ToolGroup, + ToolInfo, ToolStep, TopicAbout, TopicCandidate, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 91c43fc31..8acb7e7d5 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -102,6 +102,9 @@ import type { ChangePasswordResponses, ChatCapabilitiesData, ChatCapabilitiesResponses, + ChatToolsData, + ChatToolsErrors, + ChatToolsResponses, CheckComponentsData, CheckComponentsResponses, CheckManifestData, @@ -9968,6 +9971,9 @@ export class Chats { /** * Create Chat + * + * A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH + * /chats/{cid}), else Auto: everything you may use, limited by the rounds alone. */ public static createChat( options?: Options, @@ -9998,6 +10004,25 @@ export class Chats { }); } + /** + * Chat Tools + * + * The tool groups and tools an answer can be given (`tools` on a question or a conversation), as you have them: each + * tool with what it does (read, asks: it waits for your yes, out: it sends what it's given to another host) and + * whether you have it now, each group with why you don't when you have none of it, and the setting that changes it. + * Built-in groups first, then your extensions' (mcp:, ext:). With `chat_id`, as in that + * conversation's scope (a namespace's own memory is there only in a conversation about that namespace). Also: the most + * tool calls an answer may be given, and your latest choice, which new conversations start with. + */ + public static chatTools( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/chats/tools", + ...options, + }); + } + /** * Delete Chat */ @@ -10028,7 +10053,9 @@ export class Chats { /** * Update Chat * - * Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one). + * Rename a conversation, change what it draws on, the model that answers in it (null: the configured one), or + * what its answers may use (`tools`, null: Auto) and at most how many tool calls each makes (`max_tool_calls`, null: + * no limit but the rounds). The choice of tools and limit is also what your new conversations start with. */ public static updateChat( options: Options, @@ -10095,6 +10122,11 @@ export class Chats { * text, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the * new version, the message it follows and its versions); so does a question beside others under `parent`. * + * What the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a + * search) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's + * choice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its + * record keeps what it was given and used. + * * `/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the * conversation so far into its summary now; each streams cleared or compacted, then its reply as an answer. * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 9a4c4cfe5..e12ad2a8d 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -3383,6 +3383,18 @@ export type Chat = { * the model chosen for this conversation; null: the configured one */ model?: string | null; + /** + * Tools + * + * the tool groups and tools its answers may use; null: Auto + */ + tools?: Array | null; + /** + * Max Tool Calls + * + * at most this many tool calls in each answer; null: no limit but the rounds + */ + max_tool_calls?: number | null; /** * Created At */ @@ -3441,9 +3453,15 @@ export type ChatCapabilities = { /** * Max Steps * - * the most tool calls it makes for one answer + * the most rounds of tool calls it makes for one answer (a round makes one call or more) */ max_steps: number; + /** + * Max Tool Calls + * + * the most tool calls a question may give one answer (`max_tool_calls`) + */ + max_tool_calls?: number; /** * Check * @@ -3562,6 +3580,18 @@ export type ChatCreate = { * the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one */ model?: string | null; + /** + * Tools + * + * what its answers may use: tool groups and tools (GET /chats/tools); [] for none; left out: your latest choice (PATCH /chats/{cid}), else Auto + */ + tools?: Array | null; + /** + * Max Tool Calls + * + * at most this many tool calls in each answer (1 to ai.max_tool_calls); left out: your latest choice + */ + max_tool_calls?: number | null; }; /** @@ -3742,6 +3772,18 @@ export type ChatSummary = { * the model chosen for this conversation; null: the configured one */ model?: string | null; + /** + * Tools + * + * the tool groups and tools its answers may use; null: Auto + */ + tools?: Array | null; + /** + * Max Tool Calls + * + * at most this many tool calls in each answer; null: no limit but the rounds + */ + max_tool_calls?: number | null; /** * Created At */ @@ -3753,6 +3795,35 @@ export type ChatSummary = { [key: string]: unknown; }; +/** + * ChatTools + */ +export type ChatTools = { + /** + * Groups + * + * the built-in groups, then your extensions' (mcp:, ext:) + */ + groups: Array; + /** + * Max Tool Calls + * + * the most tool calls one answer may be given (ai.max_tool_calls) + */ + max_tool_calls: number; + /** + * your latest choice, which new conversations start with + */ + remembered: ToolChoice; + /** + * Namespaces Note + * + * what the choice of namespaces doesn't narrow + */ + namespaces_note?: string; + [key: string]: unknown; +}; + /** * ChatUpdate */ @@ -3768,6 +3839,18 @@ export type ChatUpdate = { * the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one */ model?: string | null; + /** + * Tools + * + * what its answers may use: tool groups and tools (GET /chats/tools); [] for none; null: Auto (everything you may use). Your new conversations start with it too + */ + tools?: Array | null; + /** + * Max Tool Calls + * + * at most this many tool calls in each answer (1 to ai.max_tool_calls); null: no limit but the rounds + */ + max_tool_calls?: number | null; }; /** @@ -10871,6 +10954,22 @@ export type MessageCreate = { * voice: said in a voice conversation (the question's record keeps it) */ via?: "voice" | null; + /** + * Tools + * + * what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins + */ + tools?: Array | null; + /** + * Max Tool Calls + * + * at most this many tool calls in this answer (1 to ai.max_tool_calls); null: the conversation's + */ + max_tool_calls?: number | null; + /** + * what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read + */ + scope?: ChatScope | null; }; /** @@ -21202,6 +21301,101 @@ export type TokenPair = { [key: string]: unknown; }; +/** + * ToolChoice + */ +export type ToolChoice = { + /** + * Tools + */ + tools?: Array | null; + /** + * Max Tool Calls + */ + max_tool_calls?: number | null; + [key: string]: unknown; +}; + +/** + * ToolGroup + */ +export type ToolGroup = { + /** + * Name + * + * what `tools` names it by: a built-in group, mcp: or ext: + */ + name: string; + /** + * Label + */ + label: string; + /** + * Tools + */ + tools: Array; + /** + * Asks + * + * one of its tools waits for your yes + */ + asks: boolean; + /** + * Out + * + * one of its tools sends data to another host + */ + out: boolean; + /** + * Available + * + * you have one of its tools now + */ + available: boolean; + /** + * Why + * + * why you don't, when you have none + */ + why?: string | null; + /** + * Setting + * + * the setting that would change that (section.key), for admins + */ + setting?: string | null; + [key: string]: unknown; +}; + +/** + * ToolInfo + */ +export type ToolInfo = { + /** + * Name + */ + name: string; + /** + * Effect + * + * read: it looks things up; asks: it waits for your yes; out: it sends what it's given to another host + */ + effect: "read" | "asks" | "out"; + /** + * Out + * + * it sends what it's given to another host (and asks first after outside text) + */ + out?: boolean; + /** + * Available + * + * you have it now + */ + available: boolean; + [key: string]: unknown; +}; + /** * ToolStep */ @@ -38825,6 +39019,36 @@ export type ChatCapabilitiesResponses = { export type ChatCapabilitiesResponse = ChatCapabilitiesResponses[keyof ChatCapabilitiesResponses]; +export type ChatToolsData = { + body?: never; + path?: never; + query?: { + /** + * Chat Id + */ + chat_id?: number | null; + }; + url: "/api/v1/chats/tools"; +}; + +export type ChatToolsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ChatToolsError = ChatToolsErrors[keyof ChatToolsErrors]; + +export type ChatToolsResponses = { + /** + * Successful Response + */ + 200: ChatTools; +}; + +export type ChatToolsResponse = ChatToolsResponses[keyof ChatToolsResponses]; + export type DeleteChatData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index c9f6b8a70..a1f398067 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -24891,6 +24891,7 @@ "chats" ], "summary": "Create Chat", + "description": "A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH\n/chats/{cid}), else Auto: everything you may use, limited by the rounds alone.", "operationId": "create_chat", "requestBody": { "content": { @@ -24955,6 +24956,56 @@ } } }, + "/api/v1/chats/tools": { + "get": { + "tags": [ + "chats" + ], + "summary": "Chat Tools", + "description": "The tool groups and tools an answer can be given (`tools` on a question or a conversation), as you have them: each\ntool with what it does (read, asks: it waits for your yes, out: it sends what it's given to another host) and\nwhether you have it now, each group with why you don't when you have none of it, and the setting that changes it.\nBuilt-in groups first, then your extensions' (mcp:, ext:). With `chat_id`, as in that\nconversation's scope (a namespace's own memory is there only in a conversation about that namespace). Also: the most\ntool calls an answer may be given, and your latest choice, which new conversations start with.", + "operationId": "chat_tools", + "parameters": [ + { + "name": "chat_id", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Chat Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatTools" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/chats/{cid}": { "get": { "tags": [ @@ -25002,7 +25053,7 @@ "chats" ], "summary": "Update Chat", - "description": "Rename a conversation, change what it draws on, or the model that answers in it (null: the configured one).", + "description": "Rename a conversation, change what it draws on, the model that answers in it (null: the configured one), or\nwhat its answers may use (`tools`, null: Auto) and at most how many tool calls each makes (`max_tool_calls`, null:\nno limit but the rounds). The choice of tools and limit is also what your new conversations start with.", "operationId": "update_chat", "parameters": [ { @@ -25200,7 +25251,7 @@ "chats" ], "summary": "Send Message", - "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nA conversation is a tree of versions (GET /chats/{cid}: `tree`, and each message's `versions`); the question goes\non from `parent` when given (any message of the conversation), else from the version showing (`head`). With\n`regenerate` (an answer's id) its question is answered again, as a new version beside that answer (`content` isn't\nneeded; `model` may pick another). With `edit`, one of your earlier questions is edited: this question becomes a new\nversion beside it and is answered, and the old one stays with what followed it (404 if it isn't a question in this\nconversation). Either keeps the page the question was asked from (where, and the highlighted text; not the page's\ntext, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the\nnew version, the message it follows and its versions); so does a question beside others under `parent`.\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).", + "description": "Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model\nreads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST\n/chats/{cid}/stop: what came before is saved, marked stopped), meta (the answer's record, kept with it: what produced\nit), done (the saved message id).\n\nA conversation is a tree of versions (GET /chats/{cid}: `tree`, and each message's `versions`); the question goes\non from `parent` when given (any message of the conversation), else from the version showing (`head`). With\n`regenerate` (an answer's id) its question is answered again, as a new version beside that answer (`content` isn't\nneeded; `model` may pick another). With `edit`, one of your earlier questions is edited: this question becomes a new\nversion beside it and is answered, and the old one stays with what followed it (404 if it isn't a question in this\nconversation). Either keeps the page the question was asked from (where, and the highlighted text; not the page's\ntext, which isn't kept) unless `context` is given, and streams `branch` first ({message, parent, versions}: the\nnew version, the message it follows and its versions); so does a question beside others under `parent`.\n\nWhat the answer may use: `tools` (tool groups and tools, GET /chats/tools; [] for none, when it answers from a\nsearch) and `max_tool_calls` (at most that many tool calls, in as many rounds as it takes), else the conversation's\nchoice (PATCH /chats/{cid}); and `scope`, what this answer alone draws on (the conversation keeps its own). Its\nrecord keeps what it was given and used.\n\n`/clear` makes the model start afresh (the messages stay to read) and `/compact [what to keep]` folds the\nconversation so far into its summary now; each streams cleared or compacted, then its reply as an answer.\n\nWith `browser` (the session of a browser's Lens side panel, POST /browser/sessions), the assistant can use that\nbrowser while the side panel is connected: open pages, read them, click, type and download files into the\nconversation (docs/browser-assistant.md).", "operationId": "send_message", "parameters": [ { @@ -36229,6 +36280,33 @@ "title": "Model", "description": "the model chosen for this conversation; null: the configured one" }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "the tool groups and tools its answers may use; null: Auto" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls", + "description": "at most this many tool calls in each answer; null: no limit but the rounds" + }, "created_at": { "anyOf": [ { @@ -36320,7 +36398,13 @@ "max_steps": { "type": "integer", "title": "Max Steps", - "description": "the most tool calls it makes for one answer" + "description": "the most rounds of tool calls it makes for one answer (a round makes one call or more)" + }, + "max_tool_calls": { + "type": "integer", + "title": "Max Tool Calls", + "description": "the most tool calls a question may give one answer (`max_tool_calls`)", + "default": 50 }, "check": { "type": "boolean", @@ -36517,6 +36601,35 @@ ], "title": "Model", "description": "the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 100 + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "what its answers may use: tool groups and tools (GET /chats/tools); [] for none; left out: your latest choice (PATCH /chats/{cid}), else Auto" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer", + "minimum": 1.0 + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls", + "description": "at most this many tool calls in each answer (1 to ai.max_tool_calls); left out: your latest choice" } }, "additionalProperties": false, @@ -36864,6 +36977,33 @@ "title": "Model", "description": "the model chosen for this conversation; null: the configured one" }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "the tool groups and tools its answers may use; null: Auto" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls", + "description": "at most this many tool calls in each answer; null: no limit but the rounds" + }, "created_at": { "anyOf": [ { @@ -36895,6 +37035,41 @@ ], "title": "ChatSummary" }, + "ChatTools": { + "properties": { + "groups": { + "items": { + "$ref": "#/components/schemas/ToolGroup" + }, + "type": "array", + "title": "Groups", + "description": "the built-in groups, then your extensions' (mcp:, ext:)" + }, + "max_tool_calls": { + "type": "integer", + "title": "Max Tool Calls", + "description": "the most tool calls one answer may be given (ai.max_tool_calls)" + }, + "remembered": { + "$ref": "#/components/schemas/ToolChoice", + "description": "your latest choice, which new conversations start with" + }, + "namespaces_note": { + "type": "string", + "title": "Namespaces Note", + "description": "what the choice of namespaces doesn't narrow", + "default": "Web and MCP tools reach beyond your namespaces: the namespaces you pick narrow the archive's tools only." + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "groups", + "max_tool_calls", + "remembered" + ], + "title": "ChatTools" + }, "ChatUpdate": { "properties": { "title": { @@ -36929,6 +37104,35 @@ ], "title": "Model", "description": "the model that answers in it (one of GET /chats/capabilities `models`); null: the configured one" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 100 + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "what its answers may use: tool groups and tools (GET /chats/tools); [] for none; null: Auto (everything you may use). Your new conversations start with it too" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer", + "minimum": 1.0 + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls", + "description": "at most this many tool calls in each answer (1 to ai.max_tool_calls); null: no limit but the rounds" } }, "additionalProperties": false, @@ -48974,6 +49178,46 @@ ], "title": "Via", "description": "voice: said in a voice conversation (the question's record keeps it)" + }, + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 100 + }, + { + "type": "null" + } + ], + "title": "Tools", + "description": "what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer", + "minimum": 1.0 + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls", + "description": "at most this many tool calls in this answer (1 to ai.max_tool_calls); null: the conversation's" + }, + "scope": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatScope" + }, + { + "type": "null" + } + ], + "description": "what this answer draws on, instead of the conversation's scope (which stays as it is); {} for everything you can read" } }, "additionalProperties": false, @@ -66791,6 +67035,145 @@ ], "title": "TokenPair" }, + "ToolChoice": { + "properties": { + "tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "max_tool_calls": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Tool Calls" + } + }, + "additionalProperties": true, + "type": "object", + "title": "ToolChoice" + }, + "ToolGroup": { + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "what `tools` names it by: a built-in group, mcp: or ext:" + }, + "label": { + "type": "string", + "title": "Label" + }, + "tools": { + "items": { + "$ref": "#/components/schemas/ToolInfo" + }, + "type": "array", + "title": "Tools" + }, + "asks": { + "type": "boolean", + "title": "Asks", + "description": "one of its tools waits for your yes" + }, + "out": { + "type": "boolean", + "title": "Out", + "description": "one of its tools sends data to another host" + }, + "available": { + "type": "boolean", + "title": "Available", + "description": "you have one of its tools now" + }, + "why": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Why", + "description": "why you don't, when you have none" + }, + "setting": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Setting", + "description": "the setting that would change that (section.key), for admins" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "name", + "label", + "tools", + "asks", + "out", + "available" + ], + "title": "ToolGroup" + }, + "ToolInfo": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "effect": { + "type": "string", + "enum": [ + "read", + "asks", + "out" + ], + "title": "Effect", + "description": "read: it looks things up; asks: it waits for your yes; out: it sends what it's given to another host" + }, + "out": { + "type": "boolean", + "title": "Out", + "description": "it sends what it's given to another host (and asks first after outside text)", + "default": false + }, + "available": { + "type": "boolean", + "title": "Available", + "description": "you have it now" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "name", + "effect", + "available" + ], + "title": "ToolInfo" + }, "ToolStep": { "properties": { "tool": { From 8759a60f561100bfa245b401bed942f38d9ac53f Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:43:06 +0000 Subject: [PATCH 06/12] feat(chat): a choice made as a conversation starts is the latest choice Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- docs/assistant.md | 5 +++-- fastapi_backend/app/api/v1/routes/chats.py | 7 +++++-- fastapi_backend/tests/api/test_chat_tools.py | 3 +++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 5 +++-- nextjs-frontend/openapi.json | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/assistant.md b/docs/assistant.md index 39693560d..eb127b4b1 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -221,8 +221,9 @@ tools it may use, how many tool calls it may make, and which namespaces it reads [namespace picker](#picking-the-namespace-for-a-conversation-over-everything). - **Remembered.** Each answer records what it was offered and its limits (`tools`, `limits`, `scope`), and each question records the options it was asked with ([What each answer records](#what-each-answer-records)). A - conversation keeps its `tools` and `max_tool_calls`. The person's latest choice is kept on their account, so a new - conversation starts with it (`remembered` in `GET /chats/tools`) unless `POST /chats` says otherwise. + conversation keeps its `tools` and `max_tool_calls`. The person's latest choice (the last `POST /chats` or `PATCH + /chats/` that set them) is kept on their account, so a new conversation starts with it (`remembered` in `GET + /chats/tools`) unless `POST /chats` says otherwise. - **Web and MCP tools reach beyond your namespaces.** The namespaces picked narrow the archive's tools only. A web search, an MCP server or the browser reads and sends wherever it reaches. diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py index 17a679cff..3b649b82c 100644 --- a/fastapi_backend/app/api/v1/routes/chats.py +++ b/fastapi_backend/app/api/v1/routes/chats.py @@ -149,8 +149,9 @@ def _limit(cfg: dict[str, Any], n: int | None) -> int | None: @router.post("/chats") def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body: ChatCreate | None = None) -> Created: - """A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH - /chats/{cid}), else Auto: everything you may use, limited by the rounds alone.""" + """A conversation. Its answers may use what `tools` and `max_tool_calls` say, which become your latest choice; + left out, your latest choice (here or PATCH /chats/{cid}), else Auto: everything you may use, limited by the rounds + alone.""" body = body or ChatCreate() if body.kind == "setup" and not user.admin: raise HTTPException(403, "only admins set up the server") @@ -158,6 +159,8 @@ def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body tools = _tools(_box(db, cfg, user, acl, None, request), body.tools) if "tools" in body.model_fields_set else last["tools"] limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in body.model_fields_set else _most(cfg, last["max_tool_calls"]) cid = chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind, tools, limit) + if body.kind == "chat" and {"tools", "max_tool_calls"} & body.model_fields_set: + chat.remember_tools(db, user.id, tools, limit) return Created(id=cid) diff --git a/fastapi_backend/tests/api/test_chat_tools.py b/fastapi_backend/tests/api/test_chat_tools.py index 44ecb2a9b..b89e4f743 100644 --- a/fastapi_backend/tests/api/test_chat_tools.py +++ b/fastapi_backend/tests/api/test_chat_tools.py @@ -215,6 +215,9 @@ def test_a_conversation_keeps_its_choice_and_new_ones_start_with_it(app, db, cfg new = c.post("/api/v1/chats", headers=h, json={"tools": None, "max_tool_calls": None}).json()["id"] got = c.get(f"/api/v1/chats/{new}", headers=h).json() assert (got["tools"], got["max_tool_calls"]) == (None, None) + # a choice made as one starts is the latest choice too + c.post("/api/v1/chats", headers=h, json={"tools": ["notes"], "max_tool_calls": 4}) + assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": ["notes"], "max_tool_calls": 4} c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": None, "max_tool_calls": None}) assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": None, "max_tool_calls": None} # someone else's diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 8acb7e7d5..44ef4f17a 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -9972,8 +9972,9 @@ export class Chats { /** * Create Chat * - * A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH - * /chats/{cid}), else Auto: everything you may use, limited by the rounds alone. + * A conversation. Its answers may use what `tools` and `max_tool_calls` say, which become your latest choice; + * left out, your latest choice (here or PATCH /chats/{cid}), else Auto: everything you may use, limited by the rounds + * alone. */ public static createChat( options?: Options, diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index a1f398067..3e178c3a3 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -24891,7 +24891,7 @@ "chats" ], "summary": "Create Chat", - "description": "A conversation. Its answers may use what `tools` and `max_tool_calls` say; left out, your latest choice (PATCH\n/chats/{cid}), else Auto: everything you may use, limited by the rounds alone.", + "description": "A conversation. Its answers may use what `tools` and `max_tool_calls` say, which become your latest choice;\nleft out, your latest choice (here or PATCH /chats/{cid}), else Auto: everything you may use, limited by the rounds\nalone.", "operationId": "create_chat", "requestBody": { "content": { From 73d10e79edbc70d55ab48d7799a460238877fd2b Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:57:41 +0000 Subject: [PATCH 07/12] fix(chat): setting the server up keeps every tool, whatever the latest choice Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/app/api/v1/routes/chats.py | 6 ++++-- fastapi_backend/app/schemas/chats.py | 2 +- fastapi_backend/tests/api/test_chat_tools.py | 5 +++++ nextjs-frontend/app/openapi-client/types.gen.ts | 2 +- nextjs-frontend/openapi.json | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py index 3b649b82c..651a0a22a 100644 --- a/fastapi_backend/app/api/v1/routes/chats.py +++ b/fastapi_backend/app/api/v1/routes/chats.py @@ -155,7 +155,8 @@ def create_chat(user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request, body body = body or ChatCreate() if body.kind == "setup" and not user.admin: raise HTTPException(403, "only admins set up the server") - last = chat.remembered_tools(db, user.id) + # setting the server up keeps every tool: your latest choice is for conversations about the archive + last = chat.remembered_tools(db, user.id) if body.kind == "chat" else {"tools": None, "max_tool_calls": None} tools = _tools(_box(db, cfg, user, acl, None, request), body.tools) if "tools" in body.model_fields_set else last["tools"] limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in body.model_fields_set else _most(cfg, last["max_tool_calls"]) cid = chat.create(db, user.id, body.title, _scope(acl, db, body.scope), _model(cfg, body.model), body.kind, tools, limit) @@ -245,7 +246,8 @@ def update_chat(cid: int, body: ChatUpdate, user: Writer, acl: Acl, db: Db, cfg: tools = _tools(_box(db, cfg, user, acl, c.get("scope"), request), body.tools) if "tools" in chose else c.get("tools") limit = _limit(cfg, body.max_tool_calls) if "max_tool_calls" in chose else c.get("max_tool_calls") db.q("UPDATE $r SET tools = $t, max_tool_calls = $n", r=R("chat", cid), t=tools, n=limit) - chat.remember_tools(db, user.id, tools, limit) + if c.get("kind") == "chat": + chat.remember_tools(db, user.id, tools, limit) return Ok() diff --git a/fastapi_backend/app/schemas/chats.py b/fastapi_backend/app/schemas/chats.py index 4c8a7043d..9a2ce6dd1 100644 --- a/fastapi_backend/app/schemas/chats.py +++ b/fastapi_backend/app/schemas/chats.py @@ -299,7 +299,7 @@ class MessageCreate(RequestModel): tools: list[str] | None = Field( None, max_length=100, - description="what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists " + description="what this answer may use: tool groups (archive, notes, mcp:tracker) and tools, as GET /chats/tools lists " "them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An " "admin's switch still wins", ) diff --git a/fastapi_backend/tests/api/test_chat_tools.py b/fastapi_backend/tests/api/test_chat_tools.py index b89e4f743..39f41151a 100644 --- a/fastapi_backend/tests/api/test_chat_tools.py +++ b/fastapi_backend/tests/api/test_chat_tools.py @@ -220,6 +220,11 @@ def test_a_conversation_keeps_its_choice_and_new_ones_start_with_it(app, db, cfg assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": ["notes"], "max_tool_calls": 4} c.patch(f"/api/v1/chats/{cid}", headers=h, json={"tools": None, "max_tool_calls": None}) assert c.get("/api/v1/chats/tools", headers=h).json()["remembered"] == {"tools": None, "max_tool_calls": None} + # setting the server up keeps every tool + ca, ha = s.cl["admin"] + ca.patch(f"/api/v1/chats/{ca.post('/api/v1/chats', headers=ha, json={}).json()['id']}", headers=ha, json={"tools": ["archive"]}) + setup = ca.post("/api/v1/chats", headers=ha, json={"kind": "setup"}).json()["id"] + assert ca.get(f"/api/v1/chats/{setup}", headers=ha).json()["tools"] is None # someone else's cv, hv = s.cl["viewer"] assert cv.get("/api/v1/chats/tools", headers=hv).json()["remembered"] == {"tools": None, "max_tool_calls": None} diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index e12ad2a8d..5dac559b2 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -10957,7 +10957,7 @@ export type MessageCreate = { /** * Tools * - * what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins + * what this answer may use: tool groups (archive, notes, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins */ tools?: Array | null; /** diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 3e178c3a3..3cc88e66a 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -49193,7 +49193,7 @@ } ], "title": "Tools", - "description": "what this answer may use: tool groups (archive, web, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins" + "description": "what this answer may use: tool groups (archive, notes, mcp:tracker) and tools, as GET /chats/tools lists them; [] for none (it answers from a search); null: the conversation's choice, else Auto (everything you may use). An admin's switch still wins" }, "max_tool_calls": { "anyOf": [ From 3f0b9de311e8e838007a51f32d2915f77a914aad Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 02:09:35 +0000 Subject: [PATCH 08/12] test(entities): wait for the typed search before counting merge options The find box empties while the list for what was typed loads, so on a busy runner the first list could pass the wait and then vanish. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- nextjs-frontend/__tests__/entities-curate.test.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nextjs-frontend/__tests__/entities-curate.test.tsx b/nextjs-frontend/__tests__/entities-curate.test.tsx index e7c7e7473..65c89e21e 100644 --- a/nextjs-frontend/__tests__/entities-curate.test.tsx +++ b/nextjs-frontend/__tests__/entities-curate.test.tsx @@ -79,6 +79,8 @@ describe("merging into an entity", () => { const box = screen.getByRole("combobox", { name: "Entity to merge into Northwind Labs" }); expect(box).toHaveFocus(); fireEvent.change(box, { target: { value: "north" } }); + // the list for what was typed, not the first one (which empties while the typed one loads, on a slow runner) + await waitFor(() => expect(m(Entities.listEntities).mock.calls.some(([o]) => o.query.q === "north")).toBe(true)); // itself and the entities that are always there can't be merged await waitFor(() => expect(screen.getAllByRole("option")).toHaveLength(1)); expect(screen.getByRole("option")).toHaveTextContent("North Wind Labs"); From e6b5cfa32e51c1693a65eef8a151ec93e7ba19a0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 03:19:45 +0000 Subject: [PATCH 09/12] test(settings): the AI section's server values include ai.max_tool_calls Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- nextjs-frontend/__tests__/settings-web-search.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/nextjs-frontend/__tests__/settings-web-search.test.tsx b/nextjs-frontend/__tests__/settings-web-search.test.tsx index 3081ea8a1..9e164a653 100644 --- a/nextjs-frontend/__tests__/settings-web-search.test.tsx +++ b/nextjs-frontend/__tests__/settings-web-search.test.tsx @@ -32,6 +32,7 @@ const web = (values: Record) => injection_check: "patterns", disabled_tools: [], max_steps: 6, + max_tool_calls: 50, browser_steps: 25, max_transcript_reads: 20, confirm_over_recordings: 100, From a9e21e57fb5290098b7d85dcdd8ac292b3417830 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 04:13:27 +0000 Subject: [PATCH 10/12] fix(chat): find_resources belongs to the archive's tools Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/app/domain/ai_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 554c9306c..7fde22b22 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -537,7 +537,7 @@ def builtin_names(): GROUPS = { "archive": { "label": "Archive", - "tools": ["search_transcripts", "find_similar", "list_recordings", "read_transcript", "recording_outputs"], + "tools": ["search_transcripts", "find_resources", "find_similar", "list_recordings", "read_transcript", "recording_outputs"], }, "graph": { "label": "Graph", From 98b7a8bb29d76bacd00565938a2e4f1de3b3b607 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 04:18:03 +0000 Subject: [PATCH 11/12] docs(chat): a new built-in tool joins a tool group Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- fastapi_backend/app/domain/ai_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 1248ab17e..6e9375bb2 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -551,8 +551,9 @@ def builtin_names(): BROWSER_OUT = {"browser_open", "browser_click", "browser_type", "browser_press"} # The groups of tools a question can be given (POST /chats/{cid}/messages `tools`; docs/assistant.md, "Choosing tools, -# namespaces and a limit"), in the order the web app shows them, each built-in tool in one. Extensions add their own: -# mcp: for an MCP server's tools, ext: for another extension's (extensions.Active.groups). +# namespaces and a limit"), in the order the web app shows them, each built-in tool in one: a new tool joins one here +# (test_chat_tools checks). Extensions add their own: mcp: for an MCP server's tools, ext: for another +# extension's (extensions.Active.groups). GROUPS = { "archive": { "label": "Archive", From e8073d62f73da9d026aabf019fd351800316ce4c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:53:45 +0000 Subject: [PATCH 12/12] feat(chat): a Tools chip to pick tools and a limit, and all namespaces in one tap Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- CHANGELOG.md | 2 +- docs/assistant.md | 5 +- .../__tests__/chat-tools-picker.test.tsx | 149 ++++++++++++ nextjs-frontend/components/chat/chat-app.tsx | 47 +++- nextjs-frontend/components/chat/composer.tsx | 19 +- .../components/chat/tools-picker.tsx | 224 ++++++++++++++++++ 6 files changed, 439 insertions(+), 7 deletions(-) create mode 100644 nextjs-frontend/__tests__/chat-tools-picker.test.tsx create mode 100644 nextjs-frontend/components/chat/tools-picker.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index fec5eb894..85389da5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version assistant → Most tool calls a question may ask for, 50 by default) and narrow one answer to its own namespaces (`scope`). `GET /api/v1/chats/tools` lists the groups a person has, what each tool does (reads, asks first, sends data out) and why a group is missing. An admin's switches still win, and new conversations start with the person's last - choice. The web app's Tools chip is in progress + choice. In the web app, a Tools chip beside the scope picks them, and "All my namespaces" tops the scope's list ([Choosing tools, namespaces and a limit](docs/assistant.md#choosing-tools-namespaces-and-a-limit)). - **Entities found by spaCy, fetched when picked.** Settings → Analysis → Entity extraction → spaCy now fetches spaCy (MIT) and its small model the first time an asset needs it (about 60 MB): English, German, or one for many diff --git a/docs/assistant.md b/docs/assistant.md index bb1a27587..5d010c804 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -288,7 +288,10 @@ tools it may use, how many tool calls it may make, and which namespaces it reads - **Web and MCP tools reach beyond your namespaces.** The namespaces picked narrow the archive's tools only. A web search, an MCP server or the browser reads and sends wherever it reaches. -In the web app, a Tools chip beside the composer's scope picks these (in progress). +In the web app, the **Tools** chip beside the composer's scope picks these for the conversation (Enter or a tap opens +it): Auto, a switch for each group (marked "needs approval" or "sends data out"; one you don't have says why, with a +link to its setting for admins) and a limit on tool calls per answer. A conversation not started yet opens on your +latest choice. "All my namespaces" at the top of the scope's namespaces goes back to everything in one tap. Refine later: tool choice in chat rooms and voice mode, a default choice per namespace, and the model suggesting the tools a question needs. diff --git a/nextjs-frontend/__tests__/chat-tools-picker.test.tsx b/nextjs-frontend/__tests__/chat-tools-picker.test.tsx new file mode 100644 index 000000000..a2c642342 --- /dev/null +++ b/nextjs-frontend/__tests__/chat-tools-picker.test.tsx @@ -0,0 +1,149 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, within } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import type { ChatTools } from "@/app/openapi-client/types.gen"; +import { ScopeBar } from "@/components/chat/composer"; +import { choiceLabel, settingLink, ToolsChip, type ToolChoice } from "@/components/chat/tools-picker"; + +jest.mock("@/app/openapi-client", () => ({ Chats: { chatTools: jest.fn() }, Collections: {} })); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +jest.mock("@/lib/hooks/session", () => ({ + useArchive: () => ({ + namespaces: [ + { name: "pods", recordings: 3, minutes: 40 }, + { name: "calls", recordings: 2, minutes: 10 }, + ], + }), +})); +jest.mock("@/components/import/pending", () => ({ chooseFiles: jest.fn(() => Promise.resolve([])) })); + +beforeAll(() => { + // Radix's popper measures what it places + global.ResizeObserver = class { + observe() {} + unobserve() {} + disconnect() {} + } as unknown as typeof ResizeObserver; +}); + +const group = (name: string, label: string, more: Partial = {}) => ({ + name, + label, + tools: [{ name: `${name}_tool`, effect: "read" as const, out: false, available: more.available ?? true }], + asks: false, + out: false, + available: true, + why: null, + setting: null, + ...more, +}); + +const CATALOG: ChatTools = { + groups: [ + group("archive", "Archive"), + group("notes", "Notes", { asks: true }), + group("browser", "Your browser", { + out: true, + available: false, + why: "switched off by an admin", + setting: "ai.browser", + }), + group("server", "Server", { available: false, why: "for admins" }), + ], + max_tool_calls: 12, + remembered: { tools: null, max_tool_calls: null }, + namespaces_note: + "Web and MCP tools reach beyond your namespaces: the namespaces you pick narrow the archive's tools only.", +}; + +function chip(value: ToolChoice, admin = false) { + const onChange = jest.fn(); + render(); + const button = screen.getByRole("button", { name: /Choose the tools for this conversation/ }); + return { onChange, button }; +} + +describe("the Tools chip", () => { + it("is a button, so Enter opens it, and says what is picked", () => { + const { button } = chip({ tools: null, max_tool_calls: null }); + expect(button.tagName).toBe("BUTTON"); + expect(button).toHaveTextContent("Tools: Auto"); + expect(choiceLabel({ tools: [], max_tool_calls: null })).toBe("Tools: no tools"); + expect(choiceLabel({ tools: ["archive", "notes"], max_tool_calls: 5 })).toBe("Tools: 2 picked · 5 calls"); + }); + + it("opens on Auto, with every group you have on and the others saying why", () => { + const { button } = chip({ tools: null, max_tool_calls: null }); + fireEvent.click(button); + const dialog = screen.getByRole("dialog"); + expect(within(dialog).getByRole("switch", { name: /Auto/ })).toBeChecked(); + expect(within(dialog).getByRole("switch", { name: "Archive" })).toBeChecked(); + expect(within(dialog).getByRole("switch", { name: "Server" })).toBeDisabled(); + expect(within(dialog).getByText("For admins.")).toBeInTheDocument(); + expect(within(dialog).getByText("needs approval")).toBeInTheDocument(); + expect(within(dialog).getByText("sends data out")).toBeInTheDocument(); + expect(within(dialog).getByText(/Web and MCP tools reach beyond your namespaces/)).toBeInTheDocument(); + // only admins are sent to the setting + expect(within(dialog).queryByRole("link", { name: "Open settings" })).not.toBeInTheDocument(); + }); + + it("links an admin to the setting that switches a group back on", () => { + fireEvent.click(chip({ tools: null, max_tool_calls: null }, true).button); + expect(screen.getByRole("link", { name: "Open settings" })).toHaveAttribute("href", settingLink("ai.browser")); + expect(settingLink("ai.browser")).toBe("/settings/ai?field=ai.browser"); + }); + + it("turning a group off leaves Auto with the rest of what you have", () => { + const { button, onChange } = chip({ tools: null, max_tool_calls: null }); + fireEvent.click(button); + fireEvent.click(screen.getByRole("switch", { name: "Notes" })); + expect(onChange).toHaveBeenCalledWith({ tools: ["archive"], max_tool_calls: null }); + }); + + it("picks groups, keeps single tools, and goes back to Auto", () => { + const { button, onChange } = chip({ tools: ["archive", "find_entities"], max_tool_calls: null }); + fireEvent.click(button); + expect(screen.getByRole("switch", { name: "Notes" })).not.toBeChecked(); + expect(screen.getByText("Also picked: find_entities")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("switch", { name: "Notes" })); + expect(onChange).toHaveBeenLastCalledWith({ tools: ["archive", "find_entities", "notes"], max_tool_calls: null }); + fireEvent.click(screen.getByRole("switch", { name: /Auto/ })); + expect(onChange).toHaveBeenLastCalledWith({ tools: null, max_tool_calls: null }); + }); + + it("limits the tool calls, from 1 to the server's most", () => { + let r = chip({ tools: null, max_tool_calls: null }); + fireEvent.click(r.button); + fireEvent.click(screen.getByRole("switch", { name: "Limit tool calls per answer" })); + expect(r.onChange).toHaveBeenCalledWith({ tools: null, max_tool_calls: 10 }); + cleanup(); + + r = chip({ tools: null, max_tool_calls: 12 }); + fireEvent.click(r.button); + expect(screen.getByRole("button", { name: "More tool calls" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Fewer tool calls" })); + expect(r.onChange).toHaveBeenLastCalledWith({ tools: null, max_tool_calls: 11 }); + fireEvent.change(screen.getByRole("spinbutton", { name: "Tool calls per answer" }), { target: { value: "40" } }); + expect(r.onChange).toHaveBeenCalledTimes(1); // past the most: not sent + fireEvent.change(screen.getByRole("spinbutton", { name: "Tool calls per answer" }), { target: { value: "3" } }); + expect(r.onChange).toHaveBeenLastCalledWith({ tools: null, max_tool_calls: 3 }); + }); +}); + +describe("the scope's namespaces", () => { + it("go back to all of them in one tap", () => { + const onChange = jest.fn(); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + + + , + ); + fireEvent.click(screen.getByRole("button", { name: "Scope" })); + const all = screen.getByRole("checkbox", { name: "All my namespaces" }); + expect(all).not.toBeChecked(); + fireEvent.click(all); + expect(onChange).toHaveBeenCalledWith({ namespaces: undefined }); + }); +}); diff --git a/nextjs-frontend/components/chat/chat-app.tsx b/nextjs-frontend/components/chat/chat-app.tsx index f8ed6d229..92ad8d423 100644 --- a/nextjs-frontend/components/chat/chat-app.tsx +++ b/nextjs-frontend/components/chat/chat-app.tsx @@ -19,6 +19,7 @@ import { EmptyChat } from "@/components/chat/empty"; import { NamespaceOffer, spokenPick } from "@/components/chat/namespace-offer"; import { fromApiScope, scopeFromParams, toApiScope, type Scope } from "@/components/chat/scope"; import { CitationSheet, SourcesPanel, SourcesSheet } from "@/components/chat/sources"; +import { ToolsChip, useToolCatalog, type ToolChoice } from "@/components/chat/tools-picker"; import { applyEvent, newTurn, @@ -103,6 +104,7 @@ export function ChatApp() { const linkScope = useMemo(() => scopeFromParams(new URLSearchParams(search.toString())), [search]); const [draftScope, setDraftScope] = useState(() => linkScope); const [draftModel, setDraftModel] = useState(null); // for the conversation that isn't started yet + const [draftTools, setDraftTools] = useState(null); // and its tools, when picked before it starts const [draft, setDraft] = useState(() => search.get("q") ?? ""); const [composing, setComposing] = useState(() => Boolean( @@ -320,11 +322,18 @@ export function ChatApp() { const created = await data( Chats.createChat({ client, - body: kind === "setup" ? { kind } : { scope: toApiScope(draftScope), model: draftModel ?? undefined }, + body: + kind === "setup" + ? { kind } + : { scope: toApiScope(draftScope), model: draftModel ?? undefined, ...(draftTools ?? {}) }, }), ); cid = created.id; qc.invalidateQueries({ queryKey: ["chats"] }); + if (draftTools) { + setDraftTools(null); // it's the latest choice now: the next new conversation starts with it + void qc.invalidateQueries({ queryKey: ["chat-tools"] }); + } setComposing(false); router.replace(`/chat/${cid}`); } catch (e) { @@ -339,7 +348,7 @@ export function ChatApp() { } return run(cid, question, model, attached, edit); }, - [activeId, client, draftScope, draftModel, qc, router, run, toast], + [activeId, client, draftScope, draftModel, draftTools, qc, router, run, toast], ); // What you send while an answer is being written waits its turn, so you never have to wait to type. @@ -599,6 +608,28 @@ export function ChatApp() { } }; const noProvider = llm.known && !llm.configured; + + // what the answers may use: the conversation's choice, else the one picked before it started, else your latest + const usesTools = llm.known && llm.configured && llm.tools; + const toolCatalog = useToolCatalog(activeId, usesTools); + const toolChoice: ToolChoice = + activeId != null + ? { tools: chat.data?.tools ?? null, max_tool_calls: chat.data?.max_tool_calls ?? null } + : (draftTools ?? { + tools: toolCatalog.data?.remembered.tools ?? null, + max_tool_calls: toolCatalog.data?.remembered.max_tool_calls ?? null, + }); + const pickTools = async (v: ToolChoice) => { + if (activeId == null) return setDraftTools(v); + try { + await data(Chats.updateChat({ client, path: { cid: activeId }, body: v })); + await qc.invalidateQueries({ queryKey: ["chat", activeId] }); + void qc.invalidateQueries({ queryKey: ["chats"] }); + void qc.invalidateQueries({ queryKey: ["chat-tools"] }); + } catch (e) { + toast({ title: "Couldn’t change the tools", body: e instanceof Error ? e.message : undefined, tone: "red" }); + } + }; const showThread = activeId != null || liveHere != null; // setting the server up doesn't look in the archive: no sources column to fill const setupChat = activeId != null && chat.data?.kind === "setup"; @@ -851,6 +882,16 @@ export function ChatApp() { models={llm.models} onModel={(m) => void pickModel(m)} openTick={scopeOpenTick} + toolsChip={ + usesTools && !setupChat ? ( + void pickTools(v)} + catalog={toolCatalog.data} + admin={admin} + /> + ) : undefined + } /> {queue.length > 0 && (
    @@ -920,7 +961,7 @@ export function ChatApp() { function ScopeBarWithOpen({ openTick, ...props }: Parameters[0] & { openTick: number }) { const ref = useRef(null); useEffect(() => { - if (openTick) ref.current?.querySelector("button[aria-haspopup='dialog']")?.click(); + if (openTick) ref.current?.querySelector("button[data-scope-open]")?.click(); }, [openTick]); return (
    diff --git a/nextjs-frontend/components/chat/composer.tsx b/nextjs-frontend/components/chat/composer.tsx index a23b49b54..6a6932523 100644 --- a/nextjs-frontend/components/chat/composer.tsx +++ b/nextjs-frontend/components/chat/composer.tsx @@ -145,6 +145,14 @@ function ScopePicker({
    {section === "namespaces" && (
      +
    • + onChange({ ...scope, namespaces: undefined })} + label={All my namespaces} + /> + {namespaceSize(namespaces)} +
    • {namespaces.map((n) => (
    • ))}
    • - None ticked means all your namespaces. Scope never goes beyond what you can read. + Scope never goes beyond what you can read. It narrows the archive’s tools: web and MCP tools reach beyond + your namespaces.
    )} @@ -298,7 +307,8 @@ function useCollectionNames(enabled: boolean): Map { return useMemo(() => new Map((cols.data ?? []).map((c) => [c.id, c.name])), [cols.data]); } -/** Scope chips (each shows its size), "+ Scope", and which model answers (a menu, when there's a choice). */ +/** Scope chips (each shows its size), "+ Scope", the Tools chip, and which model answers (a menu, when there's a + * choice). */ export function ScopeBar({ scope, onChange, @@ -306,6 +316,7 @@ export function ScopeBar({ tools, models = [], onModel, + toolsChip, className, }: { scope: Scope; @@ -315,6 +326,8 @@ export function ScopeBar({ /** The models people may pick (GET /chats/capabilities). */ models?: string[]; onModel?: (m: string) => void; + /** What the answers may use (the Tools chip), after the scope. */ + toolsChip?: ReactNode; className?: string; }) { const { namespaces } = useArchive(); @@ -379,6 +392,7 @@ export function ScopeBar({ + + + setOpen(false)} /> + + + ); +} + +/** Auto, a switch for each group (saying which ask first or send data out, and why one is missing), and the limit. */ +export function ToolsPicker({ + value, + onChange, + catalog, + admin, + onClose, +}: { + value: ToolChoice; + onChange: (v: ToolChoice) => void; + catalog: ChatTools | undefined; + admin: boolean; + onClose: () => void; +}) { + const groups = catalog?.groups ?? []; + const has = groups.filter((g) => g.available).map((g) => g.name); + const names = new Set(groups.map((g) => g.name)); + const auto = value.tools == null; + const on = (g: ToolGroup) => g.available && (auto || Boolean(value.tools?.includes(g.name))); + const flip = (g: ToolGroup, v: boolean) => { + const now = auto ? has : (value.tools ?? []); + const others = now.filter((n) => n !== g.name); + onChange({ ...value, tools: v ? [...others, g.name] : others }); + }; + const most = catalog?.max_tool_calls ?? 50; + // single tools picked over the API stay picked; the switches are for groups + const single = (value.tools ?? []).filter((n) => !names.has(n)); + return ( +
    + onChange({ ...value, tools: v ? null : has })} + label={ + + Auto every tool you have + + } + /> + {!catalog &&

    Loading tools…

    } +
      + {groups.map((g) => { + const link = admin && g.setting ? settingLink(g.setting) : null; + return ( +
    • +
      + flip(g, v)} label={g.label} /> + + {g.asks && needs approval} + {g.out && sends data out} + +
      + {!g.available && g.why && ( +

      + {g.why.charAt(0).toUpperCase() + g.why.slice(1)}.{" "} + {link && ( + + Open settings + + )} +

      + )} +
    • + ); + })} +
    + {single.length > 0 &&

    Also picked: {single.join(", ")}

    } + onChange({ ...value, max_tool_calls: n })} /> +

    {catalog?.namespaces_note}

    +
    + +
    +
    + ); +} + +/** The tool calls one answer may make: no limit, or a number from 1 to the server's most, with − and +. */ +function Limit({ + value, + most, + onChange, +}: { + value: number | null; + most: number; + onChange: (n: number | null) => void; +}) { + const [editing, setEditing] = useState(null); // what's typed, while it's typed + const clamp = (n: number) => Math.max(1, Math.min(most, n)); + return ( +
    + onChange(v ? clamp(FIRST_LIMIT) : null)} + label={Limit tool calls per answer} + /> + {value != null && ( +
    + + { + setEditing(e.target.value); + const n = Number(e.target.value); + if (e.target.value && Number.isInteger(n) && n >= 1 && n <= most) onChange(n); + }} + onBlur={() => setEditing(null)} + className="h-7 w-14 rounded-sm border border-border bg-background text-center text-[13px] tabular" + /> + + of at most {most} +
    + )} +
    + ); +}