From 2aa8a4dcd2503dc94c86523475a195779bd770a0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:40:16 +0000 Subject: [PATCH 1/2] feat(assistant): ask before sending data out once an answer read third-party text Once an answer has read text from outside Lens (a tool's result other than Lens's own, the page it was asked from, the namespace assistant's memories, a message in a chat room open to anyone, or an earlier answer that read any of these), tools that could carry it out of Lens wait for approval even where they'd run at once: the person's own read web tools, other servers' MCP tools, Python tools with the network, and the changes a setup conversation makes itself. The approval card says why. ai.untrusted_asks (on by default) turns it off. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01BVez82cREgCJtFk6yHReed --- docs/assistant.md | 12 +- docs/chat-rooms.md | 4 +- fastapi_backend/app/api/v1/routes/chats.py | 2 + fastapi_backend/app/domain/ai_tools.py | 49 ++++++- fastapi_backend/app/domain/bridge.py | 14 +- fastapi_backend/app/domain/extensions.py | 7 + fastapi_backend/app/domain/ops_tools.py | 7 +- fastapi_backend/app/domain/store.py | 1 + .../tests/api/test_prompt_injection.py | 124 +++++++++++++++++- .../components/settings/bodies.tsx | 1 + nextjs-frontend/components/settings/model.ts | 7 + 11 files changed, 219 insertions(+), 9 deletions(-) diff --git a/docs/assistant.md b/docs/assistant.md index 8c45265b4..db2d0a40f 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -131,6 +131,15 @@ apart from what the person asks: ``, `` and the like) and anything that looks like a marker get a space inside, so a local model server that reads them in a message can't be made to end the turn and speak as the system. +- **Asks first once it has read it** (`ai.untrusted_asks`, on by default; Settings → AI assistant → Ask before + sending data out after reading third-party text): once an answer has read text from outside Lens (a tool's result + other than Lens's own, the page it was asked from, the namespace assistant's memories, a message in a chat room + open to anyone, or an earlier answer in the conversation that read any of these), anything that could carry what + it read out of Lens waits for your approval, even where it would otherwise run at once: your own `read` web tools, + other servers' MCP tools and Python tools with the network, and the changes a setup conversation makes itself. The + approval card says why ("asked first: this answer read text from outside Lens: tool search_transcripts"). Notes and + memories are still written at once: they stay in Lens, and are marked as data when read back. + Marking makes injection less likely to work; it doesn't make it impossible (a system prompt is not a filter). What the assistant can do is still bounded by the person's reads and approvals (N-PROMPT in the repository's `threat-model.md`). @@ -241,7 +250,8 @@ Admins also get the server tools: `server_status` (what's set up and what's miss (`POST /api/v1/chats` with `"kind": "setup"`, or **Finish with the assistant** in the setup wizard once a model is connected) the admin has asked the assistant to set the server up, so it makes the changes itself and says what it changed. Every change is audited with `assistant` in its detail, and all of them stay changeable in Settings. -Turning telemetry on or off always asks first. +Turning telemetry on or off always asks first, and so does every change once the answer has read text from outside +Lens ([third-party content](#third-party-content-and-prompt-injection)). ### Routine choices diff --git a/docs/chat-rooms.md b/docs/chat-rooms.md index ce3164642..75e9fc68f 100644 --- a/docs/chat-rooms.md +++ b/docs/chat-rooms.md @@ -74,7 +74,9 @@ That's all. Without an account given, it answers as the admin who turned it on. several waiting, "yes all" or "yes 12". Only the chat usernames listed may talk to Lens and approve, so it needs that list; **Approving in the room: only in the web app** turns it off. Settings, imports, batch runs, extensions and forgetting are always approved in the web app. Each approval is in the audit log with "via chat room" and who - said yes. + said yes. Without a list, anyone in the room may be someone the account doesn't vouch for, so their messages count + as text from outside Lens: web tools and other servers' tools wait for approval in the web app too + ([third-party content](assistant.md#third-party-content-and-prompt-injection)). - **Conversations.** Each person in each room is a conversation of that account's, titled "Matterbridge · room · person", listed in Chat with the account's other conversations. Follow-ups keep their context, and older messages are folded into a running summary ([Long conversations](assistant.md)). `Lens /clear` starts diff --git a/fastapi_backend/app/api/v1/routes/chats.py b/fastapi_backend/app/api/v1/routes/chats.py index 398231090..43cd2f9c1 100644 --- a/fastapi_backend/app/api/v1/routes/chats.py +++ b/fastapi_backend/app/api/v1/routes/chats.py @@ -284,6 +284,8 @@ def answer(on: chat.Answering) -> Iterator[str]: user.admin, act=setup, said=q, + # what it read before any tool ran: the page it was asked from, and earlier answers' excerpts and tools + outside=(["page"] if page else []) + ai_tools.read_before(past, summary), ) try: answer = "" diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 5ad77c063..7524b2d63 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -329,12 +329,30 @@ def builtin_names(): } +def read_before(history, summary=None): + """Whether earlier answers in a conversation read text from outside Lens (excerpts, or tools other than Lens's own), + as a Toolbox's `outside`: the model reads those answers again, word for word or in the summary. After /clear it + reads only what came since.""" + since = (summary or {}).get("upto") or 0 if summary is not None and not summary.get("text") else 0 + for m in history: + if m["id"] <= since or m["role"] != "assistant": + continue + tools = {s.get("tool") for s in m.get("steps") or [] if s} - OWN_RESULTS - {None, "choose_namespace"} + if m.get("passages") or tools: + return ["earlier answers"] + return [] + + class Toolbox(ops_tools.OpsTools): """`admin` adds the server tools (ops_tools.py), with `base` (archive.yaml's config) to save settings over; `act` makes their changes at once instead of asking for approval.""" - def __init__(self, db, cfg, user, readable, editable, scope, chat_id, base=None, admin=False, act=False, said=""): + def __init__(self, db, cfg, user, readable, editable, scope, chat_id, base=None, admin=False, act=False, said="", outside=()): self.db, self.cfg, self.user, self.chat, self.said = db, cfg, user, chat_id, said + # where this answer has read text from outside Lens so far (`outside`: before any tool ran, such as the page it + # was asked from or a chat room's message): once it has, tools that send data out of Lens or change the server + # ask first, even where they'd act at once (wary) + self.outside = list(dict.fromkeys(outside)) self.base, self.admin, self.act = base or cfg, admin, act self.scope = scope or {} self.readable, self.editable = set(readable), set(editable) @@ -370,6 +388,8 @@ def _home_note(self): return "" sid, name, p = self.home rows = ns_assistant.labelled(self.db, ns_assistant.memories(self.db, sid, ns_assistant.PROMPT_MAX)) + if rows: + self.read_outside("memory") spaces = { r["id"]: r["space"] for r in self.db.rows( @@ -461,6 +481,19 @@ def untrusted(self, name): for the model to read as data.""" return name not in OWN_RESULTS + def read_outside(self, source): + if source not in self.outside: + self.outside.append(source) + + def wary(self): + """Whether this answer has read text from outside Lens, so that what could carry its instructions out (a web + tool, another server's tool) or change the server asks first (ai.untrusted_asks, on by default).""" + return bool(self.outside) and self.cfg["ai"].get("untrusted_asks", True) + + def why(self): + """Why something that would have run at once waits for a yes, said with it.""" + return f" (asked first: this answer read text from outside Lens: {', '.join(self.outside[:4])})" + def call(self, name, args, model=False): """A tool's result (JSON) and the step's summary. `model`: the result as the model reads it, third-party text marked as data (untrusted), with what hooks add after it, unmarked.""" @@ -478,7 +511,10 @@ def call(self, name, args, model=False): why = extensions.fill(h["spec"]["action"]["reason"], {"said": self.said, "tool": name}) return json.dumps({"error": f"blocked: {why}"}), f"{name}: blocked ({why})" extra += self._hook(h, {"said": self.said, "tool": name}) + asked = len(self.approvals) result, summary = self._run(name, fn, args) + if self.untrusted(name) and len(self.approvals) == asked: # it read something, rather than asking for a yes + self.read_outside(f"tool {name}") if hooks: for h in self.ext.matching("after_tool", name, result): extra += self._hook(h, {"said": self.said, "tool": name, "result": result[:4000]}) @@ -505,9 +541,16 @@ def _ext_tool(self, name): def run(**args): args = extensions.tool_args(t["spec"], args) what = f"{t['name']}(" + ", ".join(f"{k}={json.dumps(v, ensure_ascii=False)[:60]}" for k, v in args.items()) + ")" - if t["spec"]["effect"] == "change" and not self.act: + change = t["spec"]["effect"] == "change" + # a tool that reaches another host carries what the model put in its arguments there: after reading text + # from outside Lens, which may have put it there, it asks first + asks = change and not self.act + wary = not asks and (change or extensions.sends_out(t["spec"])) and self.wary() + if asks or wary: # the reason said only where it asks because of what it read return self._approval( - "extension", {"tool": t["name"], "extension": t["ext"], "version": t["version"], "args": args}, f"Run {what}" + "extension", + {"tool": t["name"], "extension": t["ext"], "version": t["version"], "args": args}, + f"Run {what}" + (self.why() if wary else ""), ) out = extensions.run_tool(self.db, self.cfg, t["spec"], args, toolbox=self) return out, f"Ran {what}" diff --git a/fastapi_backend/app/domain/bridge.py b/fastapi_backend/app/domain/bridge.py index 0733cf0bf..1f9788e30 100644 --- a/fastapi_backend/app/domain/bridge.py +++ b/fastapi_backend/app/domain/bridge.py @@ -290,8 +290,20 @@ def answer(db, cfg, base, account, cid, q): steps, approvals, notice = [], [], None wrote = cfg["llm"].get("model") if llm.configured(cfg) else None if llm.configured(cfg) and cfg["ai"].get("tools"): + # anyone in a room with no list of who may talk to Lens is someone the account doesn't vouch for + anyone = not [u for u in cfg["bridge"].get("users") or [] if u.strip()] box = ai_tools.Toolbox( - db, cfg, {"id": account["id"], "email": account["email"]}, readable, editable, scope, cid, base, account["admin"], said=q + db, + cfg, + {"id": account["id"], "email": account["email"]}, + readable, + editable, + scope, + cid, + base, + account["admin"], + said=q, + outside=(["chat room"] if anyone else []) + ai_tools.read_before(past, summary), ) try: text = None diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index 3a3bcaca6..c62a1ba20 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -813,6 +813,13 @@ def tool_args(spec, args): return out +def sends_out(spec): + """Whether a tool sends what it's given to another host: a web tool, another MCP server's tool, Python with the + network. A prompt tool talks to Lens's own model, and a canvas tool's calls are each another tool's.""" + run = spec["run"] + return run["type"] not in ("prompt", "graph", "python") or (run["type"] == "python" and bool(run.get("network"))) + + def run_tool(db, cfg, spec, args, model=None, toolbox=None): """Run a tool's body with checked arguments; what it gave back, for the model to read.""" from . import llm diff --git a/fastapi_backend/app/domain/ops_tools.py b/fastapi_backend/app/domain/ops_tools.py index 9976e55d6..40d08f9ab 100644 --- a/fastapi_backend/app/domain/ops_tools.py +++ b/fastapi_backend/app/domain/ops_tools.py @@ -4,7 +4,8 @@ Admins get the server tools; anyone gets `import_files` for their own attachments and `import_urls` for web pages, into namespaces they edit. A change waits for the person's approval, unless the conversation lets the assistant act (a setup conversation does: the admin -asked it to set things up): then it's made at once, audited, and said in the answer. Telemetry, which sends data +asked it to set things up): then it's made at once, audited, and said in the answer, as long as the answer hasn't read +text from outside Lens (Toolbox.wary), which could be what asked for it. Telemetry, which sends data elsewhere, always waits for approval. """ @@ -226,10 +227,12 @@ def describe_namespace(db, sid, name): class OpsTools: - """Mixed into the chat Toolbox: needs db, cfg, base, user, editable, admin, act and _approval.""" + """Mixed into the chat Toolbox: needs db, cfg, base, user, editable, admin, act, wary, why and _approval.""" def _change(self, tool, args, summary): if self.act and not (tool == "change_settings" and args.get("section") in ALWAYS_ASK): + if self.wary(): # it read text from outside Lens, which may be what asked for this + return self._approval(tool, args, summary + self.why()) out = apply(self.db, self.cfg, self.base, tool, args, self.user, self.editable, self.admin) return {"status": "done", **out}, f"{summary}: done" return self._approval(tool, args, summary) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 2bd4b8e00..1b50dca8a 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -281,6 +281,7 @@ def labels(): "context_tokens": None, # the chat model's context window; summarised early before it fills (None: 16000) "organise_notes": True, # notes nobody filed are filed in PARA, or a suggestion waits on them (notebook.py) "untrusted_marking": True, # third-party text reaches the model marked as data, not instructions (untrusted.py) + "untrusted_asks": True, # after reading it, tools that send data out or change the server ask first (ai_tools.py) "disabled_tools": [], "max_steps": 6, "max_transcript_reads": 20, diff --git a/fastapi_backend/tests/api/test_prompt_injection.py b/fastapi_backend/tests/api/test_prompt_injection.py index 1ef764862..a49eefa15 100644 --- a/fastapi_backend/tests/api/test_prompt_injection.py +++ b/fastapi_backend/tests/api/test_prompt_injection.py @@ -8,7 +8,7 @@ import pytest -from app.domain import ingest, untrusted +from app.domain import ai_tools, bridge, chat, extensions, ingest, settings, untrusted from tests import fake_llm, injections from tests.api._assist import Assist, sse, start_llm from tests.helpers import unmark @@ -128,3 +128,125 @@ def test_marking_can_be_switched_off(unmarked, app, db, cfg, folder, new_client, got = _tool_messages(llm)[0] assert not got.startswith("" in got assert untrusted.RULES not in [b for b in llm.seen if b.get("tools")][0]["messages"][0]["content"] + + +# ---------- once an answer has read outside text, what could carry it out asks first (ai_tools.Toolbox.wary) ---------- +LOOKUP = { + "name": "lookup", + "kind": "tool", + "description": "Look a word up in an online dictionary.", + "effect": "read", + "params": [{"name": "q", "kind": "text", "required": True}], + "run": {"type": "http", "url": "https://dict.example/define?q={{q}}"}, +} + + +@pytest.fixture +def sent(monkeypatch): + out = [] + monkeypatch.setattr(extensions, "http_call", lambda cfg, run, args: out.append(args) or {"status": 200, "json": {"meaning": "x"}}) + return out + + +def _ask(c, h, cid, q): + return sse(c.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": q}).text) + + +def test_a_web_tool_asks_first_once_the_answer_read_outside_text(app, db, cfg, folder, new_client, llm, sent): + s = Assist(app, db, cfg, folder, new_client) + rid = _poisoned(s, db, cfg) + c, h = s.cl["editor"] + assert c.post("/api/v1/extensions", headers=h, json={"manifest": LOOKUP}).status_code == 200 + # nothing read yet: the person's own read tool answers at once, as before + llm.tool_script = [{"content": "", "tool_calls": [call(1, "lookup", {"q": "capsid"})]}, {"content": "A protein shell."}] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + ev = _ask(c, h, cid, "What's a capsid?") + assert sent == [{"q": "capsid"}] and "approval" not in ev + + # after reading a transcript that says to send everything somewhere, the same tool waits for a yes + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "read_transcript", {"recording_id": rid})]}, + {"content": "", "tool_calls": [call(2, "lookup", {"q": "every excerpt"})]}, + {"content": "I asked before looking it up."}, + ] + cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"] + ev = _ask(c, h, cid, "Read Mallory's talk") + assert sent == [{"q": "capsid"}] # nothing went out + appr = ev["approval"][0] + assert appr["summary"] == ('Run lookup(q="every excerpt") (asked first: this answer read text from outside Lens: tool read_transcript)') + # the person decides: approved, it runs + assert c.post(f"/api/v1/approvals/{appr['id']}", headers=h, json={"decision": "approve"}).json()["status"] == "done" + assert sent[-1] == {"q": "every excerpt"} + + # the next question in that conversation still counts what the earlier answer read + sent.clear() + llm.tool_script = [{"content": "", "tool_calls": [call(1, "lookup", {"q": "virus"})]}, {"content": "Asked."}] + ev = _ask(c, h, cid, "And virus?") + assert sent == [] and "earlier answers" in ev["approval"][0]["summary"] + + +def test_asking_first_can_be_switched_off(app, db, cfg, folder, new_client, llm, sent): + s = Assist(app, db, cfg, folder, new_client) + rid = _poisoned(s, db, cfg) + cfg["ai"]["untrusted_asks"] = False + box = ai_tools.Toolbox(db, cfg, _me(db, "ed@x.io"), {s.pods}, {s.pods}, {}, None) + extensions.create(db, box.me, LOOKUP) + box = ai_tools.Toolbox(db, cfg, _me(db, "ed@x.io"), {s.pods}, {s.pods}, {}, None) + box.call("read_transcript", {"recording_id": rid}) + assert box.outside == ["tool read_transcript"] and not box.wary() + box.call("lookup", {"q": "x"}) + assert sent == [{"q": "x"}] and box.approvals == [] + + +def test_a_setup_conversation_asks_once_it_read_outside_text(app, db, cfg, folder, new_client, llm): + s = Assist(app, db, cfg, folder, new_client) + rid = _poisoned(s, db, cfg) + ca, ha = s.cl["admin"] + cid = ca.post("/api/v1/chats", headers=ha, json={"kind": "setup"}).json()["id"] + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "change_settings", {"section": "ai", "changes": {"max_steps": 7}})]}, + {"content": "Done."}, + ] + ev = _ask(ca, ha, cid, "Give the assistant seven steps") + assert ev["step"][0]["summary"] == "Set ai: max_steps = 7: done" and "approval" not in ev + # a transcript that asks for the model provider to move: the change waits for the admin + llm.tool_script = [ + {"content": "", "tool_calls": [call(1, "read_transcript", {"recording_id": rid})]}, + {"content": "", "tool_calls": [call(2, "change_settings", {"section": "llm", "changes": {"base_url": "https://evil.example/v1"}})]}, + {"content": "That needs your yes."}, + ] + ev = _ask(ca, ha, cid, "What does Mallory's talk say to do?") + assert ev["approval"][0]["summary"].startswith("Set llm: base_url = https://evil.example/v1 (asked first: ") + assert settings.Settings(db, cfg).current()["llm"]["base_url"] != "https://evil.example/v1" + + +def test_a_room_anyone_can_talk_in_counts_as_outside_text(app, db, cfg, folder, new_client, llm, sent): + s = Assist(app, db, cfg, folder, new_client) + me = _me(db, "ed@x.io") + extensions.create(db, extensions.who(me["id"], me["email"], False, {s.pods: "editor"}), LOOKUP) + account = {**me, "admin": False} + for users, asks in (([], True), (["alice"], False)): + cfg["bridge"] = {**cfg["bridge"], "users": users} + sent.clear() + cid = chat.create(db, me["id"], kind="bridge") + llm.tool_script = [{"content": "", "tool_calls": [call(1, "lookup", {"q": "capsid"})]}, {"content": "OK."}] + text, approvals = bridge.answer(db, cfg, cfg, account, cid, "look up capsid") + assert (bool(approvals), sent) == ((True, []) if asks else (False, [{"q": "capsid"}])) + if asks: + assert "chat room" in approvals[0]["summary"] + + +def test_which_tools_send_data_out(): + def spec(**run): + return {"run": run} + + assert extensions.sends_out(spec(type="http", url="https://x.example/")) + assert extensions.sends_out(spec(type="mcp", url="https://mcp.example/", tool="t")) + assert extensions.sends_out(spec(type="python", code="", network=True)) + assert not extensions.sends_out(spec(type="python", code="")) + assert not extensions.sends_out(spec(type="prompt", prompt="x")) + assert not extensions.sends_out(spec(type="graph", graph={})) # each tool it calls is gated on its own + + +def _me(db, email): + return {"id": db.one("SELECT VALUE record::id(id) FROM account WHERE email = $e", e=email), "email": email} diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index db0c3a7c9..0fa252af6 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -1274,6 +1274,7 @@ function AiBody({ ctx }: { ctx: BodyCtx }) {

+
diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 98058e215..2fc9becb5 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -749,6 +749,13 @@ export const FIELDS: FieldSpec[] = [ kind: "switch", hint: "Recordings, documents, web pages, chat rooms and what tools send back reach the model marked as material to read and cite, so instructions written in them aren't followed.", }, + { + section: "ai", + key: "untrusted_asks", + label: "Ask before sending data out after reading third-party text", + kind: "switch", + hint: "Once an answer has read a recording, a document, a web page or a chat room, web tools, other servers' tools and changes the setup chat would make at once wait for your approval.", + }, { section: "ai", key: "refine_notes", From a6c7e170beddc8b2a47d5c5973a921dc9ba76959 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 21:21:15 +0000 Subject: [PATCH 2/2] chore(openapi): regenerate the schema for 0.7.0 The v0.7.0 release bumped the package version without regenerating nextjs-frontend/openapi.json, so the generate-openapi-schema hook fails on any pull request that touches the API. Regenerated with the repository's own command; only the version changes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01BVez82cREgCJtFk6yHReed --- nextjs-frontend/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 29243ab27..85998ff17 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "Lens", - "version": "0.6.0" + "version": "0.7.0" }, "paths": { "/api/v1/auth/status": {