From 6db7154d69d1d4e64dbd5f62f1f4447a1bfd2958 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:15:25 +0000 Subject: [PATCH] feat(chat): built-in Telegram bot, on a shared chat-channel layer Paste a bot token from @BotFather in Settings > Telegram and the assistant answers there through getUpdates long polling: no Matterbridge, no public address. It answers only people let in (one click from the settings page), transcribes voice notes, holds files for the assistant to import, and keeps everything Matterbridge rooms do (namespace routing, approving by reply). channels.py is the adapter layer: Matterbridge and Telegram are adapters with their own thread, lease and status, and bridge.handle serves both. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0194jTK9wbki29B4LcS7aKHG --- CHANGELOG.md | 6 + docs/chat-rooms.md | 22 ++ fastapi_backend/app/api/v1/routes/admin.py | 19 +- fastapi_backend/app/cli.py | 4 +- fastapi_backend/app/core/runtime.py | 6 +- fastapi_backend/app/domain/bridge.py | 210 +++++++------- fastapi_backend/app/domain/channels.py | 236 ++++++++++++++++ fastapi_backend/app/domain/settings.py | 48 +++- fastapi_backend/app/domain/store.py | 14 + fastapi_backend/app/domain/telegram.py | 262 ++++++++++++++++++ fastapi_backend/app/domain/uploads.py | 11 +- fastapi_backend/app/schemas/admin.py | 25 ++ fastapi_backend/tests/api/test_telegram.py | 207 ++++++++++++++ fastapi_backend/tests/fake_telegram.py | 84 ++++++ nextjs-frontend/app/openapi-client/index.ts | 9 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 32 +++ .../app/openapi-client/types.gen.ts | 125 +++++++++ .../components/settings/bodies.tsx | 119 ++++++++ nextjs-frontend/components/settings/model.ts | 67 +++++ .../components/settings/settings-shell.tsx | 2 + nextjs-frontend/openapi.json | 187 +++++++++++++ threat-model.md | 6 +- threat-model.yaml | 2 +- 23 files changed, 1565 insertions(+), 138 deletions(-) create mode 100644 fastapi_backend/app/domain/channels.py create mode 100644 fastapi_backend/app/domain/telegram.py create mode 100644 fastapi_backend/tests/api/test_telegram.py create mode 100644 fastapi_backend/tests/fake_telegram.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5112764c6..9a16e1572 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Telegram, built in.** Paste a bot token from @BotFather in Settings › Telegram and the assistant answers there: + no Matterbridge, no public address. It talks only to people you let in (one click on the settings page), hears + voice notes, takes files for the assistant to import, and does everything it does in Matterbridge rooms + ([Chat rooms](docs/chat-rooms.md#telegram-built-in)). Chat apps now share one channel layer, so the next one is an + adapter. + - **Install on Windows, or by double-clicking.** `install.ps1` installs Lens on Windows: it gets Python 3 if needed and opens the web installer, which installs Docker Desktop (and WSL) and runs Lens in it. Each release now carries launchers that need no terminal at all: **Install Lens.app** for macOS, **Install Lens.cmd** for Windows and diff --git a/docs/chat-rooms.md b/docs/chat-rooms.md index ed28cb807..02c714985 100644 --- a/docs/chat-rooms.md +++ b/docs/chat-rooms.md @@ -4,6 +4,28 @@ Lens's assistant can answer in Slack, Discord, Telegram, Matrix, Mattermost, IRC [Matterbridge](https://github.com/42wim/matterbridge) bridges. Say "Lens, what did we decide about the launch?" (or "@Lens …") in a bridged room and it answers there, as it would in Chat. +## Telegram, built in + +Telegram needs no Matterbridge and no public address: Lens reads its own bot's messages (long polling) and answers. + +1. In Telegram, message **@BotFather**, send `/newbot` and copy the token it gives you. +2. Open **Settings → Telegram**, paste the token and turn on **Answer in Telegram**. +3. Message your bot. You show on the settings page under **Wrote to the bot, not let in**: click **Let in**. + +Anyone can find a bot, so it answers only the people in **People it talks to** (Telegram user ids or usernames); +anyone else who writes to it is told once that they need to be let in, and shows on the page. A private chat with +the bot is always for Lens; in a group it answers what names it ("Lens, …"), mentions the bot (`@your_bot`) or +replies to it. Voice notes are transcribed by the server's speech-to-text (the answer starts with what it heard), and +files sent to it (up to Telegram's 20 MB for bots) are held for the assistant, which imports them into a namespace as +a file dropped into Chat is. The page lists the chats it has seen with their ids, so a group can be given to a +namespace's assistant with **Chats for a namespace's assistant** (`-1001234567890 = pods`). Approving by replying +"yes", the account it answers as and routing between namespaces work as below. + +Give Lens a bot of its own: a bot that Matterbridge also reads answers neither reliably (**Check the bot** says so). + +Telegram is one adapter of the chat-channel layer (`app/domain/channels.py`) that Matterbridge is another of; a new +chat app adds an adapter (reading, sending, files) and gets everything in a room for free. + ## Set it up in the web app Lens can run Matterbridge for you, so adding Slack or WhatsApp takes no config file: diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index 5249bfb4a..2d093524e 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -29,6 +29,7 @@ sources, speech, store, + telegram, telemetry, tunnel, ) @@ -52,6 +53,7 @@ SemanticStatus, SpeechTestResult, Started, + TelegramStatus, TelemetryStatus, TelemetryTestResult, TunnelStatus, @@ -78,10 +80,10 @@ def update_settings(section: str, body: dict[str, Any], user: AdminWriter, reque if "*" not in hosts and here not in hosts: raise HTTPException(400, f"that list leaves out {here}, the address you're using, and would lock you out") if ( - section == "bridge" + section in ("bridge", "telegram") and body.get("enabled") and "account" not in body - and not request.app.state.archive.current()["bridge"].get("account") + and not (request.app.state.archive.current().get(section) or {}).get("account") ): body = {**body, "account": user.email} # turned on without saying who it answers as: the admin who turned it on files_were = request.app.state.settings.current()["encryption"].get("files") if section == "encryption" else None @@ -185,6 +187,19 @@ def test_bridge(user: AdminWriter, cfg: Cfg, db: Db) -> BridgeTestResult: return BridgeTestResult(ok=error is None, error=error) +@router.get("/settings/telegram") +def telegram_status(user: AdminReader, cfg: Cfg, db: Db) -> TelegramStatus: + """How the assistant's Telegram bot is doing, and the chats it has seen (their ids are what telegram.rooms names).""" + return TelegramStatus(**telegram.status(db, cfg)) + + +@router.post("/settings/telegram/test") +def test_telegram(user: AdminWriter, cfg: Cfg, db: Db) -> BridgeTestResult: + """Check that Telegram knows the bot's token, and that the account to answer as exists.""" + error = telegram.TELEGRAM.check(db, cfg) + return BridgeTestResult(ok=error is None, error=error) + + @router.post("/settings/documents/node/test") def test_conversion_node(user: AdminWriter, cfg: Cfg) -> ConversionNodeTestResult: """Check that the conversion node (documents.anytopdf_url) answers at its address and takes Lens's token.""" diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 96cbc0797..211bc0062 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -260,7 +260,7 @@ def platform_main(argv, config): import threading import time - from .domain import auth, bridge, jobs, local_llm, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel, updates + from .domain import auth, channels, jobs, local_llm, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel, updates ap = argparse.ArgumentParser(prog="lens") sub = ap.add_subparsers(dest="cmd", required=True) @@ -364,7 +364,7 @@ def password(): if not (a.steps or a.no_schedule): # and scans watched folders and runs routines, as `lens watch` does routines.start(db, C, stop, log=print) sensors.start(db, C, stop, log=print, name=wk.name) # and the sensor hub, while sensors are on - bridge.start(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat-room bridge, while it's on + channels.start_all(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat channels, while they're on tunnel.start(db, C, stop, name=wk.name, log=print) # and the Cloudflare tunnel, while it's on local_llm.start(db, C, stop, name=wk.name, log=print) # and the local chat model, while it's on updates.start(db, C, stop, log=print) # and the daily check for a new release (docs/updates.md) diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index 2e61798c8..fd02a61be 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -13,7 +13,7 @@ from app.domain import ( activity, auth, - bridge, + channels, components, content_types, jobs, @@ -82,7 +82,7 @@ def prepare(self) -> None: def start_background(self) -> None: """Inline workers, the watched-folder poller, the notifier, the routine scheduler, the sensor hub (idle while - sensors are off) and the chat-room bridge (idle while it's off). Production runs these as separate processes (`lens worker`).""" + sensors are off) and the chat channels (each idle while it's off). Production runs these as separate processes (`lens worker`).""" for i in range(max(0, int(self.current()["workers"]["inline"]))): w = jobs.Worker(self.db, self.current, name=f"api-{os.getpid()}-{i}", log=log.info) threading.Thread(target=w.loop, args=(self.stop,), daemon=True, name=f"worker-{i}").start() @@ -90,7 +90,7 @@ def start_background(self) -> None: notify.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.warning) routines.start(self.db, self.current, self.stop, log=log.info) sensors.start(self.db, self.current, self.stop, log=log.info, name=f"api-{os.getpid()}") - bridge.start(self.db, self.current, self.stop, lambda: self.base, name=f"api-{os.getpid()}", log_fn=log.warning) + channels.start_all(self.db, self.current, self.stop, lambda: self.base, name=f"api-{os.getpid()}", log_fn=log.warning) tunnel.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) local_llm.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) updates.start(self.db, self.current, self.stop, log=log.info) diff --git a/fastapi_backend/app/domain/bridge.py b/fastapi_backend/app/domain/bridge.py index e5780e217..b48e53a8a 100644 --- a/fastapi_backend/app/domain/bridge.py +++ b/fastapi_backend/app/domain/bridge.py @@ -20,26 +20,26 @@ "yes" (or "no") in the room, when bridge.approve is "low_risk" and the sender is one of bridge.users; everything else waits for approval in the web app, linked from the answer. -Refine later: voice notes, files posted in the room, one Lens account per chat user. +What happens in a room is the same in every chat channel (channels.py): Matterbridge is one adapter, Telegram's own +bot (telegram.py) another. handle() takes a message from any of them, with their voice notes (transcribed) and files +(held for the assistant to import), and answers through the adapter it came from. + +Refine later: one Lens account per chat user. """ from __future__ import annotations -import datetime as dt import hashlib import json import logging -import os import re -import threading import urllib.error import urllib.request -from . import ai_tools, auth, auto_scope, chat, graph_history, llm, matterbridge, ns_assistant, store +from . import ai_tools, auth, auto_scope, channels, chat, graph_history, llm, matterbridge, ns_assistant, store from .store import R log = logging.getLogger(__name__) -LEASE_SECONDS = 30 TIMEOUT = 15 MAX_CHARS = 4000 @@ -158,6 +158,8 @@ def addressed(cfg, m, aliases=()): if allowed and who.strip("<> ").lower() not in allowed and (m.get("userid") or "").lower() not in allowed: return None rx = re.compile(rf"^\s*@?{re.escape(name)}\b[\s,:;!?-]*", re.I) + if m.get("direct"): # a private chat, or a reply or mention the channel resolved: for Lens whatever it says + return rx.sub("", text, count=1).strip() or None if rx.match(text): return rx.sub("", text, count=1).strip() or None if b.get("answer") == "all": @@ -174,11 +176,12 @@ def _account(db, cfg): return {"id": u["id"], "email": u["email"], "admin": bool(u.get("admin"))} -def conversation(db, account, m, namespace=None): +def conversation(db, account, m, namespace=None, label="Matterbridge"): """The conversation for this person in this room, started on their first message; scoped to `namespace` (the room's namespace assistant), which follows bridge.rooms when it changes. In a room no namespace is given to, the - scope is routing's (route).""" - key = hashlib.sha1(json.dumps([m.get("gateway"), m.get("channel"), m.get("account"), m.get("username")]).encode()).hexdigest() + scope is routing's (route). `label` names the channel in its title.""" + who = m["thread"] if m.get("thread") else [m.get("gateway"), m.get("channel"), m.get("account"), m.get("username")] + key = hashlib.sha1(json.dumps(who).encode()).hexdigest() scope = {"namespaces": [namespace]} if namespace else {} cid = db.one("SELECT VALUE record::id(id) FROM chat WHERE bridge = $k AND account = $a LIMIT 1", k=key, a=account["id"]) if cid: @@ -188,7 +191,7 @@ def conversation(db, account, m, namespace=None): db.q("UPDATE $r SET scope = {}, room = NONE, pinned = NONE WHERE room = 'given'", r=R("chat", cid)) return cid where = " · ".join(x for x in [m.get("channel"), (m.get("username") or "").strip("<> ")] if x) - cid = chat.create(db, account["id"], title=f"Matterbridge · {where}" if where else "Matterbridge", scope=scope, kind="bridge") + cid = chat.create(db, account["id"], title=f"{label} · {where}" if where else label, scope=scope, kind="bridge") db.q("UPDATE $r SET bridge = $k, room = $g", r=R("chat", cid), k=key, g="given" if namespace else None) return cid @@ -272,16 +275,17 @@ def route(db, cfg, account, cid, q, ns=None): # ---------- answering ---------- -def answer(db, cfg, base, account, cid, q): - """Answer a question in a conversation as the account would be answered in the web app (without streaming). - Returns (text, approvals).""" +def answer(db, cfg, base, account, cid, q, files=()): + """Answer a question in a conversation as the account would be answered in the web app (without streaming), with + the files sent with it (held uploads: channels.hold). Returns (text, approvals).""" roles = auth.roles(db, account) readable = set(roles) editable = {s for s in roles if auth.allows(roles, s, "editor")} past = chat.history(db, cid) summary = chat.memory(db, cfg, cid) scope = (db.one("SELECT scope FROM $r", r=R("chat", cid)) or {}).get("scope") or None - chat.add(db, cid, "user", q) + chat.add(db, cid, "user", q, attachments=list(files) or None) + asked = q + chat.attached_note(list(files)) passages = chat.retrieve(db, q, readable, scope, cfg=cfg) steps, approvals, notice = [], [], None wrote = cfg["llm"].get("model") if llm.configured(cfg) else None @@ -291,7 +295,7 @@ def answer(db, cfg, base, account, cid, q): ) try: text = None - for kind, data in chat.tool_answer(cfg, box, q, past, cfg["ai"].get("max_steps") or 6, summary=summary): + for kind, data in chat.tool_answer(cfg, box, asked, past, cfg["ai"].get("max_steps") or 6, summary=summary): if kind == "step": steps.append(data) elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers @@ -309,7 +313,7 @@ def answer(db, cfg, base, account, cid, q): return "I couldn't answer that: " + str(e), box.approvals error = None try: - text = llm.chat(cfg, chat.messages_for(q, passages, past, summary)) if llm.configured(cfg) else chat.fallback(passages) + text = llm.chat(cfg, chat.messages_for(asked, passages, past, summary)) if llm.configured(cfg) else chat.fallback(passages) except llm.LLMError as e: text, error = "I couldn't answer that: " + str(e), str(e) chat.add(db, cid, "assistant", text or "(no answer)", chat.cited(text, passages), steps=steps, notice=notice, error=error, model=wrote) @@ -403,8 +407,44 @@ def reply_text(cfg, text, approvals, cid, db=None): return out -def handle(db, cfg, base, m): - """One message from a room: answered when it's for Lens. True when it was.""" +def _media(db, cfg, ch, m): + """A message's voice note as its text (when it has none and is for Lens), and a note when there's nothing to say + but files: the message as handle() reads it, and what to say instead of answering (or None).""" + direct = m.get("direct") or cfg["bridge"].get("answer") == "all" + if not direct or (m.get("text") or "").strip(): + return m, None + if m.get("voice"): + try: + return {**m, "text": channels.heard(cfg, ch, m["voice"]), "heard": True}, None + except channels.ChannelError as e: + return m, str(e) + if m.get("files"): + return {**m, "text": "I attached " + ("this file." if len(m["files"]) == 1 else "these files.")}, None + return m, None + + +def _hold(db, cfg, ch, account, m): + """The files sent with a message, held for the assistant to import, and a line for each that couldn't be.""" + files, failed = [], [] + for f in m.get("files") or []: + try: + files.append(channels.hold(db, cfg, ch, account, f)) + except (ValueError, OSError, channels.ChannelError) as e: + failed.append(f"Couldn't take {f.get('filename') or 'that file'}: {e}") + return files, failed + + +def handle(db, cfg, base, m, ch=None): + """One message from a room of channel `ch` (Matterbridge when None): answered when it's for Lens. True when it + was.""" + ch = ch or MATTERBRIDGE + cfg = channels.view(cfg, ch) + m, cant = _media(db, cfg, ch, m) + if cant: + if addressed(cfg, {**m, "text": "?"}): # only said to who could have been answered + ch.send(cfg, m, cant) + return True + return False ns = room_namespace(cfg, m) sid = {v: k for k, v in store.space_names(db).items()}.get(ns) if ns else None home = ns_assistant.profile(db, sid) if sid is not None else None @@ -413,137 +453,83 @@ def handle(db, cfg, base, m): if not q: return False account = _account(db, cfg) - cid = conversation(db, account, m, ns if sid is not None else None) + cid = conversation(db, account, m, ns if sid is not None else None, ch.label) note = None + files, failed = _hold(db, cfg, ch, account, m) try: decided = decide_in_room(db, cfg, base, account, cid, m, q) if decided: chat.add(db, cid, "user", q) chat.add(db, cid, "assistant", decided) - post(cfg, m.get("gateway"), decided, m.get("channel")) + ch.send(cfg, m, decided) return True if sid is None: said, note = route(db, cfg, account, cid, q, by_name[1] if by_name else None) if said: chat.add(db, cid, "user", q) chat.add(db, cid, "assistant", said) - post(cfg, m.get("gateway"), said, m.get("channel")) + ch.send(cfg, m, said) return True - text, approvals = answer(db, cfg, base, account, cid, q[:MAX_CHARS]) + text, approvals = answer(db, cfg, base, account, cid, q[:MAX_CHARS], files) except Exception: # noqa: BLE001 - still say something in the room - log.exception("bridge: answering failed") + log.exception("%s: answering failed", ch.key) text, approvals, note = "Something went wrong while answering. Try again.", [], None out = reply_text(cfg, text, approvals, cid, db) - post(cfg, m.get("gateway"), out + (f"\n\n{note}" if note else ""), m.get("channel")) + if m.get("heard"): # what the voice note was heard as, so a mishearing shows + out = f"(Heard: “{q}”)\n\n{out}" + out = "\n".join([*failed, out]) if failed else out + ch.send(cfg, m, out + (f"\n\n{note}" if note else "")) try: # a room conversation goes on for good: older messages are folded into its summary chat.compact(db, cfg, cid) except Exception: # noqa: BLE001 - tried again after the next answer - log.warning("bridge: couldn't update the summary of chat %s", cid, exc_info=True) + log.warning("%s: couldn't update the summary of chat %s", ch.key, cid, exc_info=True) return True -# ---------- the thread ---------- -def _later(seconds): - return (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds)).isoformat(timespec="seconds") +# ---------- Matterbridge as a channel ---------- +class Matterbridge(channels.Channel): + key, label, lease = "bridge", "Matterbridge", "lease" + def ready(self, cfg): + return ready(cfg) -def lease(db, me): - """True while this process is the one reading Matterbridge (only one may: it forgets messages once read).""" - r = R("bridge_state", "lease") - try: - db.q("CREATE $r CONTENT $d", r=r, d={"holder": me, "until": _later(LEASE_SECONDS)}) - return True - except Exception as e: # noqa: BLE001 - it's there already: take it if it's ours or has run out - if "already exists" not in str(e): - raise - return bool( - db.rows( - "UPDATE $r SET holder = $me, until = $u WHERE holder = $me OR until < $now RETURN id", - r=r, - me=me, - u=_later(LEASE_SECONDS), - now=store.now(), - ) - ) + def check(self, db, cfg): + return check(db, cfg) + + def fetch(self, db, cfg): + if matterbridge.running(cfg): # the config follows the settings; the service restarts Matterbridge on a change + problem = matterbridge.write(cfg) + if problem: + raise BridgeError(problem) + return fetch(cfg) + def send(self, cfg, m, text): + post(cfg, m.get("gateway"), text, m.get("channel")) -def release(db, me): - db.q("DELETE $r WHERE holder = $me", r=R("bridge_state", "lease"), me=me) +MATTERBRIDGE = Matterbridge() -def status(db, cfg): - """How the bridge is doing: off, incomplete (what's missing), running (in which process, when it last looked, how - many messages it answered), or error (what went wrong last).""" - why = ready(cfg) - if why == "off": - return {"state": "off"} - if why: - return {"state": "incomplete", "error": why} - s = db.one("SELECT holder, until, at, error, answered FROM $r", r=R("bridge_state", "lease")) or {} - if not s or (s.get("until") or "") < store.now(): - return {"state": "starting", "error": s.get("error")} - return {"state": "error" if s.get("error") else "running", **{k: s.get(k) for k in ("holder", "at", "error", "answered")}} +# ---------- the thread (channels.py) ---------- +def lease(db, me): + """True while this process is the one reading Matterbridge (only one may: it forgets messages once read).""" + return channels.lease(db, me, MATTERBRIDGE) -def _note(db, me, **kw): - db.q("UPDATE $r MERGE $d WHERE holder = $me", r=R("bridge_state", "lease"), d=kw, me=me) +def release(db, me): + channels.release(db, me, MATTERBRIDGE) -_HELD: set = set() # the names this process holds the lease under + +def status(db, cfg): + """How the bridge is doing (channels.status).""" + return channels.status(db, cfg, MATTERBRIDGE) def tick(db, cfg, base, me): """One look at Matterbridge: how many messages were answered.""" - if ready(cfg): - if me in _HELD: - release(db, me) - _HELD.discard(me) - return 0 - if not lease(db, me): - _HELD.discard(me) - return 0 - _HELD.add(me) - try: - if matterbridge.running(cfg): # the config follows the settings; the service restarts Matterbridge on a change - problem = matterbridge.write(cfg) - if problem: - raise BridgeError(problem) - said = fetch(cfg) - except Exception as e: - _note(db, me, error=str(e), at=store.now()) - raise - n, failed = 0, None - for m in said: # Matterbridge has forgotten them: each gets its turn even when one before it failed - try: - n += handle(db, cfg, base, m) - except Exception as e: # noqa: BLE001 - reported in the status, and the rest are still answered - log.warning("bridge: a message couldn't be answered: %s", e) - failed = str(e) - if n: - db.q("UPDATE $r SET answered = (answered ?? 0) + $n WHERE holder = $me", r=R("bridge_state", "lease"), n=n, me=me) - _note(db, me, error=failed, at=store.now()) - return n + return channels.tick(db, cfg, base, me, MATTERBRIDGE) def start(db, cfg_fn, stop, base_fn=None, name=None, log_fn=None): """The bridge thread, until `stop` is set. It does nothing while the bridge is off.""" - me = name or f"pid-{os.getpid()}" - - def loop(): - while not stop.is_set(): - cfg = cfg_fn() - try: - tick(db, cfg, base_fn() if base_fn else cfg, me) - except Exception as e: # noqa: BLE001 - keep listening - if log_fn: - log_fn(f"bridge: {type(e).__name__}: {e}") - stop.wait(max(1, int((cfg.get("bridge") or {}).get("poll_seconds") or 2))) - if me in _HELD: - try: - release(db, me) - except Exception: # noqa: BLE001 - the lease runs out on its own - pass - - th = threading.Thread(target=loop, daemon=True, name="bridge") - th.start() - return th + return channels.start(db, cfg_fn, stop, MATTERBRIDGE, base_fn, name, log_fn) diff --git a/fastapi_backend/app/domain/channels.py b/fastapi_backend/app/domain/channels.py new file mode 100644 index 000000000..07d7ad471 --- /dev/null +++ b/fastapi_backend/app/domain/channels.py @@ -0,0 +1,236 @@ +"""Chat channels: the assistant in chat apps, one adapter per kind of connection, all sharing what happens in a room +(bridge.py: who a message is for, which namespace it's about, approving by reply) and the thread that reads each. + +An adapter (a Channel) knows only its own connection: whether it's set up (ready), checking it (check), what was said +since the last look (fetch), posting a reply (send) and fetching a voice note or file sent with a message +(download). Its settings section holds the room settings bridge.py reads: account, name, answer, users, rooms, +approve. Matterbridge (bridge.MATTERBRIDGE) and Telegram (telegram.py) are adapters; each has its own thread, lease +and status, so one being down doesn't stop the other. + +Messages are dicts in Matterbridge's shape (gateway, channel, account, username, userid, text, event), and may say: + +- direct: it's for the assistant whatever it says (a private chat, a reply to it, a mention the adapter has resolved) +- voice: a voice note to transcribe (the adapter's reference to it, for download) +- files: [{ref, filename, size}], files sent with it, held for the assistant to import (import_files) +- thread: what identifies the conversation (else gateway, channel, account and username) +""" + +from __future__ import annotations + +import datetime as dt +import logging +import os +import secrets +import threading + +from . import store, uploads, voice +from .store import R + +log = logging.getLogger(__name__) +LEASE_SECONDS = 30 + + +class ChannelError(RuntimeError): + pass + + +class Channel: + """One kind of chat connection. `key` is its settings section, `label` names it in conversation titles, `lease` + is its bridge_state record (which process reads it, and how that's going).""" + + key = "bridge" + label = "Chat" + lease = "lease" + + def ready(self, cfg): + """Why it can't run, or None when it can ("off" while it's turned off).""" + raise NotImplementedError + + def check(self, db, cfg): + """None when the connection and the account it answers as check out, else what's wrong.""" + raise NotImplementedError + + def fetch(self, db, cfg): + """The messages said since the last look; once returned they aren't returned again.""" + raise NotImplementedError + + def send(self, cfg, m, text): + """Post `text` where `m` was said.""" + raise NotImplementedError + + def download(self, cfg, ref): + """The bytes of a voice note or file sent with a message.""" + raise ChannelError("this chat can't send files to Lens") + + def pause(self, cfg, reading, failed): + """Seconds to wait before the next look: `reading` when this process holds the lease, `failed` after an error.""" + return max(1, int((cfg.get(self.key) or {}).get("poll_seconds") or 2)) + + def handle(self, db, cfg, base, m): + from . import bridge + + return bridge.handle(db, cfg, base, m, self) + + +def view(cfg, ch): + """The settings as bridge.py reads them: the channel's own section as `bridge`.""" + if ch.key == "bridge": + return cfg + return {**cfg, "bridge": cfg.get(ch.key) or {}} + + +# ---------- voice notes and files ---------- +def heard(cfg, ch, ref): + """What was said in a voice note, transcribed by the server's speech-to-text (voice.py). ChannelError, saying why, + when it can't be.""" + if not voice.can_transcribe(cfg): + raise ChannelError("I can't listen to voice notes here: no speech-to-text engine is set up. Type it instead.") + try: + text = voice.transcribe(cfg, ch.download(cfg, ref))["text"] + except ValueError as e: + raise ChannelError(f"I couldn't listen to that: {e}") from None + if not text: + raise ChannelError("I couldn't make out anything in that voice note.") + return text + + +def hold(db, cfg, ch, account, f): + """A file sent with a message, kept as a held upload of the account's (as a file dropped into a conversation in the + web app), for the assistant to import. Returns {id, filename, size}; ValueError when it can't be taken.""" + name = uploads.clean_name(f.get("filename") or "file") + data = ch.download(cfg, f["ref"]) + path = uploads.incoming(cfg) / f"chat-{secrets.token_hex(8)}" + path.write_bytes(data) + try: + up = uploads.take(db, cfg, "", path, name, {"id": account["id"], "email": account["email"]}, hold=True) + finally: + path.unlink(missing_ok=True) + return {"id": up["id"], "filename": up["filename"], "size": up["size"]} + + +# ---------- the thread that reads a channel ---------- +def _later(seconds): + return (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds)).isoformat(timespec="seconds") + + +def lease(db, me, ch): + """True while this process is the one reading the channel (only one may: what's read is gone).""" + r = R("bridge_state", ch.lease) + try: + db.q("CREATE $r CONTENT $d", r=r, d={"holder": me, "until": _later(LEASE_SECONDS)}) + return True + except Exception as e: # noqa: BLE001 - it's there already: take it if it's ours or has run out + if "already exists" not in str(e): + raise + return bool( + db.rows( + "UPDATE $r SET holder = $me, until = $u WHERE holder = $me OR holder = NONE OR until < $now RETURN id", + r=r, + me=me, + u=_later(LEASE_SECONDS), + now=store.now(), + ) + ) + + +def release(db, me, ch): + db.q("UPDATE $r SET holder = NONE, until = NONE WHERE holder = $me", r=R("bridge_state", ch.lease), me=me) + + +def state(db, ch): + """The channel's bridge_state record: the lease, and whatever its adapter keeps there.""" + return db.one("SELECT * OMIT id FROM $r", r=R("bridge_state", ch.lease)) or {} + + +def keep(db, ch, **kw): + """Keep something in the channel's bridge_state record (an adapter's offset, the chats it has seen).""" + db.q("UPSERT $r MERGE $d", r=R("bridge_state", ch.lease), d=kw) + + +def status(db, cfg, ch): + """How the channel is doing: off, incomplete (what's missing), starting, running (in which process, when it last + looked, how many messages it answered), or error (what went wrong last).""" + why = ch.ready(cfg) + if why == "off": + return {"state": "off"} + if why: + return {"state": "incomplete", "error": why} + s = state(db, ch) + if not s.get("holder") or (s.get("until") or "") < store.now(): + return {"state": "starting", "error": s.get("error")} + return {"state": "error" if s.get("error") else "running", **{k: s.get(k) for k in ("holder", "at", "error", "answered")}} + + +def _note(db, me, ch, **kw): + db.q("UPDATE $r MERGE $d WHERE holder = $me", r=R("bridge_state", ch.lease), d=kw, me=me) + + +_HELD: set = set() # (channel, name) this process holds the lease of + + +def tick(db, cfg, base, me, ch): + """One look at a channel: how many messages were answered.""" + if ch.ready(cfg): + if (ch.key, me) in _HELD: + release(db, me, ch) + _HELD.discard((ch.key, me)) + return 0 + if not lease(db, me, ch): + _HELD.discard((ch.key, me)) + return 0 + _HELD.add((ch.key, me)) + try: + said = ch.fetch(db, cfg) + except Exception as e: + _note(db, me, ch, error=str(e), at=store.now()) + raise + n, failed = 0, None + for m in said: # they're gone from the channel: each gets its turn even when one before it failed + try: + n += ch.handle(db, cfg, base, m) + except Exception as e: # noqa: BLE001 - reported in the status, and the rest are still answered + log.warning("%s: a message couldn't be answered: %s", ch.key, e) + failed = str(e) + if n: + db.q("UPDATE $r SET answered = (answered ?? 0) + $n WHERE holder = $me", r=R("bridge_state", ch.lease), n=n, me=me) + _note(db, me, ch, error=failed, at=store.now()) + return n + + +def start(db, cfg_fn, stop, ch, base_fn=None, name=None, log_fn=None): + """The channel's thread, until `stop` is set. It does nothing while the channel is off.""" + me = name or f"pid-{os.getpid()}" + + def loop(): + while not stop.is_set(): + cfg = cfg_fn() + failed = False + try: + tick(db, cfg, base_fn() if base_fn else cfg, me, ch) + except Exception as e: # noqa: BLE001 - keep listening + failed = True + if log_fn: + log_fn(f"{ch.key}: {type(e).__name__}: {e}") + stop.wait(ch.pause(cfg, (ch.key, me) in _HELD, failed)) + if (ch.key, me) in _HELD: + try: + release(db, me, ch) + except Exception: # noqa: BLE001 - the lease runs out on its own + pass + _HELD.discard((ch.key, me)) + + th = threading.Thread(target=loop, daemon=True, name=ch.key) + th.start() + return th + + +def every(): + """Every channel Lens has.""" + from . import bridge, telegram + + return [bridge.MATTERBRIDGE, telegram.TELEGRAM] + + +def start_all(db, cfg_fn, stop, base_fn=None, name=None, log_fn=None): + """A thread per channel (each idle while it's off).""" + return [start(db, cfg_fn, stop, ch, base_fn, name, log_fn) for ch in every()] diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 4941151ea..bc81fc010 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -44,6 +44,7 @@ "local_llm": ("enabled", "model", "use_as_provider", "context", "threads", "gpu_layers", "port", "host"), "mail": None, "bridge": None, + "telegram": None, "matterbridge": None, "notifications": None, "telemetry": None, @@ -115,6 +116,7 @@ "speech": ("openai_api_key", "elevenlabs_api_key", "assemblyai_api_key", "deepgram_api_key"), "mail": ("password",), "bridge": ("token",), + "telegram": ("token",), "matterbridge": ("slack_token", "discord_token", "telegram_token", "matrix_password"), "telemetry": ("headers",), "fedora": ("password",), @@ -141,6 +143,8 @@ ("mail", "security"): {"starttls", "ssl", "none"}, ("bridge", "answer"): {"mention", "all"}, ("bridge", "approve"): {"off", "low_risk"}, + ("telegram", "answer"): {"mention", "all"}, + ("telegram", "approve"): {"off", "low_risk"}, ("tunnel", "mode"): {"off", "quick", "token", "managed"}, ("files", "store"): {"local", "connection"}, ("documents", "converter"): {"auto", "anytopdf", "lens"}, @@ -410,6 +414,8 @@ def _check(section, key, value, default): return _mail_setting(key, value) if section == "bridge" and key not in ("answer", "enabled", "approve"): return _bridge_setting(key, value) + if section == "telegram" and key not in ("answer", "enabled", "approve"): + return _telegram_setting(key, value) if section == "matterbridge" and key != "run": return _matterbridge_setting(key, value) if (section, key) == ("voice", "tts_base_url"): @@ -609,17 +615,21 @@ def _mail_setting(key, value): return value -def _bridge_setting(key, value): +TELEGRAM_TOKEN_RX = re.compile(r"^[0-9]{3,20}:[A-Za-z0-9_-]{30,64}$") + + +def _bridge_setting(key, value, section="bridge"): + """bridge's settings, and the room settings every chat channel shares (channels.py: `section`).""" if key == "poll_seconds": if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 300): - raise ValueError("bridge.poll_seconds is a whole number of seconds from 1 to 300") + raise ValueError(f"{section}.poll_seconds is a whole number of seconds from 1 to 300") return value if key == "users": if not (isinstance(value, list) and all(isinstance(v, str) and len(v) <= 200 for v in value) and len(value) <= 500): - raise ValueError("bridge.users is a list of chat usernames") - return list(dict.fromkeys(v.strip() for v in value if v.strip())) + raise ValueError(f"{section}.users is a list of chat usernames") + return list(dict.fromkeys(v.strip().lstrip("@") if section != "bridge" else v.strip() for v in value if v.strip())) if key == "rooms": - bad = "bridge.rooms is a list of rooms and namespaces, like team = pods or team/general = pods" + bad = f"{section}.rooms is a list of rooms and namespaces, like team = pods or team/general = pods" if not (isinstance(value, list) and len(value) <= 200 and all(isinstance(v, str) and len(v) <= 300 for v in value)): raise ValueError(bad) out = [] @@ -633,11 +643,11 @@ def _bridge_setting(key, value): out.append(f"{where} = {ns}") return list(dict.fromkeys(out)) if key not in ("url", "token", "gateway", "account", "name"): - raise ValueError(f"unknown setting bridge.{key}") + raise ValueError(f"unknown setting {section}.{key}") if value in (None, ""): return "Lens" if key == "name" else None if not (isinstance(value, str) and len(value.strip()) <= 500): - raise ValueError(f"bridge.{key} is text") + raise ValueError(f"{section}.{key} is text") value = value.strip() if key != "token" else value if key == "url": u = urllib.parse.urlsplit(value) @@ -646,13 +656,31 @@ def _bridge_setting(key, value): return value.rstrip("/") if key == "account": if not EMAIL_RX.match(value): - raise ValueError("bridge.account is the email of the Lens account it answers as") + raise ValueError(f"{section}.account is the email of the Lens account it answers as") return value.lower() if key == "name" and not re.match(r"^[\w .-]{1,40}$", value): - raise ValueError("bridge.name is a short name, such as Lens") + raise ValueError(f"{section}.name is a short name, such as Lens") return value +def _telegram_setting(key, value): + if key == "token": + value = (value or "").strip() + if value and not TELEGRAM_TOKEN_RX.match(value): + raise ValueError("telegram.token is the bot's token from @BotFather, like 123456789:AAE…") + return value or None + if key == "api_url": + if value in (None, ""): + return None + u = urllib.parse.urlsplit(value.strip()) if isinstance(value, str) else None + if not u or u.scheme not in ("http", "https") or not u.hostname: + raise ValueError("telegram.api_url is the http(s) address of a Bot API server, such as https://api.telegram.org") + return value.strip().rstrip("/") + if key in ("url", "gateway", "poll_seconds"): # Matterbridge's own; Telegram waits on getUpdates instead + raise ValueError(f"unknown setting telegram.{key}") + return _bridge_setting(key, value, "telegram") + + def _matterbridge_setting(key, value): if key.endswith(("_channels", "_chats", "_rooms", "_groups")): if not (isinstance(value, list) and len(value) <= 100 and all(isinstance(v, str) and len(v) <= 200 for v in value)): @@ -980,7 +1008,7 @@ def save(db, base, section, changes, user=None): telemetry.parse_headers(v) # ValueError when malformed if section == "matterbridge" and (len(v) > 500 or "\n" in v.strip()): raise ValueError(f"matterbridge.{k} is one line of text") - if (section, k) == ("documents", "anytopdf_token"): + if (section, k) in (("documents", "anytopdf_token"), ("telegram", "token")): v = _check(section, k, v, None) sealed[k] = seal(base, v, f"setting:{section}.{k}") elif not (isinstance(v, dict) and v.get("secret")): # the mask echoed back means "unchanged" diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index cbae2c2df..3df160117 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -363,6 +363,20 @@ def labels(): "approve": "low_risk", "poll_seconds": 2, }, + # the assistant in Telegram through a bot of your own (telegram.py), with the room settings bridge has: account, + # name, answer, users (Telegram usernames or user ids), rooms (" = namespace") and approve + "telegram": { + "enabled": False, + "token": None, + "account": None, + "name": "Lens", + "answer": "mention", + "users": [], + "rooms": [], + "approve": "low_risk", + # a Bot API server of your own (telegram-bot-api), else Telegram's + "api_url": None, + }, # Matterbridge run by Lens (matterbridge.py): with `run` on, the matterbridge service in Docker Compose gets its # config from these, a gateway per room, and the bridge talks to it; off, bridge.url is a Matterbridge of your own "matterbridge": { diff --git a/fastapi_backend/app/domain/telegram.py b/fastapi_backend/app/domain/telegram.py new file mode 100644 index 000000000..77808f73e --- /dev/null +++ b/fastapi_backend/app/domain/telegram.py @@ -0,0 +1,262 @@ +"""The assistant in Telegram, through a bot of your own: paste its token from @BotFather in Settings › Telegram and +turn it on. No Matterbridge, and no public address: Lens asks Telegram for new messages (getUpdates, long polling) +and answers with sendMessage. + +Everything that happens in a room is bridge.py's, as for Matterbridge (channels.py): it answers as one Lens account +(telegram.account), with a conversation per person per chat; a chat can be given to a namespace's assistant +(telegram.rooms, " = namespace", the ids listed in Settings once Lens has seen the chat); reversible changes +can be approved by replying "yes" (telegram.users); voice notes are transcribed by the server's speech-to-text, and +files are held for the assistant to import. + +Anyone can find a bot and message it, so it answers only the people in telegram.users (their Telegram user ids, or +usernames): someone else who writes to it is told once that they need to be let in, and shows in Settings with an +Allow button. Private chats with the bot are always for Lens. In a group it answers what names it ("Lens, …"), mentions the bot +(@your_bot) or replies to it; with BotFather's privacy mode on (the default), Telegram only sends it those anyway. + +Refine later: buttons for approvals (inline keyboards), answers read aloud as voice notes, Markdown formatting. +""" + +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.parse +import urllib.request + +from . import auth, channels + +API = "https://api.telegram.org" +TIMEOUT = 15 +LONG_POLL = 10 # seconds Telegram holds getUpdates open waiting for a message (under channels.LEASE_SECONDS) +MAX_CHARS = 4000 # Telegram allows 4096 per message +MAX_DOWNLOAD = 20 * 1024 * 1024 # what the Bot API hands out +TOKEN_RX = re.compile(r"^[0-9]{3,20}:[A-Za-z0-9_-]{30,64}$") +_BOTS: dict = {} # token: getMe's answer + + +class TelegramError(channels.ChannelError): + pass + + +def _api(cfg): + return ((cfg.get("telegram") or {}).get("api_url") or API).rstrip("/") + + +def _call(cfg, method, body=None, timeout=TIMEOUT): + token = (cfg.get("telegram") or {}).get("token") + req = urllib.request.Request( + f"{_api(cfg)}/bot{token}/{method}", + data=json.dumps(body or {}).encode(), + headers={"Content-Type": "application/json", "User-Agent": "Lens"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + got = json.loads(r.read() or b"{}") + except urllib.error.HTTPError as e: + try: + said = json.loads(e.read() or b"{}").get("description") or "" + except ValueError: + said = "" + if e.code in (401, 404): + raise TelegramError("Telegram doesn't know that bot token (check it with @BotFather)") from None + if e.code == 409: + raise TelegramError("another program is reading this bot's messages (a webhook, or Matterbridge with the same bot)") from None + raise TelegramError(f"Telegram answered HTTP {e.code}" + (f": {said}" if said else "")) from None + except (urllib.error.URLError, OSError) as e: + raise TelegramError(f"can't reach Telegram at {_api(cfg)}: {getattr(e, 'reason', e)}") from None + if not got.get("ok"): + raise TelegramError(f"Telegram said: {got.get('description') or 'no'}") + return got.get("result") + + +def bot(cfg): + """The bot the token is for: {id, username, first_name}.""" + token = cfg["telegram"]["token"] + if token not in _BOTS: + _BOTS[token] = _call(cfg, "getMe") + return _BOTS[token] + + +# ---------- what was said ---------- +def _strip_mention(text, username): + """The text without the bot's @mention (and /command@bot as /command); whether it was there.""" + if not username: + return text, False + rx = re.compile(rf"@{re.escape(username)}\b", re.I) + found = bool(rx.search(text)) + return re.sub(r"\s{2,}", " ", rx.sub("", text)).strip(" ,:") if found else text, found + + +def _files(msg): + out = [] + d = msg.get("document") + if d: + out.append({"ref": d["file_id"], "filename": d.get("file_name") or f"file-{msg['message_id']}", "size": d.get("file_size")}) + if msg.get("photo"): # the sizes it was sent in: the largest + p = max(msg["photo"], key=lambda x: x.get("file_size") or x.get("width", 0)) + out.append({"ref": p["file_id"], "filename": f"photo-{msg['message_id']}.jpg", "size": p.get("file_size")}) + v = msg.get("video") + if v: + out.append({"ref": v["file_id"], "filename": v.get("file_name") or f"video-{msg['message_id']}.mp4", "size": v.get("file_size")}) + return out + + +def message(me, msg): + """A Telegram message as a channel message (channels.py), or None for what isn't for anyone (a bot's, a service + message).""" + chat, who = msg.get("chat") or {}, msg.get("from") or {} + if not chat or who.get("is_bot"): + return None + private = chat.get("type") == "private" + text, mentioned = _strip_mention(msg.get("text") or msg.get("caption") or "", me.get("username")) + if text.split(" ")[0] == "/start": # what Telegram sends when someone opens the bot + text = "Hello! What can you do?" + replied = ((msg.get("reply_to_message") or {}).get("from") or {}).get("id") == me.get("id") + vn = msg.get("voice") or msg.get("audio") or msg.get("video_note") or {} + name = who.get("username") or " ".join(x for x in [who.get("first_name"), who.get("last_name")] if x) or str(who.get("id")) + return { + "gateway": str(chat["id"]), + "channel": "" if private else (chat.get("title") or chat.get("username") or ""), + "account": "telegram", + "username": name, + "userid": str(who.get("id") or ""), + "text": text, + "event": "", + "direct": private or mentioned or replied, + "voice": vn.get("file_id"), + "files": _files(msg), + "thread": ["telegram", chat["id"], who.get("id")], + "reply_to": None if private else msg.get("message_id"), + } + + +def _seen(db, msgs): + """The chats and people Lens has seen, latest first, for Settings (a chat's id is what telegram.rooms names, a + person's what telegram.users does).""" + k = channels.state(db, TELEGRAM) + chats = [c for c in k.get("chats") or [] if isinstance(c, dict)] + people = [p for p in k.get("people") or [] if isinstance(p, dict)] + for m in msgs: + chats = [c for c in chats if c.get("id") != m["gateway"]] + chats.insert(0, {"id": m["gateway"], "title": m["channel"] or f"private chat with {m['username']}"}) + told = next((p.get("told") for p in people if p.get("id") == m["userid"]), None) + people = [p for p in people if p.get("id") != m["userid"]] + people.insert(0, {"id": m["userid"], "name": m["username"], "told": told}) + channels.keep(db, TELEGRAM, chats=chats[:50], people=people[:50]) + + +def allowed(cfg, m): + """Whether Lens talks to the sender of `m`: they're in telegram.users, by user id or username.""" + users = {u.strip().lstrip("@").lower() for u in cfg["telegram"].get("users") or [] if u.strip()} + return bool(users) and (m.get("userid") in users or (m.get("username") or "").lower() in users) + + +STRANGER = ( + "Hi! I'm a private assistant and only talk to people my owner has let in. " + "They can let you in from Lens, in Settings › Telegram, where you now show." +) + + +# ---------- the channel ---------- +class Telegram(channels.Channel): + key, label, lease = "telegram", "Telegram", "telegram" + + def ready(self, cfg): + t = cfg.get("telegram") or {} + if not t.get("enabled"): + return "off" + if not t.get("token"): + return "the bot's token from @BotFather" + if not t.get("account"): + return "a Lens account to answer as" + return None + + def check(self, db, cfg): + t = cfg.get("telegram") or {} + if not t.get("token"): + return "paste the bot's token from @BotFather first" + mb = cfg.get("matterbridge") or {} + if mb.get("run") and mb.get("telegram_token") == t["token"]: + return "Matterbridge reads this bot too; give Lens a bot of its own, or take it out of Matterbridge" + _BOTS.pop(t["token"], None) + try: + bot(cfg) + except TelegramError as e: + return str(e) + if not t.get("account"): + return "choose the Lens account it answers as" + u = auth.find_account(db, t["account"]) + if not u or u.get("disabled"): + return f"there's no active Lens account {t['account']}" + return None + + def fetch(self, db, cfg): + me = bot(cfg) + offset = channels.state(db, self).get("offset") + body = {"timeout": LONG_POLL, "allowed_updates": ["message"]} + if offset: + body["offset"] = offset + got = _call(cfg, "getUpdates", body, timeout=TIMEOUT + LONG_POLL) or [] + if got: # the next look confirms these, and Telegram forgets them + channels.keep(db, self, offset=max(u["update_id"] for u in got) + 1) + msgs = [m for m in (message(me, u["message"]) for u in got if u.get("message")) if m] + if msgs: + _seen(db, msgs) + return msgs + + def handle(self, db, cfg, base, m): + if allowed(cfg, m): + return super().handle(db, cfg, base, m) + if m.get("direct"): # someone not let in: told once, and listed in Settings to allow + people = channels.state(db, self).get("people") or [] + if not any(p.get("id") == m["userid"] and p.get("told") for p in people): + self.send(cfg, m, STRANGER) + channels.keep(db, self, people=[{**p, "told": True} if p.get("id") == m["userid"] else p for p in people]) + return False + + def send(self, cfg, m, text): + text = text or "(no answer)" + for i in range(0, len(text), MAX_CHARS): + body = {"chat_id": m["gateway"], "text": text[i : i + MAX_CHARS]} + if m.get("reply_to") and i == 0: + body["reply_parameters"] = {"message_id": m["reply_to"], "allow_sending_without_reply": True} + _call(cfg, "sendMessage", body) + + def download(self, cfg, ref): + f = _call(cfg, "getFile", {"file_id": ref}) + if (f.get("file_size") or 0) > MAX_DOWNLOAD or not f.get("file_path"): + raise TelegramError("Telegram only hands bots files up to 20 MB; upload it in the web app instead") + url = f"{_api(cfg)}/file/bot{cfg['telegram']['token']}/{urllib.parse.quote(f['file_path'])}" + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "Lens"}), timeout=60) as r: + data = r.read(MAX_DOWNLOAD + 1) + except (urllib.error.URLError, OSError) as e: + raise TelegramError(f"couldn't fetch the file from Telegram: {getattr(e, 'reason', e)}") from None + if len(data) > MAX_DOWNLOAD: + raise TelegramError("Telegram only hands bots files up to 20 MB; upload it in the web app instead") + return data + + def pause(self, cfg, reading, failed): + if failed: + return 5 + return 0 if reading else 2 # reading: getUpdates did the waiting + + +TELEGRAM = Telegram() + + +def status(db, cfg): + """channels.status, with the bot's name and the chats and people it has seen (whether each is let in).""" + s = channels.status(db, cfg, TELEGRAM) + k = channels.state(db, TELEGRAM) + token = (cfg.get("telegram") or {}).get("token") + s["bot"] = (_BOTS.get(token) or {}).get("username") if token else None + s["chats"] = [c for c in k.get("chats") or [] if isinstance(c, dict)] + s["people"] = [ + {"id": p["id"], "name": p.get("name") or p["id"], "allowed": allowed(cfg, {"userid": p["id"], "username": p.get("name")})} + for p in k.get("people") or [] + if isinstance(p, dict) and p.get("id") + ] + return s diff --git a/fastapi_backend/app/domain/uploads.py b/fastapi_backend/app/domain/uploads.py index b579af418..51079c4e7 100644 --- a/fastapi_backend/app/domain/uploads.py +++ b/fastapi_backend/app/domain/uploads.py @@ -434,11 +434,12 @@ def _attach(db, cfg, rid, sid, dest, st, fp, by, probed): return jobs.add_steps(db, rid, ATTACH_STEPS, by=by) -def take(db, cfg, ns, path, filename, by, title=None, pipeline=None, collection=None): - """A whole file that arrived some other way (an import webhook), at `path` on the same disk as the uploads folder, - uploaded into namespace `ns` in one go: it's checked as start() checks a file, moved into place and finished as if - its last chunk had just arrived. The namespace must exist. Returns the finished upload.""" - row = start(db, cfg, ns, filename, os.path.getsize(path), by, title=title, pipeline=pipeline, collection=collection) +def take(db, cfg, ns, path, filename, by, title=None, pipeline=None, collection=None, hold=False): + """A whole file that arrived some other way (an import webhook, a chat channel), at `path` on the same disk as the + uploads folder, uploaded into namespace `ns` in one go: it's checked as start() checks a file, moved into place + and finished as if its last chunk had just arrived. The namespace must exist; with `hold` there's none yet (ns ""): + it's held for the assistant to place, as a file dropped into a conversation is. Returns the upload.""" + row = start(db, cfg, ns, filename, os.path.getsize(path), by, title=title, pipeline=pipeline, collection=collection, hold=hold) os.replace(path, _part(cfg, row["id"])) try: return received(db, cfg, row["id"]) diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index 6a843ad4b..6f2280f64 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -101,6 +101,31 @@ class Removed(ResponseModel): removed: bool +class TelegramChat(ResponseModel): + id: str = Field(description="the chat's id, for telegram.rooms (like -1001234567890 = pods)") + title: str + + +class TelegramPerson(ResponseModel): + id: str = Field(description="their Telegram user id, for telegram.users") + name: str + allowed: bool = Field(description="in telegram.users: Lens talks to them") + + +class TelegramStatus(ResponseModel): + state: Literal["off", "incomplete", "starting", "running", "error"] = Field( + description="off; incomplete (`error` says what's missing); starting (no server process has picked it up yet); " + "running; error (the last look at Telegram failed, `error` says why)" + ) + error: str | None = None + holder: str | None = Field(None, description="the server process reading the bot's messages") + at: str | None = Field(None, description="when it last looked") + answered: int | None = Field(None, description="messages it answered") + bot: str | None = Field(None, description="the bot's username, once Lens has asked Telegram") + chats: list[TelegramChat] = Field(default_factory=list, description="the chats it has seen, latest first") + people: list[TelegramPerson] = Field(default_factory=list, description="who has written to it, latest first") + + class BridgeTestResult(ResponseModel): ok: bool error: str | None = None diff --git a/fastapi_backend/tests/api/test_telegram.py b/fastapi_backend/tests/api/test_telegram.py new file mode 100644 index 000000000..494468f0b --- /dev/null +++ b/fastapi_backend/tests/api/test_telegram.py @@ -0,0 +1,207 @@ +"""The assistant in Telegram through a bot of Lens's own: set up in the app with the bot's token, it answers private +chats and what's said to it in groups, as the account it was given, hears voice notes and holds files for the +assistant to import, with no Matterbridge in between.""" + +from __future__ import annotations + +import pytest + +from app.domain import channels, settings, telegram, uploads, voice +from tests import fake_llm, fake_telegram +from tests.api._assist import start_llm +from tests.helpers import login, make_user, seed + +update = fake_telegram.update + + +@pytest.fixture +def tg(): + srv, url = fake_telegram.start() + telegram._BOTS.clear() + yield fake_telegram.Handler, url + srv.shutdown() + + +@pytest.fixture +def llm(cfg): + srv = start_llm(cfg) + yield fake_llm.Handler + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, llm): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def bot(app, db, cfg, folder, tg): + """Seeded recordings, a viewer of pods for the bot to answer as, and the bot set up for them.""" + seed(db, cfg, folder) + make_user(db, "root@x.io", "root password 1", admin=True) + make_user(db, "bot@x.io", "bot password 1", roles={"pods": "viewer"}) + h, url = tg + cfg["telegram"] = { + **cfg["telegram"], + "enabled": True, + "api_url": url, + "token": fake_telegram.TOKEN, + "account": "bot@x.io", + "users": ["alice", "bob", "carol", "10"], + } + cfg["ai"]["tools"] = False + return h + + +def run(db, cfg, me="api-1"): + return channels.tick(db, cfg, cfg, me, telegram.TELEGRAM) + + +def test_the_bot_is_set_up_and_checked_in_the_app(client, db, cfg, tg): + h, url = tg + make_user(db, "root@x.io", "root password 1", admin=True) + a = login(client, "root@x.io", "root password 1") + assert client.get("/api/v1/settings/telegram", headers=a).json()["state"] == "off" + assert client.post("/api/v1/settings/telegram/test", headers=a).json() == { + "ok": False, + "error": "paste the bot's token from @BotFather first", + } + assert client.put("/api/v1/settings/telegram", headers=a, json={"token": "not a token"}).status_code == 400 + wrong = "123456789:" + "B" * 35 + r = client.put("/api/v1/settings/telegram", headers=a, json={"token": wrong, "api_url": url + "/", "enabled": True}) + assert r.status_code == 200, r.text + saved = settings.Settings(db, cfg).current()["telegram"] + assert saved["token"] == wrong and saved["api_url"] == url and saved["account"] == "root@x.io" # who turned it on + assert wrong not in client.get("/api/v1/settings", headers=a).text # the token is a secret + assert "doesn't know that bot token" in client.post("/api/v1/settings/telegram/test", headers=a).json()["error"] + client.put("/api/v1/settings/telegram", headers=a, json={"token": fake_telegram.TOKEN}) + assert client.post("/api/v1/settings/telegram/test", headers=a).json() == {"ok": True, "error": None} + assert client.get("/api/v1/settings/telegram", headers=a).json()["state"] == "starting" + r = client.put("/api/v1/settings/telegram", headers=a, json={"users": ["@Alice", " ", "12345"], "rooms": ["-100123 = Pods"]}) + assert r.status_code == 200, r.text + saved = settings.Settings(db, cfg).current()["telegram"] + assert saved["users"] == ["Alice", "12345"] and saved["rooms"] == ["-100123 = pods"] + for bad in ({"account": "bot"}, {"answer": "sometimes"}, {"approve": "always"}, {"poll_seconds": 2}, {"api_url": "telegram"}): + assert client.put("/api/v1/settings/telegram", headers=a, json=bad).status_code == 400, bad + + +def test_matterbridge_reading_the_same_bot_is_caught(db, cfg, bot): + cfg["matterbridge"].update(run=True, telegram_token=fake_telegram.TOKEN) + assert "Matterbridge reads this bot too" in telegram.TELEGRAM.check(db, cfg) + cfg["matterbridge"].update(run=False) + assert telegram.TELEGRAM.check(db, cfg) is None + + +def test_it_answers_private_chats_and_what_is_said_to_it_in_groups(db, cfg, bot, llm): + bot.updates = [ + update("what leaves on Friday?"), # a private chat: always for Lens + update("lunch anyone?", chat_id=-100, chat_type="group", title="Team", user="bob", user_id=8), + update("@lens_test_bot when does the shipment leave?", chat_id=-100, chat_type="group", title="Team", user="bob", user_id=8), + update("Lens, hello", chat_id=-100, chat_type="group", title="Team", user="carol", user_id=9), + update( + "and on Monday?", + chat_id=-100, + chat_type="group", + title="Team", + user="carol", + user_id=9, + reply_to_message={"message_id": 1, "from": fake_telegram.BOT, "chat": {"id": -100}}, + ), + update("/start", user="dave", user_id=10, chat_id=43), + ] + assert run(db, cfg) == 5 + assert [s["chat_id"] for s in bot.sent] == ["42", "-100", "-100", "-100", "43"] + assert all(s["text"] == "OK" for s in bot.sent) # the fake model's answer + assert "reply_parameters" not in bot.sent[0] and bot.sent[1]["reply_parameters"]["message_id"] # groups: as a reply + asked = [m["messages"][-1]["content"] for m in llm.seen if m.get("messages")] + assert any(c.endswith("Question: what leaves on Friday?") for c in asked) + assert any(c.endswith("Question: when does the shipment leave?") for c in asked) + assert any(c.endswith("Question: Hello! What can you do?") for c in asked) + rows = db.rows("SELECT title, account FROM chat WHERE kind = 'bridge' ORDER BY title") + assert [r["title"] for r in rows] == ["Telegram · Team · bob", "Telegram · Team · carol", "Telegram · alice", "Telegram · dave"] + # what was read is confirmed on the next look, and isn't answered twice + last = max(u["update_id"] for u in bot.updates) + bot.sent = [] + assert run(db, cfg) == 0 and not bot.sent and bot.offsets[-1] == last + 1 and not bot.updates + # the chats it has seen, for Settings (their ids are what telegram.rooms names) + s = telegram.status(db, cfg) + assert s["state"] == "running" and s["answered"] == 5 and s["bot"] == "lens_test_bot" + assert {"id": "-100", "title": "Team"} in s["chats"] and {"id": "42", "title": "private chat with alice"} in s["chats"] + assert {"id": "10", "name": "dave", "allowed": True} in s["people"] + + +def test_only_people_let_in_are_answered(db, cfg, bot, llm): + """Anyone can find a bot: someone not in telegram.users is told once, and listed in Settings to let in.""" + bot.updates = [update("hi", user="eve", user_id=66), update("Lens, hi", user="eve", user_id=66, chat_id=-100, chat_type="group")] + assert run(db, cfg) == 0 + assert [s["text"] for s in bot.sent] == [telegram.STRANGER] # once, whatever else they say + assert not db.rows("SELECT id FROM chat WHERE kind = 'bridge'") + assert {"id": "66", "name": "eve", "allowed": False} in telegram.status(db, cfg)["people"] + bot.updates = [update("lunch?", user="mallory", user_id=67, chat_id=-100, chat_type="group")] + assert run(db, cfg) == 0 and len(bot.sent) == 1 # not said to Lens: not answered at all + cfg["telegram"]["users"] = ["66"] # let in by user id + bot.updates = [update("hi", user="eve", user_id=66)] + assert run(db, cfg) == 1 and bot.sent[-1]["text"] == "OK" + cfg["telegram"]["users"] = [] # nobody let in: nobody answered + bot.updates = [update("hi", user="eve", user_id=66)] + assert run(db, cfg) == 0 and len(bot.sent) == 2 + + +def test_a_chat_given_to_a_namespace_assistant(db, cfg, bot, llm): + cfg["telegram"]["rooms"] = ["-100 = pods"] + bot.updates = [update("Lens, when do episodes ship?", chat_id=-100, chat_type="group", title="Team")] + assert run(db, cfg) == 1 + assert db.one("SELECT VALUE scope FROM chat WHERE kind = 'bridge' LIMIT 1") == {"namespaces": ["pods"]} + + +def test_voice_notes_are_heard(db, cfg, bot, llm, monkeypatch): + bot.files["voice-1"] = b"OggS..." + monkeypatch.setattr(voice, "can_transcribe", lambda cfg: None) + bot.updates = [update(voice={"file_id": "voice-1", "duration": 2})] + assert run(db, cfg) == 1 + assert "no speech-to-text engine" in bot.sent[-1]["text"] + heard = [] + monkeypatch.setattr(voice, "can_transcribe", lambda cfg: "sensevoice") + monkeypatch.setattr(voice, "transcribe", lambda cfg, data: heard.append(data) or {"text": "what leaves on Friday?"}) + bot.updates = [update(voice={"file_id": "voice-1", "duration": 2})] + assert run(db, cfg) == 1 + assert heard == [b"OggS..."] + assert bot.sent[-1]["text"] == "(Heard: “what leaves on Friday?”)\n\nOK" + asked = [m["messages"][-1]["content"] for m in llm.seen if m.get("messages")] + assert asked[-1].endswith("Question: what leaves on Friday?") + # in a group, a voice note that isn't a reply to the bot isn't listened to + bot.updates = [update(voice={"file_id": "voice-1"}, chat_id=-100, chat_type="group", title="Team")] + assert run(db, cfg) == 0 and len(heard) == 1 + + +def test_files_are_held_for_the_assistant_to_import(db, cfg, bot, llm): + bot.files["doc-1"] = b"Episode notes.\n" + bot.updates = [update(document={"file_id": "doc-1", "file_name": "notes.txt", "file_size": 15}, caption="file this in pods")] + assert run(db, cfg) == 1 + held = db.rows("SELECT * FROM upload WHERE state = 'held'") + assert len(held) == 1 and held[0]["filename"] == "notes.txt" and held[0]["email"] == "bot@x.io" + uid = str(held[0]["id"]).split(":")[-1].strip("⟨⟩`") + assert uploads.get(db, uid)["size"] == 15 + asked = [m["messages"][-1]["content"] for m in llm.seen if m.get("messages")][-1] + assert "file this in pods" in asked and "Attached files, not in the archive yet" in asked and "notes.txt" in asked + # one that can't be taken is said, and the question is still answered + bot.files["bad"] = b"MZ" + bot.updates = [update(document={"file_id": "bad", "file_name": "tool.exe", "file_size": 2})] + assert run(db, cfg) == 1 + assert bot.sent[-1]["text"].startswith("Couldn't take tool.exe: ") and bot.sent[-1]["text"].endswith("OK") + + +def test_one_process_reads_the_bot_and_long_answers_are_split(db, cfg, bot, llm): + bot.updates = [update("hi")] + assert run(db, cfg, "api-1") == 1 + bot.updates = [update("hi again")] + assert run(db, cfg, "worker-2") == 0 # api-1 holds it + assert run(db, cfg, "api-1") == 1 + cfg["telegram"]["enabled"] = False + run(db, cfg, "api-1") # turned off: it lets go, and the offset stays + assert channels.state(db, telegram.TELEGRAM).get("offset") + cfg["telegram"]["enabled"] = True + telegram.TELEGRAM.send(cfg, {"gateway": "42"}, "y" * 9000) + assert [len(s["text"]) for s in bot.sent[-3:]] == [4000, 4000, 1000] diff --git a/fastapi_backend/tests/fake_telegram.py b/fastapi_backend/tests/fake_telegram.py new file mode 100644 index 000000000..cc771b4ed --- /dev/null +++ b/fastapi_backend/tests/fake_telegram.py @@ -0,0 +1,84 @@ +"""A tiny stand-in for Telegram's Bot API: `updates` are handed out by getUpdates (and forgotten once a later call's +offset passes them), what's sent with sendMessage lands in `sent`, and `files` are what getFile and /file/ serve.""" + +import http.server +import json +import threading + +TOKEN = "123456789:" + "A" * 35 +BOT = {"id": 999, "is_bot": True, "first_name": "Lens", "username": "lens_test_bot"} + + +class Handler(http.server.BaseHTTPRequestHandler): + updates: list = [] + sent: list = [] + files: dict = {} + offsets: list = [] + token = TOKEN + + def log_message(self, *a): + pass + + def _send(self, status, obj): + data = json.dumps(obj).encode() if not isinstance(obj, bytes) else obj + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + prefix = f"/file/bot{Handler.token}/" + if self.path.startswith(prefix) and self.path[len(prefix) :] in Handler.files: + return self._send(200, Handler.files[self.path[len(prefix) :]]) + self._send(404, b"not found") + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + bot, _, method = self.path.lstrip("/").partition("/") + if bot != f"bot{Handler.token}": + return self._send(401, {"ok": False, "error_code": 401, "description": "Unauthorized"}) + if method == "getMe": + return self._send(200, {"ok": True, "result": BOT}) + if method == "getUpdates": + off = body.get("offset") or 0 + Handler.offsets.append(off) + Handler.updates = [u for u in Handler.updates if u["update_id"] >= off] + return self._send(200, {"ok": True, "result": Handler.updates}) + if method == "sendMessage": + Handler.sent.append(body) + return self._send(200, {"ok": True, "result": {"message_id": len(Handler.sent)}}) + if method == "getFile": + fid = body.get("file_id") + if fid not in Handler.files: + return self._send(400, {"ok": False, "description": "Bad Request: file not found"}) + return self._send(200, {"ok": True, "result": {"file_id": fid, "file_size": len(Handler.files[fid]), "file_path": fid}}) + self._send(404, {"ok": False, "description": "Not Found"}) + + +_N = [0] + + +def update(text="", chat_id=42, chat_type="private", user="alice", user_id=7, title=None, **extra): + """An update with a message, as Telegram sends it.""" + _N[0] += 1 + chat = {"id": chat_id, "type": chat_type} + if title: + chat["title"] = title + msg = { + "message_id": _N[0], + "from": {"id": user_id, "is_bot": False, "first_name": user.title(), "username": user}, + "chat": chat, + "date": 1760000000, + **extra, + } + if text: + msg["text"] = text + return {"update_id": 1000 + _N[0], "message": msg} + + +def start(): + Handler.updates, Handler.sent, Handler.files, Handler.offsets, Handler.token = [], [], {}, [], TOKEN + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv, f"http://127.0.0.1:{srv.server_address[1]}" diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 32ab2f424..019793e2b 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -2521,6 +2521,12 @@ export type { TakeBackupData, TakeBackupResponse, TakeBackupResponses, + TelegramChat, + TelegramPerson, + TelegramStatus, + TelegramStatusData, + TelegramStatusResponse, + TelegramStatusResponses, TelemetryExport, TelemetryStatus, TelemetryStatusData, @@ -2578,6 +2584,9 @@ export type { TestSpeechErrors, TestSpeechResponse, TestSpeechResponses, + TestTelegramData, + TestTelegramResponse, + TestTelegramResponses, TestTelemetryData, TestTelemetryResponse, TestTelemetryResponses, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 6e5801f34..8f0fc8df4 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -1219,6 +1219,8 @@ import type { TagRecordingsResponses, TakeBackupData, TakeBackupResponses, + TelegramStatusData, + TelegramStatusResponses, TelemetryStatusData, TelemetryStatusResponses, TestBridgeData, @@ -1248,6 +1250,8 @@ import type { TestSpeechData, TestSpeechErrors, TestSpeechResponses, + TestTelegramData, + TestTelegramResponses, TestTelemetryData, TestTelemetryResponses, TokenData, @@ -2493,6 +2497,34 @@ export class Admin { }); } + /** + * Telegram Status + * + * How the assistant's Telegram bot is doing, and the chats it has seen (their ids are what telegram.rooms names). + */ + public static telegramStatus( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/settings/telegram", + ...options, + }); + } + + /** + * Test Telegram + * + * Check that Telegram knows the bot's token, and that the account to answer as exists. + */ + public static testTelegram( + options?: Options, + ): RequestResult { + return (options?.client ?? client).post({ + url: "/api/v1/settings/telegram/test", + ...options, + }); + } + /** * Test Conversion Node * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index fd83bbb05..450469e46 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -15131,6 +15131,99 @@ export type TagsChanged = { [key: string]: unknown; }; +/** + * TelegramChat + */ +export type TelegramChat = { + /** + * Id + * + * the chat's id, for telegram.rooms (like -1001234567890 = pods) + */ + id: string; + /** + * Title + */ + title: string; + [key: string]: unknown; +}; + +/** + * TelegramPerson + */ +export type TelegramPerson = { + /** + * Id + * + * their Telegram user id, for telegram.users + */ + id: string; + /** + * Name + */ + name: string; + /** + * Allowed + * + * in telegram.users: Lens talks to them + */ + allowed: boolean; + [key: string]: unknown; +}; + +/** + * TelegramStatus + */ +export type TelegramStatus = { + /** + * State + * + * off; incomplete (`error` says what's missing); starting (no server process has picked it up yet); running; error (the last look at Telegram failed, `error` says why) + */ + state: "off" | "incomplete" | "starting" | "running" | "error"; + /** + * Error + */ + error?: string | null; + /** + * Holder + * + * the server process reading the bot's messages + */ + holder?: string | null; + /** + * At + * + * when it last looked + */ + at?: string | null; + /** + * Answered + * + * messages it answered + */ + answered?: number | null; + /** + * Bot + * + * the bot's username, once Lens has asked Telegram + */ + bot?: string | null; + /** + * Chats + * + * the chats it has seen, latest first + */ + chats?: Array; + /** + * People + * + * who has written to it, latest first + */ + people?: Array; + [key: string]: unknown; +}; + /** * TelemetryExport */ @@ -19021,6 +19114,38 @@ export type TestBridgeResponses = { export type TestBridgeResponse = TestBridgeResponses[keyof TestBridgeResponses]; +export type TelegramStatusData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/telegram"; +}; + +export type TelegramStatusResponses = { + /** + * Successful Response + */ + 200: TelegramStatus; +}; + +export type TelegramStatusResponse = TelegramStatusResponses[keyof TelegramStatusResponses]; + +export type TestTelegramData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/telegram/test"; +}; + +export type TestTelegramResponses = { + /** + * Successful Response + */ + 200: BridgeTestResult; +}; + +export type TestTelegramResponse = TestTelegramResponses[keyof TestTelegramResponses]; + export type TestConversionNodeData = { body?: never; path?: never; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d7d16bf49..e588a283a 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -302,6 +302,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { return ; case "bridge": return ; + case "telegram": + return ; case "remote-access": return ; case "components": @@ -1141,6 +1143,123 @@ function BridgeBody({ ctx }: { ctx: BodyCtx }) { ); } +function TelegramBody({ ctx }: { ctx: BodyCtx }) { + const client = useApiClient(); + const qc = useQueryClient(); + const t = ctx.view.telegram; + const token = ctx.state("telegram.token"); + const status = useQuery({ + queryKey: ["telegram-status", t?.updated_at ?? null], + queryFn: () => data(Admin.telegramStatus({ client })), + refetchInterval: 5_000, // people who write to the bot show up to let in + }); + const test = useMutation({ mutationFn: () => data(Admin.testTelegram({ client })) }); + const saved = ((t?.values?.users ?? []) as string[]).filter(Boolean); + const letIn = useMutation({ + mutationFn: (id: string) => + data(Admin.updateSettings({ client, path: { section: "telegram" }, body: { users: [...saved, id] } })), + onSuccess: async () => { + await qc.invalidateQueries({ queryKey: ["settings"] }); + await qc.invalidateQueries({ queryKey: ["telegram-status"] }); + }, + }); + const s = status.data; + const waiting = (s?.people ?? []).filter((p) => !p.allowed); + return ( + <> + {s?.state === "running" ? ( + + {s.answered ? `Answered ${s.answered} message${s.answered === 1 ? "" : "s"} so far. ` : ""} + {saved.length + ? "Each person’s conversation is in Chat for the account it answers as." + : "Nobody is let in yet: message the bot from Telegram, then let yourself in below."} + + ) : s?.state === "error" ? ( + + {s.error} + + ) : s?.state === "incomplete" ? ( + + It needs {s.error}. + + ) : s?.state === "starting" ? ( + A server process picks it up within a few seconds. + ) : ( + + Create a bot with @BotFather in Telegram (/newbot), paste its token below and turn this on. + + )} + {waiting.length > 0 && ( +
+ Wrote to the bot, not let in +
    + {waiting.map((p) => ( +
  • + {p.name} + {p.id} + +
  • + ))} +
+ {letIn.isError && {letIn.error.message}} +
+ )} + + token.onChange(x)} + /> + + +
+ + +
+ + {(s?.chats ?? []).length > 0 && ( +
+ Chats it has seen +
    + {(s?.chats ?? []).map((c) => ( +
  • + {c.title} {c.id} +
  • + ))} +
+
+ )} + + +
+ + {ctx.dirty && ( + Uses the saved settings, not your unsaved changes + )} +
+ {test.data && + (test.data.ok ? ( + + The token and the account it answers as check out. + + ) : ( + + {test.data.error} + + ))} + {test.isError && {test.error.message}} + + ); +} + function MatterbridgeSecret({ ctx, id, label }: { ctx: BodyCtx; id: string; label: string }) { const m = ctx.view.matterbridge; const st = ctx.state(`matterbridge.${id}`); diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 6c59496a1..66f12e344 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -73,6 +73,7 @@ export type SectionId = | "notifications" | "mail" | "bridge" + | "telegram" | "telemetry" | "fedora" | "sensors" @@ -225,6 +226,13 @@ export const SECTIONS: SectionSpec[] = [ description: "The assistant in Slack, Discord, Telegram, Matrix and other chat rooms, through Matterbridge: it answers what’s said to it there.", }, + { + id: "telegram", + label: "Telegram", + backend: ["telegram"], + description: + "The assistant in Telegram through a bot of your own: paste its token from @BotFather. No Matterbridge and no public address needed.", + }, { id: "notifications", label: "Notifications", @@ -921,6 +929,65 @@ export const FIELDS: FieldSpec[] = [ max: 300, unit: "s", }, + // Telegram (telegram.py) + { section: "telegram", key: "enabled", label: "Answer in Telegram", kind: "switch" }, + { + section: "telegram", + key: "account", + label: "Answers as", + kind: "text", + nullable: true, + placeholder: "you, when left empty", + hint: "The Lens account it answers as: it reads what that account can read, and changes wait for approval.", + }, + { section: "telegram", key: "name", label: "Its name in groups", kind: "text" }, + { + section: "telegram", + key: "answer", + label: "In groups, answers", + kind: "select", + options: [ + { value: "mention", label: "Messages that name it, mention the bot or reply to it" }, + { value: "all", label: "Every message (turn off the bot’s privacy mode in @BotFather)" }, + ], + }, + { + section: "telegram", + key: "users", + label: "People it talks to", + kind: "lines", + mono: true, + hint: "Telegram user ids or usernames, one per line. Anyone can find a bot, so it answers nobody else; people who write to it show above, to let in with one click.", + }, + { + section: "telegram", + key: "approve", + label: "Approving in the chat", + kind: "select", + options: [ + { value: "low_risk", label: "Entity merges and edits, new namespaces" }, + { value: "off", label: "Only in the web app" }, + ], + hint: "The people above can reply “yes” or “no” to these in Telegram. Settings, imports, runs and extensions are always approved in the web app.", + }, + { + section: "telegram", + key: "rooms", + label: "Chats for a namespace’s assistant", + kind: "lines", + mono: true, + hint: "One per line, like -1001234567890 = pods (the chats it has seen are listed above with their ids): conversations there talk to that namespace’s own assistant. Elsewhere Lens picks the namespace for each question.", + }, + { + section: "telegram", + key: "api_url", + label: "Bot API server", + kind: "text", + mono: true, + nullable: true, + placeholder: "https://api.telegram.org", + hint: "Leave empty unless you run your own telegram-bot-api server.", + }, // Matterbridge run by Lens (Chat rooms) { section: "matterbridge", key: "run", label: "Run Matterbridge here", kind: "switch" }, { section: "matterbridge", key: "slack_token", label: "Slack bot token", kind: "secret" }, diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx index 7918f3209..5ef2b4d73 100644 --- a/nextjs-frontend/components/settings/settings-shell.tsx +++ b/nextjs-frontend/components/settings/settings-shell.tsx @@ -32,6 +32,7 @@ import { Users, Mail, MessagesSquare, + Send, PackageCheck, RadioTower, } from "lucide-react"; @@ -79,6 +80,7 @@ const ICON: Record = { notifications: Bell, mail: Mail, bridge: MessagesSquare, + telegram: Send, telemetry: Activity, fedora: Archive, sensors: RadioTower, diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index cfe128052..39856bebf 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2488,6 +2488,50 @@ } } }, + "/api/v1/settings/telegram": { + "get": { + "tags": [ + "admin" + ], + "summary": "Telegram Status", + "description": "How the assistant's Telegram bot is doing, and the chats it has seen (their ids are what telegram.rooms names).", + "operationId": "telegram_status", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TelegramStatus" + } + } + } + } + } + } + }, + "/api/v1/settings/telegram/test": { + "post": { + "tags": [ + "admin" + ], + "summary": "Test Telegram", + "description": "Check that Telegram knows the bot's token, and that the account to answer as exists.", + "operationId": "test_telegram", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BridgeTestResult" + } + } + } + } + } + } + }, "/api/v1/settings/documents/node/test": { "post": { "tags": [ @@ -50467,6 +50511,149 @@ ], "title": "TagsChanged" }, + "TelegramChat": { + "properties": { + "id": { + "type": "string", + "title": "Id", + "description": "the chat's id, for telegram.rooms (like -1001234567890 = pods)" + }, + "title": { + "type": "string", + "title": "Title" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "title" + ], + "title": "TelegramChat" + }, + "TelegramPerson": { + "properties": { + "id": { + "type": "string", + "title": "Id", + "description": "their Telegram user id, for telegram.users" + }, + "name": { + "type": "string", + "title": "Name" + }, + "allowed": { + "type": "boolean", + "title": "Allowed", + "description": "in telegram.users: Lens talks to them" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "name", + "allowed" + ], + "title": "TelegramPerson" + }, + "TelegramStatus": { + "properties": { + "state": { + "type": "string", + "enum": [ + "off", + "incomplete", + "starting", + "running", + "error" + ], + "title": "State", + "description": "off; incomplete (`error` says what's missing); starting (no server process has picked it up yet); running; error (the last look at Telegram failed, `error` says why)" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "holder": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Holder", + "description": "the server process reading the bot's messages" + }, + "at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "At", + "description": "when it last looked" + }, + "answered": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Answered", + "description": "messages it answered" + }, + "bot": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Bot", + "description": "the bot's username, once Lens has asked Telegram" + }, + "chats": { + "items": { + "$ref": "#/components/schemas/TelegramChat" + }, + "type": "array", + "title": "Chats", + "description": "the chats it has seen, latest first" + }, + "people": { + "items": { + "$ref": "#/components/schemas/TelegramPerson" + }, + "type": "array", + "title": "People", + "description": "who has written to it, latest first" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "state" + ], + "title": "TelegramStatus" + }, "TelemetryExport": { "properties": { "at": { diff --git a/threat-model.md b/threat-model.md index 01f0272e9..f2c322cad 100644 --- a/threat-model.md +++ b/threat-model.md @@ -101,7 +101,7 @@ optional listeners (MQTT, syslog, tunnel, local model server) run inside workers | Web app & API gateway (Web) | `nextjs-frontend/proxy.ts`, `lib/api/backend-proxy.ts`, `next.config.mjs`; `core/middleware.py` Host check and CSP | network | In | | Identity & sessions (Identity) | `routes/{auth,passkeys,oauth,external,setup,users}.py`, `domain/{auth,passkeys,oauth,external_login,iiif_auth}.py`, `core/security.py`, `nextjs-frontend/auth.ts` | DB, OIDC providers, SMTP, logs | In | | Authorization & data API (AuthZ) | `api/deps.py` (`Access`, `Writer`, `Admin*`), `domain/access.py`, `domain/ipgroups.py`, share links, `api/media.py`, `api/iiif.py`, `api/pages.py`, `api/mcp_tools.py` | DB, files | In | -| Outbound fetchers & stored secrets (Outbound) | `domain/netguard.py`, `webcapture.py`, `feeds.py`, `notify.py`, `sources.py`, `llm.py`, `semantic.py`, `speech.py`, `fedora.py`, `bridge.py`, `telemetry.py`, `settings.py` seal, `keyring.py`, `vaults.py` | network, child processes, files | In | +| Outbound fetchers & stored secrets (Outbound) | `domain/netguard.py`, `webcapture.py`, `feeds.py`, `notify.py`, `sources.py`, `llm.py`, `semantic.py`, `speech.py`, `fedora.py`, `bridge.py`, `telegram.py`, `telemetry.py`, `settings.py` seal, `keyring.py`, `vaults.py` | network, child processes, files | In | | Ingest & processing (Ingest) | `routes/{uploads,files,imports}.py`, `domain/{ingest,documents,convert,video,uploads}.py`, `templates.py`, `render.py` | child processes (ffmpeg, LibreOffice, Chromium, poppler, Tesseract), files | In | | Query engines (Query) | `domain/cypher.py`, `domain/rdf.py` (SPARQL), `domain/rdf_import.py`, `domain/search.py` | DB | In | | Assistant & extensions (Assistant) | `domain/{chat,ai_tools,ops_tools,extensions,tool_nodes,flow,custom_nodes,code_tools}.py`, MCP write tools | network (model servers), child process (code tools) | In | @@ -276,7 +276,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | `/uploads`, `/import`, `/recordings/{rid}/files` | file bytes, name, extension, size | yes, editors | data, type/class, size | none | *(documented, docs/configuration.md "Uploads accept only the types")* | | watched folders, storage sources, IMAP, iCal | remote file names, bytes, mail, events | yes, whoever writes to the remote, mails the box or serves the feed | data, resource-name, size | admin creates sources | *(inferred, Q17)* | | `POST /imports/web`, extension `http` tools, notification targets | URL | yes: editors (web, tools), owners (targets) | resource-name, x-network-destination | none | *(maintainer, 2026-10)* | -| admin settings: model, embeddings, speech, decisions, Fedora, bridge, telemetry, IIIF import, rclone remotes, IMAP, iCal, MQTT bridge, OIDC issuer | URL, credentials, `api_key_env` | admin only; documented guards still apply (P-ADMIN-GUARD) | resource-name, x-credential | admin is trusted | *(maintainer, 2026-10)* | +| admin settings: model, embeddings, speech, decisions, Fedora, bridge, Telegram bot, telemetry, IIIF import, rclone remotes, IMAP, iCal, MQTT bridge, OIDC issuer | URL, credentials, `api_key_env` | admin only; documented guards still apply (P-ADMIN-GUARD) | resource-name, x-credential | admin is trusted | *(maintainer, 2026-10)* | | replies from remote servers | web page HTML and scripts, IIIF manifests, model replies, OIDC discovery, calendars and their redirects | yes, the remote | data, topology | treat as untrusted | *(inferred, Q17)* | | chat, `/graph/ask`, MCP tools | prompt text; model-chosen tool calls | yes; also indirectly through content | data, x-prompt-injection | none | *(documented, docs/assistant.md "limited to what the person can read")* | | extensions (prompt, graph and `http` tools, hooks), custom nodes including `template` nodes (Q23) | tool spec, graph JSON, template text | yes, any writer for their own; sharing is gated | callback/code (declarative) | person enables only extensions they trust | *(documented, docs/assistant.md "other people's run only when an admin shared them")* | @@ -556,7 +556,7 @@ Exported by phase 3.6 (backtest) and de-identified. Closing rows carry their sta | N-ENUM | Hiding account or object existence outside password sign-in and signed-in 404s | Passkey 404, external messages, anonymous 401 vs 404 | correctness-only | no | *(inferred, Q6)* | | N-AUDIT | Tamper evidence or completeness of the audit and activity logs; sign-in requests are not logged there | operations record only; admins read it | correctness-only | no | *(documented, docs/activity.md "aren't logged here")* | | N-MQTT-SUBSCRIBE | Topic isolation on the MQTT hub: "anything can subscribe to it, with `+` and `#`" | Signed-in or anonymous-when-enabled clients | security-critical | yes: a hub login is not per-stream access | *(documented, docs/sensors.md "anything can subscribe to it")* | -| N-BRIDGE | Per-person access in chat rooms. The bridge answers as one account, the turning-on admin by default | Everyone in the room reads what that account reads (exposure posture: Q28) | security-critical | no | *(documented, docs/chat-rooms.md "answers as the admin who turned it on")* | +| N-BRIDGE | Per-person access in chat rooms. The bridge (Matterbridge or the Telegram bot) answers as one account, the turning-on admin by default | Everyone in the room reads what that account reads (exposure posture: Q28); the Telegram bot answers only people in `telegram.users` (anyone can find a bot), and nobody until one is let in | security-critical | no | *(documented, docs/chat-rooms.md "answers as the admin who turned it on")* | | N-PUBLIC | Privacy of a public recording's open parts: anyone may use them | Open parts only; closed parts and restricted or private recordings are P-ACCESS-PARTS | correctness-only | no | *(documented, docs/access.md "the parts of a public recording anyone may use")* | | N-DESCRIPTION | Hiding a recording's description when its parts are closed | Anyone who may see the page | correctness-only | no | *(documented, docs/access.md "never a closed part")* | | N-RDF-XML | XML parsing in RDF import. RDF/XML is refused, so XXE and billion-laughs do not apply there | RDF import only, not docx | security-critical | no | *(documented, rdf_import.py docstring "RDF/XML isn't read")* | diff --git a/threat-model.yaml b/threat-model.yaml index 872a1a204..a77a2e106 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:c7c945e760490e73dbb947e89beff868a218b89b839b4172f9d928d4853e992a" +prose_version: "threat-model.md@sha256:9f5ac9f2ac8bf790f5fb03313f216c949a30ec4daf765f8fe290e10eca8a73c9" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8}