From 316c8971ccd4d62d1a0a247cb290b48582d654c7 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:40:42 +0000 Subject: [PATCH 1/5] feat(extensions): run stdio MCP servers in a sandboxed runner MCP servers that run as a program (npx, uvx) run in the MCP runner, an opt-in container (COMPOSE_PROFILES=mcp) with none of Lens's data or keys, no capabilities, a read-only image and a network shared only with the backend. Admins add them as mcp extensions with command/args/env, and a plugin's stdio servers come along when an admin installs it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CsyA2Kuhsnhzz2HjkD3zED --- .github/workflows/mcp-runner.yml | 65 ++++ docker-compose.prod.yml | 30 +- docker-compose.yml | 29 +- docs/assistant.md | 20 +- fastapi_backend/app/domain/extensions.py | 39 ++- fastapi_backend/app/domain/marketplace.py | 29 +- fastapi_backend/app/domain/mcp_client.py | 58 +++- fastapi_backend/tests/api/test_mcp_servers.py | 78 +++++ mcp-runner/Dockerfile | 16 + mcp-runner/runner.py | 297 ++++++++++++++++++ 10 files changed, 627 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/mcp-runner.yml create mode 100644 mcp-runner/Dockerfile create mode 100644 mcp-runner/runner.py diff --git a/.github/workflows/mcp-runner.yml b/.github/workflows/mcp-runner.yml new file mode 100644 index 000000000..d9a27bc96 --- /dev/null +++ b/.github/workflows/mcp-runner.yml @@ -0,0 +1,65 @@ +name: MCP runner + +# The MCP runner (the optional mcp service, mcp-runner/): it builds, starts as docker-compose runs it (read-only, no +# capabilities), writes its token to the volume it shares with Lens, and runs real stdio MCP servers from npm and PyPI. +on: + pull_request: + paths: + - mcp-runner/** + - .github/workflows/mcp-runner.yml + push: + branches: [main] + paths: + - mcp-runner/** + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + mcp-runner: + name: Build and run servers + runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Build + run: docker build -t lens-mcp:ci mcp-runner + + - name: Start it as Lens's compose file does + run: | + docker network create mcp + docker run -d --name mcp --network mcp --read-only --tmpfs /tmp --cap-drop ALL \ + --security-opt no-new-privileges:true -v mcp-data:/data -v mcp-key:/key lens-mcp:ci + sleep 3 + docker logs mcp + + - name: Runs an npm server and a PyPI server + run: | + token="$(docker run --rm -v mcp-key:/key alpine:3.20 cat /key/token)" + call() { + docker run --rm --network mcp curlimages/curl:8.10.1 -fsS --max-time 240 -X "$1" \ + -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://mcp:8090$2" -d "$3" + } + call PUT /servers/memory '{"command": "npx", "args": ["-y", "@modelcontextprotocol/server-memory@2026.8.31"]}' + call POST /servers/memory/mcp '{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}' | tee out.json + grep -q create_entities out.json + call PUT /servers/time '{"command": "uvx", "args": ["mcp-server-time==2026.8.18"]}' + call POST /servers/time/mcp '{"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "get_current_time", "arguments": {"timezone": "UTC"}}}' | tee out.json + grep -q UTC out.json + # and refuses anyone without the token + code="$(docker run --rm --network mcp curlimages/curl:8.10.1 -s -o /dev/null -w '%{http_code}' -X PUT http://mcp:8090/servers/x -d '{}')" + test "$code" = 401 + + - name: What it said + if: failure() + run: docker logs mcp + + - name: Stop + if: always() + run: docker rm -f mcp || true diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c5b41f574..7be10f2ac 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -60,6 +60,9 @@ x-backend: &backend LENS_TELEMETRY_HEADERS: ${LENS_TELEMETRY_HEADERS:-} # the volume shared with the matterbridge service (its config, written from Settings → Chat rooms) LENS_MATTERBRIDGE_DIR: /matterbridge + # the MCP runner (the optional `mcp` service, docs/assistant.md): MCP servers that run as a program, sandboxed there + LENS_MCP_RUNNER_URL: http://mcp:8090 + LENS_MCP_TOKEN_FILE: /mcp-key/token # upgrading from the web app (docs/updates.md): the folder install.sh's upgrade agent watches, and the install folder LENS_CONTROL_DIR: /control LENS_HOST_DIR: ${LENS_DIR:-} @@ -68,11 +71,12 @@ x-backend: &backend - ./.lens-control:/control - ${AUDIO_DIR:-./local-audio}:/audio:ro - matterbridge-data:/matterbridge + - mcp-key:/mcp-key:ro depends_on: surrealdb: condition: service_healthy restart: unless-stopped - networks: [lens] + networks: [lens, mcp] services: surrealdb: @@ -139,6 +143,26 @@ services: restart: unless-stopped networks: [lens] + # The MCP runner (mcp-runner/runner.py, docs/assistant.md): MCP servers that run as a program (stdio, usually npx or + # uvx), which admins add as extensions. Opt-in: `docker compose --profile mcp up -d`, or COMPOSE_PROFILES=mcp in .env. + # The sandbox: none of Lens's data or keys, no capabilities, a read-only image, and a network shared only with the + # backend (and the internet). + mcp: + image: lens-mcp:${LENS_VERSION:-prod} + build: ./mcp-runner + profiles: [mcp] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + tmpfs: [/tmp] + mem_limit: 2g + pids_limit: 512 + volumes: + - mcp-data:/data + - mcp-key:/key + restart: unless-stopped + networks: [mcp] + # Fedora 6 (docs/fedora.md), the repository Lens can keep a copy of the archive in. Opt-in, and not needed on small # machines: `docker compose --profile fedora up`, with LENS_FEDORA_URL=http://fedora:8080/fcrepo/rest (and # LENS_FEDORA_USER / LENS_FEDORA_PASSWORD) in .env, or set it in Settings → Fedora. @@ -172,6 +196,8 @@ services: volumes: surreal-data: matterbridge-data: + mcp-data: + mcp-key: archive-data: fedora-data: opensearch-data: @@ -179,3 +205,5 @@ volumes: networks: lens: driver: bridge + mcp: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index 179d67cda..30c4a825d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,9 @@ x-backend: &backend OPENAPI_OUTPUT_FILE: ./shared-data/openapi.json # the volume shared with the matterbridge service (its config, written from Settings → Chat rooms) LENS_MATTERBRIDGE_DIR: /matterbridge + # the MCP runner (the optional `mcp` service, docs/assistant.md): MCP servers that run as a program, sandboxed there + LENS_MCP_RUNNER_URL: http://mcp:8090 + LENS_MCP_TOKEN_FILE: /mcp-key/token volumes: - ./fastapi_backend:/app # the image's own packages, not the source tree's: anonymous, so `make dev` (up --build --renew-anon-volumes) @@ -50,10 +53,11 @@ x-backend: &backend - ./local-shared-data:/app/shared-data - ${AUDIO_DIR:-./local-audio}:/audio:ro - matterbridge-data:/matterbridge + - mcp-key:/mcp-key:ro depends_on: surrealdb: condition: service_healthy - networks: [lens] + networks: [lens, mcp] services: surrealdb: @@ -131,6 +135,25 @@ services: - matterbridge-data:/matterbridge networks: [lens] + # The MCP runner (mcp-runner/runner.py, docs/assistant.md): MCP servers that run as a program (stdio, usually npx or + # uvx), which admins add as extensions. Opt-in: `docker compose --profile mcp up -d`, or COMPOSE_PROFILES=mcp in .env. + # The sandbox: none of Lens's data or keys, no capabilities, a read-only image, and a network shared only with the + # backend (and the internet). + mcp: + image: lens-mcp:dev + build: ./mcp-runner + profiles: [mcp] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + tmpfs: [/tmp] + mem_limit: 2g + pids_limit: 512 + volumes: + - mcp-data:/data + - mcp-key:/key + networks: [mcp] + # Fedora 6 (docs/fedora.md), the repository Lens can keep a copy of the archive in. Opt-in, and not needed on small # machines: `docker compose --profile fedora up`, with LENS_FEDORA_URL=http://fedora:8080/fcrepo/rest (and # LENS_FEDORA_USER / LENS_FEDORA_PASSWORD) in .env, or set it in Settings → Fedora. @@ -149,9 +172,13 @@ services: volumes: surreal-data: matterbridge-data: + mcp-data: + mcp-key: archive-data: fedora-data: networks: lens: driver: bridge + mcp: + driver: bridge diff --git a/docs/assistant.md b/docs/assistant.md index aa08c9081..f94f336ec 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -159,8 +159,9 @@ shared with namespaces (admins can share with everyone), and switched on or off - **MCP server** (admins add them): a server that speaks the Model Context Protocol over Streamable HTTP, at the address an admin gives (this machine and the local network included, as for the model's address), with the headers it needs (a token). Lens lists its tools when it's saved, and the assistant calls them as `__`: one the - server marks read-only answers at once, the others ask for approval. Servers that run as a program (stdio) aren't - run. + server marks read-only answers at once, the others ask for approval. A server that runs as a program (stdio: + `command`, `args` and `env` instead of an address, as in Claude's `.mcp.json`, usually `npx` or `uvx`) runs in the + MCP runner, below. - **Plugin**: a bundle of tools, skills, hooks and MCP servers, shared and switched off as one. Make one however suits you: @@ -206,9 +207,22 @@ version. A skill (a folder with SKILL.md) is a skill, and the assistant reads it scripts) when the instructions point at them; nothing in them runs. A command (`commands/*.md`) is a skill the assistant follows when someone asks for it, and an agent (`agents/*.md`) is a prompt tool with the agent's instructions as its system prompt. Claude Code's hooks run commands on the computer, so they're left out. A plugin's MCP servers on -the network (`.mcp.json`) come along when an admin installs it; those that run as a program are left out. Repositories are read from GitHub or GitLab (or the archive link) through the same guard as web +the network (`.mcp.json`) come along when an admin installs it, and those that run as a program too when the MCP +runner is on (ones that run a program shipped inside the plugin are left out). Repositories are read from GitHub or GitLab (or the archive link) through the same guard as web tools: public addresses only. +### The MCP runner + +MCP servers that run as a program run in a container of their own, the MCP runner +([`mcp-runner/`](https://github.com/adeelahmad/lense/tree/main/mcp-runner)), never next to Lens. It's off until you +turn it on: add `COMPOSE_PROFILES=mcp` to Lens's `.env` and run the install command again (or +`docker compose --profile mcp up -d`). The container is the sandbox: it has none of Lens's data, keys or database, runs +as an unprivileged user with every capability dropped, a read-only image and memory and process limits, and shares a +network only with Lens's backend (and the internet, which `npx` and `uvx` need to fetch the servers). Each server gets +an empty folder as its home and only the environment its extension gives it; Lens talks to the runner over HTTP with +a token the runner writes to a volume only the two share. A server starts on first use, stays up while it's used, +and stops after 15 idle minutes. Installs that don't run on Docker don't have the runner yet. + `ai.extensions: false` turns every extension off. ## Setting up and running the server by chat diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index 3a3bcaca6..27c370596 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -264,22 +264,36 @@ def _check_hook(spec, me, db=None): def _check_mcp(spec, me, db=None): - """An MCP server: its address, the headers it needs (a token), and its tools as it listed them (mcp_client.py).""" - if set(spec) - {"url", "headers", "tools"}: - raise ValueError("an MCP server is {url, headers, tools}") + """An MCP server: its address and the headers it needs (a token), or the program it runs as (command, args, env: + run in the MCP runner's sandbox), and its tools as it listed them (mcp_client.py).""" + if set(spec) - {"url", "headers", "command", "args", "env", "tools"}: + raise ValueError("an MCP server is {url, headers, tools}, or {command, args, env, tools} for one that runs as a program") if not me["admin"]: raise ValueError("only admins add MCP servers") - url = _text(spec.get("url"), "the MCP server's address", 2000, required=True) - if not re.match(r"^https?://[^/?#\s]+", url): - raise ValueError("an MCP server's address starts with http:// or https:// (servers that run as a program aren't run)") - headers = spec.get("headers") or {} - if not isinstance(headers, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in headers.items()): - raise ValueError("headers are names and text values") + + def words(v, what): + if not isinstance(v, dict) or not all(isinstance(k, str) and isinstance(x, str) for k, x in v.items()): + raise ValueError(f"{what} are names and text values") + return v + + if spec.get("command") is not None: + if spec.get("url") or spec.get("headers"): + raise ValueError("an MCP server has an address or a command, not both") + server = {"command": _text(spec.get("command"), "the MCP server's command", 200, required=True)} + args = spec.get("args") or [] + if not isinstance(args, list) or len(args) > 50 or not all(isinstance(a, str) and len(a) <= 2000 for a in args): + raise ValueError("an MCP server's args are a list of words") + server.update({"args": args or None, "env": words(spec.get("env") or {}, "env values") or None}) + else: + url = _text(spec.get("url"), "the MCP server's address (or its command)", 2000, required=True) + if not re.match(r"^https?://[^/?#\s]+", url): + raise ValueError("an MCP server's address starts with http:// or https://") + server = {"url": url, "headers": words(spec.get("headers") or {}, "headers") or None} tools = spec.get("tools") if tools is None: # asked of the server: what it offers now from . import mcp_client - tools = mcp_client.discover({"url": url, "headers": headers}) + tools = mcp_client.discover(store.clean(server)) if tools is not None: if not isinstance(tools, list) or len(tools) > 100: raise ValueError("an MCP server's tools are a list of at most 100") @@ -297,7 +311,7 @@ def _check_mcp(spec, me, db=None): ) for t in tools ] - return store.clean({"url": url, "headers": headers or None, "tools": tools}) + return store.clean({**server, "tools": tools}) def mcp_tools(item): @@ -306,7 +320,8 @@ def mcp_tools(item): s, out = item["spec"], [] for t in s.get("tools") or []: name = re.sub(r"[^A-Za-z0-9_-]", "_", f"{item['name']}__{t['name']}")[:64] - run = store.clean({"type": "mcp", "url": s["url"], "headers": s.get("headers"), "tool": t["name"]}) + where = {k: s.get(k) for k in ("url", "headers", "command", "args", "env")} + run = store.clean({"type": "mcp", **where, "tool": t["name"]}) spec = { "params": [], "schema": t.get("schema") or {"type": "object"}, diff --git a/fastapi_backend/app/domain/marketplace.py b/fastapi_backend/app/domain/marketplace.py index 20f2b3599..0136d52f9 100644 --- a/fastapi_backend/app/domain/marketplace.py +++ b/fastapi_backend/app/domain/marketplace.py @@ -15,8 +15,8 @@ What becomes what: a skill is a skill, with its text files (references, examples, scripts to read; nothing runs); a command is a skill the assistant follows when someone asks for it; an agent is a prompt tool with the agent's instructions as its system prompt. Claude Code's hooks run shell commands on the computer, so they're left out; MCP -servers on the network come along when an admin installs it (servers that run as a program don't); `preview` lists -what's left out and why. A plugin becomes one Lens plugin, owned by the person +servers come along when an admin installs it (those that run as a program run in the MCP runner's sandbox, when it +runs: mcp_client.py); `preview` lists what's left out and why. A plugin becomes one Lens plugin, owned by the person who installed it and private until they share it, and installing it again brings the new version. Only admins add marketplaces: that's the approval. People who can write extensions install from the ones added, and @@ -431,15 +431,26 @@ def _mcp(found, meta, admin, report): c = c if isinstance(c, dict) else {} kind = c.get("type") or ("stdio" if c.get("command") else "http") if kind == "stdio" or c.get("command"): - report.append( - {"kind": "mcp", "name": name, "why": "it runs a program on the computer; Lens reaches MCP servers on the network"} - ) - continue - if kind not in ("http", "streamable-http") or not isinstance(c.get("url"), str): + args = [str(x) for x in c.get("args") or [] if isinstance(x, (str, int, float))] + env = {k: str(v) for k, v in (c.get("env") or {}).items() if isinstance(k, str) and isinstance(v, (str, int, float))} + if not isinstance(c.get("command"), str): + report.append({"kind": "mcp", "name": name, "why": "it says it runs a program but not which"}) + continue + if any("CLAUDE_PLUGIN_ROOT" in x for x in [c["command"], *args, *env.values()]): + report.append( + {"kind": "mcp", "name": name, "why": "it runs a program from the plugin's own folder, which Lens doesn't run"} + ) + continue + if not mcp_client.runner(): + report.append({"kind": "mcp", "name": name, "why": mcp_client.NO_RUNNER}) + continue + server = store.clean({"command": c["command"], "args": args or None, "env": env or None}) + elif kind not in ("http", "streamable-http") or not isinstance(c.get("url"), str): report.append({"kind": "mcp", "name": name, "why": f"Lens speaks MCP over HTTP; this one uses {kind}"}) continue - headers = {k: v for k, v in (c.get("headers") or {}).items() if isinstance(k, str) and isinstance(v, str)} - server = {"url": c["url"], "headers": headers or None} + else: + headers = {k: v for k, v in (c.get("headers") or {}).items() if isinstance(k, str) and isinstance(v, str)} + server = {"url": c["url"], "headers": headers or None} try: tools = mcp_client.discover(server) except ValueError as e: diff --git a/fastapi_backend/app/domain/mcp_client.py b/fastapi_backend/app/domain/mcp_client.py index 6092b3dec..324c4102a 100644 --- a/fastapi_backend/app/domain/mcp_client.py +++ b/fastapi_backend/app/domain/mcp_client.py @@ -1,15 +1,21 @@ """Lens as an MCP client: the assistant calls the tools of MCP servers an admin added (extensions.py, kind `mcp`). A server is reached over Streamable HTTP (the MCP transport for servers on the network), and answers as JSON or as an -event stream; each use is a short session of its own: `initialize`, then `tools/list` or `tools/call`. Servers that -run as a program on this computer (stdio) aren't run. Only admins add servers, as they set the model's address, so -the address is theirs to choose: a server on this machine or the local network works. +event stream; each use is a short session of its own: `initialize`, then `tools/list` or `tools/call`. Only admins +add servers, as they set the model's address, so the address is theirs to choose: a server on this machine or the +local network works. + +A server that runs as a program (stdio: a command, usually `npx` or `uvx`) runs in the MCP runner, a container of its +own with none of Lens's data or keys (mcp-runner/runner.py, the optional `mcp` service), which Lens tells what to run +and then reaches over HTTP like any other. Without the runner, such servers aren't run. """ from __future__ import annotations +import hashlib import itertools import json +import os import urllib.error import urllib.request @@ -41,9 +47,9 @@ def _events(raw): data = [] -def _post(url, headers, body, timeout=TIMEOUT): - """One POST to the server: (status, response headers, body text). Tests replace it.""" - req = urllib.request.Request(url, data=json.dumps(body).encode(), method="POST", headers=headers) +def _post(url, headers, body, timeout=TIMEOUT, method="POST"): + """One request to the server (or the runner): (status, response headers, body text). Tests replace it.""" + req = urllib.request.Request(url, data=json.dumps(body).encode(), method=method, headers=headers) try: with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (an admin's address, http or https) return r.status, dict(r.headers.items()), r.read(MAX_REPLY).decode("utf-8", "replace") @@ -53,8 +59,44 @@ def _post(url, headers, body, timeout=TIMEOUT): raise McpError(f"couldn't reach the MCP server: {getattr(e, 'reason', e)}") from None +def runner(): + """The MCP runner's address and token, when it runs (LENS_MCP_RUNNER_URL, and the token it wrote to + LENS_MCP_TOKEN_FILE); None otherwise.""" + url = os.environ.get("LENS_MCP_RUNNER_URL", "").rstrip("/") + try: + with open(os.environ.get("LENS_MCP_TOKEN_FILE") or "/mcp-key/token") as f: + token = f.read().strip() + except OSError: + return None + return (url, token) if url and token else None + + +NO_RUNNER = ( + "MCP servers that run as a program need the MCP runner, a sandbox of their own: add COMPOSE_PROFILES=mcp to " + "Lens's .env and run the install command again (docs/assistant.md)" +) + + +def reach(server): + """Where to reach a server: its own address, or (one that runs as a program) the runner's, after telling the + runner what to run. {url, headers}.""" + if not server.get("command"): + return server + r = runner() + if not r: + raise McpError(NO_RUNNER) + conf = {"command": server["command"], "args": server.get("args") or [], "env": server.get("env") or {}} + key = "s" + hashlib.sha256(json.dumps(conf, sort_keys=True).encode()).hexdigest()[:24] + headers = {"Authorization": f"Bearer {r[1]}", "Content-Type": "application/json"} + status, _, text = _post(f"{r[0]}/servers/{key}", headers, conf, method="PUT") + if status >= 400: + raise McpError(f"the MCP runner said {status}: {text[:300]}") + return {"url": f"{r[0]}/servers/{key}/mcp", "headers": {"Authorization": headers["Authorization"]}} + + class Session: def __init__(self, server): + server = reach(server) if not str(server.get("url") or "").startswith(("http://", "https://")): raise McpError("an MCP server's address starts with http:// or https://") self.url = server["url"] @@ -106,7 +148,7 @@ def call(self, name, args): def discover(server): """A server's tools, as an MCP extension keeps them: {name, description, schema, read_only}.""" - with activity.call("mcp.list", detail={"url": server.get("url")}): + with activity.call("mcp.list", detail={"url": server.get("url") or server.get("command")}): tools = Session(server).tools() out = [] for t in tools: @@ -124,7 +166,7 @@ def discover(server): def call(run, args): """Call one of a server's tools: its text, and anything structured it gave back.""" - with activity.call("mcp.call", detail={"url": run["url"], "tool": run["tool"]}): + with activity.call("mcp.call", detail={"url": run.get("url") or run.get("command"), "tool": run["tool"]}): got = Session(run).call(run["tool"], args) texts = [c.get("text") for c in got.get("content") or [] if isinstance(c, dict) and c.get("type") == "text"] out = {"text": "\n".join(t for t in texts if t)} diff --git a/fastapi_backend/tests/api/test_mcp_servers.py b/fastapi_backend/tests/api/test_mcp_servers.py index cfd60eb18..831c1148a 100644 --- a/fastapi_backend/tests/api/test_mcp_servers.py +++ b/fastapi_backend/tests/api/test_mcp_servers.py @@ -159,3 +159,81 @@ def test_a_plugins_mcp_servers_come_with_an_admin_install(app, db, cfg, folder, # someone else: the servers are left out, and with nothing else in it, nothing to install no = c.post("/api/v1/extensions/import", headers=h, json={"url": "acme/kit", "preview": True}) assert no.status_code == 400 and "only admins add MCP servers" in no.json()["detail"] + + +FAKE_STDIO = r""" +import json, sys +for line in sys.stdin: + m = json.loads(line) + if "id" not in m: + continue + if m["method"] == "initialize": + r = {"protocolVersion": "2025-06-18", "capabilities": {"tools": {}}, "serverInfo": {"name": "fake"}} + elif m["method"] == "tools/list": + r = {"tools": [{"name": "echo", "description": "Say it back", "inputSchema": {"type": "object"}, "annotations": {"readOnlyHint": True}}]} + elif m["method"] == "tools/call": + if m["params"]["arguments"].get("die"): + sys.exit(3) + r = {"content": [{"type": "text", "text": "echo " + json.dumps(m["params"]["arguments"]) + " " + sys.argv[1]}]} + else: + print(json.dumps({"jsonrpc": "2.0", "id": m["id"], "error": {"code": -32601, "message": "no"}}), flush=True) + continue + print(json.dumps({"jsonrpc": "2.0", "id": m["id"], "result": r}), flush=True) +""" + + +@pytest.fixture +def runner(tmp_path, monkeypatch): + """The MCP runner (mcp-runner/runner.py) on a free port, and Lens told where it is.""" + import importlib.util + import pathlib + import sys + import threading + + path = pathlib.Path(__file__).resolve().parents[3] / "mcp-runner" / "runner.py" + spec = importlib.util.spec_from_file_location("lens_mcp_runner", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.HOME = str(tmp_path / "home") + srv = mod.serve(0, token="secret-token") + threading.Thread(target=srv.serve_forever, daemon=True).start() + (tmp_path / "token").write_text("secret-token") + monkeypatch.setenv("LENS_MCP_RUNNER_URL", f"http://127.0.0.1:{srv.server_address[1]}") + monkeypatch.setenv("LENS_MCP_TOKEN_FILE", str(tmp_path / "token")) + script = tmp_path / "fake_mcp.py" + script.write_text(FAKE_STDIO) + yield {"mod": mod, "command": sys.executable, "args": ["-I", str(script), "hi"]} + for s in mod.SERVERS.values(): + s.stop() + mod.SERVERS.clear() + srv.shutdown() + + +def test_a_server_that_runs_as_a_program_runs_in_the_runner(app, db, cfg, folder, new_client, runner, monkeypatch): + s = Assist(app, db, cfg, folder, new_client) + ca, ha = s.cl["admin"] + manifest = {"name": "echoer", "kind": "mcp", "command": runner["command"], "args": runner["args"], "env": {"X": "1"}} + eid = ca.post("/api/v1/extensions", headers=ha, json={"manifest": manifest}).json()["id"] + got = ca.get(f"/api/v1/extensions/{eid}", headers=ha).json() + assert [t["name"] for t in got["spec"]["tools"]] == ["echo"] and got["spec"]["command"] == runner["command"] + run = lambda args: ca.post(f"/api/v1/extensions/{eid}/test", headers=ha, json={"tool": "echoer__echo", "args": args}) # noqa: E731 + assert run({"a": 1}).json()["output"] == {"text": 'echo {"a": 1} hi'} + # one process, initialized once, kept between uses + (server,) = runner["mod"].SERVERS.values() + pid = server.proc.pid + assert run({"b": 2}).json()["output"]["text"].startswith('echo {"b": 2}') and server.proc.pid == pid + # it stops: the answer says so, and the next use starts it again + assert "stopped" in run({"die": True}).json()["detail"] + assert run({"c": 3}).json()["output"]["text"].startswith('echo {"c": 3}') and server.proc.pid != pid + + # the runner wants its token + import urllib.request + + req = urllib.request.Request(f"{runner['mod'].os.environ['LENS_MCP_RUNNER_URL']}/servers/x", data=b"{}", method="PUT") + with pytest.raises(urllib.error.HTTPError) as e: + urllib.request.urlopen(req, timeout=5) + assert e.value.code == 401 + # without the runner, such a server can't be added + monkeypatch.delenv("LENS_MCP_RUNNER_URL") + no = ca.post("/api/v1/extensions", headers=ha, json={"manifest": {**manifest, "name": "other"}}) + assert no.status_code == 400 and "COMPOSE_PROFILES=mcp" in no.json()["detail"] diff --git a/mcp-runner/Dockerfile b/mcp-runner/Dockerfile new file mode 100644 index 000000000..c196b8dea --- /dev/null +++ b/mcp-runner/Dockerfile @@ -0,0 +1,16 @@ +# Lens's MCP runner (runner.py): MCP servers that run as a program (stdio), in a container of their own, reached by +# Lens's backend over HTTP. The optional `mcp` service in Docker Compose; it has node/npx and uv/uvx, which is what +# MCP servers are usually started with. +FROM node:22-slim +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 ca-certificates git \ + && rm -rf /var/lib/apt/lists/* +COPY --from=ghcr.io/astral-sh/uv:0.8 /uv /uvx /usr/local/bin/ +COPY runner.py /usr/local/lib/lens-mcp-runner.py +RUN mkdir -p /data /key && chown node:node /data /key +USER node +ENV LENS_MCP_HOME=/data LENS_MCP_TOKEN_FILE=/key/token +VOLUME /data /key +EXPOSE 8090 +HEALTHCHECK --interval=30s --timeout=5s CMD ["python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/health', timeout=4)"] +ENTRYPOINT ["python3", "/usr/local/lib/lens-mcp-runner.py"] diff --git a/mcp-runner/runner.py b/mcp-runner/runner.py new file mode 100644 index 000000000..d60adcf3c --- /dev/null +++ b/mcp-runner/runner.py @@ -0,0 +1,297 @@ +"""Lens's MCP runner: MCP servers that run as a program (stdio), started in this container and reached by Lens over +Streamable HTTP (docs/assistant.md, fastapi_backend/app/domain/mcp_client.py). + +This container is the sandbox: it has none of Lens's data, keys or database, runs as an unprivileged user with every +capability dropped, and shares a network only with Lens's backend (and the internet, which `npx` and `uvx` need to +fetch the servers). Each server gets an empty folder of its own as its home, and only the environment it was given. + +API (each request carries `Authorization: Bearer `, the token this runner writes to $LENS_MCP_TOKEN_FILE, +which Lens reads from the volume the two share): + +- PUT /servers/ {command, args, env}: what to run under that key. The same again is a no-op; different settings + stop the old process. +- POST /servers//mcp: one JSON-RPC message, MCP's Streamable HTTP without streaming. The process starts on first + use (and again if it stopped) and is initialized once; a client's `initialize` gets that answer. Requests are passed + through with ids of the runner's own; notifications get 202. +- GET /health. + +Processes idle for IDLE seconds stop. Standard library only. +""" + +from __future__ import annotations + +import collections +import hmac +import itertools +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +VERSION = "2025-06-18" +HOME = os.environ.get("LENS_MCP_HOME", "/data") +TOKEN_FILE = os.environ.get("LENS_MCP_TOKEN_FILE", "/key/token") +PORT = int(os.environ.get("LENS_MCP_PORT", "8090")) +START_SECONDS = 180 # a first `npx -y` downloads the server +CALL_SECONDS = 120 +IDLE = 900 +MAX_SERVERS = 30 +MAX_BODY = 4_000_000 +KEY_RX = re.compile(r"^[a-z0-9_-]{1,64}$") +BASE_ENV = {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), "LANG": "C.UTF-8", "NODE_ENV": "production"} + + +class Server: + """One stdio server process, initialized once, with requests matched to their answers by id.""" + + def __init__(self, key, conf): + self.key, self.conf = key, conf + self.proc = None + self.init = None + self.lock = threading.Lock() + self.wlock = threading.Lock() + self.waiting: dict[int, tuple[threading.Event, list]] = {} + self.ids = itertools.count(1) + self.err = collections.deque(maxlen=40) + self.used = time.monotonic() + + def _start(self): + home = os.path.join(HOME, self.key) + os.makedirs(home, exist_ok=True) + env = {**BASE_ENV, "HOME": home, "TMPDIR": home, **{k: str(v) for k, v in (self.conf.get("env") or {}).items()}} + cmd = [self.conf["command"], *[str(a) for a in self.conf.get("args") or []]] + if not shutil.which(cmd[0], path=env["PATH"]): + raise RuntimeError(f"{cmd[0]} isn't installed in the MCP runner (it has node, npx, python3, uv and uvx)") + self.err.clear() + self.proc = subprocess.Popen( + cmd, cwd=home, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True + ) + threading.Thread(target=self._read, args=(self.proc,), daemon=True).start() + threading.Thread(target=self._errors, args=(self.proc,), daemon=True).start() + got = self._request( + "initialize", {"protocolVersion": VERSION, "capabilities": {}, "clientInfo": {"name": "lens", "version": "1"}}, START_SECONDS + ) + if "error" in got: + raise RuntimeError(f"the server didn't start: {got['error'].get('message')}") + self.init = got["result"] + self._write({"jsonrpc": "2.0", "method": "notifications/initialized"}) + + def _write(self, msg): + with self.wlock: + self.proc.stdin.write((json.dumps(msg) + "\n").encode()) + self.proc.stdin.flush() + + def _read(self, proc): + for line in proc.stdout: + try: + msg = json.loads(line) + except ValueError: + continue + if not isinstance(msg, dict): + continue + if "method" in msg and "id" in msg: # the server asks something of the client: ping, roots, sampling + answer = ( + {"result": {}} + if msg["method"] == "ping" + else {"result": {"roots": []}} + if msg["method"] == "roots/list" + else {"error": {"code": -32601, "message": "not supported"}} + ) + try: + self._write({"jsonrpc": "2.0", "id": msg["id"], **answer}) + except OSError: + pass + continue + hit = self.waiting.pop(msg.get("id"), None) if isinstance(msg.get("id"), int) else None + if hit: + hit[1].append(msg) + hit[0].set() + for ev, _ in list(self.waiting.values()): + ev.set() + + def _errors(self, proc): + for line in proc.stderr: + self.err.append(line.decode("utf-8", "replace").rstrip()) + + def _request(self, method, params, seconds=CALL_SECONDS): + rid = next(self.ids) + ev, box = threading.Event(), [] + self.waiting[rid] = (ev, box) + self._write({"jsonrpc": "2.0", "id": rid, "method": method, **({"params": params} if params is not None else {})}) + if not ev.wait(seconds): + self.waiting.pop(rid, None) + raise RuntimeError(f"the server didn't answer {method} within {seconds} seconds") + if not box: + tail = " / ".join(list(self.err)[-5:]) + raise RuntimeError("the server stopped" + (f": {tail}" if tail else "")) + return box[0] + + def ready(self): + with self.lock: + if self.proc is None or self.proc.poll() is not None: + self.stop() + try: + self._start() + except Exception: + self.stop() + raise + self.used = time.monotonic() + + def handle(self, msg): + """One client message: its answer (None for a notification).""" + self.ready() + if "id" not in msg: + if msg.get("method") != "notifications/initialized": + self._write(msg) + return None + if msg.get("method") == "initialize": + return {"jsonrpc": "2.0", "id": msg["id"], "result": self.init} + got = self._request(msg.get("method"), msg.get("params")) + self.used = time.monotonic() + return {"jsonrpc": "2.0", "id": msg["id"], **{k: got[k] for k in ("result", "error") if k in got}} + + def stop(self): + p, self.proc, self.init = self.proc, None, None + if p and p.poll() is None: + try: + os.killpg(p.pid, 15) + p.wait(5) + except (OSError, subprocess.TimeoutExpired): + try: + os.killpg(p.pid, 9) + except OSError: + pass + + +SERVERS: dict[str, Server] = {} +LOCK = threading.Lock() + + +def check_conf(conf): + if not isinstance(conf, dict) or not isinstance(conf.get("command"), str) or not conf["command"].strip(): + raise ValueError("a server is {command, args, env}") + args, env = conf.get("args") or [], conf.get("env") or {} + if not isinstance(args, list) or not all(isinstance(a, str) for a in args): + raise ValueError("args are a list of words") + if not isinstance(env, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in env.items()): + raise ValueError("env is names and text values") + return {"command": conf["command"].strip(), "args": args, "env": env} + + +def reap(): + while True: + time.sleep(60) + with LOCK: + for s in SERVERS.values(): + if s.proc and time.monotonic() - s.used > IDLE: + with s.lock: + s.stop() + + +class Handler(BaseHTTPRequestHandler): + server_version = "lens-mcp-runner" + + def log_message(self, fmt, *args): + sys.stderr.write(f"{self.address_string()} {fmt % args}\n") + + def _send(self, status, body=None, headers=None): + data = b"" if body is None else json.dumps(body).encode() + self.send_response(status) + for k, v in (headers or {}).items(): + self.send_header(k, v) + if body is not None: + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _allowed(self): + given = self.headers.get("Authorization", "") + return hmac.compare_digest(given.encode(), f"Bearer {self.server.token}".encode()) + + def _body(self): + n = int(self.headers.get("Content-Length") or 0) + if n > MAX_BODY: + raise ValueError("too large") + return json.loads(self.rfile.read(n) or b"null") + + def do_GET(self): + if self.path == "/health": + return self._send(200, {"ok": True, "servers": len(SERVERS)}) + self._send(404, {"error": "not found"}) + + def do_PUT(self): + m = re.fullmatch(r"/servers/([^/]+)", self.path) + if not self._allowed(): + return self._send(401, {"error": "unauthorized"}) + if not m or not KEY_RX.match(m[1]): + return self._send(404, {"error": "not found"}) + try: + conf = check_conf(self._body()) + except ValueError as e: + return self._send(400, {"error": str(e)}) + with LOCK: + s = SERVERS.get(m[1]) + if s and s.conf == conf: + return self._send(200, {"ok": True}) + if not s and len(SERVERS) >= MAX_SERVERS: + idle = sorted((x for x in SERVERS.values() if not x.proc), key=lambda x: x.used) + if not idle: + return self._send(409, {"error": f"the runner keeps at most {MAX_SERVERS} servers"}) + del SERVERS[idle[0].key] + if s: + with s.lock: + s.stop() + SERVERS[m[1]] = Server(m[1], conf) + self._send(200, {"ok": True}) + + def do_POST(self): + m = re.fullmatch(r"/servers/([^/]+)/mcp", self.path) + if not self._allowed(): + return self._send(401, {"error": "unauthorized"}) + s = SERVERS.get(m[1]) if m else None + if not s: + return self._send(404, {"error": "no such server: Lens sets it up first"}) + try: + msg = self._body() + except ValueError: + return self._send(400, {"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "not JSON"}}) + if not isinstance(msg, dict): + return self._send(400, {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "one message at a time"}}) + try: + out = s.handle(msg) + except (RuntimeError, OSError, ValueError, KeyError) as e: # it wouldn't start or answer: say why, as MCP does + return self._send(200, {"jsonrpc": "2.0", "id": msg.get("id"), "error": {"code": -32603, "message": str(e)[:2000]}}) + if out is None: + return self._send(202) + self._send(200, out, {"Mcp-Session-Id": s.key}) + + +def serve(port=PORT, token=None): + """The runner on `port` (0: any); returns the server (its .token, .server_address).""" + srv = ThreadingHTTPServer(("0.0.0.0", port), Handler) + srv.daemon_threads = True + srv.token = token or secrets.token_urlsafe(32) + return srv + + +def main(): + srv = serve() + os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True) + tmp = TOKEN_FILE + ".new" + with open(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644), "w") as f: + f.write(srv.token) + os.replace(tmp, TOKEN_FILE) + threading.Thread(target=reap, daemon=True).start() + print(f"MCP runner on :{PORT}", flush=True) + srv.serve_forever() + + +if __name__ == "__main__": + main() From 6a781c12f18f0b094bfa8fd7b8f16b7e732a6391 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:49:05 +0000 Subject: [PATCH 2/5] style(mcp-runner): format with the repo's default ruff settings Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CsyA2Kuhsnhzz2HjkD3zED --- mcp-runner/runner.py | 123 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 20 deletions(-) diff --git a/mcp-runner/runner.py b/mcp-runner/runner.py index d60adcf3c..54b36f06c 100644 --- a/mcp-runner/runner.py +++ b/mcp-runner/runner.py @@ -44,7 +44,11 @@ MAX_SERVERS = 30 MAX_BODY = 4_000_000 KEY_RX = re.compile(r"^[a-z0-9_-]{1,64}$") -BASE_ENV = {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), "LANG": "C.UTF-8", "NODE_ENV": "production"} +BASE_ENV = { + "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), + "LANG": "C.UTF-8", + "NODE_ENV": "production", +} class Server: @@ -64,21 +68,42 @@ def __init__(self, key, conf): def _start(self): home = os.path.join(HOME, self.key) os.makedirs(home, exist_ok=True) - env = {**BASE_ENV, "HOME": home, "TMPDIR": home, **{k: str(v) for k, v in (self.conf.get("env") or {}).items()}} + env = { + **BASE_ENV, + "HOME": home, + "TMPDIR": home, + **{k: str(v) for k, v in (self.conf.get("env") or {}).items()}, + } cmd = [self.conf["command"], *[str(a) for a in self.conf.get("args") or []]] if not shutil.which(cmd[0], path=env["PATH"]): - raise RuntimeError(f"{cmd[0]} isn't installed in the MCP runner (it has node, npx, python3, uv and uvx)") + raise RuntimeError( + f"{cmd[0]} isn't installed in the MCP runner (it has node, npx, python3, uv and uvx)" + ) self.err.clear() self.proc = subprocess.Popen( - cmd, cwd=home, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True + cmd, + cwd=home, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, ) threading.Thread(target=self._read, args=(self.proc,), daemon=True).start() threading.Thread(target=self._errors, args=(self.proc,), daemon=True).start() got = self._request( - "initialize", {"protocolVersion": VERSION, "capabilities": {}, "clientInfo": {"name": "lens", "version": "1"}}, START_SECONDS + "initialize", + { + "protocolVersion": VERSION, + "capabilities": {}, + "clientInfo": {"name": "lens", "version": "1"}, + }, + START_SECONDS, ) if "error" in got: - raise RuntimeError(f"the server didn't start: {got['error'].get('message')}") + raise RuntimeError( + f"the server didn't start: {got['error'].get('message')}" + ) self.init = got["result"] self._write({"jsonrpc": "2.0", "method": "notifications/initialized"}) @@ -95,7 +120,9 @@ def _read(self, proc): continue if not isinstance(msg, dict): continue - if "method" in msg and "id" in msg: # the server asks something of the client: ping, roots, sampling + if ( + "method" in msg and "id" in msg + ): # the server asks something of the client: ping, roots, sampling answer = ( {"result": {}} if msg["method"] == "ping" @@ -108,7 +135,11 @@ def _read(self, proc): except OSError: pass continue - hit = self.waiting.pop(msg.get("id"), None) if isinstance(msg.get("id"), int) else None + hit = ( + self.waiting.pop(msg.get("id"), None) + if isinstance(msg.get("id"), int) + else None + ) if hit: hit[1].append(msg) hit[0].set() @@ -123,10 +154,19 @@ def _request(self, method, params, seconds=CALL_SECONDS): rid = next(self.ids) ev, box = threading.Event(), [] self.waiting[rid] = (ev, box) - self._write({"jsonrpc": "2.0", "id": rid, "method": method, **({"params": params} if params is not None else {})}) + self._write( + { + "jsonrpc": "2.0", + "id": rid, + "method": method, + **({"params": params} if params is not None else {}), + } + ) if not ev.wait(seconds): self.waiting.pop(rid, None) - raise RuntimeError(f"the server didn't answer {method} within {seconds} seconds") + raise RuntimeError( + f"the server didn't answer {method} within {seconds} seconds" + ) if not box: tail = " / ".join(list(self.err)[-5:]) raise RuntimeError("the server stopped" + (f": {tail}" if tail else "")) @@ -154,7 +194,11 @@ def handle(self, msg): return {"jsonrpc": "2.0", "id": msg["id"], "result": self.init} got = self._request(msg.get("method"), msg.get("params")) self.used = time.monotonic() - return {"jsonrpc": "2.0", "id": msg["id"], **{k: got[k] for k in ("result", "error") if k in got}} + return { + "jsonrpc": "2.0", + "id": msg["id"], + **{k: got[k] for k in ("result", "error") if k in got}, + } def stop(self): p, self.proc, self.init = self.proc, None, None @@ -174,12 +218,18 @@ def stop(self): def check_conf(conf): - if not isinstance(conf, dict) or not isinstance(conf.get("command"), str) or not conf["command"].strip(): + if ( + not isinstance(conf, dict) + or not isinstance(conf.get("command"), str) + or not conf["command"].strip() + ): raise ValueError("a server is {command, args, env}") args, env = conf.get("args") or [], conf.get("env") or {} if not isinstance(args, list) or not all(isinstance(a, str) for a in args): raise ValueError("args are a list of words") - if not isinstance(env, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in env.items()): + if not isinstance(env, dict) or not all( + isinstance(k, str) and isinstance(v, str) for k, v in env.items() + ): raise ValueError("env is names and text values") return {"command": conf["command"].strip(), "args": args, "env": env} @@ -213,7 +263,9 @@ def _send(self, status, body=None, headers=None): def _allowed(self): given = self.headers.get("Authorization", "") - return hmac.compare_digest(given.encode(), f"Bearer {self.server.token}".encode()) + return hmac.compare_digest( + given.encode(), f"Bearer {self.server.token}".encode() + ) def _body(self): n = int(self.headers.get("Content-Length") or 0) @@ -241,9 +293,14 @@ def do_PUT(self): if s and s.conf == conf: return self._send(200, {"ok": True}) if not s and len(SERVERS) >= MAX_SERVERS: - idle = sorted((x for x in SERVERS.values() if not x.proc), key=lambda x: x.used) + idle = sorted( + (x for x in SERVERS.values() if not x.proc), key=lambda x: x.used + ) if not idle: - return self._send(409, {"error": f"the runner keeps at most {MAX_SERVERS} servers"}) + return self._send( + 409, + {"error": f"the runner keeps at most {MAX_SERVERS} servers"}, + ) del SERVERS[idle[0].key] if s: with s.lock: @@ -261,13 +318,39 @@ def do_POST(self): try: msg = self._body() except ValueError: - return self._send(400, {"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "not JSON"}}) + return self._send( + 400, + { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "not JSON"}, + }, + ) if not isinstance(msg, dict): - return self._send(400, {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "one message at a time"}}) + return self._send( + 400, + { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32600, "message": "one message at a time"}, + }, + ) try: out = s.handle(msg) - except (RuntimeError, OSError, ValueError, KeyError) as e: # it wouldn't start or answer: say why, as MCP does - return self._send(200, {"jsonrpc": "2.0", "id": msg.get("id"), "error": {"code": -32603, "message": str(e)[:2000]}}) + except ( + RuntimeError, + OSError, + ValueError, + KeyError, + ) as e: # it wouldn't start or answer: say why, as MCP does + return self._send( + 200, + { + "jsonrpc": "2.0", + "id": msg.get("id"), + "error": {"code": -32603, "message": str(e)[:2000]}, + }, + ) if out is None: return self._send(202) self._send(200, out, {"Mcp-Session-Id": s.key}) From 6471fbf00eb528fcbf6f97eeb6dae217b71941d4 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Wed, 14 Oct 2026 09:57:19 +0000 Subject: [PATCH 3/5] docs(threat-model): the MCP runner and the stdio servers it runs Records the runner as new surface (Assistant family and the dependency table) and fixes the marketplace docstring that still said plugins bring network MCP servers only. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0188zd7Y9m6tdENwXQyhCSH2 --- fastapi_backend/app/domain/marketplace.py | 2 +- threat-model.md | 3 ++- threat-model.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fastapi_backend/app/domain/marketplace.py b/fastapi_backend/app/domain/marketplace.py index 0136d52f9..f5b854500 100644 --- a/fastapi_backend/app/domain/marketplace.py +++ b/fastapi_backend/app/domain/marketplace.py @@ -404,7 +404,7 @@ def _norm(p): def _mcp(found, meta, admin, report): """A plugin's MCP servers (.mcp.json, or mcpServers in its manifest) as MCP server items, their tools listed now. - Servers on the network only (HTTP); only admins add them.""" + On the network (HTTP), or run as a program in the MCP runner when it runs; only admins add them.""" conf = meta.get("mcpServers") if isinstance(conf, str): conf = found.get(_norm(conf)) diff --git a/threat-model.md b/threat-model.md index 9496b468e..a3ceed72e 100644 --- a/threat-model.md +++ b/threat-model.md @@ -104,7 +104,7 @@ optional listeners (MQTT, syslog, tunnel, local model server) run inside workers | Outbound fetchers & stored secrets (Outbound) | `domain/netguard.py`, `webcapture.py`, `feeds.py`, `notify.py`, `sources.py`, `llm.py`, `semantic.py`, `speech.py`, `fedora.py`, `bridge.py`, `telegram.py`, `slack.py`, `mailbox.py`, `telemetry.py`, `settings.py` seal, `keyring.py`, `vaults.py` | network, child processes, files | In | | Ingest & processing (Ingest) | `routes/{uploads,files,imports}.py`, `domain/{ingest,documents,convert,video,uploads}.py`, `templates.py`, `render.py` | child processes (ffmpeg, LibreOffice, Chromium, poppler, Tesseract), files | In | | Query engines (Query) | `domain/cypher.py`, `domain/rdf.py` (SPARQL), `domain/rdf_import.py`, `domain/search.py` | DB | In | -| Assistant & extensions (Assistant) | `domain/{chat,ai_tools,ops_tools,extensions,tool_nodes,flow,custom_nodes,code_tools}.py`, MCP write tools | network (model servers), child process (code tools) | In | +| Assistant & extensions (Assistant) | `domain/{chat,ai_tools,ops_tools,extensions,tool_nodes,flow,custom_nodes,code_tools,mcp_client,marketplace}.py`, MCP write tools | network (model servers, admins' MCP servers, the MCP runner), child process (code tools) | In | | Optional services, off by default (Optional) | `domain/{sensors,mqtt,syslog,shares,s3,webdav}.py`, `tunnel.py`, `https.py`, `local_llm.py`, `components.py`, `telemetry.py` | listening sockets, downloads, child processes | In | | Packaging variants | `docker-compose.prod.yml`, `install.sh`, `cloudron/`, `packaging/{synology,qnap}`, `proxmox/` | host | §1.6 | | Dev and CI tooling, samples | `docker-compose.yml`, `make dev`, `.github/`, `ci/`, `examples/`, `overrides/`, `prod-frontend-deploy.yml`, `watcher.py`, `commands/`, `fastapi_backend/tests/` | host | Out (§1.3) | @@ -441,6 +441,7 @@ third-party source; dependencies come from `uv.lock`, the pnpm lockfile and cont | Catalog GGUF models | pinned revision and SHA-256 | in model (P-MODEL-CHECKSUM) | *(documented, docs/local-models.md "checked against their SHA-256")* | | Optional ML extras (whisper, pyannote, speechbrain, insightface, doctr) | model inference; some fetch weights on their own | upstream | *(inferred, Q19)* | | StyleTTS2 service (`styletts2/`, opt-in Compose profile) and macOS `say` | text to speech; StyleTTS2 fetches its weights from Hugging Face and GitHub on first use and listens only on the Compose network; `say` gets the text on stdin, never as arguments | upstream for the model and synthesis; the argument handling in model | *(maintainer, 2026-10)* | +| MCP runner (`mcp-runner/`, opt-in Compose profile) and the stdio MCP servers admins add (usually `npx` or `uvx` packages) | running those servers away from Lens: a container with none of Lens's data, keys or database, a non-root user, every capability dropped, `no-new-privileges`, a read-only image, memory and process limits, a network shared only with the backend (plus the internet to fetch packages), an empty home and only its extension's environment per server, and a bearer token on a volume only the two share | upstream for the servers and the packages they fetch (an admin's choice, trusted input, §1.3); the runner's isolation and token check in model | *(documented, docs/assistant.md "The MCP runner")* | | Build, lint and codegen toolchain (frontend dev dependencies, openapi-ts, linters) | building the images; in no running service | upstream; a report must show the code ships | *(inferred, Q45)* | | OIDC providers, SMTP, model and speech providers | honest responses over TLS | the admin chose them; trusted input | *(maintainer, 2026-10)* | diff --git a/threat-model.yaml b/threat-model.yaml index dc0979c46..ef8ef461c 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:275791b79d131205c80bd3e1ea47b5c20ebbe620940f45a9cb3e2c4a1aba90db" +prose_version: "threat-model.md@sha256:22eaa876a13f1f57c0e690a9dedb73ad2aa2bb8700550c4144116d764604bc41" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} From e8b51c19d9c5d167a46a0424d50183968309e291 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Wed, 14 Oct 2026 09:57:35 +0000 Subject: [PATCH 4/5] docs(devlog): stdio MCP servers in the runner, and what to refine later Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0188zd7Y9m6tdENwXQyhCSH2 --- DEVLOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/DEVLOG.md b/DEVLOG.md index b6e4739e9..85c8afb75 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,17 @@ Plans and progress for work in flight. Newest first. +## 2026-10-14 · MCP servers that run as a program + +Admins add a stdio MCP server (`command`, `args`, `env`, as in Claude's `.mcp.json`) as an `mcp` extension, and a +plugin's stdio servers come along when an admin installs it. They run in the MCP runner (`mcp-runner/`, the opt-in +`mcp` Compose profile), a container with none of Lens's data or keys, reached over HTTP with a token on a volume only +it and the backend share. Recorded in the threat model. + +Refine later: turn the runner on from Settings through the machine agent, as monitoring does, instead of +`COMPOSE_PROFILES=mcp` in `.env`; the runner on installs that don't use Docker (microsandbox, Cloudron, NAS packages); +plugin servers that run a program from the plugin's own folder; an egress allow-list for the runner's network. + ## 2026-10-10 · Pasted transcripts fetch no speech model Found by the clicks survey: on a fresh install, importing a pasted transcript queued the standard pipeline from From 3276c4efabddc6e7264b3de3b704ec30045751ec Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Wed, 14 Oct 2026 10:01:16 +0000 Subject: [PATCH 5/5] docs(changelog): MCP servers that run as a program Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0188zd7Y9m6tdENwXQyhCSH2 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90116938e..481ad1e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **MCP servers that run as a program.** Most MCP servers in marketplaces run as a program (`npx` or `uvx`), not on + the network. Admins can now add one as an MCP server extension with its `command`, `args` and `env`, as in Claude's + `.mcp.json`, and a plugin's such servers come along when an admin installs it. They run in the MCP runner, an opt-in + container with none of Lens's data or keys (`COMPOSE_PROFILES=mcp`), start on first use and stop after 15 idle + minutes ([Assistant](docs/assistant.md#the-mcp-runner)). + ## 0.8.1 October 12, 2026 {id="0.8.1"} - **The installer stops waiting when Lens can't start, and shows why.** A microsandbox install that couldn't start Lens