From 3ff45d1fc10f06de6d82004f89ca7101885429fb Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 00:42:08 +0000 Subject: [PATCH 1/4] feat(assistant): turn the MCP runner and a browser on from the web app, with ready-made MCP servers The host upgrade agent's services action now takes a profile (mcp, browser or monitoring, a fixed list) and turn=on|off, so the web app, the API (GET /admin/services, PUT /admin/services/{profile}) and the MCP tools services and set_service can start or stop any of Lens's optional containers; the old monitoring=on|off request still works. The new browser profile runs Playwright MCP in a hardened container on an internal network whose only way out is browser-proxy, netguard run on its own (public addresses, ports 80 and 443, a byte budget per connection). Add MCP server gains Ready-made: Browser (one click once its container answers, browser_run_code_unsafe off) and GitHub (a field for the token). ${NAME} values get their own fields, and a spec with keep: true reuses one MCP session between calls. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HXPWXcSCQb9hWPr7xkZu7B --- .github/workflows/browser.yml | 113 ++++++ CHANGELOG.md | 8 + browser/playwright-mcp.json | 7 + docker-compose.prod.yml | 60 +++- docker-compose.yml | 50 ++- docs/assistant.md | 51 ++- docs/contributing.md | 1 + docs/mcp.md | 10 +- docs/prometheus.md | 4 +- docs/updates.md | 11 +- fastapi_backend/app/api/mcp_tools.py | 44 +++ .../app/api/v1/routes/extensions.py | 13 +- fastapi_backend/app/api/v1/routes/updates.py | 31 +- fastapi_backend/app/domain/extensions.py | 14 +- fastapi_backend/app/domain/mcp_client.py | 48 ++- fastapi_backend/app/domain/mcp_servers.py | 40 +++ fastapi_backend/app/domain/monitoring.py | 77 +---- fastapi_backend/app/domain/netguard.py | 46 ++- fastapi_backend/app/domain/services.py | 172 ++++++++++ fastapi_backend/app/schemas/extensions.py | 15 + fastapi_backend/app/schemas/updates.py | 22 ++ fastapi_backend/tests/api/test_mcp.py | 6 + fastapi_backend/tests/api/test_mcp_servers.py | 65 ++++ fastapi_backend/tests/api/test_monitoring.py | 6 +- fastapi_backend/tests/api/test_services.py | 122 +++++++ .../tests/domain/test_netguard_forwarder.py | 105 ++++++ .../__tests__/mcp-server-ui.test.tsx | 103 +++++- nextjs-frontend/app/openapi-client/index.ts | 16 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 57 ++++ .../app/openapi-client/types.gen.ts | 204 +++++++++++ .../components/admin/service-switch.tsx | 76 +++++ .../components/extensions/mcp-server-form.tsx | 136 +++++++- nextjs-frontend/openapi.json | 321 ++++++++++++++++++ packaging/host/lens-upgrade.sh | 52 ++- packaging/host/project.json | 26 ++ packaging/host/test-services.sh | 88 +++++ threat-model.json | 25 +- threat-model.md | 17 +- threat-model.yaml | 23 +- 39 files changed, 2148 insertions(+), 137 deletions(-) create mode 100644 .github/workflows/browser.yml create mode 100644 browser/playwright-mcp.json create mode 100644 fastapi_backend/app/domain/services.py create mode 100644 fastapi_backend/tests/api/test_services.py create mode 100644 fastapi_backend/tests/domain/test_netguard_forwarder.py create mode 100644 nextjs-frontend/components/admin/service-switch.tsx create mode 100644 packaging/host/project.json create mode 100644 packaging/host/test-services.sh diff --git a/.github/workflows/browser.yml b/.github/workflows/browser.yml new file mode 100644 index 000000000..4c4cd2273 --- /dev/null +++ b/.github/workflows/browser.yml @@ -0,0 +1,113 @@ +name: Browser + +# The assistant's browser (the optional browser profile: Playwright MCP and netguard's proxy, docs/assistant.md): it +# starts as docker-compose.prod.yml runs it, lists its tools over MCP, opens a public page that stays open from one +# session to the next, can't reach a private address through its proxy, and has no way out of its network but that. +on: + pull_request: + paths: + - browser/** + - fastapi_backend/app/domain/netguard.py + - docker-compose.prod.yml + - .github/workflows/browser.yml + push: + branches: [main] + paths: + - browser/** + - fastapi_backend/app/domain/netguard.py + - docker-compose.prod.yml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + browser: + name: Start it and browse + runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }} + timeout-minutes: 20 + env: + # the compose file needs these set to be read; the browser uses none of them + ACCESS_SECRET_KEY: ci + ARCHIVE_SECRET_KEY: ci + SURREAL_PASS: ci + AUTH_SECRET: ci + steps: + - uses: actions/checkout@v4 + + - name: Start it as Lens's compose file does + run: | + docker compose -p lensci -f docker-compose.prod.yml --profile browser up -d browser browser-proxy + sleep 5 + docker compose -p lensci -f docker-compose.prod.yml ps -a + + - name: Lists its tools, and browses a public page + run: | + # mcp BODY: one request to the browser, from a container on its network as the backend is; with $SID, in + # that session + mcp() { + docker run --rm --network lensci_browser curlimages/curl:8.10.1 -sS -i --max-time 120 -X POST \ + http://browser:8931/mcp -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \ + ${SID:+-H "Mcp-Session-Id: $SID"} -d "$1" + } + open_session() { + SID="" + out="$(mcp '{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "ci", "version": "1"}}}')" + SID="$(printf '%s' "$out" | tr -d '\r' | sed -n 's/^[Mm]cp-[Ss]ession-[Ii]d: //p')" + test -n "$SID" || { echo "$out"; return 1; } + mcp '{"jsonrpc": "2.0", "method": "notifications/initialized"}' >/dev/null + } + call() { # tool arguments + mcp "{\"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/call\", \"params\": {\"name\": \"$1\", \"arguments\": $2}}" + } + for _ in $(seq 1 30); do + open_session 2>/dev/null && break + sleep 2 + done + test -n "$SID" + mcp '{"jsonrpc": "2.0", "id": 3, "method": "tools/list"}' | tee out.txt + grep -q browser_navigate out.txt && grep -q browser_snapshot out.txt + call browser_navigate '{"url": "https://example.com"}' | tee out.txt + grep -q "Example Domain" out.txt + # another session sees the same page: the one browser, shared (Lens's backend and worker each keep one) + open_session + call browser_snapshot '{}' | tee out.txt + grep -q "Example Domain" out.txt + + - name: Can't reach a private address, nor out but through its proxy + run: | + mcp() { + docker run --rm --network lensci_browser curlimages/curl:8.10.1 -sS -i --max-time 120 -X POST \ + http://browser:8931/mcp -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \ + ${SID:+-H "Mcp-Session-Id: $SID"} -d "$1" + } + SID="" + out="$(mcp '{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "ci", "version": "1"}}}')" + SID="$(printf '%s' "$out" | tr -d '\r' | sed -n 's/^[Mm]cp-[Ss]ession-[Ii]d: //p')" + mcp '{"jsonrpc": "2.0", "method": "notifications/initialized"}' >/dev/null + mcp '{"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "browser_navigate", "arguments": {"url": "https://169.254.169.254/latest/meta-data/"}}}' | tee out.txt + grep -q ERR_TUNNEL_CONNECTION_FAILED out.txt + mcp '{"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "browser_navigate", "arguments": {"url": "http://browser-proxy/"}}}' >/dev/null + docker compose -p lensci -f docker-compose.prod.yml logs browser-proxy | tee out.txt + grep -q "refused CONNECT 169.254.169.254:443" out.txt + grep -q "refused GET http://browser-proxy/: browser-proxy isn't a public address" out.txt + # straight out of the browser's container, past its proxy: no way + docker compose -p lensci -f docker-compose.prod.yml exec -T browser node -e " + require('https').get('https://example.com', () => { console.log('reached the internet'); process.exit(1) }) + .on('error', (e) => { console.log('no way out:', e.code); process.exit(0) })" + # and only the backend's name for it is served + code="$(docker run --rm --network lensci_browser curlimages/curl:8.10.1 -s -o /dev/null -w '%{http_code}' \ + -X POST -H 'Host: evil.example:8931' http://browser:8931/mcp -d '{}')" + test "$code" = 403 + + - name: What they said + if: failure() + run: docker compose -p lensci -f docker-compose.prod.yml logs + + - name: Stop + if: always() + run: docker compose -p lensci -f docker-compose.prod.yml --profile browser down -v || true diff --git a/CHANGELOG.md b/CHANGELOG.md index 087b78a1a..a24bf9077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,14 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version sealed and never shown again (manifests say `••••`), and an upgrade step seals the ones saved before. Each server's tools have a switch and "Asks first", kept in its manifest ([Adding an MCP server](docs/assistant.md#adding-an-mcp-server)). +- **A browser for the assistant, and ready-made MCP servers.** Add MCP server now starts with Ready-made: + **Browser** runs a headless Chromium ([Playwright MCP](https://github.com/microsoft/playwright-mcp)) in a container + of its own, whose only way out is a proxy to public addresses on ports 80 and 443, and adds it with one click; + **GitHub** fills in GitHub's own server with a field for its token. Values written as `${NAME}` in a pasted config + now get a field of their own too. The MCP runner, the browser and monitoring turn on and off from the web app (the + machine's upgrade agent starts and stops them), and agents do the same with `PUT /api/v1/admin/services/` + or the MCP tools `services` and `set_service`. A server whose spec says `keep: true` keeps one session between + calls ([The browser](docs/assistant.md#the-browser), [Optional containers](docs/assistant.md#optional-containers)). - **Describe an automation in words.** `POST /api/v1/automations/draft` asks the model once to turn a sentence into a draft automation to try and save, and says what didn't fit and what it couldn't express ([Describe it in words](docs/automations.md#describe-it-in-words)). diff --git a/browser/playwright-mcp.json b/browser/playwright-mcp.json new file mode 100644 index 000000000..e64772275 --- /dev/null +++ b/browser/playwright-mcp.json @@ -0,0 +1,7 @@ +{ + "browser": { + "launchOptions": { + "args": ["--force-webrtc-ip-handling-policy=disable_non_proxied_udp", "--disable-dev-shm-usage"] + } + } +} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e0f3a35a0..cbe4d1feb 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -77,6 +77,8 @@ x-backend: &backend # the MCP runner (the optional `mcp` service, docs/assistant.md): MCP servers that run as a program, sandboxed there LENS_MCP_RUNNER_URL: http://mcp:8090 LENS_MCP_TOKEN_FILE: /mcp-key/token + # the assistant's browser (the optional `browser` service, docs/assistant.md): Playwright MCP + LENS_BROWSER_URL: http://browser:8931 # upgrading from the web app (docs/updates.md): the folder install.sh's upgrade agent watches, and the install folder LENS_CONTROL_DIR: /control LENS_HOST_DIR: ${LENS_DIR:-} @@ -94,7 +96,7 @@ x-backend: &backend surrealdb: condition: service_healthy restart: unless-stopped - networks: [lens, mcp] + networks: [lens, mcp, browser] services: surrealdb: @@ -207,6 +209,55 @@ services: restart: unless-stopped networks: [mcp] + # The assistant's browser (docs/assistant.md#the-browser): headless Chromium the assistant drives over MCP + # (Playwright MCP), added in Extensions › Add MCP server › Browser. Opt-in, about 400 MB to download: that page turns + # it on (the upgrade agent adds `browser` to COMPOSE_PROFILES and starts it and its proxy), or + # `docker compose --profile browser up -d`. The sandbox: none of Lens's data or keys, no capabilities, a read-only + # image, its profile in memory only, and a network with no way out but browser-proxy, which reaches public web + # addresses only, on ports 80 and 443 (fastapi_backend/app/domain/netguard.py). One browser for everyone who uses it. + browser: + image: mcr.microsoft.com/playwright/mcp:${PLAYWRIGHT_MCP_VERSION:-v0.0.83} + profiles: [browser] + command: + - --port=8931 + - --host=0.0.0.0 + - --allowed-hosts=browser:8931 + - --shared-browser-context + - --proxy-server=http://browser-proxy:3128 + - --image-responses=omit + - --timeout-navigation=45000 + - --output-dir=/tmp/output + - --config=/etc/lens/playwright-mcp.json + stdin_open: true # it stops when its input closes + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + tmpfs: [/tmp, /home/node] + mem_limit: 1g + pids_limit: 512 + volumes: + - ./browser/playwright-mcp.json:/etc/lens/playwright-mcp.json:ro + depends_on: [browser-proxy] + restart: unless-stopped + networks: [browser] + + # The browser's only way out: netguard's proxy on its own (the standard library only), to public addresses on the + # web's ports. Nothing else: no data, no keys, nobody's user. + browser-proxy: + image: python:3.12-slim-trixie + profiles: [browser] + command: ["python", "-I", "/netguard/netguard.py", "--listen", "0.0.0.0:3128"] + user: "65534:65534" + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + mem_limit: 256m + pids_limit: 256 + volumes: + - ./fastapi_backend/app/domain/netguard.py:/netguard/netguard.py:ro + restart: unless-stopped + networks: [browser, browser-out] + # StyleTTS2 (docs/speech-providers.md#styletts2): spoken answers and podcasts read on this machine, in a voice like a # short clip you put in its voices volume. Heavy (PyTorch and a 1 GB model, fetched on first use), so opt-in: # `docker compose --profile styletts2 up -d`, then Settings → AI assistant → Voice → StyleTTS2. @@ -329,3 +380,10 @@ networks: driver: bridge mcp: driver: bridge + # the browser, the backend and the browser's proxy: no way out of it (internal) + browser: + driver: bridge + internal: true + # the browser's proxy to the internet + browser-out: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index 59152fe63..d387866f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,8 @@ x-backend: &backend # the MCP runner (the optional `mcp` service, docs/assistant.md): MCP servers that run as a program, sandboxed there LENS_MCP_RUNNER_URL: http://mcp:8090 LENS_MCP_TOKEN_FILE: /mcp-key/token + # the assistant's browser (the optional `browser` service, docs/assistant.md): Playwright MCP + LENS_BROWSER_URL: http://browser:8931 volumes: - ./fastapi_backend:/app # the image's own packages, not the source tree's: anonymous, so `make dev` (up --build --renew-anon-volumes) @@ -62,7 +64,7 @@ x-backend: &backend depends_on: surrealdb: condition: service_healthy - networks: [lens, mcp] + networks: [lens, mcp, browser] services: surrealdb: @@ -178,6 +180,47 @@ services: - mcp-key:/key networks: [mcp] + # The assistant's browser and its proxy, as in docker-compose.prod.yml (docs/assistant.md#the-browser). Opt-in: + # `docker compose --profile browser up -d`. + browser: + image: mcr.microsoft.com/playwright/mcp:${PLAYWRIGHT_MCP_VERSION:-v0.0.83} + profiles: [browser] + command: + - --port=8931 + - --host=0.0.0.0 + - --allowed-hosts=browser:8931 + - --shared-browser-context + - --proxy-server=http://browser-proxy:3128 + - --image-responses=omit + - --timeout-navigation=45000 + - --output-dir=/tmp/output + - --config=/etc/lens/playwright-mcp.json + stdin_open: true # it stops when its input closes + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + tmpfs: [/tmp, /home/node] + mem_limit: 1g + pids_limit: 512 + volumes: + - ./browser/playwright-mcp.json:/etc/lens/playwright-mcp.json:ro + depends_on: [browser-proxy] + networks: [browser] + + browser-proxy: + image: python:3.12-slim-trixie + profiles: [browser] + command: ["python", "-I", "/netguard/netguard.py", "--listen", "0.0.0.0:3128"] + user: "65534:65534" + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + mem_limit: 256m + pids_limit: 256 + volumes: + - ./fastapi_backend/app/domain/netguard.py:/netguard/netguard.py:ro + networks: [browser, browser-out] + # StyleTTS2 (docs/speech-providers.md#styletts2): spoken answers and podcasts read on this machine, in a voice like a # short clip you put in its voices volume. Heavy (PyTorch and a 1 GB model, fetched on first use), so opt-in: # `docker compose --profile styletts2 up -d`, then Settings → AI assistant → Voice → StyleTTS2. @@ -221,3 +264,8 @@ networks: driver: bridge mcp: driver: bridge + browser: + driver: bridge + internal: true # no way out but browser-proxy + browser-out: + driver: bridge diff --git a/docs/assistant.md b/docs/assistant.md index 083ef2018..751a5fa9d 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -308,11 +308,18 @@ or the command it runs as with its arguments and environment. **Paste a config** servers in, one server or many: Claude's `.mcp.json` (or `mcpServers` in its settings), VS Code's `mcp.json` (`servers`), or a server on its own as its README gives it. A server without a name is named after its package or its address (`@modelcontextprotocol/server-github` is `github`), and values written as `${NAME}`, which those apps fill in -from their own settings, are pointed out to fill in. Servers on the older SSE transport are left out with a note. +from their own settings, get a field of their own to type them in, which goes in their place when the server is tested +or saved. Servers on the older SSE transport are left out with a note. **Test** asks the server for its tools before anything is saved, and Enter adds it. Over the API: `POST /api/v1/extensions/mcp/read` reads a config (nothing is reached), `POST /api/v1/extensions/mcp/test` lists a server's tools without saving it, and `POST /api/v1/extensions` saves it as an extension of kind `mcp`. +**Ready-made** lists servers Lens knows. **Browser** adds [the browser](#the-browser) with one click once its container +runs (until then its switch is there to turn it on). **GitHub** puts GitHub's own server +(`https://api.githubcopilot.com/mcp/`) in the form with a field for a personal access token. The list is +`GET /api/v1/extensions/mcp/presets` (admins), each with the spec to save and, for the browser, whether its container +is on and answering. + Header and environment values whose names say they carry a credential (Authorization, API keys, tokens, secrets, passwords, cookies, sessions) are sealed when the server is saved, with the server's key (as notification targets and sources are, since an MCP server belongs to the whole install rather than a namespace), bound to the extension, the @@ -330,11 +337,18 @@ each change is a new version (`PATCH /api/v1/extensions/{id}/tools` with `{tool, an MCP tool still asks first after the conversation read outside text, as every tool that sends to another host does ([Third-party content and prompt injection](#third-party-content-and-prompt-injection)). +Lens opens a new MCP session for each call, except for a server whose spec says `keep: true`, one that remembers +things between calls (the browser's open page): its calls share one session, kept for ten minutes after the last one +and opened again when the server no longer knows it. That session is the server's, shared by everyone whose assistant +uses it, as its address and token are. + ### The MCP runner MCP servers that run as a program run in a container of their own, the MCP runner ([`mcp-runner/`](https://github.com/adeelahmad/lense/tree/main/mcp-runner)), never next to Lens. It's off until you -turn it on: add `COMPOSE_PROFILES=mcp` to Lens's `.env` and run the install command again (or +turn it on: **Run the MCP runner on this machine** under **Command** in **Add MCP server** has the machine's upgrade +agent add `mcp` to `COMPOSE_PROFILES` in `.env` and start it (see [Optional containers](#optional-containers)), +or by hand, add `COMPOSE_PROFILES=mcp` to Lens's `.env` and run the install command again (or `docker compose --profile mcp up -d`). The container is the sandbox: it has none of Lens's data, keys or database, runs as an unprivileged user with every capability dropped, a read-only image and memory and process limits, and shares a network only with Lens's backend (and the internet, which `npx` and `uvx` need to fetch the servers). Each server gets @@ -342,6 +356,39 @@ an empty folder as its home and only the environment its extension gives it; Len a token the runner writes to a volume only the two share. A server starts on first use, stays up while it's used, and stops after 15 idle minutes. Installs that don't run on Docker don't have the runner yet. +### The browser + +The browser is a headless Chromium the assistant drives over MCP: it opens pages, reads them, clicks, types and fills +in forms. It is [Playwright MCP](https://github.com/microsoft/playwright-mcp) (Apache-2.0), in a container of its own +(the optional `browser` profile, `mcr.microsoft.com/playwright/mcp`, pinned in the compose file: about 400 MB to +download, and at most 1 GB of memory). It's off until you turn it on: **Run the browser on this +machine** under **Ready-made** in **Add MCP server**, then **Add Browser**, which adds it as the MCP server `browser` +(its tools are `browser__browser_navigate`, `browser__browser_snapshot`, `browser__browser_click` and so on). + +- **Its sandbox.** None of Lens's data, keys or database; every capability dropped, a read-only image, memory and + process limits, and its profile (cookies, history) in memory only, gone when it stops (after an hour unused, or with + the switch). Its network has no way out but its proxy, `browser-proxy`: the same guard that captures web pages + (`netguard.py`), on its own, which goes to public addresses on ports 80 and 443 only and refuses this machine, the + local network and a cloud's metadata address. Lens's backend reaches it at `http://browser:8931/mcp`, and it + answers only to that name. +- **What asks first.** Going to a page, going back and waiting for one run at once until the conversation has read + outside text; clicking, typing, filling in forms and anything else that acts on a page ask first, and after the + conversation read a page every browser tool asks, as every tool that sends to another host does. Running code in + the browser's own server process (`browser_run_code_unsafe`) starts switched off; turn it on in the server's page + only if you mean to. +- **One browser for everyone.** Every conversation that uses it drives the same browser, with the same open pages and + cookies, so a site someone signs in to through it is signed in for the others too until it stops. +- Installs that don't run on Docker don't have the browser yet; by hand, it's `docker compose --profile browser up -d`. + +### Optional containers + +The MCP runner, the browser and monitoring ([Prometheus and Grafana](prometheus.md)) are containers Lens can run next +to it, off until an admin turns one on. The web app can't start containers itself, so the machine's upgrade agent does +([Updates](updates.md)): it adds the profile to `COMPOSE_PROFILES` in Lens's `.env`, or takes it out, and starts or +stops that profile's containers, one change at a time. Without an agent (Lens wasn't installed with `install.sh`), the +switch shows the line to run on the machine instead. Agents use the same switches: `GET /api/v1/admin/services` and +`PUT /api/v1/admin/services/{profile}` with `{"on": true}`, or the MCP tools `services` and `set_service` (admins). + `ai.extensions: false` turns every extension off. ## Setting up and running the server by chat diff --git a/docs/contributing.md b/docs/contributing.md index 5daacb53b..224c61bb8 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -22,6 +22,7 @@ Lens is one repository with several projects, and [Nx](https://nx.dev/) runs the | `docs` | `docs/` (with `mkdocs.yml`, `overrides/`, `README.md`, `CHANGELOG.md`) | `build` (strict, as CI), `serve` | | `clipper` | `extensions/browser/` | `install` (`npm ci` when the lockfile is newer than `node_modules`), `test`, `build` (every browser target, checked) | | `desktop` | `desktop/` | `lint` (`sh -n` and shellcheck on `lens-app.sh`), `test` (`lens-app.sh` checks on Linux; the Mac build stays in the Desktop workflow) | +| `host-agent` | `packaging/host/` | `lint` (`sh -n` and shellcheck on the upgrade agent), `test` (`test-services.sh`, its optional-containers action against a stand-in `docker`) | | `microsandbox` | `microsandbox/` | `lint` (`sh -n` and shellcheck on its scripts), `test` (`test-dns.sh`, the name-lookup fallback, and `test-programs.sh`, LibreOffice and Chromium on first use, in the base image with Docker; the install itself runs in the Install workflow) | ```bash diff --git a/docs/mcp.md b/docs/mcp.md index a92688f1b..898033a0d 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -45,8 +45,8 @@ Each namespace can have its own server, at `/mcp/` (`https://lens.exa that should see that namespace and nothing else: a family assistant, a project's coding agent, a client's app. It has the same tools, but sees only that namespace, whatever else the person who signs the agent in can read. Tools that take a namespace use this one when none is named, `list_namespaces` and `list_files` show only it, and the tools -about the server itself (`file_storage`, `monitoring`, `install_settings` and the ones that change them) aren't there, -even for admins. An admin counts as an owner of the namespace there. +about the server itself (`file_storage`, `monitoring`, `services`, `install_settings` and the ones that change them) +aren't there, even for admins. An admin counts as an owner of the namespace there. It is off until an owner of the namespace turns it on: **Admin → Namespaces →** the namespace **→ Connect apps**, which also copies its address, or `PATCH /api/v1/namespaces//integrations` with `{"mcp": true}`, or an agent's @@ -115,6 +115,8 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `preview_template` | what a template in a drafted automation gives before it's saved, against an activity or the trigger kind's sample; a prompt shows outside text marked as the model sees it; an error names the missing variable and its nearest name | | `file_storage` | where Lens keeps its own files (notes' images and attachments) and the connections it could use; `check` tests it. Admins only | | `set_file_storage` | keep new files on this machine or a storage connection (folder, rclone crypt); checked before it's saved. Admins only, with a write-scope token | +| `services` | Lens's optional containers ([Optional containers](assistant.md#optional-containers)): the MCP runner, the browser and monitoring, whether each is on and answering, and how the machine's last change went. Admins only | +| `set_service` | start or stop one of them (`mcp`, `browser` or `monitoring`): the machine's upgrade agent does it. Admins only, with a write-scope token | | `install_settings` | whether other devices on the network reach Lens and its edition (full or lean), whether the machine's upgrade agent can change them, and the install line to run when it can't. Admins only | | `set_install_settings` | open Lens to the network or keep it to this machine, or change the edition: the machine sets itself up again and Lens restarts ([Updates](updates.md#setting-the-machine-up-again)). Admins only, with a write-scope token | | `propose_graph_change` | ask for two entities to be merged or linked; it waits in Proposed changes unless `apply` makes it at once (it can be undone). Needs a write-scope token and editor access | @@ -136,8 +138,8 @@ which records an undoable change (proposed by default), `undo_graph_change`, `ed `merge_topics`, `delete_topic` and `undo_topic_change`, which curate entities and topics as an editor does in the web app (merges and deletes can be undone), `write_note`, which adds a note, `create_podcast`, which queues an episode, `set_nsfw` and `mark_nsfw`, which change [NSFW classification](nsfw.md) (owners or admins, and -editors), `set_namespace_integrations`, which turns a namespace's own MCP server or chat completions on or off (owners), `set_file_storage`, which changes where new files are kept (admins), and `set_monitoring` and -`set_install_settings`, which have the machine change how Lens runs (admins). +editors), `set_namespace_integrations`, which turns a namespace's own MCP server or chat completions on or off (owners), `set_file_storage`, which changes where new files are kept (admins), and `set_monitoring`, `set_service` +and `set_install_settings`, which have the machine change how Lens runs (admins). Importing stays in the web app and the API. ### What the archive says is data diff --git a/docs/prometheus.md b/docs/prometheus.md index 7d7445a86..13c752d62 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -20,7 +20,9 @@ page. upgrade agent (the one that [installs new releases](updates.md)) adds `monitoring` to `COMPOSE_PROFILES` in `.env` and starts the three containers; the page shows how it goes and then links to Grafana. Off stops and removes them (their history stays in the `prometheus-data` volume) and turns `/metrics` off again if this switch turned it on. Agents do -the same with `PUT /api/v1/admin/monitoring` (`{"on": true}`) or the MCP tools `monitoring` and `set_monitoring`. +the same with `PUT /api/v1/admin/monitoring` (`{"on": true}`) or the MCP tools `monitoring` and `set_monitoring`, +or with the switch every optional container shares, `PUT /api/v1/admin/services/monitoring` and the MCP tool +`set_service` ([Optional containers](assistant.md#optional-containers)). Grafana listens where the web app does (`LENS_BIND`: this machine only unless Lens was opened to the network) on port 3001 (`LENS_GRAFANA_PORT`), and System health links to it on the address you opened Lens at, unless diff --git a/docs/updates.md b/docs/updates.md index c6502e91c..0710dc95a 100644 --- a/docs/updates.md +++ b/docs/updates.md @@ -61,10 +61,13 @@ from those seconds can be lost. If going back fails too, the run ends `failed` a The agent installs only a release tag (`vX.Y.Z`) or the branch the install already follows, whoever writes the request, and the API refuses a release older than the one running (restore a backup instead). -The same agent turns [Charts and history](prometheus.md#prometheus-and-grafana-ready-made) on and off: a request with -`action=services` and `monitoring=on` or `off` (in the same inbox file, the one every agent watches) adds or removes -`monitoring` in `COMPOSE_PROFILES` in `.env` and starts or stops Prometheus, node-exporter and Grafana, and nothing -else; it reports in `services-status` and `services.log`. +The same agent turns Lens's [optional containers](assistant.md#optional-containers) on and off: a request with +`action=services`, a `profile` and `turn=on` or `off` (in the same inbox file, the one every agent watches) adds or +removes that profile in `COMPOSE_PROFILES` in `.env` and starts or stops its containers, and nothing else. The +profiles are fixed in the agent: `mcp` (the MCP runner), `browser` (the browser and its proxy) and `monitoring` +([Charts and history](prometheus.md#prometheus-and-grafana-ready-made): Prometheus, node-exporter and Grafana); it +refuses any other. It reports in `services-status` and `services.log`. A request from an older Lens, with +`monitoring=on` or `off` and no profile, still works. With no agent (Lens wasn't installed with install.sh, the machine has neither systemd nor launchd, or install.sh ran with `LENS_UPGRADE_AGENT=0`), the API gives the one line to run on the machine instead (`manual`). diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 4c6acdcc1..4720591ff 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -54,6 +54,7 @@ nsfw, rdf, render, + services, settings, sources, store, @@ -2007,6 +2008,49 @@ def set_monitoring(ctx: Context, on: bool) -> dict[str, Any]: return got +# ---------- Lens's optional containers: the MCP runner, the browser, monitoring (admins) ---------- +@tool( + "services", + "Optional containers", + "Lens's optional containers: the MCP runner (MCP servers that run as a program), the browser (a headless Chromium " + "the assistant drives) and monitoring. Whether each is on and answering, and how the machine's last change went. " + "Admins only.", + admin=True, +) +def services_status(ctx: Context) -> dict[str, Any]: + if not ctx.user.admin: + raise ToolError("only admins see or change Lens's optional containers") + return {"services": services.views(ctx.db), "settings": f"{ctx.web}/extensions/new/mcp"} + + +@tool( + "set_service", + "Turn an optional container on or off", + "Have this machine start (on true) or stop one of Lens's optional containers: mcp (the MCP runner), browser, or " + "monitoring (which also turns /metrics on). The machine's upgrade agent does it; services follows along. Admins " + "only, with a token that has the write scope.", + Arg("profile", "string", "which", required=True, enum=tuple(services.PROFILES)), + Arg("on", "boolean", "start (true) or stop (false)", required=True), + writes=True, + admin=True, +) +def set_service(ctx: Context, profile: str, on: bool) -> dict[str, Any]: + if not ctx.user.admin: + raise ToolError("only admins see or change Lens's optional containers") + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to change them") + try: + if profile == "monitoring": + monitoring.turn(ctx.db, ctx.request.app.state.archive.base, ctx.cfg, bool(on), by=ctx.user.email) + else: + services.ask(ctx.db, profile, bool(on), by=ctx.user.email) + except services.ServiceError as e: + raise ToolError(str(e)) from None + action = "lens.monitoring" if profile == "monitoring" else "lens.services" + auth.audit(ctx.db, ctx.user.as_audit(), action, None, {"profile": profile, "on": bool(on), "via": "mcp"}) + return services.view(ctx.db, profile) + + # ---------- how install.sh set this machine up: reachable on the network, and the edition (admins) ---------- @tool( "install_settings", diff --git a/fastapi_backend/app/api/v1/routes/extensions.py b/fastapi_backend/app/api/v1/routes/extensions.py index eb9b8d5ea..6761491ca 100644 --- a/fastapi_backend/app/api/v1/routes/extensions.py +++ b/fastapi_backend/app/api/v1/routes/extensions.py @@ -8,8 +8,8 @@ from fastapi import APIRouter, HTTPException, Request -from app.api.deps import Acl, Cfg, CurrentUser, Db, Principal, Writer, domain_errors -from app.domain import ai_tools, auth, extensions, marketplace, mcp_servers +from app.api.deps import Acl, AdminReader, Cfg, CurrentUser, Db, Principal, Writer, domain_errors +from app.domain import ai_tools, auth, extensions, marketplace, mcp_servers, services from app.schemas.common import Created, Ok from app.schemas.extensions import ( Extension, @@ -28,6 +28,7 @@ ManifestChecked, McpConfig, McpConfigText, + McpPresets, McpServerTools, McpServerTry, McpToolSwitch, @@ -114,6 +115,14 @@ def test_mcp_server(body: McpServerTry, user: Writer, db: Db) -> McpServerTools: return McpServerTools(tools=extensions.try_mcp(db, me(user), body.spec, body.extension)) +@router.get("/mcp/presets") +def mcp_presets(user: AdminReader, db: Db) -> McpPresets: + """MCP servers ready to add (admins): the browser Lens can run next to it, with whether its container is on and + running (`service`), and GitHub's own server, with the token to type in (`fill`).""" + out = [p | {"service": services.view(db, p["needs"]) if p["needs"] else None} for p in mcp_servers.presets()] + return McpPresets.model_validate({"presets": out}) + + @router.get("/{eid}") def get_extension(eid: int, user: CurrentUser, db: Db, version: int | None = None) -> ExtensionDetail: """One version (default: the current one), as a manifest too, and the list of versions.""" diff --git a/fastapi_backend/app/api/v1/routes/updates.py b/fastapi_backend/app/api/v1/routes/updates.py index ac64c85e5..070dfa183 100644 --- a/fastapi_backend/app/api/v1/routes/updates.py +++ b/fastapi_backend/app/api/v1/routes/updates.py @@ -9,7 +9,7 @@ from fastapi.responses import FileResponse from app.api.deps import AdminReader, AdminWriter, Cfg, Db, domain_errors -from app.domain import auth, backups, monitoring, updates +from app.domain import auth, backups, monitoring, services, updates from app.schemas.backups import BackupSchedule, RestoreWork from app.schemas.common import Ok from app.schemas.updates import ( @@ -19,6 +19,9 @@ InstallStatus, Monitoring, MonitoringRequest, + Service, + ServiceRequest, + Services, UpdateStatus, UpgradeRequest, ) @@ -100,6 +103,32 @@ def set_monitoring(body: MonitoringRequest, user: AdminWriter, request: Request, return Monitoring.model_validate(got) +@router.get("/admin/services") +def list_services(user: AdminReader, db: Db) -> Services: + """Lens's optional containers (the MCP runner, the browser, monitoring): whether each is on and answering, and + how the machine's last change went.""" + return Services.model_validate({"services": services.views(db)}) + + +@router.put("/admin/services/{profile}", status_code=202) +def set_service(profile: str, body: ServiceRequest, user: AdminWriter, request: Request, db: Db, cfg: Cfg) -> Service: + """Have this machine start or stop one of them: its upgrade agent adds the profile to COMPOSE_PROFILES in .env, or + takes it out, and starts or stops its containers. Monitoring also turns /metrics on, as PUT /admin/monitoring + does. GET /admin/services follows along; without an agent, 409 says the line to run on the machine instead.""" + if profile not in services.PROFILES: + raise HTTPException(404, f"Lens has no optional containers called {profile!r}") + try: + if profile == "monitoring": + monitoring.turn(db, request.app.state.archive.base, cfg, body.on, by=user.email) + else: + services.ask(db, profile, body.on, by=user.email) + except services.ServiceError as e: + raise HTTPException(409, str(e)) from None + action = "lens.monitoring" if profile == "monitoring" else "lens.services" + auth.audit(db, user.as_audit(), action, None, {"profile": profile, "on": body.on}) + return Service.model_validate(services.view(db, profile)) + + @router.get("/admin/backups") def list_backups(user: AdminReader, db: Db, cfg: Cfg) -> Backups: """The database backups in /backups, newest first; those being written; the restore running here (or diff --git a/fastapi_backend/app/domain/extensions.py b/fastapi_backend/app/domain/extensions.py index 37c21bab9..160245e74 100644 --- a/fastapi_backend/app/domain/extensions.py +++ b/fastapi_backend/app/domain/extensions.py @@ -266,11 +266,15 @@ def _check_hook(spec, me, db=None): def _check_mcp(spec, me, db=None): """An MCP server: its address and the headers it needs (a token), or the program it runs as (command, args, env: run in the MCP runner's sandbox), its tools as it listed them (mcp_client.py), the tools switched off (`off`) and - which ask first where that isn't the server's own read-only hint (`ask`: {tool: true or false}).""" - if set(spec) - {"url", "headers", "command", "args", "env", "tools", "off", "ask"}: + which ask first where that isn't the server's own read-only hint (`ask`: {tool: true or false}), and `keep` for one + that remembers things between calls, so its calls share one session (mcp_client.py).""" + if set(spec) - {"url", "headers", "command", "args", "env", "tools", "off", "ask", "keep"}: raise ValueError( - "an MCP server is {url, headers, tools, off, ask}, or {command, args, env, tools, off, ask} for one that runs as a program" + "an MCP server is {url, headers, tools, off, ask, keep}, or {command, args, env, tools, off, ask, keep} for one " + "that runs as a program" ) + if not isinstance(spec.get("keep", False), bool): + raise ValueError("an MCP server's keep is true or false: whether its calls share one session") if not me["admin"]: raise ValueError("only admins add MCP servers") @@ -303,7 +307,7 @@ def words(v, what): from . import mcp_client tools = mcp_client.discover(store.clean(server)) - server.update({"off": sorted(set(off)) or None, "ask": ask or None}) + server.update({"off": sorted(set(off)) or None, "ask": ask or None, "keep": spec.get("keep") or None}) if tools is not None: if not isinstance(tools, list) or len(tools) > 100: raise ValueError("an MCP server's tools are a list of at most 100") @@ -335,7 +339,7 @@ def mcp_tools(item, ext=None): if t["name"] in off: continue name = re.sub(r"[^A-Za-z0-9_-]", "_", f"{item['name']}__{t['name']}")[:64] - where = {k: s.get(k) for k in ("url", "headers", "command", "args", "env")} + where = {k: s.get(k) for k in ("url", "headers", "command", "args", "env", "keep")} run = store.clean({"type": "mcp", **where, "tool": t["name"], "ext": ext, "server": item["name"]}) spec = { "params": [], diff --git a/fastapi_backend/app/domain/mcp_client.py b/fastapi_backend/app/domain/mcp_client.py index 324c4102a..4ef132d29 100644 --- a/fastapi_backend/app/domain/mcp_client.py +++ b/fastapi_backend/app/domain/mcp_client.py @@ -8,6 +8,10 @@ A server that runs as a program (stdio: a command, usually `npx` or `uvx`) runs in the MCP runner, a container of its own with none of Lens's data or keys (mcp-runner/runner.py, the optional `mcp` service), which Lens tells what to run and then reaches over HTTP like any other. Without the runner, such servers aren't run. + +A server whose spec says `keep` remembers things between calls (the browser: the page it has open), so its tools are +called in one session, kept from one call to the next for KEEP seconds, and opened again when the server no longer +knows it. That session is the server's, shared by everyone whose assistant uses it, as its address and token are. """ from __future__ import annotations @@ -16,6 +20,8 @@ import itertools import json import os +import threading +import time import urllib.error import urllib.request @@ -26,13 +32,20 @@ MAX_REPLY = 2_000_000 MAX_TOOLS = 100 TIMEOUT = 60 +KEEP = 600 # seconds a kept session waits for its next call _ids = itertools.count(1) +_kept: dict[str, tuple[Session, float]] = {} +_lock = threading.Lock() class McpError(ValueError): pass +class Refused(McpError): + """The server turned the request away before running anything (it doesn't know the session, or the token).""" + + def _events(raw): """The JSON messages of an event stream's `data:` lines.""" data = [] @@ -72,8 +85,9 @@ def runner(): NO_RUNNER = ( - "MCP servers that run as a program need the MCP runner, a sandbox of their own: add COMPOSE_PROFILES=mcp to " - "Lens's .env and run the install command again (docs/assistant.md)" + "MCP servers that run as a program need the MCP runner, a sandbox of their own: turn it on in Extensions › Add " + "MCP server › Command (or add COMPOSE_PROFILES=mcp to Lens's .env and run the install command again; " + "docs/assistant.md)" ) @@ -123,7 +137,7 @@ def _send(self, method, params, note=False): return None if status >= 400: hint = ": does it need a token in its headers?" if status in (401, 403) else "" - raise McpError(f"the MCP server answered {status}{hint}") + raise (Refused if status in (400, 401, 403, 404) else McpError)(f"the MCP server answered {status}{hint}") msgs = list(_events(text)) if "text/event-stream" in low.get("content-type", "") else [json.loads(text or "null")] for m in msgs: if isinstance(m, dict) and m.get("id") == body["id"]: @@ -164,10 +178,36 @@ def discover(server): return out +def _call(run, args): + """tools/call in a new session, or, for a server that keeps its session, in the one kept for it (a new one when + that one is too old or the server turns it away).""" + if not run.get("keep"): + return Session(run).call(run["tool"], args) + where = {k: run.get(k) for k in ("url", "headers", "command", "args", "env")} + key = hashlib.sha256(json.dumps(where, sort_keys=True, default=str).encode()).hexdigest() + now = time.monotonic() + with _lock: + for k, (_, used) in list(_kept.items()): + if now - used > KEEP: + del _kept[k] + kept = _kept.get(key) + session = kept[0] if kept else Session(run) + try: + got = session.call(run["tool"], args) + except Refused: + if not kept: + raise + session = Session(run) # the server restarted or let it go: nothing ran, so once more in a new one + got = session.call(run["tool"], args) + with _lock: + _kept[key] = (session, time.monotonic()) + return got + + def call(run, args): """Call one of a server's tools: its text, and anything structured it gave back.""" with activity.call("mcp.call", detail={"url": run.get("url") or run.get("command"), "tool": run["tool"]}): - got = Session(run).call(run["tool"], args) + got = _call(run, args) texts = [c.get("text") for c in got.get("content") or [] if isinstance(c, dict) and c.get("type") == "text"] out = {"text": "\n".join(t for t in texts if t)} if got.get("structuredContent") is not None: diff --git a/fastapi_backend/app/domain/mcp_servers.py b/fastapi_backend/app/domain/mcp_servers.py index 085504bbd..1025c34dd 100644 --- a/fastapi_backend/app/domain/mcp_servers.py +++ b/fastapi_backend/app/domain/mcp_servers.py @@ -14,12 +14,15 @@ same place (`opened`). - `off` (the tools switched off) and `ask` ({tool: whether it asks first}, where that differs from the server's own read-only hint) sit in the server's spec, so they survive its tools being listed again and travel with its manifest. +- `presets` are servers ready to add: the browser Lens can run next to it (the optional `browser` profile, Playwright + MCP) and GitHub's own server, with a token to type in. """ from __future__ import annotations import hashlib import json +import os import re import urllib.parse @@ -268,6 +271,43 @@ def read(text): return {"servers": out, "skipped": skipped} +# ---------- ready-made servers (Add MCP server) ---------- +# The browser's tools that only go somewhere, and needn't ask before the conversation has read a page (they ask after, +# as every tool that reaches another host does); clicking, typing and filling in forms always ask +BROWSER_GOES = ("browser_navigate", "browser_navigate_back", "browser_wait_for") + + +def presets(): + """Servers to add with a click or two: [{name, label, description, spec, fill, needs}]. `fill` names the values to + type in first, written ${NAME} in the spec; `needs` is the optional profile it runs in (services.py), if any.""" + browser = (os.environ.get("LENS_BROWSER_URL") or "http://browser:8931").rstrip("/") + return [ + { + "name": "browser", + "label": "Browser", + "description": "Opens web pages, reads them, clicks, types and fills in forms (Playwright MCP, in Lens's browser container)", + # one session for all its calls: the page it has open stays open from one call to the next. Running code + # in the browser's own server process starts off: it reaches past the browser + "spec": { + "url": f"{browser}/mcp", + "keep": True, + "off": ["browser_run_code_unsafe"], + "ask": {t: False for t in BROWSER_GOES}, + }, + "fill": [], + "needs": "browser", + }, + { + "name": "github", + "label": "GitHub", + "description": "Issues, pull requests and code on GitHub, with a personal access token (GitHub's own MCP server)", + "spec": {"url": "https://api.githubcopilot.com/mcp/", "headers": {"Authorization": "Bearer ${GITHUB_TOKEN}"}}, + "fill": ["GITHUB_TOKEN"], + "needs": None, + }, + ] + + # ---------- upgrading ---------- def migrate_seal(db): """Upgrade step: seal the credentials MCP servers kept in plain text in their headers and env, in every version of diff --git a/fastapi_backend/app/domain/monitoring.py b/fastapi_backend/app/domain/monitoring.py index 552b5e93e..5e659471a 100644 --- a/fastapi_backend/app/domain/monitoring.py +++ b/fastapi_backend/app/domain/monitoring.py @@ -3,78 +3,38 @@ The containers can't start other containers, so the machine's upgrade agent does it, as it installs releases (updates.py, packaging/host/lens-upgrade.sh). Turning monitoring on makes sure /metrics is on with a scrape token (generated when none is set), writes that token where the monitoring Prometheus reads it (`/prometheus-token`), -then drops a request in the agent's inbox: the agent adds `monitoring` to COMPOSE_PROFILES in the install's .env and -starts the three containers; off removes it and stops them. How it went is in `/services-status`. - -The request goes in the inbox under the same name as an upgrade (`inbox/upgrade`, with `action=services`), because that -file is what every agent installed so far watches for; the agent never installs anything for it. +then asks the agent to turn the monitoring profile on (services.py): it adds `monitoring` to COMPOSE_PROFILES in the +install's .env and starts the three containers; off removes it and stops them. """ from __future__ import annotations -import datetime as dt -import os import secrets -from . import settings, store, updates +from . import services, settings, store, updates ROW = store.R("app_service", "monitoring") -ACTIVE = ("queued", "running") TOKEN_FILE = "prometheus-token" -STATUS_FILE = "services-status" GRAFANA_PORT = 3001 - - -class MonitoringError(RuntimeError): - """Monitoring can't be turned on or off; the message says why, for the admin.""" +# Monitoring can't be turned on or off; the message says why, for the admin +MonitoringError = services.ServiceError def _row(db): return db.one("SELECT * OMIT id FROM $r", r=ROW) or {} -def _put(db, run): - db.q("UPSERT $r CONTENT $d", r=ROW, d=run) - - def manual(): """The line to run on the machine when there is no agent.""" - where = os.environ.get("LENS_HOST_DIR") or "" - cd = f"cd {where} && " if where and updates.REF.match(where.lstrip("/~")) else "" - return f"{cd}docker compose -f docker-compose.prod.yml --profile monitoring up -d" + return services.manual("monitoring") def view(db): """Whether monitoring is on (as last asked), and how the agent's last run went.""" - run = _row(db) - out = { - "on": bool(run.get("on")), - "state": run.get("state"), - "error": run.get("error"), - "asked_at": run.get("asked_at"), - "finished_at": None, - "grafana_port": GRAFANA_PORT, - "agent": updates.agent(), - "manual": manual(), - } - d = updates.control_dir() - if d and run: - s = updates._kv(d / STATUS_FILE) - if s.get("id") == run.get("run"): - out["state"] = s.get("state") or out["state"] - out["finished_at"] = s.get("finished_at") - if s.get("grafana_port", "").isdigit(): - out["grafana_port"] = int(s["grafana_port"]) - if s.get("state") == "failed": - out["error"] = s.get("error") or f"the machine couldn't change it (exit {s.get('code', '?')})" - if out["state"] == "queued" and (d / "inbox" / "upgrade").exists(): - try: - waited = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(str(run.get("asked_at")))).total_seconds() - except ValueError: - waited = 0 - if waited > updates.PICKUP_SECONDS: - out["error"] = "the machine hasn't picked it up yet: is its upgrade agent running?" - return out + v = services.view(db, "monitoring", probe=False) + port = services.reported(db, "monitoring").get("grafana_port", "") + keys = ("on", "state", "error", "asked_at", "finished_at", "agent", "manual") + return {k: v[k] for k in keys} | {"grafana_port": int(port) if port.isdigit() else GRAFANA_PORT} def sync_token(cfg, create=False): @@ -97,14 +57,8 @@ def sync_token(cfg, create=False): def turn(db, base, cfg, on, by=None): """Ask the machine to start (on) or stop the monitoring containers; returns view().""" - if not updates.agent(): - raise MonitoringError("this machine has no upgrade agent to start containers: run the line shown on it instead") - d = updates.control_dir() - if (d / "inbox" / "upgrade").exists(): - raise MonitoringError("the machine is busy with another request (an upgrade?): try again in a minute") - run, now_ = _row(db), view(db) - if now_["state"] in ACTIVE and not now_["error"]: - raise MonitoringError("the machine is still changing it: wait for that to finish") + services.ready(db, "monitoring") + run = _row(db) t = cfg.get("telemetry") or {} locked = settings.locked("telemetry") changes, ours = {}, bool(run.get("turned_on_metrics")) @@ -122,10 +76,5 @@ def turn(db, base, cfg, on, by=None): if on: token = changes.get("prometheus_token") or t.get("prometheus_token") sync_token({"telemetry": {"prometheus_token": token}}, create=True) - rid = secrets.token_hex(8) - now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") - part = d / "inbox" / f".upgrade.{rid}" - part.write_text(f"id={rid}\naction=services\nmonitoring={'on' if on else 'off'}\nby={by or ''}\nat={now}\n", encoding="utf-8") - part.replace(d / "inbox" / "upgrade") - _put(db, {"run": rid, "on": bool(on), "by": by, "asked_at": now, "state": "queued", "error": None, "turned_on_metrics": ours}) + services.ask(db, "monitoring", on, by, turned_on_metrics=ours) return view(db) diff --git a/fastapi_backend/app/domain/netguard.py b/fastapi_backend/app/domain/netguard.py index ce4a2b86f..be12d3fc9 100644 --- a/fastapi_backend/app/domain/netguard.py +++ b/fastapi_backend/app/domain/netguard.py @@ -10,10 +10,16 @@ Capturing a web page, the proxy also forwards requests to the web (`forward`), on ports 80 and 443 only, to public addresses only: it resolves each host itself, refuses it when any of its addresses isn't public (or in a network an admin allowed), and connects to the address it checked, so a name can't be made to point elsewhere in between. + +Run on its own (`python netguard.py`, the optional `browser-proxy` service: standard library only), it is that +forwarding proxy for the assistant's browser, the `browser` service, whose network has no other way out +(docs/assistant.md, The browser). """ from __future__ import annotations +import argparse +import collections import ipaddress import select import socket @@ -125,6 +131,11 @@ def _refuse(self, why): self.server.refused.append(why[:300]) self.wfile.write(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + def _budget(self): + """What may still come back: the Guard's, shared by every request the page makes, or (serve) this + connection's own.""" + return [self.server.each] if self.server.each else self.server.budget + def _connect(self, host, port): if port not in PORTS: raise ValueError(f"port {port} isn't a web port") @@ -137,7 +148,7 @@ def _tunnel(self, target): with upstream: self.wfile.write(b"HTTP/1.1 200 Connection Established\r\n\r\n") self.wfile.flush() - _pipe(self.connection, upstream, self.server.budget) + _pipe(self.connection, upstream, self._budget()) def _relay(self, method, url, headers): """A plain http:// request, sent on to the address checked as one request on its own connection.""" @@ -152,7 +163,7 @@ def _relay(self, method, url, headers): length = next((int(h.split(b":", 1)[1]) for h in kept if h.lower().startswith(b"content-length:")), 0) if length: upstream.sendall(self.rfile.read(length)) - _pipe(self.connection, upstream, self.server.budget) + _pipe(self.connection, upstream, self._budget()) class _Server(socketserver.ThreadingTCPServer): @@ -163,6 +174,7 @@ class _Server(socketserver.ThreadingTCPServer): forward: bool allow: object budget: list[int] + each: int class Guard: @@ -186,6 +198,7 @@ def __init__(self, pages=None, forward=False, networks=(), max_bytes=500 * 2**20 def __enter__(self): srv = _Server(("127.0.0.1", 0), _Handler) srv.pages, srv.refused, srv.asked, srv.forward, srv.budget = self.pages, self.refused, self.asked, self.forward, [self.max_bytes] + srv.each = 0 networks = self.networks srv.allow = lambda ip: public_ip(ip, networks) threading.Thread(target=srv.serve_forever, name="netguard", daemon=True).start() @@ -244,3 +257,32 @@ def nowhere_env(env): for k in ("no_proxy", "NO_PROXY"): out.pop(k, None) return out + + +# ---------- on its own: the browser's way out (the browser-proxy service) ---------- +class _Said(collections.deque): + """What a proxy that runs for days keeps of what it refused: the last few, each also written to its log.""" + + def append(self, x): + print(f"refused {x}", flush=True) + super().append(x) + + +def forwarder(listen="0.0.0.0:3128", max_bytes=500 * 2**20): + """The forwarding proxy, listening (`serve_forever` runs it): to public addresses, on the web's ports, with at + most `max_bytes` coming back on each connection.""" + host, _, port = listen.rpartition(":") + srv = _Server((host or "0.0.0.0", int(port)), _Handler) + srv.pages, srv.refused, srv.asked = {}, _Said(maxlen=50), collections.deque(maxlen=0) + srv.forward, srv.budget, srv.each = True, [0], max_bytes + srv.allow = public_ip + return srv + + +if __name__ == "__main__": + a = argparse.ArgumentParser(description="A forwarding proxy that reaches public web addresses only.") + a.add_argument("--listen", default="0.0.0.0:3128", help="host:port (default 0.0.0.0:3128)") + listen = a.parse_args().listen + srv = forwarder(listen) + print(f"Forwarding to public addresses on ports 80 and 443, on {listen}", flush=True) + srv.serve_forever() diff --git a/fastapi_backend/app/domain/services.py b/fastapi_backend/app/domain/services.py new file mode 100644 index 000000000..535a3a767 --- /dev/null +++ b/fastapi_backend/app/domain/services.py @@ -0,0 +1,172 @@ +"""Lens's optional containers, turned on and off from the web app: the MCP runner, the browser and monitoring +(docs/assistant.md, docs/prometheus.md). + +The containers can't start other containers, so the machine's upgrade agent does it, as it installs releases +(updates.py, packaging/host/lens-upgrade.sh): a request with action=services, profile= and turn=on|off in its +inbox makes it add the profile to COMPOSE_PROFILES in the install's .env, or take it out, and start or stop that +profile's containers. It says how it went in `/services-status`, which is about the last request only, +whichever profile it was for, so what it said about a profile's request is kept with that profile once it's finished. +The agent does one request at a time, so a profile isn't turned while another is still changing. + +Monitoring adds its own steps around this (monitoring.py: /metrics and its token). A profile added here needs its +containers named in the agent's services_of too, or the agent refuses it. +""" + +from __future__ import annotations + +import datetime as dt +import os +import secrets +import socket +import urllib.parse + +from . import store, updates + +# profile: (what it's called, what it is, the address it answers on while it runs, or None) +PROFILES = { + "mcp": ( + "MCP runner", + "Runs the MCP servers that are programs (npx, uvx) in a container of their own, with none of Lens's data or keys", + "LENS_MCP_RUNNER_URL", + ), + "browser": ( + "Browser", + "A headless Chromium the assistant drives over MCP (Playwright MCP), in a container that reaches public web " + "addresses only, through a proxy that checks each one", + "LENS_BROWSER_URL", + ), + "monitoring": ( + "Monitoring", + "Prometheus, node-exporter and Grafana: charts of Lens and this machine over time", + None, + ), +} +ACTIVE = ("queued", "running") +FINAL = ("done", "failed") +STATUS_FILE = "services-status" + + +class ServiceError(RuntimeError): + """A profile can't be turned on or off; the message says why, for the admin.""" + + +def _check(profile): + if profile not in PROFILES: + raise ServiceError(f"Lens has no optional containers called {profile!r}: {', '.join(PROFILES)}") + + +def _row(db, profile): + return db.one("SELECT * OMIT id FROM $r", r=store.R("app_service", profile)) or {} + + +def _put(db, profile, run): + db.q("UPSERT $r CONTENT $d", r=store.R("app_service", profile), d=run) + + +def manual(profile): + """The line to run on the machine when there is no agent.""" + where = os.environ.get("LENS_HOST_DIR") or "" + cd = f"cd {where} && " if where and updates.REF.match(where.lstrip("/~")) else "" + return f"{cd}docker compose -f docker-compose.prod.yml --profile {profile} up -d" + + +def running(profile): + """Whether the profile's container answers now (it takes connections where Lens reaches it), or None when Lens + can't tell (monitoring, or no address set).""" + env = PROFILES[profile][2] + u = urllib.parse.urlsplit(os.environ.get(env, "") if env else "") + if not u.hostname: + return None + try: + with socket.create_connection((u.hostname, u.port or 80), timeout=1.5): + return True + except OSError: + return False + + +def reported(db, profile, run=None): + """What the agent said about the profile's last request: its status file while that's about the request, else + what was kept once it finished ({} before the agent said anything).""" + run = _row(db, profile) if run is None else run + said = dict(run.get("said") or {}) + d = updates.control_dir() + if d and run.get("run"): + s = updates._kv(d / STATUS_FILE) + if s.get("id") == run["run"]: + said = s + if s.get("state") in FINAL and run.get("said") != s: + _put(db, profile, {**run, "said": s}) + return said + + +def view(db, profile, probe=True): + """Whether the profile is on (as last asked, or, never asked here, whether it runs), and how the agent's last + change went.""" + _check(profile) + run = _row(db, profile) + label, about, _ = PROFILES[profile] + up = running(profile) if probe else None + out = { + "profile": profile, + "label": label, + "description": about, + "on": bool(run.get("on")) if run else bool(up), + "running": up, + "state": run.get("state"), + "error": run.get("error"), + "asked_at": run.get("asked_at"), + "finished_at": None, + "agent": updates.agent(), + "manual": manual(profile), + } + s = reported(db, profile, run) + if s: + out["state"] = s.get("state") or out["state"] + out["finished_at"] = s.get("finished_at") + if s.get("state") == "failed": + out["error"] = s.get("error") or f"the machine couldn't change it (exit {s.get('code', '?')})" + d = updates.control_dir() + if d and out["state"] == "queued" and (d / "inbox" / "upgrade").exists(): + try: + asked = dt.datetime.fromisoformat(str(run.get("asked_at"))) + waited = (dt.datetime.now(dt.timezone.utc) - asked).total_seconds() + except ValueError: + waited = 0 + if waited > updates.PICKUP_SECONDS: + out["error"] = "the machine hasn't picked it up yet: is its upgrade agent running?" + return out + + +def views(db): + """Every optional profile, as view() says it.""" + return [view(db, p) for p in PROFILES] + + +def ready(db, profile): + """ServiceError unless the machine can take a request for the profile now.""" + _check(profile) + if not updates.agent(): + raise ServiceError("this machine has no upgrade agent to start containers: run the line shown on it instead") + d = updates.control_dir() + if (d / "inbox" / "upgrade").exists(): + raise ServiceError("the machine is busy with another request (an upgrade?): try again in a minute") + for p in PROFILES: # one at a time: the agent's status file is about the last request only + v = view(db, p, probe=False) + if v["state"] in ACTIVE and not v["error"]: + what = "it" if p == profile else f"the {v['label'].lower()}" + raise ServiceError(f"the machine is still changing {what}: wait for that to finish") + return d + + +def ask(db, profile, on, by=None, **keep): + """Ask the machine to start (on) or stop the profile's containers; returns view(). `keep` is kept with the + request (monitoring's own fields).""" + d = ready(db, profile) + rid = secrets.token_hex(8) + now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + part = d / "inbox" / f".upgrade.{rid}" + turn = "on" if on else "off" + part.write_text(f"id={rid}\naction=services\nprofile={profile}\nturn={turn}\nby={by or ''}\nat={now}\n", encoding="utf-8") + part.replace(d / "inbox" / "upgrade") + _put(db, profile, {"run": rid, "on": bool(on), "by": by, "asked_at": now, "state": "queued", "error": None, **keep}) + return view(db, profile) diff --git a/fastapi_backend/app/schemas/extensions.py b/fastapi_backend/app/schemas/extensions.py index f91bf5da9..e3ffd9e36 100644 --- a/fastapi_backend/app/schemas/extensions.py +++ b/fastapi_backend/app/schemas/extensions.py @@ -7,6 +7,7 @@ from pydantic import Field from app.schemas.common import RequestModel, ResponseModel +from app.schemas.updates import Service Kind = Literal["tool", "skill", "hook", "plugin", "mcp"] Visibility = Literal["private", "namespace", "everyone"] @@ -115,6 +116,20 @@ class McpServerTools(ResponseModel): tools: list[dict[str, Any]] = Field(description="its tools, as it lists them now: {name, description, schema, read_only}") +class McpPreset(ResponseModel): + name: str = Field(description="what its tools are called by: name__tool") + label: str + description: str + spec: dict[str, Any] = Field(description="the server, as an MCP extension keeps it") + fill: list[str] = Field(default_factory=list, description="values written as ${NAME} in spec, to type in first") + needs: str | None = Field(None, description="the optional profile it runs in (GET /admin/services), if any") + service: Service | None = Field(None, description="that profile: whether it's on and running") + + +class McpPresets(ResponseModel): + presets: list[McpPreset] + + class McpToolSwitch(RequestModel): tool: str = Field(description="the tool's name, as the server lists it") on: bool | None = Field(None, description="switch it on or off") diff --git a/fastapi_backend/app/schemas/updates.py b/fastapi_backend/app/schemas/updates.py index 101a3f9de..37adb081b 100644 --- a/fastapi_backend/app/schemas/updates.py +++ b/fastapi_backend/app/schemas/updates.py @@ -140,3 +140,25 @@ class Monitoring(ResponseModel): class MonitoringRequest(RequestModel): on: bool = Field(description="start (true) or stop (false) Prometheus, node-exporter and Grafana") + + +class Service(ResponseModel): + profile: str = Field(description="the Docker Compose profile: mcp (the MCP runner), browser, monitoring") + label: str + description: str + on: bool = Field(description="as last asked here; never asked here: whether it runs") + running: bool | None = Field(None, description="whether it answers where Lens reaches it now; null: Lens can't tell") + state: Literal["queued", "running", "done", "failed"] | None = Field(None, description="how the machine's last change went") + error: str | None = None + asked_at: str | None = None + finished_at: str | None = None + agent: str | None = Field(None, description="what changes it on this machine (systemd, launchd); none: run `manual` there") + manual: str = Field(description="the line that turns it on by hand, on the machine Lens runs on") + + +class Services(ResponseModel): + services: list[Service] + + +class ServiceRequest(RequestModel): + on: bool = Field(description="start (true) or stop (false) the profile's containers") diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index 6e720b9df..b3f017db4 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -88,6 +88,7 @@ def test_clients_are_sent_to_sign_in(client, app, env): "set_namespace_integrations", "set_file_storage", "set_monitoring", + "set_service", "set_install_settings", "mark_activities", "decline_approval", @@ -524,6 +525,11 @@ def test_monitoring_tools(client, db, env, monkeypatch): got = tool(client, ha, "monitoring") assert (got["on"], got["agent"]) == (False, None) and got["settings"].endswith("/settings/telemetry") assert "no upgrade agent" in tool_error(client, ha, "set_monitoring", on=True) + # and every optional container, the same way + got = tool(client, ha, "services") + assert [s["profile"] for s in got["services"]] == ["mcp", "browser", "monitoring"] + assert "no upgrade agent" in tool_error(client, ha, "set_service", profile="browser", on=True) + assert "only admins" in tool_error(client, env["h"], "set_service", profile="browser", on=True) def test_install_tools(client, db, env, tmp_path, monkeypatch): diff --git a/fastapi_backend/tests/api/test_mcp_servers.py b/fastapi_backend/tests/api/test_mcp_servers.py index 1e624e4c9..4e7d2e1d1 100644 --- a/fastapi_backend/tests/api/test_mcp_servers.py +++ b/fastapi_backend/tests/api/test_mcp_servers.py @@ -416,3 +416,68 @@ def read(text): ) def test_a_server_without_a_name_gets_one(server, name): assert mcp_servers.guess_name(server) == name + + +BROWSER_TOOLS = [ + {"name": "browser_navigate", "description": "Go to a page", "inputSchema": {"type": "object"}}, + {"name": "browser_snapshot", "description": "Read the page", "inputSchema": {"type": "object"}, "annotations": {"readOnlyHint": True}}, + {"name": "browser_click", "description": "Click", "inputSchema": {"type": "object"}}, + {"name": "browser_run_code_unsafe", "description": "Run code", "inputSchema": {"type": "object"}}, +] + + +def test_the_browser_preset_keeps_one_session(app, db, cfg, folder, new_client, monkeypatch): + """The browser, added with a click from its preset: its calls share a session (the page stays open), opened again + when the server lets it go; running code in its server process starts off, and going to a page asks only once the + conversation has read outside text.""" + s = Assist(app, db, cfg, folder, new_client) + c, h = s.cl["editor"] + ca, ha = s.cl["admin"] + assert c.get("/api/v1/extensions/mcp/presets", headers=h).status_code == 403 + got = {p["name"]: p for p in ca.get("/api/v1/extensions/mcp/presets", headers=ha).json()["presets"]} + b, gh = got["browser"], got["github"] + assert (b["needs"], b["service"]["profile"], b["spec"]["url"]) == ("browser", "browser", "http://browser:8931/mcp") + assert b["spec"]["keep"] is True and b["spec"]["off"] == ["browser_run_code_unsafe"] + assert (gh["fill"], gh["service"], gh["spec"]["headers"]) == (["GITHUB_TOKEN"], None, {"Authorization": "Bearer ${GITHUB_TOKEN}"}) + + sessions, gone, calls = [], set(), [] + + def post(url, headers, body, timeout=60): + if "id" not in body: + return 202, {}, "" + if body["method"] == "initialize": + sessions.append(f"s{len(sessions) + 1}") + result = {"protocolVersion": "2025-06-18", "capabilities": {"tools": {}}, "serverInfo": {"name": "pw"}} + reply = json.dumps({"jsonrpc": "2.0", "id": body["id"], "result": result}) + return 200, {"Content-Type": "application/json", "Mcp-Session-Id": sessions[-1]}, reply + sid = headers.get("Mcp-Session-Id") + if sid in gone: + return 404, {}, "Session not found" + if body["method"] == "tools/list": + result = {"tools": BROWSER_TOOLS} + else: + calls.append((sid, body["params"]["name"])) + result = {"content": [{"type": "text", "text": f"{sid} {body['params']['name']}"}]} + return 200, {"Content-Type": "application/json"}, json.dumps({"jsonrpc": "2.0", "id": body["id"], "result": result}) + + monkeypatch.setattr(mcp_client, "_post", post) + monkeypatch.setattr(mcp_client, "_kept", {}) + manifest = {"name": b["name"], "kind": "mcp", **b["spec"]} + eid = ca.post("/api/v1/extensions", headers=ha, json={"manifest": manifest}).json()["id"] + tools = {t["name"]: t for t in extensions.tool_items(extensions.get(db, eid))} + assert sorted(tools) == ["browser__browser_click", "browser__browser_navigate", "browser__browser_snapshot"] + effects = [tools[f"browser__browser_{n}"]["spec"]["effect"] for n in ("navigate", "snapshot", "click")] + assert effects == ["read", "read", "change"] + + def use(name): + return mcp_client.call(mcp_servers.open_run(db, tools[f"browser__{name}"]["spec"]["run"]), {})["text"] + + first = use("browser_navigate").split()[0] + assert use("browser_snapshot").split()[0] == first # the same session: the page it opened + gone.add(first) # the server restarted: a new session, and the call made once more + assert use("browser_snapshot").split()[0] != first + assert [n for _, n in calls].count("browser_snapshot") == 2 + # a server that doesn't keep its session gets a new one each call, as before + monkeypatch.setattr(mcp_client, "_kept", {}) + plain = {**tools["browser__browser_snapshot"]["spec"]["run"], "keep": None} + assert len({mcp_client.call(plain, {})["text"].split()[0] for _ in range(2)}) == 2 and not mcp_client._kept diff --git a/fastapi_backend/tests/api/test_monitoring.py b/fastapi_backend/tests/api/test_monitoring.py index b9851efab..d71fda021 100644 --- a/fastapi_backend/tests/api/test_monitoring.py +++ b/fastapi_backend/tests/api/test_monitoring.py @@ -41,7 +41,7 @@ def test_turning_monitoring_on_and_off(client, db, control): r = client.put("/api/v1/admin/monitoring", json={"on": True}, headers=h) assert r.status_code == 202 and r.json()["on"] is True and r.json()["state"] == "queued" req = request_of(control) - assert (req["action"], req["monitoring"], req["by"]) == ("services", "on", "root@x.io") + assert (req["action"], req["profile"], req["turn"], req["by"]) == ("services", "monitoring", "on", "root@x.io") # /metrics turned on with a token made for it, and the token where the monitoring Prometheus reads it t = client.app.state.settings.current()["telemetry"] assert t["prometheus"] is True and len(t["prometheus_token"]) == 64 @@ -52,7 +52,7 @@ def test_turning_monitoring_on_and_off(client, db, control): # the agent picks it up and says how it went (control / "inbox" / "upgrade").unlink() - (control / "services-status").write_text(f"id={req['id']}\nmonitoring=on\nstate=done\ngrafana_port=3005\ncode=0\n") + (control / "services-status").write_text(f"id={req['id']}\nprofile=monitoring\nstate=done\ngrafana_port=3005\ncode=0\n") v = client.get("/api/v1/admin/monitoring", headers=h).json() assert (v["state"], v["grafana_port"], v["error"]) == ("done", 3005, None) @@ -62,7 +62,7 @@ def test_turning_monitoring_on_and_off(client, db, control): # off stops the containers and turns /metrics off again, since monitoring turned it on r = client.put("/api/v1/admin/monitoring", json={"on": False}, headers=h) - assert r.status_code == 202 and request_of(control)["monitoring"] == "off" + assert r.status_code == 202 and request_of(control)["turn"] == "off" assert client.app.state.settings.current()["telemetry"]["prometheus"] is False diff --git a/fastapi_backend/tests/api/test_services.py b/fastapi_backend/tests/api/test_services.py new file mode 100644 index 000000000..4e5a7788c --- /dev/null +++ b/fastapi_backend/tests/api/test_services.py @@ -0,0 +1,122 @@ +"""Lens's optional containers (the MCP runner, the browser, monitoring), turned on and off from the web app through the +machine's upgrade agent (services.py, packaging/host/lens-upgrade.sh; the agent itself: packaging/host/test-services.sh).""" + +from __future__ import annotations + +import socket + +import pytest + +from tests.helpers import login, make_user + + +@pytest.fixture +def control(tmp_path, monkeypatch): + """The control folder install.sh shares with the containers, with an agent watching it.""" + d = tmp_path / "control" + (d / "inbox").mkdir(parents=True) + (d / "agent").write_text("kind=systemd\n") + monkeypatch.setenv("LENS_CONTROL_DIR", str(d)) + return d + + +def admin(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def request_of(control): + return dict(line.split("=", 1) for line in (control / "inbox" / "upgrade").read_text().splitlines()) + + +def done(control, rid, state="done", **more): + """The agent picks the request up and says how it went.""" + (control / "inbox" / "upgrade").unlink(missing_ok=True) + extra = "".join(f"{k}={v}\n" for k, v in more.items()) + (control / "services-status").write_text(f"id={rid}\nstate={state}\n{extra}") + + +def of(client, h, profile): + return next(s for s in client.get("/api/v1/admin/services", headers=h).json()["services"] if s["profile"] == profile) + + +def test_each_profile_is_turned_on_and_off_on_its_own(client, db, control): + h = admin(client, db) + got = client.get("/api/v1/admin/services", headers=h).json()["services"] + assert [(s["profile"], s["on"], s["state"], s["agent"]) for s in got] == [ + ("mcp", False, None, "systemd"), + ("browser", False, None, "systemd"), + ("monitoring", False, None, "systemd"), + ] + assert "--profile browser up -d" in got[1]["manual"] + + r = client.put("/api/v1/admin/services/browser", json={"on": True}, headers=h) + assert r.status_code == 202 and (r.json()["on"], r.json()["state"]) == (True, "queued") + req = request_of(control) + assert (req["action"], req["profile"], req["turn"], req["by"]) == ("services", "browser", "on", "root@x.io") + # one request at a time, whichever the profile + busy = client.put("/api/v1/admin/services/mcp", json={"on": True}, headers=h) + assert busy.status_code == 409 and "busy" in busy.json()["detail"] + + # picked up, but still starting: the others wait for it + done(control, req["id"], "running") + busy = client.put("/api/v1/admin/services/mcp", json={"on": True}, headers=h) + assert busy.status_code == 409 and "still changing the browser" in busy.json()["detail"] + + done(control, req["id"]) + assert of(client, h, "browser")["state"] == "done" + assert client.put("/api/v1/admin/services/mcp", json={"on": True}, headers=h).status_code == 202 + mcp = request_of(control) + assert (mcp["profile"], mcp["turn"]) == ("mcp", "on") + done(control, mcp["id"], "failed", code=2, error="mcp runs in Docker") + # the status file is about the MCP runner now; the browser keeps what the agent said about it + assert (of(client, h, "browser")["on"], of(client, h, "browser")["state"]) == (True, "done") + assert (of(client, h, "mcp")["state"], of(client, h, "mcp")["error"]) == ("failed", "mcp runs in Docker") + + # off, the same way + r = client.put("/api/v1/admin/services/browser", json={"on": False}, headers=h) + assert r.status_code == 202 and request_of(control)["turn"] == "off" and r.json()["on"] is False + + +def test_monitoring_through_the_same_switch(client, db, control): + h = admin(client, db) + r = client.put("/api/v1/admin/services/monitoring", json={"on": True}, headers=h) + assert r.status_code == 202 and request_of(control)["profile"] == "monitoring" + # as PUT /admin/monitoring: /metrics on with a token + assert client.app.state.settings.current()["telemetry"]["prometheus"] is True + assert client.get("/api/v1/admin/monitoring", headers=h).json()["on"] is True + + +def test_no_agent_shows_the_line_to_run(client, db, monkeypatch): + monkeypatch.delenv("LENS_CONTROL_DIR", raising=False) + h = admin(client, db) + b = of(client, h, "browser") + assert b["agent"] is None and b["manual"].endswith("--profile browser up -d") + r = client.put("/api/v1/admin/services/browser", json={"on": True}, headers=h) + assert r.status_code == 409 and "run the line shown" in r.json()["detail"] + assert client.put("/api/v1/admin/services/fedora", json={"on": True}, headers=h).status_code == 404 + + +def test_only_admins(client, db, control): + make_user(db, "me@x.io", "my password 12") + h = login(client, "me@x.io", "my password 12") + assert client.get("/api/v1/admin/services", headers=h).status_code == 403 + assert client.put("/api/v1/admin/services/browser", json={"on": True}, headers=h).status_code == 403 + + +def test_whether_it_answers(client, db, monkeypatch): + """A container Lens reaches says whether it runs; one never turned on here is on while it does (set up by hand).""" + h = admin(client, db) + with socket.socket() as srv: + srv.bind(("127.0.0.1", 0)) + srv.listen() + monkeypatch.setenv("LENS_BROWSER_URL", f"http://127.0.0.1:{srv.getsockname()[1]}") + b = of(client, h, "browser") + assert (b["running"], b["on"]) == (True, True) + with socket.socket() as gone: + gone.bind(("127.0.0.1", 0)) + monkeypatch.setenv("LENS_BROWSER_URL", f"http://127.0.0.1:{gone.getsockname()[1]}") + assert (of(client, h, "browser")["running"], of(client, h, "browser")["on"]) == (False, False) + monkeypatch.delenv("LENS_BROWSER_URL") + assert of(client, h, "browser")["running"] is None + assert of(client, h, "monitoring")["running"] is None diff --git a/fastapi_backend/tests/domain/test_netguard_forwarder.py b/fastapi_backend/tests/domain/test_netguard_forwarder.py new file mode 100644 index 000000000..e223b2a3f --- /dev/null +++ b/fastapi_backend/tests/domain/test_netguard_forwarder.py @@ -0,0 +1,105 @@ +"""netguard on its own (the browser-proxy service): the browser's only way out, to public web addresses only, each +connection with its own allowance of bytes, for as long as it runs.""" + +from __future__ import annotations + +import http.server +import pathlib +import socket +import subprocess +import sys +import threading +import time + +import pytest + +from app.domain import netguard + + +def send(proxy, raw): + """One request to the proxy; what came back.""" + with socket.create_connection(proxy, timeout=10) as s: + s.sendall(raw) + out = b"" + while chunk := s.recv(65536): + out += chunk + return out + + +@pytest.fixture +def forwarder(): + srv = netguard.forwarder("127.0.0.1:0", max_bytes=1000) + threading.Thread(target=srv.serve_forever, daemon=True).start() + yield srv + srv.shutdown() + srv.server_close() + + +def test_only_public_addresses_on_the_webs_ports(forwarder, capsys): + at = forwarder.server_address + for ask in ( + b"CONNECT 127.0.0.1:443 HTTP/1.1\r\n\r\n", # this machine + b"CONNECT 169.254.169.254:80 HTTP/1.1\r\n\r\n", # a cloud's metadata address + b"GET http://10.0.0.7/admin HTTP/1.1\r\nHost: 10.0.0.7\r\n\r\n", # the local network + b"CONNECT 1.1.1.1:22 HTTP/1.1\r\n\r\n", # a public address, but not a web port + b"GET /page HTTP/1.1\r\nHost: example.org\r\n\r\n", # not a proxy request + ): + assert send(at, ask).startswith(b"HTTP/1.1 403"), ask + said = capsys.readouterr().out + assert "refused CONNECT 127.0.0.1:443" in said and "port 22 isn't a web port" in said + assert len(forwarder.refused) == 5 and list(forwarder.asked) == [] # nothing piles up while it runs + + +def test_each_connection_has_its_own_allowance(forwarder, monkeypatch): + class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - http.server's name + body = b"x" * 600 + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + site = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Page) + threading.Thread(target=site.serve_forever, daemon=True).start() + port = site.server_address[1] + monkeypatch.setattr(netguard, "PORTS", netguard.PORTS | {port}) # as if it were 80 + forwarder.allow = lambda ip: True # as if 127.0.0.1 were public + try: + ask = f"GET http://127.0.0.1:{port}/ HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n".encode() + # 600 bytes back (and the head), twice: more than the 1000 allowed, but each connection on its own + for _ in range(2): + got = send(forwarder.server_address, ask) + assert got.startswith(b"HTTP/1.0 200") and got.endswith(b"x" * 600) + finally: + site.shutdown() + site.server_close() + + +def test_it_runs_on_its_own(tmp_path): + """As the browser-proxy service runs it: the file alone, with the standard library.""" + src = pathlib.Path(netguard.__file__) + (tmp_path / "netguard.py").write_text(src.read_text()) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + p = subprocess.Popen( + [sys.executable, "-I", str(tmp_path / "netguard.py"), "--listen", f"127.0.0.1:{port}"], + cwd=tmp_path, + stdout=subprocess.PIPE, + text=True, + ) + try: + assert "Forwarding to public addresses" in (p.stdout.readline() if p.stdout else "") + for _ in range(50): + try: + got = send(("127.0.0.1", port), b"CONNECT 127.0.0.1:443 HTTP/1.1\r\n\r\n") + break + except OSError: + time.sleep(0.1) + assert got.startswith(b"HTTP/1.1 403") + finally: + p.terminate() + p.wait(10) diff --git a/nextjs-frontend/__tests__/mcp-server-ui.test.tsx b/nextjs-frontend/__tests__/mcp-server-ui.test.tsx index 8377f1b0f..0b604ee01 100644 --- a/nextjs-frontend/__tests__/mcp-server-ui.test.tsx +++ b/nextjs-frontend/__tests__/mcp-server-ui.test.tsx @@ -3,7 +3,7 @@ import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import "@testing-library/jest-dom"; import type { ReactNode } from "react"; -import { Extensions } from "@/app/openapi-client"; +import { Extensions, Updates } from "@/app/openapi-client"; import { ExtensionEditor } from "@/components/extensions/extension-editor"; import { McpServerForm } from "@/components/extensions/mcp-server-form"; import { TooltipProvider } from "@/components/ui/tooltip"; @@ -18,7 +18,9 @@ jest.mock("@/app/openapi-client", () => ({ testExtension: jest.fn(), updateExtension: jest.fn(), deleteExtension: jest.fn(), + mcpPresets: jest.fn(), }, + Updates: { listServices: jest.fn(), setService: jest.fn() }, })); jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); const push = jest.fn(); @@ -53,10 +55,109 @@ const fetcher = { fill: [], }; +const service = { + profile: "browser", + label: "Browser", + description: "A headless Chromium the assistant drives", + on: false, + running: false, + state: null, + agent: "systemd", + manual: "docker compose -f docker-compose.prod.yml --profile browser up -d", +}; +const browser = { + name: "browser", + label: "Browser", + description: "Opens web pages, reads them, clicks and types", + spec: { + url: "http://browser:8931/mcp", + keep: true, + off: ["browser_run_code_unsafe"], + ask: { browser_navigate: false }, + }, + fill: [], + needs: "browser", +}; +const github = { + name: "github", + label: "GitHub", + description: "Issues, pull requests and code on GitHub", + spec: { url: "https://api.githubcopilot.com/mcp/", headers: { Authorization: "Bearer ${GITHUB_TOKEN}" } }, + fill: ["GITHUB_TOKEN"], + needs: null, + service: null, +}; + describe("adding an MCP server", () => { beforeEach(() => { jest.clearAllMocks(); archive.admin = true; + m(Extensions.mcpPresets).mockReturnValue(ok({ presets: [] })); + }); + + it("adds the browser with one click once its container runs", async () => { + m(Extensions.mcpPresets).mockReturnValue( + ok({ presets: [{ ...browser, service: { ...service, on: true, running: true, state: "done" } }, github] }), + ); + m(Extensions.createExtension).mockReturnValue(ok({ ok: true, id: 7 })); + wrap(); + fireEvent.click(await screen.findByRole("button", { name: "Add Browser" })); + await waitFor(() => expect(push).toHaveBeenCalledWith("/extensions/7")); + // with what the form doesn't show: one session for its calls, running code off, going to a page without asking + expect(m(Extensions.createExtension).mock.calls[0][0].body.manifest).toEqual({ + name: "browser", + kind: "mcp", + description: "Opens web pages, reads them, clicks and types", + url: "http://browser:8931/mcp", + keep: true, + off: ["browser_run_code_unsafe"], + ask: { browser_navigate: false }, + }); + }); + + it("turns the browser's container on first, when it isn't running", async () => { + m(Extensions.mcpPresets).mockReturnValue(ok({ presets: [{ ...browser, service }] })); + m(Updates.listServices).mockReturnValue(ok({ services: [service] })); + m(Updates.setService).mockReturnValue(ok({ ...service, on: true, state: "queued" })); + wrap(); + expect(await screen.findByRole("button", { name: "Add Browser" })).toBeDisabled(); + const on = await screen.findByRole("switch", { name: "Run the browser on this machine" }); + expect(screen.getByText(/^Off\. A headless Chromium/)).toBeInTheDocument(); + fireEvent.click(on); + await waitFor(() => expect(m(Updates.setService)).toHaveBeenCalled()); + expect(m(Updates.setService).mock.calls[0][0]).toMatchObject({ path: { profile: "browser" }, body: { on: true } }); + }); + + it("says what to run on a machine without an upgrade agent", async () => { + const none = { ...service, agent: null }; + m(Extensions.mcpPresets).mockReturnValue(ok({ presets: [{ ...browser, service: none }] })); + m(Updates.listServices).mockReturnValue(ok({ services: [none] })); + wrap(); + expect(await screen.findByText(service.manual)).toBeInTheDocument(); + expect(screen.getByRole("switch", { name: "Run the browser on this machine" })).toBeDisabled(); + }); + + it("sets GitHub up with the token typed in its place", async () => { + m(Extensions.mcpPresets).mockReturnValue(ok({ presets: [github] })); + m(Extensions.createExtension).mockReturnValue(ok({ ok: true, id: 8 })); + wrap(); + fireEvent.click(await screen.findByRole("button", { name: "Set up GitHub" })); + expect(screen.getByLabelText("Name")).toHaveValue("github"); + const token = screen.getByLabelText("GITHUB_TOKEN"); + expect(token).toHaveFocus(); + expect(token).toHaveAttribute("type", "password"); + expect(screen.getByText(/Fill in GITHUB_TOKEN/)).toBeInTheDocument(); + fireEvent.change(token, { target: { value: "ghp_$&x" } }); + expect(screen.queryByText(/Fill in GITHUB_TOKEN/)).not.toBeInTheDocument(); + fireEvent.submit(token); + await waitFor(() => expect(push).toHaveBeenCalledWith("/extensions/8")); + expect(m(Extensions.createExtension).mock.calls[0][0].body.manifest).toEqual({ + name: "github", + kind: "mcp", + description: "Issues, pull requests and code on GitHub", + url: "https://api.githubcopilot.com/mcp/", + headers: { Authorization: "Bearer ghp_$&x" }, + }); }); it("reads a pasted config, fills in its token, tests it and saves it with Enter", async () => { diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index b510f7043..56ecba288 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1962,6 +1962,9 @@ export type { ListSensorsData, ListSensorsResponse, ListSensorsResponses, + ListServicesData, + ListServicesResponse, + ListServicesResponses, ListSessionsData, ListSessionsResponse, ListSessionsResponses, @@ -2115,6 +2118,11 @@ export type { MatterbridgeStatus, McpConfig, McpConfigText, + McpPreset, + McpPresets, + McpPresetsData, + McpPresetsResponse, + McpPresetsResponses, McpServerFound, McpServerSkipped, McpServerTools, @@ -2925,6 +2933,9 @@ export type { SensorType, SensorUpdate, SeriesPoint, + Service, + ServiceRequest, + Services, SetBudgetData, SetBudgetError, SetBudgetErrors, @@ -2965,6 +2976,11 @@ export type { SetRecordingContentTypeErrors, SetRecordingContentTypeResponse, SetRecordingContentTypeResponses, + SetServiceData, + SetServiceError, + SetServiceErrors, + SetServiceResponse, + SetServiceResponses, SetupAdmin, SetupAnalyticsView, SetupBudget, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index b17155e7b..895fb8dce 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -986,6 +986,8 @@ import type { ListSegmentEditsResponses, ListSensorsData, ListSensorsResponses, + ListServicesData, + ListServicesResponses, ListSessionsData, ListSessionsResponses, ListSharesData, @@ -1063,6 +1065,8 @@ import type { MarkRecordingNsfwData, MarkRecordingNsfwErrors, MarkRecordingNsfwResponses, + McpPresetsData, + McpPresetsResponses, MeData, MeResponses, MergeEntitiesData, @@ -1438,6 +1442,9 @@ import type { SetRecordingContentTypeData, SetRecordingContentTypeErrors, SetRecordingContentTypeResponses, + SetServiceData, + SetServiceErrors, + SetServiceResponses, SetupData, SetupErrors, SetupResponses, @@ -3673,6 +3680,41 @@ export class Updates { }); } + /** + * List Services + * + * Lens's optional containers (the MCP runner, the browser, monitoring): whether each is on and answering, and + * how the machine's last change went. + */ + public static listServices( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/admin/services", + ...options, + }); + } + + /** + * Set Service + * + * Have this machine start or stop one of them: its upgrade agent adds the profile to COMPOSE_PROFILES in .env, or + * takes it out, and starts or stops its containers. Monitoring also turns /metrics on, as PUT /admin/monitoring + * does. GET /admin/services follows along; without an agent, 409 says the line to run on the machine instead. + */ + public static setService( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/admin/services/{profile}", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * List Backups * @@ -9732,6 +9774,21 @@ export class Extensions { }); } + /** + * Mcp Presets + * + * MCP servers ready to add (admins): the browser Lens can run next to it, with whether its container is on and + * running (`service`), and GitHub's own server, with the token to type in (`fill`). + */ + public static mcpPresets( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/extensions/mcp/presets", + ...options, + }); + } + /** * Delete Extension * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 5b94dccc9..d076d849d 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -10512,6 +10512,62 @@ export type McpConfigText = { text: string; }; +/** + * McpPreset + */ +export type McpPreset = { + /** + * Name + * + * what its tools are called by: name__tool + */ + name: string; + /** + * Label + */ + label: string; + /** + * Description + */ + description: string; + /** + * Spec + * + * the server, as an MCP extension keeps it + */ + spec: { + [key: string]: unknown; + }; + /** + * Fill + * + * values written as ${NAME} in spec, to type in first + */ + fill?: Array; + /** + * Needs + * + * the optional profile it runs in (GET /admin/services), if any + */ + needs?: string | null; + /** + * that profile: whether it's on and running + */ + service?: Service | null; + [key: string]: unknown; +}; + +/** + * McpPresets + */ +export type McpPresets = { + /** + * Presets + */ + presets: Array; + [key: string]: unknown; +}; + /** * McpServerFound */ @@ -18352,6 +18408,92 @@ export type SeriesPoint = { [key: string]: unknown; }; +/** + * Service + */ +export type Service = { + /** + * Profile + * + * the Docker Compose profile: mcp (the MCP runner), browser, monitoring + */ + profile: string; + /** + * Label + */ + label: string; + /** + * Description + */ + description: string; + /** + * On + * + * as last asked here; never asked here: whether it runs + */ + on: boolean; + /** + * Running + * + * whether it answers where Lens reaches it now; null: Lens can't tell + */ + running?: boolean | null; + /** + * State + * + * how the machine's last change went + */ + state?: "queued" | "running" | "done" | "failed" | null; + /** + * Error + */ + error?: string | null; + /** + * Asked At + */ + asked_at?: string | null; + /** + * Finished At + */ + finished_at?: string | null; + /** + * Agent + * + * what changes it on this machine (systemd, launchd); none: run `manual` there + */ + agent?: string | null; + /** + * Manual + * + * the line that turns it on by hand, on the machine Lens runs on + */ + manual: string; + [key: string]: unknown; +}; + +/** + * ServiceRequest + */ +export type ServiceRequest = { + /** + * On + * + * start (true) or stop (false) the profile's containers + */ + on: boolean; +}; + +/** + * Services + */ +export type Services = { + /** + * Services + */ + services: Array; + [key: string]: unknown; +}; + /** * SetupAdmin */ @@ -26053,6 +26195,52 @@ export type SetMonitoringResponses = { export type SetMonitoringResponse = SetMonitoringResponses[keyof SetMonitoringResponses]; +export type ListServicesData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/admin/services"; +}; + +export type ListServicesResponses = { + /** + * Successful Response + */ + 200: Services; +}; + +export type ListServicesResponse = ListServicesResponses[keyof ListServicesResponses]; + +export type SetServiceData = { + body: ServiceRequest; + path: { + /** + * Profile + */ + profile: string; + }; + query?: never; + url: "/api/v1/admin/services/{profile}"; +}; + +export type SetServiceErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetServiceError = SetServiceErrors[keyof SetServiceErrors]; + +export type SetServiceResponses = { + /** + * Successful Response + */ + 202: Service; +}; + +export type SetServiceResponse = SetServiceResponses[keyof SetServiceResponses]; + export type ListBackupsData = { body?: never; path?: never; @@ -38054,6 +38242,22 @@ export type TestMcpServerResponses = { export type TestMcpServerResponse = TestMcpServerResponses[keyof TestMcpServerResponses]; +export type McpPresetsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/extensions/mcp/presets"; +}; + +export type McpPresetsResponses = { + /** + * Successful Response + */ + 200: McpPresets; +}; + +export type McpPresetsResponse = McpPresetsResponses[keyof McpPresetsResponses]; + export type DeleteExtensionData = { body?: never; path: { diff --git a/nextjs-frontend/components/admin/service-switch.tsx b/nextjs-frontend/components/admin/service-switch.tsx new file mode 100644 index 000000000..f0225d62f --- /dev/null +++ b/nextjs-frontend/components/admin/service-switch.tsx @@ -0,0 +1,76 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { Updates } from "@/app/openapi-client"; +import type { Service } from "@/app/openapi-client/types.gen"; +import { Banner } from "@/components/ui/banner"; +import { Switch } from "@/components/ui/field"; +import { data, useApiClient } from "@/lib/api/browser"; + +/** What the switch says under it: waiting for the machine, starting, stopping, running, or off. */ +export function serviceNote(s: Service): string { + if (s.state === "queued") return "Waiting for the machine to pick it up…"; + if (s.state === "running") return s.on ? `Starting the ${s.label.toLowerCase()}…` : "Stopping it…"; + if (s.on && s.running === false) + return s.state === "done" ? "On, but not answering yet: it may still be starting." : "On, but not answering."; + if (s.on) return "Running."; + return `Off. ${s.description}.`; +} + +/** Lens's optional containers (the MCP runner, the browser, monitoring), while the machine changes one. */ +export function useServices() { + const client = useApiClient(); + return useQuery({ + queryKey: ["services"], + queryFn: () => data(Updates.listServices({ client })), + refetchInterval: (q) => + q.state.data?.services?.some( + (s) => s.state === "queued" || s.state === "running" || (s.on && s.state === "done" && s.running === false), + ) + ? 2000 + : false, + }); +} + +/** One optional container on or off, started and stopped by the machine's upgrade agent. */ +export function ServiceSwitch({ profile, label }: { profile: string; label?: string }) { + const client = useApiClient(); + const qc = useQueryClient(); + const q = useServices(); + const turn = useMutation({ + mutationFn: (on: boolean) => data(Updates.setService({ client, path: { profile }, body: { on } })), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["services"] }); + void qc.invalidateQueries({ queryKey: ["mcp-presets"] }); + if (profile === "monitoring") void qc.invalidateQueries({ queryKey: ["monitoring"] }); + }, + }); + const s = q.data?.services?.find((x) => x.profile === profile); + if (!s) return null; + const busy = turn.isPending || ((s.state === "queued" || s.state === "running") && !s.error); + return ( +
+ turn.mutate(on)} + disabled={busy || !s.agent} + label={label ?? s.label} + /> + {s.agent ? ( + {serviceNote(s)} + ) : ( + + This machine has no upgrade agent to start it (Lens wasn’t installed with install.sh, or it couldn’t set one + up). On the machine, run {s.manual} + + )} + {s.error && ( + + {s.error} + + )} + {turn.isError && {turn.error.message}} +
+ ); +} diff --git a/nextjs-frontend/components/extensions/mcp-server-form.tsx b/nextjs-frontend/components/extensions/mcp-server-form.tsx index 5e3023444..660d602e4 100644 --- a/nextjs-frontend/components/extensions/mcp-server-form.tsx +++ b/nextjs-frontend/components/extensions/mcp-server-form.tsx @@ -1,13 +1,14 @@ "use client"; -import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { ClipboardPaste, Play, Plus, Server, X } from "lucide-react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ClipboardPaste, Play, Plus, Server, Sparkles, X } from "lucide-react"; import Link from "next/link"; import { useRouter } from "next/navigation"; import { useState } from "react"; import { Extensions } from "@/app/openapi-client"; -import type { McpConfig, McpServerFound } from "@/app/openapi-client/types.gen"; +import type { McpConfig, McpPreset, McpServerFound } from "@/app/openapi-client/types.gen"; +import { ServiceSwitch } from "@/components/admin/service-switch"; import { Badge } from "@/components/ui/badge"; import { Button, IconButton } from "@/components/ui/button"; import { Field, Input, Textarea } from "@/components/ui/field"; @@ -31,7 +32,11 @@ export type Draft = { command: string; args: string; env: Pair[]; + /** Values written ${NAME}, to type in: put in their place when it's sent. */ fill: string[]; + values: Record; + /** What the form doesn't show, kept as it came (a ready-made server's keep, off and ask). */ + extra: Record; }; type Tool = { name: string; description?: string; read_only?: boolean }; @@ -45,33 +50,50 @@ export const EMPTY: Draft = { args: "", env: [], fill: [], + values: {}, + extra: {}, }; +/** What the form shows of a server; the rest of its spec is kept as it came. */ +const SHOWN = ["url", "headers", "command", "args", "env"]; +const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +/** A value with the ${NAME}s typed in put in their place (as other apps write them: ${NAME}, ${input:NAME}). */ +function filled(v: string, values: Record) { + return Object.entries(values).reduce( + (out, [k, x]) => + x ? out.replace(new RegExp(`\\$\\{(?:input:|env:)?${escape(k)}(?::-[^}]*)?\\}`, "g"), () => x) : out, + v, + ); +} + const pairs = (o: unknown): Pair[] => Object.entries((o ?? {}) as Record).map(([key, value]) => ({ key, value: String(value) })); -const object = (ps: Pair[]) => Object.fromEntries(ps.filter((p) => p.key.trim()).map((p) => [p.key.trim(), p.value])); +const object = (ps: Pair[], values: Record) => + Object.fromEntries(ps.filter((p) => p.key.trim()).map((p) => [p.key.trim(), filled(p.value, values)])); -/** A server as the API takes it: {url, headers}, or {command, args, env}. */ +/** A server as the API takes it: {url, headers}, or {command, args, env}, with what the form doesn't show. */ export function specOf(d: Draft): Record { if (d.how === "url") { - const headers = object(d.headers); - return { url: d.url.trim(), ...(Object.keys(headers).length ? { headers } : {}) }; + const headers = object(d.headers, d.values); + return { url: filled(d.url.trim(), d.values), ...(Object.keys(headers).length ? { headers } : {}), ...d.extra }; } const args = d.args .split("\n") - .map((a) => a.trim()) + .map((a) => filled(a.trim(), d.values)) .filter(Boolean); - const env = object(d.env); + const env = object(d.env, d.values); return { command: d.command.trim(), ...(args.length ? { args } : {}), ...(Object.keys(env).length ? { env } : {}), + ...d.extra, }; } /** A server read from a pasted config, ready to change in the form. */ export function draftOf(f: McpServerFound): Draft { const s = f.spec as { url?: string; headers?: unknown; command?: string; args?: string[]; env?: unknown }; + const extra = Object.fromEntries(Object.entries(f.spec).filter(([k]) => !SHOWN.includes(k))); return { name: f.name, description: f.description ?? "", @@ -82,6 +104,8 @@ export function draftOf(f: McpServerFound): Draft { args: (s.args ?? []).join("\n"), env: pairs(s.env), fill: f.fill ?? [], + values: {}, + extra, }; } @@ -146,7 +170,7 @@ function Pairs({ {placeholder && (

- Fill in {placeholder}: its app took it from its own settings. + Fill in {placeholder}: type it in above, or here in its place.

)} @@ -161,6 +185,70 @@ function Pairs({ ); } +/** Servers Lens knows, ready to add: one click (the browser, once its container runs), or a token to type in first + * (GitHub), which `onPick` puts in the form. */ +function ReadyMade({ onPick }: { onPick: (f: McpPreset) => void }) { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const router = useRouter(); + const q = useQuery({ + queryKey: ["mcp-presets"], + queryFn: () => data(Extensions.mcpPresets({ client })), + // while a container it needs starts + refetchInterval: (q) => + q.state.data?.presets?.some((p) => p.service?.on && p.service.running === false) ? 3000 : false, + }); + const add = useMutation({ + mutationFn: async (p: McpPreset) => + (await data(Extensions.createExtension({ client, body: { manifest: manifestOf(draftOf(p)) } }))).id, + onSuccess: (id) => { + void qc.invalidateQueries({ queryKey: ["extensions"] }); + toast({ tone: "green", title: "Added to the assistant" }); + router.push(`/extensions/${id}`); + }, + }); + const presets = q.data?.presets ?? []; + if (!presets.length) return null; + return ( + +
    + {presets.map((p) => { + const stopped = Boolean(p.needs) && p.service?.running === false; + return ( +
  • +
    + +
    +
    {p.label}
    +
    {p.description}
    +
    + {p.fill?.length ? ( + + ) : ( + + )} +
    + {stopped && p.needs && ( + + )} + {add.isError && add.variables?.name === p.name && ( +

    + {add.error.message} +

    + )} +
  • + ); + })} +
+
+ ); +} + /** "Add MCP server": a short form (an address and its headers, or a command with its args and env), or a config * pasted from another app. Test asks the server for its tools; Enter saves. Tokens are sealed when saved. */ export function McpServerForm() { @@ -240,6 +328,7 @@ export function McpServerForm() { title="Add MCP server" meta="Give the assistant another service's tools: a server on the network, or a program run in the MCP runner" /> +
)} + {draft.fill.length > 0 && ( +
+ To type in + {draft.fill.map((f, i) => ( + edit({ values: { ...draft.values, [f]: ev.target.value } })} + /> + ))} +

+ Its settings name these instead of giving them, as its app fills them in itself; what you type goes + in their place. +

+
+ )} {draft.how === "url" ? ( <> @@ -386,7 +497,7 @@ export function McpServerForm() { label="Headers" add="Add a header" rows={draft.headers} - fill={draft.fill} + fill={draft.fill.filter((f) => !draft.values[f])} onChange={(headers) => edit({ headers })} /> @@ -407,6 +518,7 @@ export function McpServerForm() { /> )} + {({ id, describedBy }) => (