From 036880efc140718d99f734245ff57941ee3b7a7a Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 07:36:48 +0000 Subject: [PATCH 1/2] feat(mesh): another machine joins as a node with the install line and lens node Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GTrHPwWHtEvFJFq3M1YpH4 --- changelog.d/mesh-node.md | 5 + docker-compose.node.yml | 61 ++++ docs/mesh.md | 38 +- fastapi_backend/app/api/v1/routes/mesh.py | 3 +- fastapi_backend/app/cli.py | 4 + fastapi_backend/app/core/middleware.py | 15 +- fastapi_backend/app/domain/mesh.py | 15 + fastapi_backend/app/domain/mesh_node.py | 341 ++++++++++++++++++ fastapi_backend/app/schemas/mesh.py | 1 + fastapi_backend/tests/api/test_mesh.py | 1 + fastapi_backend/tests/api/test_mesh_node.py | 196 ++++++++++ install.sh | 91 ++++- mesh/sidecar.py | 2 +- .../app/openapi-client/types.gen.ts | 6 + nextjs-frontend/openapi.json | 8 +- threat-model.json | 4 +- threat-model.md | 4 +- threat-model.yaml | 4 +- 18 files changed, 775 insertions(+), 24 deletions(-) create mode 100644 changelog.d/mesh-node.md create mode 100644 docker-compose.node.yml create mode 100644 fastapi_backend/app/domain/mesh_node.py create mode 100644 fastapi_backend/tests/api/test_mesh_node.py diff --git a/changelog.d/mesh-node.md b/changelog.d/mesh-node.md new file mode 100644 index 000000000..cab5f1786 --- /dev/null +++ b/changelog.d/mesh-node.md @@ -0,0 +1,5 @@ +- **Joining a node** (in progress). Another machine joins with the install line and `LENS_JOIN` and `LENS_JOIN_TOKEN`: + it asks nothing, publishes no port, and runs Nebula and `lens node`, which joins once and then follows the primary's + config, renews its certificate and reports over the mesh. Adding a node token now returns that install line. Worker + nodes running jobs and the Settings pages follow in their own pull requests + ([On a node](docs/mesh.md#on-a-node)). diff --git a/docker-compose.node.yml b/docker-compose.node.yml new file mode 100644 index 000000000..52f8d84d2 --- /dev/null +++ b/docker-compose.node.yml @@ -0,0 +1,61 @@ +# A machine on another Lens install's mesh (docs/mesh.md, Nodes): what install.sh starts with LENS_JOIN and +# LENS_JOIN_TOKEN. No database, no web app, nothing published: `mesh` runs Nebula, and `node` (`lens node`) joins once, +# then follows the primary's config over the mesh and reports every minute. +# LENS_JOIN=http://lens.home:3000 LENS_JOIN_TOKEN=lensjoin1.… docker compose -p lens-node -f docker-compose.node.yml up -d --build +name: lens-node + +services: + mesh: + image: python:3.12-slim # the plain image: the script is the repo's, Nebula is fetched with its checksum checked + command: ["python", "-I", "/lens-mesh/sidecar.py"] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + cap_add: [NET_ADMIN] + devices: ["/dev/net/tun:/dev/net/tun"] + read_only: true + tmpfs: [/tmp] + mem_limit: 256m + pids_limit: 256 + environment: + LENS_MESH_DIR: /mesh + LENS_MESH_STATE: /state + volumes: + - ./mesh/sidecar.py:/lens-mesh/sidecar.py:ro + - mesh-data:/mesh + - mesh-state:/state + restart: unless-stopped + + node: + image: lens-backend:${LENS_VERSION:-prod} + build: + context: fastapi_backend + target: ${LENS_TARGET:-lean} + command: ["lens", "node"] + # the mesh container's network: what it reaches over Nebula, this reaches too, and nothing listens outside it + network_mode: service:mesh + depends_on: [mesh] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + tmpfs: [/tmp] + mem_limit: 512m + healthcheck: + disable: true # the image's check is the API's, which a node doesn't run + environment: + LENS_MESH_DIR: /mesh + LENS_NODE_DIR: /data/node + LENS_JOIN: ${LENS_JOIN:-} + LENS_JOIN_TOKEN: ${LENS_JOIN_TOKEN:-} # spent on the first start; never written to .env + LENS_NODE_NAME: ${LENS_NODE_NAME:-} + # a unit of an auto scaling group: its name, and the steps its worker runs (docs/scaling.md) + LENS_SCALING_UNIT: ${LENS_SCALING_UNIT:-} + LENS_WORKER_STEPS: ${LENS_WORKER_STEPS:-} + volumes: + - mesh-data:/mesh + - node-data:/data + restart: unless-stopped + +volumes: + mesh-data: + mesh-state: + node-data: diff --git a/docs/mesh.md b/docs/mesh.md index 75887bd16..e25bd337b 100644 --- a/docs/mesh.md +++ b/docs/mesh.md @@ -2,9 +2,9 @@ Status: **in progress**. Built: Lens as the mesh's certificate authority, join tokens, joining without sending the token, each machine's Nebula config, a node's renewals and reports, removing a node, the `mesh` settings, the API -under `/api/v1/mesh`, and Nebula running on the primary in the `mesh` container, relaying the database and the API. -Coming in their own pull requests: `lens node` and the install line that joins another machine, Settings › Network and -Nodes in the web app, and running code on compute nodes. +under `/api/v1/mesh`, Nebula running on the primary in the `mesh` container, relaying the database and the API, and +`lens node` with the install line that joins another machine. Coming in their own pull requests: worker nodes running +jobs, Settings › Network and Nodes in the web app, and running code on compute nodes. Lens runs on one machine until you add another: a Mac with a GPU to transcribe on, a box to run the assistant's code on, a server with more disk. Those machines talk to this install over a private network of Lens's own, a @@ -38,7 +38,10 @@ default), and withdrawn before that. The machine joins at this install's address curl -fsSL https://raw.githubusercontent.com/adeelahmad/lense/main/install.sh | LENS_JOIN=http://lens.home:3000 LENS_JOIN_TOKEN=lensjoin1.… sh ``` -*(the install line's node path lands in its own pull request; the API it calls is here)* +`LENS_JOIN` is this install's address, the web app's (the one Settings › Nodes shows); with it the install line makes +the machine a node (below) instead of installing Lens. `LENS_NODE_NAME` names it (default: the machine's own name, or +for a unit an [auto scaling group](scaling.md) starts, `LENS_SCALING_UNIT`, which is kept with `LENS_WORKER_STEPS` for +the node's worker). `POST /api/v1/mesh/tokens` answers with the whole line (`install_line`). The token never crosses the wire. The machine makes its own Nebula key pair (`nebula-cert keygen`; the private key never leaves it) and sends `POST /api/v1/mesh/join` with the token's id, a random nonce, its name, its public key and @@ -63,7 +66,30 @@ Over the mesh, with its node token (`Authorization: Bearer lensnode1.…`): | `POST /api/v1/mesh/node/renew` | a new certificate for its public key (the same or a new one), with the groups its row has now. A node renews at a third of `mesh.cert_days` before its certificate runs out | | `POST /api/v1/mesh/node/beat` | its report, every minute: whether Nebula runs, its peers, its machine, the sandboxes it can run, Nebula's last lines. Shown on the Nodes page | -A removed node is refused on all three. +A removed node is refused on all three. Nodes make these calls at the primary's mesh address (`10.73.0.1:8080` by +default), which isn't a host name the web app is served at: the API admits it for these paths alone, while the mesh +is on, and each call still needs its node token. + +## On a node + +The install line with `LENS_JOIN` makes the machine a node, in `~/lens-node` (`LENS_DIR` for another folder), with +Docker, which it installs as for Lens. It asks nothing, publishes no port and starts two containers +(`docker-compose.node.yml`, in the smaller `lean` image by default): + +- **`mesh`**, the same container as on the primary (`mesh/sidecar.py`): it makes the node's key pair and runs Nebula + with the config it's given. Nothing listens on the node but Nebula, on a port it picks. +- **`node`** runs `lens node`, in the `mesh` container's network, with the folder the two share. It needs no database + and no settings. It turns the mesh on in that folder, so the container makes its key pair; joins with the token (as + above); keeps its node token and agent key in its own volume (`node.json`, readable only by it); and writes the + config for the container. Then, every minute, over the mesh only: it fetches its config (a removed machine's + certificate on the blocklist reaches it within the minute), renews its certificate when a third of its life is + left or the container's key changed, and sends its report. + +The install line waits until the node has joined and says its mesh address. A token that was used, expired or +withdrawn isn't tried again: the line says why, and is run again with a new token. The token is in the first start's +environment only, never in `.env`, and isn't needed again: running the line again updates the node, which stays the +same node. A node that's removed keeps asking, so an undo in Settings › Nodes brings it back; `docker compose -f +docker-compose.node.yml exec node lens node status` says what it is on the mesh, without its credentials. ## On this machine @@ -158,6 +184,8 @@ names an address that forwards to it). Nebula's own isolation is upstream's. ## Refine later +- The upgrade agent on nodes: today a node is updated by running its install line again. + - A second transport for networks that block UDP. - Rotating the CA. - Compute nodes running the assistant's sandboxes through the mesh, and their load shown on the Nodes page. diff --git a/fastapi_backend/app/api/v1/routes/mesh.py b/fastapi_backend/app/api/v1/routes/mesh.py index 2b45b9a3d..e938cc95f 100644 --- a/fastapi_backend/app/api/v1/routes/mesh.py +++ b/fastapi_backend/app/api/v1/routes/mesh.py @@ -113,7 +113,8 @@ def make_mesh_token(body: MeshTokenCreate, user: AdminWriter, request: Request, settings.save(db, request.app.state.archive.base, "mesh", {"enabled": True}, user.email) auth.audit(db, user.as_audit(), "settings.save", "mesh", ["enabled"]) auth.audit(db, user.as_audit(), "mesh.token.create", f"mesh_token:{tok['id']}", {"role": body.role, "uses": body.uses}) - return MeshTokenCreated(token=MeshToken(**tok), secret=secret, join_url=web_origin(request)) + url = web_origin(request) + return MeshTokenCreated(token=MeshToken(**tok), secret=secret, join_url=url, install_line=mesh.join_line(cfg, secret, url)) @router.delete("/tokens/{token_id}") diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index bbce5c2a7..c575eeca3 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -464,6 +464,10 @@ def main(argv=None): config, i = argv[i].split("=", 1)[1], i + 1 else: break + if i < len(argv) and argv[i] == "node": # a machine on another install's mesh: no database, no settings + from .domain import mesh_node + + return mesh_node.main(argv[i + 1 :]) if i < len(argv) and argv[i] in PLATFORM_CMDS: return platform_main(argv[i:], config) if i < len(argv) and argv[i] == "ext": # extensions' manifests as files (cli_ext.py) diff --git a/fastapi_backend/app/core/middleware.py b/fastapi_backend/app/core/middleware.py index 5abac260f..fa64003c4 100644 --- a/fastapi_backend/app/core/middleware.py +++ b/fastapi_backend/app/core/middleware.py @@ -59,15 +59,26 @@ def host_name(host: str) -> str: return (host.split("]")[0] + "]" if host.startswith("[") else host.rsplit(":", 1)[0]).lower() +def mesh_call(request: Request, cfg: dict[str, Any]) -> bool: + """A node's own call over the mesh (docs/mesh.md, After joining): addressed to the primary's mesh address, which + only the mesh reaches, on the paths nodes use, while the mesh is on. Each still carries its node token.""" + from app.domain import mesh + + if not request.url.path.startswith(mesh.NODE_PATHS) or not mesh.conf(cfg)["enabled"]: + return False + return host_name(request.headers.get("host", "")) == mesh.primary_ip(cfg) + + def install(app: FastAPI) -> None: @app.middleware("http") async def guard(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response: # The Host header must be one we serve: this stops DNS-rebinding attacks against a server on localhost. conf = getattr(request.app.state, "settings", None) if conf is not None: - server = conf.current()["server"] + current = conf.current() + server = current["server"] allowed = [h.lower() for h in server["allowed_hosts"]] - if "*" not in allowed and host_name(request.headers.get("host", "")) not in allowed: + if "*" not in allowed and host_name(request.headers.get("host", "")) not in allowed and not mesh_call(request, current): return PlainTextResponse("Invalid host header", status_code=400) frames = " ".join(server["embed_frame_ancestors"]) or "'none'" else: diff --git a/fastapi_backend/app/domain/mesh.py b/fastapi_backend/app/domain/mesh.py index ca1ded1dc..ce4750415 100644 --- a/fastapi_backend/app/domain/mesh.py +++ b/fastapi_backend/app/domain/mesh.py @@ -68,6 +68,8 @@ # node's agent serves the primary only DB_PORT, API_PORT, AGENT_PORT = 8000, 8080, 7443 INTERFACE = "lens-mesh" +# what nodes call on the primary, at its mesh address (core/middleware.py lets that address in for these alone) +NODE_PATHS = ("/api/v1/mesh/node/",) TOKEN_PREFIX = "lensjoin1." NODE_PREFIX = "lensnode1." TOKEN_HOURS = (1, 24 * 30) @@ -562,6 +564,19 @@ def make_token(db, cfg, by=None, role="worker", name=None, namespaces=None, hour return _token_view(row), f"{TOKEN_PREFIX}{tid}.{secret}" +def join_line(cfg, token, url=None): + """The install line another machine joins with (docs/mesh.md, Joining): install.sh with this install's address + (the web app's: `url`, else FRONTEND_URL) and the token. Shown once, with the token.""" + from app.config import settings as env + + repo = cfg["updates"].get("repo") or "adeelahmad/lense" + url = (url or env.FRONTEND_URL).rstrip("/") + if not re.fullmatch(r"https?://[A-Za-z0-9.\-\[\]:]+(/[A-Za-z0-9._~/-]*)?", url): + raise ValueError(f"{url!r} isn't an address a machine can join at") + parse_token(token) + return f"curl -fsSL https://raw.githubusercontent.com/{repo}/main/install.sh | LENS_JOIN={url} LENS_JOIN_TOKEN={token} sh" + + def revoke_token(db, tid): if not db.one("SELECT id FROM $r", r=R("mesh_token", int(tid))): raise KeyError(tid) diff --git a/fastapi_backend/app/domain/mesh_node.py b/fastapi_backend/app/domain/mesh_node.py new file mode 100644 index 000000000..8ceceb3ee --- /dev/null +++ b/fastapi_backend/app/domain/mesh_node.py @@ -0,0 +1,341 @@ +"""`lens node`: this machine's side of another install's mesh (docs/mesh.md, Nodes). + +The node's install line (install.sh with LENS_JOIN and LENS_JOIN_TOKEN) starts two containers: `mesh` runs Nebula +(mesh/sidecar.py, the same as on the primary) and `node` runs this, in the mesh container's network, with the folder +the two share. It needs no database and no settings of its own: + +1. **Joining, once.** It turns the shared folder's lens.json on, so the container makes this machine's key pair and + publishes the public key; then it joins at the install's address with the token (mesh.join: the token never crosses + the wire), opens the sealed answer, and keeps what it holds in node.json, which only it reads. A refused token (used, + expired, withdrawn) is not tried again: the install line is run again with a new one. +2. **After that, over the mesh only.** It writes the config it was given into lens.json, which the container runs, and + every minute asks the primary for its config as it is now (a removed node's certificate on the blocklist), renews + its certificate when a third of its life is left, and reports how it runs (mesh.beat). A node the admin removed is + refused; it keeps asking, so an undo brings it back. +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +import pathlib +import platform +import re +import secrets +import socket +import time + +import httpx + +from . import mesh, updates + +BEAT_SECONDS = 60 +TICK = 2.0 +RETRY = (5, 15, 30, 60, 120) + + +def say(msg): + print(f"lens node: {msg}", flush=True) + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def machine_name(raw=None): + """A name the mesh takes (mesh.NAME_RX) from this machine's: what isn't a letter, digit, space, dot, dash or + underscore becomes a dash.""" + name = re.sub(r"[^A-Za-z0-9 ._-]", "-", (raw or socket.gethostname() or "node").strip())[:63] + name = re.sub(r"^[^A-Za-z0-9]+", "", name) + return name or "node" + + +def machine(): + """What the Nodes page shows about this machine.""" + mem = None + try: + for line in pathlib.Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemTotal:"): + mem = int(line.split()[1]) * 1024 + break + except (OSError, ValueError, IndexError): + pass + if mem is None: + try: + mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except (ValueError, OSError, AttributeError): + pass + return {"os": platform.system(), "arch": platform.machine(), "cpus": os.cpu_count(), "memory": mem} + + +def _write(path, text, mode=0o600): + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_text(text) + tmp.chmod(mode) + tmp.replace(path) + + +class Refused(Exception): + """The primary said no for good: a join token that's used, expired or withdrawn.""" + + +class Agent: + """Joins once, then keeps lens.json as the primary says and reports every minute (the module's docstring).""" + + def __init__(self, state, shared, join_url=None, token=None, name=None, http=None): + self.state, self.shared = pathlib.Path(state), pathlib.Path(shared) + self.join_url = (join_url or "").strip().rstrip("/") or None + self.token = (token or "").strip() or None + self.name = machine_name(name) + self.http = http or httpx.Client(timeout=20, follow_redirects=False) + self.node = None + self.error = None + self.removed = False + self.refused = None + self.fails = 0 + self.next_try = 0.0 + self.last_beat = 0.0 + self._reported = None + + # ---------- what's kept ---------- + @property + def path(self): + return self.state / "node.json" + + def load(self): + if self.node is None: + try: + self.node = json.loads(self.path.read_text()) + except (OSError, ValueError): + self.node = None + return self.node + + def save(self): + self.state.mkdir(parents=True, exist_ok=True) + _write(self.path, json.dumps(self.node, indent=1, sort_keys=True)) + + def api(self, path): + n = self.node + return f"http://{n['primary']['ip']}:{n['primary']['api_port']}/api/v1/mesh{path}" + + def _headers(self): + return {"Authorization": f"Bearer {self.node['node_token']}"} + + # ---------- the shared folder ---------- + def public_key(self): + try: + text = (self.shared / "host.pub").read_text() + except OSError: + return None + return text if mesh.PUBLIC_KEY_RX.fullmatch(text.replace("\r\n", "\n")) else None + + def lens_file(self, pub=None): + """What lens.json says: on and waiting for a key before joining, then on with this node's config.""" + n = self.node + if not n: + return {"on": True, "ip": None, "public_key": None, "config": None, "version": None} + return {"on": True, "ip": n["node"]["ip"], "public_key": pub, "config": n["config"], "version": n["version"]} + + def publish(self, pub=None): + text = json.dumps(self.lens_file(pub), indent=1, sort_keys=True) + path = self.shared / "lens.json" + try: + if path.read_text() == text: + return False + except OSError: + pass + _write(path, text, 0o644) # the mesh container runs as another user + return True + + def service(self): + """How the mesh container says Nebula runs (its status.json), {} when it hasn't said.""" + try: + s = json.loads((self.shared / "status.json").read_text()) + except (OSError, ValueError): + return {} + return s if isinstance(s, dict) else {} + + # ---------- the calls ---------- + def join(self, pub): + if not self.join_url: + raise Refused("there's no install to join: run the install line with LENS_JOIN and LENS_JOIN_TOKEN") + try: + tid, secret = mesh.parse_token(self.token) + except ValueError as e: + raise Refused(f"{e}: copy the install line from Settings › Nodes again") from None + nonce = secrets.token_urlsafe(24) + body = { + "token_id": tid, + "nonce": nonce, + "name": self.name, + "public_key": pub, + "proof": mesh.proof(secret, tid, nonce, self.name, pub), + "info": machine(), + } + r = self.http.post(f"{self.join_url}/api/v1/mesh/join", json=body) + if r.status_code in (400, 403): + raise Refused(_detail(r)) + r.raise_for_status() + try: + answer = mesh.open_answer(secret, nonce, tid, r.json()["sealed"]) + except ValueError as e: + raise Refused(str(e)) from None + self.node = {**answer, "public_key": pub, "joined_at": _now().isoformat(), "join_url": self.join_url} + self.save() + n = answer["node"] + say(f"joined the mesh as {n['name']} ({n['role']}, {n['ip']}), node {n['id']}") + + def follow(self, pub): + """The config as the primary has it now, a new certificate when it's due, and this machine's report.""" + r = self.http.get(self.api("/node/config"), headers=self._headers()) + if r.status_code == 401: + if not self.removed: + say("this machine was removed from the mesh in Settings › Nodes: waiting in case that's undone") + self.removed = True + return + r.raise_for_status() + if self.removed: + say("this machine is back on the mesh") + self.removed = False + got = r.json() + if pub and (_due(got.get("not_after"), got.get("renew_days")) or pub != self.node.get("public_key")): + r = self.http.post(self.api("/node/renew"), json={"public_key": pub}, headers=self._headers()) + r.raise_for_status() + got = r.json() + self.node["public_key"] = pub + say(f"renewed its certificate (good until {got.get('not_after')})") + if got["version"] != self.node.get("version") or got.get("not_after") != self.node.get("not_after"): + self.node.update(config=got["config"], version=got["version"], not_after=got.get("not_after"), renew_days=got.get("renew_days")) + self.save() + s = self.service() + report = { + "nebula": s.get("state") or ("running" if s.get("nebula") else "stopped"), + "version": updates.current(), + "machine": machine(), + "error": str(s["error"])[:500] if s.get("error") else None, # this call reached the primary: its own are past + "log": [str(x) for x in (s.get("log") or [])][-40:], + } + self.http.post(self.api("/node/beat"), json={k: v for k, v in report.items() if v is not None}, headers=self._headers()) + + # ---------- the loop ---------- + def tick(self): + if self.refused or time.monotonic() < self.next_try: + return + try: + self.shared.mkdir(parents=True, exist_ok=True) + pub = self.public_key() + if not self.load(): + self.publish() # the container makes the key pair once lens.json is on + if not pub: + self.error = "waiting for the mesh container's key" + return + self.join(pub) + self.publish(self.node.get("public_key")) + if self.service().get("state") != "running": # the primary is reached over the mesh only + self.error = "waiting for Nebula to start" + return + if time.monotonic() - self.last_beat >= BEAT_SECONDS: + self.last_beat = time.monotonic() + self.follow(pub) + self.publish(self.node.get("public_key")) + self.error, self.fails = None, 0 + except Refused as e: + self.refused = str(e) + say(f"couldn't join: {e}") + except (httpx.HTTPError, OSError, ValueError, KeyError) as e: + self.error = f"{type(e).__name__}: {e}" + wait = RETRY[min(self.fails, len(RETRY) - 1)] + self.fails += 1 + self.next_try = time.monotonic() + wait + self.last_beat = 0.0 + say(f"{self.error} (trying again in {wait} s)") + + def report(self): + """agent.json: how this agent is doing, for `lens node status` (another process) and the install line.""" + now = {"refused": self.refused, "error": self.error, "removed": self.removed, "joined": bool(self.node)} + if now != self._reported: + self.state.mkdir(parents=True, exist_ok=True) + _write(self.state / "agent.json", json.dumps(now)) + self._reported = now + + def loop(self, stop): + while not stop.is_set(): + self.tick() + try: + self.report() + except OSError as e: + say(f"can't write agent.json: {e}") + stop.wait(TICK) + + def view(self): + """`lens node status`: what this machine is on the mesh, without its credentials.""" + try: + seen = json.loads((self.state / "agent.json").read_text()) + except (OSError, ValueError): + seen = {} + n = self.load() + if not n: + return {"joined": False, "refused": seen.get("refused"), "error": seen.get("error")} + return { + "joined": True, + "node": n["node"], + "primary": n["primary"], + "not_after": n.get("not_after"), + "joined_at": n.get("joined_at"), + "join_url": n.get("join_url"), + "removed": bool(seen.get("removed")), + "error": seen.get("error"), + "nebula": self.service().get("state"), + } + + +def _detail(r): + try: + return str(r.json().get("detail") or r.text) + except ValueError: + return r.text or f"HTTP {r.status_code}" + + +def _due(not_after, days): + if not not_after: + return False + try: + ends = dt.datetime.fromisoformat(str(not_after).replace("Z", "+00:00")) + except ValueError: + return False + if ends.tzinfo is None: + ends = ends.replace(tzinfo=dt.timezone.utc) + return ends - _now() < dt.timedelta(days=int(days or 1)) + + +def agent_from_env(http=None): + return Agent( + os.environ.get("LENS_NODE_DIR") or "/data/node", + os.environ.get("LENS_MESH_DIR") or "/mesh", + os.environ.get("LENS_JOIN"), + os.environ.get("LENS_JOIN_TOKEN"), + os.environ.get("LENS_NODE_NAME") or os.environ.get("LENS_SCALING_UNIT"), + http, + ) + + +def main(argv=None): + import argparse + import signal + import threading + + ap = argparse.ArgumentParser(prog="lens node", description="this machine's side of another Lens install's mesh (docs/mesh.md)") + ap.add_argument("action", nargs="?", default="run", choices=["run", "status"]) + a = ap.parse_args(argv) + agent = agent_from_env() + if a.action == "status": + print(json.dumps(agent.view(), indent=1, ensure_ascii=False)) + return + stop = threading.Event() + signal.signal(signal.SIGTERM, lambda *_: stop.set()) + say(f"{'on the mesh as ' + agent.node['node']['name'] if agent.load() else 'joining ' + str(agent.join_url)}") + try: + agent.loop(stop) + except KeyboardInterrupt: + pass diff --git a/fastapi_backend/app/schemas/mesh.py b/fastapi_backend/app/schemas/mesh.py index e553c5e2e..cd1617bb5 100644 --- a/fastapi_backend/app/schemas/mesh.py +++ b/fastapi_backend/app/schemas/mesh.py @@ -104,6 +104,7 @@ class MeshTokenCreated(ResponseModel): token: MeshToken secret: str = Field(description="the join token itself, shown this once") join_url: str = Field(description="this install's address the machine joins at") + install_line: str = Field(description="the line to run on the machine: install.sh with LENS_JOIN and LENS_JOIN_TOKEN") class MeshJoin(RequestModel): diff --git a/fastapi_backend/tests/api/test_mesh.py b/fastapi_backend/tests/api/test_mesh.py index f6389cc4f..cce8cd3cb 100644 --- a/fastapi_backend/tests/api/test_mesh.py +++ b/fastapi_backend/tests/api/test_mesh.py @@ -77,6 +77,7 @@ def test_adding_a_node_makes_the_ca_a_token_and_turns_the_mesh_on(client, db, cf assert r.status_code == 200, r.text made = r.json() assert made["secret"].startswith(mesh.TOKEN_PREFIX) and made["join_url"].startswith("http://") + assert made["install_line"].endswith(f"install.sh | LENS_JOIN={made['join_url']} LENS_JOIN_TOKEN={made['secret']} sh") assert made["token"]["role"] == "compute" and made["token"]["uses"] == 1 and made["token"]["used"] == 0 after = client.get("/api/v1/mesh", headers=h).json() diff --git a/fastapi_backend/tests/api/test_mesh_node.py b/fastapi_backend/tests/api/test_mesh_node.py new file mode 100644 index 000000000..483a60ca1 --- /dev/null +++ b/fastapi_backend/tests/api/test_mesh_node.py @@ -0,0 +1,196 @@ +"""`lens node` (docs/mesh.md, Nodes): the agent on a machine that joins another install's mesh. It turns the folder it +shares with the mesh container on, waits for the container's public key, joins with the token from the install line +(which never crosses the wire), keeps its credentials to itself, writes its config for the container, and then, over +the mesh, follows the primary's config, renews its certificate and reports every minute. The mesh container is +played here by writing its public key; the calls go to this install's API.""" + +from __future__ import annotations + +import json +import stat + +import pytest + +from app.domain import mesh, mesh_node +from tests.api.test_mesh import admin, nebula, server_db # noqa: F401 - fixtures + + +@pytest.fixture +def folders(tmp_path): + return tmp_path / "node", tmp_path / "mesh" + + +def agent(client, folders, token, name="pi"): + a = mesh_node.Agent(folders[0], folders[1], "http://testserver/", token, name, http=client) + return a + + +def running(shared, error=None): + """The mesh container saying Nebula runs (its status.json).""" + (shared / "status.json").write_text(json.dumps({"state": "running", "nebula": True, "error": error, "log": ["up"]})) + + +def due(a): + a.last_beat = 0.0 # as a minute later + + +def test_a_machine_joins_and_follows_the_primary(client, db, cfg, nebula, server_db, folders): # noqa: F811 + h = admin(client, db) + made = client.post("/api/v1/mesh/tokens", json={"role": "worker", "namespaces": []}, headers=h).json() + node_dir, shared = folders + a = agent(client, folders, made["secret"], name="Studio Mac (M2)!") + + # first the container is asked for a key pair: lens.json on, nothing to run yet + a.tick() + lens = json.loads((shared / "lens.json").read_text()) + assert lens == {"on": True, "ip": None, "public_key": None, "config": None, "version": None} + assert a.error == "waiting for the mesh container's key" and not (node_dir / "node.json").exists() + + # the container publishes its public key: the agent joins with it + _, pub = mesh.keygen(cfg) + (shared / "host.pub").write_text(pub) + a.tick() + kept = json.loads((node_dir / "node.json").read_text()) + assert stat.S_IMODE((node_dir / "node.json").stat().st_mode) == 0o600 + assert kept["node"]["name"] == "Studio Mac -M2--" and kept["node"]["ip"] == "10.73.0.2" and kept["node"]["role"] == "worker" + assert kept["node_token"].startswith(mesh.NODE_PREFIX) and kept["agent_key"] and kept["public_key"] == pub + lens = json.loads((shared / "lens.json").read_text()) + assert lens["ip"] == "10.73.0.2" and lens["public_key"] == pub and lens["version"] == kept["version"] + assert lens["config"]["pki"]["cert"] and "lighthouse" in lens["config"] + assert made["secret"] not in (shared / "lens.json").read_text() # the container gets the config, not the credentials + + # it calls the primary over the mesh only, so not before Nebula runs; then at once, with its node token + me = client.get("/api/v1/mesh/nodes", headers=h).json()[1] + assert a.error == "waiting for Nebula to start" and not me["last_seen"] + running(shared, error="a peer is slow") + a.tick() + me = client.get("/api/v1/mesh/nodes", headers=h).json()[1] + assert me["last_seen"] and me["status"]["machine"]["cpus"] and me["status"]["version"] and a.error is None + assert me["status"]["nebula"] == "running" and me["status"]["error"] == "a peer is slow" and me["status"]["log"] == ["up"] + + # a removed node is refused, and keeps asking: an undo brings it back + client.delete(f"/api/v1/mesh/nodes/{me['id']}", headers=h) + due(a) + a.tick() + assert a.removed and json.loads((shared / "lens.json").read_text())["on"] is True + a.report() + assert mesh_node.Agent(node_dir, shared).view()["removed"] is True + client.post(f"/api/v1/mesh/nodes/{me['id']}/restore", headers=h) + due(a) + a.tick() + assert not a.removed + + # another machine is removed: the blocklist grows, so this one's config changes and lens.json follows + other = client.post("/api/v1/mesh/tokens", json={"role": "compute"}, headers=h).json() + b = agent(client, (node_dir.parent / "b", node_dir.parent / "b-mesh"), other["secret"], name="box") + b.tick() + (b.shared / "host.pub").write_text(mesh.keygen(cfg)[1]) + running(b.shared) + b.tick() + assert b.node["node"]["role"] == "compute" and b.node["node"]["ip"] == "10.73.0.3" + before = json.loads((shared / "lens.json").read_text())["version"] + client.delete(f"/api/v1/mesh/nodes/{b.node['node']['id']}", headers=h) + due(a) + a.tick() + lens = json.loads((shared / "lens.json").read_text()) + assert lens["version"] != before and lens["config"]["pki"]["blocklist"] + + # the container made a new key pair (its volume was lost): the agent has it signed, and lens.json names it + _, pub2 = mesh.keygen(cfg) + (shared / "host.pub").write_text(pub2) + due(a) + a.tick() + lens = json.loads((shared / "lens.json").read_text()) + assert lens["public_key"] == pub2 and lens["config"]["pki"]["cert"] != kept["config"]["pki"]["cert"] + assert mesh.describe(cfg, lens["config"]["pki"]["cert"])["networks"] == ["10.73.0.2/16"] + + # and one that's near its end is renewed with the same key + db.q("UPDATE $r SET not_after = '2000-01-01T00:00:00Z'", r=mesh.R("mesh_node", me["id"])) + due(a) + a.tick() + assert a.node["not_after"] > "2001" and json.loads((shared / "lens.json").read_text())["public_key"] == pub2 + + # started again (a restart of the machine), it carries on as the same node without its token + again = mesh_node.Agent(node_dir, shared, None, None, "whatever", http=client) + again.tick() + assert again.node["node"]["id"] == me["id"] and not again.refused and again.error is None + view = again.view() + assert view["joined"] and view["node"]["ip"] == "10.73.0.2" and "node_token" not in json.dumps(view) + + +def test_a_spent_or_wrong_token_is_not_tried_again(client, db, cfg, nebula, folders): # noqa: F811 + h = admin(client, db) + made = client.post("/api/v1/mesh/tokens", json={"role": "compute"}, headers=h).json() + first = agent(client, (folders[0] / "1", folders[1] / "1"), made["secret"]) + (folders[1] / "1").mkdir(parents=True) + (folders[1] / "1" / "host.pub").write_text(mesh.keygen(cfg)[1]) + first.tick() + assert first.node + + second = agent(client, folders, made["secret"]) + folders[1].mkdir(parents=True, exist_ok=True) + (folders[1] / "host.pub").write_text(mesh.keygen(cfg)[1]) + second.tick() + assert "used" in second.refused and not (folders[0] / "node.json").exists() + second.report() + assert "used" in mesh_node.Agent(*folders).view()["refused"] + calls = [] + second.http = type("Spy", (), {"post": lambda *a, **k: calls.append(a)})() + second.tick() + assert calls == [] # not tried again: the install line is run again with a new token + + bad = agent(client, (folders[0] / "3", folders[1]), "lensjoin1.nope") + bad.tick() + assert "isn't a Lens join token" in bad.refused + + +def test_a_primary_out_of_reach_is_tried_again_later(client, db, cfg, nebula, folders, monkeypatch): # noqa: F811 + import httpx + + h = admin(client, db) + made = client.post("/api/v1/mesh/tokens", json={"role": "compute"}, headers=h).json() + + class Down: + def post(self, *a, **k): + raise httpx.ConnectError("no route to host") + + a = mesh_node.Agent(*folders, "http://testserver", made["secret"], "pi", http=Down()) + folders[1].mkdir(parents=True) + (folders[1] / "host.pub").write_text(mesh.keygen(cfg)[1]) + a.tick() + assert "no route to host" in a.error and a.next_try > 0 and not a.refused + a.http, a.next_try = client, 0.0 + a.tick() + assert a.node and a.error == "waiting for Nebula to start" + + +def test_a_machines_name_is_one_the_mesh_takes(): + assert mesh_node.machine_name("pi") == "pi" + assert mesh_node.machine_name("--weird/host.local") == "weird-host.local" + assert mesh_node.machine_name("x" * 80) == "x" * 63 + assert mesh_node.machine_name("///") == "node" + assert all(mesh.NAME_RX.fullmatch(mesh_node.machine_name(n)) for n in ("pi", "a b", "é-mac", "1")) + + +def test_the_primarys_mesh_address_is_a_host_only_for_nodes_calls(client, db, cfg, nebula): # noqa: F811 + at = {"host": "10.73.0.1:8080"} + assert client.get("/api/v1/mesh/node/config", headers=at).status_code == 400 # the mesh is off: not a host here + h = admin(client, db) + client.post("/api/v1/mesh/tokens", json={"role": "compute"}, headers=h) + assert client.get("/api/v1/mesh/node/config", headers=at).status_code == 401 # let in, and still needs a node token + assert client.get("/api/v1/auth/status", headers=at).status_code == 400 # nothing else is served at that address + assert client.get("/api/v1/mesh", headers={**at, **h}).status_code == 400 + + +def test_the_install_line_names_this_install_and_the_token(cfg): + token = "lensjoin1.7." + "a" * 43 + line = mesh.join_line(cfg, token, "http://192.168.1.20:3000/") + assert line == ( + "curl -fsSL https://raw.githubusercontent.com/adeelahmad/lense/main/install.sh" + f" | LENS_JOIN=http://192.168.1.20:3000 LENS_JOIN_TOKEN={token} sh" + ) + for bad in ("http://x; rm -rf ~", "file:///etc", "http://a b"): + with pytest.raises(ValueError): + mesh.join_line(cfg, token, bad) + with pytest.raises(ValueError): + mesh.join_line(cfg, "lensjoin1.7.$(id)", "http://lens.home") diff --git a/install.sh b/install.sh index 141da9116..324f142f9 100755 --- a/install.sh +++ b/install.sh @@ -37,13 +37,28 @@ # build). Docker only; Lens is built anyway for a branch or with EXTRAS (docs/deployment.md) # LENS_IMAGES_REGISTRY where those images are (default: ghcr.io/), for a mirror. Kept for later # runs +# +# Another machine joins an install's mesh as a node (docs/mesh.md, Nodes), with the line Settings › Nodes shows: +# ... | LENS_JOIN=http://lens.home:3000 LENS_JOIN_TOKEN=lensjoin1.… sh +# LENS_JOIN the install's address (the web app's); with it this machine becomes a node, in ~/lens-node by +# default, with Docker: no web app, no database, nothing published +# LENS_JOIN_TOKEN the join token, used once and never written down +# LENS_NODE_NAME the name it joins with (default: LENS_SCALING_UNIT, an auto scaling unit's, else this machine's) set -eu # All in a function, so nothing runs until the whole script has arrived (curl | sh streams it). main() { REPO="${LENS_REPO:-adeelahmad/lense}" -DIR="${LENS_DIR:-$HOME/lens}" +# a node on another install's mesh: asked for with LENS_JOIN, or what an earlier run made in LENS_DIR +NODE=0 +if [ -n "${LENS_JOIN:-}" ]; then + NODE=1 + DIR="${LENS_DIR:-$HOME/lens-node}" +else + DIR="${LENS_DIR:-$HOME/lens}" + [ -f "$DIR/.env" ] && grep -qx 'LENS_ROLE=node' "$DIR/.env" && NODE=1 +fi REF="${LENS_REF:-main}" TARGET="${LENS_TARGET:-}" @@ -109,6 +124,7 @@ py_ok() { [ "$os" != Darwin ] || xcode-select -p >/dev/null 2>&1 # else python3 is a stub that pops up an install dialog } WIZARD="${LENS_WIZARD:-}" +[ "$NODE" = 1 ] && WIZARD=0 # a node has nothing to ask if [ -z "$WIZARD" ]; then WIZARD=0 [ ! -f "$DIR/.env" ] && [ -z "${LENS_BACKEND:-}" ] && tty_ok && WIZARD=1 @@ -151,6 +167,16 @@ elif [ "$WIZARD" = 1 ]; then echo "The web installer needs Python 3.8 or later; asking here instead." fi BACKEND="${LENS_BACKEND:-}" +if [ "$NODE" = 1 ]; then # Nebula's network device needs Docker's (docs/mesh.md) + [ "${BACKEND:-docker}" = docker ] || fail "A node runs with Docker. Leave LENS_BACKEND out." + BACKEND=docker + if [ ! -f "$DIR/.env" ]; then + case "${LENS_JOIN_TOKEN:-}" in + lensjoin1.*) ;; + *) fail "LENS_JOIN_TOKEN is missing or isn't a join token. Copy the install line from Settings › Nodes again." ;; + esac + fi +fi if [ -z "$BACKEND" ] && [ -f "$DIR/.env" ]; then # an existing install keeps what it runs on (its data is there) BACKEND="$(grep '^LENS_BACKEND=' "$DIR/.env" | cut -d= -f2- || true)" [ -n "$BACKEND" ] || BACKEND=docker @@ -364,6 +390,62 @@ else fail "Couldn't get the code from GitHub ($REF). Check the network and LENS_REF, and try again." fi cd "$DIR" +setenv() { # key value: replace the line, or add it (only this user can read the copy that replaces it) + if grep -q "^$1=" .env; then + (umask 077 && sed "s|^$1=.*|$1=$2|" .env >.env.tmp) && mv .env.tmp .env + else + printf '%s=%s\n' "$1" "$2" >>.env + fi +} + +# --- A node on another install's mesh (docs/mesh.md, Nodes): two containers, no web app, nothing published -------- +# `mesh` runs Nebula and `node` (lens node) joins with the token, once; the token goes to the container's environment +# for that first start only, never into .env. Running the line again updates the node, which stays the same node. +if [ "$NODE" = 1 ]; then + if [ -f .env ] && ! grep -qx 'LENS_ROLE=node' .env; then + fail "$DIR holds a Lens install, not a node. Put the node in a folder of its own: LENS_DIR=$HOME/lens-node" + fi + if [ ! -f .env ]; then + (umask 077 && printf '# Written by install.sh: this machine is a node on another Lens install. Keep it with the volumes.\n' >.env) + fi + chmod 600 .env + setenv LENS_ROLE node + [ -z "${LENS_JOIN:-}" ] || setenv LENS_JOIN "$LENS_JOIN" + # a unit of an auto scaling group (docs/scaling.md): its name, and the steps its worker runs, kept for later runs + for k in LENS_SCALING_UNIT LENS_WORKER_STEPS; do + eval "v=\${$k:-}" + [ -z "$v" ] || setenv "$k" "$v" + done + name="${LENS_NODE_NAME:-$(grep '^LENS_NODE_NAME=' .env | cut -d= -f2- || true)}" + name="${name:-${LENS_SCALING_UNIT:-}}" + setenv LENS_NODE_NAME "${name:-$(hostname 2>/dev/null || uname -n)}" + setenv LENS_TARGET "${TARGET:-$(grep '^LENS_TARGET=' .env | cut -d= -f2- || true)}" + grep -q '^LENS_TARGET=.' .env || setenv LENS_TARGET lean # a node reads no Office files: the smaller image + setenv LENS_REPO "$REPO" + setenv LENS_DIR "$DIR" + say "Building and starting the node (the first build takes a few minutes)..." + LENS_JOIN_TOKEN="${LENS_JOIN_TOKEN:-}" $COMPOSE -f docker-compose.node.yml up -d --build + say "Joining $(grep '^LENS_JOIN=' .env | cut -d= -f2-)..." + WAIT="${LENS_WAIT:-600}" + start=$(date +%s) + until out="$($COMPOSE -f docker-compose.node.yml exec -T node lens node status 2>/dev/null)" && printf '%s' "$out" | grep -q '"joined": true'; do + refused="$(printf '%s\n' "$out" | sed -n 's/^ *"refused": "\(.*\)",\{0,1\}$/\1/p')" + if [ -n "$refused" ]; then + $COMPOSE -f docker-compose.node.yml stop node >/dev/null 2>&1 || true + fail "The install didn't let this machine join: $refused" + fi + if [ $(($(date +%s) - start)) -ge "$WAIT" ]; then + echo "The last lines the node logged:" + $COMPOSE -f docker-compose.node.yml logs --tail 40 2>&1 .env.tmp) && mv .env.tmp .env - else - printf '%s=%s\n' "$1" "$2" >>.env - fi -} unsetenv() { # key: drop the line, if there is one if grep -q "^$1=" .env; then (umask 077 && grep -v "^$1=" .env >.env.tmp) && mv .env.tmp .env diff --git a/mesh/sidecar.py b/mesh/sidecar.py index 65317b0b8..0da7de087 100644 --- a/mesh/sidecar.py +++ b/mesh/sidecar.py @@ -412,7 +412,7 @@ def main(): stop = threading.Event() for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda *_: stop.set()) - say(f"waiting for Lens to write {shared / 'lens.json'} (it does once a node is added)") + say(f"waiting for Lens to write {shared / 'lens.json'} (it does once the mesh is on)") svc.loop(stop) return 0 diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index f47c114fe..e7a370c44 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -13248,6 +13248,12 @@ export type MeshTokenCreated = { * this install's address the machine joins at */ join_url: string; + /** + * Install Line + * + * the line to run on the machine: install.sh with LENS_JOIN and LENS_JOIN_TOKEN + */ + install_line: string; [key: string]: unknown; }; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 4175e120e..1d7abe97b 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -57054,6 +57054,11 @@ "type": "string", "title": "Join Url", "description": "this install's address the machine joins at" + }, + "install_line": { + "type": "string", + "title": "Install Line", + "description": "the line to run on the machine: install.sh with LENS_JOIN and LENS_JOIN_TOKEN" } }, "additionalProperties": true, @@ -57061,7 +57066,8 @@ "required": [ "token", "secret", - "join_url" + "join_url", + "install_line" ], "title": "MeshTokenCreated" }, diff --git a/threat-model.json b/threat-model.json index c6d9aba0d..07f955611 100644 --- a/threat-model.json +++ b/threat-model.json @@ -768,7 +768,7 @@ "parameter": "a public key to certify; a status report: whether Nebula runs, its peers, its machine, its log lines", "attacker_controllable": "yes", "component": "optional", - "caller_must_enforce": "the node token is 256 random bits kept only as a hash; a node gets its own config and a certificate for its own address and groups only; reports are capped", + "caller_must_enforce": "the node token is 256 random bits kept only as a hash; a node gets its own config and a certificate for its own address and groups only; reports are capped; they arrive at the primary's mesh address, which the API's host check admits for these paths alone while the mesh is on", "provenance": "documented", "source": "docs/mesh.md \"After joining\"" }, @@ -1531,7 +1531,7 @@ "property": "P-MESH", "violation_symptom": "info-leak, integrity-bypass", "severity_tier": "security", - "conditions": "the mesh (mesh.enabled, off until an admin adds a node) admits only machines that joined with a token an admin made, spent on use and expiring, which never crosses the wire (HMAC proof; answer sealed with a key derived from it); each machine's certificate names only its own address, role and namespaces; the rendered firewall opens the database port to worker nodes and the API port to every node on the primary, a node's agent port to the primary only; removing a node blocklists every certificate it was given on every machine and stops its node token; nodes still sign in to Lens as any client does; on the primary Nebula runs in the mesh container, the only one holding NET_ADMIN, with a key pair it makes itself and Nebula's release checked against the kept checksum; its relays listen on the mesh address only", + "conditions": "the mesh (mesh.enabled, off until an admin adds a node) admits only machines that joined with a token an admin made, spent on use and expiring, which never crosses the wire (HMAC proof; answer sealed with a key derived from it); each machine's certificate names only its own address, role and namespaces; the rendered firewall opens the database port to worker nodes and the API port to every node on the primary, a node's agent port to the primary only; removing a node blocklists every certificate it was given on every machine and stops its node token; nodes still sign in to Lens as any client does; on the primary Nebula runs in the mesh container, the only one holding NET_ADMIN, with a key pair it makes itself and Nebula's release checked against the kept checksum; its relays listen on the mesh address only; a node runs lens node and Nebula in two containers with nothing published, its node token and agent key in its own volume (0600), the join token in the first start's environment only", "provenance": "documented", "source": "docs/mesh.md \"What keeps it safe\"" } diff --git a/threat-model.md b/threat-model.md index 92a79244a..840e4a5c4 100644 --- a/threat-model.md +++ b/threat-model.md @@ -334,7 +334,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | Deep research: `/research `, `research: true` on a question, `POST /chats/{cid}/research`, `/research/{id}` (revise, stop, resume, note, podcast), the assistant's and MCP's `start_research`, MCP's `research_status`, and the `research.*` settings | a goal, answers to follow-up questions, plan edits (sub-questions, sources, tool calls, outline); settings (admins) | yes: anyone who may use the assistant, in their own conversations; indirectly through what the run reads, through the model | data, x-prompt-injection, size | none; the plan waits for the person's own approval (P-RESEARCH); the run uses only the read tools of its plan's sources, as the person with their current access, within its tool calls, minutes and budgets; its web searches and page reads don't ask again, bounded by the plan and kept in the ledger; one run per person at a time; the report's claims are checked against what they cite and marked when unsupported | *(documented, docs/assistant.md "Deep research")* | | `ask/NAME.want` in a microsandbox's `/data/.lens/programs` (LibreOffice and Chromium on first use) | a program's name | writable by the Lens user inside the sandbox | x-env-config | the start script, as root, installs only `libreoffice` or `chromium`, each a fixed list of signed Debian packages, and answers in `got/`, which only root writes; it reads nothing from the file but its name, and only deletes any other name or link | *(documented, docs/microsandbox.md "LibreOffice and Chromium")* | | `POST /api/v1/mesh/join` (a machine joining the mesh) | a join token's id, a nonce, the machine's name and Nebula public key, an HMAC of those keyed with the token's secret, facts about the machine | yes, anyone reaching the web app, once a token exists | x-credential, data | no sign-in: the token's secret never crosses the wire, the HMAC is checked in constant time, wrong proofs count towards a throttle per address (8 in 15 minutes), a token is spent on use (or after `uses`) and expires, and the answer (certificate, config, node token) is sealed with a key derived from the secret, so an eavesdropper on plain http:// learns nothing | *(documented, docs/mesh.md "Joining")* | -| `/api/v1/mesh/node/config`, `/node/renew`, `/node/beat` (a node's own calls, with its node token) | a public key to certify; a status report: whether Nebula runs, its peers, its machine, its log lines | yes, a machine that joined (a removed one is refused) | x-credential, data, x-peer | the node token is 256 random bits kept only as a hash; a node gets its own config and a certificate for its own address and groups only, and can't widen them; reports are capped and kept as its status for admins | *(documented, docs/mesh.md "After joining")* | +| `/api/v1/mesh/node/config`, `/node/renew`, `/node/beat` (a node's own calls, with its node token) | a public key to certify; a status report: whether Nebula runs, its peers, its machine, its log lines | yes, a machine that joined (a removed one is refused) | x-credential, data, x-peer | the node token is 256 random bits kept only as a hash; a node gets its own config and a certificate for its own address and groups only, and can't widen them; reports are capped and kept as its status for admins; they arrive at the primary's mesh address, which the API's host check admits for these paths alone while the mesh is on | *(documented, docs/mesh.md "After joining")* | | `POST /api/v1/mesh/tokens`, `PATCH` and `DELETE /api/v1/mesh/nodes/{id}`, `PUT /api/v1/settings/mesh` | a node's role, name, namespaces, token hours and uses; the mesh's network, port, hosts, cipher | admin only | x-env-config | admin is trusted; audited (`mesh.token.create`, `mesh.join`, `mesh.node.*`); the network can't change once machines have addresses in it | *(documented, docs/mesh.md "Settings")* | | UDP 4242 on the primary and ports 8000 and 8080 on its mesh address (the `mesh` container, while the mesh is on) | Nebula handshakes and tunnelled packets from anyone reaching the port; on the mesh, TCP to the database and the API | yes, anyone reaching UDP 4242; on the mesh, a machine with a certificate the mesh's CA signed | data, x-peer | Nebula drops what no certificate of the mesh's CA signs and anything on the blocklist; its firewall, rendered by Lens, lets only `lens-worker` reach 8000 and `lens-node` 8080; the relays listen on the mesh address only; the container is the only one holding NET_ADMIN, makes its own key pair (the private key stays in its own volume) and runs only Nebula's release with the checksum kept in Lens | *(documented, docs/mesh.md "On this machine")* | @@ -591,7 +591,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-BROWSER-GUARD | The Lens extension's side panel reads and acts only on sites the person allowed in it (and the browser granted), and opens an address only on such a site. It asks the person before a click, submit or Enter that the model marks consequential or whose control's words or posting form look so, and, once the answer has read text from outside Lens (`ai.untrusted_asks`), before opening or downloading an address or typing text the person's own message doesn't contain. It never types into password, card or file fields, never reports their values, and shows each action as it happens. | integrity-bypass | security-critical | *(documented, docs/browser-assistant.md "Letting it use your browser")* | Voided by: The consequential check is the model's flag plus a word list (`extensions/browser/src/core/act.js`): a control with no telling words that a misled model doesn't mark runs without asking; synthetic events reach only the page's own handlers. Removing a site from the options stops the side panel acting there, but leaves the browser's access to it for the clipper. Lens sends the ask (`confirm`) and the extension honours it: an extension older than this doesn't ask. | | P-SANDBOX | Code the assistant runs (`run_python`, `run_bash`; `sandbox.enabled`, off by default) runs only for people `sandbox.who` allows (editors and admins, or admins; never viewers), in a sandbox of that conversation's own on the OpenShell gateway, never in Lens's processes. The sandbox gets none of Lens's data, keys or database address: only the code and the files a run copies in, which must be attached to that conversation by that person (25 MB each, 100 MB a run). Its policy allows no egress with `sandbox.network: off`, only pypi.org, files.pythonhosted.org and registry.npmjs.org on 443 with `packages`, and those and the names in `sandbox.hosts` on 80 and 443 with `hosts`; every endpoint is held to public addresses (no private, loopback, link-local or reserved range), so a listed name that resolves to one is refused. A file leaves it only as one of the person's held uploads (25 MB) placed in a namespace by an `import_files` approval, or as a download (`share_sandbox_file`, 25 MB, 20 a conversation) kept encrypted with the conversation, served only to its person as an attachment with `nosniff`, and deleted with the conversation or their account. A sandbox stops after `sandbox.idle_minutes` idle and is deleted after `sandbox.keep_hours` unused or with its conversation. | integrity-bypass | security-critical | *(documented, docs/assistant.md "Code sandbox")* | Voided by: isolation inside the sandbox is OpenShell's (§1.9); `landlock.compatibility: best_effort` runs without Landlock on a kernel that lacks it — `domain/sandbox.py`; whoever the gateway lets in can run sandboxes of their own (in the `sandbox` profile, a holder of its client certificate); a file shared to download is readable by whoever holds `secret.key` or `ARCHIVE_SECRET_KEY` (Q14); what the code prints reaches the model as third-party text (N-PROMPT). | | P-RESEARCH | A research plan (`domain/research.py`) runs only once the person who asked for it approves it: signed in, or with a "yes" in their chat room from someone in `bridge.users`; an API key, an app or an agent (MCP, the assistant) can only decline it, unless an admin turns on `research.agents_skip_review`. The run acts as that person with their access at run time and stops if their account is disabled. It is offered only the read tools of the sources its plan names (never a tool that changes anything), makes at most the plan's tool calls (`research.max_tool_calls`, 40 by default), stops looking things up after `research.max_minutes` and writes up what it found, and checks the budgets of its namespaces and `all` before each stage, waiting or stopping as they say. Its web searches and page reads, and code runs with the network when the plan names the code sandbox, don't ask again after outside text, since the plan approved them. A person has one run going at a time. One process carries a run out at a time (a claim on `research_run:` with a heartbeat every 10 seconds, taken over only after two minutes of silence). Every call it makes is in the activity ledger for the conversation and `research:`. | integrity-bypass | correctness-only | *(documented, docs/assistant.md "Deep research")* | Voided by: `research.agents_skip_review: true` lets an agent's plan run without review — `domain/research.py` `agents_may_approve`; a run's web queries can carry archive text it read, within its plan's tool calls (the misled model of N-PROMPT); a "yes" in a room is the room's people's (N-BRIDGE). | -| P-MESH | The mesh between Lens's machines (`mesh.enabled`, off until an admin adds a node) admits only machines that joined with a token an admin made, which is spent on use and expires; the token never crosses the wire (an HMAC proves it, the answer is sealed with a key derived from it), each machine's certificate names only its own address, role and namespaces, and Nebula's firewall, rendered by Lens, lets worker nodes reach the database port and every node the API port on the primary, and the primary a node's agent port, and nothing else. Removing a node puts every certificate it was given on the blocklist every machine's config carries, and its node token stops working at once. Nodes still sign in to Lens as any client does: the mesh is a transport, not an identity. On the primary Nebula runs in the `mesh` container, the only one holding NET_ADMIN, with a key pair it makes itself and Nebula's release checked against the checksum kept in Lens; its relays listen on the mesh address only. | info-leak, integrity-bypass | security-critical | *(documented, docs/mesh.md "What keeps it safe")* | Voided by: the join request goes to whatever address is put in the install line, so a wrong address gives a stranger the token's id (not its secret); a worker node that reaches the database port holds the database credential (a trusted peer, Q9); the lighthouse must be reachable on UDP `mesh.port`; Nebula's own isolation is upstream's (§1.9) — `domain/mesh.py`. | +| P-MESH | The mesh between Lens's machines (`mesh.enabled`, off until an admin adds a node) admits only machines that joined with a token an admin made, which is spent on use and expires; the token never crosses the wire (an HMAC proves it, the answer is sealed with a key derived from it), each machine's certificate names only its own address, role and namespaces, and Nebula's firewall, rendered by Lens, lets worker nodes reach the database port and every node the API port on the primary, and the primary a node's agent port, and nothing else. Removing a node puts every certificate it was given on the blocklist every machine's config carries, and its node token stops working at once. Nodes still sign in to Lens as any client does: the mesh is a transport, not an identity. On the primary Nebula runs in the `mesh` container, the only one holding NET_ADMIN, with a key pair it makes itself and Nebula's release checked against the checksum kept in Lens; its relays listen on the mesh address only. A node runs `lens node` and Nebula in two containers with nothing published, its node token and agent key in its own volume (0600), the join token in the first start's environment only. | info-leak, integrity-bypass | security-critical | *(documented, docs/mesh.md "What keeps it safe")* | Voided by: the join request goes to whatever address is put in the install line, so a wrong address gives a stranger the token's id (not its secret); a worker node that reaches the database port holds the database credential (a trusted peer, Q9); the lighthouse must be reachable on UDP `mesh.port`; Nebula's own isolation is upstream's (§1.9) — `domain/mesh.py`. | ### Worked routing examples diff --git a/threat-model.yaml b/threat-model.yaml index 9f8181a13..bdbed200f 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -570,7 +570,7 @@ entry_points: attacker_controllable: true x-condition: "a machine that joined (a removed one is refused)" control_kinds: [x-credential, data, x-peer] - caller_must_enforce: "the node token is 256 random bits kept only as a hash; a node gets its own config and a certificate for its own address and groups only; reports are capped" + caller_must_enforce: "the node token is 256 random bits kept only as a hash; a node gets its own config and a certificate for its own address and groups only; reports are capped; they arrive at the primary's mesh address, which the API's host check admits for these paths alone while the mesh is on" provenance: {kind: documented, source: 'docs/mesh.md "After joining"'} - id: "POST /api/v1/mesh/tokens, PATCH and DELETE /api/v1/mesh/nodes/{id}, PUT /api/v1/settings/mesh" component: optional @@ -1965,7 +1965,7 @@ properties_claimed: kind: authentication components: [optional] tier: security-critical - conditions: "the mesh (mesh.enabled, off until an admin adds a node) admits only machines that joined with a token an admin made, spent on use and expiring, which never crosses the wire (HMAC proof; answer sealed with a key derived from it); each machine's certificate names only its own address, role and namespaces; the rendered firewall opens the database port to worker nodes and the API port to every node on the primary, a node's agent port to the primary only; removing a node blocklists every certificate it was given on every machine and stops its node token; nodes still sign in to Lens as any client does; on the primary Nebula runs in the mesh container, the only one holding NET_ADMIN, with a key pair it makes itself and Nebula's release checked against the kept checksum; its relays listen on the mesh address only" + conditions: "the mesh (mesh.enabled, off until an admin adds a node) admits only machines that joined with a token an admin made, spent on use and expiring, which never crosses the wire (HMAC proof; answer sealed with a key derived from it); each machine's certificate names only its own address, role and namespaces; the rendered firewall opens the database port to worker nodes and the API port to every node on the primary, a node's agent port to the primary only; removing a node blocklists every certificate it was given on every machine and stops its node token; nodes still sign in to Lens as any client does; on the primary Nebula runs in the mesh container, the only one holding NET_ADMIN, with a key pair it makes itself and Nebula's release checked against the kept checksum; its relays listen on the mesh address only; a node runs lens node and Nebula in two containers with nothing published, its node token and agent key in its own volume (0600), the join token in the first start's environment only" violation_symptoms: [info-leak, integrity-bypass] x-voided-by: "a wrong address in the install line gives a stranger the token's id (not its secret); a worker node holds the database credential (trusted peer, Q9); the lighthouse must be reachable on UDP mesh.port; Nebula's own isolation is upstream's (domain/mesh.py)" provenance: {kind: documented, source: 'docs/mesh.md "What keeps it safe"'} From 0c75c5b0be020c82457a9af2ebf40d69131be3ed Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 11:46:25 +0000 Subject: [PATCH 2/2] docs(readme): the mesh's In progress line links its open pull request Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GTrHPwWHtEvFJFq3M1YpH4 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e582f12d..877229b40 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ These are open pull requests, not yet part of Lens: ([#406](https://github.com/adeelahmad/lense/pull/406)). - **Automations:** more operators for conditions ([#434](https://github.com/adeelahmad/lense/pull/434)). - **Nodes:** Lens's own private network between the machines it runs on, a Nebula mesh it issues the certificates - for ([#553](https://github.com/adeelahmad/lense/pull/553)). + for ([#591](https://github.com/adeelahmad/lense/pull/591)). ## Planned