From ffe27a6079d5401b4af31d34a1041453e95ad359 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:23:07 +0000 Subject: [PATCH 1/3] feat(shares): serve the file tree over NFS, read-only, to allowed networks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A worker runs rclone serve nfs --read-only over Lens's own S3 server as one chosen account, behind a relay that turns away clients outside nfs.allowed_networks. Off by default; Settings → NFS share; published on host port 12049. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EKnSy4KKeNYsqhnXvWVz97 --- CHANGELOG.md | 4 + docker-compose.prod.yml | 4 + docker-compose.yml | 2 + docs/file-shares.md | 59 ++++ fastapi_backend/app/api/v1/routes/admin.py | 8 + fastapi_backend/app/cli.py | 18 +- fastapi_backend/app/domain/nfs_share.py | 259 ++++++++++++++++++ fastapi_backend/app/domain/settings.py | 33 +++ fastapi_backend/app/domain/share_relay.py | 106 +++++++ fastapi_backend/app/domain/store.py | 9 + fastapi_backend/app/schemas/admin.py | 14 + .../tests/domain/test_nfs_share.py | 163 +++++++++++ mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 4 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 16 ++ .../app/openapi-client/types.gen.ts | 79 ++++++ .../components/settings/bodies.tsx | 53 +++- nextjs-frontend/components/settings/model.ts | 39 +++ .../components/settings/settings-shell.tsx | 2 + nextjs-frontend/openapi.json | 120 ++++++++ threat-model.json | 20 +- threat-model.md | 4 + threat-model.yaml | 22 +- 23 files changed, 1035 insertions(+), 4 deletions(-) create mode 100644 docs/file-shares.md create mode 100644 fastapi_backend/app/domain/nfs_share.py create mode 100644 fastapi_backend/app/domain/share_relay.py create mode 100644 fastapi_backend/tests/domain/test_nfs_share.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5112764c6..13103fb1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **The file tree over NFS.** Settings → NFS share serves the archive's folders and files (namespaces, collections, + topics and entities as folders, assets as files) read-only over NFS, to the networks you allow, showing what one + chosen account sees. Off until turned on; published on port 12049 of the host ([File shares](docs/file-shares.md)). + - **Install on Windows, or by double-clicking.** `install.ps1` installs Lens on Windows: it gets Python 3 if needed and opens the web installer, which installs Docker Desktop (and WSL) and runs Lens in it. Each release now carries launchers that need no terminal at all: **Install Lens.app** for macOS, **Install Lens.cmd** for Windows and diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c5b41f574..99b1bfdda 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -28,6 +28,7 @@ x-backend: &backend LENS_TUNNEL_ORIGIN: http://frontend:3000 # the web app's container (and the one running the tunnel): their X-Forwarded-For names the visitor, for throttles LENS_TRUSTED_PROXY_HOSTS: frontend,backend,worker + LENS_NFS_PORT: ${LENS_NFS_PORT:-12049} # where the NFS share is published, for the mount command Settings shows # Fedora (docs/fedora.md): empty keeps it off; with the fedora profile, http://fedora:8080/fcrepo/rest LENS_FEDORA_URL: ${LENS_FEDORA_URL:-} LENS_FEDORA_USER: ${LENS_FEDORA_USER:-} @@ -103,6 +104,9 @@ services: - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_MQTT_PORT:-1883}:1883" - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_SYSLOG_PORT:-5514}:5514/udp" - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_SYSLOG_PORT:-5514}:5514/tcp" + # the file tree over NFS (docs/file-tree.md), served only once it's turned on in Settings → NFS share, and then + # only to the networks allowed there. 12049 on this machine so a host NFS server on 2049 is never in the way. + - "${LENS_SHARE_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_NFS_PORT:-12049}:2049" healthcheck: disable: true diff --git a/docker-compose.yml b/docker-compose.yml index 179d67cda..8724e6e04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ x-backend: &backend SURREAL_URL: ws://surrealdb:8000 SURREAL_USER: ${SURREAL_USER:-root} SURREAL_PASS: ${SURREAL_PASS:-root} + LENS_NFS_PORT: ${LENS_NFS_PORT:-12049} ARCHIVE_CONFIG: /app/docker/archive.yaml RUN_BACKGROUND: "false" MAIL_SERVER: mail @@ -87,6 +88,7 @@ services: - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_MQTT_PORT:-1883}:1883" - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_SYSLOG_PORT:-5514}:5514/udp" - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_SYSLOG_PORT:-5514}:5514/tcp" + - "${LENS_SHARE_BIND:-127.0.0.1}:${LENS_NFS_PORT:-12049}:2049" # the file tree over NFS, once turned on healthcheck: disable: true diff --git a/docs/file-shares.md b/docs/file-shares.md new file mode 100644 index 000000000..bc42909e0 --- /dev/null +++ b/docs/file-shares.md @@ -0,0 +1,59 @@ +# File shares + +Lens can show the archive as folders and files ([the file tree](file-tree.md)) to other machines, so a file manager, +a media player or a script reads it like any other share: namespaces, collections, topics and entities are folders, +and assets are files inside them. + +## NFS + +Settings → NFS share → **Share the file tree over NFS**. It's off until you turn it on, and then a worker starts +serving within seconds. The share is **read-only**. + +Mount it on Linux (Settings shows the command with your port filled in): + +```sh +sudo mount -t nfs -o port=12049,mountport=12049,tcp,vers=3,ro :/ /mnt/lens +``` + +On macOS: `sudo mount -t nfs -o port=12049,mountport=12049,tcp,vers=3,ro,resvport :/ /Volumes/lens`. + +| Setting | Default | What it does | +| --- | --- | --- | +| `nfs.enabled` | off | Serve the share | +| `nfs.account` | the first admin | Whose view of the tree the share shows: it holds exactly what that person sees in the web app | +| `nfs.allowed_networks` | private networks (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`, `fc00::/7`, `::1/128`) | Machines anywhere else are turned away before they read anything; changes apply without a restart | +| `nfs.port` | 2049 | The port inside Lens (with Docker, inside the worker container) | + +With Docker, the worker container publishes the share on port **12049** of the host (`LENS_NFS_PORT` in `.env` picks +another), so an NFS server the host already runs on 2049 is never in the way. Like the web app, it's reachable from +this machine only unless `LENS_BIND=0.0.0.0` (what `install.sh` sets on a server); `LENS_SHARE_BIND` opens the share +on its own while the web app stays local. + +### NFS has no login + +NFS (version 3) trusts the network: there is no password, passkey or token. So Lens: + +- serves nothing until an admin turns it on; +- serves it read-only, so nothing reaching the share can change the archive; +- shows one account's view (`nfs.account`), never more than that person may see; pick an account that sees only what + the share should show (a member of just the namespaces or collections to share), not an admin, when the network has + people who shouldn't see everything; +- turns away every client outside `nfs.allowed_networks`, and counts them in Settings. + +Anyone on an allowed network can read the share, and an address on the local network can be spoofed, so list only your +own networks, and keep the port off the internet. + +### How it works + +A worker runs `rclone serve nfs --read-only` (rclone is in Lens's image already; MIT licensed) over Lens's own S3 +server, signed in with an S3 key made for `nfs.account`. rclone listens on 127.0.0.1 inside the container; a small +relay in Lens (`domain/share_relay.py`) holds the published port and hands rclone only the connections from allowed +networks. One worker serves it at a time (a lease on `app_service:nfs`), and Settings shows which one, whose view it +shows, how many connections it turned away, and rclone's last lines when something goes wrong. Agents read the same +state at `GET /api/v1/settings/nfs/status` and change it with `PUT /api/v1/settings/nfs`. + +### Refine later + +- NFS version 4 and Kerberos (sec=krb5) for a share with a real login. +- A share per namespace or collection, each with its own account and networks. +- Packaging for the Synology and QNAP compose files, which don't publish the port yet. diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index b4fb20474..2fee56733 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -24,6 +24,7 @@ llm, local_llm, matterbridge, + nfs_share, semantic, settings, sources, @@ -48,6 +49,7 @@ LlmTestResult, LocalLlmStatus, MailTestResult, + NfsStatus, Removed, SemanticStatus, SpeechTestResult, @@ -171,6 +173,12 @@ def remove_local_model(user: AdminWriter, cfg: Cfg, db: Db, model: str = Query(. return Removed(removed=removed) +@router.get("/settings/nfs/status") +def nfs_status(user: AdminReader, cfg: Cfg, db: Db) -> NfsStatus: + """The file tree over NFS: whether it's serving, as whom, to which networks, and the command that mounts it.""" + return NfsStatus.model_validate(nfs_share.status(db, cfg)) + + @router.get("/settings/tunnel/status") def tunnel_status(user: AdminReader, cfg: Cfg, db: Db) -> TunnelStatus: """How the Cloudflare tunnel (Settings › Remote access) is doing: its address, whether it's connected, and diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 96cbc0797..352efef79 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -260,7 +260,22 @@ def platform_main(argv, config): import threading import time - from .domain import auth, bridge, jobs, local_llm, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel, updates + from .domain import ( + auth, + bridge, + jobs, + local_llm, + nfs_share, + notify, + passkeys, + routines, + sensors, + settings, + sources, + telemetry, + tunnel, + updates, + ) ap = argparse.ArgumentParser(prog="lens") sub = ap.add_subparsers(dest="cmd", required=True) @@ -367,6 +382,7 @@ def password(): bridge.start(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat-room bridge, while it's on tunnel.start(db, C, stop, name=wk.name, log=print) # and the Cloudflare tunnel, while it's on local_llm.start(db, C, stop, name=wk.name, log=print) # and the local chat model, while it's on + nfs_share.start(db, C, stop, name=wk.name, log=print) # and the file tree over NFS, while it's on updates.start(db, C, stop, log=print) # and the daily check for a new release (docs/updates.md) _stop_on_term() try: diff --git a/fastapi_backend/app/domain/nfs_share.py b/fastapi_backend/app/domain/nfs_share.py new file mode 100644 index 000000000..a4a8c1fec --- /dev/null +++ b/fastapi_backend/app/domain/nfs_share.py @@ -0,0 +1,259 @@ +"""The file tree over NFS (docs/file-tree.md, NFS): graph nodes as folders and assets as files, read-only, for the +machines on the networks you allow (Settings → NFS share). + +Off by default. Turned on, one worker runs `rclone serve nfs --read-only` (rclone is in Lens's image already) over +Lens's own S3 server, signed in with an S3 key made for the account the share shows the tree as (`nfs.account`; +nobody picked: the first admin), so the share shows exactly what that person sees in the web app and nothing else. +NFS has no sign-in of its own, so rclone listens on 127.0.0.1 only and a relay (share_relay.py) on the published port +lets in only `nfs.allowed_networks`. + +One process runs it at a time, holding a lease on the `app_service:nfs` row like the local model does, and keeps its +state there (running, the port, the last lines rclone printed). +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +import shutil +import socket +import subprocess +import threading +import time + +from . import share_relay, store + +R = store.R +ROW = R("app_service", "nfs") +LEASE_SECONDS = 30 +TICK_SECONDS = 5 +RETRY = (5, 30, 120, 600) +LOG_LINES = 40 + + +def conf(cfg): + return {**store.DEFAULTS["nfs"], **(cfg.get("nfs") or {})} + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _ts(t=None): + return (t or _now()).isoformat() + + +def state(db): + row = db.one("SELECT * FROM $r", r=ROW) or {} + row.pop("id", None) + if isinstance(row.get("log"), str): + row["log"] = json.loads(row["log"]) + return row + + +def status(db, cfg): + """What Settings → NFS share shows: on or off, running where, the mount command, and what went wrong.""" + s, c = state(db), conf(cfg) + live = bool(s.get("at") and s["at"] >= _ts(_now() - dt.timedelta(seconds=LEASE_SECONDS))) + phase = (s.get("phase") or "off") if live and c["enabled"] else "off" + published = os.environ.get("LENS_NFS_PORT") or c["port"] # Docker publishes it on another port of this machine + return { + "enabled": c["enabled"], + "phase": phase, + "port": int(published), + "allowed_networks": c["allowed_networks"], + "account": s.get("account") if phase == "running" else None, + "error": s.get("error") if c["enabled"] else None, + "refused": s.get("refused") or 0, + "log": s.get("log") or [], + "process": s.get("owner") if live else None, + "mount": f"sudo mount -t nfs -o port={published},mountport={published},tcp,vers=3,ro :/ /mnt/lens", + "rclone": bool(rclone_binary()), + } + + +def rclone_binary(): + return shutil.which("rclone") + + +def _free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _account(db, c): + """The account the share shows the tree as: nfs.account, or the first admin.""" + if c.get("account"): + row = db.one("SELECT record::id(id) AS id, email, admin, disabled FROM $r", r=R("account", int(c["account"]))) + if not row or row.get("disabled"): + raise ShareError("the account picked in Settings → NFS share is gone or disabled: pick another") + return row + rows = db.rows("SELECT record::id(id) AS id, email, admin FROM account WHERE admin = true AND disabled != true ORDER BY id LIMIT 1") + if not rows: + raise ShareError("there is no admin account to show the tree as yet") + return rows[0] + + +class ShareError(RuntimeError): + pass + + +def s3_remote(db, cfg, account): + """rclone's command-line remote and environment for Lens's own S3 server, signed in as `account`. The S3 thread's + server (s3_server.py) owns the endpoint and the keys; this only asks it for them.""" + from . import s3_server # noqa: PLC0415 - optional until the S3 server is in + + endpoint, key_id, secret = s3_server.service_key(db, cfg, account["id"], "nfs") + env = { + "RCLONE_S3_PROVIDER": "Other", + "RCLONE_S3_ENDPOINT": endpoint, + "RCLONE_S3_ACCESS_KEY_ID": key_id, + "RCLONE_S3_SECRET_ACCESS_KEY": secret, + "RCLONE_S3_FORCE_PATH_STYLE": "true", + } + return ":s3:", env + + +class Runner: + """Holds the lease and runs rclone serve nfs plus the relay while nfs.enabled is on.""" + + def __init__(self, db, cfg_fn, name, log=None, remote=s3_remote): + self.db, self.cfg_fn, self.name, self.log, self.remote = db, cfg_fn, name, log or (lambda *a: None), remote + self.proc: subprocess.Popen | None = None + self.relay: share_relay.Relay | None = None + self.sig = None + self.lines: list[str] = [] + self.phase, self.error, self.account = "off", None, None + self.fails, self.next_try = 0, 0.0 + self.lock = threading.Lock() + + def _lease(self): + now = _now() + return bool( + self.db.rows( + "UPSERT $r MERGE {owner: $me, at: $t} WHERE owner = NONE OR owner = $me OR at < $stale RETURN AFTER", + r=ROW, + me=self.name, + t=_ts(now), + stale=_ts(now - dt.timedelta(seconds=LEASE_SECONDS)), + ) + ) + + def _save(self): + with self.lock: + d = {"phase": self.phase, "error": self.error, "account": self.account, "log": json.dumps(self.lines[-LOG_LINES:]), + "refused": self.relay.refused if self.relay else 0} # fmt: skip + self.db.q("UPDATE $r MERGE $d WHERE owner = $me", r=ROW, d=d, me=self.name) + + def _say(self, line): + with self.lock: + self.lines = (self.lines + [str(line)[:300]])[-LOG_LINES:] + + def stop_serving(self): + relay, self.relay = self.relay, None + if relay: + relay.stop() + p, self.proc = self.proc, None + if p and p.poll() is None: + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + + def _read(self, proc): + for raw in proc.stdout: + line = raw.rstrip() + if line: + self._say(line) + code = proc.wait() + if self.proc is proc: + self._failed(f"rclone stopped (exit {code}); {self.lines[-1] if self.lines else 'no output'}") + self.stop_serving() + self._save() + + def _failed(self, error): + self.error, self.phase = error, "error" + self.fails += 1 + self.next_try = time.monotonic() + RETRY[min(self.fails - 1, len(RETRY) - 1)] + + def _start(self, cfg, c): + exe = rclone_binary() + if not exe: + raise ShareError("rclone isn't installed here (it is in Lens's Docker image)") + account = _account(self.db, c) + remote, env_extra = self.remote(self.db, cfg, account) + inner = _free_port() + cmd = [exe, "serve", "nfs", remote, "--addr", f"127.0.0.1:{inner}", "--read-only", "--vfs-cache-mode", "off", + "--dir-cache-time", "30s", "--no-checksum"] # fmt: skip + env = {**os.environ, **env_extra} # the key in the environment, not on the command line where `ps` shows it + self.proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) + threading.Thread(target=self._read, args=(self.proc,), daemon=True, name="nfs-rclone-log").start() + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and self.proc and self.proc.poll() is None: + try: + socket.create_connection(("127.0.0.1", inner), timeout=1).close() + break + except OSError: + time.sleep(0.2) + if not (self.proc and self.proc.poll() is None): + raise ShareError(f"rclone didn't start: {self.lines[-1] if self.lines else 'no output'}") + try: + self.relay = share_relay.Relay(c["bind"], int(c["port"]), inner, share_relay.networks(c["allowed_networks"]), self.log).start() + except OSError as e: + raise ShareError(f"port {c['port']}: {e.strerror or e}") from None + self.account = account.get("email") + self.phase, self.error, self.fails = "running", None, 0 + self.log(f"nfs: serving the file tree as {self.account} on port {self.relay.port}") + + def tick(self): + cfg = self.cfg_fn() + c = conf(cfg) + if not c["enabled"]: + if self.proc or self.relay or self.phase != "off": + self.stop_serving() + self.sig, self.phase, self.error, self.account = None, "off", None, None + if self._lease(): + self._save() + return + if not self._lease(): + if self.proc or self.relay: + self.stop_serving() + return + sig = json.dumps([c["port"], c["account"], c["bind"]]) + if sig != self.sig: + self.stop_serving() + self.sig, self.fails, self.next_try, self.error, self.lines = sig, 0, 0.0, None, [] + if self.relay is not None: + self.relay.nets = share_relay.networks(c["allowed_networks"]) # follows the setting without a restart + if (self.proc and self.proc.poll() is None) or time.monotonic() < self.next_try: + self._save() + return + try: + self._start(cfg, c) + except (ShareError, OSError, ImportError) as e: + self.stop_serving() + self._failed(str(e) if not isinstance(e, ImportError) else "this Lens has no S3 server yet: upgrade") + self._save() + + def loop(self, stop): + while not stop.is_set(): + try: + self.tick() + except Exception as e: # noqa: BLE001 - keep the runner alive; the error shows in Settings + self.error = f"{type(e).__name__}: {e}" + try: + self._save() + except Exception: # noqa: BLE001 + pass + stop.wait(TICK_SECONDS) + self.stop_serving() + self.db.q("UPDATE $r SET owner = NONE, phase = 'off' WHERE owner = $me", r=ROW, me=self.name) + + +def start(db, cfg_fn, stop, name, log=None): + r = Runner(db, cfg_fn, name, log) + threading.Thread(target=r.loop, args=(stop,), daemon=True, name="nfs-share").start() + return r diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 9f19ccfef..f34144cea 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -54,6 +54,8 @@ "fedora": None, "tunnel": None, # bind is a startup setting only + "nfs": ("enabled", "port", "allowed_networks", "account"), + # bind is a startup setting only "sensors": ( "enabled", "mqtt", @@ -412,6 +414,8 @@ def _check(section, key, value, default): return _fedora_setting(key, value) if section == "sensors": return _sensor_setting(key, value) + if section == "nfs": + return _nfs_setting(key, value) if section == "components": return _component_setting(key, value) if section == "mail" and key != "security": @@ -857,6 +861,35 @@ def _sensor_setting(key, value): return value +def _nfs_setting(key, value): + if key == "enabled": + if not isinstance(value, bool): + raise ValueError("nfs.enabled is true or false") + return value + if key == "port": + if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535): + raise ValueError("nfs.port is a whole number from 1 to 65535") + return value + if key == "account": + if value in (None, ""): + return None + if not (isinstance(value, int) and not isinstance(value, bool) and value > 0): + raise ValueError("nfs.account is an account's id, or empty for the first admin") + return value + if not isinstance(value, list) or not value: + raise ValueError("nfs.allowed_networks is a list of networks like 192.168.1.0/24 (at least one)") + out = [] + for v in value: + try: + net = ipaddress.ip_network(str(v).strip(), strict=False) + except ValueError: + raise ValueError(f"nfs.allowed_networks: {v} isn't a network like 192.168.1.0/24") from None + if net.prefixlen == 0: # NFS has no sign-in: the whole internet is never a sensible default to type + raise ValueError(f"nfs.allowed_networks: {v} is every address; list the networks your machines are on") + out.append(str(net)) + return list(dict.fromkeys(out)) + + TELEMETRY_RANGES = {"export_seconds": (5, 3600)} SERVICE_RX = re.compile(r"^[A-Za-z0-9._-]{1,64}$") diff --git a/fastapi_backend/app/domain/share_relay.py b/fastapi_backend/app/domain/share_relay.py new file mode 100644 index 000000000..125172e9a --- /dev/null +++ b/fastapi_backend/app/domain/share_relay.py @@ -0,0 +1,106 @@ +"""A TCP relay that lets in only the addresses a file share allows (docs/file-shares.md). + +Some servers Lens runs for a file share, such as `rclone serve nfs`, have no list of who may connect. The relay listens +on the published port, closes every connection from outside `networks` at once, and copies the rest to the server, +which listens on 127.0.0.1 only. +""" + +from __future__ import annotations + +import ipaddress +import selectors +import socket +import socketserver +import threading + +CHUNK = 64 * 1024 + + +def networks(values): + """The allowed ranges, as ip_network objects (bad ones are refused when the setting is saved, and skipped here).""" + out = [] + for v in values or (): + try: + out.append(ipaddress.ip_network(str(v).strip(), strict=False)) + except ValueError: + continue + return out + + +def allowed(address, nets): + """True when `address` (an IP string, IPv4-mapped IPv6 too) is in one of `nets`.""" + try: + ip = ipaddress.ip_address(address.split("%")[0]) + except ValueError: + return False + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped: + ip = ip.ipv4_mapped + return any(ip.version == n.version and ip in n for n in nets) + + +class Relay: + """Listens on (host, port) and relays allowed clients to 127.0.0.1:target. `nets` can be changed while it runs.""" + + def __init__(self, host, port, target, nets, log=None): + self.host, self.port, self.target, self.nets = host, port, target, list(nets) + self.log = log or (lambda *a: None) + self.refused = 0 + relay = self + + class Handler(socketserver.BaseRequestHandler): + def handle(self): + relay._serve(self.request, self.client_address[0]) + + family = socket.AF_INET6 if ":" in host else socket.AF_INET + + class Server(socketserver.ThreadingTCPServer): + address_family = family + allow_reuse_address = True + daemon_threads = True + + self.server = Server((host, port), Handler) + self.port = self.server.server_address[1] + self.thread = None + + def start(self): + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True, name=f"share-relay-{self.port}") + self.thread.start() + return self + + def stop(self): + self.server.shutdown() + self.server.server_close() + + def _serve(self, client, address): + if not allowed(address, self.nets): + self.refused += 1 + self.log(f"file share: refused {address} on port {self.port} (not in the allowed networks)") + client.close() + return + try: + upstream = socket.create_connection(("127.0.0.1", self.target), timeout=10) + except OSError as e: + self.log(f"file share: the server on port {self.target} isn't answering: {e}") + client.close() + return + upstream.settimeout(None) + client.settimeout(None) + sel = selectors.DefaultSelector() + sel.register(client, selectors.EVENT_READ, upstream) + sel.register(upstream, selectors.EVENT_READ, client) + try: + while True: + for key, _ in sel.select(): + data = key.fileobj.recv(CHUNK) + if not data: + return + key.data.sendall(data) + except OSError: + pass + finally: + sel.close() + for s in (client, upstream): + try: + s.close() + except OSError: + pass diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index bf6baa566..c7bd288f1 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -450,6 +450,15 @@ def labels(): "max_per_minute": 600, "triage": False, }, + # the file tree over NFS (domain/nfs_share.py, docs/file-tree.md): off until turned on; read-only, shown as `account` + # (an account id; null: the first admin), to the machines in allowed_networks. bind is a startup setting only. + "nfs": { + "enabled": False, + "bind": "0.0.0.0", + "port": 2049, + "allowed_networks": ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "fc00::/7", "::1/128"], + "account": None, + }, # notifications to webhooks and Matterbridge (docs/notifications.md): targets reach public addresses only, and the # private networks listed here (a Matterbridge on the LAN or the Docker network); app_url is where links in messages # point (null: FRONTEND_URL) diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index 6a843ad4b..9ae491206 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -97,6 +97,20 @@ class LocalLlmStatus(ResponseModel): machine: dict[str, Any] +class NfsStatus(ResponseModel): + enabled: bool + phase: Literal["off", "running", "error"] + port: int + allowed_networks: list[str] + account: str | None = Field(None, description="whose view of the file tree the share shows, while running") + error: str | None = None + refused: int = Field(0, description="connections refused since it started, from outside the allowed networks") + log: list[str] = Field(default_factory=list, description="rclone's last lines") + process: str | None = Field(None, description="the worker running it") + mount: str = Field(description="the command that mounts it on Linux") + rclone: bool = Field(description="whether rclone is installed here") + + class Removed(ResponseModel): removed: bool diff --git a/fastapi_backend/tests/domain/test_nfs_share.py b/fastapi_backend/tests/domain/test_nfs_share.py new file mode 100644 index 000000000..84635a9e1 --- /dev/null +++ b/fastapi_backend/tests/domain/test_nfs_share.py @@ -0,0 +1,163 @@ +"""The file tree over NFS: the relay that lets only the allowed networks in, the runner that starts and stops rclone +with the setting, the settings' checks and the status over the API. rclone itself is a stand-in here (it isn't +installed in CI): a script that listens where `--addr` says and echoes, which is all the runner and relay rely on.""" + +from __future__ import annotations + +import socket +import socketserver +import sys +import threading +import time + +import pytest + +from app.domain import nfs_share, settings, share_relay + +FAKE_RCLONE = """#!{python} +import socketserver, sys +args = sys.argv[1:] +assert args[:2] == ["serve", "nfs"] and "--read-only" in args, args +host, port = args[args.index("--addr") + 1].rsplit(":", 1) + +class Echo(socketserver.BaseRequestHandler): + def handle(self): + while data := self.request.recv(4096): + self.request.sendall(data) + +print("Serving NFS on", host, port, flush=True) +socketserver.ThreadingTCPServer((host, int(port)), Echo).serve_forever() +""" + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def echo(port, payload=b"ping"): + """What comes back through the port; b"" when the connection is closed on us.""" + with socket.create_connection(("127.0.0.1", port), timeout=5) as s: + s.sendall(payload) + try: + return s.recv(4096) + except ConnectionResetError: + return b"" + + +@pytest.fixture +def echo_server(): + class Echo(socketserver.BaseRequestHandler): + def handle(self): + while data := self.request.recv(4096): + self.request.sendall(data) + + srv = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Echo) + srv.daemon_threads = True + threading.Thread(target=srv.serve_forever, daemon=True).start() + yield srv.server_address[1] + srv.shutdown() + srv.server_close() + + +def test_allowed_addresses(): + nets = share_relay.networks(["192.168.0.0/16", "fc00::/7", "not a network"]) + assert len(nets) == 2 + assert share_relay.allowed("192.168.1.20", nets) + assert share_relay.allowed("::ffff:192.168.1.20", nets) # IPv4 on a dual-stack socket + assert share_relay.allowed("fd12::1", nets) + assert not share_relay.allowed("8.8.8.8", nets) + assert not share_relay.allowed("garbage", nets) + + +def test_the_relay_lets_in_only_the_allowed_networks(echo_server): + relay = share_relay.Relay("127.0.0.1", 0, echo_server, share_relay.networks(["127.0.0.0/8"])).start() + try: + assert echo(relay.port) == b"ping" + relay.nets = share_relay.networks(["10.0.0.0/8"]) # changed while running + assert echo(relay.port) == b"" + assert relay.refused == 1 + finally: + relay.stop() + + +def test_settings_are_checked(db, cfg): + settings.save(db, cfg, "nfs", {"enabled": True, "allowed_networks": ["192.168.1.7", "192.168.1.0/24", "192.168.1.7/32"]}, "t") + assert settings.Settings(db, cfg).current()["nfs"]["allowed_networks"] == ["192.168.1.7/32", "192.168.1.0/24"] + for bad in ({"allowed_networks": []}, {"allowed_networks": ["0.0.0.0/0"]}, {"allowed_networks": ["nope"]}, + {"port": 0}, {"enabled": "yes"}, {"account": "me"}, {"bind": "0.0.0.0"}): # fmt: skip + with pytest.raises(ValueError): + settings.save(db, cfg, "nfs", bad, "t") + + +def test_the_runner_follows_the_setting(db, cfg, folder, monkeypatch): + from tests.helpers import make_user + + make_user(db, "ad@x.io", "admin password 1", admin=True) + exe = folder / "rclone" + exe.write_text(FAKE_RCLONE.format(python=sys.executable)) + exe.chmod(0o755) + monkeypatch.setattr(nfs_share, "rclone_binary", lambda: str(exe)) + seen = {} + + def remote(db_, cfg_, account): + seen["account"] = account["email"] + return ":s3:", {"RCLONE_S3_SECRET_ACCESS_KEY": "s3cret"} + + port = free_port() + cfg["nfs"]["bind"] = "127.0.0.1" + settings.save(db, cfg, "nfs", {"enabled": True, "port": port}, "t") + s = settings.Settings(db, cfg) + run = nfs_share.Runner(db, s.current, "test", remote=remote) + try: + run.tick() + assert run.phase == "running", (run.error, run.lines) + assert seen["account"] == "ad@x.io" # nobody picked: the first admin + assert "s3cret" not in " ".join(run.proc.args) # the key isn't on the command line + assert echo(port) == b"ping" + st = nfs_share.status(db, s.current()) + assert st["phase"] == "running" and st["account"] == "ad@x.io" and st["process"] == "test" + + settings.save(db, cfg, "nfs", {"allowed_networks": ["10.0.0.0/8"]}, "t") + run.tick() + assert echo(port) == b"" # this machine isn't allowed any more, without a restart + run.tick() + assert nfs_share.status(db, s.current())["refused"] == 1 + + settings.save(db, cfg, "nfs", {"enabled": False}, "t") + run.tick() + assert run.proc is None and run.relay is None + assert nfs_share.status(db, s.current())["phase"] == "off" + with pytest.raises(OSError): + socket.create_connection(("127.0.0.1", port), timeout=1).close() + finally: + run.stop_serving() + + +def test_it_says_what_is_wrong(db, cfg, monkeypatch): + settings.save(db, cfg, "nfs", {"enabled": True, "port": free_port()}, "t") + s = settings.Settings(db, cfg) + monkeypatch.setattr(nfs_share, "rclone_binary", lambda: None) + run = nfs_share.Runner(db, s.current, "test") + run.tick() + assert run.phase == "error" and "rclone" in run.error + assert nfs_share.status(db, s.current())["error"] == run.error + monkeypatch.setattr(nfs_share, "rclone_binary", lambda: "/bin/true") + run.next_try = time.monotonic() - 1 + run.tick() + assert run.phase == "error" and "admin" in run.error # no account to show the tree as + + +def test_status_over_the_api(client, db, monkeypatch): + from tests.helpers import login, make_user + + monkeypatch.setenv("LENS_NFS_PORT", "12049") + make_user(db, "ad@x.io", "admin password 1", admin=True) + make_user(db, "me@x.io", "member password 1") + h = login(client, "ad@x.io", "admin password 1") + j = client.get("/api/v1/settings/nfs/status", headers=h).json() + assert j["enabled"] is False and j["phase"] == "off" and j["port"] == 12049 and "port=12049" in j["mount"] + assert client.put("/api/v1/settings/nfs", headers=h, json={"allowed_networks": ["0.0.0.0/0"]}).status_code == 400 + m = login(client, "me@x.io", "member password 1") + assert client.get("/api/v1/settings/nfs/status", headers=m).status_code == 403 diff --git a/mkdocs.yml b/mkdocs.yml index d1f90077a..6965dce66 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -89,6 +89,7 @@ nav: - Storage: storage.md - Linked data (RDF): rdf.md - Fedora repository: fedora.md + - File shares: file-shares.md - API: api.md - MCP server: mcp.md - API work the design needs: backend-gaps.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index af944d0c7..e633d2db5 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1789,6 +1789,10 @@ export type { NewTokenResponse, NewTokenResponses, NextRun, + NfsStatus, + NfsStatusData, + NfsStatusResponse, + NfsStatusResponses, NodeParam, NodeTrace, NodeType, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index c0239bb7f..a22e9ab9d 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -912,6 +912,8 @@ import type { NewTokenData, NewTokenErrors, NewTokenResponses, + NfsStatusData, + NfsStatusResponses, NotSameSpeakerData, NotSameSpeakerErrors, NotSameSpeakerResponses, @@ -2467,6 +2469,20 @@ export class Admin { }); } + /** + * Nfs Status + * + * The file tree over NFS: whether it's serving, as whom, to which networks, and the command that mounts it. + */ + public static nfsStatus( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/settings/nfs/status", + ...options, + }); + } + /** * Tunnel Status * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index b25dc3338..684fa730e 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -7848,6 +7848,69 @@ export type NextRun = { [key: string]: unknown; }; +/** + * NfsStatus + */ +export type NfsStatus = { + /** + * Enabled + */ + enabled: boolean; + /** + * Phase + */ + phase: "off" | "running" | "error"; + /** + * Port + */ + port: number; + /** + * Allowed Networks + */ + allowed_networks: Array; + /** + * Account + * + * whose view of the file tree the share shows, while running + */ + account?: string | null; + /** + * Error + */ + error?: string | null; + /** + * Refused + * + * connections refused since it started, from outside the allowed networks + */ + refused?: number; + /** + * Log + * + * rclone's last lines + */ + log?: Array; + /** + * Process + * + * the worker running it + */ + process?: string | null; + /** + * Mount + * + * the command that mounts it on Linux + */ + mount: string; + /** + * Rclone + * + * whether rclone is installed here + */ + rclone: boolean; + [key: string]: unknown; +}; + /** * NodeParam */ @@ -19051,6 +19114,22 @@ export type RemoveLocalModelResponses = { export type RemoveLocalModelResponse = RemoveLocalModelResponses[keyof RemoveLocalModelResponses]; +export type NfsStatusData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/nfs/status"; +}; + +export type NfsStatusResponses = { + /** + * Successful Response + */ + 200: NfsStatus; +}; + +export type NfsStatusResponse = NfsStatusResponses[keyof NfsStatusResponses]; + export type TunnelStatusData = { body?: never; path?: never; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index cd2cf6bdc..4d37fd672 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -5,7 +5,7 @@ import { PlugZap, RefreshCw, Terminal } from "lucide-react"; import Link from "next/link"; import { useEffect, useRef, type ReactNode } from "react"; -import { Admin, Fedora, Metadata, Sensors, Sources } from "@/app/openapi-client"; +import { Admin, Fedora, Metadata, Sensors, Sources, Users } from "@/app/openapi-client"; import type { MatterbridgeStatus } from "@/app/openapi-client/types.gen"; import { ComponentsStatus } from "@/components/settings/components-status"; import { AllTokens } from "@/components/account/all-tokens"; @@ -384,6 +384,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { return ; case "sensors": return ; + case "nfs": + return ; case "uploads": return ( <> @@ -889,6 +891,55 @@ function SensorsBody({ ctx }: { ctx: BodyCtx }) { ); } +function NfsBody({ ctx }: { ctx: BodyCtx }) { + const client = useApiClient(); + const status = useQuery({ + queryKey: ["nfs-status"], + queryFn: () => data(Admin.nfsStatus({ client })), + refetchInterval: 10_000, + }); + const users = useQuery({ queryKey: ["users"], queryFn: () => data(Users.listUsers({ client })), staleTime: 15_000 }); + const s = status.data; + return ( + <> + {s && s.enabled && ( + + {s.phase === "running" ? ( + <> + Mount it on Linux with {s.mount} + {s.refused + ? `. Turned away ${s.refused} connection${s.refused === 1 ? "" : "s"} from other networks.` + : "."} + + ) : ( + (s.error ?? "A worker starts it within seconds.") + )} + + )} + + !u.disabled).map((u) => ({ value: String(u.id), label: u.email })), + ]} + /> + + +

+ With Docker, the worker container publishes it on port {s?.port ?? 12049} of this machine, reachable from this + machine only unless LENS_BIND or{" "} + LENS_SHARE_BIND opens it to the network; set{" "} + LENS_NFS_PORT in .env to use another port. +

+ + ); +} + function MailBody({ ctx }: { ctx: BodyCtx }) { const client = useApiClient(); const m = ctx.view.mail; diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 80deb8049..f26fd211b 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -76,6 +76,7 @@ export type SectionId = | "telemetry" | "fedora" | "sensors" + | "nfs" | "uploads" | "storage" | "documents" @@ -253,6 +254,13 @@ export const SECTIONS: SectionSpec[] = [ description: "Off unless you turn it on. An MQTT hub and a syslog listener in Lens’s workers, so routers, DNS servers and devices can send to it; and how long what they send is kept.", }, + { + id: "nfs", + label: "NFS share", + backend: ["nfs"], + description: + "Off unless you turn it on. The archive as folders and files over NFS, read-only: namespaces, collections, topics and entities as folders, assets as files, for machines on the networks you allow.", + }, { id: "uploads", label: "Uploads", @@ -1677,6 +1685,32 @@ export const FIELDS: FieldSpec[] = [ max: 1440, unit: "minutes", }, + // NFS share (opt-in) + { + section: "nfs", + key: "enabled", + label: "Share the file tree over NFS", + kind: "switch", + hint: "Read-only. Off: nothing listens", + }, + { + section: "nfs", + key: "account", + label: "Show it as", + kind: "select", + nullable: true, + options: [], + hint: "The share shows what this person sees in the web app, and nothing else", + }, + { + section: "nfs", + key: "allowed_networks", + label: "Networks it’s shared with", + kind: "lines", + mono: true, + hint: "One per line, like 192.168.1.0/24. NFS has no login, so machines anywhere else are turned away", + }, + { section: "nfs", key: "port", label: "Port inside Lens", kind: "int", min: 1, max: 65535 }, // Sensors (opt-in) { section: "sensors", @@ -2328,6 +2362,7 @@ export function parse(f: FieldSpec, ui: unknown): Parsed { } case "select": if (f.key === "frame_width") return { value: Number(ui) }; + if (f.section === "nfs" && f.key === "account") return { value: ui === "" || ui == null ? null : Number(ui) }; if (f.section === "files" && f.key === "connection") return { value: ui === "" || ui == null ? null : Number(ui) }; return { value: ui === "" && f.nullable ? null : ui }; @@ -2421,6 +2456,10 @@ export function crossErrors(values: Record): Record !/^[0-9a-f.:]+(\/\d{1,3})?$/i.test(x)); if (badNet) e["notifications.networks"] = `“${badNet}” isn’t a network like 192.168.1.0/24`; + const nNets = values["nfs.allowed_networks"] as string[] | undefined; + const badNNet = nNets?.find((x) => !/^[0-9a-f.:]+(\/\d{1,3})?$/i.test(x)); + if (badNNet) e["nfs.allowed_networks"] = `“${badNNet}” isn’t a network like 192.168.1.0/24`; + else if (nNets && nNets.length === 0) e["nfs.allowed_networks"] = "List at least one network"; const sNets = values["sensors.syslog_networks"] as string[] | undefined; const badSNet = sNets?.find((x) => !/^[0-9a-f.:]+(\/\d{1,3})?$/i.test(x)); if (badSNet) e["sensors.syslog_networks"] = `“${badSNet}” isn’t a network like 192.168.1.0/24`; diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx index 7918f3209..8ec677fa4 100644 --- a/nextjs-frontend/components/settings/settings-shell.tsx +++ b/nextjs-frontend/components/settings/settings-shell.tsx @@ -34,6 +34,7 @@ import { MessagesSquare, PackageCheck, RadioTower, + FolderTree, } from "lucide-react"; import Link from "next/link"; import { useCallback, useState } from "react"; @@ -82,6 +83,7 @@ const ICON: Record = { telemetry: Activity, fedora: Archive, sensors: RadioTower, + nfs: FolderTree, uploads: Upload, storage: Database, documents: FileType, diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 60a269183..e39930f3e 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2444,6 +2444,28 @@ } } }, + "/api/v1/settings/nfs/status": { + "get": { + "tags": [ + "admin" + ], + "summary": "Nfs Status", + "description": "The file tree over NFS: whether it's serving, as whom, to which networks, and the command that mounts it.", + "operationId": "nfs_status", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NfsStatus" + } + } + } + } + } + } + }, "/api/v1/settings/tunnel/status": { "get": { "tags": [ @@ -37833,6 +37855,104 @@ ], "title": "NextRun" }, + "NfsStatus": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled" + }, + "phase": { + "type": "string", + "enum": [ + "off", + "running", + "error" + ], + "title": "Phase" + }, + "port": { + "type": "integer", + "title": "Port" + }, + "allowed_networks": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Allowed Networks" + }, + "account": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Account", + "description": "whose view of the file tree the share shows, while running" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "refused": { + "type": "integer", + "title": "Refused", + "description": "connections refused since it started, from outside the allowed networks", + "default": 0 + }, + "log": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Log", + "description": "rclone's last lines" + }, + "process": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Process", + "description": "the worker running it" + }, + "mount": { + "type": "string", + "title": "Mount", + "description": "the command that mounts it on Linux" + }, + "rclone": { + "type": "boolean", + "title": "Rclone", + "description": "whether rclone is installed here" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "phase", + "port", + "allowed_networks", + "mount", + "rclone" + ], + "title": "NfsStatus" + }, "NodeParam": { "properties": { "name": { diff --git a/threat-model.json b/threat-model.json index bbc99cbd5..4c373a7b8 100644 --- a/threat-model.json +++ b/threat-model.json @@ -451,6 +451,15 @@ "provenance": "documented", "source": "docs/sensors.md \"Sensors are off until an admin turns them on\"" }, + { + "entry_point": "NFS share (port 12049 on the host, 2049 in the worker)", + "parameter": "NFS and MOUNT RPCs, client address", + "attacker_controllable": "yes", + "component": "optional", + "caller_must_enforce": "admin enables and picks the account; operator firewalls", + "provenance": "documented", + "source": "docs/file-shares.md \"NFS has no login\"" + }, { "entry_point": ".env, archive.yaml, LENS_*, ARCHIVE_SECRET_KEY, RCLONE_BINARY", "parameter": "process configuration", @@ -998,6 +1007,14 @@ "provenance": "documented", "source": "docs/sensors.md \"Sensors are off until an admin turns them on\"" }, + { + "property": "P-NFS", + "violation_symptom": "info-leak", + "severity_tier": "security", + "conditions": "the NFS share is off until an admin turns it on, is read-only, shows only what one chosen account sees, and lets in only allowed_networks", + "provenance": "documented", + "source": "docs/file-shares.md \"NFS has no login\"" + }, { "property": "P-CHILD-SECRETS", "violation_symptom": "info-leak", @@ -1381,7 +1398,8 @@ "Treat transcript text and MCP results as untrusted prompt input in your agents", "Keep the IIIF cookie SameSite=None; Secure by serving IIIF over HTTPS", "Verify webhook-signature on receipt; treat Slack and Discord target URLs as secrets", - "Keep logs that record request paths private when sensor push tokens are in use" + "Keep logs that record request paths private when sensor push tokens are in use", + "When the NFS share is on, list only your own networks and pick an account that sees only what the share should show" ], "known_misuse": [ { diff --git a/threat-model.md b/threat-model.md index 01f0272e9..957812bcb 100644 --- a/threat-model.md +++ b/threat-model.md @@ -282,6 +282,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | extensions (prompt, graph and `http` tools, hooks), custom nodes including `template` nodes (Q23) | tool spec, graph JSON, template text | yes, any writer for their own; sharing is gated | callback/code (declarative) | person enables only extensions they trust | *(documented, docs/assistant.md "other people's run only when an admin shared them")* | | Python code tools, shipped and admin-created templates, workflows, pipelines, routines | Python, Jinja source, regexes | admin only | callback/code | admin is trusted | *(maintainer, 2026-10)* | | MQTT 1883, syslog 5514, `POST /sensors/push/{token}/{stream}` | payloads, hub logins, token in the path | yes, once enabled | data, size/rate, x-credential | admin enables; operator firewalls | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | +| NFS share (port 12049 on the host, 2049 in the worker) | NFS and MOUNT RPCs, client address | yes, once enabled, from `nfs.allowed_networks` | data, x-network-source | admin enables and picks the account; operator firewalls | *(documented, docs/file-shares.md "NFS has no login")* | | `.env`, `archive.yaml`, `LENS_*`, `ARCHIVE_SECRET_KEY`, `RCLONE_BINARY` | process configuration | no (operator) | x-env-config | keep unreadable to others | *(maintainer, 2026-10)* | | workers and DB peers | DB credential, shared database rows | no (trusted peer) | x-peer | run peers on trusted hosts | *(inferred, Q9)* | @@ -384,6 +385,7 @@ Every row is claimed, disclaimed, N/A with a reason, or unresolved. Family short | Optional | Atomicity, recursion, serialization | N/A — flat, append-only readings stored as data | no multi-step state, nesting or object reconstruction | none | *(inferred, Q34)* | | Optional | Callback / collaborator execution | claimed | tunnel token and llama-server key stay off command lines | P-CHILD-SECRETS | *(documented, docs/remote-access.md "in its environment, not on its command line")* | | Optional | Reference lifecycle | claimed | each service off until an admin turns it on | P-SENSORS | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | +| Optional | Confidentiality | claimed | NFS share off until an admin turns it on; read-only; one account's view; allowed networks only | P-NFS | *(documented, docs/file-shares.md "NFS has no login")* | | Optional | Concurrency; resource cost | disclaimed | MQTT and syslog connection counts, one thread per connection | N-VOLUME | *(documented, SECURITY.md "denial of service by sheer volume")* | ## 1.8 Assumptions and guarantees about outputs @@ -498,6 +500,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-TELEMETRY | Telemetry is off unless turned on, has no built-in destination, and never sends content, names, addresses or error messages. | info-leak | security-critical | *(documented, docs/telemetry.md "Never sent")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-MODEL-CHECKSUM | Catalog GGUF models are pinned to a revision and checked against their SHA-256. | integrity-bypass | security-critical | *(documented, docs/local-models.md "checked against their SHA-256")* | Voided by: `hf:` references have no pin or hash — `domain/local_llm.py:138-139`. | | P-SENSORS | Sensors are off until an admin enables them. MQTT needs a hub login unless `mqtt_anonymous` is on. Syslog hears only listed networks. | integrity-bypass | security-critical | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | Voided by: `sensors.mqtt_anonymous` — `domain/sensors.py:984`; `sensors.syslog_networks`, `sensors.bind`. | +| P-NFS | The NFS share is off until an admin turns it on. It is read-only, shows only what one chosen account sees in the web app, and turns away every client outside `nfs.allowed_networks` before rclone reads a byte. | info-leak | security-critical | *(documented, docs/file-shares.md "NFS has no login")* | Voided by: `nfs.allowed_networks` set wide, or a client address spoofed on the local network (NFS has no login) — `domain/share_relay.py`; Docker publishing that rewrites client addresses. | | P-CHILD-SECRETS | The tunnel token reaches cloudflared in its environment, not its command line; llama-server gets its own random API key. | info-leak | correctness-only | *(documented, docs/remote-access.md "in its environment, not on its command line")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-REMOTE-NAMES | An IMAP message is named by mailbox validity and uid, so a rebuilt mailbox never serves another message under an old name. | wrong-output | correctness-only | *(documented, docs/processing.md "an old one never reads another message")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-NOTIFY | Each notification event is claimed once across processes and retried with backoff. | wrong-output | correctness-only | *(documented, docs/notifications.md "claimed once")* | Voided by: none identified (not required for this symptom; no exhaustive search) | @@ -597,6 +600,7 @@ The operator (and, for the last rows, integrators that consume Lens output) must | After a password change or suspected theft, revoke API tokens and app grants, or disable the account | N-REVOKE-TOKENS | *(inferred, Q3)* | | List private networks in `documents.web_networks` or `notifications.networks` only on purpose | P-SSRF-GUARD, P-ADMIN-GUARD | *(documented, docs/configuration.md "loopback and cloud metadata addresses stay out")* | | When sensors are on, firewall ports 1883 and 5514 or bind them to a private address | P-SENSORS, N-MQTT-SUBSCRIBE | *(documented, docs/sensors.md "Ways in")* | +| When the NFS share is on, list only your own networks and pick an account that sees only what the share should show | P-NFS | *(documented, docs/file-shares.md "NFS has no login")* | | Give the chat-room bridge its own low-role account | N-BRIDGE | *(documented, docs/chat-rooms.md "it reads only what that account can read")* | | Run workers only on trusted hosts; they hold the DB credential | N-APP-LAYER | *(inferred, Q9)* | | Pick model and speech servers you trust with transcripts and audio | §1.8 prompts | *(maintainer, 2026-10)* | diff --git a/threat-model.yaml b/threat-model.yaml index 872a1a204..56077006b 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:c7c945e760490e73dbb947e89beff868a218b89b839b4172f9d928d4853e992a" +prose_version: "threat-model.md@sha256:0647ba2329f20585ed3cabc256ed0ca822b55437d90c39ddb1b30da4807a0eab" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} @@ -389,6 +389,16 @@ entry_points: obligation_id: enable-and-firewall-sensors caller_must_enforce: "admin enables; operator firewalls" provenance: {kind: documented, source: 'docs/sensors.md "Sensors are off until an admin turns them on"'} + - id: "NFS share (port 12049 on the host, 2049 in the worker)" + component: optional + parameters: + - name: "NFS and MOUNT RPCs, client address" + attacker_controllable: true + x-condition: "once enabled, from nfs.allowed_networks" + control_kinds: [data, x-network-source] + obligation_id: enable-and-limit-nfs + caller_must_enforce: "admin enables and picks the account; operator firewalls" + provenance: {kind: documented, source: 'docs/file-shares.md "NFS has no login"'} - id: ".env, archive.yaml, LENS_*, ARCHIVE_SECRET_KEY, RCLONE_BINARY" component: outbound parameters: @@ -523,6 +533,7 @@ contract_dimensions: - {component: optional, dimension: serialization-reconstruction, status: not-applicable, conditions: "readings stored as data; no object reconstruction", provenance: {kind: inferred, question_id: Q34}} - {component: optional, dimension: callback-execution, status: claimed, property_id: P-CHILD-SECRETS, conditions: "tunnel token and llama-server key stay off command lines", provenance: {kind: documented, source: 'docs/remote-access.md "in its environment, not on its command line"'}} - {component: optional, dimension: reference-lifecycle, status: claimed, property_id: P-SENSORS, conditions: "each service off until an admin turns it on", provenance: {kind: documented, source: 'docs/sensors.md "Sensors are off until an admin turns them on"'}} + - {component: optional, dimension: confidentiality, status: claimed, property_id: P-NFS, conditions: "NFS share off until an admin turns it on; read-only; one account's view; allowed networks only", provenance: {kind: documented, source: 'docs/file-shares.md "NFS has no login"'}} - {component: optional, dimension: concurrency-reentrancy, status: disclaimed, property_id: N-VOLUME, conditions: "MQTT and syslog connection counts, one thread per connection", provenance: {kind: documented, source: 'SECURITY.md "denial of service by sheer volume"'}} - {component: optional, dimension: resource-complexity, status: disclaimed, property_id: N-VOLUME, conditions: "MQTT and syslog connection counts, one thread per connection", provenance: {kind: documented, source: 'SECURITY.md "denial of service by sheer volume"'}} @@ -1384,6 +1395,14 @@ properties_claimed: violation_symptoms: [integrity-bypass] x-voided-by: "sensors.mqtt_anonymous (domain/sensors.py:984); sensors.syslog_networks, sensors.bind" provenance: {kind: documented, source: 'docs/sensors.md "Sensors are off until an admin turns them on"'} + - id: P-NFS + kind: confidentiality + components: [optional] + tier: security-critical + conditions: "the NFS share is off until an admin turns it on, is read-only, shows only what one chosen account sees, and lets in only allowed_networks" + violation_symptoms: [info-leak] + x-voided-by: "nfs.allowed_networks set wide, or a spoofed client address on the local network (domain/share_relay.py); Docker publishing that rewrites client addresses" + provenance: {kind: documented, source: 'docs/file-shares.md "NFS has no login"'} - id: P-CHILD-SECRETS kind: confidentiality components: [optional] @@ -1517,6 +1536,7 @@ downstream_responsibilities: - {id: dr-untrusted-prompt-input, component: assistant, statement: "Treat transcript text and MCP results as untrusted prompt input in your agents", enforces: [N-PROMPT, dmna-mcp-text-as-instructions], provenance: {kind: inferred, question_id: Q26}} - {id: dr-iiif-over-https, component: authz, statement: "Keep the IIIF cookie SameSite=None; Secure by serving IIIF over HTTPS", enforces: [], x-protects: "IIIF cookie", provenance: {kind: documented, source: 'docs/iiif.md "its cookie is `SameSite=None; Secure`"'}} - {id: dr-verify-webhook-signature, component: outbound, statement: "Verify webhook-signature on receipt; treat Slack and Discord target URLs as secrets", enforces: [dmna-chat-webhooks-signed], x-protects: "§1.8 webhook bodies", provenance: {kind: documented, source: 'domain/notify.py docstring "signed the Standard Webhooks way"'}} + - {id: dr-limit-nfs-share, component: optional, statement: "When the NFS share is on, list only your own networks and pick an account that sees only what the share should show", enforces: [P-NFS, enable-and-limit-nfs], provenance: {kind: documented, source: 'docs/file-shares.md "NFS has no login"'}} - {id: dr-private-logs-push-tokens, component: optional, statement: "Keep logs that record request paths private when sensor push tokens are in use", enforces: [P-SENSORS], provenance: {kind: documented, source: 'docs/sensors.md "Ways in"'}} # ---------------------------------------------------------------- §1.14 From 5d0e7c77d648d91087bc4a9033d4a0af8209891d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:47:19 +0000 Subject: [PATCH 2/3] fix(shares): read through the S3 server's loopback endpoint Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EKnSy4KKeNYsqhnXvWVz97 --- docs/file-shares.md | 3 ++- fastapi_backend/app/domain/nfs_share.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/file-shares.md b/docs/file-shares.md index bc42909e0..4e5124d83 100644 --- a/docs/file-shares.md +++ b/docs/file-shares.md @@ -46,7 +46,8 @@ own networks, and keep the port off the internet. ### How it works A worker runs `rclone serve nfs --read-only` (rclone is in Lens's image already; MIT licensed) over Lens's own S3 -server, signed in with an S3 key made for `nfs.account`. rclone listens on 127.0.0.1 inside the container; a small +server (a copy on 127.0.0.1 inside the worker, so the S3 share itself needn't be on), signed in with an S3 key +made for `nfs.account`, which is revoked when the share stops. rclone listens on 127.0.0.1 inside the container; a small relay in Lens (`domain/share_relay.py`) holds the published port and hands rclone only the connections from allowed networks. One worker serves it at a time (a lease on `app_service:nfs`), and Settings shows which one, whose view it shows, how many connections it turned away, and rclone's last lines when something goes wrong. Agents read the same diff --git a/fastapi_backend/app/domain/nfs_share.py b/fastapi_backend/app/domain/nfs_share.py index 73874deb3..8f3d6139a 100644 --- a/fastapi_backend/app/domain/nfs_share.py +++ b/fastapi_backend/app/domain/nfs_share.py @@ -100,12 +100,14 @@ class ShareError(RuntimeError): pass -def s3_remote(db, cfg, account): +def s3_remote(db, cfg_fn, account): """rclone's command-line remote and environment for Lens's own S3 server (s3.py), signed in as `account` with a - key s3.py keeps for this share.""" + key s3.py keeps for this share. It reads from the S3 server's loopback copy in this process, which runs whether or + not the S3 share itself is on.""" from . import s3 # noqa: PLC0415 - optional until the S3 server is in - endpoint, key_id, secret = s3.service_key(db, cfg, account["id"], "nfs") + _, key_id, secret = s3.service_key(db, cfg_fn(), account["id"], "nfs") + endpoint = s3.internal_endpoint(db, cfg_fn) env = { "RCLONE_S3_PROVIDER": "Other", "RCLONE_S3_ENDPOINT": endpoint, @@ -194,7 +196,7 @@ def _start(self, cfg, c): if not exe: raise ShareError("rclone isn't installed here (it is in Lens's Docker image)") account = _account(self.db, c) - remote, env_extra = self.remote(self.db, cfg, account) + remote, env_extra = self.remote(self.db, self.cfg_fn, account) inner = _free_port() cmd = [exe, "serve", "nfs", remote, "--addr", f"127.0.0.1:{inner}", "--read-only", "--vfs-cache-mode", "off", "--dir-cache-time", "30s", "--no-checksum"] # fmt: skip From bf51173aadbcff3c0ceffa2842b5bbf516a3f575 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 11 Oct 2026 01:02:01 +0000 Subject: [PATCH 3/3] test(login): wait for the cool-down button instead of racing its effect The throttled sign-in test read the button right after the alert appeared, but the cool-down state is set in an effect that can land a tick later, which failed CI on an unrelated PR. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EKnSy4KKeNYsqhnXvWVz97 --- nextjs-frontend/__tests__/loginForm.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nextjs-frontend/__tests__/loginForm.test.tsx b/nextjs-frontend/__tests__/loginForm.test.tsx index a04ee381d..2ada0dd3a 100644 --- a/nextjs-frontend/__tests__/loginForm.test.tsx +++ b/nextjs-frontend/__tests__/loginForm.test.tsx @@ -117,6 +117,7 @@ describe("LoginForm", () => { fillAndSubmit(); expect(await screen.findByRole("alert")).toHaveTextContent("Too many attempts"); - expect(screen.getByRole("button", { name: /try again in a few minutes/i })).toBeDisabled(); + // the cool-down starts in an effect after the alert renders, so wait for it rather than racing it + expect(await screen.findByRole("button", { name: /try again in a few minutes/i })).toBeDisabled(); }); });