diff --git a/CHANGELOG.md b/CHANGELOG.md index 57709fa36..57a6061c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,12 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version `automation_actions`, `save_automation`, `test_automation` and `run_automation`, with problems in plain words. The assistant saves them switched off and asks before switching one on or running it ([Agents](docs/automations.md#agents)). +- **File shares through Remote access.** S3 and WebDAV can now be reached at the web app's own address, `/s3` and + `/dav`, so they work through the Cloudflare tunnel and HTTPS as well as on the network, and on Cloudron, which + publishes only the web app. It's off until an admin turns on **At the web app's address too** in Settings › File + shares; then a new access key's dialog gives those addresses. Signing in is the same: a person's own access key, + read-only. MQTT, syslog, SMB and NFS aren't HTTP and stay on their own ports + ([The graph as files](docs/file-tree.md#at-the-web-apps-address), [Remote access](docs/remote-access.md#what-it-reaches)). - **Versions in conversations.** A conversation is now a tree: editing a question makes a new version beside it instead of deleting what followed, regenerating an answer (`regenerate`, with the same model or another) writes one beside it, and a question can go on from any message (`parent`). `GET /api/v1/chats/` shows each message's diff --git a/Dockerfile.cloudron b/Dockerfile.cloudron index b9cd33677..5c9c8b863 100644 --- a/Dockerfile.cloudron +++ b/Dockerfile.cloudron @@ -54,7 +54,7 @@ RUN UV_PROJECT_ENVIRONMENT=/app/code/venv UV_PYTHON_DOWNLOADS=never UV_COMPILE_B # The web app, built in the stage above; Next writes its cache to /run (the code is read-only on Cloudron) WORKDIR /app/code/frontend -COPY --from=frontend /app/package.json /app/next.config.mjs ./ +COPY --from=frontend /app/package.json /app/next.config.mjs /app/lens-relay.js ./ COPY --from=frontend /app/node_modules ./node_modules COPY --from=frontend /app/.next ./.next COPY --from=frontend /app/public ./public diff --git a/cloudron/supervisord.conf b/cloudron/supervisord.conf index 4c19b77cb..45665f8bd 100644 --- a/cloudron/supervisord.conf +++ b/cloudron/supervisord.conf @@ -54,6 +54,8 @@ redirect_stderr=true priority=40 user=cloudron directory=/app/code/frontend +; lens-relay.js: the file shares at the app's address, /s3 and /dav, once allowed in Settings › File shares +environment=NODE_OPTIONS="--require /app/code/frontend/lens-relay.js" command=/app/code/frontend/node_modules/.bin/next start --hostname 0.0.0.0 --port 3000 autorestart=true stopasgroup=true diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e0f3a35a0..227f34b16 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -151,6 +151,9 @@ services: AUTH_TRUST_HOST: "true" # the containers running the Cloudflare tunnel: their X-Forwarded-For names the visitor (lens-server.js) LENS_WEB_PROXY_HOSTS: backend,worker + # the file shares at the web app's address, /s3 and /dav (lens-relay.js), once allowed in Settings › File shares + LENS_S3_URL: http://worker:9555 + LENS_WEBDAV_URL: http://worker:9556 # no AUTH_URL: sign-in follows the address the browser is on (its LAN name, https:// address or Cloudflare tunnel) ports: # this machine only by default; LENS_BIND=0.0.0.0 to reach it from the network (install.sh does on a server) diff --git a/docker-compose.yml b/docker-compose.yml index 59152fe63..36b933e5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,6 +118,10 @@ services: AUTH_TRUST_HOST: "true" # no AUTH_URL: sign-in follows the address the browser is on (its LAN name, https:// address or Cloudflare tunnel) OPENAPI_OUTPUT_FILE: ./shared-data/openapi.json + # the file shares at the web app's address, /s3 and /dav, once allowed in Settings › File shares (lens-relay.js) + NODE_OPTIONS: --require /app/lens-relay.js + LENS_S3_URL: http://worker:9555 + LENS_WEBDAV_URL: http://worker:9556 volumes: - ./nextjs-frontend:/app - /app/node_modules # the image's, renewed with it (see the backend's /opt/venv) diff --git a/docs/configuration.md b/docs/configuration.md index 2759825f7..cc778fe21 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -90,6 +90,7 @@ whether the wizard is still pending (`wizard_pending`). | `AUTH_TRUST_HOST` | `true` behind a proxy or in Docker | | `TRUST_PROXY_HEADERS` | `true` when a reverse proxy in front of the web app sets `X-Forwarded-Host`, `-Proto` and `-For`: they're passed on to the API, which names that address in OAuth discovery and reads the visitor's address for IP groups and throttles. Off, the web app reports the `Host` the browser sent and the address it was reached from | | `LENS_WEB_PROXY_HOSTS` | hosts whose `X-Forwarded-For` the web app passes on with `TRUST_PROXY_HEADERS` off (`backend,worker` in the Docker Compose files, for the Cloudflare tunnel) | +| `LENS_S3_URL`, `LENS_WEBDAV_URL` | where the web app finds the file shares it relays at `/s3` and `/dav` once Settings › File shares allows it (`http://worker:9555` and `http://worker:9556` in the Docker Compose files; unset, this machine's ports 9555 and 9556). See [The graph as files](file-tree.md#at-the-web-apps-address) | | `LENS_HTTPS`, `LENS_HTTPS_PORT`, `LENS_HTTP_PORT`, `LENS_HTTPS_BIND` | install.sh: `LENS_HTTPS=1` publishes the HTTPS and HTTP ports (443 and 80 by default) for [HTTPS](https.md) | | `LENS_HTTPS_LISTEN` | `1` in the process that has the HTTPS ports and runs Caddy (the worker in `docker-compose.https.yml`); `LENS_CADDY`: a caddy to use instead of fetching one | diff --git a/docs/file-tree.md b/docs/file-tree.md index 26b6cddac..b4b520000 100644 --- a/docs/file-tree.md +++ b/docs/file-tree.md @@ -106,9 +106,43 @@ namespaces. Listing (PROPFIND, one level at a time) and reading (GET, with range Like S3, it speaks plain HTTP, and Basic sign-in sends the secret with every request: keep it on this machine or behind TLS on a network you don't trust. +## At the web app's address + +The web app can relay both shares at its own address: S3 at `/s3` and WebDAV at `/dav`. They then work wherever the web +app opens, including through [Remote access](remote-access.md) and [HTTPS](https.md), which serve the web app over +`https://` and reach nothing else. It's off until an admin turns on **At the web app's address too** in Settings, File +shares (`shares.remote`), on top of the shares themselves; until then requests that come that way are turned away +before anyone signs in, and the shares' own ports work as before. + +Once it's on, the dialog that shows a new access key gives these addresses: + +```ini +[lens] +type = s3 +provider = Other +access_key_id = LK… +secret_access_key = … +endpoint = https://lens.example.com/s3 +force_path_style = true +``` + +and `https://lens.example.com/dav/` for WebDAV (Finder, Windows Explorer, `davs://lens.example.com/dav/` in GNOME Files +or Dolphin, rclone's `type = webdav`). The S3 endpoint has a path, so it needs a client that takes one: rclone, the AWS +CLI and boto3 do. + +How it works: the web app hands requests under `/s3` and `/dav` to the worker's share servers before Next.js sees them +(`nextjs-frontend/lens-relay.js`; the servers are at `LENS_S3_URL` and `LENS_WEBDAV_URL`, set in the Docker Compose +files, else this machine's ports 9555 and 9556). The path and `Host` stay as the client sent them, since S3 clients +sign both, and `X-Forwarded-Prefix` tells the share server the request came through the web app. Failed WebDAV +sign-ins are counted per visitor, from `X-Forwarded-For` as the web app and the proxies Lens trusts report it +(Settings, Access & embedding, Trusted proxies). Signing in is the same as at the ports: a person's own access key, +read-only. + ## Refine later - TLS by itself for S3 and WebDAV, folders' real dates, and WebDAV locks for clients that insist on them. +- Virtual-host S3 addressing at the web app's address, for clients that can't take an endpoint with a path (MinIO's + `mc`). - Show supplementary files (captions, translations, attachments) beside each asset. - Cache an asset's text and metadata so large folders list faster. - A sidecar for each collection, topic and entity, and a folder of notes. diff --git a/docs/https.md b/docs/https.md index 462f7ccb3..3bc22a2bf 100644 --- a/docs/https.md +++ b/docs/https.md @@ -113,6 +113,9 @@ All in Settings › HTTPS (`PUT /api/v1/settings/https`; `POST /api/v1/settings/ Passkeys and sign-in follow the https:// address, and email links use the first domain unless `notifications.app_url` says otherwise. - **Changes**: a saved change restarts Caddy with the new configuration within a few seconds. +- **What it serves**: the web app, and what the web app serves at its own address: the API, the MCP server at `/mcp`, + and the file shares at `/s3` and `/dav` once Settings › File shares allows them there + ([The graph as files](file-tree.md#at-the-web-apps-address)). MQTT, syslog, SMB and NFS keep their own ports. By hand, with Docker: diff --git a/docs/remote-access.md b/docs/remote-access.md index f06c7ca1e..1a8d47bb9 100644 --- a/docs/remote-access.md +++ b/docs/remote-access.md @@ -44,6 +44,18 @@ command (the long string after `--token`). - The tunnel's address counts as one Lens is served at: passkeys and sign-in redirects use `https://`, and email links use the fixed hostname unless Settings › Notifications sets another address. +## What it reaches + +The tunnel serves the web app and what the web app serves at its own address: + +| | Through the tunnel | +| --- | --- | +| The web app, the API (`/api/v1`) and sign-in | Yes | +| The MCP server (`/mcp`, with OAuth) | Yes ([MCP server](mcp.md)) | +| Sensors that push over HTTP (`/api/v1/sensors/push/…`) | Yes ([Sensors](sensors.md)) | +| The file shares: S3 at `/s3` and WebDAV at `/dav` | Once an admin turns on **At the web app's address too** in Settings › File shares ([The graph as files](file-tree.md#at-the-web-apps-address)) | +| MQTT, syslog, SMB and NFS | No: they aren't HTTP, so they stay on their own ports, for the local network | + ## Where it sends visitors cloudflared reaches the web app at, in order: the **web app address** in Settings › Remote access, the diff --git a/fastapi_backend/app/api/deps.py b/fastapi_backend/app/api/deps.py index 4815701fc..6f28b0c16 100644 --- a/fastapi_backend/app/api/deps.py +++ b/fastapi_backend/app/api/deps.py @@ -381,33 +381,9 @@ def client_ip(request: Request) -> str: return str(addr) if addr else (request.client.host if request.client else "") -_PROXY_HOSTS: dict[str, tuple[float, tuple[str, ...]]] = {} - - -def _proxy_hosts() -> tuple[str, ...]: - """The addresses of the hosts named in LENS_TRUSTED_PROXY_HOSTS (the web app's container, `frontend` in the - Docker Compose files, whose address changes when it's recreated), looked up at most every 30 seconds.""" - import os - import socket - import time - - names = [n.strip() for n in os.environ.get("LENS_TRUSTED_PROXY_HOSTS", "").split(",") if n.strip()] - out: list[str] = [] - for name in names: - hit = _PROXY_HOSTS.get(name) - if not hit or time.monotonic() - hit[0] > 30: - try: - found = tuple(sorted({str(i[4][0]) for i in socket.getaddrinfo(name, None)})) - except OSError: - found = () - hit = _PROXY_HOSTS[name] = (time.monotonic(), found) - out += hit[1] - return tuple(out) - - def trusted_proxies(request: Request) -> tuple[str, ...]: """server.trusted_proxies, and the web app's own container (LENS_TRUSTED_PROXY_HOSTS).""" - return tuple(request.app.state.settings.current()["server"].get("trusted_proxies") or ()) + _proxy_hosts() + return tuple(request.app.state.settings.current()["server"].get("trusted_proxies") or ()) + ipgroups.proxy_hosts() def visitor_address(request: Request) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None: diff --git a/fastapi_backend/app/api/v1/routes/shares.py b/fastapi_backend/app/api/v1/routes/shares.py index 481e6df5d..9530cc2f8 100644 --- a/fastapi_backend/app/api/v1/routes/shares.py +++ b/fastapi_backend/app/api/v1/routes/shares.py @@ -18,10 +18,11 @@ @router.get("/s3") def s3_status(user: CurrentUser, db: Db, cfg: Cfg) -> S3Status: - """Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, and your - access keys.""" + """Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, whether + both are open at the web app's address too (/s3, /dav), and your access keys.""" st = shares.status(db, cfg) - return S3Status(**st["s3"], webdav=ShareServer(**st["webdav"]), keys=[S3Key(**k) for k in s3.keys_of(db, user.id)]) + keys = [S3Key(**k) for k in s3.keys_of(db, user.id)] + return S3Status(**st["s3"], webdav=ShareServer(**st["webdav"]), remote=shares.remote(cfg), keys=keys) @router.post("/s3/keys") diff --git a/fastapi_backend/app/domain/ipgroups.py b/fastapi_backend/app/domain/ipgroups.py index 605eee6ce..bcac945a0 100644 --- a/fastapi_backend/app/domain/ipgroups.py +++ b/fastapi_backend/app/domain/ipgroups.py @@ -13,6 +13,9 @@ import functools import ipaddress +import os +import socket +import time from typing import NamedTuple from . import store @@ -83,6 +86,26 @@ def address(raw): return (ip.ipv4_mapped or ip) if isinstance(ip, ipaddress.IPv6Address) else ip +_PROXY_HOSTS: dict[str, tuple[float, tuple[str, ...]]] = {} + + +def proxy_hosts() -> tuple[str, ...]: + """The addresses of the hosts named in LENS_TRUSTED_PROXY_HOSTS (the web app's container, `frontend` in the + Docker Compose files, whose address changes when it's recreated), looked up at most every 30 seconds.""" + names = [n.strip() for n in os.environ.get("LENS_TRUSTED_PROXY_HOSTS", "").split(",") if n.strip()] + out: list[str] = [] + for name in names: + hit = _PROXY_HOSTS.get(name) + if not hit or time.monotonic() - hit[0] > 30: + try: + found = tuple(sorted({str(i[4][0]) for i in socket.getaddrinfo(name, None)})) + except OSError: + found = () + hit = _PROXY_HOSTS[name] = (time.monotonic(), found) + out += hit[1] + return tuple(out) + + @functools.lru_cache(maxsize=256) def _networks(values): out = [] diff --git a/fastapi_backend/app/domain/s3.py b/fastapi_backend/app/domain/s3.py index e024cc9f0..642b7125c 100644 --- a/fastapi_backend/app/domain/s3.py +++ b/fastapi_backend/app/domain/s3.py @@ -9,6 +9,9 @@ Path-style addressing (http://host:port//), any region name. Supported: ListBuckets, HeadBucket, GetBucketLocation, ListObjects (v1 and v2, with a delimiter or recursive), HeadObject and GetObject (with Range). + +With `shares.remote` on, the web app relays it at its own address too (https:///s3//), path and +all: clients sign that path, and the bucket and key come after /s3 (shares.web_path()). """ from __future__ import annotations @@ -423,12 +426,20 @@ def _write(self): do_PUT = do_POST = do_DELETE = _write # noqa: N815 - http.server's names def _handle(self, head): + from . import shares + raw, _, query = self.path.partition("?") try: pairs = urllib.parse.parse_qsl(query, keep_blank_values=True) headers = {k.lower(): v for k, v in self.headers.items()} db, cfg = self.server.db, self.server.cfg_fn() + prefix = shares.web_path(headers, "s3") + if prefix and not shares.remote(cfg): + raise S3Error( + 403, "AccessDenied", "S3 isn't open at the web app's address: an admin turns that on in Settings, File shares" + ) acct = verify(db, cfg, self.command, raw, pairs, headers) + raw = shares.share_path(raw, prefix) tree = filetree.Tree(db, cfg, filetree.Viewer.of(db, acct)) path = urllib.parse.unquote(raw) bucket, _, key = path.lstrip("/").partition("/") diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index e707b6c8e..8b3d580c4 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -86,7 +86,7 @@ "triage", ), # bind is a startup setting only - "shares": ("enabled", "s3", "s3_port", "webdav", "webdav_port"), + "shares": ("enabled", "s3", "s3_port", "webdav", "webdav_port", "remote"), "video": ( "sample_seconds", "scene_threshold", @@ -1390,7 +1390,7 @@ def _web_setting(key, value): def _share_setting(key, value): - if key in ("enabled", "s3", "webdav"): + if key in ("enabled", "s3", "webdav", "remote"): if not isinstance(value, bool): raise ValueError(f"shares.{key} is true or false") return value diff --git a/fastapi_backend/app/domain/shares.py b/fastapi_backend/app/domain/shares.py index 64fc2a314..6320bf3fc 100644 --- a/fastapi_backend/app/domain/shares.py +++ b/fastapi_backend/app/domain/shares.py @@ -4,6 +4,10 @@ Off until an admin turns them on (`shares.enabled`, then `s3` and `webdav`); the process that runs routines (`lens worker`) follows the settings as they change, as the sensor hub does, and says what each server is doing in `app_service:share-`. + +The web app also relays them at its own address, /s3 and /dav (nextjs-frontend/lens-relay.js), so they can be reached +wherever the web app is, Settings › Remote access included. Those requests keep their path and carry X-Forwarded-Prefix, +and the servers turn them away until an admin turns on `shares.remote`. """ from __future__ import annotations @@ -15,7 +19,7 @@ from collections.abc import Callable from typing import Any -from . import s3, store, webdav +from . import ipgroups, s3, store, webdav R = store.R CHECK_SECONDS = 5 @@ -27,6 +31,11 @@ } +# protocol -> the path the web app relays it at (nextjs-frontend/lens-relay.js) +WEB_PATHS = {"s3": "/s3", "webdav": "/dav"} +LOOPBACK = ("127.0.0.0/8", "::1/128") + + def _row(proto): return R("app_service", f"share-{proto}") @@ -55,6 +64,36 @@ def settings_of(cfg): return {**store.DEFAULTS["shares"], **(cfg.get("shares") or {})} +def web_path(headers, proto): + """The path the web app relays this protocol at, when the request came that way (its X-Forwarded-Prefix), else "". + Anyone on the network could send the header too; it only gets their request turned away, or signed and linked + under that path.""" + prefix = (headers.get("x-forwarded-prefix") or "").strip() + return prefix if prefix == WEB_PATHS[proto] else "" + + +def share_path(path, prefix): + """The path a request asks for within the share: without the web app's prefix (the web app relays the path as the + client sent it, which is also the path S3 clients sign).""" + if prefix and (path == prefix or path.startswith(prefix + "/")): + return path[len(prefix) :] or "/" + return path + + +def remote(cfg): + """Whether the shares may be reached at the web app's address (shares.remote).""" + return bool(settings_of(cfg).get("remote")) + + +def visitor(handler, cfg): + """The address a request comes from, for throttles: what the web app (and proxies it trusts: server.trusted_proxies, + LENS_TRUSTED_PROXY_HOSTS, this machine) report in X-Forwarded-For, else the connection's peer.""" + peer, port = handler.client_address[0], handler.client_address[1] + trusted = tuple((cfg.get("server") or {}).get("trusted_proxies") or ()) + ipgroups.proxy_hosts() + LOOPBACK + addr = ipgroups.client_address(peer, port or 1, handler.headers.get("x-forwarded-for", ""), trusted) + return str(addr) if addr else peer + + def wanted(cfg): """{protocol: (bind, port) it should listen on, or None while it's off}.""" s = settings_of(cfg) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 2298c1e03..0f9fdc8f6 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -587,8 +587,17 @@ def labels(): "account": None, }, # the graph as files over S3 and WebDAV (shares.py, docs/file-tree.md): off until an admin turns it on. Then the process - # that runs routines (`lens worker`) serves it, read-only, on these ports. bind is a startup setting only. - "shares": {"enabled": False, "bind": "0.0.0.0", "s3": True, "s3_port": 9555, "webdav": True, "webdav_port": 9556}, + # that runs routines (`lens worker`) serves it, read-only, on these ports. bind is a startup setting only. remote: also at + # the web app's address (/s3, /dav), so through Settings › Remote access and HTTPS too; off until an admin turns it on. + "shares": { + "enabled": False, + "bind": "0.0.0.0", + "s3": True, + "s3_port": 9555, + "webdav": True, + "webdav_port": 9556, + "remote": False, + }, # notifications to webhooks and Matterbridge (docs/notifications.md): targets reach public addresses only, and the # private networks listed here (a Matterbridge on the LAN or the Docker network); app_url is where links in messages # point (null: FRONTEND_URL) diff --git a/fastapi_backend/app/domain/webdav.py b/fastapi_backend/app/domain/webdav.py index 4e3ec99d1..accae3b6d 100644 --- a/fastapi_backend/app/domain/webdav.py +++ b/fastapi_backend/app/domain/webdav.py @@ -4,6 +4,9 @@ Each person signs in with one of their access keys (s3.py): the key id as the user name and its secret as the password (HTTP Basic). They see the namespaces and collections the web app shows them. OPTIONS, PROPFIND (depth 0 or 1), GET and HEAD (with Range); anything that writes answers 403. Run by shares.py. + +With `shares.remote` on, the web app relays it at its own address too (https:///dav/), and links in listings +start with /dav (shares.web_path()). """ from __future__ import annotations @@ -32,11 +35,11 @@ def _http_date(s): return email.utils.format_datetime((t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc)).astimezone(dt.timezone.utc), usegmt=True) -def href(parts, folder): - return "/" + "/".join(urllib.parse.quote(p, safe="") for p in parts) + ("/" if folder and parts else "") +def href(parts, folder, prefix=""): + return prefix + "/" + "/".join(urllib.parse.quote(p, safe="") for p in parts) + ("/" if folder and parts else "") -def _response(parts, e): +def _response(parts, e, prefix=""): props = [f"{escape(e.name)}", f"{_http_date(e.modified)}"] if e.folder: props.append("") @@ -48,17 +51,17 @@ def _response(parts, e): f'"{e.version}"', ] return ( - f"{escape(href(parts, e.folder))}{''.join(props)}" + f"{escape(href(parts, e.folder, prefix))}{''.join(props)}" "HTTP/1.1 200 OK" ) -def multistatus(tree, parts, depth): - """The PROPFIND answer for a path: itself, and with depth 1 what's in it.""" +def multistatus(tree, parts, depth, prefix=""): + """The PROPFIND answer for a path: itself, and with depth 1 what's in it. `prefix`: the path the share is at.""" e = tree.stat("/".join(parts)) - out = [_response(parts, e)] + out = [_response(parts, e, prefix)] if depth != "0" and e.folder: - out += [_response([*parts, k.name], k) for k in tree.list("/".join(parts))] + out += [_response([*parts, k.name], k, prefix) for k in tree.list("/".join(parts))] return ('\n' + "".join(out) + "").encode() @@ -88,9 +91,27 @@ def _drain(self): return self.rfile.read(n) + def _prefix(self): + from . import shares + + return shares.web_path(self.headers, "webdav") + + def _closed(self): + """True (and turned away) when the request came through the web app while that's not allowed.""" + from . import shares + + if self._prefix() and not shares.remote(self.server.cfg_fn()): + self._send(403, b"WebDAV isn't open at the web app's address: an admin turns that on in Settings, File shares\n") + return True + return False + def _account(self): """The account the request signs in as, or None (an answer was sent).""" - ip = self.client_address[0] + from . import shares + + if self._closed(): + return None + ip = shares.visitor(self, self.server.cfg_fn()) if auth.throttled(f"webdav:{ip}"): self._send(429, b"too many failed sign-ins: try again later\n") return None @@ -109,16 +130,20 @@ def _account(self): return acct def _parts(self): - raw = self.path.partition("?")[0] + from . import shares + + raw = shares.share_path(self.path.partition("?")[0], self._prefix()) return [urllib.parse.unquote(p) for p in raw.split("/") if p] def do_OPTIONS(self): # noqa: N802 - http.server's names self._drain() - self._send(200, headers={"Allow": ALLOW, "MS-Author-Via": "DAV"}) + if not self._closed(): + self._send(200, headers={"Allow": ALLOW, "MS-Author-Via": "DAV"}) def _write(self): self._drain() - self._send(403, b"the file tree is read-only\n", headers={"Allow": ALLOW}) + if not self._closed(): + self._send(403, b"the file tree is read-only\n", headers={"Allow": ALLOW}) do_PUT = do_POST = do_DELETE = do_MKCOL = do_MOVE = do_COPY = do_PROPPATCH = do_LOCK = do_UNLOCK = _write # noqa: N815 @@ -135,7 +160,7 @@ def do_PROPFIND(self): # noqa: N802 if depth not in ("0", "1"): return self._send(403, b"PROPFIND with Depth: infinity isn't supported; ask for depth 1\n") try: - body = multistatus(self._tree(acct), self._parts(), depth) + body = multistatus(self._tree(acct), self._parts(), depth, self._prefix()) except filetree.NotFound: return self._send(404, b"not found\n") self._send(207, body, "application/xml; charset=utf-8") diff --git a/fastapi_backend/app/schemas/filetree.py b/fastapi_backend/app/schemas/filetree.py index b1be7cb74..ff6e88ef7 100644 --- a/fastapi_backend/app/schemas/filetree.py +++ b/fastapi_backend/app/schemas/filetree.py @@ -60,4 +60,8 @@ class S3Status(ShareServer): """The S3 server (these fields), the WebDAV server, and your access keys (they sign in to both).""" webdav: ShareServer + remote: bool = Field( + False, + description="whether an admin opened both at the web app's address too, /s3 and /dav (shares.remote), so remote access reaches them", + ) keys: list[S3Key] = Field(description="your access keys") diff --git a/fastapi_backend/tests/api/test_s3.py b/fastapi_backend/tests/api/test_s3.py index 14c757a4d..81cf97fc7 100644 --- a/fastapi_backend/tests/api/test_s3.py +++ b/fastapi_backend/tests/api/test_s3.py @@ -12,7 +12,7 @@ from botocore.exceptions import ClientError from app.domain import hierarchy, s3, shares, store, topics -from tests.helpers import login, make_user, seed +from tests.helpers import login, make_user, seed, web_relay @pytest.fixture @@ -34,10 +34,10 @@ def _key(client, h, **body): return r.json() -def _s3(port, kid, secret): +def _s3(port, kid, secret, path=""): return botocore.session.get_session().create_client( "s3", - endpoint_url=f"http://127.0.0.1:{port}", + endpoint_url=f"http://127.0.0.1:{port}{path}", region_name="us-east-1", aws_access_key_id=kid, aws_secret_access_key=secret, @@ -118,13 +118,45 @@ def test_what_s3_refuses(client, env): assert e.value.code == 403 +def test_at_the_web_apps_address_once_an_admin_allows_it(client, cfg, env): + """The web app relays S3 at /s3 (nextjs-frontend/lens-relay.js), so remote access reaches it: clients sign that + path, and it's turned away until an admin turns on shares.remote.""" + k = _key(client, env["hv"]) + with web_relay(env["port"], "/s3") as port: + c = _s3(port, k["id"], k["secret"], "/s3") + assert _code(lambda: c.list_buckets()) == "AccessDenied" + cfg["shares"] = {**(cfg.get("shares") or {}), "remote": True} + assert [b["Name"] for b in c.list_buckets()["Buckets"]] == ["pods"] + talks = c.list_objects_v2(Bucket="pods", Prefix="Collections/Talks/", Delimiter="/") + assert "Collections/Talks/ep2 [2].txt" in [o["Key"] for o in talks["Contents"]] + own = (env["folder"] / "ep1.txt").read_bytes() + got = c.get_object(Bucket="pods", Key="Collections/General/ep1 [1].original.txt", Range="bytes=2-5") + assert got["Body"].read() == own[2:6] + assert c.head_object(Bucket="pods", Key="Collections/General/ep1 [1].original.txt")["ContentLength"] == len(own) + link = c.generate_presigned_url("get_object", Params={"Bucket": "pods", "Key": "Collections/General/ep1 [1].original.txt"}) + assert "/s3/pods/" in link and urllib.request.urlopen(link).read() == own + assert _code(lambda: _s3(port, k["id"], "wrong", "/s3").list_buckets()) == "SignatureDoesNotMatch" + assert _code(lambda: c.put_object(Bucket="pods", Key="x.txt", Body=b"hi")) == "AccessDenied" + # at its own port, as before + assert [b["Name"] for b in _s3(env["port"], k["id"], k["secret"]).list_buckets()["Buckets"]] == ["pods"] + + +def test_admins_open_them_at_the_web_apps_address(client, db, env): + make_user(db, "ad@x.io", "admin password 1", admin=True) + ha = login(client, "ad@x.io", "admin password 1") + assert client.put("/api/v1/settings/shares", json={"remote": "yes"}, headers=ha).status_code == 400 + assert client.put("/api/v1/settings/shares", json={"remote": True}, headers=env["hv"]).status_code == 403 + assert client.put("/api/v1/settings/shares", json={"remote": True}, headers=ha).status_code == 200 + assert client.get("/api/v1/shares/s3", headers=env["hv"]).json()["remote"] is True + + def test_keys_are_made_while_signed_in(client, db, env): r = client.post("/api/v1/tokens", headers=env["hv"], json={"name": "script", "scope": "write", "days": 10}) token = {"Authorization": f"Bearer {r.json()['token']}"} assert client.post("/api/v1/shares/s3/keys", headers=token, json={}).status_code == 403 k = _key(client, env["hv"], name="nas") st = client.get("/api/v1/shares/s3", headers=env["hv"]).json() - assert st["enabled"] is False and st["port"] == 9555 and st["webdav"]["port"] == 9556 + assert st["enabled"] is False and st["port"] == 9555 and st["webdav"]["port"] == 9556 and st["remote"] is False assert [(x["id"], x["name"]) for x in st["keys"]] == [(k["id"], "nas")] and "secret" not in st["keys"][0] sealed = db.one("SELECT sealed FROM $r", r=store.R("file_key", k["id"]))["sealed"] assert k["secret"] not in sealed # kept sealed diff --git a/fastapi_backend/tests/api/test_visitors.py b/fastapi_backend/tests/api/test_visitors.py index 6f0857692..ccbcc3cdc 100644 --- a/fastapi_backend/tests/api/test_visitors.py +++ b/fastapi_backend/tests/api/test_visitors.py @@ -9,7 +9,7 @@ import pytest from fastapi.testclient import TestClient -from app.api import deps +from app.domain import ipgroups from tests.helpers import make_user WEB_APP = "172.18.0.5" @@ -27,9 +27,9 @@ def lookup(host, *a, **kw): return real(host, *a, **kw) monkeypatch.setattr(socket, "getaddrinfo", lookup) - deps._PROXY_HOSTS.clear() + ipgroups._PROXY_HOSTS.clear() yield TestClient(app, base_url="http://127.0.0.1", client=(WEB_APP, 50000)) - deps._PROXY_HOSTS.clear() + ipgroups._PROXY_HOSTS.clear() def bad_logins(client, visitor, n=8): diff --git a/fastapi_backend/tests/api/test_webdav.py b/fastapi_backend/tests/api/test_webdav.py index 45678effb..007f9afa6 100644 --- a/fastapi_backend/tests/api/test_webdav.py +++ b/fastapi_backend/tests/api/test_webdav.py @@ -10,7 +10,7 @@ import pytest from app.domain import auth, shares, webdav -from tests.helpers import login, make_user, seed +from tests.helpers import login, make_user, seed, web_relay D = "{DAV:}" @@ -27,8 +27,8 @@ def env(client, db, cfg, folder): server.stop() -def _call(env, method, path, headers=None, auth_header=True, body=None): - c = http.client.HTTPConnection("127.0.0.1", env["port"], timeout=10) +def _call(env, method, path, headers=None, auth_header=True, body=None, port=None): + c = http.client.HTTPConnection("127.0.0.1", port or env["port"], timeout=10) h = {**({"Authorization": env["auth"]} if auth_header else {}), **(headers or {})} c.request(method, path, body=body, headers=h) r = c.getresponse() @@ -77,3 +77,37 @@ def test_what_webdav_refuses(env): for _ in range(8): assert _call(env, "PROPFIND", "/", {"Depth": "0", "Authorization": wrong})[0] == 401 assert _call(env, "PROPFIND", "/", {"Depth": "0"})[0] == 429 # too many failed sign-ins from this address + + +def test_at_the_web_apps_address_once_an_admin_allows_it(env, cfg): + """The web app relays WebDAV at /dav (nextjs-frontend/lens-relay.js), so remote access reaches it; listings link + under /dav, and it's turned away, before any sign-in, until an admin turns on shares.remote.""" + with web_relay(env["port"], "/dav") as port: + assert _call(env, "OPTIONS", "/dav/", auth_header=False, port=port)[0] == 403 + assert _call(env, "PROPFIND", "/dav/", {"Depth": "1"}, port=port)[0] == 403 + cfg["shares"] = {**(cfg.get("shares") or {}), "remote": True} + status, h, _ = _call(env, "OPTIONS", "/dav/", auth_header=False, port=port) + assert status == 200 and "PROPFIND" in h["Allow"] + status, _, body = _call(env, "PROPFIND", "/dav/", {"Depth": "1"}, port=port) + assert status == 207 and [(x[0], x[1]) for x in _listing(body)] == [("/dav/", True), ("/dav/pods/", True)] + status, _, body = _call(env, "PROPFIND", "/dav/pods/Collections/General/", {"Depth": "1"}, port=port) + assert "/dav/pods/Collections/General/ep1%20%5B1%5D.txt" in [x[0] for x in _listing(body)] + status, _, body = _call(env, "GET", "/dav/pods/Collections/General/ep1%20%5B1%5D.txt", port=port) + assert status == 200 and body.startswith(b"Alice: Welcome back.") + assert _call(env, "PUT", "/dav/pods/Collections/General/x.txt", body=b"hi", port=port)[0] == 403 + # at its own port, links are as before + status, _, body = _call(env, "PROPFIND", "/", {"Depth": "1"}) + assert [x[0] for x in _listing(body)] == ["/", "/pods/"] + + +def test_through_the_web_app_each_visitor_has_its_own_throttle(env, cfg): + """Every request through the web app comes from its address; the visitor's, from X-Forwarded-For, is throttled.""" + cfg["shares"] = {**(cfg.get("shares") or {}), "remote": True} + wrong = "Basic " + base64.b64encode(f"{env['k']['id']}:nope".encode()).decode() + with web_relay(env["port"], "/dav", forwarded_for="198.51.100.1") as port: + for _ in range(8): + assert _call(env, "PROPFIND", "/dav/", {"Depth": "0", "Authorization": wrong}, port=port)[0] == 401 + assert _call(env, "PROPFIND", "/dav/", {"Depth": "0"}, port=port)[0] == 429 + with web_relay(env["port"], "/dav", forwarded_for="198.51.100.2") as port: + assert _call(env, "PROPFIND", "/dav/", {"Depth": "0"}, port=port)[0] == 207 # someone else isn't locked out + assert _call(env, "PROPFIND", "/", {"Depth": "0"})[0] == 207 diff --git a/fastapi_backend/tests/helpers.py b/fastapi_backend/tests/helpers.py index 4d5e8f89d..be597a244 100644 --- a/fastapi_backend/tests/helpers.py +++ b/fastapi_backend/tests/helpers.py @@ -2,8 +2,12 @@ from __future__ import annotations +import contextlib +import http.client +import http.server import pathlib import re +import threading import wave import numpy as np @@ -218,6 +222,54 @@ def marked(text, source): return f'\n{text}\n' +@contextlib.contextmanager +def web_relay(port, prefix, forwarded_for=None): + """What the web app does for a file share (nextjs-frontend/lens-relay.js): relays requests to the server on `port` + as they came (path and Host), saying in X-Forwarded-Prefix that it's at `prefix`, and passing X-Forwarded-For. + Yields the relay's port.""" + + class Relay(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *a): + pass + + def relay(self): + n = int(self.headers.get("content-length") or 0) + body = self.rfile.read(n) if n else None + headers = {k: v for k, v in self.headers.items() if k.lower() != "connection"} + headers["X-Forwarded-Prefix"] = prefix + if forwarded_for: + headers["X-Forwarded-For"] = forwarded_for + c = http.client.HTTPConnection("127.0.0.1", port, timeout=10) + c.request(self.command, self.path, body=body, headers=headers) + r = c.getresponse() + data = r.read() + self.send_response(r.status) + for k, v in r.getheaders(): + if k.lower() not in ("connection", "transfer-encoding") and (k.lower() != "content-length" or self.command == "HEAD"): + self.send_header(k, v) + if self.command != "HEAD": + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(data) + c.close() + + def __getattr__(self, name): + if name.startswith("do_"): + return self.relay + raise AttributeError(name) + + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Relay) + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + yield srv.server_address[1] + finally: + srv.shutdown() + srv.server_close() + + def without(monkeypatch, *names): """Hide modules of the workflow studio that land in their own steps (automation_nodes.module), so a check for "coming soon" holds whether or not they are merged yet.""" diff --git a/microsandbox/run.sh b/microsandbox/run.sh index 64c38487c..fc8c54ef9 100755 --- a/microsandbox/run.sh +++ b/microsandbox/run.sh @@ -82,10 +82,14 @@ if [ ! -f "$stamp" ]; then fi # --- Tools: SurrealDB, rclone, Node, uv, pinned like the Docker images ------------------------------------------- -fetch() { curl -fsSL --retry 5 --retry-delay 3 "$@"; } +# Archives are downloaded whole before they're unpacked: curl can't try again once it has written to a pipe, so a +# connection cut partway would hand tar half an archive +fetch() { curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors "$@"; } if [ ! -x "$OPT/bin/surreal" ] || ! "$OPT/bin/surreal" version 2>/dev/null | grep -q "${SURREAL_VERSION#v}"; then log "Getting SurrealDB $SURREAL_VERSION..." - fetch "https://download.surrealdb.com/$SURREAL_VERSION/surreal-$SURREAL_VERSION.linux-$arch.tgz" | tar -xz -C "$OPT/bin" + fetch -o /tmp/lens/surreal.tgz "https://download.surrealdb.com/$SURREAL_VERSION/surreal-$SURREAL_VERSION.linux-$arch.tgz" + tar -xzf /tmp/lens/surreal.tgz -C "$OPT/bin" + rm /tmp/lens/surreal.tgz fi if [ ! -x "$OPT/bin/rclone" ]; then log "Getting rclone $RCLONE_VERSION..." @@ -95,10 +99,14 @@ fi if [ ! -x "$OPT/node/bin/node" ]; then log "Getting Node $NODE_MAJOR..." base="https://nodejs.org/dist/latest-v$NODE_MAJOR.x" - file="$(fetch "$base/SHASUMS256.txt" | awk "/linux-$node_arch.tar.xz\$/ {print \$2}")" + sums="$(fetch "$base/SHASUMS256.txt")" + file="$(printf '%s\n' "$sums" | awk "/linux-$node_arch.tar.xz\$/ {print \$2}")" + fetch -o "/tmp/lens/$file" "$base/$file" + (cd /tmp/lens && printf '%s\n' "$sums" | grep " $file\$" | sha256sum -c --quiet -) wipe "$OPT/node" mkdir -p "$OPT/node" - fetch "$base/$file" | tar -xJ -C "$OPT/node" --strip-components 1 + tar -xJf "/tmp/lens/$file" -C "$OPT/node" --strip-components 1 + rm "/tmp/lens/$file" fi if [ ! -x "$OPT/bin/uv" ]; then log "Getting uv..." @@ -140,7 +148,7 @@ if [ "$(cat "$OPT/build-id" 2>/dev/null)" != "$build_id" ]; then mkdir -p "$OPT/web/.next" cp -a "$WEB_SRC/.next/standalone/." "$OPT/web/" cp -a "$WEB_SRC/.next/static" "$OPT/web/.next/static" - cp -a "$WEB_SRC/public" "$WEB_SRC/lens-server.js" "$OPT/web/" + cp -a "$WEB_SRC/public" "$WEB_SRC/lens-server.js" "$WEB_SRC/lens-relay.js" "$OPT/web/" wipe "$WEB_SRC" || true # the disk is replaced with the sandbox anyway chown -R lens:lens "$OPT/web" echo "$build_id" >"$OPT/build-id" diff --git a/nextjs-frontend/Dockerfile.prod b/nextjs-frontend/Dockerfile.prod index 0743b6398..107f86400 100644 --- a/nextjs-frontend/Dockerfile.prod +++ b/nextjs-frontend/Dockerfile.prod @@ -14,8 +14,8 @@ ENV NODE_ENV=production NEXT_TELEMETRY_DISABLED=1 HOSTNAME=0.0.0.0 PORT=3000 COPY --from=build --chown=node:node /app/.next/standalone ./ COPY --from=build --chown=node:node /app/.next/static ./.next/static COPY --from=build --chown=node:node /app/public ./public -COPY --from=build --chown=node:node /app/lens-server.js ./ +COPY --from=build --chown=node:node /app/lens-server.js /app/lens-relay.js ./ USER node EXPOSE 3000 -# lens-server.js starts server.js, keeping visitors from choosing the address the API sees +# lens-server.js starts server.js, keeping visitors from choosing the address the API sees and relaying the file shares CMD ["node", "lens-server.js"] diff --git a/nextjs-frontend/__tests__/lens-relay.test.ts b/nextjs-frontend/__tests__/lens-relay.test.ts new file mode 100644 index 000000000..6e8212934 --- /dev/null +++ b/nextjs-frontend/__tests__/lens-relay.test.ts @@ -0,0 +1,140 @@ +/** @jest-environment node */ +import http from "node:http"; +import type { AddressInfo } from "node:net"; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { forwardHeaders, route, shareOf, targetOf } = require("../lens-relay.js"); + +type Seen = { method?: string; url?: string; headers: http.IncomingHttpHeaders; body: string }; + +function listen(server: http.Server): Promise { + return new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port))); +} + +function call(port: number, method: string, path: string, headers: Record = {}, body = "") { + return new Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }>((resolve, reject) => { + const req = http.request({ host: "127.0.0.1", port, method, path, headers }, (r) => { + let data = ""; + r.on("data", (c) => (data += c)); + r.on("end", () => resolve({ status: r.statusCode ?? 0, headers: r.headers, body: data })); + }); + req.on("error", reject); + req.end(body); + }); +} + +describe("shareOf", () => { + it("finds the share a path is for", () => { + expect(shareOf("/s3")?.name).toBe("S3"); + expect(shareOf("/s3/pods/a%20b.txt?X-Amz-Signature=1")?.name).toBe("S3"); + expect(shareOf("/s3?list-type=2")?.name).toBe("S3"); + expect(shareOf("/dav/")?.name).toBe("WebDAV"); + expect(shareOf("/dav/pods/Collections/")?.name).toBe("WebDAV"); + }); + + it("leaves the web app's own paths to Next", () => { + for (const p of ["/", "/s3x", "/s/abc", "/davis", "/api/v1/shares/s3", "/settings?section=dav", "http://x/s3/"]) + expect(shareOf(p)).toBeNull(); + }); +}); + +describe("targetOf", () => { + it("reads where the server is from the environment, else this machine", () => { + const s3 = shareOf("/s3"); + expect(targetOf(s3, {})).toEqual({ host: "127.0.0.1", port: 9555 }); + expect(targetOf(s3, { LENS_S3_URL: "http://worker:9555" })).toEqual({ host: "worker", port: 9555 }); + expect(targetOf(shareOf("/dav"), { LENS_WEBDAV_URL: "http://[::1]:7000" })).toEqual({ host: "::1", port: 7000 }); + }); +}); + +describe("forwardHeaders", () => { + it("keeps the Host and signed headers, adds the visitor and the prefix, drops hop-by-hop ones", () => { + const out = forwardHeaders( + { + host: "lens.example.com", + authorization: "AWS4-HMAC-SHA256 x", + connection: "keep-alive", + expect: "100-continue", + }, + "::ffff:203.0.113.5", + shareOf("/s3"), + ); + expect(out).toEqual({ + host: "lens.example.com", + authorization: "AWS4-HMAC-SHA256 x", + "x-forwarded-for": "203.0.113.5", + "x-forwarded-prefix": "/s3", + }); + expect( + forwardHeaders({ "x-forwarded-for": "198.51.100.1" }, "172.18.0.4", shareOf("/dav"))["x-forwarded-for"], + ).toBe("198.51.100.1, 172.18.0.4"); + }); +}); + +describe("route", () => { + const seen: Seen = { headers: {}, body: "" }; + const share = http.createServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", () => { + Object.assign(seen, { method: req.method, url: req.url, headers: req.headers, body }); + res.writeHead(207, { "content-type": "application/xml", dav: "1" }); + res.end(""); + }); + }); + const front = http.createServer((req, res) => { + if (!route(req, res)) { + res.writeHead(404); + res.end("next"); + } + }); + let frontPort = 0; + + beforeAll(async () => { + process.env.LENS_WEBDAV_URL = `http://127.0.0.1:${await listen(share)}`; + frontPort = await listen(front); + }); + afterAll(() => { + delete process.env.LENS_WEBDAV_URL; + share.close(); + front.close(); + }); + + it("relays any method, path and body to the share, and its answer back", async () => { + const r = await call( + frontPort, + "PROPFIND", + "/dav/pods/Collections/", + { Depth: "1", Host: "lens.example.com" }, + "", + ); + expect(r.status).toBe(207); + expect(r.body).toBe(""); + expect(r.headers.dav).toBe("1"); + expect(seen.method).toBe("PROPFIND"); + expect(seen.url).toBe("/dav/pods/Collections/"); + expect(seen.body).toBe(""); + expect(seen.headers.host).toBe("lens.example.com"); + expect(seen.headers.depth).toBe("1"); + expect(seen.headers["x-forwarded-prefix"]).toBe("/dav"); + expect(seen.headers["x-forwarded-for"]).toBe("127.0.0.1"); + }); + + it("leaves other paths to Next", async () => { + expect((await call(frontPort, "GET", "/settings")).body).toBe("next"); + }); + + it("says so when the share isn't running", async () => { + const closed = http.createServer(); + const port = await listen(closed); + await new Promise((resolve) => closed.close(resolve)); + process.env.LENS_S3_URL = `http://127.0.0.1:${port}`; + try { + const r = await call(frontPort, "GET", "/s3/"); + expect(r.status).toBe(503); + expect(r.body).toContain("Settings, File shares"); + } finally { + delete process.env.LENS_S3_URL; + } + }); +}); diff --git a/nextjs-frontend/__tests__/s3-keys.test.ts b/nextjs-frontend/__tests__/s3-keys.test.ts index fc1739a90..92dde9f55 100644 --- a/nextjs-frontend/__tests__/s3-keys.test.ts +++ b/nextjs-frontend/__tests__/s3-keys.test.ts @@ -1,8 +1,27 @@ -import { rcloneConfig, webdavText } from "@/components/account/s3-model"; +import { rcloneConfig, shareAddresses, webdavText } from "@/components/account/s3-model"; + +describe("Share addresses", () => { + const page = { origin: "https://lens.example.com", hostname: "lens.example.com" }; + const ports = { port: 9555, webdav_port: 9556 }; + + it("are the share servers' ports on this host", () => { + expect(shareAddresses(page, ports, false)).toEqual({ + s3: "http://lens.example.com:9555", + webdav: "http://lens.example.com:9556/", + }); + }); + + it("are the web app's own address once an admin opened them there", () => { + expect(shareAddresses(page, ports, true)).toEqual({ + s3: "https://lens.example.com/s3", + webdav: "https://lens.example.com/dav/", + }); + }); +}); describe("S3 access keys", () => { - it("gives an rclone remote for the key, at the address the web app is on", () => { - expect(rcloneConfig("lens.local", { id: "LKABC", secret: "s3cr3t", port: 9555 })).toBe( + it("gives an rclone remote for the key", () => { + expect(rcloneConfig("http://lens.local:9555", { id: "LKABC", secret: "s3cr3t" })).toBe( [ "[lens]", "type = s3", @@ -18,7 +37,7 @@ describe("S3 access keys", () => { describe("WebDAV sign-in", () => { it("gives the address, and the key and secret as user name and password", () => { - expect(webdavText("lens.local", { id: "LKABC", secret: "s3cr3t", webdav_port: 9556 })).toBe( + expect(webdavText("http://lens.local:9556/", { id: "LKABC", secret: "s3cr3t" })).toBe( ["address = http://lens.local:9556/", "user = LKABC", "password = s3cr3t"].join("\n"), ); }); diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 01b1ee78a..5bfe99e66 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -11321,8 +11321,8 @@ export class Shares { /** * S3 Status * - * Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, and your - * access keys. + * Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, whether + * both are open at the web app's address too (/s3, /dav), and your access keys. */ public static s3Status( options?: Options, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 90338022b..832c96927 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -18101,6 +18101,12 @@ export type S3Status = { */ service?: S3Service | null; webdav: ShareServer; + /** + * Remote + * + * whether an admin opened both at the web app's address too, /s3 and /dav (shares.remote), so remote access reaches them + */ + remote?: boolean; /** * Keys * diff --git a/nextjs-frontend/components/account/s3-keys.tsx b/nextjs-frontend/components/account/s3-keys.tsx index de1c83963..c6f1c8e2d 100644 --- a/nextjs-frontend/components/account/s3-keys.tsx +++ b/nextjs-frontend/components/account/s3-keys.tsx @@ -6,7 +6,7 @@ import { useState } from "react"; import { Shares } from "@/app/openapi-client"; import type { S3Key, S3KeyCreated } from "@/app/openapi-client/types.gen"; -import { rcloneConfig, webdavText } from "@/components/account/s3-model"; +import { rcloneConfig, shareAddresses, webdavText } from "@/components/account/s3-model"; import { tokenExpiry } from "@/components/account/token-model"; import { Banner } from "@/components/ui/banner"; import { Button } from "@/components/ui/button"; @@ -57,6 +57,9 @@ export function S3Keys() { Read the archive as folders and files from any S3 client (rclone, Cyberduck, the AWS CLI). A key reads as you; nothing can be changed through it. {status.data && !status.data.enabled && " An admin turns sharing on in Settings, File shares."} + {status.data?.enabled && + status.data.remote && + " Both also open at this page’s address, wherever you reach it."}

{keys.length > 0 && newKey} @@ -113,7 +116,7 @@ export function S3Keys() { )} - setCreated(null)} /> + setCreated(null)} /> ); } @@ -151,8 +154,17 @@ function Copyable({ label, text }: { label: string; text: string }) { ); } -function ShownOnce({ created, onClose }: { created: S3KeyCreated | null; onClose: () => void }) { - const host = typeof window === "undefined" ? "localhost" : window.location.hostname; +function ShownOnce({ + created, + remote, + onClose, +}: { + created: S3KeyCreated | null; + remote: boolean; + onClose: () => void; +}) { + const page = typeof window === "undefined" ? { origin: "http://localhost", hostname: "localhost" } : window.location; + const at = created && shareAddresses(page, created, remote); return ( } > - {created && ( + {created && at && (
- - + +
)}
diff --git a/nextjs-frontend/components/account/s3-model.ts b/nextjs-frontend/components/account/s3-model.ts index d8c9f122d..48a383c55 100644 --- a/nextjs-frontend/components/account/s3-model.ts +++ b/nextjs-frontend/components/account/s3-model.ts @@ -1,23 +1,32 @@ import type { S3KeyCreated } from "@/app/openapi-client/types.gen"; +/** + * Where S3 and WebDAV clients reach the file shares: at the web app's own address (/s3 and /dav, so through remote + * access too) once an admin opened them there (shares.remote), else at the share servers' ports on this host. + */ +export function shareAddresses( + page: { origin: string; hostname: string }, + created: Pick, + remote: boolean, +): { s3: string; webdav: string } { + if (remote) return { s3: `${page.origin}/s3`, webdav: `${page.origin}/dav/` }; + return { s3: `http://${page.hostname}:${created.port}`, webdav: `http://${page.hostname}:${created.webdav_port}/` }; +} + /** The rclone remote for a key: what people paste into rclone.conf (or adapt for any S3 client). */ -export function rcloneConfig(host: string, created: Pick): string { +export function rcloneConfig(endpoint: string, created: Pick): string { return [ "[lens]", "type = s3", "provider = Other", `access_key_id = ${created.id}`, `secret_access_key = ${created.secret}`, - `endpoint = http://${host}:${created.port}`, + `endpoint = ${endpoint}`, "force_path_style = true", ].join("\n"); } /** How to mount the WebDAV share with a key: its address, then the key id and secret as user name and password. */ -export function webdavText(host: string, created: Pick): string { - return [ - `address = http://${host}:${created.webdav_port}/`, - `user = ${created.id}`, - `password = ${created.secret}`, - ].join("\n"); +export function webdavText(address: string, created: Pick): string { + return [`address = ${address}`, `user = ${created.id}`, `password = ${created.secret}`].join("\n"); } diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 3c26a7cf4..2898d6be7 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -1571,6 +1571,7 @@ function SharesBody({ ctx }: { ctx: BodyCtx }) { +

With Docker, the worker container publishes 9555 (S3) and 9556 (WebDAV) where the web app is; set{" "} LENS_S3_PORT or LENS_WEBDAV_PORT in .env diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 1a6ec79b4..4a2254ffc 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -2631,6 +2631,13 @@ export const FIELDS: FieldSpec[] = [ hint: "Read-only; mount it as a drive in Finder, Windows Explorer or a file manager", }, { section: "shares", key: "webdav_port", label: "WebDAV port", kind: "int", min: 1, max: 65535 }, + { + section: "shares", + key: "remote", + label: "At the web app’s address too", + kind: "switch", + hint: "S3 at /s3 and WebDAV at /dav wherever the web app opens, Remote access and HTTPS included. Off: only on their ports", + }, // Sensors (opt-in) { section: "sensors", diff --git a/nextjs-frontend/lens-relay.js b/nextjs-frontend/lens-relay.js new file mode 100644 index 000000000..522fae06f --- /dev/null +++ b/nextjs-frontend/lens-relay.js @@ -0,0 +1,92 @@ +/** + * Relays servers Lens runs besides the API at the web app's own address, so their clients reach them wherever the web + * app is reached, Settings › Remote access and Settings › HTTPS included. Today, the file shares + * (docs/file-tree.md#at-the-web-apps-address): /s3 to the S3 server and /dav to the WebDAV server, both run by the + * worker (fastapi_backend/app/domain/shares.py). + * + * Requests go to the share servers before Next sees them: Next answers only the usual methods (WebDAV lists folders + * with PROPFIND) and redirects paths that end in a slash. Each keeps its path and Host (S3 clients sign both) and + * carries X-Forwarded-Prefix, which tells the share server it came this way: it turns these away until an admin turns + * on "At the web app's address" in Settings › File shares (shares.remote). The servers are at LENS_S3_URL and + * LENS_WEBDAV_URL (http://worker:9555 and http://worker:9556 in the Docker Compose files), else this machine's ports. + * + * lens-server.js uses it; `node --require ./lens-relay.js` (in NODE_OPTIONS) adds it to `next dev` and `next start`. + */ +/* eslint-disable @typescript-eslint/no-require-imports */ +const http = require("node:http"); + +const SHARES = [ + { prefix: "/s3", env: "LENS_S3_URL", fallback: "http://127.0.0.1:9555", name: "S3" }, + { prefix: "/dav", env: "LENS_WEBDAV_URL", fallback: "http://127.0.0.1:9556", name: "WebDAV" }, +]; +// Hop-by-hop headers stay with each connection; Expect was answered already (Node sends 100 Continue itself). +const HOP = new Set([ + "connection", + "keep-alive", + "proxy-connection", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "expect", +]); + +/** The share a request target is for ("/dav/pods/" is WebDAV's), or null. */ +function shareOf(url) { + const path = (url || "").split("?")[0]; + return SHARES.find((s) => path === s.prefix || path.startsWith(`${s.prefix}/`)) || null; +} + +/** Where a share's server is: its host and port. */ +function targetOf(share, env = process.env) { + const u = new URL(env[share.env] || share.fallback); + return { host: u.hostname.replace(/^\[(.*)\]$/, "$1"), port: Number(u.port) || 80 }; +} + +/** The headers sent on: the request's own (Host too), without hop-by-hop ones, with X-Forwarded-For and -Prefix. */ +function forwardHeaders(headers, peer, share) { + const out = {}; + for (const [k, v] of Object.entries(headers)) if (!HOP.has(k)) out[k] = v; + const from = (peer || "").replace(/^::ffff:/, ""); + out["x-forwarded-for"] = out["x-forwarded-for"] ? `${out["x-forwarded-for"]}, ${from}` : from; + out["x-forwarded-prefix"] = share.prefix; + return out; +} + +/** Answers a request for a share by relaying it; false (nothing sent) when the request isn't for one. */ +function route(req, res) { + const share = shareOf(req.url); + if (!share) return false; + const { host, port } = targetOf(share); + const headers = forwardHeaders(req.headers, req.socket && req.socket.remoteAddress, share); + const up = http.request({ host, port, method: req.method, path: req.url, headers }, (r) => { + const out = {}; + for (const [k, v] of Object.entries(r.headers)) if (!HOP.has(k)) out[k] = v; + res.writeHead(r.statusCode || 502, r.statusMessage, out); + r.pipe(res); + }); + up.on("error", () => { + if (res.headersSent) return res.destroy(); + res.writeHead(503, { "content-type": "text/plain; charset=utf-8", "retry-after": "30" }); + res.end(`Lens's ${share.name} share isn't running here: an admin turns it on in Settings, File shares.\n`); + }); + res.on("close", () => { + if (!res.writableFinished) up.destroy(); + }); + req.pipe(up); + return true; +} + +/** Relays these requests on every HTTP server this process starts (for `node --require ./lens-relay.js`). */ +function install() { + const emit = http.Server.prototype.emit; + http.Server.prototype.emit = function (event, req, res, ...rest) { + if (event === "request" && route(req, res)) return true; + return emit.call(this, event, req, res, ...rest); + }; +} + +if (module.parent && module.parent.id === "internal/preload") install(); + +module.exports = { shareOf, targetOf, forwardHeaders, route, install }; diff --git a/nextjs-frontend/lens-server.js b/nextjs-frontend/lens-server.js index f837f0b43..c47cb25b1 100644 --- a/nextjs-frontend/lens-server.js +++ b/nextjs-frontend/lens-server.js @@ -6,11 +6,15 @@ * sign-in throttles. Here a request's X-Forwarded-For is dropped before Next sees it, unless it came from a proxy * known to set it: any peer with TRUST_PROXY_HEADERS=true (a reverse proxy in front), or one of the hosts named in * LENS_WEB_PROXY_HOSTS (the containers running the Cloudflare tunnel), looked up every 30 seconds. + * + * Requests for the file shares (/s3 and /dav) then go to the share servers rather than to Next (lens-relay.js). */ /* eslint-disable @typescript-eslint/no-require-imports */ const http = require("node:http"); const dns = require("node:dns"); +const relay = require("./lens-relay.js"); + const truthy = (v) => ["1", "true", "yes"].includes((v || "").trim().toLowerCase()); const plain = (address) => (address || "").replace(/^::ffff:/, ""); @@ -43,9 +47,12 @@ function start() { setInterval(refresh, 30_000).unref(); } const emit = http.Server.prototype.emit; - http.Server.prototype.emit = function (event, req, ...rest) { - if (event === "request") scrubForwardedFor(req.headers, req.socket && req.socket.remoteAddress, trustAll, proxies); - return emit.call(this, event, req, ...rest); + http.Server.prototype.emit = function (event, req, res, ...rest) { + if (event === "request") { + scrubForwardedFor(req.headers, req.socket && req.socket.remoteAddress, trustAll, proxies); + if (relay.route(req, res)) return true; + } + return emit.call(this, event, req, res, ...rest); }; require("./server.js"); } diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 2e81e58d3..ae087d3e7 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -29086,7 +29086,7 @@ "shares" ], "summary": "S3 Status", - "description": "Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, and your\naccess keys.", + "description": "Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, whether\nboth are open at the web app's address too (/s3, /dav), and your access keys.", "operationId": "s3_status", "responses": { "200": { @@ -61772,6 +61772,12 @@ "webdav": { "$ref": "#/components/schemas/ShareServer" }, + "remote": { + "type": "boolean", + "title": "Remote", + "description": "whether an admin opened both at the web app's address too, /s3 and /dav (shares.remote), so remote access reaches them", + "default": false + }, "keys": { "items": { "$ref": "#/components/schemas/S3Key" diff --git a/nextjs-frontend/proxy.ts b/nextjs-frontend/proxy.ts index 2a521b38b..659021c60 100644 --- a/nextjs-frontend/proxy.ts +++ b/nextjs-frontend/proxy.ts @@ -33,10 +33,11 @@ export default auth((req) => { export const config = { // Skip Auth.js, the backend paths (proxied by route handlers, see lib/api/backend-proxy.ts; /.well-known is OAuth // discovery, /mcp the MCP server, /metrics the Prometheus scrape endpoint, /id and /ns the - // archive's linked-data URIs and vocabulary), Next's files and assets. + // archive's linked-data URIs and vocabulary), the file shares (/s3 and /dav, relayed before Next by lens-relay.js), + // Next's files and assets. // Under /iiif only the backend's IIIF resources are skipped (collection, discovery, auth, /iiif//…); the app's own // IIIF pages (/iiif, /iiif/collections/…, /iiif/import, /iiif/metadata/…) need the session like any other page. matcher: [ - "/((?!api/|\\.well-known/|mcp(?:/|$)|metrics$|_next/|embed/|s/|id/|ns$|iiif/(?:collection(?!s)|discovery|auth|\\d)|reports/|static/|fonts/|favicon\\.ico|icon\\.svg|apple-icon\\.png|manifest\\.webmanifest|safari-pinned-tab\\.svg|pwa/|robots\\.txt).*)", + "/((?!api/|\\.well-known/|mcp(?:/|$)|s3(?:/|$)|dav(?:/|$)|metrics$|_next/|embed/|s/|id/|ns$|iiif/(?:collection(?!s)|discovery|auth|\\d)|reports/|static/|fonts/|favicon\\.ico|icon\\.svg|apple-icon\\.png|manifest\\.webmanifest|safari-pinned-tab\\.svg|pwa/|robots\\.txt).*)", ], }; diff --git a/packaging/qnap/qpkg/shared/docker-compose.yml b/packaging/qnap/qpkg/shared/docker-compose.yml index 21213e5aa..04921de30 100644 --- a/packaging/qnap/qpkg/shared/docker-compose.yml +++ b/packaging/qnap/qpkg/shared/docker-compose.yml @@ -84,6 +84,9 @@ services: AUTH_TRUST_HOST: "true" # the containers running the Cloudflare tunnel: their X-Forwarded-For names the visitor (lens-server.js) LENS_WEB_PROXY_HOSTS: backend,worker + # the file shares at the web app's address, /s3 and /dav (lens-relay.js), once allowed in Settings › File shares + LENS_S3_URL: http://worker:9555 + LENS_WEBDAV_URL: http://worker:9556 # no AUTH_URL: sign-in follows the address the browser is on (its LAN name, https:// address or Cloudflare tunnel) ports: - "${LENS_PORT:-3000}:3000" diff --git a/packaging/synology/package/compose/compose.yaml.in b/packaging/synology/package/compose/compose.yaml.in index f1c5a41af..a36e00970 100644 --- a/packaging/synology/package/compose/compose.yaml.in +++ b/packaging/synology/package/compose/compose.yaml.in @@ -94,6 +94,9 @@ services: AUTH_TRUST_HOST: "true" # the containers running the Cloudflare tunnel: their X-Forwarded-For names the visitor (lens-server.js) LENS_WEB_PROXY_HOSTS: backend,worker + # the file shares at the web app's address, /s3 and /dav (lens-relay.js), once allowed in Settings › File shares + LENS_S3_URL: http://worker:9555 + LENS_WEBDAV_URL: http://worker:9556 # no AUTH_URL: sign-in follows the address the browser is on (its LAN name, https:// address or Cloudflare tunnel) ports: - "@PORT@:3000" diff --git a/threat-model.json b/threat-model.json index f5c36a678..bd128d9d7 100644 --- a/threat-model.json +++ b/threat-model.json @@ -555,6 +555,15 @@ "provenance": "documented", "source": "docs/smb.md \"What keeps it safe\"" }, + { + "entry_point": "web app /s3 and /dav (file shares relayed by lens-relay.js, ahead of Next)", + "parameter": "any method, path, Host and body, passed to the share servers with X-Forwarded-Prefix and X-Forwarded-For", + "attacker_controllable": "yes", + "component": "optional", + "caller_must_enforce": "admin enables shares.enabled and shares.remote; per-person access keys, read-only; failed WebDAV sign-ins throttled per visitor from X-Forwarded-For as trusted proxies report it", + "provenance": "documented", + "source": "docs/file-tree.md \"At the web app's address\"" + }, { "entry_point": ".env, archive.yaml, LENS_*, ARCHIVE_SECRET_KEY, RCLONE_BINARY", "parameter": "process configuration", @@ -791,6 +800,14 @@ "provenance": "documented", "source": "docs/sensors.md \"Sensors are off until an admin turns them on\"" }, + { + "name": "shares.enabled, shares.remote", + "default": "off (bind 0.0.0.0); off", + "effect": "narrows P-SHARES", + "discouraged": false, + "provenance": "documented", + "source": "docs/file-tree.md \"At the web app's address\"" + }, { "name": "tunnel.mode, local_llm.enabled, telemetry", "default": "off", @@ -1216,7 +1233,7 @@ "property": "P-SHARES", "violation_symptom": "info-leak, integrity-bypass", "severity_tier": "security", - "conditions": "the file shares (S3, WebDAV) are off until an admin enables them; every request carries a person's own access key (SigV4 or Basic), reads as them and never writes", + "conditions": "the file shares (S3, WebDAV) are off until an admin enables them, and reach the web app's address (/s3, /dav) only with shares.remote too; every request carries a person's own access key (SigV4 or Basic), reads as them and never writes", "provenance": "documented", "source": "docs/file-tree.md \"Off until an admin turns it on\"" }, diff --git a/threat-model.md b/threat-model.md index 0a7265695..cb11b37c0 100644 --- a/threat-model.md +++ b/threat-model.md @@ -242,6 +242,7 @@ machine requires HTTPS, and a changed DB credential when SurrealDB runs as its o | Chromium sandbox | off as root or without user namespaces | the `netguard` proxy is the only barrier | Yes | *(documented, docs/configuration.md "runs with its sandbox where it can")* | | `sensors.enabled`, `sensors.mqtt_anonymous` | off (`bind` `0.0.0.0`); off | opens MQTT and syslog listeners; anonymous MQTT publish | Yes; anonymous is the admin's choice | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | | `shares.enabled` | off (`bind` `0.0.0.0`) | opens the read-only S3 and WebDAV servers for the asset tree | Yes | *(documented, docs/file-tree.md "Off until an admin turns it on")* | +| `shares.remote` | off | the web app relays the file shares at its own address (`/s3`, `/dav`; `nextjs-frontend/lens-relay.js`), so the tunnel and HTTPS reach them from the internet; until it's on, the share servers refuse requests carrying `X-Forwarded-Prefix` before any sign-in | Yes | *(documented, docs/file-tree.md "At the web app's address")* | | `tunnel.mode`, `local_llm.enabled`, `telemetry`, `analytics.enabled` | off | outbound tunnel (Cloudflare terminates TLS, Q30); local model server; OTLP export; GA4 Measurement Protocol export of daily counts | Yes | *(documented, docs/remote-access.md "It's off until an admin turns it on")* | | `https.enabled` (and install.sh's `LENS_HTTPS=1`, which Settings › HTTPS can also ask the upgrade agent for) | off | Caddy, fetched at run time, terminates TLS on ports 443/80 in the worker and proxies to the web app; the DNS provider's credentials are written to `data_dir/https/caddy.json` (mode 600), not a command line | Yes | *(documented, docs/https.md "Off by default")* | | Reachable from the network: install.sh's `LENS_PUBLIC` (kept as `LENS_BIND`), also switched from Settings › Access & embedding, `PUT /api/v1/admin/install` or MCP `set_install_settings` | `1` (`0.0.0.0`) on a server without a desktop, else `0` (`127.0.0.1`) | publishes the web app's port to the network over plain HTTP, and the sensor, file-share and Grafana ports with it unless `LENS_SENSOR_BIND` or `LENS_SHARE_BIND` say otherwise. Admins (an agent with an admin's write-scope token too) can open it without a shell: the host upgrade agent runs install.sh again; audited as `lens.install` | Supported on a trusted LAN only until HTTPS is set up, as the headless-host row | *(documented, docs/updates.md "Setting the machine up again")* | @@ -302,6 +303,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | NFS share (port 12049 on the host, 2049 in the worker) | NFS and MOUNT RPCs, client address | yes, once enabled, from `nfs.allowed_networks` | data, x-network-source | admin enables and picks the account; operator firewalls | *(documented, docs/file-shares.md "NFS has no login")* | | SMB share (port 1445 on the host, 4445 in the worker) | SMB2 requests, NTLM sign-in with a file access key, client address | yes, once enabled, from `smb.allowed_networks` | data, x-credential, x-network-source | admin enables; operator firewalls | *(documented, docs/smb.md "What keeps it safe")* | | S3 9555 and WebDAV 9556 (file shares) | signed S3 requests (SigV4 header or presigned link); WebDAV with an access key as Basic sign-in | yes, once enabled | x-credential, resource-name | admin enables; each request carries a person's own access key; read-only; failed WebDAV sign-ins throttled per address | *(documented, docs/file-tree.md "Off until an admin turns it on")* | +| web app `/s3` and `/dav` (file shares relayed by `lens-relay.js`, ahead of Next) | any method, path, `Host` and body passed to the share servers with `X-Forwarded-Prefix` and `X-Forwarded-For` | yes, once `shares.enabled` and `shares.remote`; from the internet through the tunnel or HTTPS | x-credential, resource-name, x-network-source | admin enables both; the same per-person access keys, read-only; failed WebDAV sign-ins throttled per visitor from `X-Forwarded-For` as trusted proxies report it (`server.trusted_proxies`, `LENS_TRUSTED_PROXY_HOSTS`, loopback); `X-Forwarded-Prefix` sent straight to a share port only gets the sender refused or signed and linked under that path | *(documented, docs/file-tree.md "At the web app's address")* | | `.env`, `archive.yaml`, `LENS_*`, `ARCHIVE_SECRET_KEY`, `RCLONE_BINARY` | process configuration | no (operator) | x-env-config | keep unreadable to others | *(maintainer, 2026-10)* | | workers and DB peers | DB credential, shared database rows | no (trusted peer) | x-peer | run peers on trusted hosts | *(inferred, Q9)* | | `.lens-control/inbox/upgrade` (host upgrade agent), `/metrics`, `.lens-control/prometheus-token` | `ref` (release tag or this install's branch; empty: the version running), with `https=`, `public=` (`0`/`1`) and `target=` (`full`/`lean`); `action=services` with `monitoring=on`/`off`; the scrape token | the inbox is writable by the containers (and local users); `/metrics` by anyone reaching the web app | x-env-config, credential | the agent installs only a release tag or the install's branch, changes only the HTTPS ports, the web app's network reach and the edition (any other value is dropped) and only toggles the `monitoring` profile, nothing else from the file, so a writer of the inbox can at most restart Lens with those choices or open it to the network; `/metrics` answers only the scrape token (constant-time compare); the token file is world-readable on the host and grants metrics only, which carry no personal content | *(documented, docs/updates.md, docs/prometheus.md "Privacy")* | @@ -541,7 +543,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-SENSORS | Sensors are off until an admin enables them. MQTT needs a hub login unless `mqtt_anonymous` is on. Syslog hears only listed networks. | integrity-bypass | security-critical | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | Voided by: `sensors.mqtt_anonymous` — `domain/sensors.py:984`; `sensors.syslog_networks`, `sensors.bind`. | | P-NFS | The NFS share is off until an admin turns it on. It is read-only, shows only what one chosen account sees in the web app, and turns away every client outside `nfs.allowed_networks` before rclone reads a byte. | info-leak | security-critical | *(documented, docs/file-shares.md "NFS has no login")* | Voided by: `nfs.allowed_networks` set wide, or a client address spoofed on the local network (NFS has no login) — `domain/share_relay.py`; Docker publishing that rewrites client addresses. | | P-SMB | The SMB share is off until an admin turns it on. It is read-only, signs people in with their own file access keys and shows each only what they see in the web app, refuses guests and SMB1 sessions, gives every connection its own NTLM challenge, and closes connections from outside `smb.allowed_networks` before reading a byte. | info-leak | security-critical | *(documented, docs/smb.md "What keeps it safe")* | Voided by: `smb.allowed_networks` set wide; SMB 2.0 has no encryption on the wire, so a key's NTLM exchange and the files read can be watched on the network — `domain/smb_share.py`. | -| P-SHARES | The file shares (S3 and WebDAV) are off until an admin enables them. Every request is signed (S3: AWS Signature Version 4) or signed in (WebDAV: Basic) with an access key someone made while signed in to the web app; it reads as them, sees only what the web app shows them, and never writes. Keys expire as API keys do, stop with a disabled account, and are revoked at once. | info-leak, integrity-bypass | security-critical | *(documented, docs/file-tree.md "Off until an admin turns it on")* | Voided by: `shares.enabled` with `LENS_BIND=0.0.0.0` sends files, S3 signatures and WebDAV's Basic secret in clear over plain HTTP on the network; key secrets are sealed with the server key (`secret.key`), not hashed, since SigV4 signs with the secret itself — `domain/s3.py`, `domain/webdav.py`. | +| P-SHARES | The file shares (S3 and WebDAV) are off until an admin enables them. Every request is signed (S3: AWS Signature Version 4) or signed in (WebDAV: Basic) with an access key someone made while signed in to the web app; it reads as them, sees only what the web app shows them, and never writes. Keys expire as API keys do, stop with a disabled account, and are revoked at once. | info-leak, integrity-bypass | security-critical | *(documented, docs/file-tree.md "Off until an admin turns it on")* | Through the web app (`/s3`, `/dav`) only with `shares.remote` on as well. Voided by: `shares.enabled` with `LENS_BIND=0.0.0.0` sends files, S3 signatures and WebDAV's Basic secret in clear over plain HTTP on the network, as does `shares.remote` when the web app itself is reached over plain HTTP; key secrets are sealed with the server key (`secret.key`), not hashed, since SigV4 signs with the secret itself — `domain/s3.py`, `domain/webdav.py`. | | P-CHILD-SECRETS | The tunnel token reaches cloudflared in its environment, not its command line; llama-server gets its own random API key. | info-leak | correctness-only | *(documented, docs/remote-access.md "in its environment, not on its command line")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-REMOTE-NAMES | An IMAP message is named by mailbox validity and uid, so a rebuilt mailbox never serves another message under an old name. | wrong-output | correctness-only | *(documented, docs/processing.md "an old one never reads another message")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-NOTIFY | Each notification event is claimed once across processes and retried with backoff. | wrong-output | correctness-only | *(documented, docs/notifications.md "claimed once")* | Voided by: none identified (not required for this symptom; no exhaustive search) | diff --git a/threat-model.yaml b/threat-model.yaml index e9b1d7b0f..f85cb4661 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:0dd7f47f6aef27ac5fa7ddc80ee776fe1b6b41a8f737822e68fbecb9b6eb9608" +prose_version: "threat-model.md@sha256:9c0063b128c9082c99f58c1acb64e303f33dfad2a8c520ef0294a25b587a165e" model_status: under-review triage_policy: strict confidence: {documented: 204, maintainer: 46, inferred: 91, assumption: 8} @@ -486,6 +486,15 @@ entry_points: obligation_id: enable-and-limit-smb caller_must_enforce: "admin enables; operator firewalls" provenance: {kind: documented, source: 'docs/smb.md "What keeps it safe"'} + - id: "web app /s3 and /dav (file shares relayed by lens-relay.js, ahead of Next)" + component: optional + parameters: + - name: "any method, path, Host and body, passed to the share servers with X-Forwarded-Prefix and X-Forwarded-For" + attacker_controllable: true + x-condition: "once shares.enabled and shares.remote; from the internet through the tunnel or HTTPS" + control_kinds: [x-credential, resource-name, x-network-source] + caller_must_enforce: "admin enables both; per-person access keys, read-only; failed WebDAV sign-ins throttled per visitor from X-Forwarded-For as trusted proxies report it" + provenance: {kind: documented, source: 'docs/file-tree.md "At the web app''s address"'} - id: ".env, archive.yaml, LENS_*, ARCHIVE_SECRET_KEY, RCLONE_BINARY" component: outbound parameters: @@ -1203,6 +1212,14 @@ build_flags: affects_properties: - {property_id: P-SHARES, effect: narrows} provenance: {kind: documented, source: 'docs/file-tree.md "Off until an admin turns it on"'} + - name: "shares.remote" + default: "off" + security_relevant: true + maintainer_stance: supported + x-scope: "the web app relays the file shares at its own address (/s3, /dav; nextjs-frontend/lens-relay.js), so the tunnel and HTTPS reach them; until on, the share servers refuse requests carrying X-Forwarded-Prefix before any sign-in" + affects_properties: + - {property_id: P-SHARES, effect: narrows} + provenance: {kind: documented, source: 'docs/file-tree.md "At the web app''s address"'} - name: "https.enabled" default: "off" security_relevant: true @@ -1646,9 +1663,9 @@ properties_claimed: kind: authentication components: [optional] tier: security-critical - conditions: "the file shares (S3, WebDAV) are off until an admin enables them; every request carries a person's own access key (SigV4 or Basic), reads as them and never writes" + conditions: "the file shares (S3, WebDAV) are off until an admin enables them, and reach the web app's address (/s3, /dav) only with shares.remote too; every request carries a person's own access key (SigV4 or Basic), reads as them and never writes" violation_symptoms: [info-leak, integrity-bypass] - x-voided-by: "plain HTTP on the network with LENS_BIND=0.0.0.0 (WebDAV's Basic secret in clear); secrets sealed with the server key, not hashed (domain/s3.py)" + x-voided-by: "plain HTTP on the network with LENS_BIND=0.0.0.0, or shares.remote with the web app itself on plain HTTP (WebDAV's Basic secret in clear); secrets sealed with the server key, not hashed (domain/s3.py)" provenance: {kind: documented, source: 'docs/file-tree.md "Off until an admin turns it on"'} - id: P-CHILD-SECRETS kind: confidentiality