From 8ce349e08fad53a22851c484c994dfca97cbb869 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:24:17 +0000 Subject: [PATCH 1/5] feat(shares): share assets read-only over S3 and WebDAV S3 (SigV4, path-style) and WebDAV (class 1, Basic with an access key) servers in the worker, off until an admin turns on File shares; each person makes access keys on their API tokens page, which read as them. Docker publishes them on 9555 and 9556 (LENS_S3_PORT, LENS_WEBDAV_PORT). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- CHANGELOG.md | 8 +- docker-compose.prod.yml | 4 + docker-compose.yml | 3 + docs/api.md | 6 + docs/configuration.md | 2 +- docs/file-tree.md | 53 ++- fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/shares.py | 51 +++ fastapi_backend/app/cli.py | 18 +- fastapi_backend/app/domain/ops_tools.py | 4 +- fastapi_backend/app/domain/s3.py | 433 ++++++++++++++++++ fastapi_backend/app/domain/settings.py | 14 + fastapi_backend/app/domain/shares.py | 135 ++++++ fastapi_backend/app/domain/store.py | 6 + fastapi_backend/app/domain/webdav.py | 188 ++++++++ fastapi_backend/app/schemas/filetree.py | 45 +- fastapi_backend/pyproject.toml | 1 + fastapi_backend/tests/api/test_s3.py | 160 +++++++ fastapi_backend/tests/api/test_webdav.py | 79 ++++ fastapi_backend/uv.lock | 16 + install.sh | 3 +- nextjs-frontend/__tests__/s3-keys.test.ts | 25 + nextjs-frontend/app/openapi-client/index.ts | 23 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 74 +++ .../app/openapi-client/types.gen.ts | 255 +++++++++++ .../components/account/s3-keys.tsx | 178 +++++++ .../components/account/s3-model.ts | 23 + nextjs-frontend/components/account/tokens.tsx | 2 + .../components/settings/bodies.tsx | 64 ++- nextjs-frontend/components/settings/model.ts | 32 ++ .../components/settings/settings-shell.tsx | 2 + nextjs-frontend/openapi.json | 405 ++++++++++++++++ threat-model.json | 8 + threat-model.md | 7 +- threat-model.yaml | 18 +- 35 files changed, 2336 insertions(+), 11 deletions(-) create mode 100644 fastapi_backend/app/api/v1/routes/shares.py create mode 100644 fastapi_backend/app/domain/s3.py create mode 100644 fastapi_backend/app/domain/shares.py create mode 100644 fastapi_backend/app/domain/webdav.py create mode 100644 fastapi_backend/tests/api/test_s3.py create mode 100644 fastapi_backend/tests/api/test_webdav.py create mode 100644 nextjs-frontend/__tests__/s3-keys.test.ts create mode 100644 nextjs-frontend/components/account/s3-keys.tsx create mode 100644 nextjs-frontend/components/account/s3-model.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e9dfdd09..7afa348a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,18 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version inside their broader topics, entities by type. Each folder holds its assets' files, with a `.json` sidecar of metadata and a `.txt` of the text; everyone sees what the web app shows them. Browse and read it through the API (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` ([The graph as files](docs/file-tree.md)). - S3, WebDAV, SMB and NFS servers that share it are in progress. + SMB and NFS shares are in progress. - **Budget meter and Costs page.** The top bar shows what the namespace picked spent against its budget, used and left, highlighted when near or over, and follows the namespace switcher; it opens the details for a day, week or month. **Costs** in the left menu shows spend by day, model and resource, and sets the namespace's budget ([Budgets](docs/budgets.md#the-meter-and-the-costs-page)). +- **The graph over S3 and WebDAV.** Turn on Settings, File shares, and any S3 client (rclone, Cyberduck, the AWS CLI) or + WebDAV client (Finder, Windows Explorer, file managers) lists and reads the graph as folders and files, + read-only, with access keys each person makes on their API tokens page ([S3](docs/file-tree.md#s3), + [WebDAV](docs/file-tree.md#webdav)). Docker publishes S3 on port 9555 (`LENS_S3_PORT`) and WebDAV on 9556 + (`LENS_WEBDAV_PORT`), where the web app is. + ## 0.7.0 October 10, 2026 {id="0.7.0"} - **Prometheus metrics.** Lens can serve `/metrics` for Prometheus and Grafana: jobs by status and pipeline, imports, diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index fcea0984e..adf4e82c8 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -103,6 +103,10 @@ services: - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_MQTT_PORT:-1883}:1883" - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_SYSLOG_PORT:-5514}:5514/udp" - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_SYSLOG_PORT:-5514}:5514/tcp" + # the asset tree over S3 and WebDAV (docs/file-tree.md): read-only, listened on only once File shares are turned + # on in the app; published where the web app is. Something else using 9555 or 9556? Set LENS_S3_PORT / LENS_WEBDAV_PORT. + - "${LENS_SHARE_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_S3_PORT:-9555}:9555" + - "${LENS_SHARE_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_WEBDAV_PORT:-9556}:9556" healthcheck: disable: true diff --git a/docker-compose.yml b/docker-compose.yml index 07bf37855..5744f1f57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,6 +87,9 @@ services: - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_MQTT_PORT:-1883}:1883" - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_SYSLOG_PORT:-5514}:5514/udp" - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_SYSLOG_PORT:-5514}:5514/tcp" + # the asset tree over S3 and WebDAV (docs/file-tree.md), once File shares are on; LENS_S3_PORT / LENS_WEBDAV_PORT + - "${LENS_SHARE_BIND:-127.0.0.1}:${LENS_S3_PORT:-9555}:9555" + - "${LENS_SHARE_BIND:-127.0.0.1}:${LENS_WEBDAV_PORT:-9556}:9556" healthcheck: disable: true diff --git a/docs/api.md b/docs/api.md index b7839a1c1..4f0bfddb5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -599,10 +599,16 @@ removes one. ``` GET /api/v1/tree?path= GET /api/v1/tree/file?path= +GET /api/v1/shares/s3 +POST /api/v1/shares/s3/keys +DELETE /api/v1/shares/s3/keys/{key_id} +GET /api/v1/shares/s3/keys ``` The archive as folders and files ([The graph as files](file-tree.md)): `tree` lists a folder (an empty path lists your namespaces; in each, `Collections`, `Topics` and `Entities`), `tree/file` reads a file with byte ranges. Read-only; everyone sees what they can read in the web app. +`shares/s3` is the [S3](file-tree.md#s3) and [WebDAV](file-tree.md#webdav) servers' state and your access keys; keys are +made only while signed in. ## comments diff --git a/docs/configuration.md b/docs/configuration.md index 550663fb8..3fce395f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -94,7 +94,7 @@ Admins can change, in Settings: transcription, speaker separation, speech provid provider and key, a local model, the AI assistant, search (word search and search by meaning), reports and graph, video, OCR, faces and objects, workers, sign-in, components, access and embedding (embed frame ancestors, transcript upload limit, allowed hosts, trusted proxies, session length), remote access, email, chat rooms, notifications, -telemetry, the Fedora repository, sensors, uploads, storage, documents, API keys, and IIIF and metadata. The API +telemetry, the Fedora repository, sensors, file shares, uploads, storage, documents, API keys, and IIIF and metadata. The API refuses an allowed-host list that leaves out the address you are using. ## Word search diff --git a/docs/file-tree.md b/docs/file-tree.md index 6ae8d42df..985b850cd 100644 --- a/docs/file-tree.md +++ b/docs/file-tree.md @@ -7,7 +7,7 @@ folders it belongs to, its metadata in a sidecar beside it. Agents and tools tha | Where | Status | | --- | --- | | The API (`GET /api/v1/tree`, `GET /api/v1/tree/file`) and the MCP tools `list_files` and `read_file` | built | -| S3 and WebDAV servers (read-only, off until turned on) | in progress | +| An [S3 server](#s3) and a [WebDAV server](#webdav) (read-only, off until turned on) | built | | SMB and NFS shares | in progress | Every way in serves the same tree, built by `app/domain/filetree.py`. @@ -54,8 +54,59 @@ content_type}]}`. An empty path lists your namespaces. `GET /api/v1/tree/file?pa The MCP server has `list_files` and `read_file` ([MCP server](mcp.md)): `read_file` returns the text of `.txt` and `.json` files, a page at a time, and the download link of everything else. +## S3 + +Off until an admin turns it on, in Settings, File shares (`shares.enabled`). Then Lens's worker serves the tree +read-only over S3 on port 9555 (`shares.s3_port`; with Docker, `LENS_S3_PORT` publishes it on another port of the +host), where the web app is: this machine only unless Lens was opened to the network (`LENS_BIND`; +`LENS_SHARE_BIND` sets it apart). Each namespace you can read is a bucket; keys are the paths above. + +Each person makes their own access keys on their API tokens page (File share access keys), while signed in to the web app; +API tokens can't make them. A key reads as its owner and sees exactly what they see. Its secret is shown once, kept +sealed with the server's key (S3's signatures are made with the secret itself, so it can't be hashed), expires like +an API key (Settings, API keys), stops working when its owner is disabled, and can be revoked at once. Making and +revoking keys is audited. `GET /api/v1/shares/s3` says whether the S3 and WebDAV servers are on and lists your keys; +`POST /api/v1/shares/s3/keys` and `DELETE /api/v1/shares/s3/keys/{id}` make and revoke them (admins list everyone's +with `GET /api/v1/shares/s3/keys`). + +Any S3 client works with path-style addressing and any region. With rclone, the remote the web app shows when you +make a key: + +```ini +[lens] +type = s3 +provider = Other +access_key_id = LK… +secret_access_key = … +endpoint = http://lens.local:9555 +force_path_style = true +``` + +then `rclone lsd lens:`, `rclone copy lens:pods/General ./general`, or `rclone mount lens: ~/lens`. Supported: +listing buckets, listing objects (v1 and v2, by folder or recursively, a page at a time), reading objects whole or +in ranges, and presigned links. Anything that writes answers AccessDenied. Requests must be signed with Signature +Version 4. + +The server speaks plain HTTP. On a network you don't trust, keep it on this machine or behind TLS (a reverse proxy), +since files travel in clear. + +## WebDAV + +Off until an admin turns it on, with S3 (Settings, File shares; `shares.webdav`, on by default once shares are). +Lens's worker serves the same tree read-only over WebDAV on port 9556 (`shares.webdav_port`; with Docker, +`LENS_WEBDAV_PORT`), so it mounts as a drive: in Finder, Go, Connect to Server; in Windows Explorer, Map network +drive; in GNOME Files or Dolphin, `dav://lens.local:9556/`; or `rclone` with `type = webdav`. + +Sign in with an access key: its id as the user name and its secret as the password. The folders at the top are your +namespaces. Listing (PROPFIND, one level at a time) and reading (GET, with ranges) work; anything that writes answers +403. Eight failed sign-ins from one address within 15 minutes are refused for a while. + +Like S3, it speaks plain HTTP, and Basic sign-in sends the secret with every request: keep it on this machine or +behind TLS on a network you don't trust. + ## Refine later +- TLS by itself for S3 and WebDAV, folders' real dates, and WebDAV locks for clients that insist on them. - Show supplementary files (captions, translations, attachments) beside each asset. - Cache an asset's text and metadata so large folders list faster. - A sidecar for each collection, topic and entity, and a folder of notes. diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index 8f09c6bab..311028d4a 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -51,6 +51,7 @@ searches, sensors, setup, + shares, sources, speakers, templates, @@ -125,6 +126,7 @@ notifications, routines, sensors, + shares, import_hooks, voice, vaults, diff --git a/fastapi_backend/app/api/v1/routes/shares.py b/fastapi_backend/app/api/v1/routes/shares.py new file mode 100644 index 000000000..481e6df5d --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/shares.py @@ -0,0 +1,51 @@ +"""File shares (docs/file-tree.md): whether the S3 and WebDAV servers are on, and people's access keys for them. + +Admins turn the server on in the settings (`shares`). Anyone signed in makes keys for themselves, which act as them; +a key's secret is shown once. Making and revoking keys is audited. +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.api.deps import AdminReader, Cfg, CurrentUser, Db, Writer, domain_errors +from app.domain import auth, s3, shares +from app.schemas.common import Ok +from app.schemas.filetree import S3Key, S3KeyCreate, S3KeyCreated, S3Status, ShareServer + +router = APIRouter(prefix="/shares", tags=["shares"]) + + +@router.get("/s3") +def s3_status(user: CurrentUser, db: Db, cfg: Cfg) -> S3Status: + """Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, and your + access keys.""" + st = shares.status(db, cfg) + return S3Status(**st["s3"], webdav=ShareServer(**st["webdav"]), keys=[S3Key(**k) for k in s3.keys_of(db, user.id)]) + + +@router.post("/s3/keys") +def create_s3_key(body: S3KeyCreate, user: Writer, db: Db, cfg: Cfg) -> S3KeyCreated: + """An access key that reads the file tree as you, over S3 and WebDAV. Only while signed in to the web app (with your passkey), + not with an API token. The secret is in this answer only.""" + if user.via != "access": + raise HTTPException(403, "make access keys while signed in") + with domain_errors(): + kid, secret = s3.create_key(db, cfg, user.id, body.name, body.days) + auth.audit(db, user.as_audit(), "s3_key.create", f"file_key:{kid}") + st = shares.status(db, cfg) + return S3KeyCreated(id=kid, secret=secret, port=st["s3"]["port"], webdav_port=st["webdav"]["port"]) + + +@router.delete("/s3/keys/{key_id}") +def revoke_s3_key(key_id: str, user: Writer, db: Db) -> Ok: + """Revoke one of your keys (admins: anyone's). It stops working at once.""" + if s3.revoke_key(db, key_id, None if user.admin and user.via != "oauth" else user.id): + auth.audit(db, user.as_audit(), "s3_key.revoke", f"file_key:{key_id}") + return Ok() + + +@router.get("/s3/keys") +def all_s3_keys(user: AdminReader, db: Db) -> list[S3Key]: + """Everyone's access keys, with their owners (admins).""" + return [S3Key(**k) for k in s3.keys_of(db)] diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 96cbc0797..b98c80b75 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -260,7 +260,22 @@ def platform_main(argv, config): import threading import time - from .domain import auth, bridge, jobs, local_llm, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel, updates + from .domain import ( + auth, + bridge, + jobs, + local_llm, + notify, + passkeys, + routines, + sensors, + settings, + shares, + sources, + telemetry, + tunnel, + updates, + ) ap = argparse.ArgumentParser(prog="lens") sub = ap.add_subparsers(dest="cmd", required=True) @@ -364,6 +379,7 @@ def password(): if not (a.steps or a.no_schedule): # and scans watched folders and runs routines, as `lens watch` does routines.start(db, C, stop, log=print) sensors.start(db, C, stop, log=print, name=wk.name) # and the sensor hub, while sensors are on + shares.start(db, C, stop, log=print, name=wk.name) # and the file shares (S3, WebDAV), while they're on bridge.start(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat-room bridge, while it's on tunnel.start(db, C, stop, name=wk.name, log=print) # and the Cloudflare tunnel, while it's on local_llm.start(db, C, stop, name=wk.name, log=print) # and the local chat model, while it's on diff --git a/fastapi_backend/app/domain/ops_tools.py b/fastapi_backend/app/domain/ops_tools.py index 9976e55d6..5a51c0399 100644 --- a/fastapi_backend/app/domain/ops_tools.py +++ b/fastapi_backend/app/domain/ops_tools.py @@ -40,8 +40,10 @@ "matterbridge", "telemetry", "sensors", + "shares", ) -ALWAYS_ASK = {"telemetry", "sensors", "files"} # sensors open ports on the network; files moves where new files go +# sensors and shares open ports on the network; files moves where new files go +ALWAYS_ASK = {"telemetry", "sensors", "shares", "files"} ADMIN_TOOLS = [ ( diff --git a/fastapi_backend/app/domain/s3.py b/fastapi_backend/app/domain/s3.py new file mode 100644 index 000000000..d4c3ab4b9 --- /dev/null +++ b/fastapi_backend/app/domain/s3.py @@ -0,0 +1,433 @@ +"""The file tree over S3 (docs/file-tree.md#s3): a read-only S3 server any S3 client mounts or syncs (rclone, +Cyberduck, the AWS CLI, s3fs, boto3), each person's namespaces as buckets. + +Off until an admin turns it on (`shares.enabled`); then the process that runs routines (`lens worker`) listens on +`shares.s3_port`, as the sensor hub does (shares.py). Every request is signed (AWS Signature Version 4, in the Authorization +header or a presigned link) with an access key someone made for themselves in the web app; the key acts as them and +sees what they see in the web app. Keys' secrets are sealed with the server's key (they must be readable: SigV4 signs +with the secret itself), shown once, and can be revoked. Writes answer AccessDenied. + +Path-style addressing (http://host:port//), any region name. Supported: ListBuckets, HeadBucket, +GetBucketLocation, ListObjects (v1 and v2, with a delimiter or recursive), HeadObject and GetObject (with Range). +""" + +from __future__ import annotations + +import base64 +import datetime as dt +import email.utils +import hashlib +import hmac +import http.server +import re +import secrets +import urllib.parse +from typing import Any +from xml.sax.saxutils import escape + +from . import auth, filetree, keyring, settings, store + +R = store.R +NS = "http://s3.amazonaws.com/doc/2006-03-01/" +KEY_RX = re.compile(r"^LK[A-Z0-9]{18}$") +ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +SKEW = dt.timedelta(minutes=15) +MAX_KEYS = 1000 +MAX_PRESIGN_SECONDS = 7 * 24 * 3600 +NAME_MAX = 80 + + +class S3Error(Exception): + def __init__(self, status: int, code: str, message: str): + super().__init__(message) + self.status, self.code, self.message = status, code, message + + +# ---------- access keys ---------- +def _context(kid): + return f"file_key:{kid}:secret" + + +def create_key(db, cfg, account, name=None, days=None): + """A new access key for an account: (key id, secret). The secret is shown this once. It lasts `days` (the API + keys' limits apply: auth.token_days; None takes their default).""" + days = auth.token_days(cfg, days) + kid = "LK" + "".join(secrets.choice(ALPHABET) for _ in range(18)) + secret = secrets.token_urlsafe(30) + db.q( + "CREATE $r CONTENT $d", + r=R("file_key", kid), + d={ + "account": int(account), + "name": (str(name or "").strip() or "S3 key")[:NAME_MAX], + "sealed": settings.seal(cfg, secret, _context(kid)), + "created_at": store.now(), + **({"expires_at": auth._later(24 * days)} if days else {}), + }, + ) + return kid, secret + + +def keys_of(db, account=None): + """Someone's keys (None: everyone's, with their owners' emails), newest first; never the secrets.""" + rows = db.rows( + "SELECT record::id(id) AS id, account, name, created_at, expires_at, last_used_at FROM file_key" + + (" WHERE account = $a" if account is not None else ""), + a=account, + ) + if account is None and rows: + emails = {a["id"]: a["email"] for a in db.rows("SELECT record::id(id) AS id, email FROM account")} + rows = [{**r, "email": emails.get(r["account"])} for r in rows] + return sorted(rows, key=lambda r: (r.get("created_at") or "", r["id"]), reverse=True) + + +def revoke_key(db, kid, account=None): + """Revoke a key (only the account's own, when given); whether there was one.""" + row = db.one("SELECT account FROM $r", r=R("file_key", str(kid))) if KEY_RX.match(str(kid)) else None + if not row or (account is not None and row["account"] != account): + return False + db.q("DELETE $r", r=R("file_key", str(kid))) + return True + + +def key_account(db, cfg, kid): + """(secret, active account) for a key id, or None.""" + if not KEY_RX.match(kid or ""): + return None + row = db.one("SELECT account, sealed, expires_at FROM $r", r=R("file_key", kid)) + if not row or (row.get("expires_at") and row["expires_at"] < store.now()): + return None + acct = auth.active_account(db, row["account"]) + if not acct: + return None + try: + secret = settings.unseal(cfg, row["sealed"], _context(kid)) + except Exception: # noqa: BLE001 - the server's key changed since: the key is dead + return None + return secret, acct + + +# ---------- Signature Version 4 ---------- +def _enc(s, safe="-_.~"): + return urllib.parse.quote(s, safe=safe) + + +def canonical_query(pairs): + return "&".join(f"{_enc(k)}={_enc(v)}" for k, v in sorted((k, v) for k, v in pairs)) + + +def canonical_path(raw): + """The path as SigV4 signs it for S3: each segment encoded once, as the client sent it decoded.""" + return "/".join(_enc(urllib.parse.unquote(p)) for p in raw.split("/")) or "/" + + +def _hmac(key, msg): + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +def signature(secret, date, region, service, to_sign): + k = _hmac(_hmac(_hmac(_hmac(("AWS4" + secret).encode(), date), region), service), "aws4_request") + return hmac.new(k, to_sign.encode(), hashlib.sha256).hexdigest() + + +def _amz_time(s): + try: + return dt.datetime.strptime(s or "", "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc) + except ValueError: + raise S3Error(403, "AccessDenied", "the request's date is missing or malformed") from None + + +def verify(db, cfg, method, raw_path, pairs, headers, now=None): + """Check a request's signature; the account it acts as. `pairs`: the query string's (name, value) pairs, `headers` + lowercased names to values.""" + now = now or dt.datetime.now(dt.timezone.utc) + q = dict(pairs) + presigned = "X-Amz-Signature" in q + if presigned: + if q.get("X-Amz-Algorithm") != "AWS4-HMAC-SHA256": + raise S3Error(400, "AuthorizationQueryParametersError", "only AWS4-HMAC-SHA256 is supported") + cred, signed, sig, amz = q.get("X-Amz-Credential", ""), q.get("X-Amz-SignedHeaders", ""), q["X-Amz-Signature"], q.get("X-Amz-Date") + try: + expires = int(q.get("X-Amz-Expires", "")) + except ValueError: + raise S3Error(400, "AuthorizationQueryParametersError", "X-Amz-Expires is a number of seconds") from None + when = _amz_time(amz) + if not 0 < expires <= MAX_PRESIGN_SECONDS or now > when + dt.timedelta(seconds=expires) or when - now > SKEW: + raise S3Error(403, "AccessDenied", "this link has expired") + payload = "UNSIGNED-PAYLOAD" + pairs = [(k, v) for k, v in pairs if k != "X-Amz-Signature"] + else: + h = headers.get("authorization", "") + if not h: + raise S3Error(403, "AccessDenied", "sign requests with an access key made in Lens (Settings, File shares)") + if not h.startswith("AWS4-HMAC-SHA256 "): + raise S3Error(400, "InvalidRequest", "only AWS Signature Version 4 is supported") + parts = dict(p.strip().split("=", 1) for p in h[len("AWS4-HMAC-SHA256 ") :].split(",") if "=" in p) + cred, signed, sig = parts.get("Credential", ""), parts.get("SignedHeaders", ""), parts.get("Signature", "") + amz = headers.get("x-amz-date") or "" + when = _amz_time(amz) if amz else _http_time(headers.get("date")) + if abs(now - when) > SKEW: + raise S3Error(403, "RequestTimeTooSkewed", "the request's time is too far from the server's") + payload = headers.get("x-amz-content-sha256") or "UNSIGNED-PAYLOAD" + scope = cred.split("/") + if len(scope) != 5 or scope[4] != "aws4_request": + raise S3Error(403, "AccessDenied", "the credential is malformed") + kid, date, region, service = scope[:4] + found = key_account(db, cfg, kid) + if not found: + raise S3Error(403, "InvalidAccessKeyId", "no such access key, or it was revoked") + secret, acct = found + names = [n for n in signed.split(";") if n] + if "host" not in names: + raise S3Error(403, "AccessDenied", "the host header has to be signed") + canon_headers = "".join(f"{n}:{' '.join(str(headers.get(n, '')).split())}\n" for n in names) + canonical = "\n".join([method, canonical_path(raw_path), canonical_query(pairs), canon_headers, ";".join(names), payload]) + to_sign = "\n".join(["AWS4-HMAC-SHA256", amz or when.strftime("%Y%m%dT%H%M%SZ"), "/".join(scope[1:4] + ["aws4_request"]), + hashlib.sha256(canonical.encode()).hexdigest()]) # fmt: skip + if not hmac.compare_digest(signature(secret, date, region, service, to_sign), sig): + raise S3Error(403, "SignatureDoesNotMatch", "the signature doesn't match: check the secret key") + db.q("UPDATE $r SET last_used_at = $t", r=R("file_key", kid), t=store.now()) + return acct + + +def _http_time(s): + try: + t = email.utils.parsedate_to_datetime(s or "") + except (TypeError, ValueError): + t = None + if t is None: + raise S3Error(403, "AccessDenied", "the request's date is missing or malformed") + return t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc) + + +# ---------- answers ---------- +def _iso(s): + """An S3 time (2026-10-10T20:05:45.000Z) for a stored ISO time.""" + try: + t = dt.datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except ValueError: + t = dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc) + t = t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc) + return t.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def _xml(body): + return ('\n' + body).encode() + + +def error_xml(e: S3Error, resource=""): + return _xml( + f"{e.code}{escape(e.message)}{escape(resource)}" + f"{secrets.token_hex(8)}" + ) + + +def list_buckets(tree, account): + buckets = "".join(f"{escape(e.name)}{_iso(None)}" for e in tree.list("")) + who = escape(str(account.get("email") or account["id"])) + return _xml(f'{who}{who}' + f"{buckets}") # fmt: skip + + +def _items(tree, bucket, prefix, delimiter): + """(key, entry or None for a common prefix), sorted as S3 sorts them (by their UTF-8 bytes).""" + folder, _, _ = prefix.rpartition("/") + base = f"{bucket}/{folder}" if folder else bucket + pre = f"{folder}/" if folder else "" + try: + entries = tree.list(base) + except (filetree.NotFound, filetree.NotAFolder): + return [] + out = [] + if delimiter == "/": + for e in entries: + key = pre + e.name + ("/" if e.folder else "") + if key.startswith(prefix): + out.append((key, None if e.folder else e)) + else: + for e in entries: + key = pre + e.name + if e.folder: + if (key + "/").startswith(prefix): # only the folders the prefix reaches into + out += [(p[len(bucket) + 1 :], f) for p, f in tree.walk(f"{base}/{e.name}")] + else: + out.append((key, e)) + out = [(k, e) for k, e in out if k.startswith(prefix)] + if delimiter: # another delimiter: group by it after the prefix + grouped, seen = [], set() + for k, e in out: + cut = k.find(delimiter, len(prefix)) + if cut < 0: + grouped.append((k, e)) + elif k[: cut + len(delimiter)] not in seen: + seen.add(k[: cut + len(delimiter)]) + grouped.append((k[: cut + len(delimiter)], None)) + out = grouped + return sorted(out, key=lambda x: x[0].encode()) + + +def list_objects(tree, bucket, q): + """ListObjects, v1 or v2 (list-type=2).""" + v2 = q.get("list-type") == "2" + prefix, delimiter = q.get("prefix", ""), q.get("delimiter", "") + try: + most = max(0, min(MAX_KEYS, int(q.get("max-keys", MAX_KEYS)))) + except ValueError: + raise S3Error(400, "InvalidArgument", "max-keys is a number") from None + if v2: + token = q.get("continuation-token") + try: + after = base64.urlsafe_b64decode(token.encode()).decode() if token else q.get("start-after", "") + except ValueError: + raise S3Error(400, "InvalidArgument", "the continuation token is not valid") from None + else: + after = q.get("marker", "") + items = [x for x in _items(tree, bucket, prefix, delimiter) if x[0].encode() > after.encode()] + page, more = items[:most], len(items) > most + url = q.get("encoding-type") == "url" + + def k(s): + return escape(urllib.parse.quote(s, safe="/") if url else s) + + body = [f"{escape(bucket)}{k(prefix)}{most}"] + if delimiter: + body.append(f"{k(delimiter)}") + if url: + body.append("url") + body.append(f"{'true' if more else 'false'}") + if v2: + body.append(f"{len(page)}") + if q.get("continuation-token"): + body.append(f"{escape(q['continuation-token'])}") + if q.get("start-after"): + body.append(f"{k(q['start-after'])}") + if more: + body.append(f"{base64.urlsafe_b64encode(page[-1][0].encode()).decode()}") + else: + body.append(f"{k(after)}") + if more and delimiter: + body.append(f"{k(page[-1][0])}") + for key, e in page: + if e is None: + body.append(f"{k(key)}") + else: + body.append( + f"{k(key)}{_iso(e.modified)}"{e.version}"" + f"{e.size}STANDARD" + ) + root = "ListBucketResult" + return _xml(f'<{root} xmlns="{NS}">' + "".join(body) + f"") + + +# ---------- the server ---------- +class Handler(http.server.BaseHTTPRequestHandler): + server_version = "Lens" + protocol_version = "HTTP/1.1" + server: Any # shares.Server + + def log_message(self, fmt, *args): # quiet: the requests aren't worth a line each + pass + + def _send(self, status, body=b"", ctype="application/xml", headers=None, head=False): + self.send_response(status) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.send_header("x-amz-request-id", secrets.token_hex(8)) + for k, v in (headers or {}).items(): + self.send_header(k, v) + self.end_headers() + if not head: + self.wfile.write(body) + + def _fail(self, e: S3Error, resource=""): + self._send(e.status, b"" if self.command == "HEAD" else error_xml(e, resource), head=self.command == "HEAD") + + def do_GET(self): + self._handle(head=False) + + def do_HEAD(self): + self._handle(head=True) + + def _write(self): + self.rfile.read(min(int(self.headers.get("content-length") or 0), 1 << 20)) # what was sent, to keep the connection + self._fail(S3Error(403, "AccessDenied", "the file tree is read-only")) + + do_PUT = do_POST = do_DELETE = _write # noqa: N815 - http.server's names + + def _handle(self, head): + raw, _, query = self.path.partition("?") + try: + pairs = urllib.parse.parse_qsl(query, keep_blank_values=True) + headers = {k.lower(): v for k, v in self.headers.items()} + db, cfg = self.server.db, self.server.cfg_fn() + acct = verify(db, cfg, self.command, raw, pairs, headers) + tree = filetree.Tree(db, cfg, filetree.Viewer.of(db, acct)) + path = urllib.parse.unquote(raw) + bucket, _, key = path.lstrip("/").partition("/") + q = dict(pairs) + if not bucket: + return self._send(200, list_buckets(tree, acct), head=head) + if bucket not in [e.name for e in tree.list("")]: + raise S3Error(404, "NoSuchBucket", "no such bucket, or not yours to read") + if not key: + if "location" in q: + return self._send(200, _xml(f''), head=head) + if head: + return self._send(200, head=True) + if set(q) & {"acl", "policy", "versioning", "lifecycle", "cors", "tagging", "uploads", "website", "object-lock"}: + raise S3Error(501, "NotImplemented", "the file tree only lists and reads") + return self._send(200, list_objects(tree, bucket, q)) + return self._object(tree, db, cfg, bucket, key, head) + except S3Error as e: + self._fail(e, raw) + except keyring.Locked: + self._fail(S3Error(403, "AccessDenied", "this namespace is a locked vault: an owner unlocks it in the web app"), raw) + except (BrokenPipeError, ConnectionResetError): + pass + except Exception as e: # noqa: BLE001 - say so, and keep serving + self.server.log(f"s3: {type(e).__name__}: {e}") + self._fail(S3Error(500, "InternalError", "something went wrong on the server"), raw) + + def _object(self, tree, db, cfg, bucket, key, head): + try: + e = tree.stat(f"{bucket}/{key}") + except filetree.NotFound: + e = None + if e is None or e.folder: + raise S3Error(404, "NoSuchKey", "no such file") + if e.encrypted: + keyring.plain_size(db, cfg, e.path) # a locked vault says so before anything is sent + start, end, status = 0, e.size - 1, 200 + rng = self.headers.get("range") + if rng and e.size: + m = re.fullmatch(r"bytes=(\d*)-(\d*)", rng.strip()) + if not m or m.groups() == ("", ""): + raise S3Error(416, "InvalidRange", "the range is not valid") + a, b = m.groups() + start, end = (max(0, e.size - int(b)), e.size - 1) if a == "" else (int(a), min(e.size - 1, int(b)) if b else e.size - 1) + if start > end or start >= e.size: + raise S3Error(416, "InvalidRange", "the range is not satisfiable") + status = 206 + when = dt.datetime.fromisoformat(_iso(e.modified).replace(".000Z", "+00:00")) + self.send_response(status) + self.send_header("Content-Type", e.content_type or "application/octet-stream") + self.send_header("Content-Length", str(max(0, end - start + 1))) + self.send_header("ETag", f'"{e.version}"') + self.send_header("Last-Modified", email.utils.format_datetime(when, usegmt=True)) + self.send_header("Accept-Ranges", "bytes") + if status == 206: + self.send_header("Content-Range", f"bytes {start}-{end}/{e.size}") + self.end_headers() + if head or end < start: + return + for chunk in e.read(db, cfg, start, end): + self.wfile.write(chunk) + + +def password_account(db, cfg, kid, secret): + """The account an access key and its secret sign in as (WebDAV's Basic sign-in), or None.""" + found = key_account(db, cfg, kid) + if not found or not hmac.compare_digest(found[0].encode(), str(secret or "").encode()): + return None + db.q("UPDATE $r SET last_used_at = $t", r=R("file_key", kid), t=store.now()) + return found[1] diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index ad1ed0554..1665999e6 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -71,6 +71,8 @@ "max_per_minute", "triage", ), + # bind is a startup setting only + "shares": ("enabled", "s3", "s3_port", "webdav", "webdav_port"), "video": ( "sample_seconds", "scene_threshold", @@ -419,6 +421,8 @@ def _check(section, key, value, default): return _backup_setting(key, value) if section == "fedora": return _fedora_setting(key, value) + if section == "shares": + return _share_setting(key, value) if section == "sensors": return _sensor_setting(key, value) if section == "components": @@ -908,6 +912,16 @@ def _sensor_setting(key, value): return value +def _share_setting(key, value): + if key in ("enabled", "s3", "webdav"): + if not isinstance(value, bool): + raise ValueError(f"shares.{key} is true or false") + return value + if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535): + raise ValueError(f"shares.{key} is a port number from 1 to 65535") + return value + + TELEMETRY_RANGES = {"export_seconds": (5, 3600)} SERVICE_RX = re.compile(r"^[A-Za-z0-9._-]{1,64}$") TOKEN_RX = re.compile(r"^[A-Za-z0-9._~-]+$") diff --git a/fastapi_backend/app/domain/shares.py b/fastapi_backend/app/domain/shares.py new file mode 100644 index 000000000..7f11de1c7 --- /dev/null +++ b/fastapi_backend/app/domain/shares.py @@ -0,0 +1,135 @@ +"""File shares (docs/file-tree.md): the servers that share the asset tree (filetree.py) on the network, read-only, with +people's own access keys (s3.py): S3 (s3.py) and WebDAV (webdav.py). + +Off until an admin turns them on (`shares.enabled`, then `s3` and `webdav`); the process that runs routines (`lens +worker`) follows the settings as they change, as the sensor hub does, and says what each server is doing in +`app_service:share-`. +""" + +from __future__ import annotations + +import http.server +import socket +import threading +import time +from collections.abc import Callable +from typing import Any + +from . import s3, store, webdav + +R = store.R +CHECK_SECONDS = 5 +RETRY_SECONDS = 30 +# protocol -> (its handler, the setting naming its port) +PROTOCOLS: dict[str, tuple[type[http.server.BaseHTTPRequestHandler], str]] = { + "s3": (s3.Handler, "s3_port"), + "webdav": (webdav.Handler, "webdav_port"), +} + + +def _row(proto): + return R("app_service", f"share-{proto}") + + +class Server(http.server.ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, bind, port, db, cfg_fn: Callable[[], dict[str, Any]], log=None, handler=s3.Handler): + self.db, self.cfg_fn, self.log = db, cfg_fn, log or (lambda *a: None) + if ":" in bind: + self.address_family = socket.AF_INET6 + super().__init__((bind, port), handler) + self.port = self.server_address[1] + + def start(self): + threading.Thread(target=self.serve_forever, daemon=True, name="share-server").start() + return self + + def stop(self): + self.shutdown() + self.server_close() + + +def settings_of(cfg): + return {**store.DEFAULTS["shares"], **(cfg.get("shares") or {})} + + +def wanted(cfg): + """{protocol: (bind, port) it should listen on, or None while it's off}.""" + s = settings_of(cfg) + return { + proto: (s.get("bind") or "0.0.0.0", int(s[port])) if s.get("enabled") and s.get(proto) else None + for proto, (_, port) in PROTOCOLS.items() + } + + +class Service: + """Starts, stops or moves each server as the settings change, and says what it's doing.""" + + def __init__(self, db, cfg_fn, log=None, name=None): + self.db, self.cfg_fn, self.log = db, cfg_fn, log or (lambda *a: None) + self.name = name or f"{socket.gethostname()}-{id(self) % 10000}" + self.servers: dict[str, Server] = {} + self.state: dict[str, dict[str, Any]] = {p: {} for p in PROTOCOLS} + self.retry = dict.fromkeys(PROTOCOLS, 0.0) + + def apply(self): + for proto, want in wanted(self.cfg_fn()).items(): + st = self.state[proto] + if want == st.get("want") and (proto in self.servers or want is None or time.monotonic() < self.retry[proto]): + continue + self._stop(proto) + if want is None: + self.state[proto] = {} + self.db.q("DELETE $r", r=_row(proto)) + continue + try: + srv = Server(want[0], want[1], self.db, self.cfg_fn, self.log, PROTOCOLS[proto][0]).start() + self.servers[proto] = srv + self.state[proto] = {"want": want, "running": True, "port": srv.port, "since": store.now()} + self.log(f"shares: {proto} on port {srv.port}") + except OSError as e: + self.state[proto] = {"want": want, "running": False, "error": f"port {want[1]}: {e.strerror or e}"} + self.retry[proto] = time.monotonic() + RETRY_SECONDS + out = {k: v for k, v in self.state[proto].items() if k != "want"} + self.db.q("UPSERT $r CONTENT $d", r=_row(proto), d=store.clean({"process": self.name, "at": store.now(), **out})) + + def _stop(self, proto): + srv = self.servers.pop(proto, None) + if srv is not None: + try: + srv.stop() + except Exception: # noqa: BLE001 + pass + + def close(self): + for proto in list(self.servers): + self._stop(proto) + + def loop(self, stop): + while True: + try: + self.apply() + except Exception as e: # noqa: BLE001 - keep following the settings + self.log(f"shares: {type(e).__name__}: {e}") + if stop.wait(CHECK_SECONDS): + break + self.close() + + +def start(db, cfg_fn, stop, log=None, name=None): + """The thread that runs the servers that are turned on, until `stop` is set.""" + svc = Service(db, cfg_fn, log, name) + threading.Thread(target=svc.loop, args=(stop,), daemon=True, name="shares").start() + return svc + + +def status(db, cfg): + """{protocol: {enabled, port, service}}: whether it's on, its port, and what the process running it says.""" + s, out = settings_of(cfg), {} + for proto, (_, port) in PROTOCOLS.items(): + row = db.one("SELECT * FROM $r", r=_row(proto)) or {} + row.pop("id", None) + out[proto] = {"enabled": bool(s.get("enabled") and s.get(proto)), "port": int(s[port]), "service": row or None} + return out diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 75a8415d8..8324adc02 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -455,6 +455,9 @@ def labels(): "max_per_minute": 600, "triage": False, }, + # the asset tree over S3 and WebDAV (shares.py, docs/file-tree.md): off until an admin turns it on. Then the process + # that runs routines (`lens worker`) serves it, read-only, on these ports. bind is a startup setting only. + "shares": {"enabled": False, "bind": "0.0.0.0", "s3": True, "s3_port": 9555, "webdav": True, "webdav_port": 9556}, # notifications to webhooks and Matterbridge (docs/notifications.md): targets reach public addresses only, and the # private networks listed here (a Matterbridge on the LAN or the Docker network); app_url is where links in messages # point (null: FRONTEND_URL) @@ -1020,6 +1023,9 @@ def close(self): "DEFINE INDEX IF NOT EXISTS external_identity_account ON external_identity FIELDS account", "DEFINE INDEX IF NOT EXISTS external_identity_provider ON external_identity FIELDS provider", "DEFINE TABLE IF NOT EXISTS external_flow SCHEMALESS", + # access keys for the file tree's S3 server (s3.py): id is the key id, the secret sealed + "DEFINE TABLE IF NOT EXISTS file_key SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS file_key_account ON file_key FIELDS account", "DEFINE TABLE IF NOT EXISTS api_token SCHEMALESS", "DEFINE INDEX IF NOT EXISTS api_token_hash ON api_token FIELDS hash UNIQUE", "DEFINE INDEX IF NOT EXISTS api_token_account ON api_token FIELDS account", diff --git a/fastapi_backend/app/domain/webdav.py b/fastapi_backend/app/domain/webdav.py new file mode 100644 index 000000000..4e3ec99d1 --- /dev/null +++ b/fastapi_backend/app/domain/webdav.py @@ -0,0 +1,188 @@ +"""The asset tree over WebDAV (docs/file-tree.md#webdav): a read-only WebDAV server (RFC 4918, class 1) that Finder, +Windows Explorer, GNOME Files, Dolphin, rclone and davfs2 mount as a drive. + +Each person signs in with one of their access keys (s3.py): the key id as the user name and its secret as the +password (HTTP Basic). They see the namespaces and collections the web app shows them. OPTIONS, PROPFIND (depth 0 +or 1), GET and HEAD (with Range); anything that writes answers 403. Run by shares.py. +""" + +from __future__ import annotations + +import base64 +import datetime as dt +import email.utils +import http.server +import re +import urllib.parse +from typing import Any +from xml.sax.saxutils import escape + +from . import auth, filetree, keyring, s3 + +ALLOW = "OPTIONS, GET, HEAD, PROPFIND" +REALM = 'Basic realm="Lens", charset="UTF-8"' +MAX_BODY = 1 << 20 + + +def _http_date(s): + try: + t = dt.datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except ValueError: + t = dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc) + return email.utils.format_datetime((t if t.tzinfo else t.replace(tzinfo=dt.timezone.utc)).astimezone(dt.timezone.utc), usegmt=True) + + +def href(parts, folder): + return "/" + "/".join(urllib.parse.quote(p, safe="") for p in parts) + ("/" if folder and parts else "") + + +def _response(parts, e): + props = [f"{escape(e.name)}", f"{_http_date(e.modified)}"] + if e.folder: + props.append("") + else: + props += [ + "", + f"{e.size}", + f"{escape(e.content_type or 'application/octet-stream')}", + f'"{e.version}"', + ] + return ( + f"{escape(href(parts, e.folder))}{''.join(props)}" + "HTTP/1.1 200 OK" + ) + + +def multistatus(tree, parts, depth): + """The PROPFIND answer for a path: itself, and with depth 1 what's in it.""" + e = tree.stat("/".join(parts)) + out = [_response(parts, e)] + if depth != "0" and e.folder: + out += [_response([*parts, k.name], k) for k in tree.list("/".join(parts))] + return ('\n' + "".join(out) + "").encode() + + +class Handler(http.server.BaseHTTPRequestHandler): + server_version = "Lens" + protocol_version = "HTTP/1.1" + server: Any # shares.Server + + def log_message(self, fmt, *args): + pass + + def _send(self, status, body=b"", ctype="text/plain; charset=utf-8", headers=None): + self.send_response(status) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.send_header("DAV", "1") + for k, v in (headers or {}).items(): + self.send_header(k, v) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _drain(self): + n = int(self.headers.get("content-length") or 0) + if n > MAX_BODY: + self.close_connection = True + return + self.rfile.read(n) + + def _account(self): + """The account the request signs in as, or None (an answer was sent).""" + ip = self.client_address[0] + if auth.throttled(f"webdav:{ip}"): + self._send(429, b"too many failed sign-ins: try again later\n") + return None + h = self.headers.get("authorization", "") + acct = None + if h[:6].lower() == "basic ": + try: + kid, _, secret = base64.b64decode(h[6:].strip()).decode().partition(":") + except ValueError: + kid, secret = "", "" + acct = s3.password_account(self.server.db, self.server.cfg_fn(), kid, secret) + if acct is None: + auth.hit(f"webdav:{ip}") + if acct is None: + self._send(401, b"sign in with an access key from Lens (your API tokens page)\n", headers={"WWW-Authenticate": REALM}) + return acct + + def _parts(self): + raw = self.path.partition("?")[0] + return [urllib.parse.unquote(p) for p in raw.split("/") if p] + + def do_OPTIONS(self): # noqa: N802 - http.server's names + self._drain() + self._send(200, headers={"Allow": ALLOW, "MS-Author-Via": "DAV"}) + + def _write(self): + self._drain() + self._send(403, b"the file tree is read-only\n", headers={"Allow": ALLOW}) + + do_PUT = do_POST = do_DELETE = do_MKCOL = do_MOVE = do_COPY = do_PROPPATCH = do_LOCK = do_UNLOCK = _write # noqa: N815 + + def _tree(self, acct): + db = self.server.db + return filetree.Tree(db, self.server.cfg_fn(), filetree.Viewer.of(db, acct)) + + def do_PROPFIND(self): # noqa: N802 + self._drain() + acct = self._account() + if acct is None: + return + depth = self.headers.get("depth", "1").strip().lower() + if depth not in ("0", "1"): + return self._send(403, b"PROPFIND with Depth: infinity isn't supported; ask for depth 1\n") + try: + body = multistatus(self._tree(acct), self._parts(), depth) + except filetree.NotFound: + return self._send(404, b"not found\n") + self._send(207, body, "application/xml; charset=utf-8") + + def do_GET(self): # noqa: N802 + self._get() + + def do_HEAD(self): # noqa: N802 + self._get() + + def _get(self): + acct = self._account() + if acct is None: + return + db, cfg = self.server.db, self.server.cfg_fn() + try: + e = self._tree(acct).stat("/".join(self._parts())) + except filetree.NotFound: + return self._send(404, b"not found\n") + if e.folder: + return self._send(200, b"a folder: open it with a WebDAV client\n") + try: + if e.encrypted: + keyring.plain_size(db, cfg, e.path) + except keyring.Locked: + return self._send(403, b"this namespace is a locked vault: an owner unlocks it in the web app\n") + start, end, status = 0, e.size - 1, 200 + m = re.fullmatch(r"bytes=(\d*)-(\d*)", (self.headers.get("range") or "").strip()) + if m and m.groups() != ("", "") and e.size: + a, b = m.groups() + start, end = (max(0, e.size - int(b)), e.size - 1) if a == "" else (int(a), min(e.size - 1, int(b)) if b else e.size - 1) + if start > end or start >= e.size: + return self._send(416, headers={"Content-Range": f"bytes */{e.size}"}) + status = 206 + self.send_response(status) + self.send_header("Content-Type", e.content_type or "application/octet-stream") + self.send_header("Content-Length", str(max(0, end - start + 1))) + self.send_header("ETag", f'"{e.version}"') + self.send_header("Last-Modified", _http_date(e.modified)) + self.send_header("Accept-Ranges", "bytes") + if status == 206: + self.send_header("Content-Range", f"bytes {start}-{end}/{e.size}") + self.end_headers() + if self.command == "HEAD" or end < start: + return + try: + for chunk in e.read(db, cfg, start, end): + self.wfile.write(chunk) + except (BrokenPipeError, ConnectionResetError): + pass diff --git a/fastapi_backend/app/schemas/filetree.py b/fastapi_backend/app/schemas/filetree.py index 28aa8e629..b1be7cb74 100644 --- a/fastapi_backend/app/schemas/filetree.py +++ b/fastapi_backend/app/schemas/filetree.py @@ -4,7 +4,7 @@ from pydantic import Field -from app.schemas.common import ResponseModel +from app.schemas.common import RequestModel, ResponseModel class TreeEntry(ResponseModel): @@ -18,3 +18,46 @@ class TreeEntry(ResponseModel): class TreeFolder(ResponseModel): path: str = Field(description="the folder, from the top: '' for the namespaces") entries: list[TreeEntry] + + +class S3Key(ResponseModel): + id: str = Field(description="the access key id") + name: str + created_at: str | None = None + expires_at: str | None = None + last_used_at: str | None = None + email: str | None = Field(None, description="its owner (the list of everyone's keys)") + + +class S3KeyCreate(RequestModel): + name: str = Field("S3 key", max_length=80) + days: int | None = Field(None, ge=0, description="how long it lasts (the API keys' limits); none: their default") + + +class S3KeyCreated(ResponseModel): + id: str = Field(description="the access key id") + secret: str = Field(description="the secret access key, shown this once") + port: int = Field(description="the S3 server's port") + webdav_port: int + + +class S3Service(ResponseModel): + process: str | None = None + at: str | None = None + running: bool | None = None + port: int | None = None + since: str | None = None + error: str | None = None + + +class ShareServer(ResponseModel): + enabled: bool = Field(description="whether an admin turned it on (shares.enabled and its own switch)") + port: int = Field(description="the port it listens on in the worker (Docker publishes LENS_S3_PORT, LENS_WEBDAV_PORT)") + service: S3Service | None = Field(None, description="what the process running it says; none while it's off") + + +class S3Status(ShareServer): + """The S3 server (these fields), the WebDAV server, and your access keys (they sign in to both).""" + + webdav: ShareServer + keys: list[S3Key] = Field(description="your access keys") diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index 784eade8f..02ca29b7f 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -74,6 +74,7 @@ dev = [ "onnxruntime>=1.18", # the objects tests run a small YOLOX-shaped model "onnx>=1.23.2", # ... which they build "mcp>=2.3.0", # the official MCP client, to test the MCP server with + "botocore>=1.43.111", ] [tool.hatch.build.targets.wheel] diff --git a/fastapi_backend/tests/api/test_s3.py b/fastapi_backend/tests/api/test_s3.py new file mode 100644 index 000000000..364d0a413 --- /dev/null +++ b/fastapi_backend/tests/api/test_s3.py @@ -0,0 +1,160 @@ +"""The file tree over S3 (docs/file-tree.md#s3): a real S3 client (botocore) lists and reads each person's namespaces +with a key they made in the web app; writes, wrong signatures and revoked keys are refused.""" + +from __future__ import annotations + +import threading +import urllib.request + +import botocore.session +import pytest +from botocore.config import Config +from botocore.exceptions import ClientError + +from app.domain import hierarchy, s3, shares, store, topics +from tests.helpers import login, make_user, seed + + +@pytest.fixture +def env(client, db, cfg, folder): + a, b, call = seed(db, cfg, folder) + topics.tag(db, topics.create(db, store.ns_id(db, "pods"), "Gene therapy"), [a]) + talks = hierarchy.create(db, store.ns_id(db, "pods"), "Talks") + db.q("UPDATE $r SET collection = $c", r=store.R("recording", b), c=talks) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + hv = login(client, "vi@x.io", "viewer password 1") + server = shares.Server("127.0.0.1", 0, db, lambda: cfg).start() + yield {"hv": hv, "port": server.port, "folder": folder} + server.stop() + + +def _key(client, h, **body): + r = client.post("/api/v1/shares/s3/keys", headers=h, json=body) + assert r.status_code == 200, r.text + return r.json() + + +def _s3(port, kid, secret): + return botocore.session.get_session().create_client( + "s3", + endpoint_url=f"http://127.0.0.1:{port}", + region_name="us-east-1", + aws_access_key_id=kid, + aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 1}), + ) + + +def test_people_read_their_namespaces_over_s3(client, env): + k = _key(client, env["hv"], name="laptop") + assert k["id"].startswith("LK") and k["port"] == 9555 and k["webdav_port"] == 9556 + c = _s3(env["port"], k["id"], k["secret"]) + assert [b["Name"] for b in c.list_buckets()["Buckets"]] == ["pods"] + top = c.list_objects_v2(Bucket="pods", Delimiter="/") + assert [p["Prefix"] for p in top["CommonPrefixes"]] == ["General/", "Talks/"] and "Contents" not in top + talks = c.list_objects_v2(Bucket="pods", Prefix="Talks/", Delimiter="/") + assert [o["Key"] for o in talks["Contents"]] == ["Talks/ep2 [2].json", "Talks/ep2 [2].original.txt", "Talks/ep2 [2].txt"] + assert [o["Key"] for o in c.list_objects_v2(Bucket="pods", Prefix="Talks/ep2 [2].t")["Contents"]] == ["Talks/ep2 [2].txt"] + # v1 listing, recursive, a page at a time + keys, marker = [], "" + while True: + page = c.list_objects(Bucket="pods", Marker=marker, MaxKeys=2) + keys += [o["Key"] for o in page.get("Contents", [])] + if not page["IsTruncated"]: + break + marker = keys[-1] + assert keys == sorted(keys, key=str.encode) and len(keys) == 6 and keys[0] == "General/ep1 [1].json" + # v2 pages + first = c.list_objects_v2(Bucket="pods", MaxKeys=4) + rest = c.list_objects_v2(Bucket="pods", ContinuationToken=first["NextContinuationToken"]) + assert [o["Key"] for o in first["Contents"] + rest["Contents"]] == keys + # objects, whole and in part, as listed + own = (env["folder"] / "ep1.txt").read_bytes() + got = c.get_object(Bucket="pods", Key="General/ep1 [1].original.txt") + assert got["Body"].read() == own and got["ContentLength"] == len(own) and len(got["ETag"].strip('"')) != 32 + part = c.get_object(Bucket="pods", Key="General/ep1 [1].original.txt", Range="bytes=3-9") + assert part["Body"].read() == own[3:10] and part["ContentRange"] == f"bytes 3-9/{len(own)}" + head = c.head_object(Bucket="pods", Key="General/ep1 [1].txt") + assert head["ContentLength"] == next(o["Size"] for o in keys_of(c, "General/") if o["Key"].endswith("ep1 [1].txt")) + assert c.get_object(Bucket="pods", Key="General/ep1 [1].txt")["Body"].read().startswith(b"Alice: Welcome back.") + # a presigned link works on its own + url = c.generate_presigned_url("get_object", Params={"Bucket": "pods", "Key": "General/ep1 [1].json"}, ExpiresIn=60) + assert b'"namespace": "pods"' in urllib.request.urlopen(url).read() + assert c.get_bucket_location(Bucket="pods")["LocationConstraint"] is None + + +def keys_of(c, prefix): + return c.list_objects_v2(Bucket="pods", Prefix=prefix)["Contents"] + + +def _code(fn): + with pytest.raises(ClientError) as e: + fn() + return e.value.response["Error"]["Code"] + + +def test_what_s3_refuses(client, env): + k = _key(client, env["hv"]) + c = _s3(env["port"], k["id"], k["secret"]) + assert _code(lambda: c.list_objects_v2(Bucket="calls")) == "NoSuchBucket" # not theirs + assert _code(lambda: c.get_object(Bucket="pods", Key="General/nope.txt")) == "NoSuchKey" + assert _code(lambda: c.put_object(Bucket="pods", Key="x.txt", Body=b"hi")) == "AccessDenied" + assert _code(lambda: c.delete_object(Bucket="pods", Key="General/ep1 [1].json")) == "AccessDenied" + assert _code(lambda: _s3(env["port"], k["id"], "wrong").list_buckets()) == "SignatureDoesNotMatch" + assert _code(lambda: _s3(env["port"], "LKAAAAAAAAAAAAAAAAAA", "x").list_buckets()) == "InvalidAccessKeyId" + # revoked: it stops working at once + assert client.delete(f"/api/v1/shares/s3/keys/{k['id']}", headers=env["hv"]).status_code == 200 + assert _code(lambda: c.list_buckets()) == "InvalidAccessKeyId" + # unsigned + with pytest.raises(urllib.error.HTTPError) as e: + urllib.request.urlopen(f"http://127.0.0.1:{env['port']}/") + assert e.value.code == 403 + + +def test_keys_are_made_while_signed_in(client, db, env): + r = client.post("/api/v1/tokens", headers=env["hv"], json={"name": "script", "scope": "write", "days": 10}) + token = {"Authorization": f"Bearer {r.json()['token']}"} + assert client.post("/api/v1/shares/s3/keys", headers=token, json={}).status_code == 403 + k = _key(client, env["hv"], name="nas") + st = client.get("/api/v1/shares/s3", headers=env["hv"]).json() + assert st["enabled"] is False and st["port"] == 9555 and st["webdav"]["port"] == 9556 + assert [(x["id"], x["name"]) for x in st["keys"]] == [(k["id"], "nas")] and "secret" not in st["keys"][0] + sealed = db.one("SELECT sealed FROM $r", r=store.R("file_key", k["id"]))["sealed"] + assert k["secret"] not in sealed # kept sealed + assert client.get("/api/v1/shares/s3/keys", headers=env["hv"]).status_code == 403 # everyone's: admins + + +def test_the_servers_follow_the_settings(db, cfg): + conf = {"enabled": False, "s3": True, "s3_port": 0, "webdav": True, "webdav_port": 0, "bind": "127.0.0.1"} + svc = shares.Service(db, lambda: {**cfg, "shares": conf}, name="t") + svc.apply() + assert not svc.servers and shares.status(db, cfg)["s3"]["service"] is None + conf["enabled"] = True + svc.apply() + assert set(svc.servers) == {"s3", "webdav"} + assert db.one("SELECT * FROM app_service:⟨share-s3⟩")["running"] is True + conf["webdav"] = False + svc.apply() + assert set(svc.servers) == {"s3"} and db.one("SELECT * FROM app_service:⟨share-webdav⟩") is None + conf["enabled"] = False + svc.apply() + assert not svc.servers and shares.status(db, cfg)["s3"]["service"] is None + stop = threading.Event() + stop.set() + svc.loop(stop) + + +def test_signature_v4_matches_aws(db, cfg): + """The example from AWS's documentation of Signature Version 4 for S3 (GET Object with a range).""" + assert ( + s3.signature( + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "20130524", + "us-east-1", + "s3", + "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n" + "7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972", + ) + == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + ) + assert s3.canonical_path("/pods/Gene%20therapy/a%20%5B1%5D.txt") == "/pods/Gene%20therapy/a%20%5B1%5D.txt" diff --git a/fastapi_backend/tests/api/test_webdav.py b/fastapi_backend/tests/api/test_webdav.py new file mode 100644 index 000000000..5415c12a6 --- /dev/null +++ b/fastapi_backend/tests/api/test_webdav.py @@ -0,0 +1,79 @@ +"""The asset tree over WebDAV (docs/file-tree.md#webdav): mounted with an access key (its id and secret as Basic +sign-in), listed with PROPFIND and read with GET, never written.""" + +from __future__ import annotations + +import base64 +import http.client +import xml.etree.ElementTree as ET + +import pytest + +from app.domain import auth, shares, webdav +from tests.helpers import login, make_user, seed + +D = "{DAV:}" + + +@pytest.fixture +def env(client, db, cfg, folder): + seed(db, cfg, folder) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + hv = login(client, "vi@x.io", "viewer password 1") + k = client.post("/api/v1/shares/s3/keys", headers=hv, json={}).json() + server = shares.Server("127.0.0.1", 0, db, lambda: cfg, handler=webdav.Handler).start() + auth._FAILS.clear() + yield {"port": server.port, "auth": "Basic " + base64.b64encode(f"{k['id']}:{k['secret']}".encode()).decode(), "k": k, "folder": folder} + server.stop() + + +def _call(env, method, path, headers=None, auth_header=True, body=None): + c = http.client.HTTPConnection("127.0.0.1", env["port"], timeout=10) + h = {**({"Authorization": env["auth"]} if auth_header else {}), **(headers or {})} + c.request(method, path, body=body, headers=h) + r = c.getresponse() + out = (r.status, dict(r.getheaders()), r.read()) + c.close() + return out + + +def _listing(body): + out = [] + for r in ET.fromstring(body).findall(f"{D}response"): + size = r.find(f".//{D}getcontentlength") + out.append((r.find(f"{D}href").text, r.find(f".//{D}collection") is not None, None if size is None else size.text)) + return out + + +def test_mount_and_read(env): + status, h, _ = _call(env, "OPTIONS", "/", auth_header=False) + assert status == 200 and h["DAV"] == "1" and "PROPFIND" in h["Allow"] + status, _, body = _call(env, "PROPFIND", "/", {"Depth": "1"}) + assert status == 207 and [(x[0], x[1]) for x in _listing(body)] == [("/", True), ("/pods/", True)] + status, _, body = _call(env, "PROPFIND", "/pods/General/", {"Depth": "1"}) + hrefs = [x[0] for x in _listing(body)] + assert hrefs[0] == "/pods/General/" and "/pods/General/ep1%20%5B1%5D.txt" in hrefs + size = next(int(x[2]) for x in _listing(body) if x[0].endswith("ep1%20%5B1%5D.txt")) + status, _, body = _call(env, "PROPFIND", "/pods/General/ep1%20%5B1%5D.txt", {"Depth": "0"}) + assert status == 207 and len(_listing(body)) == 1 + status, h, body = _call(env, "GET", "/pods/General/ep1%20%5B1%5D.txt") + assert status == 200 and len(body) == size and body.startswith(b"Alice: Welcome back.") + own = (env["folder"] / "ep1.txt").read_bytes() + status, h, body = _call(env, "GET", "/pods/General/ep1%20%5B1%5D.original.txt", {"Range": "bytes=2-5"}) + assert status == 206 and body == own[2:6] and h["Content-Range"] == f"bytes 2-5/{len(own)}" + status, h, body = _call(env, "HEAD", "/pods/General/ep1%20%5B1%5D.txt") + assert status == 200 and h["Content-Length"] == str(size) and body == b"" + + +def test_what_webdav_refuses(env): + assert _call(env, "PROPFIND", "/", {"Depth": "1"}, auth_header=False)[0] == 401 + assert _call(env, "PROPFIND", "/calls/", {"Depth": "1"})[0] == 404 # not theirs + assert _call(env, "PROPFIND", "/", {"Depth": "infinity"})[0] == 403 + assert _call(env, "GET", "/pods/General/nope.txt")[0] == 404 + assert _call(env, "PUT", "/pods/General/x.txt", body=b"hi")[0] == 403 + assert _call(env, "DELETE", "/pods/General/ep1%20%5B1%5D.txt")[0] == 403 + assert _call(env, "MKCOL", "/pods/New/")[0] == 403 + wrong = "Basic " + base64.b64encode(f"{env['k']['id']}:nope".encode()).decode() + for _ in range(8): + assert _call(env, "PROPFIND", "/", {"Depth": "0", "Authorization": wrong})[0] == 401 + assert _call(env, "PROPFIND", "/", {"Depth": "0"})[0] == 429 # too many failed sign-ins from this address diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index 89e664d29..a4bca5439 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -320,6 +320,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, ] +[[package]] +name = "botocore" +version = "1.43.111" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/43/257e97270ddd6833fd54b11e544a09b441b02f8c731bdeb29b90479be565/botocore-1.43.111.tar.gz", hash = "sha256:44d5e80962ac6cb9e85af72667b77c9586451e3328ab0ce33195380767e213d8", size = 16321685, upload-time = "2026-10-09T19:27:44.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/5b/c3ce1b227954eb0313e76e6e7c0b5b24d4c553f0e8a03e5828ff5a5918dc/botocore-1.43.111-py3-none-any.whl", hash = "sha256:f1f4c28cb2a096bf246d0bb24cbb1a01c5cb696ef499fa71b155adda7b94c90b", size = 16018923, upload-time = "2026-10-09T19:27:40.066Z" }, +] + [[package]] name = "cairocffi" version = "1.7.1" @@ -1793,6 +1807,7 @@ whisper = [ [package.dev-dependencies] dev = [ + { name = "botocore" }, { name = "coverage" }, { name = "coveralls" }, { name = "httpx" }, @@ -1859,6 +1874,7 @@ provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", [package.metadata.requires-dev] dev = [ + { name = "botocore", specifier = ">=1.43.111" }, { name = "coverage", specifier = ">=7.6" }, { name = "coveralls", specifier = ">=4.0.1,<5" }, { name = "httpx", specifier = ">=0.27" }, diff --git a/install.sh b/install.sh index f1425683f..00ee3462c 100755 --- a/install.sh +++ b/install.sh @@ -369,11 +369,12 @@ if [ "$BACKEND" = microsandbox ]; then done [ -n "$image" ] || fail "Couldn't download the base image (python:3.12-slim-bookworm): $out" # The internet, this network (mail servers, model servers) and this machine (Ollama, LM Studio); only the web app - # and the sensor ports are published, as with Docker. + # the sensor ports and the file shares' ports (S3, WebDAV) are published, as with Docker. set -- run -d -q --name lens --replace -c "$cpus" -m "${mem}M" --root-disk 16G --net public,private,host \ -p "$bind:$PORT:3000" \ -p "$sensors:${LENS_MQTT_PORT:-1883}:1883" \ -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/udp" -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/tcp" \ + -p "${LENS_SHARE_BIND:-$bind}:${LENS_S3_PORT:-9555}:9555" -p "${LENS_SHARE_BIND:-$bind}:${LENS_WEBDAV_PORT:-9556}:9556" \ -v "$PWD:/lens:ro" -v "$PWD/.lens-control:/control" --mount-named lens-data:/data \ --entrypoint /bin/sh "$image" -- /lens/microsandbox/run.sh say "Starting Lens in a microsandbox (the first start installs and builds it, which takes a while)..." diff --git a/nextjs-frontend/__tests__/s3-keys.test.ts b/nextjs-frontend/__tests__/s3-keys.test.ts new file mode 100644 index 000000000..fc1739a90 --- /dev/null +++ b/nextjs-frontend/__tests__/s3-keys.test.ts @@ -0,0 +1,25 @@ +import { rcloneConfig, webdavText } from "@/components/account/s3-model"; + +describe("S3 access keys", () => { + it("gives an rclone remote for the key, at the address the web app is on", () => { + expect(rcloneConfig("lens.local", { id: "LKABC", secret: "s3cr3t", port: 9555 })).toBe( + [ + "[lens]", + "type = s3", + "provider = Other", + "access_key_id = LKABC", + "secret_access_key = s3cr3t", + "endpoint = http://lens.local:9555", + "force_path_style = true", + ].join("\n"), + ); + }); +}); + +describe("WebDAV sign-in", () => { + it("gives the address, and the key and secret as user name and password", () => { + expect(webdavText("lens.local", { id: "LKABC", secret: "s3cr3t", webdav_port: 9556 })).toBe( + ["address = http://lens.local:9556/", "user = LKABC", "password = s3cr3t"].join("\n"), + ); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index c0c5fe61b..db6719ade 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -38,6 +38,7 @@ export { Searches, Sensors, Setup, + Shares, Sources, Speakers, Templates, @@ -107,6 +108,9 @@ export type { AddVaultPasskeyErrors, AddVaultPasskeyResponse, AddVaultPasskeyResponses, + AllS3KeysData, + AllS3KeysResponse, + AllS3KeysResponses, AnswerActionData, AnswerActionError, AnswerActionErrors, @@ -464,6 +468,11 @@ export type { CreateRoutineErrors, CreateRoutineResponse, CreateRoutineResponses, + CreateS3KeyData, + CreateS3KeyError, + CreateS3KeyErrors, + CreateS3KeyResponse, + CreateS3KeyResponses, CreateSearchData, CreateSearchError, CreateSearchErrors, @@ -2345,6 +2354,11 @@ export type { RevokeGrantResponses, RevokeResponse, RevokeResponses, + RevokeS3KeyData, + RevokeS3KeyError, + RevokeS3KeyErrors, + RevokeS3KeyResponse, + RevokeS3KeyResponses, RevokeShareData, RevokeShareError, RevokeShareErrors, @@ -2396,6 +2410,14 @@ export type { RunWorkflowErrors, RunWorkflowResponse, RunWorkflowResponses, + S3Key, + S3KeyCreate, + S3KeyCreated, + S3Service, + S3Status, + S3StatusData, + S3StatusResponse, + S3StatusResponses, SaveCollectionFieldsData, SaveCollectionFieldsError, SaveCollectionFieldsErrors, @@ -2559,6 +2581,7 @@ export type { ShareCreate, SharedContext, ShareLink, + ShareServer, ShareSite, SigninLink, SigninLinkAnswer, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 9fd8e329a..678940344 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -40,6 +40,8 @@ import type { AddVaultPasskeyData, AddVaultPasskeyErrors, AddVaultPasskeyResponses, + AllS3KeysData, + AllS3KeysResponses, AnswerActionData, AnswerActionErrors, AnswerActionResponses, @@ -201,6 +203,9 @@ import type { CreateRoutineData, CreateRoutineErrors, CreateRoutineResponses, + CreateS3KeyData, + CreateS3KeyErrors, + CreateS3KeyResponses, CreateSearchData, CreateSearchErrors, CreateSearchResponses, @@ -1154,6 +1159,9 @@ import type { RevokeGrantErrors, RevokeGrantResponses, RevokeResponses, + RevokeS3KeyData, + RevokeS3KeyErrors, + RevokeS3KeyResponses, RevokeShareData, RevokeShareErrors, RevokeShareResponses, @@ -1178,6 +1186,8 @@ import type { RunWorkflowData, RunWorkflowErrors, RunWorkflowResponses, + S3StatusData, + S3StatusResponses, SaveCollectionFieldsData, SaveCollectionFieldsErrors, SaveCollectionFieldsResponses, @@ -9438,6 +9448,70 @@ export class Sensors { } } +export class Shares { + /** + * S3 Status + * + * Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, and your + * access keys. + */ + public static s3Status( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/shares/s3", + ...options, + }); + } + + /** + * All S3 Keys + * + * Everyone's access keys, with their owners (admins). + */ + public static allS3Keys( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/shares/s3/keys", + ...options, + }); + } + + /** + * Create S3 Key + * + * An access key that reads the file tree as you, over S3 and WebDAV. Only while signed in to the web app (with your passkey), + * not with an API token. The secret is in this answer only. + */ + public static createS3Key( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/shares/s3/keys", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Revoke S3 Key + * + * Revoke one of your keys (admins: anyone's). It stops working at once. + */ + public static revokeS3Key( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/shares/s3/keys/{key_id}", + ...options, + }); + } +} + export class ImportHooks { /** * List Import Hooks diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index cd5e91ce6..60b7d0f46 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -13399,6 +13399,149 @@ export type RoutineUpdate = { enabled?: boolean | null; }; +/** + * S3Key + */ +export type S3Key = { + /** + * Id + * + * the access key id + */ + id: string; + /** + * Name + */ + name: string; + /** + * Created At + */ + created_at?: string | null; + /** + * Expires At + */ + expires_at?: string | null; + /** + * Last Used At + */ + last_used_at?: string | null; + /** + * Email + * + * its owner (the list of everyone's keys) + */ + email?: string | null; + [key: string]: unknown; +}; + +/** + * S3KeyCreate + */ +export type S3KeyCreate = { + /** + * Name + */ + name?: string; + /** + * Days + * + * how long it lasts (the API keys' limits); none: their default + */ + days?: number | null; +}; + +/** + * S3KeyCreated + */ +export type S3KeyCreated = { + /** + * Id + * + * the access key id + */ + id: string; + /** + * Secret + * + * the secret access key, shown this once + */ + secret: string; + /** + * Port + * + * the S3 server's port + */ + port: number; + /** + * Webdav Port + */ + webdav_port: number; + [key: string]: unknown; +}; + +/** + * S3Service + */ +export type S3Service = { + /** + * Process + */ + process?: string | null; + /** + * At + */ + at?: string | null; + /** + * Running + */ + running?: boolean | null; + /** + * Port + */ + port?: number | null; + /** + * Since + */ + since?: string | null; + /** + * Error + */ + error?: string | null; + [key: string]: unknown; +}; + +/** + * S3Status + * + * The S3 server (these fields), the WebDAV server, and your access keys (they sign in to both). + */ +export type S3Status = { + /** + * Enabled + * + * whether an admin turned it on (shares.enabled and its own switch) + */ + enabled: boolean; + /** + * Port + * + * the port it listens on in the worker (Docker publishes LENS_S3_PORT, LENS_WEBDAV_PORT) + */ + port: number; + /** + * what the process running it says; none while it's off + */ + service?: S3Service | null; + webdav: ShareServer; + /** + * Keys + * + * your access keys + */ + keys: Array; + [key: string]: unknown; +}; + /** * SavedSearch */ @@ -14774,6 +14917,29 @@ export type ShareLink = { [key: string]: unknown; }; +/** + * ShareServer + */ +export type ShareServer = { + /** + * Enabled + * + * whether an admin turned it on (shares.enabled and its own switch) + */ + enabled: boolean; + /** + * Port + * + * the port it listens on in the worker (Docker publishes LENS_S3_PORT, LENS_WEBDAV_PORT) + */ + port: number; + /** + * what the process running it says; none while it's off + */ + service?: S3Service | null; + [key: string]: unknown; +}; + /** * ShareSite */ @@ -33390,6 +33556,95 @@ export type PushStreamResponses = { export type PushStreamResponse = PushStreamResponses[keyof PushStreamResponses]; +export type S3StatusData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/shares/s3"; +}; + +export type S3StatusResponses = { + /** + * Successful Response + */ + 200: S3Status; +}; + +export type S3StatusResponse = S3StatusResponses[keyof S3StatusResponses]; + +export type AllS3KeysData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/shares/s3/keys"; +}; + +export type AllS3KeysResponses = { + /** + * Response Shares-All S3 Keys + * + * Successful Response + */ + 200: Array; +}; + +export type AllS3KeysResponse = AllS3KeysResponses[keyof AllS3KeysResponses]; + +export type CreateS3KeyData = { + body: S3KeyCreate; + path?: never; + query?: never; + url: "/api/v1/shares/s3/keys"; +}; + +export type CreateS3KeyErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type CreateS3KeyError = CreateS3KeyErrors[keyof CreateS3KeyErrors]; + +export type CreateS3KeyResponses = { + /** + * Successful Response + */ + 200: S3KeyCreated; +}; + +export type CreateS3KeyResponse = CreateS3KeyResponses[keyof CreateS3KeyResponses]; + +export type RevokeS3KeyData = { + body?: never; + path: { + /** + * Key Id + */ + key_id: string; + }; + query?: never; + url: "/api/v1/shares/s3/keys/{key_id}"; +}; + +export type RevokeS3KeyErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RevokeS3KeyError = RevokeS3KeyErrors[keyof RevokeS3KeyErrors]; + +export type RevokeS3KeyResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RevokeS3KeyResponse = RevokeS3KeyResponses[keyof RevokeS3KeyResponses]; + export type ListImportHooksData = { body?: never; path: { diff --git a/nextjs-frontend/components/account/s3-keys.tsx b/nextjs-frontend/components/account/s3-keys.tsx new file mode 100644 index 000000000..6ab6ba21f --- /dev/null +++ b/nextjs-frontend/components/account/s3-keys.tsx @@ -0,0 +1,178 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Check, Copy, FolderTree, Plus } from "lucide-react"; +import { useState } from "react"; + +import { Shares } from "@/app/openapi-client"; +import type { S3Key, S3KeyCreated } from "@/app/openapi-client/types.gen"; +import { rcloneConfig, webdavText } from "@/components/account/s3-model"; +import { tokenExpiry } from "@/components/account/token-model"; +import { Banner } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { Dialog } from "@/components/ui/dialog"; +import { EmptyState, SkeletonRows } from "@/components/ui/states"; +import { Table, Td, Th, THead, Tr } from "@/components/ui/table"; +import { useToast } from "@/components/ui/toast"; +import { data, useApiClient } from "@/lib/api/browser"; +import { relative, shortDate } from "@/lib/format"; +import { cn } from "@/lib/utils"; + +/** Access keys for the file tree's S3 server (docs/file-tree.md#s3): each person makes and revokes their own. */ +export function S3Keys() { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const status = useQuery({ queryKey: ["shares", "s3"], queryFn: () => data(Shares.s3Status({ client })) }); + const [created, setCreated] = useState(null); + const create = useMutation({ + mutationFn: () => data(Shares.createS3Key({ client, body: { name: "S3 key" } })), + onSuccess: (r) => { + setCreated(r); + void qc.invalidateQueries({ queryKey: ["shares", "s3"] }); + }, + }); + const revoke = useMutation({ + mutationFn: (k: S3Key) => data(Shares.revokeS3Key({ client, path: { key_id: k.id } })), + onSuccess: (_r, k) => { + void qc.invalidateQueries({ queryKey: ["shares", "s3"] }); + toast({ title: `Revoked ${k.id}`, body: "Anything signed in with it stops working now.", tone: "green" }); + }, + }); + const keys = status.data?.keys ?? []; + const newKey = ( + + ); + return ( +
+
+
+

File share access keys

+

+ Read the archive as folders and files from any S3 client (rclone, Cyberduck, the AWS CLI). A key reads as + you; nothing can be changed through it. + {status.data && !status.data.enabled && " An admin turns sharing on in Settings, File shares."} +

+
+ {keys.length > 0 && newKey} +
+ {create.isError && {create.error.message}} + {status.isPending ? ( + + ) : !keys.length ? ( + } title="No access keys yet" actions={newKey}> + Make one for each computer or app you read the archive from. + + ) : ( +
+ + + + + + + + + + + + {keys.map((k) => { + const exp = tokenExpiry(k.expires_at); + return ( + + + + + + + + ); + })} + +
Access keyCreatedExpiresLast used + Actions +
+ {k.id} + {shortDate(k.created_at)}{exp.label} + {k.last_used_at ? relative(k.last_used_at) : "never"} + + +
+
+ )} + setCreated(null)} /> +
+ ); +} + +function Copyable({ label, text }: { label: string; text: string }) { + const [copied, setCopied] = useState(false); + const copy = async () => { + try { + await navigator.clipboard.writeText(text); + setCopied(true); + } catch { + /* clipboard blocked: the text stays selectable */ + } + }; + return ( +
+ {label} +
+
+          {text}
+        
+ +
+
+ ); +} + +function ShownOnce({ created, onClose }: { created: S3KeyCreated | null; onClose: () => void }) { + const host = typeof window === "undefined" ? "localhost" : window.location.hostname; + return ( + { + if (!o) onClose(); + }} + title="Copy your key now" + description="The secret is shown this once. Other S3 clients take the same endpoint, key and secret, with path-style addressing." + actions={ + + } + > + {created && ( +
+ + +
+ )} +
+ ); +} diff --git a/nextjs-frontend/components/account/s3-model.ts b/nextjs-frontend/components/account/s3-model.ts new file mode 100644 index 000000000..d8c9f122d --- /dev/null +++ b/nextjs-frontend/components/account/s3-model.ts @@ -0,0 +1,23 @@ +import type { S3KeyCreated } from "@/app/openapi-client/types.gen"; + +/** The rclone remote for a key: what people paste into rclone.conf (or adapt for any S3 client). */ +export function rcloneConfig(host: string, created: Pick): string { + return [ + "[lens]", + "type = s3", + "provider = Other", + `access_key_id = ${created.id}`, + `secret_access_key = ${created.secret}`, + `endpoint = http://${host}:${created.port}`, + "force_path_style = true", + ].join("\n"); +} + +/** How to mount the WebDAV share with a key: its address, then the key id and secret as user name and password. */ +export function webdavText(host: string, created: Pick): string { + return [ + `address = http://${host}:${created.webdav_port}/`, + `user = ${created.id}`, + `password = ${created.secret}`, + ].join("\n"); +} diff --git a/nextjs-frontend/components/account/tokens.tsx b/nextjs-frontend/components/account/tokens.tsx index e47129d92..66c723eb4 100644 --- a/nextjs-frontend/components/account/tokens.tsx +++ b/nextjs-frontend/components/account/tokens.tsx @@ -6,6 +6,7 @@ import { useState } from "react"; import { Tokens } from "@/app/openapi-client"; import type { ApiToken } from "@/app/openapi-client/types.gen"; +import { S3Keys } from "@/components/account/s3-keys"; import { daysError, expiresOn, SCOPE_LABEL, tokenExpiry } from "@/components/account/token-model"; import { isUnreachable } from "@/components/errors/error-states"; import { ChoiceCards } from "@/components/settings/controls"; @@ -140,6 +141,7 @@ export function TokensPage() { )} + setCreating(false)} diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d773c4df8..1da113eea 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -5,7 +5,7 @@ import { PlugZap, RefreshCw, Terminal } from "lucide-react"; import Link from "next/link"; import { useEffect, useRef, type ReactNode } from "react"; -import { Admin, Fedora, Metadata, Sensors, Sources } from "@/app/openapi-client"; +import { Admin, Fedora, Metadata, Sensors, Shares, Sources } from "@/app/openapi-client"; import type { MatterbridgeStatus } from "@/app/openapi-client/types.gen"; import { BackupsPanel } from "@/components/settings/backups-panel"; import { ComponentsStatus } from "@/components/settings/components-status"; @@ -388,6 +388,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { return ; case "sensors": return ; + case "shares": + return ; case "uploads": return ( <> @@ -968,6 +970,66 @@ function SensorsBody({ ctx }: { ctx: BodyCtx }) { ); } +type ShareServer = { + enabled: boolean; + port: number; + service?: { running?: boolean | null; port?: number | null; error?: string | null } | null; +}; + +function shareText(name: string, st: ShareServer): string { + const svc = st.service; + if (svc?.error) return `${name} can’t start: ${svc.error}`; + if (svc?.running) return `${name} is serving on port ${svc.port}`; + return st.enabled ? `${name} is starting…` : `${name} is off`; +} + +function SharesBody({ ctx }: { ctx: BodyCtx }) { + const client = useApiClient(); + const status = useQuery({ + queryKey: ["shares", "s3", ctx.view.shares?.updated_at ?? null], + queryFn: () => data(Shares.s3Status({ client })), + refetchInterval: 15_000, + }); + const st = status.data; + const failed = Boolean(st?.service?.error || st?.webdav.service?.error); + const running = Boolean(st?.service?.running || st?.webdav.service?.running); + const on = ctx.values["shares.enabled"] === true; + return ( + <> + {st && ( + + Your access keys + + } + > + {on + ? `${shareText("S3", st)}. ${shareText("WebDAV", st.webdav)}. People sign in with an access key from their API tokens page.` + : "Turn it on below; the workers start listening within seconds."} + + )} + +
+ + +
+
+ + +
+

+ With Docker, the worker container publishes 9555 (S3) and 9556 (WebDAV) where the web app is; set{" "} + LENS_S3_PORT or LENS_WEBDAV_PORT in .env + to publish them on other ports of the host. + {on ? "" : " Nothing listens until it’s on."} +

+ + ); +} + function MailBody({ ctx }: { ctx: BodyCtx }) { const client = useApiClient(); const m = ctx.view.mail; diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 691f5f44c..e2557d626 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -84,6 +84,7 @@ export type SectionId = | "telemetry" | "fedora" | "sensors" + | "shares" | "uploads" | "storage" | "backups" @@ -262,6 +263,13 @@ export const SECTIONS: SectionSpec[] = [ description: "Off unless you turn it on. An MQTT hub and a syslog listener in Lens’s workers, so routers, DNS servers and devices can send to it; and how long what they send is kept.", }, + { + id: "shares", + label: "File shares", + backend: ["shares"], + description: + "Off unless you turn it on. The archive’s assets as folders and files (namespaces and collections, a metadata file beside each asset), served read-only over S3 and WebDAV by Lens’s workers; people make their own access keys.", + }, { id: "uploads", label: "Uploads", @@ -1771,6 +1779,30 @@ export const FIELDS: FieldSpec[] = [ max: 365, hint: "The oldest go when a new one is taken. Backups you take yourself stay until you delete them", }, + // File shares (opt-in) + { + section: "shares", + key: "enabled", + label: "Share the file tree", + kind: "switch", + hint: "Off: nothing listens. The API and the MCP tools read the tree either way", + }, + { + section: "shares", + key: "s3", + label: "S3", + kind: "switch", + hint: "Read-only; each person signs in with their own access key", + }, + { section: "shares", key: "s3_port", label: "S3 port", kind: "int", min: 1, max: 65535 }, + { + section: "shares", + key: "webdav", + label: "WebDAV", + kind: "switch", + hint: "Read-only; mount it as a drive in Finder, Windows Explorer or a file manager", + }, + { section: "shares", key: "webdav_port", label: "WebDAV port", kind: "int", min: 1, max: 65535 }, // Sensors (opt-in) { section: "sensors", diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx index 2d308770f..2f9cd072a 100644 --- a/nextjs-frontend/components/settings/settings-shell.tsx +++ b/nextjs-frontend/components/settings/settings-shell.tsx @@ -19,6 +19,7 @@ import { HardDrive, FileLock, FileType, + FolderTree, Fingerprint, Globe, ScanText, @@ -83,6 +84,7 @@ const ICON: Record = { telemetry: Activity, fedora: Archive, sensors: RadioTower, + shares: FolderTree, uploads: Upload, storage: Database, backups: DatabaseBackup, diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 0504d1752..3bc68ad02 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -24314,6 +24314,137 @@ } } }, + "/api/v1/shares/s3": { + "get": { + "tags": [ + "shares" + ], + "summary": "S3 Status", + "description": "Whether the S3 server is on and on which port, what the worker running it says, the same for WebDAV, and your\naccess keys.", + "operationId": "s3_status", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/S3Status" + } + } + } + } + } + } + }, + "/api/v1/shares/s3/keys": { + "get": { + "tags": [ + "shares" + ], + "summary": "All S3 Keys", + "description": "Everyone's access keys, with their owners (admins).", + "operationId": "all_s3_keys", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/S3Key" + }, + "type": "array", + "title": "Response Shares-All S3 Keys" + } + } + } + } + } + }, + "post": { + "tags": [ + "shares" + ], + "summary": "Create S3 Key", + "description": "An access key that reads the file tree as you, over S3 and WebDAV. Only while signed in to the web app (with your passkey),\nnot with an API token. The secret is in this answer only.", + "operationId": "create_s3_key", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/S3KeyCreate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/S3KeyCreated" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/shares/s3/keys/{key_id}": { + "delete": { + "tags": [ + "shares" + ], + "summary": "Revoke S3 Key", + "description": "Revoke one of your keys (admins: anyone's). It stops working at once.", + "operationId": "revoke_s3_key", + "parameters": [ + { + "name": "key_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Key Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ok" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/import-hooks": { "get": { "tags": [ @@ -48877,6 +49008,248 @@ "type": "object", "title": "RoutineUpdate" }, + "S3Key": { + "properties": { + "id": { + "type": "string", + "title": "Id", + "description": "the access key id" + }, + "name": { + "type": "string", + "title": "Name" + }, + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "expires_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "last_used_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Used At" + }, + "email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Email", + "description": "its owner (the list of everyone's keys)" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "name" + ], + "title": "S3Key" + }, + "S3KeyCreate": { + "properties": { + "name": { + "type": "string", + "maxLength": 80, + "title": "Name", + "default": "S3 key" + }, + "days": { + "anyOf": [ + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } + ], + "title": "Days", + "description": "how long it lasts (the API keys' limits); none: their default" + } + }, + "additionalProperties": false, + "type": "object", + "title": "S3KeyCreate" + }, + "S3KeyCreated": { + "properties": { + "id": { + "type": "string", + "title": "Id", + "description": "the access key id" + }, + "secret": { + "type": "string", + "title": "Secret", + "description": "the secret access key, shown this once" + }, + "port": { + "type": "integer", + "title": "Port", + "description": "the S3 server's port" + }, + "webdav_port": { + "type": "integer", + "title": "Webdav Port" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "secret", + "port", + "webdav_port" + ], + "title": "S3KeyCreated" + }, + "S3Service": { + "properties": { + "process": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Process" + }, + "at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "At" + }, + "running": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Running" + }, + "port": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Port" + }, + "since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Since" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "additionalProperties": true, + "type": "object", + "title": "S3Service" + }, + "S3Status": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "whether an admin turned it on (shares.enabled and its own switch)" + }, + "port": { + "type": "integer", + "title": "Port", + "description": "the port it listens on in the worker (Docker publishes LENS_S3_PORT, LENS_WEBDAV_PORT)" + }, + "service": { + "anyOf": [ + { + "$ref": "#/components/schemas/S3Service" + }, + { + "type": "null" + } + ], + "description": "what the process running it says; none while it's off" + }, + "webdav": { + "$ref": "#/components/schemas/ShareServer" + }, + "keys": { + "items": { + "$ref": "#/components/schemas/S3Key" + }, + "type": "array", + "title": "Keys", + "description": "your access keys" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "port", + "webdav", + "keys" + ], + "title": "S3Status", + "description": "The S3 server (these fields), the WebDAV server, and your access keys (they sign in to both)." + }, "SavedSearch": { "properties": { "id": { @@ -51297,6 +51670,38 @@ ], "title": "ShareLink" }, + "ShareServer": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "whether an admin turned it on (shares.enabled and its own switch)" + }, + "port": { + "type": "integer", + "title": "Port", + "description": "the port it listens on in the worker (Docker publishes LENS_S3_PORT, LENS_WEBDAV_PORT)" + }, + "service": { + "anyOf": [ + { + "$ref": "#/components/schemas/S3Service" + }, + { + "type": "null" + } + ], + "description": "what the process running it says; none while it's off" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "port" + ], + "title": "ShareServer" + }, "ShareSite": { "properties": { "origin": { diff --git a/threat-model.json b/threat-model.json index 5a165bd1c..b811b994b 100644 --- a/threat-model.json +++ b/threat-model.json @@ -998,6 +998,14 @@ "provenance": "documented", "source": "docs/sensors.md \"Sensors are off until an admin turns them on\"" }, + { + "property": "P-SHARES", + "violation_symptom": "info-leak, integrity-bypass", + "severity_tier": "security", + "conditions": "the file shares (S3, WebDAV) are off until an admin enables them; every request carries a person's own access key (SigV4 or Basic), reads as them and never writes", + "provenance": "documented", + "source": "docs/file-tree.md \"Off until an admin turns it on\"" + }, { "property": "P-CHILD-SECRETS", "violation_symptom": "info-leak", diff --git a/threat-model.md b/threat-model.md index 5a1bc3ece..b616e8a43 100644 --- a/threat-model.md +++ b/threat-model.md @@ -105,7 +105,7 @@ optional listeners (MQTT, syslog, tunnel, local model server) run inside workers | Ingest & processing (Ingest) | `routes/{uploads,files,imports}.py`, `domain/{ingest,documents,convert,video,uploads}.py`, `templates.py`, `render.py` | child processes (ffmpeg, LibreOffice, Chromium, poppler, Tesseract), files | In | | Query engines (Query) | `domain/cypher.py`, `domain/rdf.py` (SPARQL), `domain/rdf_import.py`, `domain/search.py` | DB | In | | Assistant & extensions (Assistant) | `domain/{chat,ai_tools,ops_tools,extensions,tool_nodes,flow,custom_nodes,code_tools}.py`, MCP write tools | network (model servers), child process (code tools) | In | -| Optional services, off by default (Optional) | `domain/{sensors,mqtt,syslog}.py`, `tunnel.py`, `local_llm.py`, `components.py`, `telemetry.py` | listening sockets, downloads, child processes | In | +| Optional services, off by default (Optional) | `domain/{sensors,mqtt,syslog,shares,s3,webdav}.py`, `tunnel.py`, `local_llm.py`, `components.py`, `telemetry.py` | listening sockets, downloads, child processes | In | | Packaging variants | `docker-compose.prod.yml`, `install.sh`, `cloudron/`, `packaging/{synology,qnap}`, `proxmox/` | host | §1.6 | | Dev and CI tooling, samples | `docker-compose.yml`, `make dev`, `.github/`, `ci/`, `examples/`, `overrides/`, `prod-frontend-deploy.yml`, `watcher.py`, `commands/`, `fastapi_backend/tests/` | host | Out (§1.3) | @@ -208,7 +208,7 @@ notification and telemetry targets; telemetry is off by default and there is no | Optional ML extras | some download weights on first use on their own (Hugging Face, ModelScope) | *(inferred, Q19)* | | Env proxies | plain `urllib` callers honour `HTTP(S)_PROXY`; the guarded openers ignore it | *(assumption, Q19)* | | Child processes | rclone, ffmpeg/ffprobe, LibreOffice, Chromium, Tesseract, poppler tools, antiword/catdoc, cloudflared, llama-server, `unshare` + Python for code tools, uv/pip; all as argument lists | *(assumption, Q20)* | -| Listening sockets | API and web app; a `netguard` proxy on loopback during captures; MQTT 1883 and syslog 5514 on `sensors.bind` (default `0.0.0.0`) when sensors are on; llama-server on its port (0.0.0.0 in containers, with its own API key) | *(assumption, Q19)* | +| Listening sockets | API and web app; a `netguard` proxy on loopback during captures; MQTT 1883 and syslog 5514 on `sensors.bind` (default `0.0.0.0`) when sensors are on; the file shares' S3 server on 9555 and WebDAV server on 9556 (`shares.bind`) when file shares are on; llama-server on its port (0.0.0.0 in containers, with its own API key) | *(assumption, Q19)* | | Logs | the setup code and a `/setup?code=` link at WARNING; sign-in and reset links when mail is not set | *(documented, docs/authentication.md "the API prints in its log")* | | Files | everything under `data_dir`, including `secret.key` (mode 600) when `ARCHIVE_SECRET_KEY` is unset; temp dirs for imports, rclone config, LibreOffice and Chromium profiles | *(inferred, Q19)* | | Process-wide state | `rdflib` `SPARQL_LOAD_GRAPHS = False` on each SPARQL call; `sys.path` gains `data_dir/python` (components); a signal handler in the CLI | *(assumption, Q19)* | @@ -239,6 +239,7 @@ machine requires HTTPS, and a changed DB credential when SurrealDB runs as its o | `documents.web_networks`, `LENS_WEB_NETWORKS`; `notifications.networks` | empty | widen the outbound guard to listed private networks | Yes (operator and admin input) | *(documented, docs/configuration.md "loopback and cloud metadata addresses stay out")* | | Chromium sandbox | off as root or without user namespaces | the `netguard` proxy is the only barrier | Yes | *(documented, docs/configuration.md "runs with its sandbox where it can")* | | `sensors.enabled`, `sensors.mqtt_anonymous` | off (`bind` `0.0.0.0`); off | opens MQTT and syslog listeners; anonymous MQTT publish | Yes; anonymous is the admin's choice | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | +| `shares.enabled` | off (`bind` `0.0.0.0`) | opens the read-only S3 and WebDAV servers for the asset tree | Yes | *(documented, docs/file-tree.md "Off until an admin turns it on")* | | `tunnel.mode`, `local_llm.enabled`, `telemetry` | off | outbound tunnel (Cloudflare terminates TLS, Q30); local model server; OTLP export | Yes | *(documented, docs/remote-access.md "It's off until an admin turns it on")* | | `components.auto` | on | unpinned-hash pip installs at runtime | Yes | *(maintainer, 2026-10)* | | `encryption.files` | on for new archives | file encryption at rest | Yes | *(documented, docs/encryption.md "A new archive turns this on at its first start")* | @@ -282,6 +283,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | extensions (prompt, graph and `http` tools, hooks), custom nodes including `template` nodes (Q23) | tool spec, graph JSON, template text | yes, any writer for their own; sharing is gated | callback/code (declarative) | person enables only extensions they trust | *(documented, docs/assistant.md "other people's run only when an admin shared them")* | | Python code tools, shipped and admin-created templates, workflows, pipelines, routines | Python, Jinja source, regexes | admin only | callback/code | admin is trusted | *(maintainer, 2026-10)* | | MQTT 1883, syslog 5514, `POST /sensors/push/{token}/{stream}` | payloads, hub logins, token in the path | yes, once enabled | data, size/rate, x-credential | admin enables; operator firewalls | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | +| S3 9555 and WebDAV 9556 (file shares) | signed S3 requests (SigV4 header or presigned link); WebDAV with an access key as Basic sign-in | yes, once enabled | x-credential, resource-name | admin enables; each request carries a person's own access key; read-only; failed WebDAV sign-ins throttled per address | *(documented, docs/file-tree.md "Off until an admin turns it on")* | | `.env`, `archive.yaml`, `LENS_*`, `ARCHIVE_SECRET_KEY`, `RCLONE_BINARY` | process configuration | no (operator) | x-env-config | keep unreadable to others | *(maintainer, 2026-10)* | | workers and DB peers | DB credential, shared database rows | no (trusted peer) | x-peer | run peers on trusted hosts | *(inferred, Q9)* | @@ -499,6 +501,7 @@ Paths in the Voided by column are relative to `fastapi_backend/app/`. Searches c | P-TELEMETRY | Telemetry is off unless turned on, has no built-in destination, and never sends content, names, addresses or error messages. | info-leak | security-critical | *(documented, docs/telemetry.md "Never sent")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-MODEL-CHECKSUM | Catalog GGUF models are pinned to a revision and checked against their SHA-256. | integrity-bypass | security-critical | *(documented, docs/local-models.md "checked against their SHA-256")* | Voided by: `hf:` references have no pin or hash — `domain/local_llm.py:138-139`. | | P-SENSORS | Sensors are off until an admin enables them. MQTT needs a hub login unless `mqtt_anonymous` is on. Syslog hears only listed networks. | integrity-bypass | security-critical | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | Voided by: `sensors.mqtt_anonymous` — `domain/sensors.py:984`; `sensors.syslog_networks`, `sensors.bind`. | +| P-SHARES | The file shares (S3 and WebDAV) are off until an admin enables them. Every request is signed (S3: AWS Signature Version 4) or signed in (WebDAV: Basic) with an access key someone made while signed in to the web app; it reads as them, sees only what the web app shows them, and never writes. Keys expire as API keys do, stop with a disabled account, and are revoked at once. | info-leak, integrity-bypass | security-critical | *(documented, docs/file-tree.md "Off until an admin turns it on")* | Voided by: `shares.enabled` with `LENS_BIND=0.0.0.0` sends files, S3 signatures and WebDAV's Basic secret in clear over plain HTTP on the network; key secrets are sealed with the server key (`secret.key`), not hashed, since SigV4 signs with the secret itself — `domain/s3.py`, `domain/webdav.py`. | | P-CHILD-SECRETS | The tunnel token reaches cloudflared in its environment, not its command line; llama-server gets its own random API key. | info-leak | correctness-only | *(documented, docs/remote-access.md "in its environment, not on its command line")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-REMOTE-NAMES | An IMAP message is named by mailbox validity and uid, so a rebuilt mailbox never serves another message under an old name. | wrong-output | correctness-only | *(documented, docs/processing.md "an old one never reads another message")* | Voided by: none identified (not required for this symptom; no exhaustive search) | | P-NOTIFY | Each notification event is claimed once across processes and retried with backoff. | wrong-output | correctness-only | *(documented, docs/notifications.md "claimed once")* | Voided by: none identified (not required for this symptom; no exhaustive search) | diff --git a/threat-model.yaml b/threat-model.yaml index a80ae5597..9d311dde3 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:42d9bc92f48ce93ea2d58412a61b086dde8ab4522af95f628304d1b33919832f" +prose_version: "threat-model.md@sha256:cb427fc9050a27088e6cb9cc1e01350537c4f56ddc95ea1d022bb49fec675661" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} @@ -1024,6 +1024,14 @@ build_flags: affects_properties: - {property_id: P-SENSORS, effect: narrows} provenance: {kind: documented, source: 'docs/sensors.md "Sensors are off until an admin turns them on"'} + - name: "shares.enabled" + default: "off" + security_relevant: true + maintainer_stance: supported + x-scope: "opens the read-only S3 and WebDAV servers for the asset tree on shares.s3_port and shares.webdav_port" + affects_properties: + - {property_id: P-SHARES, effect: narrows} + provenance: {kind: documented, source: 'docs/file-tree.md "Off until an admin turns it on"'} - name: "tunnel.mode, local_llm.enabled, telemetry" default: "off" security_relevant: true @@ -1384,6 +1392,14 @@ properties_claimed: violation_symptoms: [integrity-bypass] x-voided-by: "sensors.mqtt_anonymous (domain/sensors.py:984); sensors.syslog_networks, sensors.bind" provenance: {kind: documented, source: 'docs/sensors.md "Sensors are off until an admin turns them on"'} + - id: P-SHARES + kind: authentication + components: [optional] + tier: security-critical + conditions: "the file shares (S3, WebDAV) are off until an admin enables them; every request carries a person's own access key (SigV4 or Basic), reads as them and never writes" + violation_symptoms: [info-leak, integrity-bypass] + x-voided-by: "plain HTTP on the network with LENS_BIND=0.0.0.0 (WebDAV's Basic secret in clear); secrets sealed with the server key, not hashed (domain/s3.py)" + provenance: {kind: documented, source: 'docs/file-tree.md "Off until an admin turns it on"'} - id: P-CHILD-SECRETS kind: confidentiality components: [optional] From 07df46ffb181c669cf12033f7fecdbc2258b4452 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:42:51 +0000 Subject: [PATCH 2/5] feat(shares): key lookup and service keys for SMB and NFS Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- fastapi_backend/app/domain/s3.py | 55 +++++++++++++++++++++++++++- fastapi_backend/tests/api/test_s3.py | 13 +++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/fastapi_backend/app/domain/s3.py b/fastapi_backend/app/domain/s3.py index d4c3ab4b9..468dd8c7c 100644 --- a/fastapi_backend/app/domain/s3.py +++ b/fastapi_backend/app/domain/s3.py @@ -91,8 +91,9 @@ def revoke_key(db, kid, account=None): def key_account(db, cfg, kid): - """(secret, active account) for a key id, or None.""" - if not KEY_RX.match(kid or ""): + """(secret, active account) for a key id (any case: SMB lowercases user names), or None.""" + kid = str(kid or "").upper() + if not KEY_RX.match(kid): return None row = db.one("SELECT account, sealed, expires_at FROM $r", r=R("file_key", kid)) if not row or (row.get("expires_at") and row["expires_at"] < store.now()): @@ -107,6 +108,56 @@ def key_account(db, cfg, kid): return secret, acct +def lookup_key(db, cfg, kid): + """{account: id, secret} for a live key id (any case), or None: for servers that check the secret themselves + (SMB's NTLM).""" + found = key_account(db, cfg, kid) + return {"account": found[1]["id"], "secret": found[0]} if found else None + + +def touch(db, kid): + """Note that a key was just used.""" + kid = str(kid or "").upper() + if KEY_RX.match(kid): + db.q("UPDATE $r SET last_used_at = $t", r=R("file_key", kid), t=store.now()) + + +def service_key(db, cfg, account, label): + """(in-network S3 endpoint, key id, secret) for a service acting as an account (NFS: label "nfs"), made once per + (account, label) and reused after; it doesn't expire. Revoke it with revoke_service_key.""" + from . import shares + + port = int(shares.settings_of(cfg)["s3_port"]) + endpoint = f"http://worker:{port}" + for row in db.rows("SELECT record::id(id) AS id FROM file_key WHERE account = $a AND service = $l", a=int(account), l=str(label)): + found = key_account(db, cfg, row["id"]) + if found: + return endpoint, row["id"], found[0] + db.q("DELETE $r", r=R("file_key", row["id"])) + kid = "LK" + "".join(secrets.choice(ALPHABET) for _ in range(18)) + secret = secrets.token_urlsafe(30) + db.q( + "CREATE $r CONTENT $d", + r=R("file_key", kid), + d={ + "account": int(account), + "name": f"{label} (service)"[:NAME_MAX], + "service": str(label), + "sealed": settings.seal(cfg, secret, _context(kid)), + "created_at": store.now(), + }, + ) + return endpoint, kid, secret + + +def revoke_service_key(db, account, label): + """Revoke a service's key for an account; whether there was one.""" + rows = db.rows("SELECT record::id(id) AS id FROM file_key WHERE account = $a AND service = $l", a=int(account), l=str(label)) + for row in rows: + db.q("DELETE $r", r=R("file_key", row["id"])) + return bool(rows) + + # ---------- Signature Version 4 ---------- def _enc(s, safe="-_.~"): return urllib.parse.quote(s, safe=safe) diff --git a/fastapi_backend/tests/api/test_s3.py b/fastapi_backend/tests/api/test_s3.py index 364d0a413..f6990529d 100644 --- a/fastapi_backend/tests/api/test_s3.py +++ b/fastapi_backend/tests/api/test_s3.py @@ -124,6 +124,19 @@ def test_keys_are_made_while_signed_in(client, db, env): assert client.get("/api/v1/shares/s3/keys", headers=env["hv"]).status_code == 403 # everyone's: admins +def test_other_servers_use_the_same_keys(client, db, cfg, env): + """SMB checks a key's secret itself (any case: NTLM lowercases user names); NFS signs in with a service key.""" + uid = db.one("SELECT record::id(id) AS id FROM account WHERE email = 'vi@x.io'")["id"] + k = _key(client, env["hv"]) + assert s3.lookup_key(db, cfg, k["id"].lower()) == {"account": uid, "secret": k["secret"]} + assert s3.lookup_key(db, cfg, "LKNOTAKEY") is None + endpoint, kid, secret = s3.service_key(db, cfg, uid, "nfs") + assert endpoint == "http://worker:9555" and s3.service_key(db, cfg, uid, "nfs") == (endpoint, kid, secret) + c = _s3(env["port"], kid, secret) + assert [b["Name"] for b in c.list_buckets()["Buckets"]] == ["pods"] + assert s3.revoke_service_key(db, uid, "nfs") and s3.lookup_key(db, cfg, kid) is None + + def test_the_servers_follow_the_settings(db, cfg): conf = {"enabled": False, "s3": True, "s3_port": 0, "webdav": True, "webdav_port": 0, "bind": "127.0.0.1"} svc = shares.Service(db, lambda: {**cfg, "shares": conf}, name="t") From 1b26d1c8fc1edb48f4f71c0418cf5e1f2520a0d2 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:43:56 +0000 Subject: [PATCH 3/5] feat(shares): an in-process S3 endpoint for NFS Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- fastapi_backend/app/domain/s3.py | 17 +++++++++++++++++ fastapi_backend/tests/api/test_s3.py | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/fastapi_backend/app/domain/s3.py b/fastapi_backend/app/domain/s3.py index 468dd8c7c..e024cc9f0 100644 --- a/fastapi_backend/app/domain/s3.py +++ b/fastapi_backend/app/domain/s3.py @@ -21,6 +21,7 @@ import http.server import re import secrets +import threading import urllib.parse from typing import Any from xml.sax.saxutils import escape @@ -150,6 +151,22 @@ def service_key(db, cfg, account, label): return endpoint, kid, secret +_internal: dict[str, Any] = {} +_internal_lock = threading.Lock() + + +def internal_endpoint(db, cfg_fn): + """The URL of an S3 server for this process alone (loopback, a free port), started on first use whatever the + share settings say: for servers in the same process that read the tree through S3 (NFS). `cfg_fn` gives the + current settings.""" + from . import shares + + with _internal_lock: + if "server" not in _internal: + _internal["server"] = shares.Server("127.0.0.1", 0, db, cfg_fn).start() + return f"http://127.0.0.1:{_internal['server'].port}" + + def revoke_service_key(db, account, label): """Revoke a service's key for an account; whether there was one.""" rows = db.rows("SELECT record::id(id) AS id FROM file_key WHERE account = $a AND service = $l", a=int(account), l=str(label)) diff --git a/fastapi_backend/tests/api/test_s3.py b/fastapi_backend/tests/api/test_s3.py index f6990529d..cd9971c91 100644 --- a/fastapi_backend/tests/api/test_s3.py +++ b/fastapi_backend/tests/api/test_s3.py @@ -134,6 +134,10 @@ def test_other_servers_use_the_same_keys(client, db, cfg, env): assert endpoint == "http://worker:9555" and s3.service_key(db, cfg, uid, "nfs") == (endpoint, kid, secret) c = _s3(env["port"], kid, secret) assert [b["Name"] for b in c.list_buckets()["Buckets"]] == ["pods"] + inside = s3.internal_endpoint(db, lambda: cfg) + assert inside.startswith("http://127.0.0.1:") and s3.internal_endpoint(db, lambda: cfg) == inside + c = _s3(int(inside.rsplit(":", 1)[1]), kid, secret) + assert [b["Name"] for b in c.list_buckets()["Buckets"]] == ["pods"] assert s3.revoke_service_key(db, uid, "nfs") and s3.lookup_key(db, cfg, kid) is None From 4df656ce751e89fbca5e4808ec78d98e5ad36279 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 21:58:12 +0000 Subject: [PATCH 4/5] test(shares): S3 and WebDAV over the graph layout Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- docker-compose.prod.yml | 2 +- docker-compose.yml | 2 +- docs/file-tree.md | 7 ++--- fastapi_backend/app/domain/shares.py | 2 +- fastapi_backend/app/domain/store.py | 2 +- fastapi_backend/tests/api/test_s3.py | 33 ++++++++++++++---------- fastapi_backend/tests/api/test_webdav.py | 18 ++++++------- 7 files changed, 37 insertions(+), 29 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index adf4e82c8..89ed05a8b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -103,7 +103,7 @@ services: - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_MQTT_PORT:-1883}:1883" - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_SYSLOG_PORT:-5514}:5514/udp" - "${LENS_SENSOR_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_SYSLOG_PORT:-5514}:5514/tcp" - # the asset tree over S3 and WebDAV (docs/file-tree.md): read-only, listened on only once File shares are turned + # the graph as files over S3 and WebDAV (docs/file-tree.md): read-only, listened on only once File shares are turned # on in the app; published where the web app is. Something else using 9555 or 9556? Set LENS_S3_PORT / LENS_WEBDAV_PORT. - "${LENS_SHARE_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_S3_PORT:-9555}:9555" - "${LENS_SHARE_BIND:-${LENS_BIND:-127.0.0.1}}:${LENS_WEBDAV_PORT:-9556}:9556" diff --git a/docker-compose.yml b/docker-compose.yml index 5744f1f57..9d0420df6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,7 +87,7 @@ services: - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_MQTT_PORT:-1883}:1883" - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_SYSLOG_PORT:-5514}:5514/udp" - "${LENS_SENSOR_BIND:-127.0.0.1}:${LENS_SYSLOG_PORT:-5514}:5514/tcp" - # the asset tree over S3 and WebDAV (docs/file-tree.md), once File shares are on; LENS_S3_PORT / LENS_WEBDAV_PORT + # the graph as files over S3 and WebDAV (docs/file-tree.md), once File shares are on; LENS_S3_PORT / LENS_WEBDAV_PORT - "${LENS_SHARE_BIND:-127.0.0.1}:${LENS_S3_PORT:-9555}:9555" - "${LENS_SHARE_BIND:-127.0.0.1}:${LENS_WEBDAV_PORT:-9556}:9556" healthcheck: diff --git a/docs/file-tree.md b/docs/file-tree.md index 985b850cd..392faaacd 100644 --- a/docs/file-tree.md +++ b/docs/file-tree.md @@ -2,7 +2,8 @@ Lens shows its graph as an ordinary tree of folders and files: graph nodes (collections, topics, entities) are folders, nested along the graph's links, and every asset (a recording, document or image) is a few files inside the -folders it belongs to, its metadata in a sidecar beside it. Agents and tools that already understand files can read Lens without learning its API. +folders it belongs to, its metadata in a sidecar beside it. Agents and tools that already understand files can read +Lens without learning its API. | Where | Status | | --- | --- | @@ -82,8 +83,8 @@ endpoint = http://lens.local:9555 force_path_style = true ``` -then `rclone lsd lens:`, `rclone copy lens:pods/General ./general`, or `rclone mount lens: ~/lens`. Supported: -listing buckets, listing objects (v1 and v2, by folder or recursively, a page at a time), reading objects whole or +then `rclone lsd lens:`, `rclone copy "lens:pods/Topics/Gene therapy" ./gene-therapy`, or `rclone mount lens: ~/lens`. +Supported: listing buckets, listing objects (v1 and v2, by folder or recursively, a page at a time), reading objects whole or in ranges, and presigned links. Anything that writes answers AccessDenied. Requests must be signed with Signature Version 4. diff --git a/fastapi_backend/app/domain/shares.py b/fastapi_backend/app/domain/shares.py index 7f11de1c7..64fc2a314 100644 --- a/fastapi_backend/app/domain/shares.py +++ b/fastapi_backend/app/domain/shares.py @@ -1,4 +1,4 @@ -"""File shares (docs/file-tree.md): the servers that share the asset tree (filetree.py) on the network, read-only, with +"""File shares (docs/file-tree.md): the servers that share the graph as files (filetree.py) on the network, read-only, with people's own access keys (s3.py): S3 (s3.py) and WebDAV (webdav.py). Off until an admin turns them on (`shares.enabled`, then `s3` and `webdav`); the process that runs routines (`lens diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 8324adc02..011e8eeb7 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -455,7 +455,7 @@ def labels(): "max_per_minute": 600, "triage": False, }, - # the asset tree over S3 and WebDAV (shares.py, docs/file-tree.md): off until an admin turns it on. Then the process + # the graph as files over S3 and WebDAV (shares.py, docs/file-tree.md): off until an admin turns it on. Then the process # that runs routines (`lens worker`) serves it, read-only, on these ports. bind is a startup setting only. "shares": {"enabled": False, "bind": "0.0.0.0", "s3": True, "s3_port": 9555, "webdav": True, "webdav_port": 9556}, # notifications to webhooks and Matterbridge (docs/notifications.md): targets reach public addresses only, and the diff --git a/fastapi_backend/tests/api/test_s3.py b/fastapi_backend/tests/api/test_s3.py index cd9971c91..14c757a4d 100644 --- a/fastapi_backend/tests/api/test_s3.py +++ b/fastapi_backend/tests/api/test_s3.py @@ -51,10 +51,16 @@ def test_people_read_their_namespaces_over_s3(client, env): c = _s3(env["port"], k["id"], k["secret"]) assert [b["Name"] for b in c.list_buckets()["Buckets"]] == ["pods"] top = c.list_objects_v2(Bucket="pods", Delimiter="/") - assert [p["Prefix"] for p in top["CommonPrefixes"]] == ["General/", "Talks/"] and "Contents" not in top - talks = c.list_objects_v2(Bucket="pods", Prefix="Talks/", Delimiter="/") - assert [o["Key"] for o in talks["Contents"]] == ["Talks/ep2 [2].json", "Talks/ep2 [2].original.txt", "Talks/ep2 [2].txt"] - assert [o["Key"] for o in c.list_objects_v2(Bucket="pods", Prefix="Talks/ep2 [2].t")["Contents"]] == ["Talks/ep2 [2].txt"] + assert [p["Prefix"] for p in top["CommonPrefixes"]] == ["Collections/", "Entities/", "Topics/"] and "Contents" not in top + talks = c.list_objects_v2(Bucket="pods", Prefix="Collections/Talks/", Delimiter="/") + assert [o["Key"] for o in talks["Contents"]] == [ + "Collections/Talks/ep2 [2].json", + "Collections/Talks/ep2 [2].original.txt", + "Collections/Talks/ep2 [2].txt", + ] + assert [o["Key"] for o in c.list_objects_v2(Bucket="pods", Prefix="Collections/Talks/ep2 [2].t")["Contents"]] == [ + "Collections/Talks/ep2 [2].txt" + ] # v1 listing, recursive, a page at a time keys, marker = [], "" while True: @@ -63,22 +69,23 @@ def test_people_read_their_namespaces_over_s3(client, env): if not page["IsTruncated"]: break marker = keys[-1] - assert keys == sorted(keys, key=str.encode) and len(keys) == 6 and keys[0] == "General/ep1 [1].json" + assert keys == sorted(keys, key=str.encode) and keys[0] == "Collections/General/ep1 [1].json" + assert len([k for k in keys if k.startswith("Collections/")]) == 6 and "Topics/Gene therapy/ep1 [1].txt" in keys # v2 pages first = c.list_objects_v2(Bucket="pods", MaxKeys=4) rest = c.list_objects_v2(Bucket="pods", ContinuationToken=first["NextContinuationToken"]) assert [o["Key"] for o in first["Contents"] + rest["Contents"]] == keys # objects, whole and in part, as listed own = (env["folder"] / "ep1.txt").read_bytes() - got = c.get_object(Bucket="pods", Key="General/ep1 [1].original.txt") + got = c.get_object(Bucket="pods", Key="Collections/General/ep1 [1].original.txt") assert got["Body"].read() == own and got["ContentLength"] == len(own) and len(got["ETag"].strip('"')) != 32 - part = c.get_object(Bucket="pods", Key="General/ep1 [1].original.txt", Range="bytes=3-9") + part = c.get_object(Bucket="pods", Key="Collections/General/ep1 [1].original.txt", Range="bytes=3-9") assert part["Body"].read() == own[3:10] and part["ContentRange"] == f"bytes 3-9/{len(own)}" - head = c.head_object(Bucket="pods", Key="General/ep1 [1].txt") - assert head["ContentLength"] == next(o["Size"] for o in keys_of(c, "General/") if o["Key"].endswith("ep1 [1].txt")) - assert c.get_object(Bucket="pods", Key="General/ep1 [1].txt")["Body"].read().startswith(b"Alice: Welcome back.") + head = c.head_object(Bucket="pods", Key="Collections/General/ep1 [1].txt") + assert head["ContentLength"] == next(o["Size"] for o in keys_of(c, "Collections/General/") if o["Key"].endswith("ep1 [1].txt")) + assert c.get_object(Bucket="pods", Key="Collections/General/ep1 [1].txt")["Body"].read().startswith(b"Alice: Welcome back.") # a presigned link works on its own - url = c.generate_presigned_url("get_object", Params={"Bucket": "pods", "Key": "General/ep1 [1].json"}, ExpiresIn=60) + url = c.generate_presigned_url("get_object", Params={"Bucket": "pods", "Key": "Collections/General/ep1 [1].json"}, ExpiresIn=60) assert b'"namespace": "pods"' in urllib.request.urlopen(url).read() assert c.get_bucket_location(Bucket="pods")["LocationConstraint"] is None @@ -97,9 +104,9 @@ def test_what_s3_refuses(client, env): k = _key(client, env["hv"]) c = _s3(env["port"], k["id"], k["secret"]) assert _code(lambda: c.list_objects_v2(Bucket="calls")) == "NoSuchBucket" # not theirs - assert _code(lambda: c.get_object(Bucket="pods", Key="General/nope.txt")) == "NoSuchKey" + assert _code(lambda: c.get_object(Bucket="pods", Key="Collections/General/nope.txt")) == "NoSuchKey" assert _code(lambda: c.put_object(Bucket="pods", Key="x.txt", Body=b"hi")) == "AccessDenied" - assert _code(lambda: c.delete_object(Bucket="pods", Key="General/ep1 [1].json")) == "AccessDenied" + assert _code(lambda: c.delete_object(Bucket="pods", Key="Collections/General/ep1 [1].json")) == "AccessDenied" assert _code(lambda: _s3(env["port"], k["id"], "wrong").list_buckets()) == "SignatureDoesNotMatch" assert _code(lambda: _s3(env["port"], "LKAAAAAAAAAAAAAAAAAA", "x").list_buckets()) == "InvalidAccessKeyId" # revoked: it stops working at once diff --git a/fastapi_backend/tests/api/test_webdav.py b/fastapi_backend/tests/api/test_webdav.py index 5415c12a6..45678effb 100644 --- a/fastapi_backend/tests/api/test_webdav.py +++ b/fastapi_backend/tests/api/test_webdav.py @@ -50,18 +50,18 @@ def test_mount_and_read(env): assert status == 200 and h["DAV"] == "1" and "PROPFIND" in h["Allow"] status, _, body = _call(env, "PROPFIND", "/", {"Depth": "1"}) assert status == 207 and [(x[0], x[1]) for x in _listing(body)] == [("/", True), ("/pods/", True)] - status, _, body = _call(env, "PROPFIND", "/pods/General/", {"Depth": "1"}) + status, _, body = _call(env, "PROPFIND", "/pods/Collections/General/", {"Depth": "1"}) hrefs = [x[0] for x in _listing(body)] - assert hrefs[0] == "/pods/General/" and "/pods/General/ep1%20%5B1%5D.txt" in hrefs + assert hrefs[0] == "/pods/Collections/General/" and "/pods/Collections/General/ep1%20%5B1%5D.txt" in hrefs size = next(int(x[2]) for x in _listing(body) if x[0].endswith("ep1%20%5B1%5D.txt")) - status, _, body = _call(env, "PROPFIND", "/pods/General/ep1%20%5B1%5D.txt", {"Depth": "0"}) + status, _, body = _call(env, "PROPFIND", "/pods/Collections/General/ep1%20%5B1%5D.txt", {"Depth": "0"}) assert status == 207 and len(_listing(body)) == 1 - status, h, body = _call(env, "GET", "/pods/General/ep1%20%5B1%5D.txt") + status, h, body = _call(env, "GET", "/pods/Collections/General/ep1%20%5B1%5D.txt") assert status == 200 and len(body) == size and body.startswith(b"Alice: Welcome back.") own = (env["folder"] / "ep1.txt").read_bytes() - status, h, body = _call(env, "GET", "/pods/General/ep1%20%5B1%5D.original.txt", {"Range": "bytes=2-5"}) + status, h, body = _call(env, "GET", "/pods/Collections/General/ep1%20%5B1%5D.original.txt", {"Range": "bytes=2-5"}) assert status == 206 and body == own[2:6] and h["Content-Range"] == f"bytes 2-5/{len(own)}" - status, h, body = _call(env, "HEAD", "/pods/General/ep1%20%5B1%5D.txt") + status, h, body = _call(env, "HEAD", "/pods/Collections/General/ep1%20%5B1%5D.txt") assert status == 200 and h["Content-Length"] == str(size) and body == b"" @@ -69,9 +69,9 @@ def test_what_webdav_refuses(env): assert _call(env, "PROPFIND", "/", {"Depth": "1"}, auth_header=False)[0] == 401 assert _call(env, "PROPFIND", "/calls/", {"Depth": "1"})[0] == 404 # not theirs assert _call(env, "PROPFIND", "/", {"Depth": "infinity"})[0] == 403 - assert _call(env, "GET", "/pods/General/nope.txt")[0] == 404 - assert _call(env, "PUT", "/pods/General/x.txt", body=b"hi")[0] == 403 - assert _call(env, "DELETE", "/pods/General/ep1%20%5B1%5D.txt")[0] == 403 + assert _call(env, "GET", "/pods/Collections/General/nope.txt")[0] == 404 + assert _call(env, "PUT", "/pods/Collections/General/x.txt", body=b"hi")[0] == 403 + assert _call(env, "DELETE", "/pods/Collections/General/ep1%20%5B1%5D.txt")[0] == 403 assert _call(env, "MKCOL", "/pods/New/")[0] == 403 wrong = "Basic " + base64.b64encode(f"{env['k']['id']}:nope".encode()).decode() for _ in range(8): From 28690c1bd454ee2afc9093c178ce78cd566a3a68 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 22:40:14 +0000 Subject: [PATCH 5/5] fix(install): find microsandbox's latest release without GitHub's API limit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017MjvZcE48RUyNdztPNBgid --- install.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index cda44673a..7ff6ecb4e 100755 --- a/install.sh +++ b/install.sh @@ -180,7 +180,17 @@ if [ "$BACKEND" = microsandbox ]; then if ! have msb; then say "Installing microsandbox (install.microsandbox.dev)..." have curl || fail "curl is needed to install microsandbox." - curl -fsSL https://install.microsandbox.dev | sh + msb_installer="$(curl -fsSL https://install.microsandbox.dev)" || fail "Couldn't download the microsandbox installer." + # Its installer asks GitHub's API for the latest release, which refuses busy shared addresses (CI, offices); the + # release page's redirect names it without that limit, so hand the installer the version when we can find it. + msb_repo="$(printf '%s\n' "$msb_installer" | sed -n 's/^GITHUB_REPO="\(.*\)"$/\1/p' | head -1)" + msb_tag="" + [ -n "$msb_repo" ] && msb_tag="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "https://github.com/$msb_repo/releases/latest" 2>/dev/null | sed -n 's#.*/releases/tag/##p')" + if [ -n "$msb_tag" ]; then + printf '%s\n' "$msb_installer" | sed "s|^ get_latest_version\$| VERSION=$msb_tag|" | sh + else + printf '%s\n' "$msb_installer" | sh + fi have msb || fail "microsandbox didn't install. See https://docs.microsandbox.dev, then run this again." fi if [ "$os" = Linux ] && [ ! -e /dev/kvm ]; then