From 83621a0167bd64ef8c145f2eb1697da8b922933a Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:10:40 +0000 Subject: [PATCH] feat(https): serve Lens over HTTPS with certificates from Let's Encrypt or ZeroSSL Lens runs Caddy in the process that has the HTTPS ports: certificates over ACME (HTTP, TLS-ALPN or DNS with any caddy-dns provider), renewed on their own, set up in the https settings section. install.sh LENS_HTTPS=1 publishes 443 and 80. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YB1b1hox6iFRdfWcpoKZwK --- CHANGELOG.md | 7 + docker-compose.https.yml | 18 + docs/configuration.md | 2 + docs/https.md | 111 +++ fastapi_backend/app/api/deps.py | 3 + fastapi_backend/app/api/v1/routes/admin.py | 17 + fastapi_backend/app/cli.py | 18 +- fastapi_backend/app/core/runtime.py | 2 + fastapi_backend/app/domain/https.py | 702 ++++++++++++++++++ fastapi_backend/app/domain/settings.py | 14 + fastapi_backend/app/domain/store.py | 30 +- fastapi_backend/app/email.py | 10 +- fastapi_backend/app/schemas/admin.py | 30 + fastapi_backend/tests/api/test_https.py | 290 ++++++++ install.sh | 34 +- microsandbox/run.sh | 2 + mkdocs.yml | 1 + nextjs-frontend/app/openapi-client/index.ts | 10 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 34 + .../app/openapi-client/types.gen.ts | 156 ++++ nextjs-frontend/openapi.json | 213 ++++++ threat-model.md | 3 +- threat-model.yaml | 10 +- 23 files changed, 1704 insertions(+), 13 deletions(-) create mode 100644 docker-compose.https.yml create mode 100644 docs/https.md create mode 100644 fastapi_backend/app/domain/https.py create mode 100644 fastapi_backend/tests/api/test_https.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e9dfdd09..a66513760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version metadata and a `.txt` of the text; everyone sees what the web app shows them. Browse and read it through the API (`/api/v1/tree`) and the MCP tools `list_files` and `read_file` ([The graph as files](docs/file-tree.md)). S3, WebDAV, SMB and NFS servers that share it are in progress. + +- **HTTPS with automatic certificates.** Lens can serve itself at `https://` your domain with a certificate it gets + and renews on its own from Let's Encrypt or ZeroSSL (or any ACME authority), proving the domain over HTTP, TLS-ALPN + or DNS, with Cloudflare, Route 53 and the other caddy-dns providers, so wildcards and machines the internet can't + reach work too. Run the install line with `LENS_HTTPS=1` to open ports 443 and 80, then set it up with + `PUT /api/v1/settings/https`; the Settings page comes next. Off by default ([HTTPS](docs/https.md)). + - **Budget meter and Costs page.** The top bar shows what the namespace picked spent against its budget, used and left, highlighted when near or over, and follows the namespace switcher; it opens the details for a day, week or month. **Costs** in the left menu shows spend by day, model and resource, and sets the namespace's budget diff --git a/docker-compose.https.yml b/docker-compose.https.yml new file mode 100644 index 000000000..89d8bf190 --- /dev/null +++ b/docker-compose.https.yml @@ -0,0 +1,18 @@ +# HTTPS with certificates Lens gets and renews itself (docs/https.md), set up in Settings › HTTPS. +# install.sh adds this file when LENS_HTTPS=1 is in .env (or the web app asked for it): the worker gets the HTTPS +# and HTTP ports, and runs Caddy there while HTTPS is on in Settings. By hand: +# docker compose -f docker-compose.prod.yml -f docker-compose.https.yml up -d +# Something else on this machine using 443 or 80? Set LENS_HTTPS_PORT / LENS_HTTP_PORT to other ports (with the DNS +# challenge, port 80 isn't needed at all). +services: + backend: + environment: + LENS_HTTPS_PORT: ${LENS_HTTPS_PORT:-443} + worker: + environment: + LENS_HTTPS_LISTEN: "1" + LENS_HTTPS_PORT: ${LENS_HTTPS_PORT:-443} + ports: + # Caddy listens on 8443 and 8080 in the container (it runs as an ordinary user, who can't use ports below 1024) + - "${LENS_HTTPS_BIND:-0.0.0.0}:${LENS_HTTPS_PORT:-443}:8443" + - "${LENS_HTTPS_BIND:-0.0.0.0}:${LENS_HTTP_PORT:-80}:8080" diff --git a/docs/configuration.md b/docs/configuration.md index 550663fb8..b0a4f1640 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -76,6 +76,8 @@ this machine or the Docker host, with their models), `PUT /api/v1/setup/storage` | `AUTH_TRUST_HOST` | `true` behind a proxy or in Docker | | `TRUST_PROXY_HEADERS` | `true` when a reverse proxy in front of the web app sets `X-Forwarded-Host`, `-Proto` and `-For`: they're passed on to the API, which names that address in OAuth discovery and reads the visitor's address for IP groups and throttles. Off, the web app reports the `Host` the browser sent and the address it was reached from | | `LENS_WEB_PROXY_HOSTS` | hosts whose `X-Forwarded-For` the web app passes on with `TRUST_PROXY_HEADERS` off (`backend,worker` in the Docker Compose files, for the Cloudflare tunnel) | +| `LENS_HTTPS`, `LENS_HTTPS_PORT`, `LENS_HTTP_PORT`, `LENS_HTTPS_BIND` | install.sh: `LENS_HTTPS=1` publishes the HTTPS and HTTP ports (443 and 80 by default) for [HTTPS](https.md) | +| `LENS_HTTPS_LISTEN` | `1` in the process that has the HTTPS ports and runs Caddy (the worker in `docker-compose.https.yml`); `LENS_CADDY`: a caddy to use instead of fetching one | ## archive.yaml diff --git a/docs/https.md b/docs/https.md new file mode 100644 index 000000000..1914c06dd --- /dev/null +++ b/docs/https.md @@ -0,0 +1,111 @@ +# HTTPS with automatic certificates + +Lens can serve itself at `https://lens.example.com` with a certificate it gets and renews on its own, from +[Let's Encrypt](https://letsencrypt.org) or [ZeroSSL](https://zerossl.com), with no reverse proxy to set up. It runs +[Caddy](https://caddyserver.com), which answers on the HTTPS port, gets a certificate for each domain over ACME, +renews it about a month before it runs out and passes visitors on to the web app. Passkeys work there, since browsers +allow them on https:// addresses. + +Status: **built** in the API and the installer; the **Settings › HTTPS** page in the web app is in progress (until +then, `PUT /api/v1/settings/https` sets it). Off by default. + +For a machine the internet can't reach at all, [Remote access](remote-access.md) (a Cloudflare tunnel) is the +other way to an https:// address. + +## Turning it on + +1. Point the domain's DNS at this machine (an A or AAAA record), unless you use the DNS challenge. +2. Give Lens the HTTPS ports: run the install line again with `LENS_HTTPS=1`. It publishes ports 443 and 80 to the + worker, where Caddy runs, and remembers it in `.env`, so upgrades keep them. + + ```sh + curl -fsSL https://raw.githubusercontent.com/adeelahmad/lense/main/install.sh | LENS_HTTPS=1 sh + ``` + + Something else on the machine uses 443 or 80? `LENS_HTTPS_PORT` and `LENS_HTTP_PORT` choose others (Lens's + addresses then carry the port, like `https://lens.example.com:8443`). `LENS_HTTPS_BIND` limits them to one + address. Rootless Docker can't publish ports below 1024 unless `net.ipv4.ip_unprivileged_port_start` allows it. +3. In **Settings › HTTPS**: the domain, an email address for the certificate authority (optional; it warns you before + a certificate expires), and turn it on. + +Settings › HTTPS shows whether a process has the ports, the certificates Lens holds and when each runs out, and +Caddy's last lines, so a domain that can't be verified says why. + +## Proving the domain is yours + +| Challenge | What needs to reach this machine | Wildcards | +| --- | --- | --- | +| **HTTP** (default) | Port 80 or 443 from the internet: the certificate authority visits `http:///.well-known/acme-challenge/` or the HTTPS port itself (TLS-ALPN) | No | +| **TLS-ALPN only** | Port 443 | No | +| **DNS** | Nothing: Lens adds a TXT record through your DNS provider's API | Yes, `*.example.com` | + +The DNS challenge suits a machine on a home network or behind a firewall, as long as the address people use reaches +it (on the LAN, say). Pick the provider and give what it asks for: + +| Provider | Fields | +| --- | --- | +| Cloudflare | an API token with Zone › Zone › Read and Zone › DNS › Edit | +| Amazon Route 53 | access key ID and secret of an IAM user allowed to list the hosted zones and change their records; region and hosted zone ID optional | +| DigitalOcean, Duck DNS, Gandi, deSEC, Netlify, Vultr, Linode, IONOS, Bunny | an API token | +| Porkbun, Namecheap, OVHcloud, DNSimple, netcup, Azure, Google Cloud DNS, PowerDNS, RFC 2136 (BIND, Knot) | the fields Settings lists | + +Any other [caddy-dns](https://github.com/caddy-dns) plugin works too: its name as the provider (like `hetzner`), and +its fields as JSON. `dns_package` names a plugin outside caddy-dns (`github.com//`). + +## Certificate authorities + +| `ca` | | +| --- | --- | +| `both` (default) | Let's Encrypt, and ZeroSSL when Let's Encrypt can't issue (an outage, a rate limit) | +| `letsencrypt` | Let's Encrypt only. `staging: true` uses its staging service while you try things out (browsers don't trust those certificates) | +| `zerossl` | ZeroSSL only. Lens gets ZeroSSL's account credentials with the email address; or give the EAB key ID and HMAC key from your ZeroSSL account | +| `custom` | any ACME directory (`ca_url`), like a company's own step-ca; EAB credentials when it needs them | + +## Settings + +All in Settings › HTTPS (the `https` section of `PUT /api/v1/settings/https`): + +| Key | Default | | +| --- | --- | --- | +| `enabled` | `false` | | +| `domains` | `[]` | the names to serve, like `lens.example.com` | +| `email` | `""` | the ACME account's contact | +| `ca`, `ca_url`, `staging` | `both` | see above | +| `eab_key_id`, `eab_hmac_key` | | external account binding (the HMAC key is a secret) | +| `challenge` | `http` | `http`, `tls-alpn` or `dns` | +| `dns_provider`, `dns_package`, `dns_credentials` | | the DNS challenge's provider, plugin and fields (a JSON object; a secret) | +| `dns_resolvers`, `propagation` | | resolvers to check the TXT record with, and how long to wait for it (`2m`) | +| `redirect_http` | `true` | send http:// to https:// | +| `hsts` | `false` | tell browsers to use https:// only, for a year. Turn it on once HTTPS works: turning HTTPS off later leaves browsers unable to reach the http:// address | +| `origin` | | the web app, as the worker reaches it (`LENS_HTTPS_ORIGIN`, else `LENS_TUNNEL_ORIGIN`: `http://frontend:3000` in Docker) | +| `https_port`, `http_port` | `8443`, `8080` | where Caddy listens in the container (install.sh publishes 443 and 80 to them) | + +## How it runs + +- **Caddy** is fetched into `data_dir/bin` the first time it's needed: Caddy's release from GitHub, checked against + its published checksums, or, for a DNS provider, a build with that provider's plugin from Caddy's download service + (`caddyserver.com/api/download`). A `caddy` on `PATH`, or `LENS_CADDY`, is used instead when it has the plugin. +- **Where**: only a process with `LENS_HTTPS_LISTEN=1` runs it. install.sh's `docker-compose.https.yml` gives the + worker the ports and sets it; in a [microsandbox](microsandbox.md) it's the sandbox. Of those, one process holds a + lease and runs Caddy; another takes over within 30 seconds if it stops. +- **What it keeps**: the certificates and the ACME account in `data_dir/https/caddy`, in the archive volume, so a + rebuilt container doesn't ask for them again (certificate authorities limit how often). The configuration Lens + writes, `data_dir/https/caddy.json`, holds the DNS provider's credentials, so it's readable only by Lens (mode 600) + and passed by path, not on a command line. +- **Visitors**: Caddy passes the browser's `Host` on with `X-Forwarded-For` and `X-Forwarded-Proto: https`. The web + app trusts `X-Forwarded-For` from the worker (`LENS_WEB_PROXY_HOSTS`), so throttles and IP groups see the visitor. + Passkeys and sign-in follow the https:// address, and email links use the first domain unless + `notifications.app_url` says otherwise. +- **Changes**: a saved change restarts Caddy with the new configuration within a few seconds. + +By hand, with Docker: + +```sh +docker compose -f docker-compose.prod.yml -f docker-compose.https.yml up -d +``` + +## Refine later + +- A button in Settings that opens the ports through the upgrade agent, so step 2 needs no terminal. +- The install wizard asking for the domain. +- Caddy in the image, for machines that can't download it. diff --git a/fastapi_backend/app/api/deps.py b/fastapi_backend/app/api/deps.py index 4f073d1c9..ee12457fc 100644 --- a/fastapi_backend/app/api/deps.py +++ b/fastapi_backend/app/api/deps.py @@ -383,6 +383,7 @@ def web_origin(request: Request) -> str: FRONTEND_URL. Called directly, the API's own address.""" from app.config import settings as env from app.core.middleware import host_name + from app.domain import https h = request.headers host = (h.get("x-forwarded-host") or "").split(",")[0].strip() @@ -400,6 +401,8 @@ def web_origin(request: Request) -> str: shaped = re.fullmatch(r"[A-Za-z0-9.\-]+(:\d+)?|\[[0-9A-Fa-f:.]+\](:\d+)?", host) if shaped and host.lower() in _tunnel_hosts(request, cfg): return f"https://{host.lower()}" # Cloudflare serves it over https; cloudflared reaches the web app over http + if shaped and host_name(host) in https.hosts(cfg): + return f"https://{host.lower()}" # Caddy serves it over https (Settings › HTTPS), maybe at another port if shaped and proto in ("http", "https") and (trusted or host_name(host) in _web_hosts(cfg)): return f"{proto}://{host}" return env.FRONTEND_URL.rstrip("/") diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index b4fb20474..c49c71f0c 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -19,6 +19,7 @@ blobs, bridge, decide, + https, jobs, keyring, llm, @@ -44,6 +45,8 @@ FileStoreTestResult, FileStoreTry, Health, + HttpsProvider, + HttpsStatus, IndexQueued, LlmTestResult, LocalLlmStatus, @@ -178,6 +181,20 @@ def tunnel_status(user: AdminReader, cfg: Cfg, db: Db) -> TunnelStatus: return TunnelStatus(**tunnel.status(db, cfg)) +@router.get("/settings/https/status") +def https_status(user: AdminReader, cfg: Cfg, db: Db) -> HttpsStatus: + """How HTTPS (Settings › HTTPS) is doing: whether a process has the HTTPS ports and runs Caddy, the certificates it + holds and when they run out, and Caddy's last lines.""" + return HttpsStatus(**https.status(db, cfg)) + + +@router.get("/settings/https/providers") +def https_providers(user: AdminReader) -> list[HttpsProvider]: + """The DNS providers Settings › HTTPS lists for the DNS challenge, with the fields each needs. Any other caddy-dns + plugin works too, by name.""" + return [HttpsProvider(**p) for p in https.providers()] + + @router.post("/settings/bridge/test") def test_bridge(user: AdminWriter, cfg: Cfg, db: Db) -> BridgeTestResult: """Check that Matterbridge answers at its address with its token, and that the account to answer as exists.""" diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 96cbc0797..ed89440ca 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -260,7 +260,22 @@ def platform_main(argv, config): import threading import time - from .domain import auth, bridge, jobs, local_llm, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel, updates + from .domain import ( + auth, + bridge, + https, + jobs, + local_llm, + notify, + passkeys, + routines, + sensors, + settings, + sources, + telemetry, + tunnel, + updates, + ) ap = argparse.ArgumentParser(prog="lens") sub = ap.add_subparsers(dest="cmd", required=True) @@ -366,6 +381,7 @@ def password(): sensors.start(db, C, stop, log=print, name=wk.name) # and the sensor hub, while sensors are on bridge.start(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat-room bridge, while it's on tunnel.start(db, C, stop, name=wk.name, log=print) # and the Cloudflare tunnel, while it's on + https.start(db, C, stop, name=wk.name, log=print) # and HTTPS, where the ports are (LENS_HTTPS_LISTEN) local_llm.start(db, C, stop, name=wk.name, log=print) # and the local chat model, while it's on updates.start(db, C, stop, log=print) # and the daily check for a new release (docs/updates.md) _stop_on_term() diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py index 2e61798c8..3c745b2db 100644 --- a/fastapi_backend/app/core/runtime.py +++ b/fastapi_backend/app/core/runtime.py @@ -16,6 +16,7 @@ bridge, components, content_types, + https, jobs, local_llm, notify, @@ -92,6 +93,7 @@ def start_background(self) -> None: sensors.start(self.db, self.current, self.stop, log=log.info, name=f"api-{os.getpid()}") bridge.start(self.db, self.current, self.stop, lambda: self.base, name=f"api-{os.getpid()}", log_fn=log.warning) tunnel.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) + https.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) local_llm.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info) updates.start(self.db, self.current, self.stop, log=log.info) diff --git a/fastapi_backend/app/domain/https.py b/fastapi_backend/app/domain/https.py new file mode 100644 index 000000000..ef9a8c67d --- /dev/null +++ b/fastapi_backend/app/domain/https.py @@ -0,0 +1,702 @@ +"""HTTPS with certificates Lens gets and renews itself (Settings › HTTPS), from Let's Encrypt or ZeroSSL over ACME. + +Lens runs Caddy (https://caddyserver.com), a single program that answers on the HTTPS port, gets a certificate for +each domain, renews it before it runs out and passes visitors on to the web app. Off by default. Two ways to prove +the domain is yours: + +* **http**: the certificate authority visits http:///.well-known/acme-challenge/ (port 80) or the HTTPS + port itself (TLS-ALPN). The domain's DNS points at this machine and ports 80 and 443 reach it. +* **dns**: Lens adds a TXT record through the DNS provider's API (Cloudflare, Route 53 and the other caddy-dns + plugins), so nothing has to reach this machine to get the certificate, and wildcard names (*.example.com) work. + +Caddy is fetched the first time it's needed (data_dir/bin): Caddy's release from GitHub, checked against its +checksums, or for a DNS provider a build with that provider's plugin from Caddy's download service. A caddy on PATH +(or LENS_CADDY) that has the plugin is used instead. + +Only a process that has the HTTPS ports runs it (LENS_HTTPS_LISTEN=1: install.sh's docker-compose.https.yml gives +the worker the ports and sets it). Of those, one holds a lease on `app_service:https`, like the Cloudflare tunnel's, +and keeps what Caddy is doing there for the web app. +""" + +from __future__ import annotations + +import datetime as dt +import hashlib +import json +import os +import pathlib +import platform +import re +import shutil +import subprocess +import tarfile +import threading +import time +import urllib.parse + +import httpx + +from . import store + +R = store.R +ROW = R("app_service", "https") +CAS = ("letsencrypt", "zerossl", "both", "custom") +CHALLENGES = ("http", "tls-alpn", "dns") +LEASE_SECONDS = 30 +TICK_SECONDS = 5 +RETRY = (5, 15, 60, 300) +LOG_LINES = 40 +CADDY_VERSION = "2.10.2" +RELEASES = f"https://github.com/caddyserver/caddy/releases/download/v{CADDY_VERSION}/" +BUILDS = "https://caddyserver.com/api/download" +LETSENCRYPT = "https://acme-v02.api.letsencrypt.org/directory" +LETSENCRYPT_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory" +ZEROSSL = "https://acme.zerossl.com/v2/DV90" +HOST = re.compile(r"(?=.{1,253}$)([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}") +MODULE = re.compile(r"[a-z0-9][a-z0-9_-]{0,63}") +PACKAGE = re.compile(r"(github\.com|gitlab\.com|codeberg\.org)/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(/v[0-9]+)?") +DURATION = re.compile(r"[0-9]+(ms|s|m|h)") + +# The DNS providers the web app lists, with what each needs (the caddy-dns plugin's fields; secret ones are +# passwords). Any other caddy-dns plugin works too: its name, and its fields as JSON. +PROVIDERS: dict[str, dict] = { + "cloudflare": {"label": "Cloudflare", "fields": [("api_token", "API token (Zone.Zone read, Zone.DNS edit)", True)]}, + "route53": { + "label": "Amazon Route 53", + "fields": [ + ("access_key_id", "Access key ID", False), + ("secret_access_key", "Secret access key", True), + ("region", "Region (optional)", False), + ("hosted_zone_id", "Hosted zone ID (optional)", False), + ], + }, + "digitalocean": {"label": "DigitalOcean", "fields": [("auth_token", "API token", True)]}, + "duckdns": {"label": "Duck DNS", "fields": [("api_token", "Token", True)]}, + "porkbun": {"label": "Porkbun", "fields": [("api_key", "API key", True), ("api_secret_key", "Secret API key", True)]}, + "namecheap": { + "label": "Namecheap", + "fields": [("api_key", "API key", True), ("user", "User name", False), ("client_ip", "This server's public IP (optional)", False)], + }, + "gandi": {"label": "Gandi", "fields": [("bearer_token", "Personal access token", True)]}, + "ovh": { + "label": "OVHcloud", + "fields": [ + ("endpoint", "Endpoint (ovh-eu, ovh-ca, ...)", False), + ("application_key", "Application key", False), + ("application_secret", "Application secret", True), + ("consumer_key", "Consumer key", True), + ], + }, + "desec": {"label": "deSEC", "fields": [("token", "Token", True)]}, + "dnsimple": { + "label": "DNSimple", + "fields": [("api_access_token", "API access token", True), ("account_id", "Account ID (optional)", False)], + }, + "netlify": {"label": "Netlify", "fields": [("personal_access_token", "Personal access token", True)]}, + "vultr": {"label": "Vultr", "fields": [("auth_token", "API key", True)]}, + "linode": {"label": "Linode (Akamai)", "fields": [("api_token", "API token", True)]}, + "ionos": {"label": "IONOS", "fields": [("auth_api_token", "API key (prefix.secret)", True)]}, + "bunny": {"label": "Bunny DNS", "fields": [("access_key", "API key", True)]}, + "netcup": { + "label": "netcup", + "fields": [("customer_number", "Customer number", False), ("api_key", "API key", True), ("api_password", "API password", True)], + }, + "azure": { + "label": "Azure DNS", + "fields": [ + ("tenant_id", "Tenant ID", False), + ("client_id", "Client ID", False), + ("client_secret", "Client secret", True), + ("subscription_id", "Subscription ID", False), + ("resource_group_name", "Resource group", False), + ], + }, + "googleclouddns": {"label": "Google Cloud DNS", "fields": [("gcp_project", "Project ID", False)]}, + "powerdns": { + "label": "PowerDNS", + "fields": [("server_url", "API address", False), ("api_token", "API key", True), ("server_id", "Server ID (optional)", False)], + }, + "rfc2136": { + "label": "RFC 2136 (BIND, Knot, ...)", + "fields": [ + ("server", "Server (host:53)", False), + ("key_name", "TSIG key name", False), + ("key_alg", "TSIG algorithm (hmac-sha256)", False), + ("key", "TSIG secret", True), + ], + }, +} + + +class HttpsError(RuntimeError): + """Setting HTTPS up didn't work; the message says why, for the admin.""" + + +def _now(): + return dt.datetime.now(dt.timezone.utc) + + +def _ts(t=None): + return (t or _now()).isoformat(timespec="seconds") + + +def conf(cfg): + return cfg.get("https") or {} + + +def enabled(cfg): + return bool(conf(cfg).get("enabled")) + + +def domains(cfg): + """The names to get certificates for, lower case.""" + return [d.strip().lower().rstrip(".") for d in conf(cfg).get("domains") or [] if isinstance(d, str) and d.strip()] + + +def hosts(cfg): + """The host names the web app is served at over HTTPS (not wildcards), while HTTPS is on.""" + return [d for d in domains(cfg) if not d.startswith("*.")] if enabled(cfg) else [] + + +def public_port(): + """The port HTTPS is reached at from outside (LENS_HTTPS_PORT, which install.sh publishes), 443 by default.""" + try: + return int(os.environ.get("LENS_HTTPS_PORT") or 443) + except ValueError: + return 443 + + +def public_url(cfg): + """https://, for links in emails, or "" when HTTPS is off.""" + h = hosts(cfg) + if not h: + return "" + port = public_port() + return f"https://{h[0]}" + ("" if port == 443 else f":{port}") + + +def origin(cfg): + """Where Caddy sends visitors: the web app, as this server reaches it.""" + from app.config import settings as env + + return ( + conf(cfg).get("origin") or os.environ.get("LENS_HTTPS_ORIGIN") or os.environ.get("LENS_TUNNEL_ORIGIN") or env.FRONTEND_URL + ).rstrip("/") + + +def credentials(cfg): + """The DNS provider's fields ({name: value}), from the sealed dns_credentials (JSON).""" + raw = conf(cfg).get("dns_credentials") + if not raw or not isinstance(raw, str): + return {} + try: + d = json.loads(raw) + except ValueError: + return {} + return {str(k): v for k, v in d.items()} if isinstance(d, dict) else {} + + +def parse_credentials(value): + """Settings › HTTPS: dns_credentials is a JSON object of the provider's fields. ValueError when it isn't.""" + try: + d = json.loads(value) + except ValueError: + raise ValueError("https.dns_credentials is a JSON object of the DNS provider's fields") from None + if not isinstance(d, dict) or not all( + isinstance(k, str) and re.fullmatch(r"[a-z0-9_]{1,64}", k) and isinstance(v, (str, int, bool)) and len(str(v)) <= 4000 + for k, v in d.items() + ): + raise ValueError('https.dns_credentials is a JSON object of the DNS provider\'s fields: {"api_token": "..."}') + return value + + +def package(cfg): + """The Go package of the DNS provider's Caddy plugin: dns_package, else github.com/caddy-dns/.""" + t = conf(cfg) + return (t.get("dns_package") or "").strip() or f"github.com/caddy-dns/{(t.get('dns_provider') or '').strip()}" + + +def check(cfg): + """Settings › HTTPS before saving: ValueError for what can't work.""" + t = conf(cfg) + names = domains(cfg) + challenge = t.get("challenge") or "http" + for d in names: + bare = d[2:] if d.startswith("*.") else d + if not HOST.fullmatch(bare): + raise ValueError(f"{d} isn't a domain name; enter names like lens.example.com") + if d.startswith("*.") and challenge != "dns": + raise ValueError("a wildcard name (*.example.com) needs the DNS challenge") + if len(names) > 50: + raise ValueError("up to 50 domain names") + email = (t.get("email") or "").strip() + if email and not re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", email): + raise ValueError("the email address for the certificate authority isn't an email address") + ca = t.get("ca") or "both" + if ca == "custom": + u = urllib.parse.urlsplit(t.get("ca_url") or "") + if u.scheme != "https" or not u.hostname: + raise ValueError("a custom certificate authority is its ACME directory's https:// address") + if bool(t.get("eab_key_id")) != bool(t.get("eab_hmac_key")): + raise ValueError("external account binding needs both the key ID and the HMAC key") + if challenge == "dns": + prov = (t.get("dns_provider") or "").strip() + if not MODULE.fullmatch(prov): + raise ValueError("choose the DNS provider that holds the domain, like cloudflare or route53") + if not PACKAGE.fullmatch(package(cfg)): + raise ValueError("the DNS plugin is a Go package, like github.com/caddy-dns/cloudflare") + for r in t.get("dns_resolvers") or []: + if not re.fullmatch(r"[A-Za-z0-9.:\[\]-]{1,100}", str(r)): + raise ValueError("DNS resolvers are addresses, like 1.1.1.1 or 1.1.1.1:53") + if t.get("propagation") and not DURATION.fullmatch(str(t["propagation"])): + raise ValueError("how long to wait for the DNS record is a duration, like 2m") + for k in ("https_port", "http_port"): + v = t.get(k) + if v is not None and not (isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 65535): + raise ValueError(f"https.{k} is a port number") + if t.get("https_port") and t.get("https_port") == t.get("http_port"): + raise ValueError("the HTTPS and HTTP ports are different ports") + o = urllib.parse.urlsplit(origin(cfg)) + if o.scheme != "http" or not o.hostname: + raise ValueError("the web app's address for HTTPS is an http:// address, like http://frontend:3000") + if t.get("enabled") and not names: + raise ValueError("enter the domain name Lens is reached at, like lens.example.com") + + +# ---------- Caddy's configuration ---------- +def _issuers(cfg): + t = conf(cfg) + challenge = t.get("challenge") or "http" + if challenge == "dns": + dns: dict = {"provider": {"name": t["dns_provider"].strip(), **credentials(cfg)}} + if t.get("dns_resolvers"): + dns["resolvers"] = [str(r) for r in t["dns_resolvers"]] + if t.get("propagation"): + dns["propagation_timeout"] = str(t["propagation"]) + challenges: dict = {"http": {"disabled": True}, "tls-alpn": {"disabled": True}, "dns": dns} + elif challenge == "tls-alpn": + challenges = {"http": {"disabled": True}} + else: + challenges = {} + + def acme(url, eab=False): + i: dict = {"module": "acme", "ca": url} + if (t.get("email") or "").strip(): + i["email"] = t["email"].strip() + if challenges: + i["challenges"] = challenges + if eab and t.get("eab_key_id") and t.get("eab_hmac_key"): + i["external_account"] = {"key_id": t["eab_key_id"], "mac_key": t["eab_hmac_key"]} + return i + + ca = t.get("ca") or "both" + le = acme(LETSENCRYPT_STAGING if t.get("staging") else LETSENCRYPT) + if ca == "letsencrypt": + return [le] + if ca == "zerossl": + return [acme(ZEROSSL, eab=True)] + if ca == "custom": + return [acme(t["ca_url"], eab=True)] + # both: Let's Encrypt, and ZeroSSL when Let's Encrypt can't (ZeroSSL has no staging service) + return [le] if t.get("staging") else [le, acme(ZEROSSL, eab=True)] + + +def caddy_config(cfg, storage): + """Caddy's JSON configuration: HTTPS for the domains, passed on to the web app; HTTP sent to HTTPS.""" + t = conf(cfg) + https_port, http_port = int(t.get("https_port") or 8443), int(t.get("http_port") or 8080) + names = domains(cfg) + o = urllib.parse.urlsplit(origin(cfg)) + upstream = f"{o.hostname}:{o.port or 80}" + handle: list[dict] = [] + if t.get("hsts"): + handle.append({"handler": "headers", "response": {"set": {"Strict-Transport-Security": ["max-age=31536000"]}}}) + handle.append({"handler": "reverse_proxy", "upstreams": [{"dial": upstream}]}) + servers: dict = { + "lens": { + "listen": [f":{https_port}"], + "routes": [{"match": [{"host": names}], "handle": handle, "terminal": True}], + # its own redirect below: Caddy's would send people to the port inside the container + "automatic_https": {"disable_redirects": True}, + } + } + if t.get("redirect_http", True): + port = public_port() + target = "https://{http.request.host}" + ("" if port == 443 else f":{port}") + "{http.request.uri}" + servers["redirect"] = { + "listen": [f":{http_port}"], + "routes": [ + { + "match": [{"host": names}], + "handle": [{"handler": "static_response", "status_code": 308, "headers": {"Location": [target]}}], + } + ], + } + return { + "admin": {"disabled": True}, + "logging": {"logs": {"default": {"level": "INFO", "encoder": {"format": "json"}}}}, + "storage": {"module": "file_system", "root": str(storage)}, + "apps": { + "http": {"http_port": http_port, "https_port": https_port, "servers": servers}, + "tls": {"automation": {"policies": [{"subjects": names, "issuers": _issuers(cfg)}]}}, + }, + } + + +def certificates(storage): + """The certificates Caddy keeps: [{names, expires, issuer}], soonest to expire first.""" + from cryptography import x509 + + out = [] + for p in sorted(pathlib.Path(storage).glob("certificates/*/*/*.crt")): + try: + c = x509.load_pem_x509_certificate(p.read_bytes()) + except (OSError, ValueError): + continue + try: + names = c.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type(x509.DNSName) + except x509.ExtensionNotFound: + names = [] + org = c.issuer.get_attributes_for_oid(x509.NameOID.ORGANIZATION_NAME) + out.append( + { + "names": list(names), + "expires": c.not_valid_after_utc.isoformat(timespec="seconds"), + "issuer": str(org[0].value) if org else c.issuer.rfc4514_string(), + } + ) + return sorted(out, key=lambda c: c["expires"]) + + +# ---------- what other processes see ---------- +def state(db): + row = db.one("SELECT * FROM $r", r=ROW) or {} + row.pop("id", None) + if row.get("log") and isinstance(row["log"], str): + row["log"] = json.loads(row["log"]) + if row.get("certificates") and isinstance(row["certificates"], str): + row["certificates"] = json.loads(row["certificates"]) + return row + + +def status(db, cfg): + s = state(db) + live = bool(s.get("at") and s["at"] >= _ts(_now() - dt.timedelta(seconds=LEASE_SECONDS))) + on = enabled(cfg) + names = set(domains(cfg)) + certs = [c for c in s.get("certificates") or [] if names & set(c.get("names") or [])] + return { + "enabled": on, + "running": live and bool(s.get("running")), + "ports_open": live, + "url": public_url(cfg) or None, + "certificates": certs, + "error": s.get("error") if on else None, + "log": s.get("log") or [], + "origin": origin(cfg), + "process": s.get("owner") if live else None, + } + + +def providers(): + """Settings › HTTPS: the DNS providers listed, with their fields.""" + return [ + {"name": n, "label": p["label"], "fields": [{"name": f, "label": lbl, "secret": sec} for f, lbl, sec in p["fields"]]} + for n, p in PROVIDERS.items() + ] + + +# ---------- Caddy ---------- +def _platform(): + system = platform.system().lower() + machine = platform.machine().lower() + arch = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64", "armv7l": "armv7", "armv6l": "armv6"}.get(machine) + if system not in ("linux", "darwin") or not arch: + raise HttpsError(f"install caddy yourself on this system ({system} {machine}) and set LENS_CADDY to it") + return system, arch + + +_MODULES: dict[str, tuple[float, set[str]]] = {} + + +def modules(exe): + """The modules a caddy has (cached while the file is unchanged).""" + try: + mtime = os.path.getmtime(exe) + except OSError: + return set() + hit = _MODULES.get(exe) + if hit and hit[0] == mtime: + return hit[1] + try: + out = subprocess.run([exe, "list-modules"], capture_output=True, text=True, timeout=30).stdout + except (OSError, subprocess.TimeoutExpired): + return set() + found = {line.strip() for line in out.splitlines() if line.strip() and " " not in line.strip()} + _MODULES[exe] = (mtime, found) + return found + + +def _wanted(cfg): + t = conf(cfg) + return f"dns.providers.{t['dns_provider'].strip()}" if (t.get("challenge") == "dns" and t.get("dns_provider")) else None + + +def binary(cfg): + """A caddy that has what the settings need: LENS_CADDY, caddy on PATH, or one Lens fetched; None when none does.""" + want = _wanted(cfg) + bin_dir = pathlib.Path(cfg["data_dir"]) / "bin" + fetched = bin_dir / (f"caddy-{conf(cfg)['dns_provider'].strip()}" if want else "caddy") + for exe in (os.environ.get("LENS_CADDY"), shutil.which("caddy"), str(fetched)): + if exe and os.path.isfile(exe) and os.access(exe, os.X_OK) and (not want or want in modules(exe)): + return exe + return None + + +def _download(url, dest, say=None): + tmp = dest.with_name(dest.name + ".part") + if say: + say(f"downloading {url}") + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=300) as r: + r.raise_for_status() + with open(tmp, "wb") as f: + for chunk in r.iter_bytes(): + f.write(chunk) + except httpx.HTTPError as e: + tmp.unlink(missing_ok=True) + raise HttpsError(f"couldn't download Caddy from {urllib.parse.urlsplit(url).hostname}: {e}") from None + return tmp + + +def fetch_binary(cfg, say=None): + """Download Caddy into data_dir/bin: the release from GitHub (checksum checked), or with the DNS provider's + plugin, a build from Caddy's download service. Returns its path.""" + system, arch = _platform() + dest = pathlib.Path(cfg["data_dir"]) / "bin" + dest.mkdir(parents=True, exist_ok=True) + want = _wanted(cfg) + if want: + prov = conf(cfg)["dns_provider"].strip() + params = {"os": system, "arch": arch[:3] if arch.startswith("armv") else arch, "p": package(cfg)} + if arch.startswith("armv"): + params["arm"] = arch[-1] + out = dest / f"caddy-{prov}" + tmp = _download(f"{BUILDS}?{urllib.parse.urlencode(params)}", out, say) + tmp.chmod(0o755) + tmp.replace(out) + _MODULES.pop(str(out), None) + if want not in modules(str(out)): + raise HttpsError(f"the Caddy built with {package(cfg)} has no {want} module; check the DNS provider's name") + return str(out) + name = f"caddy_{CADDY_VERSION}_{'mac' if system == 'darwin' else system}_{arch}.tar.gz" + try: + sums = httpx.get(RELEASES + f"caddy_{CADDY_VERSION}_checksums.txt", follow_redirects=True, timeout=60) + sums.raise_for_status() + except httpx.HTTPError as e: + raise HttpsError(f"couldn't download Caddy's checksums from GitHub: {e}") from None + expect = next((line.split()[0] for line in sums.text.splitlines() if line.strip().endswith(" " + name)), None) + if not expect: + raise HttpsError(f"Caddy {CADDY_VERSION} has no download for {system} {arch}") + tmp = _download(RELEASES + name, dest / name, say) + h = hashlib.sha512() + with open(tmp, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + if h.hexdigest() != expect: + tmp.unlink(missing_ok=True) + raise HttpsError("the Caddy download doesn't match its checksum; try again") + out = dest / "caddy" + with tarfile.open(tmp) as t: + member = next((m for m in t.getmembers() if m.name == "caddy" and m.isfile()), None) + if not member: + raise HttpsError("the Caddy download has no program in it") + src = t.extractfile(member) + assert src is not None + with src, open(out.with_name("caddy.new"), "wb") as f: + shutil.copyfileobj(src, f) + tmp.unlink() + out.with_name("caddy.new").chmod(0o755) + out.with_name("caddy.new").replace(out) + _MODULES.pop(str(out), None) + return str(out) + + +# ---------- the runner ---------- +def _line(raw): + """A Caddy log line, shortened for Settings › HTTPS, and what it says: (text, obtained?, problem or None).""" + try: + d = json.loads(raw) + except ValueError: + return raw[:300], False, None + if not isinstance(d, dict): + return raw[:300], False, None + msg = str(d.get("msg") or "") + who = ", ".join(d.get("identifiers") or []) or str(d.get("identifier") or d.get("domain") or "") + err = str(d.get("error") or "") + text = " ".join(x for x in (str(d.get("level") or "").upper(), msg, who, err) if x)[:300] + obtained = msg.startswith("certificate obtained successfully") or msg == "certificate renewed successfully" + problem = f"{msg}{' (' + who + ')' if who else ''}: {err}"[:300] if d.get("level") == "error" and err else None + return text, obtained, problem + + +def listens_here(): + """Whether this process has the HTTPS ports (LENS_HTTPS_LISTEN=1).""" + return (os.environ.get("LENS_HTTPS_LISTEN") or "").strip().lower() in ("1", "true", "yes", "on") + + +class Runner: + """Holds the lease and runs Caddy while Settings › HTTPS says to, in a process that has the ports.""" + + def __init__(self, db, cfg_fn, name, log=None): + self.db, self.cfg_fn, self.name, self.log = db, cfg_fn, name, log or (lambda *a: None) + self.proc: subprocess.Popen | None = None + self.sig = None + self.lines: list[str] = [] + self.error = None + self.fails = 0 + self.next_try = 0.0 + self.certs_at = 0.0 + self.certs: list[dict] = [] + self.lock = threading.Lock() + + def _lease(self): + now = _now() + return bool( + self.db.rows( + "UPSERT $r MERGE {owner: $me, at: $t} WHERE owner = NONE OR owner = $me OR at < $stale RETURN AFTER", + r=ROW, + me=self.name, + t=_ts(now), + stale=_ts(now - dt.timedelta(seconds=LEASE_SECONDS)), + ) + ) + + def storage(self, cfg): + return pathlib.Path(cfg["data_dir"]) / "https" + + def _save(self, cfg=None): + if cfg is not None and time.monotonic() - self.certs_at > 60: + self.certs_at = time.monotonic() + try: + self.certs = certificates(self.storage(cfg) / "caddy") + except Exception: # noqa: BLE001 - a certificate it can't read just isn't listed + pass + with self.lock: + d = { + "running": self.proc is not None and self.proc.poll() is None, + "error": self.error, + "log": json.dumps(self.lines[-LOG_LINES:]), + "certificates": json.dumps(self.certs), + } + self.db.q("UPDATE $r MERGE $d WHERE owner = $me", r=ROW, d=d, me=self.name) + + def stop_process(self): + p, self.proc = self.proc, None + if p and p.poll() is None: + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + + def _read(self, proc): + assert proc.stdout is not None + for raw in proc.stdout: + raw = raw.rstrip() + if not raw: + continue + text, obtained, problem = _line(raw) + with self.lock: + self.lines = (self.lines + [text])[-LOG_LINES:] + if obtained: + self.error, self.fails, self.certs_at = None, 0, 0.0 + elif problem: + self.error = problem # Caddy keeps trying; this says why it has no certificate yet + if obtained or problem: + self._save() + code = proc.wait() + if self.proc is proc: + with self.lock: + self.error = f"Caddy stopped (exit {code}); {self.lines[-1] if self.lines else 'no output'}" + self.fails += 1 + self.next_try = time.monotonic() + RETRY[min(self.fails - 1, len(RETRY) - 1)] + self._save() + + def tick(self): + cfg = self.cfg_fn() + if not self._lease(): + if self.proc: # someone else runs it now + self.stop_process() + return + if not enabled(cfg): # the lease still says a process has the ports, for Settings › HTTPS + if self.proc: + self.stop_process() + self.error, self.sig = None, None + self._save() + return + root = self.storage(cfg) + try: + check(cfg) + conf_json = json.dumps(caddy_config(cfg, root / "caddy"), sort_keys=True) + except (ValueError, KeyError) as e: + self.stop_process() + self.error = str(e) + self._save() + return + sig = json.dumps([conf_json, _wanted(cfg), package(cfg)]) + if sig != self.sig: + self.stop_process() + self.sig, self.fails, self.next_try, self.error, self.lines = sig, 0, 0.0, None, [] + if self.proc and self.proc.poll() is None: + self._save(cfg) + return + if time.monotonic() < self.next_try: + self._save(cfg) + return + try: + exe = binary(cfg) or fetch_binary(cfg, say=lambda m: self.lines.append(m)) + root.mkdir(parents=True, exist_ok=True) + os.chmod(root, 0o700) # the configuration holds the DNS provider's credentials + path = root / "caddy.json" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(conf_json) + except (HttpsError, OSError) as e: + self.error = str(e) + self.fails += 1 + self.next_try = time.monotonic() + RETRY[min(self.fails - 1, len(RETRY) - 1)] + self._save(cfg) + return + env = {k: v for k, v in os.environ.items() if not k.startswith("CADDY")} + env["HOME"] = str(root) # Caddy keeps its own files under $HOME otherwise + env["XDG_DATA_HOME"], env["XDG_CONFIG_HOME"] = str(root / "data"), str(root / "config") + self.proc = subprocess.Popen( + [exe, "run", "--config", str(path)], env=env, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1 + ) + threading.Thread(target=self._read, args=(self.proc,), daemon=True, name="caddy-log").start() + self.log("https: started Caddy for %s", ", ".join(domains(cfg))) + self._save(cfg) + + def loop(self, stop): + while not stop.is_set(): + try: + self.tick() + except Exception as e: # noqa: BLE001 - keep the runner alive; the error shows in Settings + self.error = f"{type(e).__name__}: {e}" + try: + self._save() + except Exception: # noqa: BLE001 + pass + stop.wait(TICK_SECONDS) + self.stop_process() + self.db.q("UPDATE $r SET owner = NONE, running = false WHERE owner = $me", r=ROW, me=self.name) + + +def start(db, cfg_fn, stop, name, log=None): + """Run the HTTPS runner in a thread: idle unless this process has the ports and HTTPS is on.""" + if not listens_here(): + return None + r = Runner(db, cfg_fn, name, log) + threading.Thread(target=r.loop, args=(stop,), daemon=True, name="https").start() + return r diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index ad1ed0554..1596998dd 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -54,6 +54,7 @@ "encryption": None, "fedora": None, "tunnel": None, + "https": None, # bind is a startup setting only "sensors": ( "enabled", @@ -120,6 +121,7 @@ "telemetry": ("headers", "prometheus_token"), "fedora": ("password",), "tunnel": ("token", "api_token"), + "https": ("dns_credentials", "eab_hmac_key"), "documents": ("anytopdf_token",), } ENUMS = { @@ -143,6 +145,8 @@ ("bridge", "answer"): {"mention", "all"}, ("bridge", "approve"): {"off", "low_risk"}, ("tunnel", "mode"): {"off", "quick", "token", "managed"}, + ("https", "ca"): {"letsencrypt", "zerossl", "both", "custom"}, + ("https", "challenge"): {"http", "tls-alpn", "dns"}, ("files", "store"): {"local", "connection"}, ("documents", "converter"): {"auto", "anytopdf", "lens"}, } @@ -1050,6 +1054,10 @@ def save(db, base, section, changes, user=None): raise ValueError(f"matterbridge.{k} is one line of text") if (section, k) in (("documents", "anytopdf_token"), ("telemetry", "prometheus_token")): v = _check(section, k, v, None) + if (section, k) == ("https", "dns_credentials"): + from . import https + + https.parse_credentials(v) sealed[k] = seal(base, v, f"setting:{section}.{k}") elif not (isinstance(v, dict) and v.get("secret")): # the mask echoed back means "unchanged" raise ValueError(f"{section}.{k} must be text") @@ -1097,6 +1105,12 @@ def save(db, base, section, changes, user=None): merged = {**(base.get("tunnel") or {}), **data, **{k: "set" for k in sealed}} tunnel.check({"tunnel": merged}) + if section == "https": + from . import https + + merged = {**(base.get("https") or {}), **data, **{k: "set" for k in sealed}} + merged.pop("dns_credentials", None) # its fields aren't needed to check the rest + https.check({"https": merged}) if ( section == "server" and "allowed_hosts" in data diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 75a8415d8..bf7a37594 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -473,6 +473,33 @@ def labels(): # tunnel and DNS record for hostname with api_token). origin: the web app as this server reaches it (default # LENS_TUNNEL_ORIGIN, else FRONTEND_URL). token and api_token are secrets. "tunnel": {"mode": "off", "hostname": "", "token": None, "api_token": None, "origin": ""}, + # HTTPS with certificates from Let's Encrypt or ZeroSSL, got and renewed by Caddy run by Lens (domain/https.py). Off + # by default. domains: the names to serve (and get certificates for); ca: letsencrypt, zerossl, both (Let's Encrypt + # then ZeroSSL) or custom (ca_url, an ACME directory); challenge: http (ports 80 and 443 reach this machine), tls-alpn + # (only 443) or dns (a TXT record through dns_provider's API: any caddy-dns plugin, dns_package to use another one; + # dns_credentials is its fields as JSON). eab_*: external account binding. https_port and http_port are where Caddy + # listens in the container (install.sh publishes 443 and 80 to them). dns_credentials and eab_hmac_key are secrets. + "https": { + "enabled": False, + "domains": [], + "email": "", + "ca": "both", + "ca_url": "", + "staging": False, + "challenge": "http", + "dns_provider": "", + "dns_package": "", + "dns_credentials": None, + "dns_resolvers": [], + "propagation": "", + "eab_key_id": "", + "eab_hmac_key": None, + "redirect_http": True, + "hsts": False, + "origin": "", + "https_port": 8443, + "http_port": 8080, + }, "notifications": {"enabled": True, "networks": [], "poll_seconds": 5, "max_attempts": 6, "app_url": None}, # OpenTelemetry traces and metrics (docs/telemetry.md): off unless an admin turns it on, and sent only to the OTLP/HTTP # endpoint set here (e.g. a collector at http://localhost:4318). headers is a secret: key=value pairs for the @@ -967,7 +994,8 @@ def close(self): # Note: on 2.x, CONTAINS against an indexed field also returns nothing; use string::contains() there. # settings, people and access "DEFINE TABLE IF NOT EXISTS app_setting SCHEMALESS", - # the Cloudflare tunnel: which process runs cloudflared, and what it's doing (app/domain/tunnel.py) + # the Cloudflare tunnel: which process runs cloudflared, and what it's doing (app/domain/tunnel.py); HTTPS: which + # process runs Caddy (app/domain/https.py) "DEFINE TABLE IF NOT EXISTS app_service SCHEMALESS", # each namespace's data key, wrapped by the keys that can open it (app/domain/keyring.py) "DEFINE TABLE IF NOT EXISTS data_key SCHEMALESS", diff --git a/fastapi_backend/app/email.py b/fastapi_backend/app/email.py index be3eb4d06..df065f3c5 100644 --- a/fastapi_backend/app/email.py +++ b/fastapi_backend/app/email.py @@ -21,12 +21,14 @@ def mail_enabled(cfg: dict[str, Any]) -> bool: def app_url(cfg: dict[str, Any], db: Any = None) -> str: """Where people open the web app, for links in emails: notifications.app_url when set, else the Cloudflare tunnel's - fixed hostname (Settings › Remote access), else FRONTEND_URL. With `db`, for a link that's read right away (a chat - room's answer), a quick tunnel's address counts too while it's up; emails don't use it, since it changes.""" - from app.domain import tunnel + fixed hostname (Settings › Remote access), else the first HTTPS domain (Settings › HTTPS), else FRONTEND_URL. With + `db`, for a link that's read right away (a chat room's answer), a quick tunnel's address counts too while it's up; + emails don't use it, since it changes.""" + from app.domain import https, tunnel host = tunnel.hostname(cfg) or (tunnel.public_host(db) if db is not None else "") - return str((cfg.get("notifications") or {}).get("app_url") or (f"https://{host}" if host else settings.FRONTEND_URL)).rstrip("/") + url = f"https://{host}" if host else https.public_url(cfg) or settings.FRONTEND_URL + return str((cfg.get("notifications") or {}).get("app_url") or url).rstrip("/") def _mailer(cfg: dict[str, Any]) -> Any: diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index 6743dc51c..789ede8e7 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -57,6 +57,36 @@ class TunnelStatus(ResponseModel): process: str | None = Field(None, description="the server process running cloudflared") +class HttpsCertificate(ResponseModel): + names: list[str] = Field(description="the domain names it's for") + expires: str = Field(description="when it runs out (Caddy renews it about a month before)") + issuer: str = Field(description="the certificate authority that issued it") + + +class HttpsStatus(ResponseModel): + enabled: bool + running: bool = Field(description="Caddy is running in a server process") + ports_open: bool = Field(description="a server process has the HTTPS ports (install.sh opened them)") + url: str | None = Field(None, description="the https:// address people open Lens at") + certificates: list[HttpsCertificate] = Field(default_factory=list) + error: str | None = Field(None, description="why there's no certificate yet, when there isn't") + log: list[str] = Field(default_factory=list, description="Caddy's last lines") + origin: str = Field(description="where Caddy sends visitors: the web app, as the server reaches it") + process: str | None = Field(None, description="the server process with the HTTPS ports") + + +class HttpsProviderField(ResponseModel): + name: str + label: str + secret: bool + + +class HttpsProvider(ResponseModel): + name: str = Field(description="the caddy-dns plugin's name, like cloudflare") + label: str + fields: list[HttpsProviderField] + + class LocalModel(ResponseModel): id: str label: str diff --git a/fastapi_backend/tests/api/test_https.py b/fastapi_backend/tests/api/test_https.py new file mode 100644 index 000000000..9178149c6 --- /dev/null +++ b/fastapi_backend/tests/api/test_https.py @@ -0,0 +1,290 @@ +"""HTTPS with certificates Lens gets and renews itself: set up in Settings › HTTPS, Caddy runs in the one process that +has the HTTPS ports, the certificates show in the app, and passkeys and email links use the https:// address. Caddy +is faked: it reads the configuration Lens writes, says it got a certificate and keeps one where Caddy would.""" + +from __future__ import annotations + +import datetime as dt +import json +import pathlib +import sys +import textwrap +import time + +import pytest + +from app import email +from app.domain import https, settings +from tests.helpers import login, make_user + +FAKE = textwrap.dedent( + """\ + #!{python} + import datetime as dt, json, os, pathlib, sys, time + args = sys.argv[1:] + if args[0] == "list-modules": + print("http.handlers.reverse_proxy") + print("tls.issuance.acme") + for m in os.environ.get("FAKE_MODULES", "").split(","): + if m: + print(m) + sys.exit(0) + conf = json.loads(pathlib.Path(args[args.index("--config") + 1]).read_text()) + pathlib.Path(os.environ["FAKE_SEEN"]).write_text(json.dumps(conf)) + policy = conf["apps"]["tls"]["automation"]["policies"][0] + for name in policy["subjects"]: + if "broken" in name: + print(json.dumps({{"level": "error", "msg": "could not get certificate from issuer", "identifier": name, + "error": "HTTP 400 urn:ietf:params:acme:error:dns - no valid A records found"}}), flush=True) + continue + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(x509.NameOID.COMMON_NAME, name)]) + issuer = x509.Name([x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, "Let's Encrypt")]) + now = dt.datetime.now(dt.timezone.utc) + cert = (x509.CertificateBuilder().subject_name(subject).issuer_name(issuer).public_key(key.public_key()) + .serial_number(1).not_valid_before(now).not_valid_after(now + dt.timedelta(days=90)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName(name)]), False).sign(key, hashes.SHA256())) + d = pathlib.Path(conf["storage"]["root"]) / "certificates" / "acme-v02.api.letsencrypt.org-directory" / name.replace("*", "wildcard_") + d.mkdir(parents=True, exist_ok=True) + (d / (name.replace("*", "wildcard_") + ".crt")).write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + print(json.dumps({{"level": "info", "msg": "certificate obtained successfully", "identifier": name}}), flush=True) + while True: + time.sleep(0.2) + """ +) + + +@pytest.fixture +def caddy(tmp_path, monkeypatch): + """A fake caddy on PATH, in a process that has the HTTPS ports.""" + d = tmp_path / "bin" + d.mkdir() + exe = d / "caddy" + exe.write_text(FAKE.format(python=sys.executable)) + exe.chmod(0o755) + seen = tmp_path / "seen.json" + monkeypatch.setenv("PATH", f"{d}:{__import__('os').environ['PATH']}") + monkeypatch.setenv("FAKE_SEEN", str(seen)) + monkeypatch.setenv("LENS_HTTPS_LISTEN", "1") + monkeypatch.delenv("LENS_CADDY", raising=False) + https._MODULES.clear() + return seen + + +def wait(fn, timeout=15): + end = time.monotonic() + timeout + while time.monotonic() < end: + v = fn() + if v: + return v + time.sleep(0.05) + raise AssertionError("timed out") + + +def runner(db, cfg, name="worker-1"): + return https.Runner(db, lambda: settings.Settings(db, cfg).current(), name) + + +def test_https_is_set_up_in_the_app(client, db, cfg): + make_user(db, "root@x.io", "root password 1", admin=True) + a = login(client, "root@x.io", "root password 1") + s = client.get("/api/v1/settings/https/status", headers=a).json() + assert not s["enabled"] and not s["running"] and not s["ports_open"] and s["url"] is None + providers = client.get("/api/v1/settings/https/providers", headers=a).json() + cloudflare = next(p for p in providers if p["name"] == "cloudflare") + assert cloudflare["fields"] == [{"name": "api_token", "label": "API token (Zone.Zone read, Zone.DNS edit)", "secret": True}] + for bad in ( + {"enabled": True}, # no domain + {"domains": ["not a domain"]}, + {"domains": ["*.example.com"]}, # a wildcard needs the DNS challenge + {"ca": "someone"}, + {"ca": "custom", "ca_url": "http://ca.example.com/dir"}, + {"challenge": "dns", "dns_provider": "Cloud Flare"}, + {"challenge": "dns", "dns_provider": "cloudflare", "dns_package": "example.com/x"}, + {"dns_credentials": "api_token=x"}, + {"dns_credentials": '{"api token": "x"}'}, + {"eab_key_id": "kid"}, # without its HMAC key + {"https_port": 8080}, # the same as the HTTP port + {"origin": "frontend:3000"}, + {"email": "nobody"}, + ): + r = client.put("/api/v1/settings/https", headers=a, json=bad) + assert r.status_code == 400, bad + r = client.put( + "/api/v1/settings/https", + headers=a, + json={ + "enabled": True, + "domains": ["Lens.Example.com", "*.example.com"], + "email": "admin@example.com", + "challenge": "dns", + "dns_provider": "cloudflare", + "dns_credentials": json.dumps({"api_token": "cf-secret"}), + }, + ) + assert r.status_code == 200, r.text + assert "cf-secret" not in client.get("/api/v1/settings", headers=a).text # the credentials are a secret + current = settings.Settings(db, cfg).current() + assert https.credentials(current) == {"api_token": "cf-secret"} and https.hosts(current) == ["lens.example.com"] + s = client.get("/api/v1/settings/https/status", headers=a).json() + assert s["enabled"] and s["url"] == "https://lens.example.com" and not s["ports_open"] # nothing has the ports yet + # a change that keeps the saved credentials (the mask echoed back) + r = client.put("/api/v1/settings/https", headers=a, json={"email": "", "dns_credentials": {"secret": True}}) + assert r.status_code == 200, r.text + assert https.credentials(settings.Settings(db, cfg).current()) == {"api_token": "cf-secret"} + assert client.put("/api/v1/settings/https", headers=a, json={"enabled": False}).status_code == 200 + viewer = make_user(db, "v@x.io", "viewer password 1") + assert viewer + assert client.get("/api/v1/settings/https/status", headers=login(client, "v@x.io", "viewer password 1")).status_code == 403 + + +def test_caddys_configuration_follows_the_settings(cfg, monkeypatch): + monkeypatch.setenv("LENS_HTTPS_PORT", "8443") + cfg["https"] = { + **cfg["https"], + "enabled": True, + "domains": ["lens.example.com"], + "email": "a@example.com", + "hsts": True, + "origin": "http://frontend:3000", + } + c = https.caddy_config(cfg, "/data/https/caddy") + assert c["admin"] == {"disabled": True} and c["storage"]["root"] == "/data/https/caddy" + lens = c["apps"]["http"]["servers"]["lens"] + assert lens["listen"] == [":8443"] and lens["routes"][0]["match"] == [{"host": ["lens.example.com"]}] + assert lens["routes"][0]["handle"][-1] == {"handler": "reverse_proxy", "upstreams": [{"dial": "frontend:3000"}]} + assert lens["routes"][0]["handle"][0]["response"]["set"]["Strict-Transport-Security"] + # http:// goes to https:// at the port people use (published by install.sh), not the one in the container + redirect = c["apps"]["http"]["servers"]["redirect"] + assert redirect["listen"] == [":8080"] + assert redirect["routes"][0]["handle"][0]["headers"]["Location"] == ["https://{http.request.host}:8443{http.request.uri}"] + issuers = c["apps"]["tls"]["automation"]["policies"][0]["issuers"] + assert [i["ca"] for i in issuers] == [https.LETSENCRYPT, https.ZEROSSL] # both: ZeroSSL when Let's Encrypt can't + assert all(i["email"] == "a@example.com" and "challenges" not in i for i in issuers) + + cfg["https"].update(ca="letsencrypt", staging=True, challenge="tls-alpn", redirect_http=False) + c = https.caddy_config(cfg, "/s") + assert "redirect" not in c["apps"]["http"]["servers"] + [le] = c["apps"]["tls"]["automation"]["policies"][0]["issuers"] + assert le["ca"] == https.LETSENCRYPT_STAGING and le["challenges"] == {"http": {"disabled": True}} + + cfg["https"].update( + ca="zerossl", + challenge="dns", + domains=["*.example.com"], + dns_provider="route53", + dns_credentials=json.dumps({"access_key_id": "AK", "secret_access_key": "SK"}), + dns_resolvers=["1.1.1.1"], + propagation="2m", + eab_key_id="kid", + eab_hmac_key="mac", + ) + https.check(cfg) + [z] = https.caddy_config(cfg, "/s")["apps"]["tls"]["automation"]["policies"][0]["issuers"] + assert z["ca"] == https.ZEROSSL and z["external_account"] == {"key_id": "kid", "mac_key": "mac"} + assert z["challenges"] == { + "http": {"disabled": True}, + "tls-alpn": {"disabled": True}, + "dns": { + "provider": {"name": "route53", "access_key_id": "AK", "secret_access_key": "SK"}, + "resolvers": ["1.1.1.1"], + "propagation_timeout": "2m", + }, + } + assert https.package(cfg) == "github.com/caddy-dns/route53" + + +def test_caddy_runs_in_one_process_with_the_ports_and_shows_its_certificates(db, cfg, caddy): + cfg["https"] = {**cfg["https"], "enabled": True, "domains": ["lens.example.com"], "origin": "http://frontend:3000"} + one, two = runner(db, cfg, "worker-1"), runner(db, cfg, "worker-2") + try: + one.tick() + two.tick() # the lease is taken: it doesn't run a second Caddy + assert one.proc and not two.proc + wait(lambda: any("certificate obtained" in line for line in https.status(db, cfg)["log"])) + one.certs_at = 0.0 + one.tick() + s = https.status(db, cfg) + assert s["running"] and s["ports_open"] and s["process"] == "worker-1" and s["error"] is None + [cert] = s["certificates"] + assert cert["names"] == ["lens.example.com"] and cert["issuer"] == "Let's Encrypt" + assert cert["expires"] > (dt.datetime.now(dt.UTC) + dt.timedelta(days=80)).isoformat() + written = pathlib.Path(cfg["data_dir"]) / "https" / "caddy.json" + assert oct(written.stat().st_mode & 0o777) == "0o600" # it may hold the DNS provider's credentials + assert json.loads(caddy.read_text())["apps"]["tls"]["automation"]["policies"][0]["subjects"] == ["lens.example.com"] + # turned off: Caddy stops, and the process still says it has the ports + cfg["https"]["enabled"] = False + one.tick() + assert one.proc is None + s = https.status(db, cfg) + assert not s["running"] and s["ports_open"] + finally: + one.stop_process() + two.stop_process() + + +def test_a_domain_that_cant_be_verified_says_why(db, cfg, caddy): + cfg["https"] = {**cfg["https"], "enabled": True, "domains": ["broken.example.com"], "origin": "http://frontend:3000"} + r = runner(db, cfg) + try: + r.tick() + s = wait(lambda: (st := https.status(db, cfg))["error"] and st) + assert "no valid A records" in s["error"] and "broken.example.com" in s["error"] and not s["certificates"] + cfg["https"]["domains"] = ["lens.example.com"] + r.tick() # a changed setting starts Caddy again straight away, and the error goes once it has the certificate + wait(lambda: https.status(db, cfg)["error"] is None) + finally: + r.stop_process() + + +def test_a_dns_provider_needs_a_caddy_with_its_plugin(db, cfg, caddy, monkeypatch): + cfg["https"] = { + **cfg["https"], + "enabled": True, + "domains": ["*.example.com"], + "challenge": "dns", + "dns_provider": "cloudflare", + "origin": "http://frontend:3000", + } + fetched = [] + + def fetch(c, say=None): + fetched.append(https.package(c)) + raise https.HttpsError("couldn't download Caddy from caddyserver.com: offline") + + monkeypatch.setattr(https, "fetch_binary", fetch) + assert https.binary(cfg) is None # the caddy on PATH has no cloudflare plugin + r = runner(db, cfg) + try: + r.tick() + assert fetched == ["github.com/caddy-dns/cloudflare"] and r.proc is None + assert "offline" in https.status(db, cfg)["error"] + monkeypatch.setenv("FAKE_MODULES", "dns.providers.cloudflare") + https._MODULES.clear() + assert https.binary(cfg) # one that has it is used as it is + finally: + r.stop_process() + + +def test_nothing_runs_where_the_ports_arent(db, cfg, monkeypatch): + monkeypatch.delenv("LENS_HTTPS_LISTEN", raising=False) + assert https.start(db, lambda: cfg, None, "api-1") is None + + +def test_passkeys_and_email_links_use_the_https_address(app, client, db, cfg, monkeypatch): + cfg["https"] = {**cfg["https"], "enabled": True, "domains": ["lens.example.com"]} + # Caddy passes the browser's host on; the browser is on https://lens.example.com + o = client.post("/api/v1/auth/passkey/options", headers={"x-forwarded-host": "lens.example.com", "x-forwarded-proto": "http"}) + assert o.status_code == 200 and o.json()["options"]["rpId"] == "lens.example.com" + assert email.app_url(cfg) == "https://lens.example.com" + monkeypatch.setenv("LENS_HTTPS_PORT", "8443") + o = client.post("/api/v1/auth/passkey/options", headers={"x-forwarded-host": "lens.example.com:8443", "x-forwarded-proto": "https"}) + assert o.json()["options"]["rpId"] == "lens.example.com" + assert email.app_url(cfg) == "https://lens.example.com:8443" + cfg["https"] = {**cfg["https"], "enabled": False} + db.next_id("settings") # the app's settings are read again + o = client.post("/api/v1/auth/passkey/options", headers={"x-forwarded-host": "lens.example.com", "x-forwarded-proto": "https"}) + assert o.json()["options"]["rpId"] == "localhost" diff --git a/install.sh b/install.sh index f1425683f..fff61e288 100755 --- a/install.sh +++ b/install.sh @@ -19,6 +19,9 @@ # GITHUB_TOKEN a GitHub token to use when downloading (not needed; avoids GitHub's rate limits) # LENS_NO_OPEN 1: don't open a browser # LENS_YES 1: install what Lens needs without asking (it only asks at a terminal) +# LENS_HTTPS 1: publish the HTTPS and HTTP ports (443, 80; LENS_HTTPS_PORT / LENS_HTTP_PORT for others) so Lens +# can serve https:// with certificates it gets itself, set up in Settings › HTTPS (docs/https.md); +# 0: don't (default: as the last run left it, else 0) # LENS_UPGRADE_AGENT 0: don't set up the agent that lets the web app install new releases (docs/updates.md) # LENS_WIZARD 1: answer everything in the web installer, in the browser (the default for a first install at a # terminal, where Python 3 is there); 0: here in the terminal @@ -307,6 +310,16 @@ if [ "$LENS_PUBLIC" = 1 ]; then [ -n "$ip" ] && host="$ip" fi +# --- HTTPS (docs/https.md): the ports Caddy answers on, run by Lens while Settings › HTTPS has it on --------------- +if [ -z "${LENS_HTTPS:-}" ] && [ -f .env ]; then # as the last run left it + LENS_HTTPS="$(grep '^LENS_HTTPS=' .env | cut -d= -f2- || true)" +fi +[ "${LENS_HTTPS:-0}" = 1 ] || LENS_HTTPS=0 +HTTPS_PORT="${LENS_HTTPS_PORT:-$( [ -f .env ] && grep '^LENS_HTTPS_PORT=' .env | cut -d= -f2- || true)}" +HTTP_PORT="${LENS_HTTP_PORT:-$( [ -f .env ] && grep '^LENS_HTTP_PORT=' .env | cut -d= -f2- || true)}" +HTTPS_PORT="${HTTPS_PORT:-443}" +HTTP_PORT="${HTTP_PORT:-80}" + # --- Secrets, once: changing them later would sign everyone out and make stored credentials unreadable ------------ setenv() { # key value: replace the line, or add it if grep -q "^$1=" .env; then @@ -330,6 +343,11 @@ setenv LENS_BIND "$bind" setenv LENS_PORT "$PORT" setenv FRONTEND_URL "http://$host:$PORT" setenv LENS_REPO "$REPO" +setenv LENS_HTTPS "$LENS_HTTPS" +if [ "$LENS_HTTPS" = 1 ]; then + setenv LENS_HTTPS_PORT "$HTTPS_PORT" + setenv LENS_HTTP_PORT "$HTTP_PORT" +fi setenv LENS_DIR "$DIR" # from the web installer (docs/configuration.md): the first namespace, and settings Lens saves once at its first start for k in LENS_NAMESPACE LENS_SETUP_SETTINGS; do @@ -373,8 +391,11 @@ if [ "$BACKEND" = microsandbox ]; then set -- run -d -q --name lens --replace -c "$cpus" -m "${mem}M" --root-disk 16G --net public,private,host \ -p "$bind:$PORT:3000" \ -p "$sensors:${LENS_MQTT_PORT:-1883}:1883" \ - -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/udp" -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/tcp" \ - -v "$PWD:/lens:ro" -v "$PWD/.lens-control:/control" --mount-named lens-data:/data \ + -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/udp" -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/tcp" + if [ "$LENS_HTTPS" = 1 ]; then # Caddy listens on 8443 and 8080 inside (docs/https.md) + set -- "$@" -p "${LENS_HTTPS_BIND:-0.0.0.0}:$HTTPS_PORT:8443" -p "${LENS_HTTPS_BIND:-0.0.0.0}:$HTTP_PORT:8080" + fi + set -- "$@" -v "$PWD:/lens:ro" -v "$PWD/.lens-control:/control" --mount-named lens-data:/data \ --entrypoint /bin/sh "$image" -- /lens/microsandbox/run.sh say "Starting Lens in a microsandbox (the first start installs and builds it, which takes a while)..." # The command that starts it, kept for a restart of the machine: a sandbox started again (msb start) boots without @@ -412,10 +433,13 @@ if [ "$BACKEND" = microsandbox ]; then PLIST fi else + files="-f docker-compose.prod.yml" + [ "$LENS_HTTPS" = 1 ] && files="$files -f docker-compose.https.yml" # the HTTPS ports (docs/https.md) say "Building and starting Lens (the first build takes a few minutes)..." - $COMPOSE -f docker-compose.prod.yml up -d --build - LOGS="$COMPOSE -f docker-compose.prod.yml logs" - STOP="$COMPOSE -f docker-compose.prod.yml down" + # shellcheck disable=SC2086 + $COMPOSE $files up -d --build + LOGS="$COMPOSE $files logs" + STOP="$COMPOSE $files down" WAIT=300 fi diff --git a/microsandbox/run.sh b/microsandbox/run.sh index e3f6ee7e8..5949f0bcc 100755 --- a/microsandbox/run.sh +++ b/microsandbox/run.sh @@ -123,6 +123,8 @@ export ARCHIVE_ALLOWED_HOSTS="${ARCHIVE_ALLOWED_HOSTS:-localhost,127.0.0.1}" export LENS_TUNNEL_ORIGIN=http://127.0.0.1:3000 export LENS_TRUSTED_PROXY_HOSTS=localhost export LENS_WEB_PROXY_HOSTS=localhost +# HTTPS (docs/https.md): install.sh published 443 and 80 to the sandbox's 8443 and 8080, where Caddy listens +[ "${LENS_HTTPS:-0}" = 1 ] && export LENS_HTTPS_LISTEN=1 LENS_HTTPS_PORT="${LENS_HTTPS_PORT:-443}" export API_BASE_URL=http://127.0.0.1:8000 export AUTH_TRUST_HOST=true NODE_ENV=production NEXT_TELEMETRY_DISABLED=1 export HOME=/home/lens XDG_CACHE_HOME="$DATA/.cache" diff --git a/mkdocs.yml b/mkdocs.yml index 10d285ab0..0a2d90170 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -69,6 +69,7 @@ nav: - Components: components.md - Updates and backups: updates.md - Remote access: remote-access.md + - HTTPS: https.md - Agents and apps: - MCP server: mcp.md - API: api.md diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index c0c5fe61b..81cff73c6 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1316,6 +1316,16 @@ export type { HomeSuggestionsResponses, HookItem, HookPushed, + HttpsCertificate, + HttpsProvider, + HttpsProviderField, + HttpsProvidersData, + HttpsProvidersResponse, + HttpsProvidersResponses, + HttpsStatus, + HttpsStatusData, + HttpsStatusResponse, + HttpsStatusResponses, HttpValidationError, HubLogin, HubLoginCreate, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 9fd8e329a..4fd670a0b 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -659,6 +659,10 @@ import type { HomeSuggestionsData, HomeSuggestionsErrors, HomeSuggestionsResponses, + HttpsProvidersData, + HttpsProvidersResponses, + HttpsStatusData, + HttpsStatusResponses, ImportArchiveData, ImportArchiveErrors, ImportArchiveResponses, @@ -2559,6 +2563,36 @@ export class Admin { }); } + /** + * Https Status + * + * How HTTPS (Settings › HTTPS) is doing: whether a process has the HTTPS ports and runs Caddy, the certificates it + * holds and when they run out, and Caddy's last lines. + */ + public static httpsStatus( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/settings/https/status", + ...options, + }); + } + + /** + * Https Providers + * + * The DNS providers Settings › HTTPS lists for the DNS challenge, with the fields each needs. Any other caddy-dns + * plugin works too, by name. + */ + public static httpsProviders( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/settings/https/providers", + ...options, + }); + } + /** * Test Bridge * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index cd5e91ce6..f27b999db 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -6023,6 +6023,128 @@ export type HookPushed = { [key: string]: unknown; }; +/** + * HttpsCertificate + */ +export type HttpsCertificate = { + /** + * Names + * + * the domain names it's for + */ + names: Array; + /** + * Expires + * + * when it runs out (Caddy renews it about a month before) + */ + expires: string; + /** + * Issuer + * + * the certificate authority that issued it + */ + issuer: string; + [key: string]: unknown; +}; + +/** + * HttpsProvider + */ +export type HttpsProvider = { + /** + * Name + * + * the caddy-dns plugin's name, like cloudflare + */ + name: string; + /** + * Label + */ + label: string; + /** + * Fields + */ + fields: Array; + [key: string]: unknown; +}; + +/** + * HttpsProviderField + */ +export type HttpsProviderField = { + /** + * Name + */ + name: string; + /** + * Label + */ + label: string; + /** + * Secret + */ + secret: boolean; + [key: string]: unknown; +}; + +/** + * HttpsStatus + */ +export type HttpsStatus = { + /** + * Enabled + */ + enabled: boolean; + /** + * Running + * + * Caddy is running in a server process + */ + running: boolean; + /** + * Ports Open + * + * a server process has the HTTPS ports (install.sh opened them) + */ + ports_open: boolean; + /** + * Url + * + * the https:// address people open Lens at + */ + url?: string | null; + /** + * Certificates + */ + certificates?: Array; + /** + * Error + * + * why there's no certificate yet, when there isn't + */ + error?: string | null; + /** + * Log + * + * Caddy's last lines + */ + log?: Array; + /** + * Origin + * + * where Caddy sends visitors: the web app, as the server reaches it + */ + origin: string; + /** + * Process + * + * the server process with the HTTPS ports + */ + process?: string | null; + [key: string]: unknown; +}; + /** * HubLogin */ @@ -20026,6 +20148,40 @@ export type TunnelStatusResponses = { export type TunnelStatusResponse = TunnelStatusResponses[keyof TunnelStatusResponses]; +export type HttpsStatusData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/https/status"; +}; + +export type HttpsStatusResponses = { + /** + * Successful Response + */ + 200: HttpsStatus; +}; + +export type HttpsStatusResponse = HttpsStatusResponses[keyof HttpsStatusResponses]; + +export type HttpsProvidersData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/https/providers"; +}; + +export type HttpsProvidersResponses = { + /** + * Response Admin-Https Providers + * + * Successful Response + */ + 200: Array; +}; + +export type HttpsProvidersResponse = HttpsProvidersResponses[keyof HttpsProvidersResponses]; + export type TestBridgeData = { body?: never; path?: never; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 0504d1752..12d041177 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2466,6 +2466,54 @@ } } }, + "/api/v1/settings/https/status": { + "get": { + "tags": [ + "admin" + ], + "summary": "Https Status", + "description": "How HTTPS (Settings \u203a HTTPS) is doing: whether a process has the HTTPS ports and runs Caddy, the certificates it\nholds and when they run out, and Caddy's last lines.", + "operationId": "https_status", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HttpsStatus" + } + } + } + } + } + } + }, + "/api/v1/settings/https/providers": { + "get": { + "tags": [ + "admin" + ], + "summary": "Https Providers", + "description": "The DNS providers Settings \u203a HTTPS lists for the DNS challenge, with the fields each needs. Any other caddy-dns\nplugin works too, by name.", + "operationId": "https_providers", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/HttpsProvider" + }, + "type": "array", + "title": "Response Admin-Https Providers" + } + } + } + } + } + } + }, "/api/v1/settings/bridge/test": { "post": { "tags": [ @@ -36114,6 +36162,171 @@ ], "title": "HookPushed" }, + "HttpsCertificate": { + "properties": { + "names": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Names", + "description": "the domain names it's for" + }, + "expires": { + "type": "string", + "title": "Expires", + "description": "when it runs out (Caddy renews it about a month before)" + }, + "issuer": { + "type": "string", + "title": "Issuer", + "description": "the certificate authority that issued it" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "names", + "expires", + "issuer" + ], + "title": "HttpsCertificate" + }, + "HttpsProvider": { + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "the caddy-dns plugin's name, like cloudflare" + }, + "label": { + "type": "string", + "title": "Label" + }, + "fields": { + "items": { + "$ref": "#/components/schemas/HttpsProviderField" + }, + "type": "array", + "title": "Fields" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "name", + "label", + "fields" + ], + "title": "HttpsProvider" + }, + "HttpsProviderField": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "label": { + "type": "string", + "title": "Label" + }, + "secret": { + "type": "boolean", + "title": "Secret" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "name", + "label", + "secret" + ], + "title": "HttpsProviderField" + }, + "HttpsStatus": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled" + }, + "running": { + "type": "boolean", + "title": "Running", + "description": "Caddy is running in a server process" + }, + "ports_open": { + "type": "boolean", + "title": "Ports Open", + "description": "a server process has the HTTPS ports (install.sh opened them)" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url", + "description": "the https:// address people open Lens at" + }, + "certificates": { + "items": { + "$ref": "#/components/schemas/HttpsCertificate" + }, + "type": "array", + "title": "Certificates" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "why there's no certificate yet, when there isn't" + }, + "log": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Log", + "description": "Caddy's last lines" + }, + "origin": { + "type": "string", + "title": "Origin", + "description": "where Caddy sends visitors: the web app, as the server reaches it" + }, + "process": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Process", + "description": "the server process with the HTTPS ports" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "running", + "ports_open", + "origin" + ], + "title": "HttpsStatus" + }, "HubLogin": { "properties": { "id": { diff --git a/threat-model.md b/threat-model.md index 5a1bc3ece..ed00401af 100644 --- a/threat-model.md +++ b/threat-model.md @@ -105,7 +105,7 @@ optional listeners (MQTT, syslog, tunnel, local model server) run inside workers | Ingest & processing (Ingest) | `routes/{uploads,files,imports}.py`, `domain/{ingest,documents,convert,video,uploads}.py`, `templates.py`, `render.py` | child processes (ffmpeg, LibreOffice, Chromium, poppler, Tesseract), files | In | | Query engines (Query) | `domain/cypher.py`, `domain/rdf.py` (SPARQL), `domain/rdf_import.py`, `domain/search.py` | DB | In | | Assistant & extensions (Assistant) | `domain/{chat,ai_tools,ops_tools,extensions,tool_nodes,flow,custom_nodes,code_tools}.py`, MCP write tools | network (model servers), child process (code tools) | In | -| Optional services, off by default (Optional) | `domain/{sensors,mqtt,syslog}.py`, `tunnel.py`, `local_llm.py`, `components.py`, `telemetry.py` | listening sockets, downloads, child processes | In | +| Optional services, off by default (Optional) | `domain/{sensors,mqtt,syslog}.py`, `tunnel.py`, `https.py`, `local_llm.py`, `components.py`, `telemetry.py` | listening sockets, downloads, child processes | In | | Packaging variants | `docker-compose.prod.yml`, `install.sh`, `cloudron/`, `packaging/{synology,qnap}`, `proxmox/` | host | §1.6 | | Dev and CI tooling, samples | `docker-compose.yml`, `make dev`, `.github/`, `ci/`, `examples/`, `overrides/`, `prod-frontend-deploy.yml`, `watcher.py`, `commands/`, `fastapi_backend/tests/` | host | Out (§1.3) | @@ -240,6 +240,7 @@ machine requires HTTPS, and a changed DB credential when SurrealDB runs as its o | Chromium sandbox | off as root or without user namespaces | the `netguard` proxy is the only barrier | Yes | *(documented, docs/configuration.md "runs with its sandbox where it can")* | | `sensors.enabled`, `sensors.mqtt_anonymous` | off (`bind` `0.0.0.0`); off | opens MQTT and syslog listeners; anonymous MQTT publish | Yes; anonymous is the admin's choice | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | | `tunnel.mode`, `local_llm.enabled`, `telemetry` | off | outbound tunnel (Cloudflare terminates TLS, Q30); local model server; OTLP export | Yes | *(documented, docs/remote-access.md "It's off until an admin turns it on")* | +| `https.enabled` (and install.sh's `LENS_HTTPS=1`) | off | Caddy, fetched at run time, terminates TLS on ports 443/80 in the worker and proxies to the web app; the DNS provider's credentials are written to `data_dir/https/caddy.json` (mode 600), not a command line | Yes | *(documented, docs/https.md "Off by default")* | | `components.auto` | on | unpinned-hash pip installs at runtime | Yes | *(maintainer, 2026-10)* | | `encryption.files` | on for new archives | file encryption at rest | Yes | *(documented, docs/encryption.md "A new archive turns this on at its first start")* | | `ai.extensions`, `ai.disabled_tools` | on, none | turn assistant extensions or named tools off | Yes | *(documented, docs/assistant.md "Extending the assistant")* | diff --git a/threat-model.yaml b/threat-model.yaml index a80ae5597..c92375912 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:42d9bc92f48ce93ea2d58412a61b086dde8ab4522af95f628304d1b33919832f" +prose_version: "threat-model.md@sha256:c967c66629827d692bf760cc49d929494430dcd778a3dc96cc5b2d27a8078807" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} @@ -1024,6 +1024,14 @@ build_flags: affects_properties: - {property_id: P-SENSORS, effect: narrows} provenance: {kind: documented, source: 'docs/sensors.md "Sensors are off until an admin turns them on"'} + - name: "https.enabled" + default: "off" + security_relevant: true + maintainer_stance: supported + x-scope: "Caddy, fetched at run time, terminates TLS on 443/80 in the worker (install.sh LENS_HTTPS=1 publishes them) and proxies to the web app; DNS provider credentials go to data_dir/https/caddy.json (mode 600), not a command line" + affects_properties: + - {property_id: P-CHILD-SECRETS, effect: preserves} + provenance: {kind: documented, source: 'docs/https.md "Off by default"'} - name: "tunnel.mode, local_llm.enabled, telemetry" default: "off" security_relevant: true