From 0ef8519f4003133c08669c45a23c636a4ca7d391 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 22:48:11 +0000 Subject: [PATCH 1/2] feat: turn Prometheus and Grafana on from the web app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "Charts and history" switch on System health and in Settings → Telemetry asks the host upgrade agent to add or remove the monitoring compose profile and start or stop Prometheus, node-exporter and Grafana. Turning it on also turns /metrics on with a generated scrape token, which Prometheus reads from the control folder, and links System health to Grafana on this host. API: GET/PUT /admin/monitoring; MCP: monitoring, set_monitoring. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XCh396xzDRUrSuH1xohcXR --- docker-compose.prod.yml | 11 +- docs/prometheus.md | 35 ++-- docs/updates.md | 5 + fastapi_backend/app/api/mcp_tools.py | 52 +++++- fastapi_backend/app/api/v1/routes/admin.py | 3 + fastapi_backend/app/api/v1/routes/updates.py | 25 ++- fastapi_backend/app/domain/monitoring.py | 131 ++++++++++++++ fastapi_backend/app/schemas/updates.py | 15 ++ fastapi_backend/tests/api/test_mcp.py | 11 ++ fastapi_backend/tests/api/test_monitoring.py | 128 ++++++++++++++ monitoring/prometheus.yml | 2 +- .../__tests__/monitoring-switch.test.ts | 30 ++++ nextjs-frontend/app/openapi-client/index.ts | 10 ++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 39 ++++ .../app/openapi-client/types.gen.ts | 102 +++++++++++ nextjs-frontend/components/admin/health.tsx | 22 +-- .../components/admin/monitoring-switch.tsx | 80 +++++++++ .../components/settings/bodies.tsx | 7 +- nextjs-frontend/components/settings/model.ts | 2 +- nextjs-frontend/openapi.json | 167 ++++++++++++++++++ packaging/host/lens-upgrade.sh | 67 ++++++- threat-model.md | 1 + 22 files changed, 907 insertions(+), 38 deletions(-) create mode 100644 fastapi_backend/app/domain/monitoring.py create mode 100644 fastapi_backend/tests/api/test_monitoring.py create mode 100644 nextjs-frontend/__tests__/monitoring-switch.test.ts create mode 100644 nextjs-frontend/components/admin/monitoring-switch.tsx diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2d34abc74..0ddffe3e4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -189,9 +189,10 @@ services: networks: [lens] # Prometheus, node-exporter and Grafana (docs/prometheus.md): charts of Lens and this machine over time, ready made. - # Opt-in: in .env set COMPOSE_PROFILES=monitoring, LENS_PROMETHEUS=on, LENS_PROMETHEUS_TOKEN (openssl rand -hex 32) - # and LENS_GRAFANA_URL=http://localhost:3001, then `docker compose up -d`. Grafana listens where the web app does - # (LENS_BIND), on LENS_GRAFANA_PORT (3001); anyone who reaches it may view, and only the admin may change things. + # Opt-in: Admin → System health or Settings → Telemetry turns it on (the upgrade agent adds `monitoring` to + # COMPOSE_PROFILES and starts these). Prometheus scrapes with the token Lens writes to .lens-control/prometheus-token, + # or LENS_PROMETHEUS_TOKEN when set. Grafana listens where the web app does (LENS_BIND), on LENS_GRAFANA_PORT (3001); + # anyone who reaches it may view, and only the admin may change things. prometheus: image: prom/prometheus:${PROMETHEUS_VERSION:-v3.5.0} profiles: [monitoring] @@ -199,7 +200,8 @@ services: entrypoint: ["/bin/sh", "-c"] command: - >- - printf %s "$$LENS_PROMETHEUS_TOKEN" > /prometheus/lens-token && + { if [ -n "$$LENS_PROMETHEUS_TOKEN" ]; then printf %s "$$LENS_PROMETHEUS_TOKEN" > /prometheus/lens-token; + else ln -sf /control/prometheus-token /prometheus/lens-token; fi; } && exec /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=$${LENS_METRICS_DAYS:-30}d environment: @@ -207,6 +209,7 @@ services: LENS_METRICS_DAYS: ${LENS_METRICS_DAYS:-30} volumes: - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./.lens-control:/control:ro # prometheus-token, written by Lens when monitoring is turned on - prometheus-data:/prometheus restart: unless-stopped networks: [lens] diff --git a/docs/prometheus.md b/docs/prometheus.md index 647b5d930..f505cc38f 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -13,21 +13,26 @@ endpoint: Lens sends nothing anywhere. For pushing traces and metrics to an Open Lens doesn't build its own charts: with Docker, the `monitoring` profile runs [Prometheus](https://prometheus.io/), [node-exporter](https://github.com/prometheus/node_exporter) (the machine's CPU, memory, disk and network) and [Grafana](https://grafana.com/oss/grafana/) next to Lens, already connected, with the Lens dashboard as Grafana's home -page. Add to `.env`: - -```sh -COMPOSE_PROFILES=monitoring # with others, comma-separated (matterbridge,monitoring) -LENS_PROMETHEUS=on -LENS_PROMETHEUS_TOKEN=3f9c... # openssl rand -hex 32 -LENS_GRAFANA_URL=http://localhost:3001 # the address you open Grafana at; System health links to it -# GRAFANA_ADMIN_PASSWORD=... # optional; otherwise admin/admin, and Grafana asks to change it -# LENS_METRICS_DAYS=30 # how long Prometheus keeps history -``` - -then `docker compose -f docker-compose.prod.yml up -d`. Grafana listens where the web app does (`LENS_BIND`: this -machine only unless Lens was opened to the network) on port 3001 (`LENS_GRAFANA_PORT`). Anyone who reaches it can view -the dashboards, which hold no personal content; only Grafana's admin can change them. Prometheus and node-exporter are -reachable only inside Lens's Docker network. +page. + +**Turn it on in the web app**: **Admin → System health** or **Settings → Telemetry → Prometheus**, switch on +**Charts and history**. That's all: Lens turns `/metrics` on, makes a scrape token if there is none, and the machine's +upgrade agent (the one that [installs new releases](updates.md)) adds `monitoring` to `COMPOSE_PROFILES` in `.env` and +starts the three containers; the page shows how it goes and then links to Grafana. Off stops and removes them (their +history stays in the `prometheus-data` volume) and turns `/metrics` off again if this switch turned it on. Agents do +the same with `PUT /api/v1/admin/monitoring` (`{"on": true}`) or the MCP tools `monitoring` and `set_monitoring`. + +Grafana listens where the web app does (`LENS_BIND`: this machine only unless Lens was opened to the network) on port +3001 (`LENS_GRAFANA_PORT`), and System health links to it on the address you opened Lens at, unless +**Grafana address** says otherwise. Anyone who reaches it can view the dashboards, which hold no personal content; +only Grafana's admin can change them (admin/admin until changed, or `GRAFANA_ADMIN_PASSWORD`). Prometheus and +node-exporter are reachable only inside Lens's Docker network. Prometheus reads the scrape token from +`.lens-control/prometheus-token`, which Lens keeps in step with Settings (or from `LENS_PROMETHEUS_TOKEN` when set), and +keeps 30 days of history (`LENS_METRICS_DAYS`). + +Without an agent (Lens not installed with `install.sh`, or it couldn't set one up) the switch shows the one line to +run on the machine instead: `docker compose -f docker-compose.prod.yml --profile monitoring up -d`. An install in a +microsandbox has no Docker to run them in. **Admin → System health** shows the server now (CPU, memory, load, uptime) and links to Grafana for history. diff --git a/docs/updates.md b/docs/updates.md index a1ddcd754..509cf2007 100644 --- a/docs/updates.md +++ b/docs/updates.md @@ -45,6 +45,11 @@ The containers see the control folder as `/control` (`LENS_CONTROL_DIR`). `POST The agent installs only a release tag (`vX.Y.Z`) or the branch the install already follows, whoever writes the request, and the API refuses a release older than the one running (restore a backup instead). +The same agent turns [Charts and history](prometheus.md#prometheus-and-grafana-ready-made) on and off: a request with +`action=services` and `monitoring=on` or `off` (in the same inbox file, the one every agent watches) adds or removes +`monitoring` in `COMPOSE_PROFILES` in `.env` and starts or stops Prometheus, node-exporter and Grafana, and nothing +else; it reports in `services-status` and `services.log`. + With no agent (Lens wasn't installed with install.sh, the machine has neither systemd nor launchd, or install.sh ran with `LENS_UPGRADE_AGENT=0`), the API gives the one line to run on the machine instead (`manual`). diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 6c9b03988..a61aff51f 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -27,7 +27,22 @@ from app.api.v1.routes import recordings as recording_routes from app.api.v1.routes import search as search_routes from app.api.v1.routes import topics as topic_routes -from app.domain import analyze, auth, blobs, feeds, filetree, graph_history, library, notebook, rdf, render, settings, sources, store +from app.domain import ( + analyze, + auth, + blobs, + feeds, + filetree, + graph_history, + library, + monitoring, + notebook, + rdf, + render, + settings, + sources, + store, +) from app.domain import entities as ents from app.domain import speakers as spk from app.domain import topics as topicmod @@ -1558,3 +1573,38 @@ def set_file_storage( settings.save(ctx.db, ctx.request.app.state.archive.base, "files", changes, ctx.user.email) auth.audit(ctx.db, ctx.user.as_audit(), "settings.save", "files", sorted(changes) + ["mcp"]) return {"saved": changes, "check": checked} + + +# ---------- monitoring: Prometheus and Grafana next to Lens (admins) ---------- +@tool( + "monitoring", + "Monitoring", + "Whether Prometheus, node-exporter and Grafana run next to Lens (charts of Lens and this machine over time), how " + "the machine's last change went, and Grafana's port. Admins only.", +) +def monitoring_status(ctx: Context) -> dict[str, Any]: + if not ctx.user.admin: + raise ToolError("only admins see or change monitoring") + return monitoring.view(ctx.db) | {"settings": f"{ctx.web}/settings/telemetry"} + + +@tool( + "set_monitoring", + "Turn monitoring on or off", + "Have this machine start (on true) or stop Prometheus, node-exporter and Grafana. On also turns /metrics on with " + "a scrape token when it has none. The machine's upgrade agent does it; monitoring follows along. Admins only, with " + "a token that has the write scope.", + Arg("on", "boolean", "start (true) or stop (false)", required=True), + writes=True, +) +def set_monitoring(ctx: Context, on: bool) -> dict[str, Any]: + if not ctx.user.admin: + raise ToolError("only admins see or change monitoring") + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to change monitoring") + try: + got = monitoring.turn(ctx.db, ctx.request.app.state.archive.base, ctx.cfg, bool(on), by=ctx.user.email) + except monitoring.MonitoringError as e: + raise ToolError(str(e)) from None + auth.audit(ctx.db, ctx.user.as_audit(), "lens.monitoring", None, {"on": bool(on), "via": "mcp"}) + return got diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index 4f6cf609a..369a562d3 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -26,6 +26,7 @@ local_llm, machine, matterbridge, + monitoring, semantic, settings, sources, @@ -92,6 +93,8 @@ def update_settings(section: str, body: dict[str, Any], user: AdminWriter, reque with domain_errors(): settings.save(db, request.app.state.archive.base, section, body, user.email) auth.audit(db, user.as_audit(), "settings.save", section, sorted(body)) + if section == "telemetry" and "prometheus_token" in body: + monitoring.sync_token(request.app.state.settings.current()) # the monitoring Prometheus scrapes with the new one if section == "encryption" and "files" in body and bool(body["files"]) != bool(files_were): cfg = request.app.state.settings.current() keyring.convert(db, cfg, bool(cfg["encryption"].get("files"))) # the files already kept follow, in the background diff --git a/fastapi_backend/app/api/v1/routes/updates.py b/fastapi_backend/app/api/v1/routes/updates.py index 669cfbe12..0823d8b52 100644 --- a/fastapi_backend/app/api/v1/routes/updates.py +++ b/fastapi_backend/app/api/v1/routes/updates.py @@ -4,14 +4,14 @@ import contextlib -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Request from fastapi.responses import FileResponse from app.api.deps import AdminReader, AdminWriter, Cfg, Db, domain_errors -from app.domain import auth, backups, updates +from app.domain import auth, backups, monitoring, updates from app.schemas.backups import BackupSchedule, RestoreWork from app.schemas.common import Ok -from app.schemas.updates import Backup, Backups, UpdateStatus, UpgradeRequest +from app.schemas.updates import Backup, Backups, Monitoring, MonitoringRequest, UpdateStatus, UpgradeRequest router = APIRouter(tags=["updates"]) @@ -46,6 +46,25 @@ def start_upgrade(user: AdminWriter, db: Db, cfg: Cfg, body: UpgradeRequest | No return UpdateStatus.model_validate(updates.view(db, cfg)) +@router.get("/admin/monitoring") +def get_monitoring(user: AdminReader, db: Db) -> Monitoring: + """Whether Prometheus, node-exporter and Grafana run next to Lens (the monitoring profile), and how the machine's + last change went.""" + return Monitoring.model_validate(monitoring.view(db)) + + +@router.put("/admin/monitoring", status_code=202) +def set_monitoring(body: MonitoringRequest, user: AdminWriter, request: Request, db: Db, cfg: Cfg) -> Monitoring: + """Have this machine start or stop Prometheus, node-exporter and Grafana. On also turns /metrics on with a scrape + token when it has none. GET /admin/monitoring follows along.""" + try: + got = monitoring.turn(db, request.app.state.archive.base, cfg, body.on, by=user.email) + except monitoring.MonitoringError as e: + raise HTTPException(409, str(e)) from None + auth.audit(db, user.as_audit(), "lens.monitoring", None, {"on": body.on}) + return Monitoring.model_validate(got) + + @router.get("/admin/backups") def list_backups(user: AdminReader, db: Db, cfg: Cfg) -> Backups: """The database backups in /backups, newest first; those being written; the restore running here (or diff --git a/fastapi_backend/app/domain/monitoring.py b/fastapi_backend/app/domain/monitoring.py new file mode 100644 index 000000000..552b5e93e --- /dev/null +++ b/fastapi_backend/app/domain/monitoring.py @@ -0,0 +1,131 @@ +"""Turning the monitoring profile (Prometheus, node-exporter, Grafana) on and off from the web app (docs/prometheus.md). + +The containers can't start other containers, so the machine's upgrade agent does it, as it installs releases +(updates.py, packaging/host/lens-upgrade.sh). Turning monitoring on makes sure /metrics is on with a scrape token +(generated when none is set), writes that token where the monitoring Prometheus reads it (`/prometheus-token`), +then drops a request in the agent's inbox: the agent adds `monitoring` to COMPOSE_PROFILES in the install's .env and +starts the three containers; off removes it and stops them. How it went is in `/services-status`. + +The request goes in the inbox under the same name as an upgrade (`inbox/upgrade`, with `action=services`), because that +file is what every agent installed so far watches for; the agent never installs anything for it. +""" + +from __future__ import annotations + +import datetime as dt +import os +import secrets + +from . import settings, store, updates + +ROW = store.R("app_service", "monitoring") +ACTIVE = ("queued", "running") +TOKEN_FILE = "prometheus-token" +STATUS_FILE = "services-status" +GRAFANA_PORT = 3001 + + +class MonitoringError(RuntimeError): + """Monitoring can't be turned on or off; the message says why, for the admin.""" + + +def _row(db): + return db.one("SELECT * OMIT id FROM $r", r=ROW) or {} + + +def _put(db, run): + db.q("UPSERT $r CONTENT $d", r=ROW, d=run) + + +def manual(): + """The line to run on the machine when there is no agent.""" + where = os.environ.get("LENS_HOST_DIR") or "" + cd = f"cd {where} && " if where and updates.REF.match(where.lstrip("/~")) else "" + return f"{cd}docker compose -f docker-compose.prod.yml --profile monitoring up -d" + + +def view(db): + """Whether monitoring is on (as last asked), and how the agent's last run went.""" + run = _row(db) + out = { + "on": bool(run.get("on")), + "state": run.get("state"), + "error": run.get("error"), + "asked_at": run.get("asked_at"), + "finished_at": None, + "grafana_port": GRAFANA_PORT, + "agent": updates.agent(), + "manual": manual(), + } + d = updates.control_dir() + if d and run: + s = updates._kv(d / STATUS_FILE) + if s.get("id") == run.get("run"): + out["state"] = s.get("state") or out["state"] + out["finished_at"] = s.get("finished_at") + if s.get("grafana_port", "").isdigit(): + out["grafana_port"] = int(s["grafana_port"]) + if s.get("state") == "failed": + out["error"] = s.get("error") or f"the machine couldn't change it (exit {s.get('code', '?')})" + if out["state"] == "queued" and (d / "inbox" / "upgrade").exists(): + try: + waited = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(str(run.get("asked_at")))).total_seconds() + except ValueError: + waited = 0 + if waited > updates.PICKUP_SECONDS: + out["error"] = "the machine hasn't picked it up yet: is its upgrade agent running?" + return out + + +def sync_token(cfg, create=False): + """Keep the token the monitoring Prometheus scrapes with the same as /metrics's, once monitoring was turned on.""" + d = updates.control_dir() + token = ((cfg or {}).get("telemetry") or {}).get("prometheus_token") + if not d or not d.is_dir(): + return + f = d / TOKEN_FILE + if not create and not f.exists(): + return + if not token: + f.unlink(missing_ok=True) + return + part = d / f".{TOKEN_FILE}.{secrets.token_hex(4)}" + part.write_text(str(token), encoding="utf-8") + part.chmod(0o644) # Prometheus runs as its own user; the token reads metrics only + part.replace(f) + + +def turn(db, base, cfg, on, by=None): + """Ask the machine to start (on) or stop the monitoring containers; returns view().""" + if not updates.agent(): + raise MonitoringError("this machine has no upgrade agent to start containers: run the line shown on it instead") + d = updates.control_dir() + if (d / "inbox" / "upgrade").exists(): + raise MonitoringError("the machine is busy with another request (an upgrade?): try again in a minute") + run, now_ = _row(db), view(db) + if now_["state"] in ACTIVE and not now_["error"]: + raise MonitoringError("the machine is still changing it: wait for that to finish") + t = cfg.get("telemetry") or {} + locked = settings.locked("telemetry") + changes, ours = {}, bool(run.get("turned_on_metrics")) + if on: + if not t.get("prometheus") and "prometheus" not in locked: + changes["prometheus"] = True + ours = True + if not t.get("prometheus_token") and "prometheus_token" not in locked: + changes["prometheus_token"] = secrets.token_hex(32) + elif ours and t.get("prometheus") and "prometheus" not in locked: + changes["prometheus"] = False # monitoring turned /metrics on: off again with it + ours = False + if changes: + settings.save(db, base, "telemetry", changes, by) + if on: + token = changes.get("prometheus_token") or t.get("prometheus_token") + sync_token({"telemetry": {"prometheus_token": token}}, create=True) + rid = secrets.token_hex(8) + now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + part = d / "inbox" / f".upgrade.{rid}" + part.write_text(f"id={rid}\naction=services\nmonitoring={'on' if on else 'off'}\nby={by or ''}\nat={now}\n", encoding="utf-8") + part.replace(d / "inbox" / "upgrade") + _put(db, {"run": rid, "on": bool(on), "by": by, "asked_at": now, "state": "queued", "error": None, "turned_on_metrics": ours}) + return view(db) diff --git a/fastapi_backend/app/schemas/updates.py b/fastapi_backend/app/schemas/updates.py index 30b35ab46..cc927371e 100644 --- a/fastapi_backend/app/schemas/updates.py +++ b/fastapi_backend/app/schemas/updates.py @@ -79,3 +79,18 @@ class RemoteBackups(ResponseModel): last_name: str | None = None last_error: str | None = Field(None, description="why copying the last backup failed") copying: CopyWork + + +class Monitoring(ResponseModel): + on: bool = Field(description="the monitoring profile (Prometheus, node-exporter, Grafana) as last asked") + state: Literal["queued", "running", "done", "failed"] | None = Field(None, description="how the machine's last change went") + error: str | None = None + asked_at: str | None = None + finished_at: str | None = None + grafana_port: int = Field(3001, description="the port Grafana listens on, where the web app does") + agent: str | None = Field(None, description="what changes it on this machine (systemd, launchd); none: run `manual` there") + manual: str = Field(description="the line that turns it on by hand, on the machine Lens runs on") + + +class MonitoringRequest(RequestModel): + on: bool = Field(description="start (true) or stop (false) Prometheus, node-exporter and Grafana") diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index cfee929b3..786c57868 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -77,6 +77,7 @@ def test_clients_are_sent_to_sign_in(client, app, env): "write_note", "create_podcast", "set_file_storage", + "set_monitoring", ] assert next(t for t in tools if t["name"] == "search")["inputSchema"]["required"] == ["query"] # API tokens and sessions work too @@ -456,3 +457,13 @@ def test_file_storage_tools(client, db, env, folder): assert out["saved"] == {"store": "local"} and out["check"]["ok"] assert client.app.state.archive.current()["files"]["store"] == "local" assert "only admins" in tool_error(client, env["h"], "set_file_storage", store="local") + + +def test_monitoring_tools(client, db, env, monkeypatch): + monkeypatch.delenv("LENS_CONTROL_DIR", raising=False) + assert "only admins" in tool_error(client, env["h"], "monitoring") + make_user(db, "root@x.io", "root password 1", admin=True) + ha = login(client, "root@x.io", "root password 1") + got = tool(client, ha, "monitoring") + assert (got["on"], got["agent"]) == (False, None) and got["settings"].endswith("/settings/telemetry") + assert "no upgrade agent" in tool_error(client, ha, "set_monitoring", on=True) diff --git a/fastapi_backend/tests/api/test_monitoring.py b/fastapi_backend/tests/api/test_monitoring.py new file mode 100644 index 000000000..b9851efab --- /dev/null +++ b/fastapi_backend/tests/api/test_monitoring.py @@ -0,0 +1,128 @@ +"""Turning the monitoring profile (Prometheus, node-exporter, Grafana) on and off from the web app (docs/prometheus.md).""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import subprocess + +import pytest + +from tests.helpers import login, make_user + +AGENT = pathlib.Path(__file__).resolve().parents[3] / "packaging" / "host" / "lens-upgrade.sh" + + +@pytest.fixture +def control(tmp_path, monkeypatch): + """The control folder install.sh shares with the containers, with an agent watching it.""" + d = tmp_path / "control" + (d / "inbox").mkdir(parents=True) + (d / "agent").write_text("kind=systemd\n") + monkeypatch.setenv("LENS_CONTROL_DIR", str(d)) + return d + + +def admin(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def request_of(control): + return dict(line.split("=", 1) for line in (control / "inbox" / "upgrade").read_text().splitlines()) + + +def test_turning_monitoring_on_and_off(client, db, control): + h = admin(client, db) + v = client.get("/api/v1/admin/monitoring", headers=h).json() + assert (v["on"], v["state"], v["agent"], v["grafana_port"]) == (False, None, "systemd", 3001) + + r = client.put("/api/v1/admin/monitoring", json={"on": True}, headers=h) + assert r.status_code == 202 and r.json()["on"] is True and r.json()["state"] == "queued" + req = request_of(control) + assert (req["action"], req["monitoring"], req["by"]) == ("services", "on", "root@x.io") + # /metrics turned on with a token made for it, and the token where the monitoring Prometheus reads it + t = client.app.state.settings.current()["telemetry"] + assert t["prometheus"] is True and len(t["prometheus_token"]) == 64 + assert (control / "prometheus-token").read_text() == t["prometheus_token"] + assert client.get("/metrics", headers={"Authorization": f"Bearer {t['prometheus_token']}"}).status_code == 200 + # one request at a time + assert client.put("/api/v1/admin/monitoring", json={"on": False}, headers=h).status_code == 409 + + # the agent picks it up and says how it went + (control / "inbox" / "upgrade").unlink() + (control / "services-status").write_text(f"id={req['id']}\nmonitoring=on\nstate=done\ngrafana_port=3005\ncode=0\n") + v = client.get("/api/v1/admin/monitoring", headers=h).json() + assert (v["state"], v["grafana_port"], v["error"]) == ("done", 3005, None) + + # a new scrape token in Settings reaches Prometheus too + assert client.put("/api/v1/settings/telemetry", json={"prometheus_token": "b" * 32}, headers=h).status_code == 200 + assert (control / "prometheus-token").read_text() == "b" * 32 + + # off stops the containers and turns /metrics off again, since monitoring turned it on + r = client.put("/api/v1/admin/monitoring", json={"on": False}, headers=h) + assert r.status_code == 202 and request_of(control)["monitoring"] == "off" + assert client.app.state.settings.current()["telemetry"]["prometheus"] is False + + +def test_a_failed_change_says_why(client, db, control): + h = admin(client, db) + client.put("/api/v1/admin/monitoring", json={"on": True}, headers=h) + rid = request_of(control)["id"] + (control / "inbox" / "upgrade").unlink() + (control / "services-status").write_text(f"id={rid}\nstate=failed\ncode=2\nerror=monitoring runs in Docker\n") + v = client.get("/api/v1/admin/monitoring", headers=h).json() + assert (v["state"], v["error"]) == ("failed", "monitoring runs in Docker") + # and it can be tried again + assert client.put("/api/v1/admin/monitoring", json={"on": True}, headers=h).status_code == 202 + + +def test_no_agent_shows_the_line_to_run(client, db, monkeypatch): + monkeypatch.delenv("LENS_CONTROL_DIR", raising=False) + h = admin(client, db) + v = client.get("/api/v1/admin/monitoring", headers=h).json() + assert v["agent"] is None and "--profile monitoring up -d" in v["manual"] + r = client.put("/api/v1/admin/monitoring", json={"on": True}, headers=h) + assert r.status_code == 409 and "run the line shown" in r.json()["detail"] + + +def test_only_admins(client, db, control): + make_user(db, "me@x.io", "my password 12") + h = login(client, "me@x.io", "my password 12") + assert client.get("/api/v1/admin/monitoring", headers=h).status_code == 403 + assert client.put("/api/v1/admin/monitoring", json={"on": True}, headers=h).status_code == 403 + + +@pytest.mark.skipif(not shutil.which("sh") or not shutil.which("paste"), reason="needs a POSIX shell") +def test_the_agent_changes_compose_profiles(tmp_path): + """packaging/host/lens-upgrade.sh, with a stand-in for docker: .env's COMPOSE_PROFILES and the compose commands.""" + inst, bin_ = tmp_path / "lens", tmp_path / "bin" + (inst / ".lens-control" / "inbox").mkdir(parents=True) + bin_.mkdir() + calls = tmp_path / "calls" + (bin_ / "docker").write_text(f'#!/bin/sh\necho "docker $*" >> {calls}\n') + (bin_ / "docker").chmod(0o755) + env = inst / ".env" + env.write_text("ACCESS_SECRET_KEY=x\nCOMPOSE_PROFILES=matterbridge\nLENS_GRAFANA_PORT=3005\n") + env_vars = {**os.environ, "PATH": f"{bin_}:{os.environ['PATH']}"} + + def ask(on): + (inst / ".lens-control" / "inbox" / "upgrade").write_text(f"id=r{on}\naction=services\nmonitoring={on}\n") + subprocess.run(["sh", str(AGENT), str(inst)], check=True, env=env_vars, timeout=30) + status = (inst / ".lens-control" / "services-status").read_text() + return dict(line.split("=", 1) for line in status.splitlines()) + + s = ask("on") + assert (s["id"], s["state"], s["grafana_port"]) == ("ron", "done", "3005") + assert "COMPOSE_PROFILES=matterbridge,monitoring" in env.read_text().splitlines() + assert "ACCESS_SECRET_KEY=x" in env.read_text().splitlines() + assert calls.read_text().splitlines()[-1].endswith("--profile monitoring up -d prometheus node-exporter grafana") + assert not (inst / ".lens-control" / "inbox" / "upgrade").exists() + + s = ask("off") + assert s["state"] == "done" and "COMPOSE_PROFILES=matterbridge" in env.read_text().splitlines() + assert calls.read_text().splitlines()[-1].endswith("rm -s -f prometheus node-exporter grafana") + + s = ask("sideways") + assert s["state"] == "failed" and "on or off" in s["error"] diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 64805cf34..be0cd7dfa 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -9,7 +9,7 @@ scrape_configs: metrics_path: /metrics authorization: type: Bearer - credentials_file: /prometheus/lens-token # written from LENS_PROMETHEUS_TOKEN when the container starts + credentials_file: /prometheus/lens-token # LENS_PROMETHEUS_TOKEN, else a link to the token Lens writes; read each scrape static_configs: - targets: ["backend:8000"] diff --git a/nextjs-frontend/__tests__/monitoring-switch.test.ts b/nextjs-frontend/__tests__/monitoring-switch.test.ts new file mode 100644 index 000000000..d53b87171 --- /dev/null +++ b/nextjs-frontend/__tests__/monitoring-switch.test.ts @@ -0,0 +1,30 @@ +/** @jest-environment node */ +import { grafanaAddress, monitoringNote } from "@/components/admin/monitoring-switch"; + +jest.mock("@/app/openapi-client", () => ({})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: null }) })); + +const base = { on: true, state: "done" as const, grafana_port: 3001, manual: "docker compose …", agent: "systemd" }; + +describe("grafanaAddress", () => { + it("prefers the address set in Settings", () => { + expect(grafanaAddress("https://grafana.example", base, "lens.local")).toBe("https://grafana.example"); + }); + it("links to the Grafana monitoring runs, on this host, once it is running", () => { + expect(grafanaAddress(null, base, "lens.local")).toBe("http://lens.local:3001"); + expect(grafanaAddress(null, { ...base, grafana_port: 3005 }, "::1")).toBe("http://[::1]:3005"); + expect(grafanaAddress(null, { ...base, state: "running" }, "lens.local")).toBeNull(); + expect(grafanaAddress(null, { ...base, on: false }, "lens.local")).toBeNull(); + expect(grafanaAddress(undefined, undefined, "lens.local")).toBeNull(); + }); +}); + +describe("monitoringNote", () => { + it("says what the machine is doing", () => { + expect(monitoringNote({ ...base, state: "queued" })).toMatch(/Waiting/); + expect(monitoringNote({ ...base, state: "running" })).toMatch(/Starting/); + expect(monitoringNote({ ...base, on: false, state: "running" })).toMatch(/Stopping/); + expect(monitoringNote(base)).toMatch(/Running/); + expect(monitoringNote({ ...base, on: false, state: null })).toMatch(/^Off/); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 25196f9b0..dd72a4380 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1049,6 +1049,9 @@ export type { GetMediaError, GetMediaErrors, GetMediaResponses, + GetMonitoringData, + GetMonitoringResponse, + GetMonitoringResponses, GetNamespaceAssistantData, GetNamespaceAssistantError, GetNamespaceAssistantErrors, @@ -1875,6 +1878,8 @@ export type { MetadataEdit, MeUpdate, ModelSpend, + Monitoring, + MonitoringRequest, MonthStats, MoveMentionData, MoveMentionError, @@ -2531,6 +2536,11 @@ export type { SetMemberErrors, SetMemberResponse, SetMemberResponses, + SetMonitoringData, + SetMonitoringError, + SetMonitoringErrors, + SetMonitoringResponse, + SetMonitoringResponses, SetNamespaceFacesModeData, SetNamespaceFacesModeError, SetNamespaceFacesModeErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 4fd670a0b..84cc0ca0d 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -509,6 +509,8 @@ import type { GetMediaData, GetMediaErrors, GetMediaResponses, + GetMonitoringData, + GetMonitoringResponses, GetNamespaceAssistantData, GetNamespaceAssistantErrors, GetNamespaceAssistantResponses, @@ -1244,6 +1246,9 @@ import type { SetMemberData, SetMemberErrors, SetMemberResponses, + SetMonitoringData, + SetMonitoringErrors, + SetMonitoringResponses, SetNamespaceFacesModeData, SetNamespaceFacesModeErrors, SetNamespaceFacesModeResponses, @@ -2986,6 +2991,40 @@ export class Updates { }); } + /** + * Get Monitoring + * + * Whether Prometheus, node-exporter and Grafana run next to Lens (the monitoring profile), and how the machine's + * last change went. + */ + public static getMonitoring( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/admin/monitoring", + ...options, + }); + } + + /** + * Set Monitoring + * + * Have this machine start or stop Prometheus, node-exporter and Grafana. On also turns /metrics on with a scrape + * token when it has none. GET /admin/monitoring follows along. + */ + public static setMonitoring( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/admin/monitoring", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * List Backups * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 30bada334..4a6d7432e 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -8239,6 +8239,67 @@ export type ModelSpend = { [key: string]: unknown; }; +/** + * Monitoring + */ +export type Monitoring = { + /** + * On + * + * the monitoring profile (Prometheus, node-exporter, Grafana) as last asked + */ + on: boolean; + /** + * State + * + * how the machine's last change went + */ + state?: "queued" | "running" | "done" | "failed" | null; + /** + * Error + */ + error?: string | null; + /** + * Asked At + */ + asked_at?: string | null; + /** + * Finished At + */ + finished_at?: string | null; + /** + * Grafana Port + * + * the port Grafana listens on, where the web app does + */ + grafana_port?: number; + /** + * Agent + * + * what changes it on this machine (systemd, launchd); none: run `manual` there + */ + agent?: string | null; + /** + * Manual + * + * the line that turns it on by hand, on the machine Lens runs on + */ + manual: string; + [key: string]: unknown; +}; + +/** + * MonitoringRequest + */ +export type MonitoringRequest = { + /** + * On + * + * start (true) or stop (false) Prometheus, node-exporter and Grafana + */ + on: boolean; +}; + /** * MonthStats */ @@ -20810,6 +20871,47 @@ export type StartUpgradeResponses = { export type StartUpgradeResponse = StartUpgradeResponses[keyof StartUpgradeResponses]; +export type GetMonitoringData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/admin/monitoring"; +}; + +export type GetMonitoringResponses = { + /** + * Successful Response + */ + 200: Monitoring; +}; + +export type GetMonitoringResponse = GetMonitoringResponses[keyof GetMonitoringResponses]; + +export type SetMonitoringData = { + body: MonitoringRequest; + path?: never; + query?: never; + url: "/api/v1/admin/monitoring"; +}; + +export type SetMonitoringErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SetMonitoringError = SetMonitoringErrors[keyof SetMonitoringErrors]; + +export type SetMonitoringResponses = { + /** + * Successful Response + */ + 202: Monitoring; +}; + +export type SetMonitoringResponse = SetMonitoringResponses[keyof SetMonitoringResponses]; + export type ListBackupsData = { body?: never; path?: never; diff --git a/nextjs-frontend/components/admin/health.tsx b/nextjs-frontend/components/admin/health.tsx index d3baa10dc..7ad9cf8ad 100644 --- a/nextjs-frontend/components/admin/health.tsx +++ b/nextjs-frontend/components/admin/health.tsx @@ -7,6 +7,7 @@ import type { ReactNode } from "react"; import { Admin, Jobs } from "@/app/openapi-client"; import { AdminFrame } from "@/components/admin/admin-frame"; +import { grafanaAddress, MonitoringSwitch, useMonitoring } from "@/components/admin/monitoring-switch"; import { isUnreachable, ServerUnreachable } from "@/components/errors/error-states"; import { Button } from "@/components/ui/button"; import { EmptyState, Skeleton } from "@/components/ui/states"; @@ -103,6 +104,7 @@ export function HealthPage() { queryFn: () => data(Admin.getHealth({ client })), refetchInterval: REFRESH_MS, }); + const monitoring = useMonitoring(); const settings = useQuery({ queryKey: ["settings"], queryFn: async () => @@ -326,7 +328,14 @@ export function HealthPage() { - + ); } @@ -386,16 +395,9 @@ function ServerNow({ machine: m, grafana }: { machine?: Machine | null; grafana? > Charts and history in Grafana - ) : ( - - For charts and history, run Prometheus and Grafana with the monitoring profile and set its address in{" "} - - Settings → Telemetry - - . - - )} + ) : null} + {!grafana && }
{tiles.map(([name, value, detail]) => (
diff --git a/nextjs-frontend/components/admin/monitoring-switch.tsx b/nextjs-frontend/components/admin/monitoring-switch.tsx new file mode 100644 index 000000000..5b66370d5 --- /dev/null +++ b/nextjs-frontend/components/admin/monitoring-switch.tsx @@ -0,0 +1,80 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { Updates } from "@/app/openapi-client"; +import type { Monitoring } from "@/app/openapi-client/types.gen"; +import { Banner } from "@/components/ui/banner"; +import { Switch } from "@/components/ui/field"; +import { data, useApiClient } from "@/lib/api/browser"; + +/** Grafana's address: the one set in Settings, else where the monitoring profile runs it (this host, its port). */ +export function grafanaAddress( + configured: string | null | undefined, + m: Pick | null | undefined, + hostname: string, +): string | null { + if (configured) return configured; + if (!m?.on || m.state !== "done" || !hostname) return null; + const host = hostname.includes(":") ? `[${hostname}]` : hostname; // an IPv6 address + return `http://${host}:${m.grafana_port ?? 3001}`; +} + +/** What the switch says under it: waiting for the machine, starting, stopping, running or off. */ +export function monitoringNote(m: Monitoring): string { + if (m.state === "queued") return "Waiting for the machine to pick it up…"; + if (m.state === "running") return m.on ? "Starting Prometheus, node-exporter and Grafana…" : "Stopping them…"; + if (m.on && m.state === "done") return "Running: Prometheus keeps the history, Grafana shows it."; + return "Off. On starts Prometheus, node-exporter and Grafana next to Lens (about 150-250 MB of memory)."; +} + +/** Whether the monitoring containers run, while the machine changes it. */ +export function useMonitoring() { + const client = useApiClient(); + return useQuery({ + queryKey: ["monitoring"], + queryFn: () => data(Updates.getMonitoring({ client })), + refetchInterval: (q) => (q.state.data?.state === "queued" || q.state.data?.state === "running" ? 2000 : false), + }); +} + +/** Monitoring on or off: Prometheus, node-exporter and Grafana, started by the machine's upgrade agent. */ +export function MonitoringSwitch() { + const client = useApiClient(); + const qc = useQueryClient(); + const q = useMonitoring(); + const turn = useMutation({ + mutationFn: (on: boolean) => data(Updates.setMonitoring({ client, body: { on } })), + onSuccess: (m) => { + qc.setQueryData(["monitoring"], m); + void qc.invalidateQueries({ queryKey: ["settings"] }); // on turns /metrics on, with a token + }, + }); + const m = q.data; + if (!m) return null; + const busy = turn.isPending || ((m.state === "queued" || m.state === "running") && !m.error); + return ( +
+ turn.mutate(on)} + disabled={busy || !m.agent} + label="Charts and history (Prometheus and Grafana)" + /> + {m.agent ? ( + {monitoringNote(m)} + ) : ( + + This machine has no upgrade agent to start them (Lens wasn’t installed with install.sh, or it couldn’t set one + up). On the machine, run {m.manual} + + )} + {m.error && ( + + {m.error} + + )} + {turn.isError && {turn.error.message}} +
+ ); +} diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 83d2c0e3c..f7473c57f 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -7,6 +7,7 @@ import { useEffect, useRef, type ReactNode } from "react"; import { Admin, Fedora, Metadata, Sensors, Sources } from "@/app/openapi-client"; import type { MatterbridgeStatus } from "@/app/openapi-client/types.gen"; +import { MonitoringSwitch } from "@/components/admin/monitoring-switch"; import { BackupsPanel } from "@/components/settings/backups-panel"; import { ComponentsStatus } from "@/components/settings/components-status"; import { AllTokens } from "@/components/account/all-tokens"; @@ -617,9 +618,11 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) {

For Grafana and other Prometheus tools: jobs, imports, model calls, cost, budgets, workers and disk, served at{" "} /metrics on this address with the same privacy as telemetry. A scraper sends - the token as Authorization: Bearer. The monitoring profile runs Prometheus - and Grafana next to Lens, ready made, with charts of Lens and this machine (docs/prometheus.md). + the token as Authorization: Bearer. Charts and history runs Prometheus and + Grafana next to Lens, ready made, with charts of Lens and this machine, and sets up the token itself + (docs/prometheus.md).

+ {Boolean(saved.prometheus) && !promSecret.set && !promToken.value && ( Set one below; until then /metrics answers no one. diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 60c7fd79d..ef33b467c 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -2074,7 +2074,7 @@ export const FIELDS: FieldSpec[] = [ nullable: true, mono: true, placeholder: "http://localhost:3001", - hint: "Where Grafana shows Lens's charts; System health links to it. The monitoring profile runs one at :3001", + hint: "Where Grafana shows Lens's charts; System health links to it. Empty: the one Charts and history runs, on this host at :3001", }, { section: "telemetry", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 75812f9fc..45350f533 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -3354,6 +3354,68 @@ } } }, + "/api/v1/admin/monitoring": { + "get": { + "tags": [ + "updates" + ], + "summary": "Get Monitoring", + "description": "Whether Prometheus, node-exporter and Grafana run next to Lens (the monitoring profile), and how the machine's\nlast change went.", + "operationId": "get_monitoring", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Monitoring" + } + } + } + } + } + }, + "put": { + "tags": [ + "updates" + ], + "summary": "Set Monitoring", + "description": "Have this machine start or stop Prometheus, node-exporter and Grafana. On also turns /metrics on with a scrape\ntoken when it has none. GET /admin/monitoring follows along.", + "operationId": "set_monitoring", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitoringRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Monitoring" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/admin/backups": { "get": { "tags": [ @@ -39910,6 +39972,111 @@ ], "title": "ModelSpend" }, + "Monitoring": { + "properties": { + "on": { + "type": "boolean", + "title": "On", + "description": "the monitoring profile (Prometheus, node-exporter, Grafana) as last asked" + }, + "state": { + "anyOf": [ + { + "type": "string", + "enum": [ + "queued", + "running", + "done", + "failed" + ] + }, + { + "type": "null" + } + ], + "title": "State", + "description": "how the machine's last change went" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "asked_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Asked At" + }, + "finished_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Finished At" + }, + "grafana_port": { + "type": "integer", + "title": "Grafana Port", + "description": "the port Grafana listens on, where the web app does", + "default": 3001 + }, + "agent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Agent", + "description": "what changes it on this machine (systemd, launchd); none: run `manual` there" + }, + "manual": { + "type": "string", + "title": "Manual", + "description": "the line that turns it on by hand, on the machine Lens runs on" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "on", + "manual" + ], + "title": "Monitoring" + }, + "MonitoringRequest": { + "properties": { + "on": { + "type": "boolean", + "title": "On", + "description": "start (true) or stop (false) Prometheus, node-exporter and Grafana" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "on" + ], + "title": "MonitoringRequest" + }, "MonthStats": { "properties": { "month": { diff --git a/packaging/host/lens-upgrade.sh b/packaging/host/lens-upgrade.sh index 29c132abb..3b54e59f0 100755 --- a/packaging/host/lens-upgrade.sh +++ b/packaging/host/lens-upgrade.sh @@ -8,9 +8,68 @@ # # It takes a release tag (vX.Y.Z), or the branch this install already follows; nothing else, whoever wrote the # request. Run by hand: sh packaging/host/lens-upgrade.sh +# +# A request with action=services turns the monitoring containers on or off instead (docs/prometheus.md): it adds or +# removes `monitoring` in COMPOSE_PROFILES in .env and starts or stops Prometheus, node-exporter and Grafana, and says +# how it went in .lens-control/services-status. It comes in the same file as an upgrade, the one every agent watches. set -eu umask 022 # the containers read the status and the log +# Monitoring on or off (action=services): COMPOSE_PROFILES in .env, then Docker Compose starts or stops the three +services() { + sstatus() { # state [extra key=value lines] + { + echo "id=$id" + echo "monitoring=$monitoring" + echo "state=$1" + echo "started_at=$started" + echo "grafana_port=$port" + shift + for kv in "$@"; do echo "$kv"; done + } >"$CTL/services-status.tmp" && mv "$CTL/services-status.tmp" "$CTL/services-status" + } + started="$(now)" + port="$(env_of LENS_GRAFANA_PORT | tr -cd 0-9)" + port="${port:-3001}" + case "$monitoring" in + on | off) ;; + *) + sstatus failed "finished_at=$(now)" "code=2" "error=monitoring is on or off" + return 0 + ;; + esac + if [ "$(env_of LENS_BACKEND)" = microsandbox ]; then + sstatus failed "finished_at=$(now)" "code=2" "error=monitoring runs in Docker; this install runs in a microsandbox" + return 0 + fi + sstatus running + # the other profiles stay as they are + profiles="$(env_of COMPOSE_PROFILES | tr ',' '\n' | tr -cd 'A-Za-z0-9_\n-' | grep -v -e '^monitoring$' -e '^$' | paste -s -d, - || true)" + [ "$monitoring" = on ] && profiles="${profiles:+$profiles,}monitoring" + if [ -f "$DIR/.env" ]; then # rewritten in place: it keeps its owner and permissions + grep -v '^COMPOSE_PROFILES=' "$DIR/.env" >"$DIR/.env.tmp" || true + [ -z "$profiles" ] || echo "COMPOSE_PROFILES=$profiles" >>"$DIR/.env.tmp" + cat "$DIR/.env.tmp" >"$DIR/.env" && rm -f "$DIR/.env.tmp" + fi + if docker compose version >/dev/null 2>&1; then compose="docker compose"; else compose="docker-compose"; fi + files="-f docker-compose.prod.yml" + [ "$(env_of LENS_HTTPS)" = 1 ] && files="$files -f docker-compose.https.yml" + set +e + # shellcheck disable=SC2086 + if [ "$monitoring" = on ]; then + (cd "$DIR" && $compose $files --profile monitoring up -d prometheus node-exporter grafana) >"$CTL/services.log" 2>&1 "$CTL/services.log" 2>&1 /dev/null || true' EXIT val() { sed -n "s/^$1=//p" "$2" | head -1 | tr -cd 'A-Za-z0-9._/-'; } id="$(val id "$REQ")" ref="$(val ref "$REQ")" +action="$(val action "$REQ")" +monitoring="$(val monitoring "$REQ")" rm -f "$REQ" now() { date -u +%Y-%m-%dT%H:%M:%SZ; } +env_of() { [ -f "$DIR/.env" ] && sed -n "s/^$1=//p" "$DIR/.env" | head -1; } +if [ "$action" = services ]; then + services + exit 0 +fi status() { # state [extra key=value lines] { echo "id=$id" @@ -50,7 +116,6 @@ fi status running # The same settings this install was made with (install.sh keeps the port and the address itself) -env_of() { [ -f "$DIR/.env" ] && sed -n "s/^$1=//p" "$DIR/.env" | head -1; } set +e ( cd "$DIR" && diff --git a/threat-model.md b/threat-model.md index c6f63c6ac..980f0cbb8 100644 --- a/threat-model.md +++ b/threat-model.md @@ -286,6 +286,7 @@ classified by gate only *(inferred, Q8)*; unread modules are listed in Q34. | MQTT 1883, syslog 5514, `POST /sensors/push/{token}/{stream}` | payloads, hub logins, token in the path | yes, once enabled | data, size/rate, x-credential | admin enables; operator firewalls | *(documented, docs/sensors.md "Sensors are off until an admin turns them on")* | | `.env`, `archive.yaml`, `LENS_*`, `ARCHIVE_SECRET_KEY`, `RCLONE_BINARY` | process configuration | no (operator) | x-env-config | keep unreadable to others | *(maintainer, 2026-10)* | | workers and DB peers | DB credential, shared database rows | no (trusted peer) | x-peer | run peers on trusted hosts | *(inferred, Q9)* | +| `.lens-control/inbox/upgrade` (host upgrade agent), `/metrics`, `.lens-control/prometheus-token` | `ref` (release tag or this install's branch), `action=services` with `monitoring=on`/`off`; the scrape token | the inbox is writable by the containers (and local users); `/metrics` by anyone reaching the web app | x-env-config, credential | the agent installs only a release tag or the install's branch and only toggles the `monitoring` profile, nothing else from the file; `/metrics` answers only the scrape token (constant-time compare); the token file is world-readable on the host and grants metrics only, which carry no personal content | *(documented, docs/updates.md, docs/prometheus.md "Privacy")* | **Stateful postconditions.** A refused or failed one-time secret check consumes nothing; a used one is deleted before Lens trusts it *(documented, docs/authentication.md "works once")*. An upload file only ever holds whole chunks From 5f8ee4897137ea0c56ba469418d663fd189f0ee0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 22:49:57 +0000 Subject: [PATCH 2/2] docs: add the control inbox and scrape token to the threat model companions Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XCh396xzDRUrSuH1xohcXR --- threat-model.json | 12 +++++++++++- threat-model.yaml | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/threat-model.json b/threat-model.json index 9fcfe88ad..6578b71ba 100644 --- a/threat-model.json +++ b/threat-model.json @@ -49,7 +49,8 @@ "routes not read one by one (about 370)", "media, report, embed, IIIF content", "/iiif/auth/token, /probe, /logout", - "workers and DB peers" + "workers and DB peers", + "host control inbox and the monitoring scrape token" ], "touches": [ "filesystem" @@ -478,6 +479,15 @@ "caller_must_enforce": "run peers on trusted hosts", "provenance": "inferred", "source": "open question Q9" + }, + { + "entry_point": "host control inbox and the monitoring scrape token", + "parameter": "inbox/upgrade request (release ref, or action=services with monitoring=on|off); /metrics scrape token and .lens-control/prometheus-token", + "attacker_controllable": "yes", + "component": "authz", + "caller_must_enforce": "the agent installs only a release tag or the install's branch and only toggles the monitoring profile; the token is compared in constant time and grants metrics only", + "provenance": "documented", + "source": "docs/updates.md, docs/prometheus.md" } ], "environment": { diff --git a/threat-model.yaml b/threat-model.yaml index 88781ae2a..64e5bf5b6 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:6dedfe71cd1ec807c7a04b3338c03cb01f70b61882d620d46a236bbe6e257399" +prose_version: "threat-model.md@sha256:deaa72376c5893773518be5790b22bd9e8b913cf7f76a87142a1e7c233cba055" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8} @@ -418,6 +418,23 @@ entry_points: obligation_id: trusted-peer-hosts caller_must_enforce: "run peers on trusted hosts" provenance: {kind: inferred, question_id: Q9} + - id: "host control inbox and the monitoring scrape token" + component: authz + parameters: + - name: "inbox/upgrade request: release ref, or action=services with monitoring=on|off" + attacker_controllable: true + x-condition: "writable by the containers and local users" + control_kinds: [x-env-config] + obligation_id: agent-allow-list + caller_must_enforce: "the agent installs only a release tag or the install's branch, and only toggles the monitoring profile" + provenance: {kind: documented, source: "docs/updates.md"} + - name: "/metrics scrape token, .lens-control/prometheus-token" + attacker_controllable: true + x-condition: "anyone reaching the web app; the file is world-readable on the host" + control_kinds: [credential] + obligation_id: metrics-token + caller_must_enforce: "token compared in constant time; metrics carry no personal content" + provenance: {kind: documented, source: "docs/prometheus.md Privacy"} # ---------------------------------------------------------------- §1.7 contract-dimension matrix # Every in-scope component × the eight required dimensions appears exactly once.