From 5364f392ad3c49f7af5be1b9c0b9761a1bc1471e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:27:48 +0000 Subject: [PATCH] feat(monitoring): Prometheus, node-exporter and Grafana as an opt-in compose profile The monitoring profile runs Prometheus scraping Lens's /metrics with the scrape token and node-exporter for the machine, and Grafana with the Lens dashboard (now with machine panels) as its home. System health shows CPU, memory, load and uptime now and links to Grafana (telemetry.grafana_url / LENS_GRAFANA_URL) for history. The compose file passes LENS_PROMETHEUS and LENS_PROMETHEUS_TOKEN to the API, and lens_jobs reports 0 for statuses with no jobs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XCh396xzDRUrSuH1xohcXR --- CHANGELOG.md | 5 + docker-compose.prod.yml | 64 ++++ docs/grafana/lens-dashboard.json | 316 ++++++++++++++++++ docs/prometheus.md | 33 +- fastapi_backend/app/api/v1/routes/admin.py | 2 + fastapi_backend/app/domain/machine.py | 79 +++++ fastapi_backend/app/domain/prometheus.py | 4 +- fastapi_backend/app/domain/settings.py | 7 + fastapi_backend/app/domain/store.py | 2 + fastapi_backend/app/schemas/admin.py | 14 + fastapi_backend/tests/api/test_telemetry.py | 11 + .../grafana/provisioning/dashboards/lens.yml | 8 + .../grafana/provisioning/datasources/lens.yml | 9 + monitoring/prometheus.yml | 18 + .../__tests__/health-duration.test.ts | 14 + nextjs-frontend/app/openapi-client/index.ts | 1 + .../app/openapi-client/types.gen.ts | 63 ++++ nextjs-frontend/components/admin/health.tsx | 81 ++++- .../components/settings/bodies.tsx | 5 +- nextjs-frontend/components/settings/model.ts | 10 + .../components/settings/section.tsx | 1 + nextjs-frontend/openapi.json | 130 +++++++ 22 files changed, 871 insertions(+), 6 deletions(-) create mode 100644 monitoring/grafana/provisioning/dashboards/lens.yml create mode 100644 monitoring/grafana/provisioning/datasources/lens.yml create mode 100644 monitoring/prometheus.yml create mode 100644 nextjs-frontend/__tests__/health-duration.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 10ea98810..efcaa1a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Charts of Lens and its server in Grafana.** A `monitoring` profile runs Prometheus, node-exporter and Grafana next to + Lens, connected and with the Lens dashboard (now with the machine's CPU, memory, disk and network) as Grafana's home. + Admin → System health shows CPU, memory, load and uptime now and links to Grafana for history + ([Prometheus](docs/prometheus.md#prometheus-and-grafana-ready-made)). + - **Prometheus metrics.** Lens can serve `/metrics` for Prometheus and Grafana: jobs by status and pipeline, imports, model calls, tokens, cost and failures by model, budgets, workers and disk. Off by default; turned on in Settings → Telemetry (or `LENS_PROMETHEUS=on`), it answers only a scraper with the scrape token, and follows `LENS_BIND`. A diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c5b41f574..0a2ed37b8 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -58,6 +58,11 @@ x-backend: &backend LENS_TELEMETRY: ${LENS_TELEMETRY:-} LENS_TELEMETRY_ENDPOINT: ${LENS_TELEMETRY_ENDPOINT:-} LENS_TELEMETRY_HEADERS: ${LENS_TELEMETRY_HEADERS:-} + # Prometheus (docs/prometheus.md): /metrics, off unless LENS_PROMETHEUS=on with a LENS_PROMETHEUS_TOKEN; the + # monitoring profile below scrapes it and charts it in Grafana at LENS_GRAFANA_URL + LENS_PROMETHEUS: ${LENS_PROMETHEUS:-} + LENS_PROMETHEUS_TOKEN: ${LENS_PROMETHEUS_TOKEN:-} + LENS_GRAFANA_URL: ${LENS_GRAFANA_URL:-} # the volume shared with the matterbridge service (its config, written from Settings → Chat rooms) LENS_MATTERBRIDGE_DIR: /matterbridge # upgrading from the web app (docs/updates.md): the folder install.sh's upgrade agent watches, and the install folder @@ -169,12 +174,71 @@ services: - opensearch-data:/usr/share/opensearch/data networks: [lens] + # Prometheus, node-exporter and Grafana (docs/prometheus.md): charts of Lens and this machine over time, ready made. + # Opt-in: in .env set COMPOSE_PROFILES=monitoring, LENS_PROMETHEUS=on, LENS_PROMETHEUS_TOKEN (openssl rand -hex 32) + # and LENS_GRAFANA_URL=http://localhost:3001, then `docker compose up -d`. Grafana listens where the web app does + # (LENS_BIND), on LENS_GRAFANA_PORT (3001); anyone who reaches it may view, and only the admin may change things. + prometheus: + image: prom/prometheus:${PROMETHEUS_VERSION:-v3.5.0} + profiles: [monitoring] + security_opt: ["no-new-privileges:true"] + entrypoint: ["/bin/sh", "-c"] + command: + - >- + printf %s "$$LENS_PROMETHEUS_TOKEN" > /prometheus/lens-token && + exec /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus + --storage.tsdb.retention.time=$${LENS_METRICS_DAYS:-30}d + environment: + LENS_PROMETHEUS_TOKEN: ${LENS_PROMETHEUS_TOKEN:-} + LENS_METRICS_DAYS: ${LENS_METRICS_DAYS:-30} + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + restart: unless-stopped + networks: [lens] + + node-exporter: + image: prom/node-exporter:${NODE_EXPORTER_VERSION:-v1.9.1} + profiles: [monitoring] + security_opt: ["no-new-privileges:true"] + command: ["--path.rootfs=/host"] + pid: host + volumes: + - /:/host:ro,rslave + restart: unless-stopped + networks: [lens] + + grafana: + image: grafana/grafana-oss:${GRAFANA_VERSION:-12.0.2} + profiles: [monitoring] + security_opt: ["no-new-privileges:true"] + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} # Grafana asks to change admin on first sign-in + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/lens-dashboard.json + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + ports: + - "${LENS_BIND:-127.0.0.1}:${LENS_GRAFANA_PORT:-3001}:3000" + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./docs/grafana:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + depends_on: [prometheus] + restart: unless-stopped + networks: [lens] + volumes: surreal-data: matterbridge-data: archive-data: fedora-data: opensearch-data: + prometheus-data: + grafana-data: networks: lens: diff --git a/docs/grafana/lens-dashboard.json b/docs/grafana/lens-dashboard.json index c0a81b4b9..67401496b 100644 --- a/docs/grafana/lens-dashboard.json +++ b/docs/grafana/lens-dashboard.json @@ -588,6 +588,322 @@ "overrides": [] }, "options": {} + }, + { + "id": 17, + "type": "row", + "title": "Machine (node-exporter, with the monitoring profile)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 36, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 18, + "title": "Lens reachable", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 37, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "up{job=\"lens\"}", + "legendFormat": "lens", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 19, + "title": "Machine uptime", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 4, + "y": 37, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "time() - node_boot_time_seconds", + "legendFormat": "uptime", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 20, + "title": "CPUs", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 37, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "count(node_cpu_seconds_total{mode=\"idle\"})", + "legendFormat": "CPUs", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 21, + "title": "Memory total", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 37, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "node_memory_MemTotal_bytes", + "legendFormat": "total", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 22, + "title": "Load (1 min)", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 37, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "node_load1", + "legendFormat": "load", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 23, + "title": "Network in", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 20, + "y": 37, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(node_network_receive_bytes_total{device!=\"lo\"}[5m]))", + "legendFormat": "in", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 24, + "title": "CPU used", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))", + "legendFormat": "CPU", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 25, + "title": "Memory used", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 41, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "legendFormat": "memory", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 26, + "title": "Disk used", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 41, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})", + "legendFormat": "/", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": {} } ], "annotations": { diff --git a/docs/prometheus.md b/docs/prometheus.md index d4adfe1fb..647b5d930 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -8,6 +8,33 @@ calls, tokens and cost, budgets, workers and the disk. endpoint: Lens sends nothing anywhere. For pushing traces and metrics to an OpenTelemetry collector instead, see [Telemetry](telemetry.md); the two are independent. +## Prometheus and Grafana, ready made + +Lens doesn't build its own charts: with Docker, the `monitoring` profile runs [Prometheus](https://prometheus.io/), +[node-exporter](https://github.com/prometheus/node_exporter) (the machine's CPU, memory, disk and network) and +[Grafana](https://grafana.com/oss/grafana/) next to Lens, already connected, with the Lens dashboard as Grafana's home +page. Add to `.env`: + +```sh +COMPOSE_PROFILES=monitoring # with others, comma-separated (matterbridge,monitoring) +LENS_PROMETHEUS=on +LENS_PROMETHEUS_TOKEN=3f9c... # openssl rand -hex 32 +LENS_GRAFANA_URL=http://localhost:3001 # the address you open Grafana at; System health links to it +# GRAFANA_ADMIN_PASSWORD=... # optional; otherwise admin/admin, and Grafana asks to change it +# LENS_METRICS_DAYS=30 # how long Prometheus keeps history +``` + +then `docker compose -f docker-compose.prod.yml up -d`. Grafana listens where the web app does (`LENS_BIND`: this +machine only unless Lens was opened to the network) on port 3001 (`LENS_GRAFANA_PORT`). Anyone who reaches it can view +the dashboards, which hold no personal content; only Grafana's admin can change them. Prometheus and node-exporter are +reachable only inside Lens's Docker network. + +**Admin → System health** shows the server now (CPU, memory, load, uptime) and links to Grafana for history. + +On a small machine, the three add roughly 150-250 MB of memory; they are off unless the profile is on. Prometheus +(Apache 2.0), node-exporter (Apache 2.0) and Grafana OSS (AGPL 3.0) run as their own unmodified images, nothing of them +is built into Lens. + ## Turning it on - **Settings → Telemetry → Prometheus**: turn on **Prometheus endpoint** and set a **Scrape token** (16 to 200 @@ -48,9 +75,11 @@ Raspberry Pi is happy at that rate. ## Grafana dashboard -[`grafana/lens-dashboard.json`](grafana/lens-dashboard.json) is a starting dashboard: queue, running and failed jobs, +[`grafana/lens-dashboard.json`](grafana/lens-dashboard.json) is the dashboard the monitoring profile loads, and a +starting point for a Grafana of your own: queue, running and failed jobs, model cost over the last day, workers, disk; then jobs by status and pipeline, model calls, tokens, cost and failures -by model, budgets used, resources by status, imports and storage. In Grafana: **Dashboards → New → Import**, upload the +by model, budgets used, resources by status, imports and storage; and, from node-exporter, the machine's CPU, memory, +disk, load, network and uptime. In your own Grafana: **Dashboards → New → Import**, upload the file and pick your Prometheus data source. ## What it measures diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index b4fb20474..598d52c73 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -23,6 +23,7 @@ keyring, llm, local_llm, + machine, matterbridge, semantic, settings, @@ -326,6 +327,7 @@ def n(table: str) -> int: "sources": [{"id": x["id"], "name": x["name"], "type": x["type"], "health": x.get("health")} for x in sources.list_sources(db)], "disk": {"path": data_dir, "free_gb": round(disk.free / 1e9, 1), "total_gb": round(disk.total / 1e9, 1)}, "counts": {"recordings": n("recording"), "speakers": n("speaker"), "accounts": n("account")}, + "machine": machine.live(), } ) diff --git a/fastapi_backend/app/domain/machine.py b/fastapi_backend/app/domain/machine.py index cecf9bb50..ba14ac441 100644 --- a/fastapi_backend/app/domain/machine.py +++ b/fastapi_backend/app/domain/machine.py @@ -8,6 +8,8 @@ import platform import shutil import subprocess +import threading +import time def _memory_gb(): @@ -89,3 +91,80 @@ def probe(data_dir=None): out["disk_free_gb"] = None out["cuda"] = bool(out["gpus"]) return out + + +# ---------- right now, for Admin → System health (history lives in Prometheus and Grafana: docs/prometheus.md) ---------- +STARTED = time.time() # this process's start, for its uptime +_cpu = {"last": None} +_cpu_lock = threading.Lock() + + +def _cpu_times(): + try: + with open("/proc/stat", encoding="ascii") as f: + parts = [int(x) for x in f.readline().split()[1:]] + except (OSError, ValueError): + return None + return sum(parts), parts[3] + (parts[4] if len(parts) > 4 else 0) # all, idle + iowait + + +def cpu_pct(): + """Busy share of all CPUs since the last call (over a moment the first time), 0-100; None without /proc/stat.""" + now = _cpu_times() + if now is None: + return None + with _cpu_lock: + last, _cpu["last"] = _cpu["last"], now + if last is None: + time.sleep(0.2) + last, now = now, _cpu_times() or now + with _cpu_lock: + _cpu["last"] = now + total, idle = now[0] - last[0], now[1] - last[1] + return round(100.0 * (total - idle) / total, 1) if total > 0 else 0.0 + + +def _meminfo(): + info = {} + try: + with open("/proc/meminfo", encoding="ascii") as f: + for line in f: + k, _, v = line.partition(":") + info[k] = int(v.split()[0]) * 1024 + except (OSError, ValueError, IndexError): + return None, None + total, avail = info.get("MemTotal"), info.get("MemAvailable", info.get("MemFree")) + return (total - avail, total) if total and avail is not None else (None, None) + + +def _first_number(path, prefix=None, scale=1): + try: + with open(path, encoding="ascii") as f: + for line in f: + if prefix is None or line.startswith(prefix): + return int(float(line.split()[1 if prefix else 0]) * scale) + except (OSError, ValueError, IndexError): + pass + return None + + +def live(): + """The machine and this process now: CPU, memory, load, the API's memory and uptimes. Fields a system doesn't + report (no /proc outside Linux and Docker) are None.""" + used, total = _meminfo() + try: + load = [round(x, 2) for x in os.getloadavg()] + except (AttributeError, OSError): + load = [None, None, None] + return { + "cpu_pct": cpu_pct(), + "cpus": os.cpu_count(), + "mem_used": used, + "mem_total": total, + "load1": load[0], + "load5": load[1], + "load15": load[2], + "rss": _first_number("/proc/self/status", "VmRSS:", 1024), + "uptime": int(time.time() - STARTED), + "host_uptime": _first_number("/proc/uptime"), + } diff --git a/fastapi_backend/app/domain/prometheus.py b/fastapi_backend/app/domain/prometheus.py index c359c70c1..172ba587e 100644 --- a/fastapi_backend/app/domain/prometheus.py +++ b/fastapi_backend/app/domain/prometheus.py @@ -79,7 +79,9 @@ def text(self): # ---------- what's measured ---------- def _jobs(db, out, now): - for status, n in sorted(jobs.counts(db).items()): + got = jobs.counts(db) + for status in [*jobs.ACTIVE, *jobs.FINAL, *sorted(set(got) - {*jobs.ACTIVE, *jobs.FINAL})]: # 0 rather than missing + n = got.get(status, 0) out.add("lens_jobs", n, {"status": status}, "Jobs by status (queued is the queue's depth).") for r in db.rows("SELECT pipeline, status, count() AS n FROM job WHERE pipeline != NONE GROUP BY pipeline, status"): out.add("lens_pipeline_jobs", r["n"], {"pipeline": r.get("pipeline"), "status": r.get("status")}, "Jobs by pipeline and status.") diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index c9585bcf4..78d0a31eb 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -169,6 +169,7 @@ ("telemetry", "headers"): "LENS_TELEMETRY_HEADERS", ("telemetry", "prometheus"): "LENS_PROMETHEUS", ("telemetry", "prometheus_token"): "LENS_PROMETHEUS_TOKEN", + ("telemetry", "grafana_url"): "LENS_GRAFANA_URL", ("fedora", "enabled"): "LENS_FEDORA", ("fedora", "url"): "LENS_FEDORA_URL", ("fedora", "user"): "LENS_FEDORA_USER", @@ -877,6 +878,12 @@ def _telemetry_setting(key, value): if not isinstance(value, bool): raise ValueError(f"telemetry.{key} is true or false") return value + if key == "grafana_url": + if value in (None, ""): + return None + if not (isinstance(value, str) and VIEWER_URL.match(value.strip())): + raise ValueError("telemetry.grafana_url is Grafana's http(s) address, like http://localhost:3001") + return value.strip().rstrip("/") if key == "endpoint": if value is None or value == "": return None diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 514b5cec4..d7642ba4a 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -492,6 +492,8 @@ def labels(): # the scraper sends as a bearer token, and nothing is served until one is set "prometheus": False, "prometheus_token": None, + # where Grafana shows Lens's charts (docs/prometheus.md), for a link on Admin → System health; none: no link + "grafana_url": None, }, # new releases (docs/updates.md): check asks GitHub for repo's latest release every check_hours, for the web app's # banner; auto_install installs one as soon as it's seen, through the machine's upgrade agent (install.sh sets it up) diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index 6a843ad4b..fc0d7bb21 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -275,6 +275,19 @@ class HealthCounts(ResponseModel): accounts: int +class HealthMachine(ResponseModel): + cpu_pct: float | None = Field(default=None, description="busy share of all CPUs, 0-100; null where /proc isn't there") + cpus: int | None = None + mem_used: int | None = Field(default=None, description="bytes") + mem_total: int | None = Field(default=None, description="bytes") + load1: float | None = None + load5: float | None = None + load15: float | None = None + rss: int | None = Field(default=None, description="the API process's memory, bytes") + uptime: int = Field(description="seconds since the API started") + host_uptime: int | None = Field(default=None, description="seconds since the machine started") + + class Health(ResponseModel): database: HealthDatabase queue: dict[str, int] = Field(description="jobs by status") @@ -282,3 +295,4 @@ class Health(ResponseModel): sources: list[HealthSource] disk: HealthDisk counts: HealthCounts + machine: HealthMachine | None = Field(default=None, description="CPU, memory, load and uptime now") diff --git a/fastapi_backend/tests/api/test_telemetry.py b/fastapi_backend/tests/api/test_telemetry.py index dc90423b6..5d3d40f85 100644 --- a/fastapi_backend/tests/api/test_telemetry.py +++ b/fastapi_backend/tests/api/test_telemetry.py @@ -252,3 +252,14 @@ def test_a_new_workers_first_job_is_traced(client, db, cfg, folder, collector): assert worker.drain() == 1 telemetry.flush() assert "job" in {s["name"] for s in fake_otlp.Handler.spans} + + +def test_grafana_link_and_machine_now(client, db): + h = admin(client, db) + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["grafana_url"] is None + assert client.put("/api/v1/settings/telemetry", json={"grafana_url": "ftp://x"}, headers=h).status_code == 400 + r = client.put("/api/v1/settings/telemetry", json={"grafana_url": "http://localhost:3001/"}, headers=h) + assert r.status_code == 200 + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["grafana_url"] == "http://localhost:3001" + m = client.get("/api/v1/admin/health", headers=h).json()["machine"] + assert m["uptime"] >= 0 and m["cpus"] >= 1 diff --git a/monitoring/grafana/provisioning/dashboards/lens.yml b/monitoring/grafana/provisioning/dashboards/lens.yml new file mode 100644 index 000000000..817b0f94d --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/lens.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: Lens + folder: Lens + type: file + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/lens.yml b/monitoring/grafana/provisioning/datasources/lens.yml new file mode 100644 index 000000000..433ef5c80 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/lens.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 000000000..64805cf34 --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,18 @@ +# Prometheus for the `monitoring` profile (docs/prometheus.md): scrapes Lens's /metrics with the scrape token and the +# machine through node-exporter, and keeps LENS_METRICS_DAYS of history (30 by default). +global: + scrape_interval: 60s + scrape_timeout: 20s + +scrape_configs: + - job_name: lens + metrics_path: /metrics + authorization: + type: Bearer + credentials_file: /prometheus/lens-token # written from LENS_PROMETHEUS_TOKEN when the container starts + static_configs: + - targets: ["backend:8000"] + + - job_name: machine + static_configs: + - targets: ["node-exporter:9100"] diff --git a/nextjs-frontend/__tests__/health-duration.test.ts b/nextjs-frontend/__tests__/health-duration.test.ts new file mode 100644 index 000000000..d5c1d1bd5 --- /dev/null +++ b/nextjs-frontend/__tests__/health-duration.test.ts @@ -0,0 +1,14 @@ +/** @jest-environment node */ +import { duration } from "@/components/admin/health"; + +jest.mock("@/app/openapi-client", () => ({})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: null }) })); + +describe("duration", () => { + it("says how long something has been up", () => { + expect(duration(420)).toBe("7 min"); + expect(duration(5 * 3600 + 12 * 60)).toBe("5 h 12 min"); + expect(duration(3 * 86400 + 4 * 3600)).toBe("3 d 4 h"); + expect(duration(null)).toBe("–"); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index af944d0c7..ed86cb69f 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1222,6 +1222,7 @@ export type { HealthCounts, HealthDatabase, HealthDisk, + HealthMachine, HealthSource, Heard, HeldRunDecision, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index a8fc4a7c1..3c981cd41 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -5314,6 +5314,10 @@ export type Health = { sources: Array; disk: HealthDisk; counts: HealthCounts; + /** + * CPU, memory, load and uptime now + */ + machine?: HealthMachine | null; [key: string]: unknown; }; @@ -5380,6 +5384,65 @@ export type HealthDisk = { [key: string]: unknown; }; +/** + * HealthMachine + */ +export type HealthMachine = { + /** + * Cpu Pct + * + * busy share of all CPUs, 0-100; null where /proc isn't there + */ + cpu_pct?: number | null; + /** + * Cpus + */ + cpus?: number | null; + /** + * Mem Used + * + * bytes + */ + mem_used?: number | null; + /** + * Mem Total + * + * bytes + */ + mem_total?: number | null; + /** + * Load1 + */ + load1?: number | null; + /** + * Load5 + */ + load5?: number | null; + /** + * Load15 + */ + load15?: number | null; + /** + * Rss + * + * the API process's memory, bytes + */ + rss?: number | null; + /** + * Uptime + * + * seconds since the API started + */ + uptime: number; + /** + * Host Uptime + * + * seconds since the machine started + */ + host_uptime?: number | null; + [key: string]: unknown; +}; + /** * HealthSource */ diff --git a/nextjs-frontend/components/admin/health.tsx b/nextjs-frontend/components/admin/health.tsx index 87de35889..d3baa10dc 100644 --- a/nextjs-frontend/components/admin/health.tsx +++ b/nextjs-frontend/components/admin/health.tsx @@ -11,7 +11,7 @@ import { isUnreachable, ServerUnreachable } from "@/components/errors/error-stat import { Button } from "@/components/ui/button"; import { EmptyState, Skeleton } from "@/components/ui/states"; import { data, useApiClient } from "@/lib/api/browser"; -import { count, relative } from "@/lib/format"; +import { bytes, count, relative } from "@/lib/format"; import { jobSteps } from "@/lib/hooks/jobs"; import { cn } from "@/lib/utils"; @@ -326,6 +326,85 @@ export function HealthPage() { + ); } + +/** Seconds as "3 d 4 h", "5 h 12 min" or "7 min". */ +export function duration(s: number | null | undefined): string { + if (s === null || s === undefined) return "–"; + const d = Math.floor(s / 86400); + const h = Math.floor((s % 86400) / 3600); + const m = Math.floor((s % 3600) / 60); + if (d) return `${d} d ${h} h`; + if (h) return `${h} h ${m} min`; + return `${m} min`; +} + +type Machine = { + cpu_pct?: number | null; + cpus?: number | null; + mem_used?: number | null; + mem_total?: number | null; + load1?: number | null; + load5?: number | null; + load15?: number | null; + rss?: number | null; + uptime: number; + host_uptime?: number | null; +}; + +/** The server now (CPU, memory, load, uptime), and where its history is: Grafana, from the monitoring profile. */ +function ServerNow({ machine: m, grafana }: { machine?: Machine | null; grafana?: string | null }) { + if (!m) return null; + const memPct = m.mem_used != null && m.mem_total ? Math.round((m.mem_used / m.mem_total) * 100) : null; + const tiles: [string, string, string][] = [ + ["CPU", m.cpu_pct != null ? `${Math.round(m.cpu_pct)}%` : "–", `${m.cpus ?? "?"} CPUs`], + [ + "Memory", + memPct != null ? `${memPct}%` : "–", + m.mem_total ? `${bytes(m.mem_used)} of ${bytes(m.mem_total)} · Lens API ${bytes(m.rss)}` : "Not reported here", + ], + ["Load", m.load1 != null ? String(m.load1) : "–", `5 min ${m.load5 ?? "–"} · 15 min ${m.load15 ?? "–"}`], + [ + "Uptime", + duration(m.uptime), + m.host_uptime != null ? `machine up ${duration(m.host_uptime)}` : "since Lens started", + ], + ]; + return ( +
+
+ Server now + {grafana ? ( + + Charts and history in Grafana + + ) : ( + + For charts and history, run Prometheus and Grafana with the monitoring profile and set its address in{" "} + + Settings → Telemetry + + . + + )} +
+
+ {tiles.map(([name, value, detail]) => ( +
+ {name} + {value} + {detail} +
+ ))} +
+
+ ); +} diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 636b51e80..e311e18cc 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -613,8 +613,8 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) {

For Grafana and other Prometheus tools: jobs, imports, model calls, cost, budgets, workers and disk, served at{" "} /metrics on this address with the same privacy as telemetry. A scraper sends - the token as Authorization: Bearer; see docs/prometheus.md for a scrape - config and a sample dashboard. + the token as Authorization: Bearer. The monitoring profile runs Prometheus + and Grafana next to Lens, ready made, with charts of Lens and this machine (docs/prometheus.md).

{Boolean(saved.prometheus) && !promSecret.set && !promToken.value && ( @@ -634,6 +634,7 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) { value={promToken.value as string | undefined} onChange={(x) => promToken.onChange(x)} /> + ); } diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 9e2a23823..df52c54d5 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -1966,6 +1966,16 @@ export const FIELDS: FieldSpec[] = [ hint: "Serve /metrics for a Prometheus scraper; off by default, and answered only with the scrape token", }, { section: "telemetry", key: "prometheus_token", label: "Scrape token", kind: "secret" }, + { + section: "telemetry", + key: "grafana_url", + label: "Grafana address", + kind: "text", + nullable: true, + mono: true, + placeholder: "http://localhost:3001", + hint: "Where Grafana shows Lens's charts; System health links to it. The monitoring profile runs one at :3001", + }, { section: "telemetry", key: "prices", diff --git a/nextjs-frontend/components/settings/section.tsx b/nextjs-frontend/components/settings/section.tsx index 9cfd075db..59efaf404 100644 --- a/nextjs-frontend/components/settings/section.tsx +++ b/nextjs-frontend/components/settings/section.tsx @@ -47,6 +47,7 @@ const ENV: Record = { "telemetry.headers": "LENS_TELEMETRY_HEADERS", "telemetry.prometheus": "LENS_PROMETHEUS", "telemetry.prometheus_token": "LENS_PROMETHEUS_TOKEN", + "telemetry.grafana_url": "LENS_GRAFANA_URL", "fedora.enabled": "LENS_FEDORA", "fedora.url": "LENS_FEDORA_URL", "fedora.user": "LENS_FEDORA_USER", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index e10371c6f..69afa80d0 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -33586,6 +33586,17 @@ }, "counts": { "$ref": "#/components/schemas/HealthCounts" + }, + "machine": { + "anyOf": [ + { + "$ref": "#/components/schemas/HealthMachine" + }, + { + "type": "null" + } + ], + "description": "CPU, memory, load and uptime now" } }, "additionalProperties": true, @@ -33684,6 +33695,125 @@ ], "title": "HealthDisk" }, + "HealthMachine": { + "properties": { + "cpu_pct": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cpu Pct", + "description": "busy share of all CPUs, 0-100; null where /proc isn't there" + }, + "cpus": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Cpus" + }, + "mem_used": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mem Used", + "description": "bytes" + }, + "mem_total": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mem Total", + "description": "bytes" + }, + "load1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Load1" + }, + "load5": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Load5" + }, + "load15": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Load15" + }, + "rss": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rss", + "description": "the API process's memory, bytes" + }, + "uptime": { + "type": "integer", + "title": "Uptime", + "description": "seconds since the API started" + }, + "host_uptime": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Host Uptime", + "description": "seconds since the machine started" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "uptime" + ], + "title": "HealthMachine" + }, "HealthSource": { "properties": { "id": {