From 05f303a38996f1d037a83fe0a56eb10192774c73 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:05:50 +0000 Subject: [PATCH] feat(telemetry): Prometheus /metrics endpoint with a sample Grafana dashboard Off by default; turned on in Settings > Telemetry (or LENS_PROMETHEUS), /metrics answers only a scraper with the scrape token and is served where the web app is, so it follows LENS_BIND. Exposes jobs by status and pipeline, routine runs, resources and imports, the activity ledger's calls, tokens, cost and failures by model, budgets, workers and disk. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XCh396xzDRUrSuH1xohcXR --- CHANGELOG.md | 5 + docs/grafana/lens-dashboard.json | 596 ++++++++++++++++++ docs/prometheus.md | 96 +++ docs/telemetry.md | 2 + fastapi_backend/app/api/metrics.py | 26 + fastapi_backend/app/domain/prometheus.py | 240 +++++++ fastapi_backend/app/domain/settings.py | 17 +- fastapi_backend/app/domain/store.py | 4 + fastapi_backend/app/main.py | 2 + fastapi_backend/tests/api/test_prometheus.py | 72 +++ mkdocs.yml | 1 + nextjs-frontend/app/metrics/route.ts | 6 + .../components/settings/bodies.tsx | 27 + nextjs-frontend/components/settings/model.ts | 16 + .../components/settings/section.tsx | 2 + nextjs-frontend/proxy.ts | 4 +- 16 files changed, 1111 insertions(+), 5 deletions(-) create mode 100644 docs/grafana/lens-dashboard.json create mode 100644 docs/prometheus.md create mode 100644 fastapi_backend/app/api/metrics.py create mode 100644 fastapi_backend/app/domain/prometheus.py create mode 100644 fastapi_backend/tests/api/test_prometheus.py create mode 100644 nextjs-frontend/app/metrics/route.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5112764c6..10ea98810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Prometheus metrics.** Lens can serve `/metrics` for Prometheus and Grafana: jobs by status and pipeline, imports, + model calls, tokens, cost and failures by model, budgets, workers and disk. Off by default; turned on in Settings → + Telemetry (or `LENS_PROMETHEUS=on`), it answers only a scraper with the scrape token, and follows `LENS_BIND`. A + sample Grafana dashboard comes with it ([Prometheus](docs/prometheus.md)). + - **Install on Windows, or by double-clicking.** `install.ps1` installs Lens on Windows: it gets Python 3 if needed and opens the web installer, which installs Docker Desktop (and WSL) and runs Lens in it. Each release now carries launchers that need no terminal at all: **Install Lens.app** for macOS, **Install Lens.cmd** for Windows and diff --git a/docs/grafana/lens-dashboard.json b/docs/grafana/lens-dashboard.json new file mode 100644 index 000000000..c0a81b4b9 --- /dev/null +++ b/docs/grafana/lens-dashboard.json @@ -0,0 +1,596 @@ +{ + "title": "Lens", + "uid": "lens-overview", + "schemaVersion": 39, + "version": 1, + "editable": true, + "tags": [ + "lens" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "1m", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "label": "Prometheus", + "current": {} + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Queued jobs", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(lens_jobs{status=\"queued\"})", + "legendFormat": "queued", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 2, + "title": "Running jobs", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(lens_jobs{status=\"running\"})", + "legendFormat": "running", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 3, + "title": "Failed jobs", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(lens_jobs{status=\"failed\"})", + "legendFormat": "failed", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 4, + "title": "Model cost, last 24h", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(lens_activity_cost_usd{window=\"24h\"})", + "legendFormat": "USD", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 5, + "title": "Workers up", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "lens_workers_up", + "legendFormat": "up", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 6, + "title": "Disk free", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "lens_disk_free_bytes", + "legendFormat": "free", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 7, + "title": "Jobs by status", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (status) (lens_jobs)", + "legendFormat": "{{status}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 8, + "title": "Jobs by pipeline", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (pipeline) (lens_pipeline_jobs{status=~\"queued|running|held\"})", + "legendFormat": "{{pipeline}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 9, + "title": "Model calls per hour, by model", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 12, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (model) (lens_activity_calls{window=\"1h\",kind=\"out\"})", + "legendFormat": "{{model}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 10, + "title": "Tokens per hour, by model", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 12, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (model, direction) (lens_activity_tokens{window=\"1h\"})", + "legendFormat": "{{model}} {{direction}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 11, + "title": "Cost per hour, by model", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 12, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (model) (lens_activity_cost_usd{window=\"1h\"})", + "legendFormat": "{{model}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 12, + "title": "Failed calls per hour", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 20, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (kind, model) (lens_activity_failed{window=\"1h\"})", + "legendFormat": "{{kind}} {{model}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 13, + "title": "Budgets used", + "type": "bargauge", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 20, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "lens_budget_share", + "legendFormat": "{{resource}} ({{period}})", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 14, + "title": "Resources by status", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 20, + "w": 8, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (status) (lens_resources)", + "legendFormat": "{{status}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 15, + "title": "Imports", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "lens_imports{window=\"1h\"}", + "legendFormat": "last hour", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "lens_imports{window=\"24h\"}", + "legendFormat": "last 24h", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 16, + "title": "Storage", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 28, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "lens_data_bytes", + "legendFormat": "data folder", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "lens_disk_free_bytes", + "legendFormat": "disk free", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + } + ], + "annotations": { + "list": [] + } +} diff --git a/docs/prometheus.md b/docs/prometheus.md new file mode 100644 index 000000000..d4adfe1fb --- /dev/null +++ b/docs/prometheus.md @@ -0,0 +1,96 @@ +# Prometheus + +Lens can serve a [Prometheus](https://prometheus.io/) scrape endpoint at `/metrics`, so Prometheus, Grafana Agent, +VictoriaMetrics or any other scraper can chart what Lens is doing and what it costs: jobs and pipelines, imports, model +calls, tokens and cost, budgets, workers and the disk. + +**It is off unless you turn it on**, and even then it answers only a scraper that sends the scrape token. It is a pull +endpoint: Lens sends nothing anywhere. For pushing traces and metrics to an OpenTelemetry collector instead, see +[Telemetry](telemetry.md); the two are independent. + +## Turning it on + +- **Settings → Telemetry → Prometheus**: turn on **Prometheus endpoint** and set a **Scrape token** (16 to 200 + letters, digits, `.`, `_`, `~` and `-`; `openssl rand -hex 32` makes a good one). The token is stored encrypted and + never shown again. +- **.env**, which wins over Settings and shows those fields locked there: + + ```sh + LENS_PROMETHEUS=on + LENS_PROMETHEUS_TOKEN=3f9c... # openssl rand -hex 32 + ``` + +Until a token is set, `/metrics` answers 401 to everyone. While it is off, it answers 404. + +## Where it is + +`/metrics` is served by the web app's address, like the API (`http://:3000/metrics` on a default install), so it +follows `LENS_BIND`: on this machine only, unless Lens was opened to the network. A Prometheus on the same machine +scrapes it as it is; one elsewhere needs Lens reachable from it (or the remote access tunnel). The API answers on +`/metrics` itself too, for a scraper inside the Docker network (`http://backend:8000/metrics`). + +## Scrape config + +```yaml +scrape_configs: + - job_name: lens + scrape_interval: 60s + metrics_path: /metrics + authorization: + type: Bearer + credentials_file: /etc/prometheus/lens-token # the scrape token, on one line + static_configs: + - targets: ["localhost:3000"] +``` + +Each scrape runs a few small queries on the database (grouped counts and sums), so once a minute is plenty; a +Raspberry Pi is happy at that rate. + +## Grafana dashboard + +[`grafana/lens-dashboard.json`](grafana/lens-dashboard.json) is a starting dashboard: queue, running and failed jobs, +model cost over the last day, workers, disk; then jobs by status and pipeline, model calls, tokens, cost and failures +by model, budgets used, resources by status, imports and storage. In Grafana: **Dashboards → New → Import**, upload the +file and pick your Prometheus data source. + +## What it measures + +Everything is read from what Lens already keeps when Prometheus scrapes, so the numbers are the same ones the web app +shows. All are gauges. + +| Metric | Labels | What it is | +| --- | --- | --- | +| `lens_build_info` | `version` | Always 1; the version running. | +| `lens_jobs` | `status` | Jobs by status: `queued` (the queue's depth), `running`, `held` (over budget), `succeeded`, `failed`, `cancelled`. | +| `lens_pipeline_jobs` | `pipeline`, `status` | The same, for jobs a pipeline queued. | +| `lens_routine_runs` | `status` | Routine runs kept, by status. | +| `lens_resources` | `status` | Resources (recordings and documents) by status; `error` counts failed imports and processing. | +| `lens_imports` | `window` | Resources imported in the last `1h` or `24h`. | +| `lens_activity_calls` | `window`, `kind`, `model` | Calls in the [activity ledger](activity.md): `in` (API changes), `out` (model calls, embeddings, webhooks…), `run` (jobs and routine runs ending). | +| `lens_activity_failed` | `window`, `kind`, `model` | Those that failed. | +| `lens_activity_seconds` | `window`, `kind`, `model` | Time spent in them. | +| `lens_activity_tokens` | `window`, `kind`, `model`, `direction` | Tokens in and out. | +| `lens_activity_cost_usd` | `window`, `kind`, `model` | Estimated cost, from the prices in Settings → Telemetry. | +| `lens_activity_unpriced` | `window`, `kind`, `model` | Priced calls whose cost is missing (no token counts): the cost shown is a floor. | +| `lens_budget_limit_usd`, `lens_budget_limit_tokens` | `resource`, `period` | Each [budget](budgets.md)'s caps. | +| `lens_budget_spent_usd`, `lens_budget_spent_tokens` | `resource`, `period` | What it spent this period. | +| `lens_budget_share` | `resource`, `period` | The largest share of its caps spent (1 is the cap). | +| `lens_budget_over` | `resource`, `period` | 1 when over its cap. | +| `lens_workers`, `lens_workers_up` | | Workers that have checked in; those seen in the last 2 minutes and not paused. | +| `lens_disk_total_bytes`, `lens_disk_free_bytes` | | The disk the data folder is on. | +| `lens_data_bytes` | | The data folder's size (database and files), measured in the background at most every 10 minutes. | +| `lens_scrape_errors`, `lens_scrape_seconds` | | Parts of the scrape that failed (see the API's log), and how long it took. | + +`window` is `1h` or `24h`: the ledger's totals over the last hour and day, rather than counters, because the ledger +drops rows after `activity.keep_days` and a counter must never go down. For a month's spend, use the budgets. + +## Privacy + +Like telemetry, the metrics carry no personal content: no titles, file names, namespace names, prompts, answers, +people or addresses. Labels are statuses, ids (`routine:3`, a pipeline's id), model names and windows. + +## Refine later + +- Job step durations as histograms (from `step_stat`). +- Per-namespace breakdowns, opt-in, since namespace names are personal. +- The database's own size apart from files. diff --git a/docs/telemetry.md b/docs/telemetry.md index 0c781c1f3..ea8774b1a 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -9,6 +9,8 @@ never sends telemetry to the Lens project or anyone else. Turned on, it sends on a collector on the same machine or network, which can pass it on to Jaeger, Grafana Tempo and Prometheus, SigNoz, Honeycomb, or whatever you use. +To have Prometheus scrape Lens instead, see [Prometheus](prometheus.md). + ## Turning it on Any of these: diff --git a/fastapi_backend/app/api/metrics.py b/fastapi_backend/app/api/metrics.py new file mode 100644 index 000000000..6ba47dbc2 --- /dev/null +++ b/fastapi_backend/app/api/metrics.py @@ -0,0 +1,26 @@ +"""GET /metrics: the Prometheus scrape endpoint (docs/prometheus.md), off by default and answered only with its token.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request +from fastapi.responses import PlainTextResponse, Response + +from app.api.deps import Cfg, Db +from app.domain import prometheus + +router = APIRouter(tags=["metrics"]) +CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8" + + +@router.get("/metrics", include_in_schema=False) +def metrics(request: Request, db: Db, cfg: Cfg) -> Response: + """Prometheus' text format, for a scraper that sends `Authorization: Bearer `.""" + if not prometheus.enabled(cfg): + return PlainTextResponse("Not Found\n", status_code=404) + if not prometheus.allowed(cfg, request.headers.get("authorization")): + return PlainTextResponse( + "a scrape token is needed: Authorization: Bearer \n", + status_code=401, + headers={"WWW-Authenticate": 'Bearer realm="lens-metrics"'}, + ) + return Response(prometheus.render(db), media_type=CONTENT_TYPE, headers={"Cache-Control": "no-store"}) diff --git a/fastapi_backend/app/domain/prometheus.py b/fastapi_backend/app/domain/prometheus.py new file mode 100644 index 000000000..c359c70c1 --- /dev/null +++ b/fastapi_backend/app/domain/prometheus.py @@ -0,0 +1,240 @@ +"""A Prometheus scrape endpoint, /metrics (docs/prometheus.md): what Lens is doing and what it costs, for Grafana. + +Off by default (telemetry.prometheus). Turned on, GET /metrics answers only a scraper that sends the token set in +telemetry.prometheus_token (`Authorization: Bearer `), and is served where the web app is, so it follows +LENS_BIND: this machine only unless Lens was opened to the network. + +Everything is read when Prometheus scrapes, from the tables Lens already keeps: jobs by status and pipeline, routine +runs, resources by status and imports, the activity ledger (calls, tokens, cost and failures by kind and model over +the last hour and day), budgets, workers and the disk. Like telemetry, it carries no personal content: no titles, +file names, namespace names, prompts or user ids; only statuses, ids, model names, counts and sizes. +""" + +from __future__ import annotations + +import datetime as dt +import hmac +import logging +import os +import shutil +import threading +import time + +from . import __version__, activity, budgets, jobs, library + +log = logging.getLogger("lens") +WINDOWS = (("1h", dt.timedelta(hours=1)), ("24h", dt.timedelta(hours=24))) +SIZE_SECONDS = 600 # how long a measured data folder size is reused: walking a big archive isn't free +WORKER_FRESH = 120 # seconds since a worker's last heartbeat for it to count as up + +_size = {"at": 0.0, "bytes": None, "path": None, "busy": False} +_size_lock = threading.Lock() + + +def _section(cfg): + return (cfg or {}).get("telemetry") or {} + + +def enabled(cfg): + return bool(_section(cfg).get("prometheus")) + + +def allowed(cfg, authorization): + """Whether an Authorization header carries the scrape token. No token set: nobody may scrape.""" + token = _section(cfg).get("prometheus_token") + if not token or not authorization: + return False + scheme, _, given = authorization.partition(" ") + return scheme.lower() == "bearer" and hmac.compare_digest(given.strip().encode(), str(token).encode()) + + +# ---------- the text format ---------- +def _esc(v): + return str(v).replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +class Out: + """Metrics in Prometheus' text exposition format, each family's HELP and TYPE once, before its samples.""" + + def __init__(self): + self.families = {} + + def add(self, name, value, labels=None, help="", kind="gauge"): + fam = self.families.setdefault(name, {"help": help, "kind": kind, "samples": []}) + if value is None: + return + fam["samples"].append((labels or {}, float(value))) + + def text(self): + lines = [] + for name, f in self.families.items(): + lines.append(f"# HELP {name} {f['help']}") + lines.append(f"# TYPE {name} {f['kind']}") + for labels, v in f["samples"]: + lab = ",".join(f'{k}="{_esc(x)}"' for k, x in labels.items()) + num = repr(v) if v != int(v) else str(int(v)) + lines.append(f"{name}{{{lab}}} {num}" if lab else f"{name} {num}") + return "\n".join(lines) + "\n" + + +# ---------- what's measured ---------- +def _jobs(db, out, now): + for status, n in sorted(jobs.counts(db).items()): + out.add("lens_jobs", n, {"status": status}, "Jobs by status (queued is the queue's depth).") + for r in db.rows("SELECT pipeline, status, count() AS n FROM job WHERE pipeline != NONE GROUP BY pipeline, status"): + out.add("lens_pipeline_jobs", r["n"], {"pipeline": r.get("pipeline"), "status": r.get("status")}, "Jobs by pipeline and status.") + out.families.setdefault("lens_pipeline_jobs", {"help": "Jobs by pipeline and status.", "kind": "gauge", "samples": []}) + + +def _routines(db, out, now): + for r in db.rows("SELECT status, count() AS n FROM routine_run GROUP BY status"): + out.add("lens_routine_runs", r["n"], {"status": r.get("status") or "?"}, "Routine runs kept, by status.") + out.families.setdefault("lens_routine_runs", {"help": "Routine runs kept, by status.", "kind": "gauge", "samples": []}) + + +def _resources(db, out, now): + got = {r.get("status"): r["n"] for r in db.rows("SELECT status, count() AS n FROM recording GROUP BY status")} + for s in library.STATUSES: + out.add( + "lens_resources", + got.get(s, 0), + {"status": s}, + "Resources (recordings and documents) by status; error is failed imports or processing.", + ) + for name, span in WINDOWS: + since = (now - span).isoformat(timespec="seconds") + n = db.one("SELECT count() AS n FROM recording WHERE created_at >= $s GROUP ALL", s=since) or {} + out.add("lens_imports", n.get("n") or 0, {"window": name}, "Resources imported in the window.") + + +def _ledger(db, out, now): + help_ = { + "lens_activity_calls": "Calls in the activity ledger over the window, by kind (in, out, run) and model.", + "lens_activity_failed": "Calls that failed over the window, by kind and model.", + "lens_activity_tokens": "Tokens over the window, by model and direction (in, out).", + "lens_activity_cost_usd": "Estimated cost in USD over the window, by model (from the prices in Settings).", + "lens_activity_unpriced": "Priced calls whose cost is missing (no token counts) over the window: the cost is a floor.", + "lens_activity_seconds": "Time spent in calls over the window, by kind and model.", + } + for name in help_: + out.families.setdefault(name, {"help": help_[name], "kind": "gauge", "samples": []}) + for window, span in WINDOWS: + since = (now - span).isoformat(timespec="seconds") + rows = db.rows( + "SELECT kind, model, count() AS calls, count(ok = false) AS failed, math::sum(tokens_in ?? 0) AS tokens_in, " + "math::sum(tokens_out ?? 0) AS tokens_out, math::sum(cost_usd ?? 0) AS cost_usd, math::sum(ms ?? 0) AS ms, " + f"count({activity._unpriced_sql()}) AS unpriced FROM activity WHERE at >= $s GROUP BY kind, model", + s=since, + ) + for r in rows: + kind, model = r.get("kind") or "?", r.get("model") or "" + lab = {"window": window, "kind": kind, "model": model} + out.add("lens_activity_calls", r.get("calls") or 0, lab) + out.add("lens_activity_failed", r.get("failed") or 0, lab) + out.add("lens_activity_seconds", (r.get("ms") or 0) / 1000, lab) + if kind == "run": # a run row repeats what its calls cost: counted once, on the calls + continue + if r.get("tokens_in") or r.get("tokens_out") or model: + out.add("lens_activity_tokens", r.get("tokens_in") or 0, {**lab, "direction": "in"}) + out.add("lens_activity_tokens", r.get("tokens_out") or 0, {**lab, "direction": "out"}) + out.add("lens_activity_cost_usd", round(float(r.get("cost_usd") or 0), 6), lab) + out.add("lens_activity_unpriced", r.get("unpriced") or 0, lab) + + +def _budgets(db, out, now): + help_ = { + "lens_budget_limit_usd": "A budget's cap in USD, by resource and period.", + "lens_budget_limit_tokens": "A budget's cap in tokens, by resource and period.", + "lens_budget_spent_usd": "What a budgeted resource spent this period, in USD.", + "lens_budget_spent_tokens": "Tokens a budgeted resource used this period.", + "lens_budget_share": "The largest share of a budget's caps spent this period (1 is the cap).", + "lens_budget_over": "1 when a budget is over its cap, else 0.", + } + for name in help_: + out.families.setdefault(name, {"help": help_[name], "kind": "gauge", "samples": []}) + for b in budgets.all_budgets(db): + st = budgets.status(db, b["resource"], b, now) + lab = {"resource": b["resource"], "period": b.get("period") or "month"} + out.add("lens_budget_limit_usd", b.get("usd"), lab) + out.add("lens_budget_limit_tokens", b.get("tokens"), lab) + spent = st.get("spent") or {} + out.add("lens_budget_spent_usd", spent.get("usd"), lab) + out.add("lens_budget_spent_tokens", spent.get("tokens"), lab) + out.add("lens_budget_share", st.get("share"), lab) + out.add("lens_budget_over", 1 if st.get("state") == "over" else 0, lab) + + +def _workers(db, out, now): + rows = db.rows("SELECT record::id(id) AS name, heartbeat_at, paused FROM worker") + up = 0 + for w in rows: + try: + beat = dt.datetime.fromisoformat(str(w.get("heartbeat_at"))) + if beat.tzinfo is None: + beat = beat.replace(tzinfo=dt.timezone.utc) + fresh = (now - beat).total_seconds() <= WORKER_FRESH + except ValueError: + fresh = False + up += 1 if fresh and not w.get("paused") else 0 + out.add("lens_workers", len(rows), None, "Workers that have checked in.") + out.add("lens_workers_up", up, None, f"Workers that checked in within {WORKER_FRESH} seconds and aren't paused.") + + +def _measure(path): + total = 0 + for root, _dirs, files in os.walk(path, onerror=lambda e: None): + for f in files: + try: + total += os.lstat(os.path.join(root, f)).st_size + except OSError: + pass + with _size_lock: + _size.update(at=time.monotonic(), bytes=total, path=path, busy=False) + + +def data_size(path): + """The data folder's size in bytes, measured in the background at most every SIZE_SECONDS; None until first measured.""" + with _size_lock: + stale = _size["path"] != path or time.monotonic() - _size["at"] > SIZE_SECONDS or _size["bytes"] is None + if stale and not _size["busy"]: + _size["busy"] = True + threading.Thread(target=_measure, args=(path,), name="lens-data-size", daemon=True).start() + return _size["bytes"] if _size["path"] == path else None + + +def _storage(db, out, now): + path = str(db.data_dir or "") + if not path or not os.path.isdir(path): + return + try: + du = shutil.disk_usage(path) + except OSError: + du = None + if du: + out.add("lens_disk_total_bytes", du.total, None, "Size of the disk the data folder is on.") + out.add("lens_disk_free_bytes", du.free, None, "Free space on the disk the data folder is on.") + out.add( + "lens_data_bytes", + data_size(path), + None, + f"Size of the data folder (database and files), measured at most every {SIZE_SECONDS} seconds.", + ) + + +def render(db, now=None): + """Every metric, as Prometheus' text format. A part that fails is skipped and counted in lens_scrape_errors.""" + t = time.monotonic() + now = now or dt.datetime.now(dt.timezone.utc) + out = Out() + out.add("lens_build_info", 1, {"version": __version__}, "The version of Lens running.") + errors = 0 + parts = (_jobs, _routines, _resources, _ledger, _budgets, _workers, _storage) + for part in parts: + try: + part(db, out, now) + except Exception: # noqa: BLE001 - one broken query mustn't lose the rest of the scrape + log.exception("prometheus: %s failed", part.__name__) + errors += 1 + out.add("lens_scrape_errors", errors, None, "Parts of this scrape that failed (see the API's log).") + out.add("lens_scrape_seconds", round(time.monotonic() - t, 4), None, "How long this scrape took.") + return out.text() diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 4941151ea..257b0fd41 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -116,7 +116,7 @@ "mail": ("password",), "bridge": ("token",), "matterbridge": ("slack_token", "discord_token", "telegram_token", "matrix_password"), - "telemetry": ("headers",), + "telemetry": ("headers", "prometheus_token"), "fedora": ("password",), "tunnel": ("token", "api_token"), "documents": ("anytopdf_token",), @@ -167,6 +167,8 @@ ("telemetry", "enabled"): "LENS_TELEMETRY", ("telemetry", "endpoint"): "LENS_TELEMETRY_ENDPOINT", ("telemetry", "headers"): "LENS_TELEMETRY_HEADERS", + ("telemetry", "prometheus"): "LENS_PROMETHEUS", + ("telemetry", "prometheus_token"): "LENS_PROMETHEUS_TOKEN", ("fedora", "enabled"): "LENS_FEDORA", ("fedora", "url"): "LENS_FEDORA_URL", ("fedora", "user"): "LENS_FEDORA_USER", @@ -851,10 +853,19 @@ def _sensor_setting(key, value): TELEMETRY_RANGES = {"export_seconds": (5, 3600)} SERVICE_RX = re.compile(r"^[A-Za-z0-9._-]{1,64}$") +TOKEN_RX = re.compile(r"^[A-Za-z0-9._~-]+$") def _telemetry_setting(key, value): - if key in ("enabled", "traces", "metrics"): + if key == "prometheus_token": + if value in (None, ""): + return None + if not (isinstance(value, str) and 16 <= len(value.strip()) <= 200 and TOKEN_RX.match(value.strip())): + raise ValueError( + "telemetry.prometheus_token is 16 to 200 letters, digits, ., _, ~ and -, like the output of openssl rand -hex 32" + ) + return value.strip() + if key in ("enabled", "traces", "metrics", "prometheus"): if not isinstance(value, bool): raise ValueError(f"telemetry.{key} is true or false") return value @@ -980,7 +991,7 @@ def save(db, base, section, changes, user=None): telemetry.parse_headers(v) # ValueError when malformed if section == "matterbridge" and (len(v) > 500 or "\n" in v.strip()): raise ValueError(f"matterbridge.{k} is one line of text") - if (section, k) == ("documents", "anytopdf_token"): + if (section, k) in (("documents", "anytopdf_token"), ("telemetry", "prometheus_token")): v = _check(section, k, v, None) sealed[k] = seal(base, v, f"setting:{section}.{k}") elif not (isinstance(v, dict) and v.get("secret")): # the mask echoed back means "unchanged" diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index cbae2c2df..0003599dd 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -481,6 +481,10 @@ def labels(): "export_seconds": 60, "service_name": "lens", "prices": {}, + # a Prometheus scrape endpoint at /metrics (docs/prometheus.md): off by default; prometheus_token is a secret + # the scraper sends as a bearer token, and nothing is served until one is set + "prometheus": False, + "prometheus_token": None, }, # new releases (docs/updates.md): check asks GitHub for repo's latest release every check_hours, for the web app's # banner; auto_install installs one as soon as it's seen, through the machine's upgrade agent (install.sh sets it up) diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py index 96e31f1df..ba386c867 100644 --- a/fastapi_backend/app/main.py +++ b/fastapi_backend/app/main.py @@ -19,6 +19,7 @@ from app.api.iiif import router as iiif_router from app.api.linked_data import router as linked_data_router from app.api.mcp import router as mcp_router +from app.api.metrics import router as metrics_router from app.api.pages import router as pages_router from app.api.v1.router import api_router from app.api.v1.routes.oauth import bearer_challenge @@ -89,6 +90,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.include_router(api_router, prefix="/api/v1") app.include_router(oauth_well_known) app.include_router(mcp_router) + app.include_router(metrics_router) app.add_exception_handler(StarletteHTTPException, bearer_challenge) # type: ignore[arg-type] app.include_router(iiif_router) app.include_router(linked_data_router) diff --git a/fastapi_backend/tests/api/test_prometheus.py b/fastapi_backend/tests/api/test_prometheus.py new file mode 100644 index 000000000..17bcbc15f --- /dev/null +++ b/fastapi_backend/tests/api/test_prometheus.py @@ -0,0 +1,72 @@ +"""The Prometheus scrape endpoint: off by default, answered only with its token, and nothing personal in it.""" + +from __future__ import annotations + +from app.domain import activity, budgets, jobs, prometheus +from tests.helpers import login, make_user, seed + +TOKEN = "0123456789abcdef0123456789abcdef" + + +def admin(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def bearer(token=TOKEN): + return {"Authorization": f"Bearer {token}"} + + +def test_off_by_default(client, db): + h = admin(client, db) + t = client.get("/api/v1/settings", headers=h).json()["telemetry"] + assert t["values"]["prometheus"] is False + assert t["values"]["prometheus_token"] == {"secret": True, "set": False} + assert client.get("/metrics", headers=bearer()).status_code == 404 + + +def test_needs_its_token(client, db): + h = admin(client, db) + assert client.put("/api/v1/settings/telemetry", json={"prometheus": True}, headers=h).status_code == 200 + # on, but no token set: nobody may scrape + assert client.get("/metrics", headers=bearer()).status_code == 401 + assert client.put("/api/v1/settings/telemetry", json={"prometheus_token": "short"}, headers=h).status_code == 400 + assert client.put("/api/v1/settings/telemetry", json={"prometheus_token": TOKEN}, headers=h).status_code == 200 + r = client.get("/metrics") + assert r.status_code == 401 and r.headers["www-authenticate"].startswith("Bearer") + assert client.get("/metrics", headers=bearer("x" * 32)).status_code == 401 + assert client.get("/metrics", headers=h).status_code == 401 # a session isn't the scrape token + r = client.get("/metrics", headers=bearer()) + assert r.status_code == 200 and r.headers["content-type"].startswith("text/plain; version=0.0.4") + assert 'lens_build_info{version="' in r.text + # the token stays sealed + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["prometheus_token"] == {"secret": True, "set": True} + assert client.put("/api/v1/settings/telemetry", json={"prometheus": False}, headers=h).status_code == 200 + assert client.get("/metrics", headers=bearer()).status_code == 404 + + +def test_what_it_measures(db, cfg, folder): + rid = seed(db, cfg, folder)[0] + jobs.enqueue(db, rid, ["analyze"]) + activity.record("out", "llm.chat", ["routine:3"], cfg=cfg, db=db, model="gpt-x", tokens_in=1000, tokens_out=200, cost_usd=0.25, ms=1500) + activity.record("out", "llm.chat", ["routine:3"], cfg=cfg, db=db, model="gpt-x", ok=False, error="Timeout", ms=500) + budgets.put(db, "routine:3", usd=0.2, period="day") + (folder / "data").mkdir(exist_ok=True) + text = prometheus.render(db) + lines = set(text.splitlines()) + assert 'lens_jobs{status="queued"} 1' in lines + assert 'lens_resources{status="analyzed"} 3' in lines + assert 'lens_imports{window="24h"} 3' in lines + assert 'lens_activity_calls{window="1h",kind="out",model="gpt-x"} 2' in lines + assert 'lens_activity_failed{window="1h",kind="out",model="gpt-x"} 1' in lines + assert 'lens_activity_tokens{window="24h",kind="out",model="gpt-x",direction="in"} 1000' in lines + assert 'lens_activity_cost_usd{window="24h",kind="out",model="gpt-x"} 0.25' in lines + assert 'lens_budget_limit_usd{resource="routine:3",period="day"} 0.2' in lines + assert 'lens_budget_over{resource="routine:3",period="day"} 1' in lines + assert "lens_disk_free_bytes " in text and "lens_scrape_errors 0" in lines + # every family says what it is, once + assert text.count("# TYPE lens_jobs gauge") == 1 + # nothing personal: no namespace names, file names or text + samples = "\n".join(x for x in text.splitlines() if not x.startswith("#")) + for word in ("pods", '"calls"', "ep1", "root@x.io"): + assert word not in samples diff --git a/mkdocs.yml b/mkdocs.yml index d1f90077a..d68539ef2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,7 @@ nav: - Chat rooms: chat-rooms.md - Sensors: sensors.md - Telemetry: telemetry.md + - Prometheus: prometheus.md - Activity and costs: activity.md - Budgets: budgets.md - IIIF: iiif.md diff --git a/nextjs-frontend/app/metrics/route.ts b/nextjs-frontend/app/metrics/route.ts new file mode 100644 index 000000000..75a1d5ba0 --- /dev/null +++ b/nextjs-frontend/app/metrics/route.ts @@ -0,0 +1,6 @@ +import { handlers } from "@/lib/api/backend-proxy"; + +// The Prometheus scrape endpoint (docs/prometheus.md), answered by the API: off by default, and only for a scraper +// that sends the token set in Settings › Telemetry. Served here so it follows LENS_BIND like the web app. +export const dynamic = "force-dynamic"; +export const { GET } = handlers; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d7d16bf49..6156b8766 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -525,6 +525,8 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) { const saved = t?.values ?? {}; const secret = (saved.headers ?? {}) as { set?: boolean }; const headers = ctx.state("telemetry.headers"); + const promSecret = (saved.prometheus_token ?? {}) as { set?: boolean }; + const promToken = ctx.state("telemetry.prometheus_token"); const locked = t?.locked ?? []; const status = useQuery({ queryKey: ["telemetry-status", t?.updated_at ?? null], @@ -606,6 +608,31 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) { ))} {test.isError && {test.error.message}} + Prometheus +

+ For Grafana and other Prometheus tools: jobs, imports, model calls, cost, budgets, workers and disk, served at{" "} + /metrics on this address with the same privacy as telemetry. A scraper sends + the token as Authorization: Bearer; see docs/prometheus.md for a scrape + config and a sample dashboard. +

+ {Boolean(saved.prometheus) && !promSecret.set && !promToken.value && ( + + Set one below; until then /metrics answers no one. + + )} + + promToken.onChange(x)} + /> ); } diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 6c59496a1..f87e3e182 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -1924,6 +1924,14 @@ export const FIELDS: FieldSpec[] = [ mono: true, hint: "How this server shows in your tracing tool", }, + { + section: "telemetry", + key: "prometheus", + label: "Prometheus endpoint", + kind: "switch", + hint: "Serve /metrics for a Prometheus scraper; off by default, and answered only with the scrape token", + }, + { section: "telemetry", key: "prometheus_token", label: "Scrape token", kind: "secret" }, { section: "telemetry", key: "prices", @@ -2494,6 +2502,14 @@ export function why(c: Change): string | null { if (origin(c.before) !== origin(c.after)) return "Saved headers are removed, since they were for the old collector: enter them again if the new one needs them."; } + if (id === "telemetry.prometheus") + return c.after + ? "/metrics answers a scraper that sends the scrape token; nothing is served until a token is set." + : "/metrics stops answering."; + if (id === "telemetry.prometheus_token") + return c.after === "" + ? "The token is removed; /metrics answers no one." + : "Scrapers must send the new token; stored encrypted."; if (id === "telemetry.headers") return c.after === "" ? "The headers are removed." : "Sent with every export; stored encrypted."; if (id === "llm.api_key") diff --git a/nextjs-frontend/components/settings/section.tsx b/nextjs-frontend/components/settings/section.tsx index e2a294857..9cfd075db 100644 --- a/nextjs-frontend/components/settings/section.tsx +++ b/nextjs-frontend/components/settings/section.tsx @@ -45,6 +45,8 @@ const ENV: Record = { "telemetry.enabled": "LENS_TELEMETRY", "telemetry.endpoint": "LENS_TELEMETRY_ENDPOINT", "telemetry.headers": "LENS_TELEMETRY_HEADERS", + "telemetry.prometheus": "LENS_PROMETHEUS", + "telemetry.prometheus_token": "LENS_PROMETHEUS_TOKEN", "fedora.enabled": "LENS_FEDORA", "fedora.url": "LENS_FEDORA_URL", "fedora.user": "LENS_FEDORA_USER", diff --git a/nextjs-frontend/proxy.ts b/nextjs-frontend/proxy.ts index b16f4188a..2a521b38b 100644 --- a/nextjs-frontend/proxy.ts +++ b/nextjs-frontend/proxy.ts @@ -32,11 +32,11 @@ export default auth((req) => { export const config = { // Skip Auth.js, the backend paths (proxied by route handlers, see lib/api/backend-proxy.ts; /.well-known is OAuth - // discovery, /mcp the MCP server, /id and /ns the + // discovery, /mcp the MCP server, /metrics the Prometheus scrape endpoint, /id and /ns the // archive's linked-data URIs and vocabulary), Next's files and assets. // Under /iiif only the backend's IIIF resources are skipped (collection, discovery, auth, /iiif//…); the app's own // IIIF pages (/iiif, /iiif/collections/…, /iiif/import, /iiif/metadata/…) need the session like any other page. matcher: [ - "/((?!api/|\\.well-known/|mcp(?:/|$)|_next/|embed/|s/|id/|ns$|iiif/(?:collection(?!s)|discovery|auth|\\d)|reports/|static/|fonts/|favicon\\.ico|icon\\.svg|apple-icon\\.png|manifest\\.webmanifest|safari-pinned-tab\\.svg|pwa/|robots\\.txt).*)", + "/((?!api/|\\.well-known/|mcp(?:/|$)|metrics$|_next/|embed/|s/|id/|ns$|iiif/(?:collection(?!s)|discovery|auth|\\d)|reports/|static/|fonts/|favicon\\.ico|icon\\.svg|apple-icon\\.png|manifest\\.webmanifest|safari-pinned-tab\\.svg|pwa/|robots\\.txt).*)", ], };