From 25ca07a2e45bac5284ad270e5b191a595a9866e6 Mon Sep 17 00:00:00 2001
From: Adeel Ahmad
Date: Fri, 2 Oct 2026 13:57:47 +0000
Subject: [PATCH 1/2] feat(telemetry): opt-in OpenTelemetry traces and metrics
Off by default and only on when an admin turns it on (Settings, the setup
wizard's new last step, or LENS_TELEMETRY=on in .env; LENS_TELEMETRY=off
keeps it off). Sent only to the OTLP/HTTP endpoint that is set; there is
no built-in destination.
- Traces: API requests by route template, jobs and each step, routines
and their actions, workflow runs, and model calls (GenAI conventions)
- Metrics: request, step and model-call durations, job/routine/workflow
outcomes, tokens, and estimated cost from per-model prices
- No personal content: no paths, queries, prompts, answers, transcript
text, file names, titles, people, hosts or error messages
- Settings shows on/off, where it sends and the last exports, and can
send a test span; every process follows changes without a restart
Co-Authored-By: Claude Opus 5.5 (1M context)
Claude-Session: https://claude.ai/code/session_01CFYD76r8LUcnfTYKBwRfPn
---
CHANGELOG.md | 9 +
docker-compose.prod.yml | 4 +
docs/configuration.md | 1 +
docs/telemetry.md | 113 ++++
fastapi_backend/.env.example | 5 +
fastapi_backend/app/api/v1/routes/admin.py | 17 +-
fastapi_backend/app/api/v1/routes/setup.py | 16 +-
fastapi_backend/app/cli.py | 6 +-
fastapi_backend/app/domain/analyze.py | 6 +-
fastapi_backend/app/domain/jobs.py | 35 +-
fastapi_backend/app/domain/llm.py | 33 +-
fastapi_backend/app/domain/routines.py | 13 +-
fastapi_backend/app/domain/settings.py | 74 ++-
fastapi_backend/app/domain/setup.py | 19 +
fastapi_backend/app/domain/store.py | 14 +
fastapi_backend/app/domain/telemetry.py | 531 ++++++++++++++++++
fastapi_backend/app/domain/workflows.py | 14 +-
fastapi_backend/app/main.py | 5 +-
fastapi_backend/app/schemas/admin.py | 23 +
fastapi_backend/app/schemas/setup.py | 14 +
fastapi_backend/pyproject.toml | 3 +
fastapi_backend/tests/api/test_telemetry.py | 226 ++++++++
fastapi_backend/tests/fake_llm.py | 6 +-
fastapi_backend/tests/fake_otlp.py | 77 +++
fastapi_backend/uv.lock | 6 +
mkdocs.yml | 1 +
.../__tests__/settings-admin-models.test.ts | 29 +
.../__tests__/setupWizard.test.tsx | 37 +-
nextjs-frontend/app/openapi-client/sdk.gen.ts | 48 ++
.../app/openapi-client/types.gen.ts | 164 +++++-
.../components/settings/bodies.tsx | 101 +++-
nextjs-frontend/components/settings/model.ts | 120 ++++
.../components/settings/section.tsx | 3 +
.../components/settings/settings-shell.tsx | 2 +
.../components/setup/setup-wizard.tsx | 160 +++++-
nextjs-frontend/openapi.json | 271 ++++++++-
packaging/qnap/qpkg/shared/docker-compose.yml | 4 +
37 files changed, 2151 insertions(+), 59 deletions(-)
create mode 100644 docs/telemetry.md
create mode 100644 fastapi_backend/app/domain/telemetry.py
create mode 100644 fastapi_backend/tests/api/test_telemetry.py
create mode 100644 fastapi_backend/tests/fake_otlp.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ffafa4848..e487a064f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,15 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
## Unreleased
+- **Opt-in telemetry.** Lens can send OpenTelemetry traces and metrics about its own work to a collector you choose:
+ API requests by route, jobs and each step, routines, workflow runs, and model calls with their tokens and an
+ estimated cost (from per-model prices you set). It is off by default and never on unless you turn it on, in
+ Settings → Telemetry, the setup wizard's new last step, or `LENS_TELEMETRY=on` with `LENS_TELEMETRY_ENDPOINT` in
+ .env (`LENS_TELEMETRY=off` keeps it off). It goes only to that endpoint, over OTLP/HTTP; there is no built-in
+ destination. Spans and metrics carry ids, step types, model names, counts and timings, never transcript, prompt or
+ answer text, file names, titles, people, paths or addresses. Settings shows whether it is on and how the last
+ exports went, and can send a test span; the API and each worker follow a change without a restart
+ (docs/telemetry.md).
- **Install on Proxmox VE with one command.** A [community helper script](https://community-scripts.org/docs/ct/detailed_guide)
(`proxmox/ct/lens.sh`, run in the Proxmox host's shell) creates a Debian 13 LXC container running Lens without
Docker: SurrealDB 3.2.4, the API, a job worker and the web app as systemd services, with fresh secrets and the web
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 71a4e2581..8733f6a3e 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -38,6 +38,10 @@ x-backend: &backend
LENS_LLM_MODEL: ${LENS_LLM_MODEL:-}
LENS_LLM_API_KEY: ${LENS_LLM_API_KEY:-}
LENS_LLM_VISION_MODEL: ${LENS_LLM_VISION_MODEL:-}
+ # opt-in telemetry (docs/telemetry.md): unset, it stays off unless turned on in Settings
+ LENS_TELEMETRY: ${LENS_TELEMETRY:-}
+ LENS_TELEMETRY_ENDPOINT: ${LENS_TELEMETRY_ENDPOINT:-}
+ LENS_TELEMETRY_HEADERS: ${LENS_TELEMETRY_HEADERS:-}
volumes:
- archive-data:/data
- ${AUDIO_DIR:-./local-audio}:/audio:ro
diff --git a/docs/configuration.md b/docs/configuration.md
index c8cab5a62..1cb93b30d 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -31,6 +31,7 @@ with AES-GCM and are write-only: the API reports whether one is set, never its v
| `LENS_NAMESPACE` | | create the first namespace at startup, while there is none |
| `LENS_LLM_BASE_URL` / `LENS_LLM_MODEL` / `LENS_LLM_API_KEY` / `LENS_LLM_VISION_MODEL` | | the model provider; wins over Settings, which show these locked |
| `LENS_SETUP_WIZARD` | | `off`: never show the setup wizard |
+| `LENS_TELEMETRY` / `LENS_TELEMETRY_ENDPOINT` / `LENS_TELEMETRY_HEADERS` | off | opt-in OpenTelemetry traces and metrics, sent only to this OTLP/HTTP endpoint ([Telemetry](telemetry.md)); `LENS_TELEMETRY=off` keeps it off whatever Settings say |
| `FRONTEND_URL` | `http://localhost:3000` | the web app's address as people use it: links in emails, and where apps send people to sign in ([OAuth](authentication.md#oauth)) |
| `CORS_ORIGINS` | `["http://localhost:3000"]` | origins allowed to call the API from a browser |
| `OPENAPI_URL` | `/openapi.json` | `""` disables `/docs` and the schema |
diff --git a/docs/telemetry.md b/docs/telemetry.md
new file mode 100644
index 000000000..0c781c1f3
--- /dev/null
+++ b/docs/telemetry.md
@@ -0,0 +1,113 @@
+# Telemetry
+
+Lens can send traces and metrics about its own work to an [OpenTelemetry](https://opentelemetry.io/) collector, to see
+how it performs: which API requests are slow, how long each job step takes, how often jobs fail, and how many tokens
+model calls use and what they cost.
+
+**It is off unless you turn it on.** Nothing is collected while it is off, and Lens has no built-in destination: it
+never sends telemetry to the Lens project or anyone else. Turned on, it sends only to the endpoint you set, typically
+a collector on the same machine or network, which can pass it on to Jaeger, Grafana Tempo and Prometheus, SigNoz,
+Honeycomb, or whatever you use.
+
+## Turning it on
+
+Any of these:
+
+- **Settings → Telemetry**: turn on **Send telemetry** and set the **OTLP endpoint**. **Send a test span** checks the
+ address and headers first, even while it is off. The section shows whether it is on, where it sends, and how the
+ last exports went.
+- **The setup wizard** asks on a fresh install (its last step); it is off unless you choose on.
+- **.env**, for the API and every worker:
+
+ ```sh
+ LENS_TELEMETRY=on # off keeps it off, whatever Settings say
+ LENS_TELEMETRY_ENDPOINT=http://localhost:4318 # the collector's OTLP/HTTP address
+ LENS_TELEMETRY_HEADERS=Authorization=Bearer%20xyz # optional: key=value pairs, comma-separated, URL-encoded
+ ```
+
+ What .env sets wins over Settings, which show those fields locked.
+
+Each process (the API, each `lens worker`, `lens watch`) follows the settings on its own and picks up a change within a
+few seconds, with no restart. The endpoint is the collector's base address: traces go to `/v1/traces` and metrics to
+`/v1/metrics` under it, over OTLP/HTTP with protobuf.
+
+In Docker, `localhost` is the container itself: point the endpoint at a collector on the `lens` network (for example
+`http://otel-collector:4318`, below) or at `http://host.docker.internal:4318` for one on the host.
+
+## What is sent
+
+| Signal | What | Attributes |
+| --- | --- | --- |
+| Trace | each API request, named by its route template (`GET /api/v1/recordings/{rid}`) | method, route, status code |
+| Trace | each job, with a span per step (`step transcribe`) | job, recording and pipeline ids; step type and outcome |
+| Trace | routines and their actions, workflow runs | routine, workflow and recording ids; action type; trigger |
+| Trace | each call to the model server (`chat llama3.1`) | model, server host, temperature, tokens in and out, estimated cost |
+| Metric | `http.server.request.duration` | method, route, status code |
+| Metric | `lens.job.step.duration`, `lens.jobs` | step type, outcome |
+| Metric | `lens.routine.runs`, `lens.workflow.runs` | outcome |
+| Metric | `gen_ai.client.operation.duration`, `gen_ai.client.token.usage` | model, token type |
+| Metric | `lens.llm.cost` (USD) | model |
+
+Every span and metric carries `service.name` (Settings → Telemetry, default `lens`), `service.version`, a random
+`service.instance.id` per process, and `lens.process.role` (`api`, `worker` or `watcher`). Model calls follow the
+OpenTelemetry [GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/).
+
+**Never sent:** transcript text, prompts or answers, file names, titles, namespace or speaker names, people, email
+addresses, client IP addresses, request paths or query strings (only route templates), host names, or error messages
+(only the error's type, since a message can quote content).
+
+### Tokens and cost
+
+Token counts come from the model server's reply (`usage`). Most servers report them; a streamed chat answer has them
+only when the server sends usage in its last chunk.
+
+Cost is an estimate from prices you set: **Model prices** in Settings → Telemetry, one line per model with its input
+and output price in dollars per million tokens (`gpt-4o-mini 0.15 0.60`). The prices in Settings → AI assistant count
+for the configured model too. A model without a price gets no cost, so local models cost nothing.
+
+## Other settings
+
+| Setting | Default | |
+| --- | --- | --- |
+| Traces / Metrics | on / on | send either or both |
+| Share of traces kept | `1` | `0.1` keeps one trace in ten (head sampling) |
+| Send metrics every | `60` seconds | |
+| Service name | `lens` | how this server shows in your tracing tool |
+
+In archive.yaml they live under `telemetry:` (`enabled`, `endpoint`, `traces`, `metrics`, `sample_ratio`,
+`export_seconds`, `service_name`, `prices`).
+
+## A collector next to Lens
+
+A minimal [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) that takes Lens's traces and metrics and
+prints them (swap the `debug` exporter for your backend's):
+
+```yaml
+# otel-collector.yaml
+receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: 0.0.0.0:4318
+exporters:
+ debug:
+ verbosity: basic
+service:
+ pipelines:
+ traces: { receivers: [otlp], exporters: [debug] }
+ metrics: { receivers: [otlp], exporters: [debug] }
+```
+
+Beside `docker-compose.prod.yml`, in a `docker-compose.override.yml` or with `-f`:
+
+```yaml
+services:
+ otel-collector:
+ image: otel/opentelemetry-collector:latest
+ command: ["--config=/etc/otel-collector.yaml"]
+ volumes:
+ - ./otel-collector.yaml:/etc/otel-collector.yaml:ro
+ networks: [lens]
+```
+
+Then set the endpoint to `http://otel-collector:4318` and turn telemetry on.
diff --git a/fastapi_backend/.env.example b/fastapi_backend/.env.example
index bfcb43013..3ffac6851 100644
--- a/fastapi_backend/.env.example
+++ b/fastapi_backend/.env.example
@@ -52,5 +52,10 @@ CORS_ORIGINS=["http://localhost:3000"]
# LENS_LLM_API_KEY=
# LENS_SETUP_WIZARD=off # never show the wizard
+# --- Telemetry (docs/telemetry.md): off unless turned on here or in Settings, and sent only to this endpoint ---
+# LENS_TELEMETRY=on # on, or off to keep it off whatever Settings say
+# LENS_TELEMETRY_ENDPOINT=http://localhost:4318 # your OpenTelemetry collector's OTLP/HTTP address
+# LENS_TELEMETRY_HEADERS= # optional: key=value pairs, comma-separated, URL-encoded
+
# Uncomment to disable /docs and /openapi.json
# OPENAPI_URL=""
diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py
index 457b48487..3ae6f8aa6 100644
--- a/fastapi_backend/app/api/v1/routes/admin.py
+++ b/fastapi_backend/app/api/v1/routes/admin.py
@@ -12,8 +12,8 @@
from app.api.deps import AdminReader, AdminWriter, Cfg, Db, domain_errors
from app.core.middleware import host_name
-from app.domain import auth, jobs, llm, settings, sources, store
-from app.schemas.admin import AuditEntry, Health, LlmTestResult, Started
+from app.domain import auth, jobs, llm, settings, sources, store, telemetry
+from app.schemas.admin import AuditEntry, Health, LlmTestResult, Started, TelemetryStatus, TelemetryTestResult
from app.schemas.auth import AccountToken
from app.schemas.common import Ok
@@ -54,6 +54,19 @@ def test_llm(user: AdminWriter, cfg: Cfg) -> LlmTestResult:
return LlmTestResult(ok=True, reply=reply.strip()[:40], ms=int((time.time() - t0) * 1000), model=cfg["llm"]["model"])
+@router.get("/settings/telemetry/status")
+def telemetry_status(user: AdminReader, cfg: Cfg) -> TelemetryStatus:
+ """Whether telemetry is on, where it goes, and how the API process's last exports went. Off by default."""
+ return TelemetryStatus.model_validate(telemetry.status(cfg))
+
+
+@router.post("/settings/telemetry/test")
+def test_telemetry(user: AdminWriter, cfg: Cfg) -> TelemetryTestResult:
+ """Send one test span to the saved endpoint now (on or off), to check the address and headers."""
+ ok, error, ms = telemetry.test_export(cfg)
+ return TelemetryTestResult(ok=ok, error=error, ms=ms)
+
+
@router.get("/audit")
def list_audit(user: AdminReader, db: Db, limit: int = Query(200, ge=1, le=1000)) -> list[AuditEntry]:
"""Who changed what, newest first."""
diff --git a/fastapi_backend/app/api/v1/routes/setup.py b/fastapi_backend/app/api/v1/routes/setup.py
index 944bacbb3..12278d5be 100644
--- a/fastapi_backend/app/api/v1/routes/setup.py
+++ b/fastapi_backend/app/api/v1/routes/setup.py
@@ -1,4 +1,4 @@
-"""The first-run setup wizard (admins): the first namespace, the model provider and storage, then finishing it.
+"""The first-run setup wizard (admins): the first namespace, the model provider, storage and telemetry, then finishing it.
The first admin is created before this, with the setup code (POST /auth/setup) or from the environment. Fields the
environment sets are left out of every save and reported as locked (domain/setup.py).
@@ -10,7 +10,7 @@
from app.api.deps import AdminReader, AdminWriter, Db
from app.domain import auth, setup
-from app.schemas.setup import SetupFinish, SetupLlm, SetupNamespace, SetupSaved, SetupStorage, SetupView
+from app.schemas.setup import SetupFinish, SetupLlm, SetupNamespace, SetupSaved, SetupStorage, SetupTelemetry, SetupView
router = APIRouter(prefix="/setup", tags=["setup"])
@@ -66,6 +66,18 @@ def save_storage(body: SetupStorage, user: AdminWriter, request: Request, db: Db
return SetupSaved(saved=[k for k in ("max_upload_mb", "folder") if getattr(body, k) is not None], watch=wid)
+@router.put("/telemetry")
+def save_telemetry(body: SetupTelemetry, user: AdminWriter, request: Request, db: Db) -> SetupSaved:
+ """Opt in to telemetry: traces and metrics sent only to this endpoint (an OTLP collector). Off unless chosen."""
+ try:
+ saved = setup.save_telemetry(db, request.app.state.archive.base, body.enabled, body.endpoint, user.email)
+ except ValueError as e:
+ raise HTTPException(400, str(e)) from None
+ if saved:
+ auth.audit(db, user.as_audit(), "settings.save", "telemetry", saved)
+ return SetupSaved(saved=saved)
+
+
@router.post("/finish")
def finish(body: SetupFinish, user: AdminWriter, db: Db) -> SetupSaved:
"""Finish (or skip) the wizard; the web app stops showing it. Everything stays changeable in Settings."""
diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py
index dca51c10b..eb0231e01 100644
--- a/fastapi_backend/app/cli.py
+++ b/fastapi_backend/app/cli.py
@@ -188,7 +188,7 @@ def platform_main(argv, config):
import threading
import time
- from .domain import auth, jobs, notify, routines, settings, sources
+ from .domain import auth, jobs, notify, routines, settings, sources, telemetry
ap = argparse.ArgumentParser(prog="lens")
sub = ap.add_subparsers(dest="cmd", required=True)
@@ -250,6 +250,7 @@ def password():
auth.update_account(db, acct["id"], password=password())
print(f"password changed for {a.email}")
elif a.cmd == "worker":
+ telemetry.set_role("worker")
wk = jobs.Worker(db, C, a.name, a.steps.split(",") if a.steps else None, log=print)
if a.once:
print(f"ran {wk.drain()} job(s)")
@@ -263,9 +264,11 @@ def password():
except KeyboardInterrupt:
stop.set()
elif a.once:
+ telemetry.set_role("watcher")
print(f"scanned {sources.poll_due(db, C(), print)} folder(s)")
print(f"ran {routines.run_due(db, C(), print)} routine(s)")
else:
+ telemetry.set_role("watcher")
print("watching storage sources and running routines; Ctrl-C to stop")
_stop_on_term()
try:
@@ -278,6 +281,7 @@ def password():
except (ValueError, KeyError) as e:
raise SystemExit(str(e)) from None
finally:
+ telemetry.shutdown() # sends what is buffered
db.close()
diff --git a/fastapi_backend/app/domain/analyze.py b/fastapi_backend/app/domain/analyze.py
index 27bea280f..f40c492dd 100644
--- a/fastapi_backend/app/domain/analyze.py
+++ b/fastapi_backend/app/domain/analyze.py
@@ -9,7 +9,7 @@
import urllib.request
from collections import Counter, defaultdict
-from . import store
+from . import store, telemetry
STOP = set(
"""a about above after again against all almost also am an and any are aren as at be because been before being
@@ -628,8 +628,8 @@ def _llm(cfg, user, system=SUMMARY_SYSTEM):
req = urllib.request.Request(
l["base_url"].rstrip("/") + "/chat/completions", data=json.dumps(body).encode(), headers=headers, method="POST"
)
- with urllib.request.urlopen(req, timeout=l.get("timeout", 300)) as r:
- text = json.loads(r.read().decode())["choices"][0]["message"]["content"]
+ with telemetry.model_call(cfg, body) as call, urllib.request.urlopen(req, timeout=l.get("timeout", 300)) as r:
+ text = call.reply(json.loads(r.read().decode()))["choices"][0]["message"]["content"]
text = re.sub(r"^```(?:json)?|```$", "", text.strip()).strip()
obj = json.loads(text[text.find("{") : text.rfind("}") + 1])
missing = [k for k in SUMMARY_NEEDS if k not in obj]
diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py
index 427d34ae6..e57df379b 100644
--- a/fastapi_backend/app/domain/jobs.py
+++ b/fastapi_backend/app/domain/jobs.py
@@ -13,7 +13,7 @@
import threading
import time
-from . import analyze, ingest, pipelines, render, speakers as spk, store
+from . import analyze, ingest, pipelines, render, speakers as spk, store, telemetry
R = store.R
PIPELINE = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "summarize", "report"]
@@ -435,6 +435,7 @@ def started(k):
return time.time()
def finished(k, outcome, note, t0, outputs=None):
+ telemetry.record("lens.job.step.duration", time.time() - t0, {"lens.step": _spec(steps[k])["type"], "lens.step.outcome": outcome})
runs[k] = store.clean(
{
**runs[k],
@@ -512,15 +513,17 @@ def beat():
i, t0 = i + 1, None
continue
before = _saved(db, rid)
- try:
- STEPS[step](db, cfg_fn(), rid, say, spec)
- except Skip as e:
- say(f"{step} skipped: {e}")
- finished(i, "skipped", str(e), t0, _outputs(before, _saved(db, rid)))
- else:
- note = said[0]
- say(f"{spec.get('name') or step} done in {time.time() - t0:.1f}s")
- finished(i, "done", note, t0, _outputs(before, _saved(db, rid)))
+ with telemetry.span(f"step {step}", {"lens.step": step, "lens.step.index": i, "lens.job.id": str(jid)}) as sp:
+ try:
+ STEPS[step](db, cfg_fn(), rid, say, spec)
+ except Skip as e:
+ say(f"{step} skipped: {e}")
+ finished(i, "skipped", str(e), t0, _outputs(before, _saved(db, rid)))
+ else:
+ note = said[0]
+ say(f"{spec.get('name') or step} done in {time.time() - t0:.1f}s")
+ finished(i, "done", note, t0, _outputs(before, _saved(db, rid)))
+ sp.set_attribute("lens.step.outcome", runs[i]["outcome"])
_timed(db, rid, spec, runs[i]["seconds"], runs[i]["outcome"] == "skipped")
i, t0 = i + 1, None
except (Exception, SystemExit) as e: # noqa: BLE001 - recorded on the job; SystemExit too (a missing engine says so)
@@ -738,7 +741,17 @@ def run_once(self):
if not job:
return False
self.register(job["id"])
- run_job(self.db, self.cfg_fn, job, self.name, self.can, self.log)
+ pipe = job.get("pipeline") if isinstance(job.get("pipeline"), dict) else {}
+ attrs = {
+ "lens.job.id": str(job["id"]),
+ "lens.recording.id": str(job["recording"]),
+ "lens.pipeline.id": str(pipe["id"]) if pipe.get("id") is not None else ("standard" if pipe else None),
+ "lens.pipeline.version": pipe.get("version"),
+ }
+ with telemetry.span("job", attrs) as sp:
+ outcome = run_job(self.db, self.cfg_fn, job, self.name, self.can, self.log)
+ sp.set_attribute("lens.job.outcome", outcome)
+ telemetry.record("lens.jobs", 1, {"lens.job.outcome": outcome})
self.register()
return True
diff --git a/fastapi_backend/app/domain/llm.py b/fastapi_backend/app/domain/llm.py
index d0c0609d9..03c23ba3d 100644
--- a/fastapi_backend/app/domain/llm.py
+++ b/fastapi_backend/app/domain/llm.py
@@ -11,6 +11,8 @@
import urllib.error
import urllib.request
+from . import telemetry
+
class LLMError(RuntimeError):
pass
@@ -91,8 +93,8 @@ def chat(cfg, messages, model=None, max_tokens=None, temperature=0.2):
payload = {"model": model or cfg["llm"]["model"], "messages": messages, "temperature": temperature}
if max_tokens:
payload["max_tokens"] = max_tokens
- with _post(cfg, payload) as r:
- return _content(_load(r))
+ with telemetry.model_call(cfg, payload) as call, _post(cfg, payload) as r:
+ return _content(call.reply(_load(r)))
def _visible(pieces):
@@ -125,10 +127,18 @@ def stream_chat(cfg, messages, model=None, temperature=0.2):
def _stream(cfg, model, messages, temperature):
- r = _post(cfg, {"model": model, "messages": messages, "temperature": temperature, "stream": True})
+ payload = {"model": model, "messages": messages, "temperature": temperature, "stream": True}
+ # not the current span: the generator may resume in another thread (a streamed response)
+ call = telemetry.model_call(cfg, payload, current=False)
+ error = None
+ try:
+ r = _post(cfg, payload)
+ except LLMError as e:
+ call.end(e)
+ raise
try:
if "text/event-stream" not in (r.headers.get("Content-Type") or ""):
- yield _content(_load(r)) # the server ignored stream=true
+ yield _content(call.reply(_load(r))) # the server ignored stream=true
return
finished = False
try:
@@ -141,7 +151,8 @@ def _stream(cfg, model, messages, temperature):
finished = True
break
try:
- choice = json.loads(data)["choices"][0]
+ chunk = call.reply(json.loads(data)) # the last chunk may carry the usage, with no choices
+ choice = chunk["choices"][0]
delta = (choice.get("delta") or {}).get("content")
except (ValueError, KeyError, IndexError, TypeError, AttributeError):
continue
@@ -152,8 +163,12 @@ def _stream(cfg, model, messages, temperature):
raise LLMError(f"the LLM server stopped mid-answer: {e}") from None
if not finished:
raise LLMError("the LLM server stopped mid-answer")
+ except BaseException as e:
+ error = e
+ raise
finally:
r.close()
+ call.end(None if isinstance(error, GeneratorExit) else error)
def parse_json(text):
@@ -211,8 +226,8 @@ def json_out(cfg, system, user, schema, model=None):
"response_format": {"type": "json_schema", "json_schema": {"name": "output", "schema": schema, "strict": False}},
}
try:
- with _post(cfg, payload) as r:
- text = _content(_load(r))
+ with telemetry.model_call(cfg, payload) as call, _post(cfg, payload) as r:
+ text = _content(call.reply(_load(r)))
except LLMError as e:
if not str(e)[:3] in ("400", "422", "404"):
raise
@@ -235,8 +250,8 @@ def chat_message(cfg, messages, tools=None, model=None, temperature=0.2):
if tools:
payload.update(tools=tools, tool_choice="auto")
try:
- with _post(cfg, payload) as r:
- j = _load(r)
+ with telemetry.model_call(cfg, payload) as call, _post(cfg, payload) as r:
+ j = call.reply(_load(r))
except LLMError as e:
if tools and str(e)[:3] in ("400", "404", "422", "501"):
raise ToolsUnsupported(str(e)) from None
diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py
index 6d36fd00d..f6893512b 100644
--- a/fastapi_backend/app/domain/routines.py
+++ b/fastapi_backend/app/domain/routines.py
@@ -17,7 +17,7 @@
import datetime as dt
-from . import jobs, organize, schedule, sources, store, workflows
+from . import jobs, organize, schedule, sources, store, telemetry, workflows
R = store.R
ACTIONS = ("sync", "pipeline", "workflow")
@@ -327,6 +327,13 @@ def _takes_new(db, a):
def run(db, cfg, rid, trigger="manual", by=None, propose_only=False, log=None):
"""Run a routine now, here; the run's id. Each action is tried even when one before it failed."""
+ with telemetry.span(
+ "routine", {"lens.routine.id": str(rid), "lens.routine.trigger": trigger, "lens.routine.propose_only": propose_only}
+ ):
+ return _run(db, cfg, rid, trigger, by, propose_only, log)
+
+
+def _run(db, cfg, rid, trigger, by, propose_only, log):
routine = get(db, rid)
since = routine.get("seen_recording") or 0
seen = {"since": since, "marks": [], "failed": False}
@@ -350,7 +357,8 @@ def say(msg):
say(f"{k}. {a['type']}")
seen["pending"] = None
try:
- got = _action(db, cfg, routine, run_id, a, seen, propose_only, say)
+ with telemetry.span(f"action {a['type']}", {"lens.routine.action": a["type"], "lens.routine.action.index": k}):
+ got = _action(db, cfg, routine, run_id, a, seen, propose_only, say)
results.append({"type": a["type"], "status": "done", "result": got})
if seen["pending"] is not None:
seen["marks"].append(seen["pending"])
@@ -365,6 +373,7 @@ def say(msg):
for s in ("applied", "proposed")
}
status = "done" if not failed else ("error" if failed == len(results) else "partly")
+ telemetry.record("lens.routine.runs", 1, {"lens.outcome": status, "lens.routine.trigger": trigger})
db.q(
"UPDATE $r MERGE $d",
r=R("routine_run", run_id),
diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py
index fbf123dda..b766e1557 100644
--- a/fastapi_backend/app/domain/settings.py
+++ b/fastapi_backend/app/domain/settings.py
@@ -18,7 +18,7 @@
import secrets
import threading
-from . import convert, ipgroups, objects, store
+from . import convert, ipgroups, objects, store, telemetry
R = store.R
EDITABLE = {
@@ -34,6 +34,7 @@
"iiif": None,
"ai": None,
"notifications": None,
+ "telemetry": None,
"video": (
"sample_seconds",
"scene_threshold",
@@ -57,7 +58,7 @@
# the LibreOffice and Chromium paths are startup settings only (the web app can't choose what the server runs)
"documents": ("page_pixels", "thumb_pixels", "ocr_below_chars", "max_pages", "convert_seconds", "attachment_resources"),
}
-SECRETS = {"llm": ("api_key",)}
+SECRETS = {"llm": ("api_key",), "telemetry": ("headers",)}
ENUMS = {
("transcribe", "engine"): {"sensevoice", "whisper", "mlx-whisper"},
("transcribe", "device"): {"auto", "cpu", "cuda", "mps"},
@@ -72,13 +73,17 @@
("video", "object_engine"): {"yolox", "ultralytics", "off"},
}
# Settings the environment (.env) sets, which win over archive.yaml and the app and show as locked there: the
-# break-glass allowed hosts, and the model provider so an install can be configured without the setup wizard.
+# break-glass allowed hosts, the model provider so an install can be configured without the setup wizard, and
+# telemetry (LENS_TELEMETRY=off keeps it off whatever the app says).
ENV_OVERRIDES = {
("server", "allowed_hosts"): "ARCHIVE_ALLOWED_HOSTS",
("llm", "base_url"): "LENS_LLM_BASE_URL",
("llm", "model"): "LENS_LLM_MODEL",
("llm", "api_key"): "LENS_LLM_API_KEY",
("llm", "vision_model"): "LENS_LLM_VISION_MODEL",
+ ("telemetry", "enabled"): "LENS_TELEMETRY",
+ ("telemetry", "endpoint"): "LENS_TELEMETRY_ENDPOINT",
+ ("telemetry", "headers"): "LENS_TELEMETRY_HEADERS",
}
@@ -87,8 +92,11 @@ def env_value(section, key):
raw = os.environ.get(ENV_OVERRIDES.get((section, key)) or "", "").strip()
if not raw:
return None
- if isinstance(store.DEFAULTS.get(section, {}).get(key), list):
+ default = store.DEFAULTS.get(section, {}).get(key)
+ if isinstance(default, list):
return [x.strip() for x in raw.split(",") if x.strip()]
+ if isinstance(default, bool):
+ return raw.lower() in ("1", "true", "on", "yes")
return raw
@@ -189,6 +197,7 @@ def current(self):
with self._lock:
if self._cfg is None or v != self._ver:
self._cfg, self._ver = effective(self.db, self.base), v
+ telemetry.apply(self._cfg) # turned on, changed or off in the app: this process follows
return self._cfg
@@ -260,6 +269,8 @@ def _check(section, key, value, default):
return _upload_setting(key, value)
if section == "notifications":
return _notify_setting(key, value)
+ if section == "telemetry":
+ return _telemetry_setting(key, value)
if (section, key) == ("documents", "attachment_resources"):
if not isinstance(value, bool):
raise ValueError("documents.attachment_resources is true or false")
@@ -335,6 +346,59 @@ def _notify_setting(key, value):
return value
+TELEMETRY_RANGES = {"export_seconds": (5, 3600)}
+SERVICE_RX = re.compile(r"^[A-Za-z0-9._-]{1,64}$")
+
+
+def _telemetry_setting(key, value):
+ if key in ("enabled", "traces", "metrics"):
+ if not isinstance(value, bool):
+ raise ValueError(f"telemetry.{key} is true or false")
+ return value
+ if key == "endpoint":
+ if value is None or value == "":
+ return None
+ if not (isinstance(value, str) and VIEWER_URL.match(value.strip())):
+ raise ValueError("telemetry.endpoint is the OTLP/HTTP address of a collector, like http://localhost:4318")
+ v = value.strip().rstrip("/")
+ if v.endswith(("/v1/traces", "/v1/metrics")):
+ raise ValueError("telemetry.endpoint is the collector's base address, without /v1/traces or /v1/metrics")
+ return v
+ if key == "sample_ratio":
+ if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 1):
+ raise ValueError("telemetry.sample_ratio is a number from 0 to 1")
+ return float(value)
+ if key == "service_name":
+ if not (isinstance(value, str) and SERVICE_RX.match(value.strip())):
+ raise ValueError("telemetry.service_name is up to 64 letters, digits, ., _ and -")
+ return value.strip()
+ if key == "prices":
+ return _prices(value)
+ lo, hi = TELEMETRY_RANGES[key]
+ if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi):
+ raise ValueError(f"telemetry.{key} is a whole number from {lo} to {hi}")
+ return value
+
+
+def _prices(value):
+ """{model: {input, output}}: USD per million tokens, for the cost estimates."""
+ msg = "telemetry.prices gives each model its input and output price in USD per million tokens"
+ if value is None:
+ return {}
+ if not isinstance(value, dict) or len(value) > 200:
+ raise ValueError(msg)
+ out = {}
+ for model, p in value.items():
+ name = str(model).strip()
+ if not name or len(name) > 200 or not isinstance(p, dict) or set(p) - {"input", "output"}:
+ raise ValueError(msg)
+ nums = {k: p.get(k, 0) for k in ("input", "output")}
+ if not all(isinstance(n, (int, float)) and not isinstance(n, bool) and 0 <= n <= 1_000_000 for n in nums.values()):
+ raise ValueError(msg)
+ out[name] = {k: float(n) for k, n in nums.items()}
+ return out
+
+
def _upload_setting(key, value):
if key == "extensions":
ok = isinstance(value, list) and value and all(isinstance(x, str) for x in value)
@@ -363,6 +427,8 @@ def save(db, base, section, changes, user=None):
if v is None or v == "":
sealed.pop(k, None)
elif isinstance(v, str):
+ if (section, k) == ("telemetry", "headers"):
+ telemetry.parse_headers(v) # ValueError when malformed
sealed[k] = seal(base, v, f"setting:{section}.{k}")
elif not (isinstance(v, dict) and v.get("secret")): # the mask echoed back means "unchanged"
raise ValueError(f"{section}.{k} must be text")
diff --git a/fastapi_backend/app/domain/setup.py b/fastapi_backend/app/domain/setup.py
index 53e20a9d6..45ffe96ca 100644
--- a/fastapi_backend/app/domain/setup.py
+++ b/fastapi_backend/app/domain/setup.py
@@ -8,6 +8,7 @@
LENS_ADMIN_EMAIL, LENS_ADMIN_PASSWORD, LENS_ADMIN_NAME the first admin, created at startup (no setup code needed)
LENS_NAMESPACE the first namespace, created at startup while there is none
LENS_LLM_BASE_URL, LENS_LLM_MODEL, LENS_LLM_API_KEY the model provider (settings.ENV_OVERRIDES)
+ LENS_TELEMETRY, LENS_TELEMETRY_ENDPOINT opt-in telemetry (off unless set on; docs/telemetry.md)
LENS_SETUP_WIZARD=off never show the wizard (everything comes from .env)
The first admin still needs the one-time setup code (or the environment), so a stranger can't claim a public server.
@@ -101,6 +102,11 @@ def view(db, cfg):
"max_upload_mb": shown["uploads"]["max_mb"],
"watches": len(db.values("SELECT VALUE id FROM watch_path")),
},
+ "telemetry": {
+ "enabled": bool(shown["telemetry"].get("enabled")),
+ "endpoint": shown["telemetry"].get("endpoint"),
+ "locked": [k for k in settings.locked("telemetry") if k in ("enabled", "endpoint")],
+ },
}
@@ -123,6 +129,19 @@ def save_llm(db, cfg, values, user=None):
return sorted(keep)
+def save_telemetry(db, cfg, enabled, endpoint, user=None):
+ """Opt in to telemetry (or stay out), leaving out what the environment sets. On needs an endpoint."""
+ locked = settings.locked("telemetry")
+ endpoint = (endpoint or "").strip() or None
+ if enabled and not endpoint and "endpoint" not in locked:
+ raise ValueError("telemetry needs an endpoint to send to, like http://localhost:4318")
+ values = {"enabled": bool(enabled), **({"endpoint": endpoint} if endpoint or enabled else {})} # off keeps the address
+ keep = {k: v for k, v in values.items() if k not in locked}
+ if keep:
+ settings.save(db, cfg, "telemetry", keep, user)
+ return sorted(keep)
+
+
def save_storage(db, cfg, max_upload_mb=None, folder=None, namespace=None, user=None):
"""The upload limit, and optionally a folder on this machine (inside sources.local_roots) watched into a
namespace. Returns the watch's id, if one was made."""
diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
index 7f6d3b1a3..baff2199b 100644
--- a/fastapi_backend/app/domain/store.py
+++ b/fastapi_backend/app/domain/store.py
@@ -255,6 +255,20 @@ def labels():
# private networks listed here (a Matterbridge on the LAN or the Docker network); app_url is where links in messages
# point (null: FRONTEND_URL)
"notifications": {"enabled": True, "networks": [], "poll_seconds": 5, "max_attempts": 6, "app_url": None},
+ # OpenTelemetry traces and metrics (docs/telemetry.md): off unless an admin turns it on, and sent only to the OTLP/HTTP
+ # endpoint set here (e.g. a collector at http://localhost:4318). headers is a secret: key=value pairs for the
+ # endpoint's auth. prices: {model: {input, output}} in USD per million tokens, for cost estimates.
+ "telemetry": {
+ "enabled": False,
+ "endpoint": None,
+ "headers": None,
+ "traces": True,
+ "metrics": True,
+ "sample_ratio": 1.0,
+ "export_seconds": 60,
+ "service_name": "lens",
+ "prices": {},
+ },
# IIIF: identifiers are built from base_url (set it to the stable public HTTPS address; null: the request's address)
"iiif": {
"base_url": None,
diff --git a/fastapi_backend/app/domain/telemetry.py b/fastapi_backend/app/domain/telemetry.py
new file mode 100644
index 000000000..253d0c6ce
--- /dev/null
+++ b/fastapi_backend/app/domain/telemetry.py
@@ -0,0 +1,531 @@
+"""Opt-in telemetry: OpenTelemetry traces and metrics, sent only to an OTLP endpoint an admin sets.
+
+Off by default, and nothing is collected or sent while it is off: no phone-home, no built-in endpoint. Turned on
+(Settings → Telemetry, the setup wizard, or LENS_TELEMETRY=on with LENS_TELEMETRY_ENDPOINT in .env), each process
+(the API and every `lens worker`) sends what it does to that endpoint over OTLP/HTTP, typically a collector on the same
+machine:
+
+* traces: API requests (by route template), jobs and their steps, routines, workflow runs and model calls;
+* metrics: request, step and model-call durations, job and routine outcomes, model tokens and estimated cost.
+
+Spans and metrics carry no personal content: no transcript or prompt text, no file names or titles, no namespace names,
+no client addresses or user ids; only route templates, step types, record ids, model names, token counts and timings.
+
+The OpenTelemetry SDK is imported only when telemetry is on. Settings are re-read when someone saves them
+(settings.Settings.current), so turning it on or off applies without a restart.
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import re
+import threading
+import time
+import urllib.parse
+import uuid
+from contextlib import contextmanager
+
+log = logging.getLogger("lens")
+
+_lock = threading.Lock()
+_state = {"key": None, "tracer": None, "meter": None, "tp": None, "mp": None, "instruments": {}, "cfg": {}}
+_role = ["api"]
+_instance = str(uuid.uuid4()) # service.instance.id: random per process, so no host name leaves the machine
+_last = {"traces": None, "metrics": None} # the last export from this process: {at, ok, error}
+HEADER_RX = re.compile(r"^[A-Za-z0-9!#$%&'*+.^_`|~-]+$")
+
+
+def parse_headers(text):
+ """`key=value,key2=value2` (OTEL_EXPORTER_OTLP_HEADERS' form, values URL-encoded) to a dict; ValueError if malformed."""
+ out = {}
+ for part in (text or "").split(","):
+ if not part.strip():
+ continue
+ k, sep, v = part.partition("=")
+ k = k.strip()
+ if not sep or not HEADER_RX.match(k):
+ raise ValueError("telemetry.headers is key=value pairs separated by commas, like Authorization=Bearer%20abc")
+ out[k] = urllib.parse.unquote(v.strip())
+ return out
+
+
+def _section(cfg):
+ return (cfg or {}).get("telemetry") or {}
+
+
+def _wanted(t):
+ """The settings that decide what is set up; None when nothing should run."""
+ if not t.get("enabled") or not t.get("endpoint"):
+ return None
+ if not (t.get("traces") or t.get("metrics")):
+ return None
+ return (
+ t["endpoint"].rstrip("/"),
+ t.get("headers") if isinstance(t.get("headers"), str) else None,
+ bool(t.get("traces")),
+ bool(t.get("metrics")),
+ float(t.get("sample_ratio") if t.get("sample_ratio") is not None else 1.0),
+ int(t.get("export_seconds") or 60),
+ t.get("service_name") or "lens",
+ _role[0],
+ )
+
+
+def prices(cfg):
+ """USD per million tokens by model: telemetry.prices, over the assistant's prices for the configured model
+ (Settings → AI assistant), so a price set there needn't be set twice."""
+ ai, model = (cfg or {}).get("ai") or {}, ((cfg or {}).get("llm") or {}).get("model")
+ out = {}
+ if model and ai.get("price_in") is not None and ai.get("price_out") is not None:
+ out[model] = {"input": ai["price_in"], "output": ai["price_out"]}
+ return {**out, **(_section(cfg).get("prices") or {})}
+
+
+def set_role(role):
+ """What this process is (api, worker, watcher): sent as lens.process.role."""
+ _role[0] = role
+
+
+def apply(cfg):
+ """Start, change or stop telemetry to match the configuration. Cheap when nothing changed."""
+ t = _section(cfg)
+ want = _wanted(t)
+ _state["cfg"] = {"prices": prices(cfg)}
+ if want == _state["key"]:
+ return
+ with _lock:
+ if want == _state["key"]:
+ return
+ # the old providers send what they hold in the background: a request that picked up the change (this runs
+ # under settings.Settings.current) never waits on a collector that is slow or gone
+ _stop(wait=False)
+ _last.update(traces=None, metrics=None)
+ if want is not None:
+ try:
+ _start(*want)
+ except Exception as e: # noqa: BLE001 - telemetry never stops the archive
+ log.warning("telemetry: couldn't start (%s: %s); carrying on without it", type(e).__name__, e)
+ _stop(wait=False)
+ elif _state["key"] is not None:
+ log.info("telemetry: off")
+ _state["key"] = want
+
+
+def _stop(wait=True):
+ providers = [p for p in (_state["tp"], _state["mp"]) if p is not None]
+ _state.update(tracer=None, meter=None, tp=None, mp=None, instruments={})
+
+ def close():
+ for p in providers:
+ try:
+ p.shutdown()
+ except Exception: # noqa: BLE001 - shutting down a provider whose endpoint is gone
+ pass
+
+ if wait:
+ close()
+ elif providers:
+ threading.Thread(target=close, daemon=True, name="telemetry-stop").start()
+
+
+def _note(kind, ok, error=None):
+ _last[kind] = {"at": time.time(), "ok": ok, "error": error}
+
+
+def _start(endpoint, headers, traces, metrics, ratio, every, service, role):
+ from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+ from opentelemetry.sdk.metrics import MeterProvider
+ from opentelemetry.sdk.metrics.export import MetricExportResult, PeriodicExportingMetricReader
+ from opentelemetry.sdk.resources import Resource
+ from opentelemetry.sdk.trace import TracerProvider
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
+ from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+
+ from . import __version__
+
+ hdrs = parse_headers(headers) if headers else {}
+ resource = Resource.create(
+ {"service.name": service, "service.version": __version__, "service.instance.id": _instance, "lens.process.role": role}
+ )
+
+ class Spans(OTLPSpanExporter):
+ def export(self, spans):
+ try:
+ r = super().export(spans)
+ except Exception as e: # noqa: BLE001
+ _note("traces", False, f"{type(e).__name__}: {e}"[:300])
+ raise
+ _note("traces", r == SpanExportResult.SUCCESS, None if r == SpanExportResult.SUCCESS else "the endpoint refused the traces")
+ return r
+
+ class Metrics(OTLPMetricExporter):
+ def export(self, metrics_data, timeout_millis=10000, **kw):
+ try:
+ r = super().export(metrics_data, timeout_millis=timeout_millis, **kw)
+ except Exception as e: # noqa: BLE001
+ _note("metrics", False, f"{type(e).__name__}: {e}"[:300])
+ raise
+ ok = r == MetricExportResult.SUCCESS
+ _note("metrics", ok, None if ok else "the endpoint refused the metrics")
+ return r
+
+ if traces:
+ tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(max(0.0, min(1.0, ratio)))))
+ tp.add_span_processor(BatchSpanProcessor(Spans(endpoint=endpoint + "/v1/traces", headers=hdrs, timeout=10)))
+ _state.update(tp=tp, tracer=tp.get_tracer("lens", __version__))
+ if metrics:
+ reader = PeriodicExportingMetricReader(
+ Metrics(endpoint=endpoint + "/v1/metrics", headers=hdrs, timeout=10), export_interval_millis=max(5, every) * 1000
+ )
+ mp = MeterProvider(resource=resource, metric_readers=[reader])
+ _state.update(mp=mp, meter=mp.get_meter("lens", __version__))
+ log.info(
+ "telemetry: on, sending %s to %s",
+ " and ".join(x for x, on in (("traces", traces), ("metrics", metrics)) if on),
+ _safe_url(endpoint),
+ )
+
+
+def _safe_url(url):
+ """The endpoint without any user:password in it, for logs and the status."""
+ try:
+ p = urllib.parse.urlsplit(url)
+ return urllib.parse.urlunsplit(
+ (p.scheme, p.hostname + (f":{p.port}" if p.port else "") if p.hostname else p.netloc, p.path, "", "")
+ )
+ except ValueError:
+ return "(invalid address)"
+
+
+def flush(timeout_millis=10000):
+ """Send what is buffered now (tests; a worker about to exit)."""
+ for p in (_state["tp"], _state["mp"]):
+ if p is not None:
+ p.force_flush(timeout_millis)
+
+
+def shutdown():
+ """Send what is buffered and stop (process exit)."""
+ with _lock:
+ _stop()
+ _state["key"] = None
+
+
+atexit.register(shutdown)
+
+
+def active():
+ return _state["tracer"] is not None or _state["meter"] is not None
+
+
+def status(cfg):
+ """What this process does with telemetry, for Settings."""
+ t = _section(cfg)
+ return {
+ "enabled": bool(t.get("enabled")),
+ "endpoint": _safe_url(t["endpoint"]) if t.get("endpoint") else None,
+ "traces": bool(_state["tracer"]),
+ "metrics": bool(_state["meter"]),
+ "last_traces": _last["traces"],
+ "last_metrics": _last["metrics"],
+ }
+
+
+# ---------- spans ----------
+class _NoSpan:
+ """What a span is while telemetry is off: nothing."""
+
+ def set_attribute(self, key, value):
+ pass
+
+ def set_attributes(self, attrs):
+ pass
+
+ def record_exception(self, e):
+ pass
+
+ def set_status(self, *a, **k):
+ pass
+
+ def update_name(self, name):
+ pass
+
+ def end(self):
+ pass
+
+ def is_recording(self):
+ return False
+
+
+NO_SPAN = _NoSpan()
+
+
+def _clean(attrs):
+ return {k: v for k, v in (attrs or {}).items() if v is not None}
+
+
+@contextmanager
+def span(name, attrs=None, kind=None):
+ """A span around a block (the current span while it runs), or nothing while traces are off. An exception that
+ leaves the block marks the span failed with its type (not its message, which may quote content) and goes on."""
+ tracer = _state["tracer"]
+ if tracer is None:
+ yield NO_SPAN
+ return
+ from opentelemetry.trace import SpanKind
+
+ with tracer.start_as_current_span(
+ name, kind=kind or SpanKind.INTERNAL, attributes=_clean(attrs), record_exception=False, set_status_on_exception=False
+ ) as s:
+ try:
+ yield s
+ except BaseException as e:
+ fail(s, e)
+ raise
+
+
+def start_span(name, attrs=None, kind=None):
+ """A span that isn't made current (for generators, which may resume in another context); end() it."""
+ tracer = _state["tracer"]
+ if tracer is None:
+ return NO_SPAN
+ from opentelemetry.trace import SpanKind
+
+ return tracer.start_span(name, kind=kind or SpanKind.INTERNAL, attributes=_clean(attrs))
+
+
+def fail(s, e):
+ """Mark a span failed with an exception's type (not its message, which may quote content)."""
+ if not s.is_recording():
+ return
+ from opentelemetry.trace import Status, StatusCode
+
+ s.set_attribute("error.type", type(e).__name__)
+ s.set_status(Status(StatusCode.ERROR, type(e).__name__))
+
+
+# ---------- metrics ----------
+def _instrument(kind, name, unit, description):
+ meter = _state["meter"]
+ if meter is None:
+ return None
+ inst = _state["instruments"].get(name)
+ if inst is None:
+ make = {"counter": meter.create_counter, "histogram": meter.create_histogram}[kind]
+ inst = _state["instruments"][name] = make(name, unit=unit, description=description)
+ return inst
+
+
+METRICS = {
+ "http.server.request.duration": ("histogram", "s", "API request duration"),
+ "lens.job.step.duration": ("histogram", "s", "How long a job's step took"),
+ "lens.jobs": ("counter", "{job}", "Job runs, by how they ended"),
+ "lens.routine.runs": ("counter", "{run}", "Routine runs, by how they ended"),
+ "lens.workflow.runs": ("counter", "{run}", "Workflow runs, by how they ended"),
+ "gen_ai.client.operation.duration": ("histogram", "s", "Model call duration"),
+ "gen_ai.client.token.usage": ("histogram", "{token}", "Tokens per model call"),
+ "lens.llm.cost": ("counter", "USD", "Estimated model cost, from the prices set in Settings → Telemetry"),
+}
+
+
+def record(name, value, attrs=None):
+ """Add to a counter or record in a histogram (see METRICS); nothing while metrics are off."""
+ kind, unit, description = METRICS[name]
+ inst = _instrument(kind, name, unit, description)
+ if inst is None:
+ return
+ try:
+ (inst.add if kind == "counter" else inst.record)(value, _clean(attrs))
+ except Exception: # noqa: BLE001 - a metric never breaks the work it measures
+ pass
+
+
+# ---------- model calls ----------
+def cost(model, input_tokens, output_tokens, prices=None):
+ """Estimated cost in USD from the per-million-token prices for the model, or None when it has no price."""
+ p = (prices if prices is not None else _state["cfg"].get("prices") or {}).get(model)
+ if not isinstance(p, dict):
+ return None
+ return round((input_tokens or 0) / 1e6 * float(p.get("input") or 0) + (output_tokens or 0) / 1e6 * float(p.get("output") or 0), 8)
+
+
+class ModelCall:
+ """One call to the model server: a client span (gen_ai.* attributes), its duration, tokens and estimated cost.
+ Neither the prompt nor the reply is recorded."""
+
+ def __init__(self, cfg, payload, operation="chat", current=True):
+ self.on = active()
+ if not self.on:
+ return
+ l = (cfg or {}).get("llm") or {}
+ self.model = payload.get("model")
+ self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None
+ server = urllib.parse.urlsplit(l.get("base_url") or "")
+ attrs = {
+ "gen_ai.operation.name": operation,
+ "gen_ai.provider.name": "openai", # the OpenAI-compatible API, whoever serves it
+ "gen_ai.request.model": self.model,
+ "gen_ai.request.temperature": payload.get("temperature"),
+ "gen_ai.request.max_tokens": payload.get("max_tokens"),
+ "gen_ai.output.type": "json" if payload.get("response_format") else "text",
+ "lens.llm.tools": len(payload.get("tools") or []) or None,
+ "lens.llm.stream": True if payload.get("stream") else None,
+ "server.address": server.hostname,
+ "server.port": server.port,
+ }
+ from opentelemetry.trace import SpanKind
+
+ name = f"{operation} {self.model}"
+ if current:
+ self._cm = span(name, attrs, SpanKind.CLIENT)
+ self.span = self._cm.__enter__()
+ else:
+ self._cm, self.span = None, start_span(name, attrs, SpanKind.CLIENT)
+
+ def reply(self, j):
+ """Note the usage and model of a reply (a parsed JSON body or a streamed chunk); returns it."""
+ if self.on and isinstance(j, dict):
+ if isinstance(j.get("usage"), dict):
+ self.usage = j["usage"]
+ if isinstance(j.get("model"), str):
+ self.response_model = j["model"]
+ return j
+
+ def end(self, error=None):
+ if not self.on or self.done:
+ return
+ self.done = True
+ seconds = time.monotonic() - self.t0
+ base = {"gen_ai.operation.name": self.operation, "gen_ai.provider.name": "openai", "gen_ai.request.model": self.model}
+ if self.response_model:
+ base["gen_ai.response.model"] = self.response_model
+ self.span.set_attribute("gen_ai.response.model", self.response_model)
+ if error is not None:
+ fail(self.span, error)
+ record("gen_ai.client.operation.duration", seconds, {**base, "error.type": type(error).__name__})
+ else:
+ record("gen_ai.client.operation.duration", seconds, base)
+ u = self.usage or {}
+ tin, tout = u.get("prompt_tokens", u.get("input_tokens")), u.get("completion_tokens", u.get("output_tokens"))
+ if isinstance(tin, int):
+ self.span.set_attribute("gen_ai.usage.input_tokens", tin)
+ record("gen_ai.client.token.usage", tin, {**base, "gen_ai.token.type": "input"})
+ if isinstance(tout, int):
+ self.span.set_attribute("gen_ai.usage.output_tokens", tout)
+ record("gen_ai.client.token.usage", tout, {**base, "gen_ai.token.type": "output"})
+ if isinstance(tin, int) or isinstance(tout, int):
+ usd = cost(self.model, tin if isinstance(tin, int) else 0, tout if isinstance(tout, int) else 0)
+ if usd is not None:
+ self.span.set_attribute("lens.llm.cost_usd", usd)
+ record("lens.llm.cost", usd, base)
+ if self._cm is not None:
+ self._cm.__exit__(None, None, None)
+ else:
+ self.span.end()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, et, e, tb):
+ self.end(e)
+ return False
+
+
+def model_call(cfg, payload, operation="chat", current=True):
+ """`with model_call(cfg, payload) as call: ... call.reply(json)`: a measured model call (a no-op while off)."""
+ return ModelCall(cfg, payload, operation, current)
+
+
+# ---------- the API ----------
+class Middleware:
+ """A server span and a duration for each API request, named by its route template (`GET /api/v1/recordings/{rid}`),
+ never by its path or query, which can carry names. Installed innermost, where the router has chosen the route."""
+
+ def __init__(self, app):
+ self.app = app
+
+ async def __call__(self, scope, receive, send):
+ if scope["type"] != "http" or not active():
+ await self.app(scope, receive, send)
+ return
+ from opentelemetry import context, trace
+ from opentelemetry.trace import SpanKind, Status, StatusCode
+
+ method = scope.get("method") or "GET"
+ status = [None]
+
+ async def sending(message):
+ if message["type"] == "http.response.start":
+ status[0] = message["status"]
+ await send(message)
+
+ t0, s, error = time.monotonic(), start_span(method, {"http.request.method": method}, SpanKind.SERVER), None
+ token = context.attach(trace.set_span_in_context(s)) if s.is_recording() else None
+ try:
+ await self.app(scope, receive, sending)
+ except Exception as e:
+ error = e
+ raise
+ finally:
+ if token is not None:
+ context.detach(token)
+ route = getattr(scope.get("route"), "path", None) or "unmatched"
+ code = status[0] or 500
+ attrs = {"http.request.method": method, "http.route": route, "http.response.status_code": code}
+ if error is not None:
+ attrs["error.type"] = type(error).__name__
+ s.update_name(f"{method} {route}")
+ s.set_attributes(attrs)
+ if error is not None:
+ fail(s, error)
+ elif code >= 500:
+ s.set_status(Status(StatusCode.ERROR))
+ s.end()
+ record("http.server.request.duration", time.monotonic() - t0, attrs)
+
+
+def test_export(cfg, timeout=5):
+ """Send one test span to the configured endpoint now, from a provider of its own; (ok, error, ms)."""
+ t = _section(cfg)
+ if not t.get("endpoint"):
+ return False, "set an endpoint first", None
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+ from opentelemetry.sdk.resources import Resource
+ from opentelemetry.sdk.trace import TracerProvider
+ from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
+
+ from . import __version__
+
+ try:
+ hdrs = parse_headers(t.get("headers")) if isinstance(t.get("headers"), str) else {}
+ except ValueError as e:
+ return False, str(e), None
+ got = {}
+
+ class Once(OTLPSpanExporter):
+ def export(self, spans):
+ try:
+ got["r"] = super().export(spans)
+ except Exception as e: # noqa: BLE001
+ got["e"] = f"{type(e).__name__}: {e}"[:300]
+ raise
+ return got["r"]
+
+ tp = TracerProvider(resource=Resource.create({"service.name": t.get("service_name") or "lens", "service.version": __version__}))
+ tp.add_span_processor(SimpleSpanProcessor(Once(endpoint=t["endpoint"].rstrip("/") + "/v1/traces", headers=hdrs, timeout=timeout)))
+ t0 = time.monotonic()
+ try:
+ with tp.get_tracer("lens").start_as_current_span("lens.telemetry.test"):
+ pass
+ finally:
+ tp.shutdown()
+ ms = int((time.monotonic() - t0) * 1000)
+ if got.get("r") == SpanExportResult.SUCCESS:
+ return True, None, ms
+ return (
+ False,
+ got.get("e") or f"the endpoint at {_safe_url(t['endpoint'])} didn't accept the span (check the address and the collector's log)",
+ ms,
+ )
diff --git a/fastapi_backend/app/domain/workflows.py b/fastapi_backend/app/domain/workflows.py
index b19207ba6..e048b81a4 100644
--- a/fastapi_backend/app/domain/workflows.py
+++ b/fastapi_backend/app/domain/workflows.py
@@ -30,7 +30,7 @@
import re
-from . import analyze, fields as fieldmod, llm, metadata, organize, pipelines, store, templates
+from . import analyze, fields as fieldmod, llm, metadata, organize, pipelines, store, telemetry, templates
R = store.R
RECORDING_NODES = ("input", "llm", "pick", "condition", "merge", "extract_rules", "extract_llm", "output", "field", "save_entities")
@@ -437,6 +437,18 @@ def save_entities(db, cfg, rid, ents):
def run(db, cfg, rid, wid, version=None, say=print, user=None):
"""Run one version (default: the current one) of a workflow on a recording. Returns what each node did:
{node id: done | skipped}."""
+ attrs = {"lens.workflow.id": str(wid), "lens.workflow.version": version, "lens.recording.id": str(rid)}
+ with telemetry.span("workflow", attrs):
+ try:
+ out = _run(db, cfg, rid, wid, version, say, user)
+ except Exception:
+ telemetry.record("lens.workflow.runs", 1, {"lens.outcome": "failed"})
+ raise
+ telemetry.record("lens.workflow.runs", 1, {"lens.outcome": "done"})
+ return out
+
+
+def _run(db, cfg, rid, wid, version, say, user):
w = get(db, wid, version)
if w["scope"] != "recording":
raise ValueError(f"workflow {w['name']} organises the graph; a routine runs it, not a pipeline")
diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py
index 6f7343a44..d7dd41eff 100644
--- a/fastapi_backend/app/main.py
+++ b/fastapi_backend/app/main.py
@@ -24,7 +24,7 @@
from app.config import settings
from app.core import middleware
from app.core.runtime import Archive
-from app.domain import __version__, store
+from app.domain import __version__, store, telemetry
from app.domain.render import WEB_DIR
from app.utils import simple_generate_unique_route_id
@@ -56,6 +56,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
yield
finally:
archive.close()
+ telemetry.shutdown()
app = FastAPI(
title=settings.PROJECT_NAME,
@@ -68,6 +69,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
ready.prepare()
_attach(app, ready)
+ # innermost, so the router has picked the route it names spans by; a no-op while telemetry is off
+ app.add_middleware(telemetry.Middleware)
app.add_middleware(
CORSMiddleware,
allow_origins=sorted(settings.CORS_ORIGINS),
diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py
index f2b956ab1..03e40d5a2 100644
--- a/fastapi_backend/app/schemas/admin.py
+++ b/fastapi_backend/app/schemas/admin.py
@@ -23,6 +23,29 @@ class LlmTestResult(ResponseModel):
model: str | None = None
+class TelemetryExport(ResponseModel):
+ at: float
+ ok: bool
+ error: str | None = None
+
+
+class TelemetryStatus(ResponseModel):
+ """What the API process does with telemetry now; each worker follows the same settings on its own."""
+
+ enabled: bool
+ endpoint: str | None = None
+ traces: bool
+ metrics: bool
+ last_traces: TelemetryExport | None = None
+ last_metrics: TelemetryExport | None = None
+
+
+class TelemetryTestResult(ResponseModel):
+ ok: bool
+ error: str | None = None
+ ms: int | None = None
+
+
class AuditEntry(ResponseModel):
at: str
email: str | None = None
diff --git a/fastapi_backend/app/schemas/setup.py b/fastapi_backend/app/schemas/setup.py
index 93e409b5a..0e67559d6 100644
--- a/fastapi_backend/app/schemas/setup.py
+++ b/fastapi_backend/app/schemas/setup.py
@@ -30,12 +30,20 @@ class SetupStorageView(ResponseModel):
watches: int
+class SetupTelemetryView(ResponseModel):
+ enabled: bool
+ endpoint: str | None = None
+ # what .env sets (LENS_TELEMETRY, LENS_TELEMETRY_ENDPOINT): shown locked
+ locked: list[str]
+
+
class SetupView(ResponseModel):
pending: bool
admin: SetupAdmin
namespace: SetupNamespaceView
llm: SetupLlmView
storage: SetupStorageView
+ telemetry: SetupTelemetryView
class SetupNamespace(RequestModel):
@@ -57,6 +65,12 @@ class SetupStorage(RequestModel):
namespace: str | None = None
+class SetupTelemetry(RequestModel):
+ enabled: bool = False
+ # the OTLP/HTTP address of a collector, like http://localhost:4318; needed to turn telemetry on
+ endpoint: str | None = Field(default=None, max_length=500)
+
+
class SetupFinish(RequestModel):
skipped: bool = False
diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml
index b28c3566d..f9755610c 100644
--- a/fastapi_backend/pyproject.toml
+++ b/fastapi_backend/pyproject.toml
@@ -18,6 +18,9 @@ dependencies = [
"pillow>=10",
"markdown>=3.5",
"cryptography>=42",
+ "opentelemetry-api>=1.27,<2",
+ "opentelemetry-sdk>=1.27,<2",
+ "opentelemetry-exporter-otlp-proto-http>=1.27,<2",
]
[project.optional-dependencies]
diff --git a/fastapi_backend/tests/api/test_telemetry.py b/fastapi_backend/tests/api/test_telemetry.py
new file mode 100644
index 000000000..e4860c5bd
--- /dev/null
+++ b/fastapi_backend/tests/api/test_telemetry.py
@@ -0,0 +1,226 @@
+"""Opt-in telemetry: off by default, sent only to the configured OTLP endpoint, and nothing personal in it."""
+
+from __future__ import annotations
+
+import threading
+
+import pytest
+
+from app.domain import jobs, telemetry
+from tests import fake_llm, fake_otlp
+from tests.helpers import drain, login, make_user, seed
+
+
+@pytest.fixture
+def collector():
+ srv, url = fake_otlp.start()
+ yield url
+ telemetry.shutdown() # telemetry is per process: the next test starts with it off
+ srv.shutdown()
+
+
+@pytest.fixture
+def model(cfg):
+ srv, url = fake_llm.start()
+ cfg["llm"].update(base_url=url, model="fake")
+ fake_llm.Handler.usage = {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500}
+ yield fake_llm.Handler
+ fake_llm.Handler.usage = None
+ srv.shutdown()
+
+
+@pytest.fixture
+def app(cfg, db, model):
+ from app.main import create_app
+
+ return create_app(cfg, db, background=False)
+
+
+def admin(client, db):
+ make_user(db, "root@x.io", "root password 1", admin=True)
+ return login(client, "root@x.io", "root password 1")
+
+
+def test_off_by_default_sends_nothing(client, db, cfg, folder, collector):
+ rid = seed(db, cfg, folder)[0]
+ h = admin(client, db)
+ t = client.get("/api/v1/settings", headers=h).json()["telemetry"]
+ assert t["values"]["enabled"] is False and t["values"]["endpoint"] is None
+ assert t["values"]["headers"] == {"secret": True, "set": False} and t["locked"] == []
+ # an endpoint alone doesn't turn it on
+ assert client.put("/api/v1/settings/telemetry", json={"endpoint": collector}, headers=h).status_code == 200
+ assert client.get(f"/api/v1/recordings/{rid}", headers=h).status_code == 200
+ jobs.enqueue(db, rid, ["analyze"])
+ drain(db, cfg)
+ assert client.post("/api/v1/settings/llm/test", headers=h).json()["ok"]
+ assert not telemetry.active()
+ telemetry.flush()
+ assert fake_otlp.Handler.bodies == []
+ s = client.get("/api/v1/settings/telemetry/status", headers=h).json()
+ assert (s["enabled"], s["traces"], s["metrics"], s["last_traces"]) == (False, False, False, None)
+
+
+def test_settings_are_checked(client, db, collector):
+ h = admin(client, db)
+
+ def put(body):
+ return client.put("/api/v1/settings/telemetry", json=body, headers=h)
+
+ for bad in (
+ {"endpoint": "ftp://collector:4318"},
+ {"endpoint": "http://collector:4318/v1/traces"},
+ {"enabled": "yes"},
+ {"sample_ratio": 1.5},
+ {"export_seconds": 1},
+ {"service_name": "lens server"},
+ {"prices": {"gpt": {"input": -1, "output": 1}}},
+ {"prices": {"gpt": {"input": 1, "cached": 1}}},
+ {"prices": ["gpt"]},
+ {"headers": "no-equals-sign"},
+ {"colour": "blue"},
+ ):
+ assert put(bad).status_code == 400, bad
+ assert put({"endpoint": "http://collector:4318/", "prices": {" gpt-4o ": {"input": 2.5, "output": 10}}}).status_code == 200
+ v = client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]
+ assert v["endpoint"] == "http://collector:4318" and v["prices"] == {"gpt-4o": {"input": 2.5, "output": 10.0}}
+ assert put({"headers": "Authorization=Bearer%20abc,X-Tenant=lens"}).status_code == 200
+ assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"] == {"secret": True, "set": True}
+ assert put({"prices": {}}).status_code == 200
+
+
+def test_traces_and_metrics_go_to_the_endpoint(client, db, cfg, folder, collector, model):
+ cfg["ai"]["tools"] = False # the plain, streamed answer
+ rid = seed(db, cfg, folder)[0]
+ h = admin(client, db)
+ r = client.put(
+ "/api/v1/settings/telemetry",
+ headers=h,
+ json={
+ "enabled": True,
+ "endpoint": collector,
+ "headers": "Authorization=Bearer%20t0k",
+ "prices": {"fake": {"input": 1.0, "output": 2.0}},
+ },
+ )
+ assert r.status_code == 200, r.text
+ # the next request picks the change up
+ assert client.get(f"/api/v1/recordings/{rid}", params={"q": "private words"}, headers=h).status_code == 200
+ assert telemetry.active()
+ assert client.get("/api/v1/recordings/999999", headers=h).status_code == 404
+ assert client.post("/api/v1/settings/llm/test", headers=h).json()["ok"]
+ jobs.enqueue(db, rid, ["analyze"])
+ drain(db, cfg)
+ cid = client.post("/api/v1/chats", headers=h, json={}).json()["id"]
+ assert "The shipment" in client.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "When does it ship?"}).text
+ telemetry.flush()
+
+ spans = {s["name"]: s for s in fake_otlp.Handler.spans}
+ got = [s["attrs"] for s in fake_otlp.Handler.spans if s["name"] == "GET /api/v1/recordings/{rid}"]
+ assert [(a["http.route"], a["http.response.status_code"]) for a in got] == [("/api/v1/recordings/{rid}", c) for c in (200, 404)]
+ got = spans["GET /api/v1/recordings/{rid}"]
+ assert got["resource"]["service.name"] == "lens" and got["resource"]["lens.process.role"] == "api"
+ assert "host.name" not in got["resource"]
+ job, step = spans["job"], spans["step analyze"]
+ assert (job["attrs"]["lens.recording.id"], job["attrs"]["lens.job.outcome"]) == (str(rid), "succeeded")
+ assert "lens.pipeline.id" not in job["attrs"] # steps asked for by hand, not a pipeline
+ assert step["attrs"]["lens.step.outcome"] == "done"
+ calls = [s for s in fake_otlp.Handler.spans if s["name"] == "chat fake"]
+ assert len(calls) == 2 # the test and the streamed answer
+ for c in calls:
+ a = c["attrs"]
+ assert (a["gen_ai.request.model"], a["gen_ai.usage.input_tokens"], a["gen_ai.usage.output_tokens"]) == ("fake", 1000, 500)
+ assert a["lens.llm.cost_usd"] == pytest.approx(0.002) and a["server.address"] == "127.0.0.1"
+ assert {h_.get("authorization") for h_ in fake_otlp.Handler.headers} == {"Bearer t0k"}
+
+ names = {p["name"] for p in fake_otlp.Handler.points}
+ assert {"http.server.request.duration", "lens.jobs", "lens.job.step.duration", "gen_ai.client.token.usage"} <= names
+ assert {"gen_ai.client.operation.duration", "lens.llm.cost"} <= names
+ cost = sum(p["value"] for p in fake_otlp.Handler.points if p["name"] == "lens.llm.cost")
+ assert cost == pytest.approx(0.004)
+ tokens = {"input": 0.0, "output": 0.0}
+ for p in fake_otlp.Handler.points:
+ if p["name"] == "gen_ai.client.token.usage":
+ tokens[p["attrs"]["gen_ai.token.type"]] += p["value"]
+ assert tokens == {"input": 2000, "output": 1000}
+
+ # nothing personal: no query strings, prompts, answers, titles or transcript text
+ raw = b"".join(fake_otlp.Handler.bodies)
+ for text in (b"private words", b"When does it ship", b"The shipment", b"ep1", b"capsid", b"root@x.io"):
+ assert text not in raw, text
+
+ s = client.get("/api/v1/settings/telemetry/status", headers=h).json()
+ assert (s["enabled"], s["endpoint"], s["traces"], s["metrics"]) == (True, collector, True, True)
+ assert s["last_traces"]["ok"] and s["last_metrics"]["ok"]
+
+ # off again: the next request stops it, and nothing more is sent
+ assert client.put("/api/v1/settings/telemetry", json={"enabled": False}, headers=h).status_code == 200
+ client.get("/api/v1/jobs", headers=h)
+ assert not telemetry.active()
+ for t in threading.enumerate(): # the old providers send what they held, in the background
+ if t.name == "telemetry-stop":
+ t.join(15)
+ fake_otlp.reset()
+ client.get(f"/api/v1/recordings/{rid}", headers=h)
+ assert fake_otlp.Handler.bodies == []
+
+
+def test_send_a_test_span(client, db, collector):
+ h = admin(client, db)
+ r = client.post("/api/v1/settings/telemetry/test", headers=h).json()
+ assert (r["ok"], r["error"]) == (False, "set an endpoint first")
+ client.put("/api/v1/settings/telemetry", json={"endpoint": collector}, headers=h)
+ r = client.post("/api/v1/settings/telemetry/test", headers=h).json()
+ assert r["ok"] and r["ms"] >= 0, r
+ assert [s["name"] for s in fake_otlp.Handler.spans] == ["lens.telemetry.test"]
+ assert not telemetry.active() # testing doesn't turn it on
+ fake_otlp.Handler.refuse = True
+ r = client.post("/api/v1/settings/telemetry/test", headers=h).json()
+ assert not r["ok"] and r["error"]
+
+
+def test_environment_turns_it_on_or_keeps_it_off(client, db, collector, monkeypatch):
+ h = admin(client, db)
+ monkeypatch.setenv("LENS_TELEMETRY", "off")
+ client.put("/api/v1/settings/telemetry", json={"enabled": True, "endpoint": collector}, headers=h)
+ t = client.get("/api/v1/settings", headers=h).json()["telemetry"]
+ assert t["values"]["enabled"] is False and t["locked"] == ["enabled"]
+ assert not telemetry.active()
+ assert client.get("/api/v1/setup", headers=h).json()["telemetry"]["locked"] == ["enabled"]
+
+ monkeypatch.setenv("LENS_TELEMETRY", "on")
+ monkeypatch.setenv("LENS_TELEMETRY_ENDPOINT", collector + "/")
+ client.put("/api/v1/settings/telemetry", json={"enabled": False}, headers=h) # a save rebuilds the settings
+ client.get("/api/v1/jobs", headers=h)
+ assert telemetry.active()
+ assert client.get("/api/v1/settings/telemetry/status", headers=h).json()["endpoint"] == collector + "/"
+
+
+def test_setup_wizard_opt_in(client, db, collector):
+ h = admin(client, db)
+ v = client.get("/api/v1/setup", headers=h).json()["telemetry"]
+ assert v == {"enabled": False, "endpoint": None, "locked": []}
+ r = client.put("/api/v1/setup/telemetry", json={"enabled": True}, headers=h)
+ assert r.status_code == 400 and "endpoint" in r.json()["detail"]
+ assert client.put("/api/v1/setup/telemetry", json={"enabled": True, "endpoint": "nope"}, headers=h).status_code == 400
+ r = client.put("/api/v1/setup/telemetry", json={"enabled": True, "endpoint": collector}, headers=h)
+ assert r.json()["saved"] == ["enabled", "endpoint"]
+ assert client.get("/api/v1/setup", headers=h).json()["telemetry"] == {"enabled": True, "endpoint": collector, "locked": []}
+ # off keeps the address for later
+ client.put("/api/v1/setup/telemetry", json={"enabled": False}, headers=h)
+ assert client.get("/api/v1/setup", headers=h).json()["telemetry"] == {"enabled": False, "endpoint": collector, "locked": []}
+ audit = client.get("/api/v1/audit", headers=h).json()
+ assert any(a["action"] == "settings.save" and a["target"] == "telemetry" for a in audit)
+
+
+def test_cost_and_headers():
+ prices = {"m": {"input": 3.0, "output": 15.0}}
+ assert telemetry.cost("m", 2_000_000, 100_000, prices) == pytest.approx(7.5)
+ assert telemetry.cost("other", 10, 10, prices) is None
+ cfg = {"llm": {"model": "m"}, "ai": {"price_in": 1.0, "price_out": 2.0}, "telemetry": {"prices": {"x": {"input": 0, "output": 1}}}}
+ assert telemetry.prices(cfg) == {"m": {"input": 1.0, "output": 2.0}, "x": {"input": 0, "output": 1}}
+ cfg["telemetry"]["prices"]["m"] = {"input": 5.0, "output": 5.0}
+ assert telemetry.prices(cfg)["m"] == {"input": 5.0, "output": 5.0}
+ assert telemetry.parse_headers("a=1, Authorization=Bearer%20x") == {"a": "1", "Authorization": "Bearer x"}
+ assert telemetry.parse_headers("") == {}
+ with pytest.raises(ValueError):
+ telemetry.parse_headers("bad header=1")
diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py
index e6b947e7f..f6bd0a0d3 100644
--- a/fastapi_backend/tests/fake_llm.py
+++ b/fastapi_backend/tests/fake_llm.py
@@ -12,6 +12,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
tool_script = [] # assistant messages to return, in order, when a request offers tools
reject_tools = False # behave like a server whose model can't call tools
blind = False # behave like a server whose model can't see images
+ usage = None # token counts to report with each answer (and as a streamed answer's last chunk), like OpenAI
def _json(self, obj):
data = json.dumps(obj).encode()
@@ -46,6 +47,8 @@ def do_POST(self):
self.end_headers()
for piece in ["The shipment ", "leaves on Friday [1]."]:
self.wfile.write(f"data: {json.dumps({'choices': [{'delta': {'content': piece}}]})}\n\n".encode())
+ if Handler.usage:
+ self.wfile.write(f"data: {json.dumps({'choices': [], 'usage': Handler.usage})}\n\n".encode())
self.wfile.write(b"data: [DONE]\n\n")
return
seen = _picture(body)
@@ -115,7 +118,8 @@ def do_POST(self):
)
else:
content = "OK"
- data = json.dumps({"choices": [{"message": {"content": content}}]}).encode()
+ extra = {"usage": Handler.usage, "model": body.get("model")} if Handler.usage else {}
+ data = json.dumps({"choices": [{"message": {"content": content}}], **extra}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
diff --git a/fastapi_backend/tests/fake_otlp.py b/fastapi_backend/tests/fake_otlp.py
new file mode 100644
index 000000000..df34e8a79
--- /dev/null
+++ b/fastapi_backend/tests/fake_otlp.py
@@ -0,0 +1,77 @@
+"""A tiny OTLP/HTTP collector for tests: keeps the spans and metric points it is sent, decoded."""
+
+import http.server
+import threading
+
+from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+
+
+def _value(v):
+ kind = v.WhichOneof("value")
+ return getattr(v, kind) if kind else None
+
+
+def _attrs(kvs):
+ return {kv.key: _value(kv.value) for kv in kvs}
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ spans = [] # {name, kind, attrs, status, resource}
+ points = [] # {name, unit, attrs, value}
+ headers = [] # the request headers of each export
+ bodies = [] # the raw requests, to check nothing personal is in them
+ refuse = False # answer 400, like a collector that won't take what it's sent
+
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers["Content-Length"]))
+ Handler.headers.append({k.lower(): v for k, v in self.headers.items()})
+ Handler.bodies.append(body)
+ if Handler.refuse:
+ self.send_response(400)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
+ if self.path == "/v1/traces":
+ req = ExportTraceServiceRequest()
+ req.ParseFromString(body)
+ for rs in req.resource_spans:
+ res = _attrs(rs.resource.attributes)
+ for ss in rs.scope_spans:
+ for s in ss.spans:
+ Handler.spans.append(
+ {"name": s.name, "kind": s.kind, "attrs": _attrs(s.attributes), "status": s.status.code, "resource": res}
+ )
+ elif self.path == "/v1/metrics":
+ req = ExportMetricsServiceRequest()
+ req.ParseFromString(body)
+ for rm in req.resource_metrics:
+ for sm in rm.scope_metrics:
+ for m in sm.metrics:
+ data = getattr(m, m.WhichOneof("data"))
+ for p in data.data_points:
+ value = p.sum if hasattr(p, "bucket_counts") else _number(p)
+ Handler.points.append({"name": m.name, "unit": m.unit, "attrs": _attrs(p.attributes), "value": value})
+ self.send_response(200)
+ self.send_header("Content-Type", "application/x-protobuf")
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+
+ def log_message(self, *a):
+ pass
+
+
+def _number(p):
+ kind = p.WhichOneof("value")
+ return getattr(p, kind) if kind else None
+
+
+def reset():
+ Handler.spans, Handler.points, Handler.headers, Handler.bodies, Handler.refuse = [], [], [], [], False
+
+
+def start():
+ reset()
+ srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ return srv, f"http://127.0.0.1:{srv.server_address[1]}"
diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock
index f7e11941c..20a5daa50 100644
--- a/fastapi_backend/uv.lock
+++ b/fastapi_backend/uv.lock
@@ -1584,6 +1584,9 @@ dependencies = [
{ name = "jinja2" },
{ name = "markdown" },
{ name = "numpy" },
+ { name = "opentelemetry-api" },
+ { name = "opentelemetry-exporter-otlp-proto-http" },
+ { name = "opentelemetry-sdk" },
{ name = "pillow" },
{ name = "pydantic-settings" },
{ name = "pyjwt" },
@@ -1679,6 +1682,9 @@ requires-dist = [
{ name = "onnxruntime", marker = "extra == 'insightface'" },
{ name = "onnxruntime", marker = "extra == 'objects'" },
{ name = "opencv-python-headless", marker = "extra == 'faces'" },
+ { name = "opentelemetry-api", specifier = ">=1.27,<2" },
+ { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27,<2" },
+ { name = "opentelemetry-sdk", specifier = ">=1.27,<2" },
{ name = "pillow", specifier = ">=10" },
{ name = "pyannote-audio", marker = "extra == 'pyannote'" },
{ name = "pydantic-settings", specifier = ">=2.5.2,<3" },
diff --git a/mkdocs.yml b/mkdocs.yml
index 88bc8b5f4..952a8ce6c 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -71,6 +71,7 @@ nav:
- Video: video.md
- Chat and batch runs: assistant.md
- Notifications: notifications.md
+ - Telemetry: telemetry.md
- IIIF: iiif.md
- API: api.md
- API work the design needs: backend-gaps.md
diff --git a/nextjs-frontend/__tests__/settings-admin-models.test.ts b/nextjs-frontend/__tests__/settings-admin-models.test.ts
index 7586483d9..b0ba60e11 100644
--- a/nextjs-frontend/__tests__/settings-admin-models.test.ts
+++ b/nextjs-frontend/__tests__/settings-admin-models.test.ts
@@ -193,6 +193,35 @@ describe("settings fields", () => {
}),
).toBe("Adds https://docs.lens.local.");
});
+
+ it("handles telemetry: off by default, prices as lines, an endpoint to turn it on", () => {
+ expect(spec("telemetry.enabled").kind).toBe("switch");
+ const prices = spec("telemetry.prices");
+ expect(toUi(prices, { "gpt-4o-mini": { input: 0.15, output: 0.6 }, local: { input: 0, output: 0 } })).toBe(
+ "gpt-4o-mini 0.15 0.6\nlocal 0 0",
+ );
+ expect(parse(prices, " gpt-4o-mini 0.15 0.60 \n\n my model 1 2")).toEqual({
+ value: { "gpt-4o-mini": { input: 0.15, output: 0.6 }, "my model": { input: 1, output: 2 } },
+ });
+ expect(parse(prices, "gpt-4o 2.5")).toEqual({
+ error: "Line 1: write the model, then its input and output price, like gpt-4o-mini 0.15 0.60",
+ });
+ expect(parse(prices, "a 1 2\nb -1 2")).toHaveProperty("error");
+ expect(parse(prices, "")).toEqual({ value: {} });
+ expect(show(prices, { a: { input: 1, output: 2 } })).toBe("1 model");
+ expect(show(prices, {})).toBe("none");
+ expect(crossErrors({ "telemetry.enabled": true, "telemetry.endpoint": null })["telemetry.endpoint"]).toMatch(
+ /address to send/,
+ );
+ expect(crossErrors({ "telemetry.enabled": false, "telemetry.endpoint": null })).toEqual({});
+ expect(crossErrors({ "telemetry.endpoint": "localhost:4318" })["telemetry.endpoint"]).toMatch(/http/);
+ expect(crossErrors({ "telemetry.endpoint": "http://c:4318/v1/traces" })["telemetry.endpoint"]).toMatch(
+ /without \/v1/,
+ );
+ const on = { id: "telemetry.enabled", field: spec("telemetry.enabled"), before: false, after: true };
+ expect(why(on)).toMatch(/start sending/);
+ expect(why({ ...on, before: true, after: false })).toMatch(/Nothing more is sent/);
+ });
});
describe("audit log", () => {
diff --git a/nextjs-frontend/__tests__/setupWizard.test.tsx b/nextjs-frontend/__tests__/setupWizard.test.tsx
index 264b9a6b0..267d57404 100644
--- a/nextjs-frontend/__tests__/setupWizard.test.tsx
+++ b/nextjs-frontend/__tests__/setupWizard.test.tsx
@@ -13,9 +13,10 @@ jest.mock("@/app/openapi-client", () => ({
saveNamespace: jest.fn(),
saveLlm: jest.fn(),
saveStorage: jest.fn(),
+ saveTelemetry: jest.fn(),
finish: jest.fn(),
},
- Admin: { testLlm: jest.fn() },
+ Admin: { testLlm: jest.fn(), testTelemetry: jest.fn() },
}));
jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) }));
const replace = jest.fn();
@@ -38,6 +39,7 @@ const VIEW = (over: Partial = {}): SetupView => ({
max_upload_mb: 4096,
watches: 0,
},
+ telemetry: { enabled: false, endpoint: null, locked: [] },
...over,
});
@@ -53,12 +55,13 @@ function wrap(ui: ReactNode) {
beforeEach(() => jest.clearAllMocks());
describe("SetupWizard", () => {
- it("walks through namespace, model provider and storage, then finishes", async () => {
+ it("walks through namespace, model provider, storage and telemetry, then finishes", async () => {
m(Setup.getSetup).mockImplementation(() => ok(VIEW()));
m(Setup.saveNamespace).mockImplementation(() => ok({ ok: true }));
m(Setup.saveLlm).mockImplementation(() => ok({ ok: true }));
m(Admin.testLlm).mockImplementation(() => ok({ ok: true, reply: "OK", ms: 120, model: "llama3" }));
m(Setup.saveStorage).mockImplementation(() => ok({ ok: true }));
+ m(Setup.saveTelemetry).mockImplementation(() => ok({ ok: true }));
m(Setup.finish).mockImplementation(() => ok({ ok: true }));
wrap();
@@ -86,14 +89,42 @@ describe("SetupWizard", () => {
});
fireEvent.change(await screen.findByLabelText(/Largest upload/), { target: { value: "2048" } });
+ fireEvent.click(screen.getByRole("button", { name: "Save and continue" }));
+ await waitFor(() => expect(Setup.saveStorage).toHaveBeenCalled());
+ expect(m(Setup.saveStorage).mock.calls[0][0].body).toEqual({ max_upload_mb: 2048, folder: null, namespace: null });
+
+ // telemetry is off unless chosen: finishing with it off sends nothing anywhere
+ expect(await screen.findByText(/off unless you turn it on/)).toBeInTheDocument();
+ expect(screen.queryByLabelText("OTLP endpoint")).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("button", { name: "Save and finish" }));
await waitFor(() =>
expect(Setup.finish).toHaveBeenCalledWith(expect.objectContaining({ body: { skipped: false } })),
);
- expect(m(Setup.saveStorage).mock.calls[0][0].body).toEqual({ max_upload_mb: 2048, folder: null, namespace: null });
+ expect(m(Setup.saveTelemetry).mock.calls[0][0].body).toEqual({ enabled: false, endpoint: null });
await waitFor(() => expect(replace).toHaveBeenCalledWith("/"));
});
+ it("opts in to telemetry with an endpoint, after a test span", async () => {
+ m(Setup.getSetup).mockImplementation(() => ok(VIEW({ namespace: { existing: ["media"], locked: false } })));
+ m(Setup.saveTelemetry).mockImplementation(() => ok({ ok: true }));
+ m(Admin.testTelemetry).mockImplementation(() => ok({ ok: true, ms: 12 }));
+ m(Setup.finish).mockImplementation(() => ok({ ok: true }));
+ wrap();
+ fireEvent.click(await screen.findByRole("button", { name: /Telemetry/ }));
+ fireEvent.click(await screen.findByRole("radio", { name: "On" }));
+ const save = screen.getByRole("button", { name: "Save and finish" });
+ expect(save).toBeDisabled(); // on needs somewhere to send to
+ fireEvent.change(screen.getByLabelText("OTLP endpoint"), { target: { value: "localhost:4318" } });
+ expect(save).toBeDisabled();
+ fireEvent.change(screen.getByLabelText("OTLP endpoint"), { target: { value: "http://localhost:4318" } });
+ fireEvent.click(screen.getByRole("button", { name: "Send a test span" }));
+ expect(await screen.findByText(/took a test span in 12 ms/)).toBeInTheDocument();
+ expect(m(Setup.saveTelemetry).mock.calls[0][0].body).toEqual({ enabled: false, endpoint: "http://localhost:4318" });
+ fireEvent.click(save);
+ await waitFor(() => expect(Setup.finish).toHaveBeenCalled());
+ expect(m(Setup.saveTelemetry).mock.calls[1][0].body).toEqual({ enabled: true, endpoint: "http://localhost:4318" });
+ });
+
it("shows what .env sets as locked", async () => {
m(Setup.getSetup).mockImplementation(() =>
ok(
diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts
index 33fb4de2d..22b0c44e3 100644
--- a/nextjs-frontend/app/openapi-client/sdk.gen.ts
+++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts
@@ -81,6 +81,10 @@ import type {
UpdateSettingsErrors,
TestLlmData,
TestLlmResponses,
+ TelemetryStatusData,
+ TelemetryStatusResponses,
+ TestTelemetryData,
+ TestTelemetryResponses,
ListAuditData,
ListAuditResponses,
ListAuditErrors,
@@ -104,6 +108,9 @@ import type {
SaveStorageData,
SaveStorageResponses,
SaveStorageErrors,
+ SaveTelemetryData,
+ SaveTelemetryResponses,
+ SaveTelemetryErrors,
FinishData,
FinishResponses,
FinishErrors,
@@ -1273,6 +1280,32 @@ export class Admin {
});
}
+ /**
+ * Telemetry Status
+ * Whether telemetry is on, where it goes, and how the API process's last exports went. Off by default.
+ */
+ public static telemetryStatus(
+ options?: Options,
+ ) {
+ return (options?.client ?? client).get({
+ url: "/api/v1/settings/telemetry/status",
+ ...options,
+ });
+ }
+
+ /**
+ * Test Telemetry
+ * Send one test span to the saved endpoint now (on or off), to check the address and headers.
+ */
+ public static testTelemetry(
+ options?: Options,
+ ) {
+ return (options?.client ?? client).post({
+ url: "/api/v1/settings/telemetry/test",
+ ...options,
+ });
+ }
+
/**
* List Audit
* Who changed what, newest first.
@@ -1392,6 +1425,21 @@ export class Setup {
});
}
+ /**
+ * Save Telemetry
+ * Opt in to telemetry: traces and metrics sent only to this endpoint (an OTLP collector). Off unless chosen.
+ */
+ public static saveTelemetry(options: Options) {
+ return (options.client ?? client).put({
+ url: "/api/v1/setup/telemetry",
+ ...options,
+ headers: {
+ "Content-Type": "application/json",
+ ...options.headers,
+ },
+ });
+ }
+
/**
* Finish
* Finish (or skip) the wizard; the web app stops showing it. Everything stays changeable in Settings.
diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts
index da746f357..cbe72d70b 100644
--- a/nextjs-frontend/app/openapi-client/types.gen.ts
+++ b/nextjs-frontend/app/openapi-client/types.gen.ts
@@ -9147,6 +9147,39 @@ export type SetupStorageView = {
[key: string]: unknown | string | boolean | Array | number;
};
+/**
+ * SetupTelemetry
+ */
+export type SetupTelemetry = {
+ /**
+ * Enabled
+ */
+ enabled?: boolean;
+ /**
+ * Endpoint
+ */
+ endpoint?: string | null;
+};
+
+/**
+ * SetupTelemetryView
+ */
+export type SetupTelemetryView = {
+ /**
+ * Enabled
+ */
+ enabled: boolean;
+ /**
+ * Endpoint
+ */
+ endpoint?: string | null;
+ /**
+ * Locked
+ */
+ locked: Array;
+ [key: string]: unknown | boolean | string | null | Array | undefined;
+};
+
/**
* SetupView
*/
@@ -9159,7 +9192,15 @@ export type SetupView = {
namespace: SetupNamespaceView;
llm: SetupLlmView;
storage: SetupStorageView;
- [key: string]: unknown | boolean | SetupAdmin | SetupNamespaceView | SetupLlmView | SetupStorageView;
+ telemetry: SetupTelemetryView;
+ [key: string]:
+ | unknown
+ | boolean
+ | SetupAdmin
+ | SetupNamespaceView
+ | SetupLlmView
+ | SetupStorageView
+ | SetupTelemetryView;
};
/**
@@ -10107,6 +10148,70 @@ export type TagsChanged = {
[key: string]: unknown | boolean | number | undefined;
};
+/**
+ * TelemetryExport
+ */
+export type TelemetryExport = {
+ /**
+ * At
+ */
+ at: number;
+ /**
+ * Ok
+ */
+ ok: boolean;
+ /**
+ * Error
+ */
+ error?: string | null;
+ [key: string]: unknown | number | boolean | string | null | undefined;
+};
+
+/**
+ * TelemetryStatus
+ * What the API process does with telemetry now; each worker follows the same settings on its own.
+ */
+export type TelemetryStatus = {
+ /**
+ * Enabled
+ */
+ enabled: boolean;
+ /**
+ * Endpoint
+ */
+ endpoint?: string | null;
+ /**
+ * Traces
+ */
+ traces: boolean;
+ /**
+ * Metrics
+ */
+ metrics: boolean;
+ last_traces?: TelemetryExport | null;
+ last_metrics?: TelemetryExport | null;
+ [key: string]: unknown | boolean | string | null | TelemetryExport | null | TelemetryExport | null | undefined;
+};
+
+/**
+ * TelemetryTestResult
+ */
+export type TelemetryTestResult = {
+ /**
+ * Ok
+ */
+ ok: boolean;
+ /**
+ * Error
+ */
+ error?: string | null;
+ /**
+ * Ms
+ */
+ ms?: number | null;
+ [key: string]: unknown | boolean | string | null | number | null | undefined;
+};
+
/**
* Template
*/
@@ -12382,6 +12487,38 @@ export type TestLlmResponses = {
export type TestLlmResponse = TestLlmResponses[keyof TestLlmResponses];
+export type TelemetryStatusData = {
+ body?: never;
+ path?: never;
+ query?: never;
+ url: "/api/v1/settings/telemetry/status";
+};
+
+export type TelemetryStatusResponses = {
+ /**
+ * Successful Response
+ */
+ 200: TelemetryStatus;
+};
+
+export type TelemetryStatusResponse = TelemetryStatusResponses[keyof TelemetryStatusResponses];
+
+export type TestTelemetryData = {
+ body?: never;
+ path?: never;
+ query?: never;
+ url: "/api/v1/settings/telemetry/test";
+};
+
+export type TestTelemetryResponses = {
+ /**
+ * Successful Response
+ */
+ 200: TelemetryTestResult;
+};
+
+export type TestTelemetryResponse = TestTelemetryResponses[keyof TestTelemetryResponses];
+
export type ListAuditData = {
body?: never;
path?: never;
@@ -12583,6 +12720,31 @@ export type SaveStorageResponses = {
export type SaveStorageResponse = SaveStorageResponses[keyof SaveStorageResponses];
+export type SaveTelemetryData = {
+ body: SetupTelemetry;
+ path?: never;
+ query?: never;
+ url: "/api/v1/setup/telemetry";
+};
+
+export type SaveTelemetryErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type SaveTelemetryError = SaveTelemetryErrors[keyof SaveTelemetryErrors];
+
+export type SaveTelemetryResponses = {
+ /**
+ * Successful Response
+ */
+ 200: SetupSaved;
+};
+
+export type SaveTelemetryResponse = SaveTelemetryResponses[keyof SaveTelemetryResponses];
+
export type FinishData = {
body: SetupFinish;
path?: never;
diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx
index d730ba670..585dec9c6 100644
--- a/nextjs-frontend/components/settings/bodies.tsx
+++ b/nextjs-frontend/components/settings/bodies.tsx
@@ -1,6 +1,6 @@
"use client";
-import { useMutation, useQueries } from "@tanstack/react-query";
+import { useMutation, useQueries, useQuery } from "@tanstack/react-query";
import { PlugZap, RefreshCw, Terminal } from "lucide-react";
import Link from "next/link";
import { useEffect, useRef, type ReactNode } from "react";
@@ -320,6 +320,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) {
>
);
+ case "telemetry":
+ return ;
case "uploads":
return (
<>
@@ -414,6 +416,103 @@ function LlmBody({ ctx }: { ctx: BodyCtx }) {
);
}
+function exportLine(e: { at: number; ok: boolean; error?: string | null } | null | undefined, what: string) {
+ if (!e) return null;
+ const when = new Date(e.at * 1000).toLocaleTimeString();
+ return e.ok ? `${what} last sent at ${when}` : `${what} failed at ${when}: ${e.error ?? "refused"}`;
+}
+
+function TelemetryBody({ ctx }: { ctx: BodyCtx }) {
+ const client = useApiClient();
+ const t = ctx.view.telemetry;
+ const saved = t?.values ?? {};
+ const secret = (saved.headers ?? {}) as { set?: boolean };
+ const headers = ctx.state("telemetry.headers");
+ const locked = t?.locked ?? [];
+ const status = useQuery({
+ queryKey: ["telemetry-status", t?.updated_at ?? null],
+ queryFn: () => data(Admin.telemetryStatus({ client })),
+ refetchInterval: 30_000,
+ });
+ const test = useMutation({ mutationFn: () => data(Admin.testTelemetry({ client })) });
+ const on = Boolean(saved.enabled);
+ const where = (saved.endpoint as string | null | undefined) ?? null;
+ const s = status.data;
+ const lines = [exportLine(s?.last_traces, "Traces"), exportLine(s?.last_metrics, "Metrics")].filter(Boolean);
+ const failed = Boolean((s?.last_traces && !s.last_traces.ok) || (s?.last_metrics && !s.last_metrics.ok));
+ return (
+ <>
+ {on && where ? (
+
+ {lines.length
+ ? lines.join(" · ")
+ : "Nothing sent from the server yet; the first export follows its first traced work."}
+
+ ) : on ? (
+
+ Set the endpoint below; until then nothing is collected or sent.
+
+ ) : (
+
+ Lens never sends telemetry anywhere unless you turn it on here, in the setup wizard or with{" "}
+ LENS_TELEMETRY=on in .env, and then only to the endpoint you set.
+
+ )}
+
+
+ headers.onChange(x)}
+ />
+
+ Optional: key=value pairs, separated by commas and URL-encoded, for a collector that needs a token.
+
+
+
+
+
+
+
+
+
+
+
+
+ What is sent: route templates (never paths or queries), job steps and how they ended, record ids, model names,
+ token counts, durations and estimated cost. Never transcript, prompt or answer text, file names, titles,
+ namespace names, people or addresses. Each worker follows these settings on its own.
+
+
+ } onClick={() => test.mutate()} disabled={test.isPending || !where}>
+ {test.isPending ? "Sending…" : "Send a test span"}
+
+
+ {ctx.dirty
+ ? "Uses the saved settings, not your unsaved changes"
+ : "Sends one span named lens.telemetry.test, even while telemetry is off"}
+
+
+ {test.data &&
+ (test.data.ok ? (
+
+ {test.data.ms} ms
+
+ ) : (
+
+ {test.data.error}
+
+ ))}
+ {test.isError && {test.error.message}}
+ >
+ );
+}
+
function AiBody({ ctx }: { ctx: BodyCtx }) {
const tools = ctx.state("ai.disabled_tools");
const off = (tools.value as string[]) ?? [];
diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts
index 6b61aaa2a..10456c828 100644
--- a/nextjs-frontend/components/settings/model.ts
+++ b/nextjs-frontend/components/settings/model.ts
@@ -73,6 +73,7 @@ export type SectionId =
| "workers"
| "access"
| "notifications"
+ | "telemetry"
| "uploads"
| "documents"
| "tokens"
@@ -162,6 +163,13 @@ export const SECTIONS: SectionSpec[] = [
description:
"Where namespaces may send notifications (webhooks, Matterbridge, Slack, Discord), how often the notifier looks, and how often a failed send is tried again.",
},
+ {
+ id: "telemetry",
+ label: "Telemetry",
+ backend: ["telemetry"],
+ description:
+ "Off unless you turn it on. Traces and metrics from the server and its workers, sent only to an OpenTelemetry collector you run or choose; nothing goes anywhere else.",
+ },
{
id: "uploads",
label: "Uploads",
@@ -922,6 +930,73 @@ export const FIELDS: FieldSpec[] = [
max: 20,
hint: "A failed send waits 30 seconds, then four times longer each time, up to 6 hours",
},
+ // Telemetry (opt-in)
+ {
+ section: "telemetry",
+ key: "enabled",
+ label: "Send telemetry",
+ kind: "switch",
+ hint: "Off by default; while off, nothing is collected or sent",
+ },
+ {
+ section: "telemetry",
+ key: "endpoint",
+ label: "OTLP endpoint",
+ kind: "text",
+ nullable: true,
+ mono: true,
+ placeholder: "http://localhost:4318",
+ hint: "An OpenTelemetry collector’s OTLP/HTTP address; traces go to /v1/traces and metrics to /v1/metrics under it",
+ },
+ { section: "telemetry", key: "headers", label: "Headers", kind: "secret" },
+ {
+ section: "telemetry",
+ key: "traces",
+ label: "Traces",
+ kind: "switch",
+ hint: "API requests, jobs and their steps, routines, workflows and model calls",
+ },
+ {
+ section: "telemetry",
+ key: "metrics",
+ label: "Metrics",
+ kind: "switch",
+ hint: "Durations, job outcomes, model tokens and estimated cost",
+ },
+ {
+ section: "telemetry",
+ key: "sample_ratio",
+ label: "Share of traces kept",
+ kind: "number",
+ min: 0,
+ max: 1,
+ hint: "1 keeps every trace, 0.1 one in ten",
+ },
+ {
+ section: "telemetry",
+ key: "export_seconds",
+ label: "Send metrics every (seconds)",
+ kind: "int",
+ min: 5,
+ max: 3600,
+ },
+ {
+ section: "telemetry",
+ key: "service_name",
+ label: "Service name",
+ kind: "text",
+ mono: true,
+ hint: "How this server shows in your tracing tool",
+ },
+ {
+ section: "telemetry",
+ key: "prices",
+ label: "Model prices for cost estimates",
+ kind: "lines",
+ mono: true,
+ placeholder: "gpt-4o-mini 0.15 0.60",
+ hint: "One model per line: its name, then the input and output price in dollars per million tokens. The AI assistant’s prices count for the configured model",
+ },
{
section: "tokens",
key: "oauth_access_minutes",
@@ -1130,6 +1205,7 @@ export function toUi(f: FieldSpec, v: unknown): unknown {
case "checks":
return Array.isArray(v) ? v.map(String) : [];
case "lines":
+ if (f.section === "telemetry" && f.key === "prices") return pricesToLines(v);
return Array.isArray(v) ? v.join("\n") : "";
case "days":
return typeof v === "number" ? String(Math.round((v / 24) * 100) / 100) : "";
@@ -1142,6 +1218,33 @@ export function toUi(f: FieldSpec, v: unknown): unknown {
export type Parsed = { value: unknown } | { error: string };
+export type Prices = Record;
+
+/** telemetry.prices as lines: "model input output". */
+export function pricesToLines(v: unknown): string {
+ if (!v || typeof v !== "object") return "";
+ return Object.entries(v as Prices)
+ .map(([model, p]) => `${model} ${p?.input ?? 0} ${p?.output ?? 0}`)
+ .join("\n");
+}
+
+/** Lines of "model input output" (dollars per million tokens) back to telemetry.prices. */
+export function linesToPrices(text: string): Parsed {
+ const out: Prices = {};
+ const lines = text
+ .split("\n")
+ .map((l) => l.trim())
+ .filter(Boolean);
+ for (let i = 0; i < lines.length; i++) {
+ const parts = lines[i].split(/\s+/);
+ const nums = parts.slice(-2).map(Number);
+ if (parts.length < 3 || nums.some((n) => !Number.isFinite(n) || n < 0))
+ return { error: `Line ${i + 1}: write the model, then its input and output price, like gpt-4o-mini 0.15 0.60` };
+ out[parts.slice(0, -2).join(" ")] = { input: nums[0], output: nums[1] };
+ }
+ return { value: out };
+}
+
/** Parses a control's value back to what the backend stores, checking it like the backend does. */
export function parse(f: FieldSpec, ui: unknown): Parsed {
switch (f.kind) {
@@ -1152,6 +1255,7 @@ export function parse(f: FieldSpec, ui: unknown): Parsed {
case "checks":
return { value: ui as string[] };
case "lines":
+ if (f.section === "telemetry" && f.key === "prices") return linesToPrices(String(ui ?? ""));
return {
value: String(ui ?? "")
.split("\n")
@@ -1259,6 +1363,12 @@ export function crossErrors(values: Record): Record !/^[0-9a-f.:]+(\/\d{1,3})?$/i.test(x));
if (badNet) e["notifications.networks"] = `“${badNet}” isn’t a network like 192.168.1.0/24`;
+ const otlp = values["telemetry.endpoint"] as string | null | undefined;
+ if (otlp && !URL_RX.test(otlp)) e["telemetry.endpoint"] = "Use an http(s) address, such as http://localhost:4318";
+ else if (otlp && /\/v1\/(traces|metrics)\/?$/.test(otlp))
+ e["telemetry.endpoint"] = "Use the collector’s base address, without /v1/traces or /v1/metrics";
+ else if (values["telemetry.enabled"] === true && "telemetry.endpoint" in values && !otlp)
+ e["telemetry.endpoint"] = "Set the collector’s address to send telemetry to";
const rights = values["iiif.rights"] as string | null | undefined;
if (rights && !RIGHTS_RX.test(rights))
e["iiif.rights"] = "Pick a Creative Commons licence or a RightsStatements.org statement";
@@ -1295,6 +1405,10 @@ export function show(f: FieldSpec, v: unknown): string {
if (f.kind === "switch") return f.key === "cross_namespace" ? (v === "off" ? "off" : "on") : v ? "on" : "off";
if (f.kind === "days") return `${Math.round(((v as number) / 24) * 100) / 100} days`;
if (f.key === "rights") return rightsShort(v as string);
+ if (f.section === "telemetry" && f.key === "prices") {
+ const n = Object.keys((v as Prices) ?? {}).length;
+ return n ? `${n} model${n === 1 ? "" : "s"}` : "none";
+ }
if (Array.isArray(v)) {
if (f.kind === "checks" && f.section === "ai") return `${v.length} off`;
return v.length > 3 ? `${v.length} ${f.kind === "lines" ? "lines" : "items"}` : v.join(", ") || "none";
@@ -1328,6 +1442,12 @@ export function why(c: Change): string | null {
if (id === "transcribe.engine" || id === "transcribe.language")
return "Applies to new transcriptions; existing transcripts stay until reprocessed.";
if (id === "workers.inline") return "Takes effect when the server restarts.";
+ if (id === "telemetry.enabled")
+ return c.after
+ ? "The server and its workers start sending traces and metrics to the endpoint within a few seconds."
+ : "Nothing more is sent; what was sent stays with your collector.";
+ if (id === "telemetry.headers")
+ return c.after === "" ? "The headers are removed." : "Sent with every export; stored encrypted.";
if (id === "llm.api_key")
return c.after === ""
? "The key is removed; LLM steps and chat stop until a new one is set."
diff --git a/nextjs-frontend/components/settings/section.tsx b/nextjs-frontend/components/settings/section.tsx
index 67e448b99..60f314274 100644
--- a/nextjs-frontend/components/settings/section.tsx
+++ b/nextjs-frontend/components/settings/section.tsx
@@ -42,6 +42,9 @@ const ENV: Record = {
"llm.model": "LENS_LLM_MODEL",
"llm.api_key": "LENS_LLM_API_KEY",
"llm.vision_model": "LENS_LLM_VISION_MODEL",
+ "telemetry.enabled": "LENS_TELEMETRY",
+ "telemetry.endpoint": "LENS_TELEMETRY_ENDPOINT",
+ "telemetry.headers": "LENS_TELEMETRY_HEADERS",
};
const CONFIRM_BASE = "CHANGE ALL IDENTIFIERS";
diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx
index 11671fbe6..e2fbc7ecb 100644
--- a/nextjs-frontend/components/settings/settings-shell.tsx
+++ b/nextjs-frontend/components/settings/settings-shell.tsx
@@ -2,6 +2,7 @@
import { useQuery } from "@tanstack/react-query";
import {
+ Activity,
AudioLines,
Bell,
Bot,
@@ -48,6 +49,7 @@ const ICON: Record = {
workers: Cpu,
access: ShieldCheck,
notifications: Bell,
+ telemetry: Activity,
uploads: Upload,
documents: FileType,
tokens: KeyRound,
diff --git a/nextjs-frontend/components/setup/setup-wizard.tsx b/nextjs-frontend/components/setup/setup-wizard.tsx
index 4538bfbdd..ea64c487e 100644
--- a/nextjs-frontend/components/setup/setup-wizard.tsx
+++ b/nextjs-frontend/components/setup/setup-wizard.tsx
@@ -5,7 +5,7 @@ import { Check, Lock } from "lucide-react";
import { useRouter } from "next/navigation";
import { useEffect, useState, type ReactNode } from "react";
-import { Admin, Setup, type LlmTestResult, type SetupView } from "@/app/openapi-client";
+import { Admin, Setup, type LlmTestResult, type SetupView, type TelemetryTestResult } from "@/app/openapi-client";
import { AuthAlert, AuthBrand } from "@/components/auth/auth-card";
import { SegmentedChoice } from "@/components/settings/controls";
import { Button } from "@/components/ui/button";
@@ -20,6 +20,7 @@ const STEPS = [
{ id: "namespace", label: "Namespace" },
{ id: "llm", label: "Model provider" },
{ id: "storage", label: "Storage" },
+ { id: "telemetry", label: "Telemetry" },
] as const;
type StepId = (typeof STEPS)[number]["id"];
@@ -29,15 +30,17 @@ const LLM_ENV: Record = {
model: "LENS_LLM_MODEL",
api_key: "LENS_LLM_API_KEY",
};
+const TELEMETRY_ENV: Record = { enabled: "LENS_TELEMETRY", endpoint: "LENS_TELEMETRY_ENDPOINT" };
const NS = /^[a-z0-9][a-z0-9_-]{0,40}$/;
+const URL_RX = /^https?:\/\/[^\s]+$/;
function errorText(e: unknown): string {
return e instanceof ApiError ? e.message : "Something went wrong. Please try again.";
}
/**
- * First-run setup for a fresh install (the admin already exists): the first namespace, the model provider and
- * storage, one step at a time. Each step can be skipped, and the whole wizard too; everything stays in Settings.
+ * First-run setup for a fresh install (the admin already exists): the first namespace, the model provider, storage
+ * and whether to send telemetry (off unless chosen), one step at a time. Each step can be skipped, and the whole wizard too; everything stays in Settings.
* Fields .env sets are shown locked, since the environment wins.
*/
export function SetupWizard() {
@@ -96,7 +99,7 @@ export function SetupWizard() {
-
+
{STEPS.map((s, i) => {
const on = s.id === step;
const ok = done.has(s.id);
@@ -138,8 +141,10 @@ export function SetupWizard() {
next("namespace", saved)} />
) : step === "llm" ? (
next("llm", saved)} />
+ ) : step === "storage" ? (
+ next("storage", saved)} />
) : (
- next("storage", saved)} />
+ next("telemetry", saved)} />
)}
{finish.error && {errorText(finish.error)}}
@@ -397,15 +402,7 @@ function LlmStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean) =
);
}
-function StorageStep({
- view,
- pending,
- onNext,
-}: {
- view: SetupView;
- pending: boolean;
- onNext: (saved: boolean) => void;
-}) {
+function StorageStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean) => void }) {
const client = useApiClient();
const s = view.storage;
const [maxMb, setMaxMb] = useState(String(s.max_upload_mb));
@@ -516,10 +513,141 @@ function StorageStep({
onNext(false)}
onSave={() => save.mutate()}
- saveText="Save and finish"
- pending={save.isPending || pending}
+ pending={save.isPending}
disabled={!validMb || (Boolean(folder.trim()) && !namespace)}
/>
>
);
}
+
+function TelemetryStep({
+ view,
+ pending,
+ onNext,
+}: {
+ view: SetupView;
+ pending: boolean;
+ onNext: (saved: boolean) => void;
+}) {
+ const client = useApiClient();
+ const t = view.telemetry;
+ const locked = new Set(t.locked);
+ const [on, setOn] = useState(t.enabled ? "on" : "off");
+ const [endpoint, setEndpoint] = useState(t.endpoint ?? "");
+ const [tested, setTested] = useState(null);
+ useEffect(() => setTested(null), [endpoint]);
+ const body = () => ({ enabled: on === "on", endpoint: endpoint.trim() || null });
+ const save = useMutation({ mutationFn: () => data(Setup.saveTelemetry({ client, body: body() })) });
+ const test = useMutation({
+ mutationFn: async () => {
+ // the test uses the saved address; saving it with telemetry off sends nothing else
+ await data(Setup.saveTelemetry({ client, body: { enabled: t.enabled, endpoint: endpoint.trim() } }));
+ return data(Admin.testTelemetry({ client }));
+ },
+ onSuccess: setTested,
+ });
+ const wantsEndpoint = on === "on" && !locked.has("endpoint");
+ const badEndpoint = Boolean(endpoint.trim()) && !URL_RX.test(endpoint.trim());
+ const missing = wantsEndpoint && !endpoint.trim();
+
+ return (
+ <>
+
+ Lens can send traces and metrics about its own work (requests, jobs, model calls with token counts and estimated
+ cost) to an OpenTelemetry collector you run, to see how it performs. It is off unless you turn it on, and
+ nothing is ever sent anywhere else.
+
+
+ ) : (
+ "You can change this any time in Settings → Telemetry."
+ )
+ }
+ >
+ {() => (
+ undefined : setOn}
+ options={[
+ { value: "off", label: "Off" },
+ { value: "on", label: "On" },
+ ]}
+ className="w-full max-w-[220px]"
+ />
+ )}
+
+ {on === "on" && (
+ <>
+
+ ) : (
+ "Your collector's OTLP/HTTP address, for example http://localhost:4318"
+ )
+ }
+ error={badEndpoint ? "Use an http(s) address" : undefined}
+ >
+ {({ id, describedBy, invalid }) => (
+ setEndpoint(e.target.value)}
+ disabled={locked.has("endpoint")}
+ autoComplete="off"
+ spellCheck={false}
+ />
+ )}
+
+
+ Sent: route templates, job steps, record ids, model names, token counts, durations and costs. Never
+ transcripts, prompts, file names, titles or people.
+
+ >
+ )}
+ {tested &&
+ (tested.ok ? (
+ The collector took a test span in {tested.ms} ms.
+ ) : (
+ The collector didn't take the test span: {tested.error}
+ ))}
+ {(save.error || test.error) && {errorText(save.error || test.error)}}
+