diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd5b3a64..bfbd3229b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Opt-in telemetry.** Lens can send OpenTelemetry traces and metrics about its own work to a collector you choose: + API requests by route, jobs and each step, routines, workflow runs, and model calls with their tokens and an + estimated cost (from per-model prices you set). It is off by default and never on unless you turn it on, in + Settings → Telemetry, the setup wizard's new last step, or `LENS_TELEMETRY=on` with `LENS_TELEMETRY_ENDPOINT` in + .env (`LENS_TELEMETRY=off` keeps it off). It goes only to that endpoint, over OTLP/HTTP; there is no built-in + destination. Spans and metrics carry ids, step types, model names, counts and timings, never transcript, prompt or + answer text, file names, titles, people, paths or addresses. Settings shows whether it is on and how the last + exports went, and can send a test span; the API and each worker follow a change without a restart + (docs/telemetry.md). - **Security: frontend dependencies patched.** The web app's lockfile now pulls fixed versions of form-data, ws, brace-expansion, minimatch, picomatch, glob, js-yaml, flatted, browserslist, Babel and the other packages GitHub flagged, each kept inside the major version its parent asks for (`overrides` in `nextjs-frontend/pnpm-workspace.yaml`). diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 71a4e2581..8733f6a3e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -38,6 +38,10 @@ x-backend: &backend LENS_LLM_MODEL: ${LENS_LLM_MODEL:-} LENS_LLM_API_KEY: ${LENS_LLM_API_KEY:-} LENS_LLM_VISION_MODEL: ${LENS_LLM_VISION_MODEL:-} + # opt-in telemetry (docs/telemetry.md): unset, it stays off unless turned on in Settings + LENS_TELEMETRY: ${LENS_TELEMETRY:-} + LENS_TELEMETRY_ENDPOINT: ${LENS_TELEMETRY_ENDPOINT:-} + LENS_TELEMETRY_HEADERS: ${LENS_TELEMETRY_HEADERS:-} volumes: - archive-data:/data - ${AUDIO_DIR:-./local-audio}:/audio:ro diff --git a/docs/configuration.md b/docs/configuration.md index c8cab5a62..1cb93b30d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,6 +31,7 @@ with AES-GCM and are write-only: the API reports whether one is set, never its v | `LENS_NAMESPACE` | | create the first namespace at startup, while there is none | | `LENS_LLM_BASE_URL` / `LENS_LLM_MODEL` / `LENS_LLM_API_KEY` / `LENS_LLM_VISION_MODEL` | | the model provider; wins over Settings, which show these locked | | `LENS_SETUP_WIZARD` | | `off`: never show the setup wizard | +| `LENS_TELEMETRY` / `LENS_TELEMETRY_ENDPOINT` / `LENS_TELEMETRY_HEADERS` | off | opt-in OpenTelemetry traces and metrics, sent only to this OTLP/HTTP endpoint ([Telemetry](telemetry.md)); `LENS_TELEMETRY=off` keeps it off whatever Settings say | | `FRONTEND_URL` | `http://localhost:3000` | the web app's address as people use it: links in emails, and where apps send people to sign in ([OAuth](authentication.md#oauth)) | | `CORS_ORIGINS` | `["http://localhost:3000"]` | origins allowed to call the API from a browser | | `OPENAPI_URL` | `/openapi.json` | `""` disables `/docs` and the schema | diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 000000000..0c781c1f3 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,113 @@ +# Telemetry + +Lens can send traces and metrics about its own work to an [OpenTelemetry](https://opentelemetry.io/) collector, to see +how it performs: which API requests are slow, how long each job step takes, how often jobs fail, and how many tokens +model calls use and what they cost. + +**It is off unless you turn it on.** Nothing is collected while it is off, and Lens has no built-in destination: it +never sends telemetry to the Lens project or anyone else. Turned on, it sends only to the endpoint you set, typically +a collector on the same machine or network, which can pass it on to Jaeger, Grafana Tempo and Prometheus, SigNoz, +Honeycomb, or whatever you use. + +## Turning it on + +Any of these: + +- **Settings → Telemetry**: turn on **Send telemetry** and set the **OTLP endpoint**. **Send a test span** checks the + address and headers first, even while it is off. The section shows whether it is on, where it sends, and how the + last exports went. +- **The setup wizard** asks on a fresh install (its last step); it is off unless you choose on. +- **.env**, for the API and every worker: + + ```sh + LENS_TELEMETRY=on # off keeps it off, whatever Settings say + LENS_TELEMETRY_ENDPOINT=http://localhost:4318 # the collector's OTLP/HTTP address + LENS_TELEMETRY_HEADERS=Authorization=Bearer%20xyz # optional: key=value pairs, comma-separated, URL-encoded + ``` + + What .env sets wins over Settings, which show those fields locked. + +Each process (the API, each `lens worker`, `lens watch`) follows the settings on its own and picks up a change within a +few seconds, with no restart. The endpoint is the collector's base address: traces go to `/v1/traces` and metrics to +`/v1/metrics` under it, over OTLP/HTTP with protobuf. + +In Docker, `localhost` is the container itself: point the endpoint at a collector on the `lens` network (for example +`http://otel-collector:4318`, below) or at `http://host.docker.internal:4318` for one on the host. + +## What is sent + +| Signal | What | Attributes | +| --- | --- | --- | +| Trace | each API request, named by its route template (`GET /api/v1/recordings/{rid}`) | method, route, status code | +| Trace | each job, with a span per step (`step transcribe`) | job, recording and pipeline ids; step type and outcome | +| Trace | routines and their actions, workflow runs | routine, workflow and recording ids; action type; trigger | +| Trace | each call to the model server (`chat llama3.1`) | model, server host, temperature, tokens in and out, estimated cost | +| Metric | `http.server.request.duration` | method, route, status code | +| Metric | `lens.job.step.duration`, `lens.jobs` | step type, outcome | +| Metric | `lens.routine.runs`, `lens.workflow.runs` | outcome | +| Metric | `gen_ai.client.operation.duration`, `gen_ai.client.token.usage` | model, token type | +| Metric | `lens.llm.cost` (USD) | model | + +Every span and metric carries `service.name` (Settings → Telemetry, default `lens`), `service.version`, a random +`service.instance.id` per process, and `lens.process.role` (`api`, `worker` or `watcher`). Model calls follow the +OpenTelemetry [GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). + +**Never sent:** transcript text, prompts or answers, file names, titles, namespace or speaker names, people, email +addresses, client IP addresses, request paths or query strings (only route templates), host names, or error messages +(only the error's type, since a message can quote content). + +### Tokens and cost + +Token counts come from the model server's reply (`usage`). Most servers report them; a streamed chat answer has them +only when the server sends usage in its last chunk. + +Cost is an estimate from prices you set: **Model prices** in Settings → Telemetry, one line per model with its input +and output price in dollars per million tokens (`gpt-4o-mini 0.15 0.60`). The prices in Settings → AI assistant count +for the configured model too. A model without a price gets no cost, so local models cost nothing. + +## Other settings + +| Setting | Default | | +| --- | --- | --- | +| Traces / Metrics | on / on | send either or both | +| Share of traces kept | `1` | `0.1` keeps one trace in ten (head sampling) | +| Send metrics every | `60` seconds | | +| Service name | `lens` | how this server shows in your tracing tool | + +In archive.yaml they live under `telemetry:` (`enabled`, `endpoint`, `traces`, `metrics`, `sample_ratio`, +`export_seconds`, `service_name`, `prices`). + +## A collector next to Lens + +A minimal [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) that takes Lens's traces and metrics and +prints them (swap the `debug` exporter for your backend's): + +```yaml +# otel-collector.yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: basic +service: + pipelines: + traces: { receivers: [otlp], exporters: [debug] } + metrics: { receivers: [otlp], exporters: [debug] } +``` + +Beside `docker-compose.prod.yml`, in a `docker-compose.override.yml` or with `-f`: + +```yaml +services: + otel-collector: + image: otel/opentelemetry-collector:latest + command: ["--config=/etc/otel-collector.yaml"] + volumes: + - ./otel-collector.yaml:/etc/otel-collector.yaml:ro + networks: [lens] +``` + +Then set the endpoint to `http://otel-collector:4318` and turn telemetry on. diff --git a/fastapi_backend/.env.example b/fastapi_backend/.env.example index bfcb43013..3ffac6851 100644 --- a/fastapi_backend/.env.example +++ b/fastapi_backend/.env.example @@ -52,5 +52,10 @@ CORS_ORIGINS=["http://localhost:3000"] # LENS_LLM_API_KEY= # LENS_SETUP_WIZARD=off # never show the wizard +# --- Telemetry (docs/telemetry.md): off unless turned on here or in Settings, and sent only to this endpoint --- +# LENS_TELEMETRY=on # on, or off to keep it off whatever Settings say +# LENS_TELEMETRY_ENDPOINT=http://localhost:4318 # your OpenTelemetry collector's OTLP/HTTP address +# LENS_TELEMETRY_HEADERS= # optional: key=value pairs, comma-separated, URL-encoded + # Uncomment to disable /docs and /openapi.json # OPENAPI_URL="" diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index 457b48487..3ae6f8aa6 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -12,8 +12,8 @@ from app.api.deps import AdminReader, AdminWriter, Cfg, Db, domain_errors from app.core.middleware import host_name -from app.domain import auth, jobs, llm, settings, sources, store -from app.schemas.admin import AuditEntry, Health, LlmTestResult, Started +from app.domain import auth, jobs, llm, settings, sources, store, telemetry +from app.schemas.admin import AuditEntry, Health, LlmTestResult, Started, TelemetryStatus, TelemetryTestResult from app.schemas.auth import AccountToken from app.schemas.common import Ok @@ -54,6 +54,19 @@ def test_llm(user: AdminWriter, cfg: Cfg) -> LlmTestResult: return LlmTestResult(ok=True, reply=reply.strip()[:40], ms=int((time.time() - t0) * 1000), model=cfg["llm"]["model"]) +@router.get("/settings/telemetry/status") +def telemetry_status(user: AdminReader, cfg: Cfg) -> TelemetryStatus: + """Whether telemetry is on, where it goes, and how the API process's last exports went. Off by default.""" + return TelemetryStatus.model_validate(telemetry.status(cfg)) + + +@router.post("/settings/telemetry/test") +def test_telemetry(user: AdminWriter, cfg: Cfg) -> TelemetryTestResult: + """Send one test span to the saved endpoint now (on or off), to check the address and headers.""" + ok, error, ms = telemetry.test_export(cfg) + return TelemetryTestResult(ok=ok, error=error, ms=ms) + + @router.get("/audit") def list_audit(user: AdminReader, db: Db, limit: int = Query(200, ge=1, le=1000)) -> list[AuditEntry]: """Who changed what, newest first.""" diff --git a/fastapi_backend/app/api/v1/routes/setup.py b/fastapi_backend/app/api/v1/routes/setup.py index 944bacbb3..12278d5be 100644 --- a/fastapi_backend/app/api/v1/routes/setup.py +++ b/fastapi_backend/app/api/v1/routes/setup.py @@ -1,4 +1,4 @@ -"""The first-run setup wizard (admins): the first namespace, the model provider and storage, then finishing it. +"""The first-run setup wizard (admins): the first namespace, the model provider, storage and telemetry, then finishing it. The first admin is created before this, with the setup code (POST /auth/setup) or from the environment. Fields the environment sets are left out of every save and reported as locked (domain/setup.py). @@ -10,7 +10,7 @@ from app.api.deps import AdminReader, AdminWriter, Db from app.domain import auth, setup -from app.schemas.setup import SetupFinish, SetupLlm, SetupNamespace, SetupSaved, SetupStorage, SetupView +from app.schemas.setup import SetupFinish, SetupLlm, SetupNamespace, SetupSaved, SetupStorage, SetupTelemetry, SetupView router = APIRouter(prefix="/setup", tags=["setup"]) @@ -66,6 +66,18 @@ def save_storage(body: SetupStorage, user: AdminWriter, request: Request, db: Db return SetupSaved(saved=[k for k in ("max_upload_mb", "folder") if getattr(body, k) is not None], watch=wid) +@router.put("/telemetry") +def save_telemetry(body: SetupTelemetry, user: AdminWriter, request: Request, db: Db) -> SetupSaved: + """Opt in to telemetry: traces and metrics sent only to this endpoint (an OTLP collector). Off unless chosen.""" + try: + saved = setup.save_telemetry(db, request.app.state.archive.base, body.enabled, body.endpoint, user.email) + except ValueError as e: + raise HTTPException(400, str(e)) from None + if saved: + auth.audit(db, user.as_audit(), "settings.save", "telemetry", saved) + return SetupSaved(saved=saved) + + @router.post("/finish") def finish(body: SetupFinish, user: AdminWriter, db: Db) -> SetupSaved: """Finish (or skip) the wizard; the web app stops showing it. Everything stays changeable in Settings.""" diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index dca51c10b..eb0231e01 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -188,7 +188,7 @@ def platform_main(argv, config): import threading import time - from .domain import auth, jobs, notify, routines, settings, sources + from .domain import auth, jobs, notify, routines, settings, sources, telemetry ap = argparse.ArgumentParser(prog="lens") sub = ap.add_subparsers(dest="cmd", required=True) @@ -250,6 +250,7 @@ def password(): auth.update_account(db, acct["id"], password=password()) print(f"password changed for {a.email}") elif a.cmd == "worker": + telemetry.set_role("worker") wk = jobs.Worker(db, C, a.name, a.steps.split(",") if a.steps else None, log=print) if a.once: print(f"ran {wk.drain()} job(s)") @@ -263,9 +264,11 @@ def password(): except KeyboardInterrupt: stop.set() elif a.once: + telemetry.set_role("watcher") print(f"scanned {sources.poll_due(db, C(), print)} folder(s)") print(f"ran {routines.run_due(db, C(), print)} routine(s)") else: + telemetry.set_role("watcher") print("watching storage sources and running routines; Ctrl-C to stop") _stop_on_term() try: @@ -278,6 +281,7 @@ def password(): except (ValueError, KeyError) as e: raise SystemExit(str(e)) from None finally: + telemetry.shutdown() # sends what is buffered db.close() diff --git a/fastapi_backend/app/domain/analyze.py b/fastapi_backend/app/domain/analyze.py index 27bea280f..f40c492dd 100644 --- a/fastapi_backend/app/domain/analyze.py +++ b/fastapi_backend/app/domain/analyze.py @@ -9,7 +9,7 @@ import urllib.request from collections import Counter, defaultdict -from . import store +from . import store, telemetry STOP = set( """a about above after again against all almost also am an and any are aren as at be because been before being @@ -628,8 +628,8 @@ def _llm(cfg, user, system=SUMMARY_SYSTEM): req = urllib.request.Request( l["base_url"].rstrip("/") + "/chat/completions", data=json.dumps(body).encode(), headers=headers, method="POST" ) - with urllib.request.urlopen(req, timeout=l.get("timeout", 300)) as r: - text = json.loads(r.read().decode())["choices"][0]["message"]["content"] + with telemetry.model_call(cfg, body) as call, urllib.request.urlopen(req, timeout=l.get("timeout", 300)) as r: + text = call.reply(json.loads(r.read().decode()))["choices"][0]["message"]["content"] text = re.sub(r"^```(?:json)?|```$", "", text.strip()).strip() obj = json.loads(text[text.find("{") : text.rfind("}") + 1]) missing = [k for k in SUMMARY_NEEDS if k not in obj] diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 427d34ae6..e57df379b 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -13,7 +13,7 @@ import threading import time -from . import analyze, ingest, pipelines, render, speakers as spk, store +from . import analyze, ingest, pipelines, render, speakers as spk, store, telemetry R = store.R PIPELINE = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "summarize", "report"] @@ -435,6 +435,7 @@ def started(k): return time.time() def finished(k, outcome, note, t0, outputs=None): + telemetry.record("lens.job.step.duration", time.time() - t0, {"lens.step": _spec(steps[k])["type"], "lens.step.outcome": outcome}) runs[k] = store.clean( { **runs[k], @@ -512,15 +513,17 @@ def beat(): i, t0 = i + 1, None continue before = _saved(db, rid) - try: - STEPS[step](db, cfg_fn(), rid, say, spec) - except Skip as e: - say(f"{step} skipped: {e}") - finished(i, "skipped", str(e), t0, _outputs(before, _saved(db, rid))) - else: - note = said[0] - say(f"{spec.get('name') or step} done in {time.time() - t0:.1f}s") - finished(i, "done", note, t0, _outputs(before, _saved(db, rid))) + with telemetry.span(f"step {step}", {"lens.step": step, "lens.step.index": i, "lens.job.id": str(jid)}) as sp: + try: + STEPS[step](db, cfg_fn(), rid, say, spec) + except Skip as e: + say(f"{step} skipped: {e}") + finished(i, "skipped", str(e), t0, _outputs(before, _saved(db, rid))) + else: + note = said[0] + say(f"{spec.get('name') or step} done in {time.time() - t0:.1f}s") + finished(i, "done", note, t0, _outputs(before, _saved(db, rid))) + sp.set_attribute("lens.step.outcome", runs[i]["outcome"]) _timed(db, rid, spec, runs[i]["seconds"], runs[i]["outcome"] == "skipped") i, t0 = i + 1, None except (Exception, SystemExit) as e: # noqa: BLE001 - recorded on the job; SystemExit too (a missing engine says so) @@ -738,7 +741,17 @@ def run_once(self): if not job: return False self.register(job["id"]) - run_job(self.db, self.cfg_fn, job, self.name, self.can, self.log) + pipe = job.get("pipeline") if isinstance(job.get("pipeline"), dict) else {} + attrs = { + "lens.job.id": str(job["id"]), + "lens.recording.id": str(job["recording"]), + "lens.pipeline.id": str(pipe["id"]) if pipe.get("id") is not None else ("standard" if pipe else None), + "lens.pipeline.version": pipe.get("version"), + } + with telemetry.span("job", attrs) as sp: + outcome = run_job(self.db, self.cfg_fn, job, self.name, self.can, self.log) + sp.set_attribute("lens.job.outcome", outcome) + telemetry.record("lens.jobs", 1, {"lens.job.outcome": outcome}) self.register() return True diff --git a/fastapi_backend/app/domain/llm.py b/fastapi_backend/app/domain/llm.py index d0c0609d9..03c23ba3d 100644 --- a/fastapi_backend/app/domain/llm.py +++ b/fastapi_backend/app/domain/llm.py @@ -11,6 +11,8 @@ import urllib.error import urllib.request +from . import telemetry + class LLMError(RuntimeError): pass @@ -91,8 +93,8 @@ def chat(cfg, messages, model=None, max_tokens=None, temperature=0.2): payload = {"model": model or cfg["llm"]["model"], "messages": messages, "temperature": temperature} if max_tokens: payload["max_tokens"] = max_tokens - with _post(cfg, payload) as r: - return _content(_load(r)) + with telemetry.model_call(cfg, payload) as call, _post(cfg, payload) as r: + return _content(call.reply(_load(r))) def _visible(pieces): @@ -125,10 +127,18 @@ def stream_chat(cfg, messages, model=None, temperature=0.2): def _stream(cfg, model, messages, temperature): - r = _post(cfg, {"model": model, "messages": messages, "temperature": temperature, "stream": True}) + payload = {"model": model, "messages": messages, "temperature": temperature, "stream": True} + # not the current span: the generator may resume in another thread (a streamed response) + call = telemetry.model_call(cfg, payload, current=False) + error = None + try: + r = _post(cfg, payload) + except LLMError as e: + call.end(e) + raise try: if "text/event-stream" not in (r.headers.get("Content-Type") or ""): - yield _content(_load(r)) # the server ignored stream=true + yield _content(call.reply(_load(r))) # the server ignored stream=true return finished = False try: @@ -141,7 +151,8 @@ def _stream(cfg, model, messages, temperature): finished = True break try: - choice = json.loads(data)["choices"][0] + chunk = call.reply(json.loads(data)) # the last chunk may carry the usage, with no choices + choice = chunk["choices"][0] delta = (choice.get("delta") or {}).get("content") except (ValueError, KeyError, IndexError, TypeError, AttributeError): continue @@ -152,8 +163,12 @@ def _stream(cfg, model, messages, temperature): raise LLMError(f"the LLM server stopped mid-answer: {e}") from None if not finished: raise LLMError("the LLM server stopped mid-answer") + except BaseException as e: + error = e + raise finally: r.close() + call.end(None if isinstance(error, GeneratorExit) else error) def parse_json(text): @@ -211,8 +226,8 @@ def json_out(cfg, system, user, schema, model=None): "response_format": {"type": "json_schema", "json_schema": {"name": "output", "schema": schema, "strict": False}}, } try: - with _post(cfg, payload) as r: - text = _content(_load(r)) + with telemetry.model_call(cfg, payload) as call, _post(cfg, payload) as r: + text = _content(call.reply(_load(r))) except LLMError as e: if not str(e)[:3] in ("400", "422", "404"): raise @@ -235,8 +250,8 @@ def chat_message(cfg, messages, tools=None, model=None, temperature=0.2): if tools: payload.update(tools=tools, tool_choice="auto") try: - with _post(cfg, payload) as r: - j = _load(r) + with telemetry.model_call(cfg, payload) as call, _post(cfg, payload) as r: + j = call.reply(_load(r)) except LLMError as e: if tools and str(e)[:3] in ("400", "404", "422", "501"): raise ToolsUnsupported(str(e)) from None diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index 6d36fd00d..f6893512b 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -17,7 +17,7 @@ import datetime as dt -from . import jobs, organize, schedule, sources, store, workflows +from . import jobs, organize, schedule, sources, store, telemetry, workflows R = store.R ACTIONS = ("sync", "pipeline", "workflow") @@ -327,6 +327,13 @@ def _takes_new(db, a): def run(db, cfg, rid, trigger="manual", by=None, propose_only=False, log=None): """Run a routine now, here; the run's id. Each action is tried even when one before it failed.""" + with telemetry.span( + "routine", {"lens.routine.id": str(rid), "lens.routine.trigger": trigger, "lens.routine.propose_only": propose_only} + ): + return _run(db, cfg, rid, trigger, by, propose_only, log) + + +def _run(db, cfg, rid, trigger, by, propose_only, log): routine = get(db, rid) since = routine.get("seen_recording") or 0 seen = {"since": since, "marks": [], "failed": False} @@ -350,7 +357,8 @@ def say(msg): say(f"{k}. {a['type']}") seen["pending"] = None try: - got = _action(db, cfg, routine, run_id, a, seen, propose_only, say) + with telemetry.span(f"action {a['type']}", {"lens.routine.action": a["type"], "lens.routine.action.index": k}): + got = _action(db, cfg, routine, run_id, a, seen, propose_only, say) results.append({"type": a["type"], "status": "done", "result": got}) if seen["pending"] is not None: seen["marks"].append(seen["pending"]) @@ -365,6 +373,7 @@ def say(msg): for s in ("applied", "proposed") } status = "done" if not failed else ("error" if failed == len(results) else "partly") + telemetry.record("lens.routine.runs", 1, {"lens.outcome": status, "lens.routine.trigger": trigger}) db.q( "UPDATE $r MERGE $d", r=R("routine_run", run_id), diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index fbf123dda..b766e1557 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -18,7 +18,7 @@ import secrets import threading -from . import convert, ipgroups, objects, store +from . import convert, ipgroups, objects, store, telemetry R = store.R EDITABLE = { @@ -34,6 +34,7 @@ "iiif": None, "ai": None, "notifications": None, + "telemetry": None, "video": ( "sample_seconds", "scene_threshold", @@ -57,7 +58,7 @@ # the LibreOffice and Chromium paths are startup settings only (the web app can't choose what the server runs) "documents": ("page_pixels", "thumb_pixels", "ocr_below_chars", "max_pages", "convert_seconds", "attachment_resources"), } -SECRETS = {"llm": ("api_key",)} +SECRETS = {"llm": ("api_key",), "telemetry": ("headers",)} ENUMS = { ("transcribe", "engine"): {"sensevoice", "whisper", "mlx-whisper"}, ("transcribe", "device"): {"auto", "cpu", "cuda", "mps"}, @@ -72,13 +73,17 @@ ("video", "object_engine"): {"yolox", "ultralytics", "off"}, } # Settings the environment (.env) sets, which win over archive.yaml and the app and show as locked there: the -# break-glass allowed hosts, and the model provider so an install can be configured without the setup wizard. +# break-glass allowed hosts, the model provider so an install can be configured without the setup wizard, and +# telemetry (LENS_TELEMETRY=off keeps it off whatever the app says). ENV_OVERRIDES = { ("server", "allowed_hosts"): "ARCHIVE_ALLOWED_HOSTS", ("llm", "base_url"): "LENS_LLM_BASE_URL", ("llm", "model"): "LENS_LLM_MODEL", ("llm", "api_key"): "LENS_LLM_API_KEY", ("llm", "vision_model"): "LENS_LLM_VISION_MODEL", + ("telemetry", "enabled"): "LENS_TELEMETRY", + ("telemetry", "endpoint"): "LENS_TELEMETRY_ENDPOINT", + ("telemetry", "headers"): "LENS_TELEMETRY_HEADERS", } @@ -87,8 +92,11 @@ def env_value(section, key): raw = os.environ.get(ENV_OVERRIDES.get((section, key)) or "", "").strip() if not raw: return None - if isinstance(store.DEFAULTS.get(section, {}).get(key), list): + default = store.DEFAULTS.get(section, {}).get(key) + if isinstance(default, list): return [x.strip() for x in raw.split(",") if x.strip()] + if isinstance(default, bool): + return raw.lower() in ("1", "true", "on", "yes") return raw @@ -189,6 +197,7 @@ def current(self): with self._lock: if self._cfg is None or v != self._ver: self._cfg, self._ver = effective(self.db, self.base), v + telemetry.apply(self._cfg) # turned on, changed or off in the app: this process follows return self._cfg @@ -260,6 +269,8 @@ def _check(section, key, value, default): return _upload_setting(key, value) if section == "notifications": return _notify_setting(key, value) + if section == "telemetry": + return _telemetry_setting(key, value) if (section, key) == ("documents", "attachment_resources"): if not isinstance(value, bool): raise ValueError("documents.attachment_resources is true or false") @@ -335,6 +346,59 @@ def _notify_setting(key, value): return value +TELEMETRY_RANGES = {"export_seconds": (5, 3600)} +SERVICE_RX = re.compile(r"^[A-Za-z0-9._-]{1,64}$") + + +def _telemetry_setting(key, value): + if key in ("enabled", "traces", "metrics"): + if not isinstance(value, bool): + raise ValueError(f"telemetry.{key} is true or false") + return value + if key == "endpoint": + if value is None or value == "": + return None + if not (isinstance(value, str) and VIEWER_URL.match(value.strip())): + raise ValueError("telemetry.endpoint is the OTLP/HTTP address of a collector, like http://localhost:4318") + v = value.strip().rstrip("/") + if v.endswith(("/v1/traces", "/v1/metrics")): + raise ValueError("telemetry.endpoint is the collector's base address, without /v1/traces or /v1/metrics") + return v + if key == "sample_ratio": + if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 1): + raise ValueError("telemetry.sample_ratio is a number from 0 to 1") + return float(value) + if key == "service_name": + if not (isinstance(value, str) and SERVICE_RX.match(value.strip())): + raise ValueError("telemetry.service_name is up to 64 letters, digits, ., _ and -") + return value.strip() + if key == "prices": + return _prices(value) + lo, hi = TELEMETRY_RANGES[key] + if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi): + raise ValueError(f"telemetry.{key} is a whole number from {lo} to {hi}") + return value + + +def _prices(value): + """{model: {input, output}}: USD per million tokens, for the cost estimates.""" + msg = "telemetry.prices gives each model its input and output price in USD per million tokens" + if value is None: + return {} + if not isinstance(value, dict) or len(value) > 200: + raise ValueError(msg) + out = {} + for model, p in value.items(): + name = str(model).strip() + if not name or len(name) > 200 or not isinstance(p, dict) or set(p) - {"input", "output"}: + raise ValueError(msg) + nums = {k: p.get(k, 0) for k in ("input", "output")} + if not all(isinstance(n, (int, float)) and not isinstance(n, bool) and 0 <= n <= 1_000_000 for n in nums.values()): + raise ValueError(msg) + out[name] = {k: float(n) for k, n in nums.items()} + return out + + def _upload_setting(key, value): if key == "extensions": ok = isinstance(value, list) and value and all(isinstance(x, str) for x in value) @@ -363,6 +427,8 @@ def save(db, base, section, changes, user=None): if v is None or v == "": sealed.pop(k, None) elif isinstance(v, str): + if (section, k) == ("telemetry", "headers"): + telemetry.parse_headers(v) # ValueError when malformed sealed[k] = seal(base, v, f"setting:{section}.{k}") elif not (isinstance(v, dict) and v.get("secret")): # the mask echoed back means "unchanged" raise ValueError(f"{section}.{k} must be text") diff --git a/fastapi_backend/app/domain/setup.py b/fastapi_backend/app/domain/setup.py index 53e20a9d6..45ffe96ca 100644 --- a/fastapi_backend/app/domain/setup.py +++ b/fastapi_backend/app/domain/setup.py @@ -8,6 +8,7 @@ LENS_ADMIN_EMAIL, LENS_ADMIN_PASSWORD, LENS_ADMIN_NAME the first admin, created at startup (no setup code needed) LENS_NAMESPACE the first namespace, created at startup while there is none LENS_LLM_BASE_URL, LENS_LLM_MODEL, LENS_LLM_API_KEY the model provider (settings.ENV_OVERRIDES) + LENS_TELEMETRY, LENS_TELEMETRY_ENDPOINT opt-in telemetry (off unless set on; docs/telemetry.md) LENS_SETUP_WIZARD=off never show the wizard (everything comes from .env) The first admin still needs the one-time setup code (or the environment), so a stranger can't claim a public server. @@ -101,6 +102,11 @@ def view(db, cfg): "max_upload_mb": shown["uploads"]["max_mb"], "watches": len(db.values("SELECT VALUE id FROM watch_path")), }, + "telemetry": { + "enabled": bool(shown["telemetry"].get("enabled")), + "endpoint": shown["telemetry"].get("endpoint"), + "locked": [k for k in settings.locked("telemetry") if k in ("enabled", "endpoint")], + }, } @@ -123,6 +129,19 @@ def save_llm(db, cfg, values, user=None): return sorted(keep) +def save_telemetry(db, cfg, enabled, endpoint, user=None): + """Opt in to telemetry (or stay out), leaving out what the environment sets. On needs an endpoint.""" + locked = settings.locked("telemetry") + endpoint = (endpoint or "").strip() or None + if enabled and not endpoint and "endpoint" not in locked: + raise ValueError("telemetry needs an endpoint to send to, like http://localhost:4318") + values = {"enabled": bool(enabled), **({"endpoint": endpoint} if endpoint or enabled else {})} # off keeps the address + keep = {k: v for k, v in values.items() if k not in locked} + if keep: + settings.save(db, cfg, "telemetry", keep, user) + return sorted(keep) + + def save_storage(db, cfg, max_upload_mb=None, folder=None, namespace=None, user=None): """The upload limit, and optionally a folder on this machine (inside sources.local_roots) watched into a namespace. Returns the watch's id, if one was made.""" diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 7f6d3b1a3..baff2199b 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -255,6 +255,20 @@ def labels(): # private networks listed here (a Matterbridge on the LAN or the Docker network); app_url is where links in messages # point (null: FRONTEND_URL) "notifications": {"enabled": True, "networks": [], "poll_seconds": 5, "max_attempts": 6, "app_url": None}, + # OpenTelemetry traces and metrics (docs/telemetry.md): off unless an admin turns it on, and sent only to the OTLP/HTTP + # endpoint set here (e.g. a collector at http://localhost:4318). headers is a secret: key=value pairs for the + # endpoint's auth. prices: {model: {input, output}} in USD per million tokens, for cost estimates. + "telemetry": { + "enabled": False, + "endpoint": None, + "headers": None, + "traces": True, + "metrics": True, + "sample_ratio": 1.0, + "export_seconds": 60, + "service_name": "lens", + "prices": {}, + }, # IIIF: identifiers are built from base_url (set it to the stable public HTTPS address; null: the request's address) "iiif": { "base_url": None, diff --git a/fastapi_backend/app/domain/telemetry.py b/fastapi_backend/app/domain/telemetry.py new file mode 100644 index 000000000..253d0c6ce --- /dev/null +++ b/fastapi_backend/app/domain/telemetry.py @@ -0,0 +1,531 @@ +"""Opt-in telemetry: OpenTelemetry traces and metrics, sent only to an OTLP endpoint an admin sets. + +Off by default, and nothing is collected or sent while it is off: no phone-home, no built-in endpoint. Turned on +(Settings → Telemetry, the setup wizard, or LENS_TELEMETRY=on with LENS_TELEMETRY_ENDPOINT in .env), each process +(the API and every `lens worker`) sends what it does to that endpoint over OTLP/HTTP, typically a collector on the same +machine: + +* traces: API requests (by route template), jobs and their steps, routines, workflow runs and model calls; +* metrics: request, step and model-call durations, job and routine outcomes, model tokens and estimated cost. + +Spans and metrics carry no personal content: no transcript or prompt text, no file names or titles, no namespace names, +no client addresses or user ids; only route templates, step types, record ids, model names, token counts and timings. + +The OpenTelemetry SDK is imported only when telemetry is on. Settings are re-read when someone saves them +(settings.Settings.current), so turning it on or off applies without a restart. +""" + +from __future__ import annotations + +import atexit +import logging +import re +import threading +import time +import urllib.parse +import uuid +from contextlib import contextmanager + +log = logging.getLogger("lens") + +_lock = threading.Lock() +_state = {"key": None, "tracer": None, "meter": None, "tp": None, "mp": None, "instruments": {}, "cfg": {}} +_role = ["api"] +_instance = str(uuid.uuid4()) # service.instance.id: random per process, so no host name leaves the machine +_last = {"traces": None, "metrics": None} # the last export from this process: {at, ok, error} +HEADER_RX = re.compile(r"^[A-Za-z0-9!#$%&'*+.^_`|~-]+$") + + +def parse_headers(text): + """`key=value,key2=value2` (OTEL_EXPORTER_OTLP_HEADERS' form, values URL-encoded) to a dict; ValueError if malformed.""" + out = {} + for part in (text or "").split(","): + if not part.strip(): + continue + k, sep, v = part.partition("=") + k = k.strip() + if not sep or not HEADER_RX.match(k): + raise ValueError("telemetry.headers is key=value pairs separated by commas, like Authorization=Bearer%20abc") + out[k] = urllib.parse.unquote(v.strip()) + return out + + +def _section(cfg): + return (cfg or {}).get("telemetry") or {} + + +def _wanted(t): + """The settings that decide what is set up; None when nothing should run.""" + if not t.get("enabled") or not t.get("endpoint"): + return None + if not (t.get("traces") or t.get("metrics")): + return None + return ( + t["endpoint"].rstrip("/"), + t.get("headers") if isinstance(t.get("headers"), str) else None, + bool(t.get("traces")), + bool(t.get("metrics")), + float(t.get("sample_ratio") if t.get("sample_ratio") is not None else 1.0), + int(t.get("export_seconds") or 60), + t.get("service_name") or "lens", + _role[0], + ) + + +def prices(cfg): + """USD per million tokens by model: telemetry.prices, over the assistant's prices for the configured model + (Settings → AI assistant), so a price set there needn't be set twice.""" + ai, model = (cfg or {}).get("ai") or {}, ((cfg or {}).get("llm") or {}).get("model") + out = {} + if model and ai.get("price_in") is not None and ai.get("price_out") is not None: + out[model] = {"input": ai["price_in"], "output": ai["price_out"]} + return {**out, **(_section(cfg).get("prices") or {})} + + +def set_role(role): + """What this process is (api, worker, watcher): sent as lens.process.role.""" + _role[0] = role + + +def apply(cfg): + """Start, change or stop telemetry to match the configuration. Cheap when nothing changed.""" + t = _section(cfg) + want = _wanted(t) + _state["cfg"] = {"prices": prices(cfg)} + if want == _state["key"]: + return + with _lock: + if want == _state["key"]: + return + # the old providers send what they hold in the background: a request that picked up the change (this runs + # under settings.Settings.current) never waits on a collector that is slow or gone + _stop(wait=False) + _last.update(traces=None, metrics=None) + if want is not None: + try: + _start(*want) + except Exception as e: # noqa: BLE001 - telemetry never stops the archive + log.warning("telemetry: couldn't start (%s: %s); carrying on without it", type(e).__name__, e) + _stop(wait=False) + elif _state["key"] is not None: + log.info("telemetry: off") + _state["key"] = want + + +def _stop(wait=True): + providers = [p for p in (_state["tp"], _state["mp"]) if p is not None] + _state.update(tracer=None, meter=None, tp=None, mp=None, instruments={}) + + def close(): + for p in providers: + try: + p.shutdown() + except Exception: # noqa: BLE001 - shutting down a provider whose endpoint is gone + pass + + if wait: + close() + elif providers: + threading.Thread(target=close, daemon=True, name="telemetry-stop").start() + + +def _note(kind, ok, error=None): + _last[kind] = {"at": time.time(), "ok": ok, "error": error} + + +def _start(endpoint, headers, traces, metrics, ratio, every, service, role): + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import MetricExportResult, PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + + from . import __version__ + + hdrs = parse_headers(headers) if headers else {} + resource = Resource.create( + {"service.name": service, "service.version": __version__, "service.instance.id": _instance, "lens.process.role": role} + ) + + class Spans(OTLPSpanExporter): + def export(self, spans): + try: + r = super().export(spans) + except Exception as e: # noqa: BLE001 + _note("traces", False, f"{type(e).__name__}: {e}"[:300]) + raise + _note("traces", r == SpanExportResult.SUCCESS, None if r == SpanExportResult.SUCCESS else "the endpoint refused the traces") + return r + + class Metrics(OTLPMetricExporter): + def export(self, metrics_data, timeout_millis=10000, **kw): + try: + r = super().export(metrics_data, timeout_millis=timeout_millis, **kw) + except Exception as e: # noqa: BLE001 + _note("metrics", False, f"{type(e).__name__}: {e}"[:300]) + raise + ok = r == MetricExportResult.SUCCESS + _note("metrics", ok, None if ok else "the endpoint refused the metrics") + return r + + if traces: + tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(max(0.0, min(1.0, ratio))))) + tp.add_span_processor(BatchSpanProcessor(Spans(endpoint=endpoint + "/v1/traces", headers=hdrs, timeout=10))) + _state.update(tp=tp, tracer=tp.get_tracer("lens", __version__)) + if metrics: + reader = PeriodicExportingMetricReader( + Metrics(endpoint=endpoint + "/v1/metrics", headers=hdrs, timeout=10), export_interval_millis=max(5, every) * 1000 + ) + mp = MeterProvider(resource=resource, metric_readers=[reader]) + _state.update(mp=mp, meter=mp.get_meter("lens", __version__)) + log.info( + "telemetry: on, sending %s to %s", + " and ".join(x for x, on in (("traces", traces), ("metrics", metrics)) if on), + _safe_url(endpoint), + ) + + +def _safe_url(url): + """The endpoint without any user:password in it, for logs and the status.""" + try: + p = urllib.parse.urlsplit(url) + return urllib.parse.urlunsplit( + (p.scheme, p.hostname + (f":{p.port}" if p.port else "") if p.hostname else p.netloc, p.path, "", "") + ) + except ValueError: + return "(invalid address)" + + +def flush(timeout_millis=10000): + """Send what is buffered now (tests; a worker about to exit).""" + for p in (_state["tp"], _state["mp"]): + if p is not None: + p.force_flush(timeout_millis) + + +def shutdown(): + """Send what is buffered and stop (process exit).""" + with _lock: + _stop() + _state["key"] = None + + +atexit.register(shutdown) + + +def active(): + return _state["tracer"] is not None or _state["meter"] is not None + + +def status(cfg): + """What this process does with telemetry, for Settings.""" + t = _section(cfg) + return { + "enabled": bool(t.get("enabled")), + "endpoint": _safe_url(t["endpoint"]) if t.get("endpoint") else None, + "traces": bool(_state["tracer"]), + "metrics": bool(_state["meter"]), + "last_traces": _last["traces"], + "last_metrics": _last["metrics"], + } + + +# ---------- spans ---------- +class _NoSpan: + """What a span is while telemetry is off: nothing.""" + + def set_attribute(self, key, value): + pass + + def set_attributes(self, attrs): + pass + + def record_exception(self, e): + pass + + def set_status(self, *a, **k): + pass + + def update_name(self, name): + pass + + def end(self): + pass + + def is_recording(self): + return False + + +NO_SPAN = _NoSpan() + + +def _clean(attrs): + return {k: v for k, v in (attrs or {}).items() if v is not None} + + +@contextmanager +def span(name, attrs=None, kind=None): + """A span around a block (the current span while it runs), or nothing while traces are off. An exception that + leaves the block marks the span failed with its type (not its message, which may quote content) and goes on.""" + tracer = _state["tracer"] + if tracer is None: + yield NO_SPAN + return + from opentelemetry.trace import SpanKind + + with tracer.start_as_current_span( + name, kind=kind or SpanKind.INTERNAL, attributes=_clean(attrs), record_exception=False, set_status_on_exception=False + ) as s: + try: + yield s + except BaseException as e: + fail(s, e) + raise + + +def start_span(name, attrs=None, kind=None): + """A span that isn't made current (for generators, which may resume in another context); end() it.""" + tracer = _state["tracer"] + if tracer is None: + return NO_SPAN + from opentelemetry.trace import SpanKind + + return tracer.start_span(name, kind=kind or SpanKind.INTERNAL, attributes=_clean(attrs)) + + +def fail(s, e): + """Mark a span failed with an exception's type (not its message, which may quote content).""" + if not s.is_recording(): + return + from opentelemetry.trace import Status, StatusCode + + s.set_attribute("error.type", type(e).__name__) + s.set_status(Status(StatusCode.ERROR, type(e).__name__)) + + +# ---------- metrics ---------- +def _instrument(kind, name, unit, description): + meter = _state["meter"] + if meter is None: + return None + inst = _state["instruments"].get(name) + if inst is None: + make = {"counter": meter.create_counter, "histogram": meter.create_histogram}[kind] + inst = _state["instruments"][name] = make(name, unit=unit, description=description) + return inst + + +METRICS = { + "http.server.request.duration": ("histogram", "s", "API request duration"), + "lens.job.step.duration": ("histogram", "s", "How long a job's step took"), + "lens.jobs": ("counter", "{job}", "Job runs, by how they ended"), + "lens.routine.runs": ("counter", "{run}", "Routine runs, by how they ended"), + "lens.workflow.runs": ("counter", "{run}", "Workflow runs, by how they ended"), + "gen_ai.client.operation.duration": ("histogram", "s", "Model call duration"), + "gen_ai.client.token.usage": ("histogram", "{token}", "Tokens per model call"), + "lens.llm.cost": ("counter", "USD", "Estimated model cost, from the prices set in Settings → Telemetry"), +} + + +def record(name, value, attrs=None): + """Add to a counter or record in a histogram (see METRICS); nothing while metrics are off.""" + kind, unit, description = METRICS[name] + inst = _instrument(kind, name, unit, description) + if inst is None: + return + try: + (inst.add if kind == "counter" else inst.record)(value, _clean(attrs)) + except Exception: # noqa: BLE001 - a metric never breaks the work it measures + pass + + +# ---------- model calls ---------- +def cost(model, input_tokens, output_tokens, prices=None): + """Estimated cost in USD from the per-million-token prices for the model, or None when it has no price.""" + p = (prices if prices is not None else _state["cfg"].get("prices") or {}).get(model) + if not isinstance(p, dict): + return None + return round((input_tokens or 0) / 1e6 * float(p.get("input") or 0) + (output_tokens or 0) / 1e6 * float(p.get("output") or 0), 8) + + +class ModelCall: + """One call to the model server: a client span (gen_ai.* attributes), its duration, tokens and estimated cost. + Neither the prompt nor the reply is recorded.""" + + def __init__(self, cfg, payload, operation="chat", current=True): + self.on = active() + if not self.on: + return + l = (cfg or {}).get("llm") or {} + self.model = payload.get("model") + self.operation, self.t0, self.done, self.response_model, self.usage = operation, time.monotonic(), False, None, None + server = urllib.parse.urlsplit(l.get("base_url") or "") + attrs = { + "gen_ai.operation.name": operation, + "gen_ai.provider.name": "openai", # the OpenAI-compatible API, whoever serves it + "gen_ai.request.model": self.model, + "gen_ai.request.temperature": payload.get("temperature"), + "gen_ai.request.max_tokens": payload.get("max_tokens"), + "gen_ai.output.type": "json" if payload.get("response_format") else "text", + "lens.llm.tools": len(payload.get("tools") or []) or None, + "lens.llm.stream": True if payload.get("stream") else None, + "server.address": server.hostname, + "server.port": server.port, + } + from opentelemetry.trace import SpanKind + + name = f"{operation} {self.model}" + if current: + self._cm = span(name, attrs, SpanKind.CLIENT) + self.span = self._cm.__enter__() + else: + self._cm, self.span = None, start_span(name, attrs, SpanKind.CLIENT) + + def reply(self, j): + """Note the usage and model of a reply (a parsed JSON body or a streamed chunk); returns it.""" + if self.on and isinstance(j, dict): + if isinstance(j.get("usage"), dict): + self.usage = j["usage"] + if isinstance(j.get("model"), str): + self.response_model = j["model"] + return j + + def end(self, error=None): + if not self.on or self.done: + return + self.done = True + seconds = time.monotonic() - self.t0 + base = {"gen_ai.operation.name": self.operation, "gen_ai.provider.name": "openai", "gen_ai.request.model": self.model} + if self.response_model: + base["gen_ai.response.model"] = self.response_model + self.span.set_attribute("gen_ai.response.model", self.response_model) + if error is not None: + fail(self.span, error) + record("gen_ai.client.operation.duration", seconds, {**base, "error.type": type(error).__name__}) + else: + record("gen_ai.client.operation.duration", seconds, base) + u = self.usage or {} + tin, tout = u.get("prompt_tokens", u.get("input_tokens")), u.get("completion_tokens", u.get("output_tokens")) + if isinstance(tin, int): + self.span.set_attribute("gen_ai.usage.input_tokens", tin) + record("gen_ai.client.token.usage", tin, {**base, "gen_ai.token.type": "input"}) + if isinstance(tout, int): + self.span.set_attribute("gen_ai.usage.output_tokens", tout) + record("gen_ai.client.token.usage", tout, {**base, "gen_ai.token.type": "output"}) + if isinstance(tin, int) or isinstance(tout, int): + usd = cost(self.model, tin if isinstance(tin, int) else 0, tout if isinstance(tout, int) else 0) + if usd is not None: + self.span.set_attribute("lens.llm.cost_usd", usd) + record("lens.llm.cost", usd, base) + if self._cm is not None: + self._cm.__exit__(None, None, None) + else: + self.span.end() + + def __enter__(self): + return self + + def __exit__(self, et, e, tb): + self.end(e) + return False + + +def model_call(cfg, payload, operation="chat", current=True): + """`with model_call(cfg, payload) as call: ... call.reply(json)`: a measured model call (a no-op while off).""" + return ModelCall(cfg, payload, operation, current) + + +# ---------- the API ---------- +class Middleware: + """A server span and a duration for each API request, named by its route template (`GET /api/v1/recordings/{rid}`), + never by its path or query, which can carry names. Installed innermost, where the router has chosen the route.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or not active(): + await self.app(scope, receive, send) + return + from opentelemetry import context, trace + from opentelemetry.trace import SpanKind, Status, StatusCode + + method = scope.get("method") or "GET" + status = [None] + + async def sending(message): + if message["type"] == "http.response.start": + status[0] = message["status"] + await send(message) + + t0, s, error = time.monotonic(), start_span(method, {"http.request.method": method}, SpanKind.SERVER), None + token = context.attach(trace.set_span_in_context(s)) if s.is_recording() else None + try: + await self.app(scope, receive, sending) + except Exception as e: + error = e + raise + finally: + if token is not None: + context.detach(token) + route = getattr(scope.get("route"), "path", None) or "unmatched" + code = status[0] or 500 + attrs = {"http.request.method": method, "http.route": route, "http.response.status_code": code} + if error is not None: + attrs["error.type"] = type(error).__name__ + s.update_name(f"{method} {route}") + s.set_attributes(attrs) + if error is not None: + fail(s, error) + elif code >= 500: + s.set_status(Status(StatusCode.ERROR)) + s.end() + record("http.server.request.duration", time.monotonic() - t0, attrs) + + +def test_export(cfg, timeout=5): + """Send one test span to the configured endpoint now, from a provider of its own; (ok, error, ms).""" + t = _section(cfg) + if not t.get("endpoint"): + return False, "set an endpoint first", None + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult + + from . import __version__ + + try: + hdrs = parse_headers(t.get("headers")) if isinstance(t.get("headers"), str) else {} + except ValueError as e: + return False, str(e), None + got = {} + + class Once(OTLPSpanExporter): + def export(self, spans): + try: + got["r"] = super().export(spans) + except Exception as e: # noqa: BLE001 + got["e"] = f"{type(e).__name__}: {e}"[:300] + raise + return got["r"] + + tp = TracerProvider(resource=Resource.create({"service.name": t.get("service_name") or "lens", "service.version": __version__})) + tp.add_span_processor(SimpleSpanProcessor(Once(endpoint=t["endpoint"].rstrip("/") + "/v1/traces", headers=hdrs, timeout=timeout))) + t0 = time.monotonic() + try: + with tp.get_tracer("lens").start_as_current_span("lens.telemetry.test"): + pass + finally: + tp.shutdown() + ms = int((time.monotonic() - t0) * 1000) + if got.get("r") == SpanExportResult.SUCCESS: + return True, None, ms + return ( + False, + got.get("e") or f"the endpoint at {_safe_url(t['endpoint'])} didn't accept the span (check the address and the collector's log)", + ms, + ) diff --git a/fastapi_backend/app/domain/workflows.py b/fastapi_backend/app/domain/workflows.py index b19207ba6..e048b81a4 100644 --- a/fastapi_backend/app/domain/workflows.py +++ b/fastapi_backend/app/domain/workflows.py @@ -30,7 +30,7 @@ import re -from . import analyze, fields as fieldmod, llm, metadata, organize, pipelines, store, templates +from . import analyze, fields as fieldmod, llm, metadata, organize, pipelines, store, telemetry, templates R = store.R RECORDING_NODES = ("input", "llm", "pick", "condition", "merge", "extract_rules", "extract_llm", "output", "field", "save_entities") @@ -437,6 +437,18 @@ def save_entities(db, cfg, rid, ents): def run(db, cfg, rid, wid, version=None, say=print, user=None): """Run one version (default: the current one) of a workflow on a recording. Returns what each node did: {node id: done | skipped}.""" + attrs = {"lens.workflow.id": str(wid), "lens.workflow.version": version, "lens.recording.id": str(rid)} + with telemetry.span("workflow", attrs): + try: + out = _run(db, cfg, rid, wid, version, say, user) + except Exception: + telemetry.record("lens.workflow.runs", 1, {"lens.outcome": "failed"}) + raise + telemetry.record("lens.workflow.runs", 1, {"lens.outcome": "done"}) + return out + + +def _run(db, cfg, rid, wid, version, say, user): w = get(db, wid, version) if w["scope"] != "recording": raise ValueError(f"workflow {w['name']} organises the graph; a routine runs it, not a pipeline") diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py index 6f7343a44..d7dd41eff 100644 --- a/fastapi_backend/app/main.py +++ b/fastapi_backend/app/main.py @@ -24,7 +24,7 @@ from app.config import settings from app.core import middleware from app.core.runtime import Archive -from app.domain import __version__, store +from app.domain import __version__, store, telemetry from app.domain.render import WEB_DIR from app.utils import simple_generate_unique_route_id @@ -56,6 +56,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: yield finally: archive.close() + telemetry.shutdown() app = FastAPI( title=settings.PROJECT_NAME, @@ -68,6 +69,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: ready.prepare() _attach(app, ready) + # innermost, so the router has picked the route it names spans by; a no-op while telemetry is off + app.add_middleware(telemetry.Middleware) app.add_middleware( CORSMiddleware, allow_origins=sorted(settings.CORS_ORIGINS), diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index f2b956ab1..03e40d5a2 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -23,6 +23,29 @@ class LlmTestResult(ResponseModel): model: str | None = None +class TelemetryExport(ResponseModel): + at: float + ok: bool + error: str | None = None + + +class TelemetryStatus(ResponseModel): + """What the API process does with telemetry now; each worker follows the same settings on its own.""" + + enabled: bool + endpoint: str | None = None + traces: bool + metrics: bool + last_traces: TelemetryExport | None = None + last_metrics: TelemetryExport | None = None + + +class TelemetryTestResult(ResponseModel): + ok: bool + error: str | None = None + ms: int | None = None + + class AuditEntry(ResponseModel): at: str email: str | None = None diff --git a/fastapi_backend/app/schemas/setup.py b/fastapi_backend/app/schemas/setup.py index 93e409b5a..0e67559d6 100644 --- a/fastapi_backend/app/schemas/setup.py +++ b/fastapi_backend/app/schemas/setup.py @@ -30,12 +30,20 @@ class SetupStorageView(ResponseModel): watches: int +class SetupTelemetryView(ResponseModel): + enabled: bool + endpoint: str | None = None + # what .env sets (LENS_TELEMETRY, LENS_TELEMETRY_ENDPOINT): shown locked + locked: list[str] + + class SetupView(ResponseModel): pending: bool admin: SetupAdmin namespace: SetupNamespaceView llm: SetupLlmView storage: SetupStorageView + telemetry: SetupTelemetryView class SetupNamespace(RequestModel): @@ -57,6 +65,12 @@ class SetupStorage(RequestModel): namespace: str | None = None +class SetupTelemetry(RequestModel): + enabled: bool = False + # the OTLP/HTTP address of a collector, like http://localhost:4318; needed to turn telemetry on + endpoint: str | None = Field(default=None, max_length=500) + + class SetupFinish(RequestModel): skipped: bool = False diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index 2d91726c0..3170a29a4 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -18,6 +18,9 @@ dependencies = [ "pillow>=10", "markdown>=3.5", "cryptography>=42", + "opentelemetry-api>=1.27,<2", + "opentelemetry-sdk>=1.27,<2", + "opentelemetry-exporter-otlp-proto-http>=1.27,<2", ] [project.optional-dependencies] diff --git a/fastapi_backend/tests/api/test_telemetry.py b/fastapi_backend/tests/api/test_telemetry.py new file mode 100644 index 000000000..e4860c5bd --- /dev/null +++ b/fastapi_backend/tests/api/test_telemetry.py @@ -0,0 +1,226 @@ +"""Opt-in telemetry: off by default, sent only to the configured OTLP endpoint, and nothing personal in it.""" + +from __future__ import annotations + +import threading + +import pytest + +from app.domain import jobs, telemetry +from tests import fake_llm, fake_otlp +from tests.helpers import drain, login, make_user, seed + + +@pytest.fixture +def collector(): + srv, url = fake_otlp.start() + yield url + telemetry.shutdown() # telemetry is per process: the next test starts with it off + srv.shutdown() + + +@pytest.fixture +def model(cfg): + srv, url = fake_llm.start() + cfg["llm"].update(base_url=url, model="fake") + fake_llm.Handler.usage = {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500} + yield fake_llm.Handler + fake_llm.Handler.usage = None + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, model): + from app.main import create_app + + return create_app(cfg, db, background=False) + + +def admin(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def test_off_by_default_sends_nothing(client, db, cfg, folder, collector): + rid = seed(db, cfg, folder)[0] + h = admin(client, db) + t = client.get("/api/v1/settings", headers=h).json()["telemetry"] + assert t["values"]["enabled"] is False and t["values"]["endpoint"] is None + assert t["values"]["headers"] == {"secret": True, "set": False} and t["locked"] == [] + # an endpoint alone doesn't turn it on + assert client.put("/api/v1/settings/telemetry", json={"endpoint": collector}, headers=h).status_code == 200 + assert client.get(f"/api/v1/recordings/{rid}", headers=h).status_code == 200 + jobs.enqueue(db, rid, ["analyze"]) + drain(db, cfg) + assert client.post("/api/v1/settings/llm/test", headers=h).json()["ok"] + assert not telemetry.active() + telemetry.flush() + assert fake_otlp.Handler.bodies == [] + s = client.get("/api/v1/settings/telemetry/status", headers=h).json() + assert (s["enabled"], s["traces"], s["metrics"], s["last_traces"]) == (False, False, False, None) + + +def test_settings_are_checked(client, db, collector): + h = admin(client, db) + + def put(body): + return client.put("/api/v1/settings/telemetry", json=body, headers=h) + + for bad in ( + {"endpoint": "ftp://collector:4318"}, + {"endpoint": "http://collector:4318/v1/traces"}, + {"enabled": "yes"}, + {"sample_ratio": 1.5}, + {"export_seconds": 1}, + {"service_name": "lens server"}, + {"prices": {"gpt": {"input": -1, "output": 1}}}, + {"prices": {"gpt": {"input": 1, "cached": 1}}}, + {"prices": ["gpt"]}, + {"headers": "no-equals-sign"}, + {"colour": "blue"}, + ): + assert put(bad).status_code == 400, bad + assert put({"endpoint": "http://collector:4318/", "prices": {" gpt-4o ": {"input": 2.5, "output": 10}}}).status_code == 200 + v = client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"] + assert v["endpoint"] == "http://collector:4318" and v["prices"] == {"gpt-4o": {"input": 2.5, "output": 10.0}} + assert put({"headers": "Authorization=Bearer%20abc,X-Tenant=lens"}).status_code == 200 + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"] == {"secret": True, "set": True} + assert put({"prices": {}}).status_code == 200 + + +def test_traces_and_metrics_go_to_the_endpoint(client, db, cfg, folder, collector, model): + cfg["ai"]["tools"] = False # the plain, streamed answer + rid = seed(db, cfg, folder)[0] + h = admin(client, db) + r = client.put( + "/api/v1/settings/telemetry", + headers=h, + json={ + "enabled": True, + "endpoint": collector, + "headers": "Authorization=Bearer%20t0k", + "prices": {"fake": {"input": 1.0, "output": 2.0}}, + }, + ) + assert r.status_code == 200, r.text + # the next request picks the change up + assert client.get(f"/api/v1/recordings/{rid}", params={"q": "private words"}, headers=h).status_code == 200 + assert telemetry.active() + assert client.get("/api/v1/recordings/999999", headers=h).status_code == 404 + assert client.post("/api/v1/settings/llm/test", headers=h).json()["ok"] + jobs.enqueue(db, rid, ["analyze"]) + drain(db, cfg) + cid = client.post("/api/v1/chats", headers=h, json={}).json()["id"] + assert "The shipment" in client.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "When does it ship?"}).text + telemetry.flush() + + spans = {s["name"]: s for s in fake_otlp.Handler.spans} + got = [s["attrs"] for s in fake_otlp.Handler.spans if s["name"] == "GET /api/v1/recordings/{rid}"] + assert [(a["http.route"], a["http.response.status_code"]) for a in got] == [("/api/v1/recordings/{rid}", c) for c in (200, 404)] + got = spans["GET /api/v1/recordings/{rid}"] + assert got["resource"]["service.name"] == "lens" and got["resource"]["lens.process.role"] == "api" + assert "host.name" not in got["resource"] + job, step = spans["job"], spans["step analyze"] + assert (job["attrs"]["lens.recording.id"], job["attrs"]["lens.job.outcome"]) == (str(rid), "succeeded") + assert "lens.pipeline.id" not in job["attrs"] # steps asked for by hand, not a pipeline + assert step["attrs"]["lens.step.outcome"] == "done" + calls = [s for s in fake_otlp.Handler.spans if s["name"] == "chat fake"] + assert len(calls) == 2 # the test and the streamed answer + for c in calls: + a = c["attrs"] + assert (a["gen_ai.request.model"], a["gen_ai.usage.input_tokens"], a["gen_ai.usage.output_tokens"]) == ("fake", 1000, 500) + assert a["lens.llm.cost_usd"] == pytest.approx(0.002) and a["server.address"] == "127.0.0.1" + assert {h_.get("authorization") for h_ in fake_otlp.Handler.headers} == {"Bearer t0k"} + + names = {p["name"] for p in fake_otlp.Handler.points} + assert {"http.server.request.duration", "lens.jobs", "lens.job.step.duration", "gen_ai.client.token.usage"} <= names + assert {"gen_ai.client.operation.duration", "lens.llm.cost"} <= names + cost = sum(p["value"] for p in fake_otlp.Handler.points if p["name"] == "lens.llm.cost") + assert cost == pytest.approx(0.004) + tokens = {"input": 0.0, "output": 0.0} + for p in fake_otlp.Handler.points: + if p["name"] == "gen_ai.client.token.usage": + tokens[p["attrs"]["gen_ai.token.type"]] += p["value"] + assert tokens == {"input": 2000, "output": 1000} + + # nothing personal: no query strings, prompts, answers, titles or transcript text + raw = b"".join(fake_otlp.Handler.bodies) + for text in (b"private words", b"When does it ship", b"The shipment", b"ep1", b"capsid", b"root@x.io"): + assert text not in raw, text + + s = client.get("/api/v1/settings/telemetry/status", headers=h).json() + assert (s["enabled"], s["endpoint"], s["traces"], s["metrics"]) == (True, collector, True, True) + assert s["last_traces"]["ok"] and s["last_metrics"]["ok"] + + # off again: the next request stops it, and nothing more is sent + assert client.put("/api/v1/settings/telemetry", json={"enabled": False}, headers=h).status_code == 200 + client.get("/api/v1/jobs", headers=h) + assert not telemetry.active() + for t in threading.enumerate(): # the old providers send what they held, in the background + if t.name == "telemetry-stop": + t.join(15) + fake_otlp.reset() + client.get(f"/api/v1/recordings/{rid}", headers=h) + assert fake_otlp.Handler.bodies == [] + + +def test_send_a_test_span(client, db, collector): + h = admin(client, db) + r = client.post("/api/v1/settings/telemetry/test", headers=h).json() + assert (r["ok"], r["error"]) == (False, "set an endpoint first") + client.put("/api/v1/settings/telemetry", json={"endpoint": collector}, headers=h) + r = client.post("/api/v1/settings/telemetry/test", headers=h).json() + assert r["ok"] and r["ms"] >= 0, r + assert [s["name"] for s in fake_otlp.Handler.spans] == ["lens.telemetry.test"] + assert not telemetry.active() # testing doesn't turn it on + fake_otlp.Handler.refuse = True + r = client.post("/api/v1/settings/telemetry/test", headers=h).json() + assert not r["ok"] and r["error"] + + +def test_environment_turns_it_on_or_keeps_it_off(client, db, collector, monkeypatch): + h = admin(client, db) + monkeypatch.setenv("LENS_TELEMETRY", "off") + client.put("/api/v1/settings/telemetry", json={"enabled": True, "endpoint": collector}, headers=h) + t = client.get("/api/v1/settings", headers=h).json()["telemetry"] + assert t["values"]["enabled"] is False and t["locked"] == ["enabled"] + assert not telemetry.active() + assert client.get("/api/v1/setup", headers=h).json()["telemetry"]["locked"] == ["enabled"] + + monkeypatch.setenv("LENS_TELEMETRY", "on") + monkeypatch.setenv("LENS_TELEMETRY_ENDPOINT", collector + "/") + client.put("/api/v1/settings/telemetry", json={"enabled": False}, headers=h) # a save rebuilds the settings + client.get("/api/v1/jobs", headers=h) + assert telemetry.active() + assert client.get("/api/v1/settings/telemetry/status", headers=h).json()["endpoint"] == collector + "/" + + +def test_setup_wizard_opt_in(client, db, collector): + h = admin(client, db) + v = client.get("/api/v1/setup", headers=h).json()["telemetry"] + assert v == {"enabled": False, "endpoint": None, "locked": []} + r = client.put("/api/v1/setup/telemetry", json={"enabled": True}, headers=h) + assert r.status_code == 400 and "endpoint" in r.json()["detail"] + assert client.put("/api/v1/setup/telemetry", json={"enabled": True, "endpoint": "nope"}, headers=h).status_code == 400 + r = client.put("/api/v1/setup/telemetry", json={"enabled": True, "endpoint": collector}, headers=h) + assert r.json()["saved"] == ["enabled", "endpoint"] + assert client.get("/api/v1/setup", headers=h).json()["telemetry"] == {"enabled": True, "endpoint": collector, "locked": []} + # off keeps the address for later + client.put("/api/v1/setup/telemetry", json={"enabled": False}, headers=h) + assert client.get("/api/v1/setup", headers=h).json()["telemetry"] == {"enabled": False, "endpoint": collector, "locked": []} + audit = client.get("/api/v1/audit", headers=h).json() + assert any(a["action"] == "settings.save" and a["target"] == "telemetry" for a in audit) + + +def test_cost_and_headers(): + prices = {"m": {"input": 3.0, "output": 15.0}} + assert telemetry.cost("m", 2_000_000, 100_000, prices) == pytest.approx(7.5) + assert telemetry.cost("other", 10, 10, prices) is None + cfg = {"llm": {"model": "m"}, "ai": {"price_in": 1.0, "price_out": 2.0}, "telemetry": {"prices": {"x": {"input": 0, "output": 1}}}} + assert telemetry.prices(cfg) == {"m": {"input": 1.0, "output": 2.0}, "x": {"input": 0, "output": 1}} + cfg["telemetry"]["prices"]["m"] = {"input": 5.0, "output": 5.0} + assert telemetry.prices(cfg)["m"] == {"input": 5.0, "output": 5.0} + assert telemetry.parse_headers("a=1, Authorization=Bearer%20x") == {"a": "1", "Authorization": "Bearer x"} + assert telemetry.parse_headers("") == {} + with pytest.raises(ValueError): + telemetry.parse_headers("bad header=1") diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py index e6b947e7f..f6bd0a0d3 100644 --- a/fastapi_backend/tests/fake_llm.py +++ b/fastapi_backend/tests/fake_llm.py @@ -12,6 +12,7 @@ class Handler(http.server.BaseHTTPRequestHandler): tool_script = [] # assistant messages to return, in order, when a request offers tools reject_tools = False # behave like a server whose model can't call tools blind = False # behave like a server whose model can't see images + usage = None # token counts to report with each answer (and as a streamed answer's last chunk), like OpenAI def _json(self, obj): data = json.dumps(obj).encode() @@ -46,6 +47,8 @@ def do_POST(self): self.end_headers() for piece in ["The shipment ", "leaves on Friday [1]."]: self.wfile.write(f"data: {json.dumps({'choices': [{'delta': {'content': piece}}]})}\n\n".encode()) + if Handler.usage: + self.wfile.write(f"data: {json.dumps({'choices': [], 'usage': Handler.usage})}\n\n".encode()) self.wfile.write(b"data: [DONE]\n\n") return seen = _picture(body) @@ -115,7 +118,8 @@ def do_POST(self): ) else: content = "OK" - data = json.dumps({"choices": [{"message": {"content": content}}]}).encode() + extra = {"usage": Handler.usage, "model": body.get("model")} if Handler.usage else {} + data = json.dumps({"choices": [{"message": {"content": content}}], **extra}).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(data))) diff --git a/fastapi_backend/tests/fake_otlp.py b/fastapi_backend/tests/fake_otlp.py new file mode 100644 index 000000000..df34e8a79 --- /dev/null +++ b/fastapi_backend/tests/fake_otlp.py @@ -0,0 +1,77 @@ +"""A tiny OTLP/HTTP collector for tests: keeps the spans and metric points it is sent, decoded.""" + +import http.server +import threading + +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + + +def _value(v): + kind = v.WhichOneof("value") + return getattr(v, kind) if kind else None + + +def _attrs(kvs): + return {kv.key: _value(kv.value) for kv in kvs} + + +class Handler(http.server.BaseHTTPRequestHandler): + spans = [] # {name, kind, attrs, status, resource} + points = [] # {name, unit, attrs, value} + headers = [] # the request headers of each export + bodies = [] # the raw requests, to check nothing personal is in them + refuse = False # answer 400, like a collector that won't take what it's sent + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + Handler.headers.append({k.lower(): v for k, v in self.headers.items()}) + Handler.bodies.append(body) + if Handler.refuse: + self.send_response(400) + self.send_header("Content-Length", "0") + self.end_headers() + return + if self.path == "/v1/traces": + req = ExportTraceServiceRequest() + req.ParseFromString(body) + for rs in req.resource_spans: + res = _attrs(rs.resource.attributes) + for ss in rs.scope_spans: + for s in ss.spans: + Handler.spans.append( + {"name": s.name, "kind": s.kind, "attrs": _attrs(s.attributes), "status": s.status.code, "resource": res} + ) + elif self.path == "/v1/metrics": + req = ExportMetricsServiceRequest() + req.ParseFromString(body) + for rm in req.resource_metrics: + for sm in rm.scope_metrics: + for m in sm.metrics: + data = getattr(m, m.WhichOneof("data")) + for p in data.data_points: + value = p.sum if hasattr(p, "bucket_counts") else _number(p) + Handler.points.append({"name": m.name, "unit": m.unit, "attrs": _attrs(p.attributes), "value": value}) + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *a): + pass + + +def _number(p): + kind = p.WhichOneof("value") + return getattr(p, kind) if kind else None + + +def reset(): + Handler.spans, Handler.points, Handler.headers, Handler.bodies, Handler.refuse = [], [], [], [], False + + +def start(): + reset() + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv, f"http://127.0.0.1:{srv.server_address[1]}" diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index 5ef23876b..eb3dfc415 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -1584,6 +1584,9 @@ dependencies = [ { name = "jinja2" }, { name = "markdown" }, { name = "numpy" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "pillow" }, { name = "pydantic-settings" }, { name = "pyjwt" }, @@ -1679,6 +1682,9 @@ requires-dist = [ { name = "onnxruntime", marker = "extra == 'insightface'" }, { name = "onnxruntime", marker = "extra == 'objects'" }, { name = "opencv-python-headless", marker = "extra == 'faces'" }, + { name = "opentelemetry-api", specifier = ">=1.27,<2" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27,<2" }, + { name = "opentelemetry-sdk", specifier = ">=1.27,<2" }, { name = "pillow", specifier = ">=10" }, { name = "pyannote-audio", marker = "extra == 'pyannote'" }, { name = "pydantic-settings", specifier = ">=2.5.2,<3" }, diff --git a/mkdocs.yml b/mkdocs.yml index 88bc8b5f4..952a8ce6c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,6 +71,7 @@ nav: - Video: video.md - Chat and batch runs: assistant.md - Notifications: notifications.md + - Telemetry: telemetry.md - IIIF: iiif.md - API: api.md - API work the design needs: backend-gaps.md diff --git a/nextjs-frontend/__tests__/settings-admin-models.test.ts b/nextjs-frontend/__tests__/settings-admin-models.test.ts index 7586483d9..b0ba60e11 100644 --- a/nextjs-frontend/__tests__/settings-admin-models.test.ts +++ b/nextjs-frontend/__tests__/settings-admin-models.test.ts @@ -193,6 +193,35 @@ describe("settings fields", () => { }), ).toBe("Adds https://docs.lens.local."); }); + + it("handles telemetry: off by default, prices as lines, an endpoint to turn it on", () => { + expect(spec("telemetry.enabled").kind).toBe("switch"); + const prices = spec("telemetry.prices"); + expect(toUi(prices, { "gpt-4o-mini": { input: 0.15, output: 0.6 }, local: { input: 0, output: 0 } })).toBe( + "gpt-4o-mini 0.15 0.6\nlocal 0 0", + ); + expect(parse(prices, " gpt-4o-mini 0.15 0.60 \n\n my model 1 2")).toEqual({ + value: { "gpt-4o-mini": { input: 0.15, output: 0.6 }, "my model": { input: 1, output: 2 } }, + }); + expect(parse(prices, "gpt-4o 2.5")).toEqual({ + error: "Line 1: write the model, then its input and output price, like gpt-4o-mini 0.15 0.60", + }); + expect(parse(prices, "a 1 2\nb -1 2")).toHaveProperty("error"); + expect(parse(prices, "")).toEqual({ value: {} }); + expect(show(prices, { a: { input: 1, output: 2 } })).toBe("1 model"); + expect(show(prices, {})).toBe("none"); + expect(crossErrors({ "telemetry.enabled": true, "telemetry.endpoint": null })["telemetry.endpoint"]).toMatch( + /address to send/, + ); + expect(crossErrors({ "telemetry.enabled": false, "telemetry.endpoint": null })).toEqual({}); + expect(crossErrors({ "telemetry.endpoint": "localhost:4318" })["telemetry.endpoint"]).toMatch(/http/); + expect(crossErrors({ "telemetry.endpoint": "http://c:4318/v1/traces" })["telemetry.endpoint"]).toMatch( + /without \/v1/, + ); + const on = { id: "telemetry.enabled", field: spec("telemetry.enabled"), before: false, after: true }; + expect(why(on)).toMatch(/start sending/); + expect(why({ ...on, before: true, after: false })).toMatch(/Nothing more is sent/); + }); }); describe("audit log", () => { diff --git a/nextjs-frontend/__tests__/setupWizard.test.tsx b/nextjs-frontend/__tests__/setupWizard.test.tsx index 264b9a6b0..267d57404 100644 --- a/nextjs-frontend/__tests__/setupWizard.test.tsx +++ b/nextjs-frontend/__tests__/setupWizard.test.tsx @@ -13,9 +13,10 @@ jest.mock("@/app/openapi-client", () => ({ saveNamespace: jest.fn(), saveLlm: jest.fn(), saveStorage: jest.fn(), + saveTelemetry: jest.fn(), finish: jest.fn(), }, - Admin: { testLlm: jest.fn() }, + Admin: { testLlm: jest.fn(), testTelemetry: jest.fn() }, })); jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); const replace = jest.fn(); @@ -38,6 +39,7 @@ const VIEW = (over: Partial = {}): SetupView => ({ max_upload_mb: 4096, watches: 0, }, + telemetry: { enabled: false, endpoint: null, locked: [] }, ...over, }); @@ -53,12 +55,13 @@ function wrap(ui: ReactNode) { beforeEach(() => jest.clearAllMocks()); describe("SetupWizard", () => { - it("walks through namespace, model provider and storage, then finishes", async () => { + it("walks through namespace, model provider, storage and telemetry, then finishes", async () => { m(Setup.getSetup).mockImplementation(() => ok(VIEW())); m(Setup.saveNamespace).mockImplementation(() => ok({ ok: true })); m(Setup.saveLlm).mockImplementation(() => ok({ ok: true })); m(Admin.testLlm).mockImplementation(() => ok({ ok: true, reply: "OK", ms: 120, model: "llama3" })); m(Setup.saveStorage).mockImplementation(() => ok({ ok: true })); + m(Setup.saveTelemetry).mockImplementation(() => ok({ ok: true })); m(Setup.finish).mockImplementation(() => ok({ ok: true })); wrap(); @@ -86,14 +89,42 @@ describe("SetupWizard", () => { }); fireEvent.change(await screen.findByLabelText(/Largest upload/), { target: { value: "2048" } }); + fireEvent.click(screen.getByRole("button", { name: "Save and continue" })); + await waitFor(() => expect(Setup.saveStorage).toHaveBeenCalled()); + expect(m(Setup.saveStorage).mock.calls[0][0].body).toEqual({ max_upload_mb: 2048, folder: null, namespace: null }); + + // telemetry is off unless chosen: finishing with it off sends nothing anywhere + expect(await screen.findByText(/off unless you turn it on/)).toBeInTheDocument(); + expect(screen.queryByLabelText("OTLP endpoint")).not.toBeInTheDocument(); fireEvent.click(screen.getByRole("button", { name: "Save and finish" })); await waitFor(() => expect(Setup.finish).toHaveBeenCalledWith(expect.objectContaining({ body: { skipped: false } })), ); - expect(m(Setup.saveStorage).mock.calls[0][0].body).toEqual({ max_upload_mb: 2048, folder: null, namespace: null }); + expect(m(Setup.saveTelemetry).mock.calls[0][0].body).toEqual({ enabled: false, endpoint: null }); await waitFor(() => expect(replace).toHaveBeenCalledWith("/")); }); + it("opts in to telemetry with an endpoint, after a test span", async () => { + m(Setup.getSetup).mockImplementation(() => ok(VIEW({ namespace: { existing: ["media"], locked: false } }))); + m(Setup.saveTelemetry).mockImplementation(() => ok({ ok: true })); + m(Admin.testTelemetry).mockImplementation(() => ok({ ok: true, ms: 12 })); + m(Setup.finish).mockImplementation(() => ok({ ok: true })); + wrap(); + fireEvent.click(await screen.findByRole("button", { name: /Telemetry/ })); + fireEvent.click(await screen.findByRole("radio", { name: "On" })); + const save = screen.getByRole("button", { name: "Save and finish" }); + expect(save).toBeDisabled(); // on needs somewhere to send to + fireEvent.change(screen.getByLabelText("OTLP endpoint"), { target: { value: "localhost:4318" } }); + expect(save).toBeDisabled(); + fireEvent.change(screen.getByLabelText("OTLP endpoint"), { target: { value: "http://localhost:4318" } }); + fireEvent.click(screen.getByRole("button", { name: "Send a test span" })); + expect(await screen.findByText(/took a test span in 12 ms/)).toBeInTheDocument(); + expect(m(Setup.saveTelemetry).mock.calls[0][0].body).toEqual({ enabled: false, endpoint: "http://localhost:4318" }); + fireEvent.click(save); + await waitFor(() => expect(Setup.finish).toHaveBeenCalled()); + expect(m(Setup.saveTelemetry).mock.calls[1][0].body).toEqual({ enabled: true, endpoint: "http://localhost:4318" }); + }); + it("shows what .env sets as locked", async () => { m(Setup.getSetup).mockImplementation(() => ok( diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 33fb4de2d..22b0c44e3 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -81,6 +81,10 @@ import type { UpdateSettingsErrors, TestLlmData, TestLlmResponses, + TelemetryStatusData, + TelemetryStatusResponses, + TestTelemetryData, + TestTelemetryResponses, ListAuditData, ListAuditResponses, ListAuditErrors, @@ -104,6 +108,9 @@ import type { SaveStorageData, SaveStorageResponses, SaveStorageErrors, + SaveTelemetryData, + SaveTelemetryResponses, + SaveTelemetryErrors, FinishData, FinishResponses, FinishErrors, @@ -1273,6 +1280,32 @@ export class Admin { }); } + /** + * Telemetry Status + * Whether telemetry is on, where it goes, and how the API process's last exports went. Off by default. + */ + public static telemetryStatus( + options?: Options, + ) { + return (options?.client ?? client).get({ + url: "/api/v1/settings/telemetry/status", + ...options, + }); + } + + /** + * Test Telemetry + * Send one test span to the saved endpoint now (on or off), to check the address and headers. + */ + public static testTelemetry( + options?: Options, + ) { + return (options?.client ?? client).post({ + url: "/api/v1/settings/telemetry/test", + ...options, + }); + } + /** * List Audit * Who changed what, newest first. @@ -1392,6 +1425,21 @@ export class Setup { }); } + /** + * Save Telemetry + * Opt in to telemetry: traces and metrics sent only to this endpoint (an OTLP collector). Off unless chosen. + */ + public static saveTelemetry(options: Options) { + return (options.client ?? client).put({ + url: "/api/v1/setup/telemetry", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + /** * Finish * Finish (or skip) the wizard; the web app stops showing it. Everything stays changeable in Settings. diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index cd214d622..1a7e74d02 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -9091,6 +9091,39 @@ export type SetupStorageView = { [key: string]: unknown | string | boolean | Array | number; }; +/** + * SetupTelemetry + */ +export type SetupTelemetry = { + /** + * Enabled + */ + enabled?: boolean; + /** + * Endpoint + */ + endpoint?: string | null; +}; + +/** + * SetupTelemetryView + */ +export type SetupTelemetryView = { + /** + * Enabled + */ + enabled: boolean; + /** + * Endpoint + */ + endpoint?: string | null; + /** + * Locked + */ + locked: Array; + [key: string]: unknown | boolean | string | null | Array | undefined; +}; + /** * SetupView */ @@ -9103,7 +9136,9 @@ export type SetupView = { namespace: SetupNamespaceView; llm: SetupLlmView; storage: SetupStorageView; - [key: string]: unknown | boolean | SetupAdmin | SetupNamespaceView | SetupLlmView | SetupStorageView; + telemetry: SetupTelemetryView; + [key: string]: + unknown | boolean | SetupAdmin | SetupNamespaceView | SetupLlmView | SetupStorageView | SetupTelemetryView; }; /** @@ -10046,6 +10081,70 @@ export type TagsChanged = { [key: string]: unknown | boolean | number | undefined; }; +/** + * TelemetryExport + */ +export type TelemetryExport = { + /** + * At + */ + at: number; + /** + * Ok + */ + ok: boolean; + /** + * Error + */ + error?: string | null; + [key: string]: unknown | number | boolean | string | null | undefined; +}; + +/** + * TelemetryStatus + * What the API process does with telemetry now; each worker follows the same settings on its own. + */ +export type TelemetryStatus = { + /** + * Enabled + */ + enabled: boolean; + /** + * Endpoint + */ + endpoint?: string | null; + /** + * Traces + */ + traces: boolean; + /** + * Metrics + */ + metrics: boolean; + last_traces?: TelemetryExport | null; + last_metrics?: TelemetryExport | null; + [key: string]: unknown | boolean | string | null | TelemetryExport | null | TelemetryExport | null | undefined; +}; + +/** + * TelemetryTestResult + */ +export type TelemetryTestResult = { + /** + * Ok + */ + ok: boolean; + /** + * Error + */ + error?: string | null; + /** + * Ms + */ + ms?: number | null; + [key: string]: unknown | boolean | string | null | number | null | undefined; +}; + /** * Template */ @@ -12303,6 +12402,38 @@ export type TestLlmResponses = { export type TestLlmResponse = TestLlmResponses[keyof TestLlmResponses]; +export type TelemetryStatusData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/telemetry/status"; +}; + +export type TelemetryStatusResponses = { + /** + * Successful Response + */ + 200: TelemetryStatus; +}; + +export type TelemetryStatusResponse = TelemetryStatusResponses[keyof TelemetryStatusResponses]; + +export type TestTelemetryData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/telemetry/test"; +}; + +export type TestTelemetryResponses = { + /** + * Successful Response + */ + 200: TelemetryTestResult; +}; + +export type TestTelemetryResponse = TestTelemetryResponses[keyof TestTelemetryResponses]; + export type ListAuditData = { body?: never; path?: never; @@ -12504,6 +12635,31 @@ export type SaveStorageResponses = { export type SaveStorageResponse = SaveStorageResponses[keyof SaveStorageResponses]; +export type SaveTelemetryData = { + body: SetupTelemetry; + path?: never; + query?: never; + url: "/api/v1/setup/telemetry"; +}; + +export type SaveTelemetryErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type SaveTelemetryError = SaveTelemetryErrors[keyof SaveTelemetryErrors]; + +export type SaveTelemetryResponses = { + /** + * Successful Response + */ + 200: SetupSaved; +}; + +export type SaveTelemetryResponse = SaveTelemetryResponses[keyof SaveTelemetryResponses]; + export type FinishData = { body: SetupFinish; path?: never; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d730ba670..585dec9c6 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -1,6 +1,6 @@ "use client"; -import { useMutation, useQueries } from "@tanstack/react-query"; +import { useMutation, useQueries, useQuery } from "@tanstack/react-query"; import { PlugZap, RefreshCw, Terminal } from "lucide-react"; import Link from "next/link"; import { useEffect, useRef, type ReactNode } from "react"; @@ -320,6 +320,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) {

); + case "telemetry": + return ; case "uploads": return ( <> @@ -414,6 +416,103 @@ function LlmBody({ ctx }: { ctx: BodyCtx }) { ); } +function exportLine(e: { at: number; ok: boolean; error?: string | null } | null | undefined, what: string) { + if (!e) return null; + const when = new Date(e.at * 1000).toLocaleTimeString(); + return e.ok ? `${what} last sent at ${when}` : `${what} failed at ${when}: ${e.error ?? "refused"}`; +} + +function TelemetryBody({ ctx }: { ctx: BodyCtx }) { + const client = useApiClient(); + const t = ctx.view.telemetry; + const saved = t?.values ?? {}; + const secret = (saved.headers ?? {}) as { set?: boolean }; + const headers = ctx.state("telemetry.headers"); + const locked = t?.locked ?? []; + const status = useQuery({ + queryKey: ["telemetry-status", t?.updated_at ?? null], + queryFn: () => data(Admin.telemetryStatus({ client })), + refetchInterval: 30_000, + }); + const test = useMutation({ mutationFn: () => data(Admin.testTelemetry({ client })) }); + const on = Boolean(saved.enabled); + const where = (saved.endpoint as string | null | undefined) ?? null; + const s = status.data; + const lines = [exportLine(s?.last_traces, "Traces"), exportLine(s?.last_metrics, "Metrics")].filter(Boolean); + const failed = Boolean((s?.last_traces && !s.last_traces.ok) || (s?.last_metrics && !s.last_metrics.ok)); + return ( + <> + {on && where ? ( + + {lines.length + ? lines.join(" · ") + : "Nothing sent from the server yet; the first export follows its first traced work."} + + ) : on ? ( + + Set the endpoint below; until then nothing is collected or sent. + + ) : ( + + Lens never sends telemetry anywhere unless you turn it on here, in the setup wizard or with{" "} + LENS_TELEMETRY=on in .env, and then only to the endpoint you set. + + )} + + + headers.onChange(x)} + /> +

+ Optional: key=value pairs, separated by commas and URL-encoded, for a collector that needs a token. +

+
+ + +
+
+ + + +
+ +

+ What is sent: route templates (never paths or queries), job steps and how they ended, record ids, model names, + token counts, durations and estimated cost. Never transcript, prompt or answer text, file names, titles, + namespace names, people or addresses. Each worker follows these settings on its own. +

+
+ + + {ctx.dirty + ? "Uses the saved settings, not your unsaved changes" + : "Sends one span named lens.telemetry.test, even while telemetry is off"} + +
+ {test.data && + (test.data.ok ? ( + + {test.data.ms} ms + + ) : ( + + {test.data.error} + + ))} + {test.isError && {test.error.message}} + + ); +} + function AiBody({ ctx }: { ctx: BodyCtx }) { const tools = ctx.state("ai.disabled_tools"); const off = (tools.value as string[]) ?? []; diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 9533c55d3..827cce18d 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -63,6 +63,7 @@ export type SectionId = | "workers" | "access" | "notifications" + | "telemetry" | "uploads" | "documents" | "tokens" @@ -152,6 +153,13 @@ export const SECTIONS: SectionSpec[] = [ description: "Where namespaces may send notifications (webhooks, Matterbridge, Slack, Discord), how often the notifier looks, and how often a failed send is tried again.", }, + { + id: "telemetry", + label: "Telemetry", + backend: ["telemetry"], + description: + "Off unless you turn it on. Traces and metrics from the server and its workers, sent only to an OpenTelemetry collector you run or choose; nothing goes anywhere else.", + }, { id: "uploads", label: "Uploads", @@ -912,6 +920,73 @@ export const FIELDS: FieldSpec[] = [ max: 20, hint: "A failed send waits 30 seconds, then four times longer each time, up to 6 hours", }, + // Telemetry (opt-in) + { + section: "telemetry", + key: "enabled", + label: "Send telemetry", + kind: "switch", + hint: "Off by default; while off, nothing is collected or sent", + }, + { + section: "telemetry", + key: "endpoint", + label: "OTLP endpoint", + kind: "text", + nullable: true, + mono: true, + placeholder: "http://localhost:4318", + hint: "An OpenTelemetry collector’s OTLP/HTTP address; traces go to /v1/traces and metrics to /v1/metrics under it", + }, + { section: "telemetry", key: "headers", label: "Headers", kind: "secret" }, + { + section: "telemetry", + key: "traces", + label: "Traces", + kind: "switch", + hint: "API requests, jobs and their steps, routines, workflows and model calls", + }, + { + section: "telemetry", + key: "metrics", + label: "Metrics", + kind: "switch", + hint: "Durations, job outcomes, model tokens and estimated cost", + }, + { + section: "telemetry", + key: "sample_ratio", + label: "Share of traces kept", + kind: "number", + min: 0, + max: 1, + hint: "1 keeps every trace, 0.1 one in ten", + }, + { + section: "telemetry", + key: "export_seconds", + label: "Send metrics every (seconds)", + kind: "int", + min: 5, + max: 3600, + }, + { + section: "telemetry", + key: "service_name", + label: "Service name", + kind: "text", + mono: true, + hint: "How this server shows in your tracing tool", + }, + { + section: "telemetry", + key: "prices", + label: "Model prices for cost estimates", + kind: "lines", + mono: true, + placeholder: "gpt-4o-mini 0.15 0.60", + hint: "One model per line: its name, then the input and output price in dollars per million tokens. The AI assistant’s prices count for the configured model", + }, { section: "tokens", key: "oauth_access_minutes", @@ -1120,6 +1195,7 @@ export function toUi(f: FieldSpec, v: unknown): unknown { case "checks": return Array.isArray(v) ? v.map(String) : []; case "lines": + if (f.section === "telemetry" && f.key === "prices") return pricesToLines(v); return Array.isArray(v) ? v.join("\n") : ""; case "days": return typeof v === "number" ? String(Math.round((v / 24) * 100) / 100) : ""; @@ -1132,6 +1208,33 @@ export function toUi(f: FieldSpec, v: unknown): unknown { export type Parsed = { value: unknown } | { error: string }; +export type Prices = Record; + +/** telemetry.prices as lines: "model input output". */ +export function pricesToLines(v: unknown): string { + if (!v || typeof v !== "object") return ""; + return Object.entries(v as Prices) + .map(([model, p]) => `${model} ${p?.input ?? 0} ${p?.output ?? 0}`) + .join("\n"); +} + +/** Lines of "model input output" (dollars per million tokens) back to telemetry.prices. */ +export function linesToPrices(text: string): Parsed { + const out: Prices = {}; + const lines = text + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + for (let i = 0; i < lines.length; i++) { + const parts = lines[i].split(/\s+/); + const nums = parts.slice(-2).map(Number); + if (parts.length < 3 || nums.some((n) => !Number.isFinite(n) || n < 0)) + return { error: `Line ${i + 1}: write the model, then its input and output price, like gpt-4o-mini 0.15 0.60` }; + out[parts.slice(0, -2).join(" ")] = { input: nums[0], output: nums[1] }; + } + return { value: out }; +} + /** Parses a control's value back to what the backend stores, checking it like the backend does. */ export function parse(f: FieldSpec, ui: unknown): Parsed { switch (f.kind) { @@ -1142,6 +1245,7 @@ export function parse(f: FieldSpec, ui: unknown): Parsed { case "checks": return { value: ui as string[] }; case "lines": + if (f.section === "telemetry" && f.key === "prices") return linesToPrices(String(ui ?? "")); return { value: String(ui ?? "") .split("\n") @@ -1249,6 +1353,12 @@ export function crossErrors(values: Record): Record !/^[0-9a-f.:]+(\/\d{1,3})?$/i.test(x)); if (badNet) e["notifications.networks"] = `“${badNet}” isn’t a network like 192.168.1.0/24`; + const otlp = values["telemetry.endpoint"] as string | null | undefined; + if (otlp && !URL_RX.test(otlp)) e["telemetry.endpoint"] = "Use an http(s) address, such as http://localhost:4318"; + else if (otlp && /\/v1\/(traces|metrics)\/?$/.test(otlp)) + e["telemetry.endpoint"] = "Use the collector’s base address, without /v1/traces or /v1/metrics"; + else if (values["telemetry.enabled"] === true && "telemetry.endpoint" in values && !otlp) + e["telemetry.endpoint"] = "Set the collector’s address to send telemetry to"; const rights = values["iiif.rights"] as string | null | undefined; if (rights && !RIGHTS_RX.test(rights)) e["iiif.rights"] = "Pick a Creative Commons licence or a RightsStatements.org statement"; @@ -1285,6 +1395,10 @@ export function show(f: FieldSpec, v: unknown): string { if (f.kind === "switch") return f.key === "cross_namespace" ? (v === "off" ? "off" : "on") : v ? "on" : "off"; if (f.kind === "days") return `${Math.round(((v as number) / 24) * 100) / 100} days`; if (f.key === "rights") return rightsShort(v as string); + if (f.section === "telemetry" && f.key === "prices") { + const n = Object.keys((v as Prices) ?? {}).length; + return n ? `${n} model${n === 1 ? "" : "s"}` : "none"; + } if (Array.isArray(v)) { if (f.kind === "checks" && f.section === "ai") return `${v.length} off`; return v.length > 3 ? `${v.length} ${f.kind === "lines" ? "lines" : "items"}` : v.join(", ") || "none"; @@ -1318,6 +1432,12 @@ export function why(c: Change): string | null { if (id === "transcribe.engine" || id === "transcribe.language") return "Applies to new transcriptions; existing transcripts stay until reprocessed."; if (id === "workers.inline") return "Takes effect when the server restarts."; + if (id === "telemetry.enabled") + return c.after + ? "The server and its workers start sending traces and metrics to the endpoint within a few seconds." + : "Nothing more is sent; what was sent stays with your collector."; + if (id === "telemetry.headers") + return c.after === "" ? "The headers are removed." : "Sent with every export; stored encrypted."; if (id === "llm.api_key") return c.after === "" ? "The key is removed; LLM steps and chat stop until a new one is set." diff --git a/nextjs-frontend/components/settings/section.tsx b/nextjs-frontend/components/settings/section.tsx index 67e448b99..60f314274 100644 --- a/nextjs-frontend/components/settings/section.tsx +++ b/nextjs-frontend/components/settings/section.tsx @@ -42,6 +42,9 @@ const ENV: Record = { "llm.model": "LENS_LLM_MODEL", "llm.api_key": "LENS_LLM_API_KEY", "llm.vision_model": "LENS_LLM_VISION_MODEL", + "telemetry.enabled": "LENS_TELEMETRY", + "telemetry.endpoint": "LENS_TELEMETRY_ENDPOINT", + "telemetry.headers": "LENS_TELEMETRY_HEADERS", }; const CONFIRM_BASE = "CHANGE ALL IDENTIFIERS"; diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx index 11671fbe6..e2fbc7ecb 100644 --- a/nextjs-frontend/components/settings/settings-shell.tsx +++ b/nextjs-frontend/components/settings/settings-shell.tsx @@ -2,6 +2,7 @@ import { useQuery } from "@tanstack/react-query"; import { + Activity, AudioLines, Bell, Bot, @@ -48,6 +49,7 @@ const ICON: Record = { workers: Cpu, access: ShieldCheck, notifications: Bell, + telemetry: Activity, uploads: Upload, documents: FileType, tokens: KeyRound, diff --git a/nextjs-frontend/components/setup/setup-wizard.tsx b/nextjs-frontend/components/setup/setup-wizard.tsx index 4538bfbdd..ea64c487e 100644 --- a/nextjs-frontend/components/setup/setup-wizard.tsx +++ b/nextjs-frontend/components/setup/setup-wizard.tsx @@ -5,7 +5,7 @@ import { Check, Lock } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useState, type ReactNode } from "react"; -import { Admin, Setup, type LlmTestResult, type SetupView } from "@/app/openapi-client"; +import { Admin, Setup, type LlmTestResult, type SetupView, type TelemetryTestResult } from "@/app/openapi-client"; import { AuthAlert, AuthBrand } from "@/components/auth/auth-card"; import { SegmentedChoice } from "@/components/settings/controls"; import { Button } from "@/components/ui/button"; @@ -20,6 +20,7 @@ const STEPS = [ { id: "namespace", label: "Namespace" }, { id: "llm", label: "Model provider" }, { id: "storage", label: "Storage" }, + { id: "telemetry", label: "Telemetry" }, ] as const; type StepId = (typeof STEPS)[number]["id"]; @@ -29,15 +30,17 @@ const LLM_ENV: Record = { model: "LENS_LLM_MODEL", api_key: "LENS_LLM_API_KEY", }; +const TELEMETRY_ENV: Record = { enabled: "LENS_TELEMETRY", endpoint: "LENS_TELEMETRY_ENDPOINT" }; const NS = /^[a-z0-9][a-z0-9_-]{0,40}$/; +const URL_RX = /^https?:\/\/[^\s]+$/; function errorText(e: unknown): string { return e instanceof ApiError ? e.message : "Something went wrong. Please try again."; } /** - * First-run setup for a fresh install (the admin already exists): the first namespace, the model provider and - * storage, one step at a time. Each step can be skipped, and the whole wizard too; everything stays in Settings. + * First-run setup for a fresh install (the admin already exists): the first namespace, the model provider, storage + * and whether to send telemetry (off unless chosen), one step at a time. Each step can be skipped, and the whole wizard too; everything stays in Settings. * Fields .env sets are shown locked, since the environment wins. */ export function SetupWizard() { @@ -96,7 +99,7 @@ export function SetupWizard() {

-
    +
      {STEPS.map((s, i) => { const on = s.id === step; const ok = done.has(s.id); @@ -138,8 +141,10 @@ export function SetupWizard() { next("namespace", saved)} /> ) : step === "llm" ? ( next("llm", saved)} /> + ) : step === "storage" ? ( + next("storage", saved)} /> ) : ( - next("storage", saved)} /> + next("telemetry", saved)} /> )} {finish.error && {errorText(finish.error)}} @@ -397,15 +402,7 @@ function LlmStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean) = ); } -function StorageStep({ - view, - pending, - onNext, -}: { - view: SetupView; - pending: boolean; - onNext: (saved: boolean) => void; -}) { +function StorageStep({ view, onNext }: { view: SetupView; onNext: (saved: boolean) => void }) { const client = useApiClient(); const s = view.storage; const [maxMb, setMaxMb] = useState(String(s.max_upload_mb)); @@ -516,10 +513,141 @@ function StorageStep({ onNext(false)} onSave={() => save.mutate()} - saveText="Save and finish" - pending={save.isPending || pending} + pending={save.isPending} disabled={!validMb || (Boolean(folder.trim()) && !namespace)} /> ); } + +function TelemetryStep({ + view, + pending, + onNext, +}: { + view: SetupView; + pending: boolean; + onNext: (saved: boolean) => void; +}) { + const client = useApiClient(); + const t = view.telemetry; + const locked = new Set(t.locked); + const [on, setOn] = useState(t.enabled ? "on" : "off"); + const [endpoint, setEndpoint] = useState(t.endpoint ?? ""); + const [tested, setTested] = useState(null); + useEffect(() => setTested(null), [endpoint]); + const body = () => ({ enabled: on === "on", endpoint: endpoint.trim() || null }); + const save = useMutation({ mutationFn: () => data(Setup.saveTelemetry({ client, body: body() })) }); + const test = useMutation({ + mutationFn: async () => { + // the test uses the saved address; saving it with telemetry off sends nothing else + await data(Setup.saveTelemetry({ client, body: { enabled: t.enabled, endpoint: endpoint.trim() } })); + return data(Admin.testTelemetry({ client })); + }, + onSuccess: setTested, + }); + const wantsEndpoint = on === "on" && !locked.has("endpoint"); + const badEndpoint = Boolean(endpoint.trim()) && !URL_RX.test(endpoint.trim()); + const missing = wantsEndpoint && !endpoint.trim(); + + return ( + <> + + Lens can send traces and metrics about its own work (requests, jobs, model calls with token counts and estimated + cost) to an OpenTelemetry collector you run, to see how it performs. It is off unless you turn it on, and + nothing is ever sent anywhere else. + + + ) : ( + "You can change this any time in Settings → Telemetry." + ) + } + > + {() => ( + undefined : setOn} + options={[ + { value: "off", label: "Off" }, + { value: "on", label: "On" }, + ]} + className="w-full max-w-[220px]" + /> + )} + + {on === "on" && ( + <> + + ) : ( + "Your collector's OTLP/HTTP address, for example http://localhost:4318" + ) + } + error={badEndpoint ? "Use an http(s) address" : undefined} + > + {({ id, describedBy, invalid }) => ( + setEndpoint(e.target.value)} + disabled={locked.has("endpoint")} + autoComplete="off" + spellCheck={false} + /> + )} + +

      + Sent: route templates, job steps, record ids, model names, token counts, durations and costs. Never + transcripts, prompts, file names, titles or people. +

      + + )} + {tested && + (tested.ok ? ( + The collector took a test span in {tested.ms} ms. + ) : ( + The collector didn't take the test span: {tested.error} + ))} + {(save.error || test.error) && {errorText(save.error || test.error)}} +
      + {on === "on" ? ( + + ) : ( + + )} +
      + + +
      +
      + + ); +} diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 0660efe19..61091cc14 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -1171,6 +1171,50 @@ } } }, + "/api/v1/settings/telemetry/status": { + "get": { + "tags": [ + "admin" + ], + "summary": "Telemetry Status", + "description": "Whether telemetry is on, where it goes, and how the API process's last exports went. Off by default.", + "operationId": "telemetry_status", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TelemetryStatus" + } + } + } + } + } + } + }, + "/api/v1/settings/telemetry/test": { + "post": { + "tags": [ + "admin" + ], + "summary": "Test Telemetry", + "description": "Send one test span to the saved endpoint now (on or off), to check the address and headers.", + "operationId": "test_telemetry", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TelemetryTestResult" + } + } + } + } + } + } + }, "/api/v1/audit": { "get": { "tags": [ @@ -1482,6 +1526,48 @@ } } }, + "/api/v1/setup/telemetry": { + "put": { + "tags": [ + "setup" + ], + "summary": "Save Telemetry", + "description": "Opt in to telemetry: traces and metrics sent only to this endpoint (an OTLP collector). Off unless chosen.", + "operationId": "save_telemetry", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetupTelemetry" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetupSaved" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/setup/finish": { "post": { "tags": [ @@ -29097,6 +29183,63 @@ ], "title": "SetupStorageView" }, + "SetupTelemetry": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "default": false + }, + "endpoint": { + "anyOf": [ + { + "type": "string", + "maxLength": 500 + }, + { + "type": "null" + } + ], + "title": "Endpoint" + } + }, + "additionalProperties": false, + "type": "object", + "title": "SetupTelemetry" + }, + "SetupTelemetryView": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled" + }, + "endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Endpoint" + }, + "locked": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Locked" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "locked" + ], + "title": "SetupTelemetryView" + }, "SetupView": { "properties": { "pending": { @@ -29114,6 +29257,9 @@ }, "storage": { "$ref": "#/components/schemas/SetupStorageView" + }, + "telemetry": { + "$ref": "#/components/schemas/SetupTelemetryView" } }, "additionalProperties": true, @@ -29123,7 +29269,8 @@ "admin", "namespace", "llm", - "storage" + "storage", + "telemetry" ], "title": "SetupView" }, @@ -30609,6 +30756,128 @@ ], "title": "TagsChanged" }, + "TelemetryExport": { + "properties": { + "at": { + "type": "number", + "title": "At" + }, + "ok": { + "type": "boolean", + "title": "Ok" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "at", + "ok" + ], + "title": "TelemetryExport" + }, + "TelemetryStatus": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled" + }, + "endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Endpoint" + }, + "traces": { + "type": "boolean", + "title": "Traces" + }, + "metrics": { + "type": "boolean", + "title": "Metrics" + }, + "last_traces": { + "anyOf": [ + { + "$ref": "#/components/schemas/TelemetryExport" + }, + { + "type": "null" + } + ] + }, + "last_metrics": { + "anyOf": [ + { + "$ref": "#/components/schemas/TelemetryExport" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "enabled", + "traces", + "metrics" + ], + "title": "TelemetryStatus", + "description": "What the API process does with telemetry now; each worker follows the same settings on its own." + }, + "TelemetryTestResult": { + "properties": { + "ok": { + "type": "boolean", + "title": "Ok" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ok" + ], + "title": "TelemetryTestResult" + }, "Template": { "properties": { "id": { diff --git a/packaging/qnap/qpkg/shared/docker-compose.yml b/packaging/qnap/qpkg/shared/docker-compose.yml index 2eab62b62..076b020b1 100644 --- a/packaging/qnap/qpkg/shared/docker-compose.yml +++ b/packaging/qnap/qpkg/shared/docker-compose.yml @@ -32,6 +32,10 @@ x-backend: &backend LENS_LLM_MODEL: ${LENS_LLM_MODEL:-} LENS_LLM_API_KEY: ${LENS_LLM_API_KEY:-} LENS_LLM_VISION_MODEL: ${LENS_LLM_VISION_MODEL:-} + # opt-in telemetry (docs/telemetry.md): unset, it stays off unless turned on in Settings + LENS_TELEMETRY: ${LENS_TELEMETRY:-} + LENS_TELEMETRY_ENDPOINT: ${LENS_TELEMETRY_ENDPOINT:-} + LENS_TELEMETRY_HEADERS: ${LENS_TELEMETRY_HEADERS:-} LENS_SETUP_WIZARD: ${LENS_SETUP_WIZARD:-} volumes: - ./archive.yaml:/qnap/archive.yaml:ro