From f0a606cf6088ef10be77f54a0d645b4e690b0305 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Fri, 2 Oct 2026 14:13:18 +0000 Subject: [PATCH] fix(telemetry): trace a worker's first job and keep collector credentials safe - A worker started with --steps now reads the saved settings at start, so telemetry is on before its first job's span - The endpoint can't carry user:password (it is stored and shown as is; credentials go in the sealed headers) or be a link-local address - Moving the endpoint to another host drops the saved headers, so a collector's token never goes to a new host; Settings says so on review Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CFYD76r8LUcnfTYKBwRfPn --- fastapi_backend/app/domain/jobs.py | 1 + fastapi_backend/app/domain/settings.py | 29 +++++++++++++++++++ fastapi_backend/tests/api/test_telemetry.py | 28 ++++++++++++++++++ .../__tests__/settings-admin-models.test.ts | 9 ++++++ nextjs-frontend/components/settings/model.ts | 11 +++++++ 5 files changed, 78 insertions(+) diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index e57df379b..53e9da681 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -705,6 +705,7 @@ def control(db, name, action, by=None): class Worker: def __init__(self, db, cfg_fn, name=None, steps=None, log=None): self.db, self.cfg_fn, self.log = db, cfg_fn, log + cfg_fn() # reads the saved settings now, so telemetry (if on) is set up before the first job's span self.name = name or f"{socket.gethostname()}-{os.getpid()}" self.can = set(steps or cfg_fn()["workers"]["steps"]) & set(STEPS) if "llm" in self.can: # a workflow needs what an llm step needs; lists written before workflows existed run them too diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index b766e1557..fd5d3cbb2 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -17,6 +17,7 @@ import re import secrets import threading +import urllib.parse from . import convert, ipgroups, objects, store, telemetry @@ -363,6 +364,21 @@ def _telemetry_setting(key, value): v = value.strip().rstrip("/") if v.endswith(("/v1/traces", "/v1/metrics")): raise ValueError("telemetry.endpoint is the collector's base address, without /v1/traces or /v1/metrics") + try: + u = urllib.parse.urlsplit(v) + host = u.hostname or "" + u.port # noqa: B018 - ValueError for a port out of range + except ValueError: + raise ValueError("telemetry.endpoint isn't a valid address") from None + if u.username is not None or u.password is not None: + # the endpoint is stored and shown as it is; a token belongs in the sealed headers + raise ValueError("telemetry.endpoint can't carry a user name or password; put credentials in telemetry.headers") + try: + link_local = ipaddress.ip_address(host).is_link_local + except ValueError: + link_local = False + if link_local: + raise ValueError("telemetry.endpoint can't be a link-local address (169.254.0.0/16, fe80::/10)") return v if key == "sample_ratio": if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 1): @@ -380,6 +396,14 @@ def _telemetry_setting(key, value): return value +def _origin(url): + try: + u = urllib.parse.urlsplit(url or "") + return (u.scheme, u.hostname, u.port) + except ValueError: + return None + + def _prices(value): """{model: {input, output}}: USD per million tokens, for the cost estimates.""" msg = "telemetry.prices gives each model its input and output price in USD per million tokens" @@ -435,7 +459,12 @@ def save(db, base, section, changes, user=None): continue if k not in defaults: raise ValueError(f"unknown setting {section}.{k}") + before = data.get(k, (base.get(section) or {}).get(k)) data[k] = _check(section, k, v, defaults[k]) + if (section, k) == ("telemetry", "endpoint") and "headers" not in (changes or {}) and _origin(before) != _origin(data[k]): + # the saved headers (an auth token) were for the old collector: they don't go to a new host + if before: + sealed.pop("headers", None) if section == "speakers": m = data.get("match_threshold", defaults["match_threshold"]) r = data.get("review_threshold", defaults["review_threshold"]) diff --git a/fastapi_backend/tests/api/test_telemetry.py b/fastapi_backend/tests/api/test_telemetry.py index e4860c5bd..b85679d51 100644 --- a/fastapi_backend/tests/api/test_telemetry.py +++ b/fastapi_backend/tests/api/test_telemetry.py @@ -77,6 +77,11 @@ def put(body): {"prices": {"gpt": {"input": 1, "cached": 1}}}, {"prices": ["gpt"]}, {"headers": "no-equals-sign"}, + {"endpoint": "http://user:secret@collector:4318"}, + {"endpoint": "http://token@collector:4318"}, + {"endpoint": "http://169.254.169.254:4318"}, + {"endpoint": "http://[fe80::1]:4318"}, + {"endpoint": "http://collector:99999"}, {"colour": "blue"}, ): assert put(bad).status_code == 400, bad @@ -87,6 +92,15 @@ def put(body): assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"] == {"secret": True, "set": True} assert put({"prices": {}}).status_code == 200 + # the saved headers stay with their collector: a new path keeps them, a new host drops them + assert put({"endpoint": "http://collector:4318/otlp"}).status_code == 200 + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"]["set"] is True + assert put({"endpoint": "http://elsewhere:4318"}).status_code == 200 + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"]["set"] is False + # unless new ones come with the move + assert put({"endpoint": "http://third:4318", "headers": "X-Key=1"}).status_code == 200 + assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"]["set"] is True + def test_traces_and_metrics_go_to_the_endpoint(client, db, cfg, folder, collector, model): cfg["ai"]["tools"] = False # the plain, streamed answer @@ -224,3 +238,17 @@ def test_cost_and_headers(): assert telemetry.parse_headers("") == {} with pytest.raises(ValueError): telemetry.parse_headers("bad header=1") + + +def test_a_new_workers_first_job_is_traced(client, db, cfg, folder, collector): + """A worker reads the saved settings when it starts, so telemetry is on before its first job's span.""" + from app.domain import settings + + rid = seed(db, cfg, folder)[0] + settings.save(db, cfg, "telemetry", {"enabled": True, "endpoint": collector}) + jobs.enqueue(db, rid, ["analyze"]) + telemetry.shutdown() # as in a fresh worker process + worker = jobs.Worker(db, settings.Settings(db, cfg).current, name="w1", steps=["analyze"]) # lens worker --steps + assert worker.drain() == 1 + telemetry.flush() + assert "job" in {s["name"] for s in fake_otlp.Handler.spans} diff --git a/nextjs-frontend/__tests__/settings-admin-models.test.ts b/nextjs-frontend/__tests__/settings-admin-models.test.ts index b0ba60e11..942b753d2 100644 --- a/nextjs-frontend/__tests__/settings-admin-models.test.ts +++ b/nextjs-frontend/__tests__/settings-admin-models.test.ts @@ -221,6 +221,15 @@ describe("settings fields", () => { const on = { id: "telemetry.enabled", field: spec("telemetry.enabled"), before: false, after: true }; expect(why(on)).toMatch(/start sending/); expect(why({ ...on, before: true, after: false })).toMatch(/Nothing more is sent/); + const moved = { + id: "telemetry.endpoint", + field: spec("telemetry.endpoint"), + before: "http://a:4318", + after: "http://b:4318", + }; + expect(why(moved)).toMatch(/headers are removed/); + expect(why({ ...moved, after: "http://a:4318/otlp" })).toBeNull(); + expect(why({ ...moved, before: null })).toBeNull(); }); }); diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 827cce18d..0549a2078 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -1436,6 +1436,17 @@ export function why(c: Change): string | null { return c.after ? "The server and its workers start sending traces and metrics to the endpoint within a few seconds." : "Nothing more is sent; what was sent stays with your collector."; + if (id === "telemetry.endpoint" && c.before && c.after !== c.before) { + const origin = (u: unknown) => { + try { + return new URL(String(u)).origin; + } catch { + return String(u); + } + }; + if (origin(c.before) !== origin(c.after)) + return "Saved headers are removed, since they were for the old collector: enter them again if the new one needs them."; + } if (id === "telemetry.headers") return c.after === "" ? "The headers are removed." : "Sent with every export; stored encrypted."; if (id === "llm.api_key")