Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions fastapi_backend/app/domain/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,7 @@ def control(db, name, action, by=None):
class Worker:
def __init__(self, db, cfg_fn, name=None, steps=None, log=None):
self.db, self.cfg_fn, self.log = db, cfg_fn, log
cfg_fn() # reads the saved settings now, so telemetry (if on) is set up before the first job's span
self.name = name or f"{socket.gethostname()}-{os.getpid()}"
self.can = set(steps or cfg_fn()["workers"]["steps"]) & set(STEPS)
if "llm" in self.can: # a workflow needs what an llm step needs; lists written before workflows existed run them too
Expand Down
29 changes: 29 additions & 0 deletions fastapi_backend/app/domain/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import re
import secrets
import threading
import urllib.parse

from . import convert, ipgroups, objects, store, telemetry

Expand Down Expand Up @@ -363,6 +364,21 @@ def _telemetry_setting(key, value):
v = value.strip().rstrip("/")
if v.endswith(("/v1/traces", "/v1/metrics")):
raise ValueError("telemetry.endpoint is the collector's base address, without /v1/traces or /v1/metrics")
try:
u = urllib.parse.urlsplit(v)
host = u.hostname or ""
u.port # noqa: B018 - ValueError for a port out of range
except ValueError:
raise ValueError("telemetry.endpoint isn't a valid address") from None
if u.username is not None or u.password is not None:
# the endpoint is stored and shown as it is; a token belongs in the sealed headers
raise ValueError("telemetry.endpoint can't carry a user name or password; put credentials in telemetry.headers")
try:
link_local = ipaddress.ip_address(host).is_link_local
except ValueError:
link_local = False
if link_local:
raise ValueError("telemetry.endpoint can't be a link-local address (169.254.0.0/16, fe80::/10)")
return v
if key == "sample_ratio":
if not (isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 1):
Expand All @@ -380,6 +396,14 @@ def _telemetry_setting(key, value):
return value


def _origin(url):
try:
u = urllib.parse.urlsplit(url or "")
return (u.scheme, u.hostname, u.port)
except ValueError:
return None


def _prices(value):
"""{model: {input, output}}: USD per million tokens, for the cost estimates."""
msg = "telemetry.prices gives each model its input and output price in USD per million tokens"
Expand Down Expand Up @@ -435,7 +459,12 @@ def save(db, base, section, changes, user=None):
continue
if k not in defaults:
raise ValueError(f"unknown setting {section}.{k}")
before = data.get(k, (base.get(section) or {}).get(k))
data[k] = _check(section, k, v, defaults[k])
if (section, k) == ("telemetry", "endpoint") and "headers" not in (changes or {}) and _origin(before) != _origin(data[k]):
# the saved headers (an auth token) were for the old collector: they don't go to a new host
if before:
sealed.pop("headers", None)
if section == "speakers":
m = data.get("match_threshold", defaults["match_threshold"])
r = data.get("review_threshold", defaults["review_threshold"])
Expand Down
28 changes: 28 additions & 0 deletions fastapi_backend/tests/api/test_telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ def put(body):
{"prices": {"gpt": {"input": 1, "cached": 1}}},
{"prices": ["gpt"]},
{"headers": "no-equals-sign"},
{"endpoint": "http://user:secret@collector:4318"},
{"endpoint": "http://token@collector:4318"},
{"endpoint": "http://169.254.169.254:4318"},
{"endpoint": "http://[fe80::1]:4318"},
{"endpoint": "http://collector:99999"},
{"colour": "blue"},
):
assert put(bad).status_code == 400, bad
Expand All @@ -87,6 +92,15 @@ def put(body):
assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"] == {"secret": True, "set": True}
assert put({"prices": {}}).status_code == 200

# the saved headers stay with their collector: a new path keeps them, a new host drops them
assert put({"endpoint": "http://collector:4318/otlp"}).status_code == 200
assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"]["set"] is True
assert put({"endpoint": "http://elsewhere:4318"}).status_code == 200
assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"]["set"] is False
# unless new ones come with the move
assert put({"endpoint": "http://third:4318", "headers": "X-Key=1"}).status_code == 200
assert client.get("/api/v1/settings", headers=h).json()["telemetry"]["values"]["headers"]["set"] is True


def test_traces_and_metrics_go_to_the_endpoint(client, db, cfg, folder, collector, model):
cfg["ai"]["tools"] = False # the plain, streamed answer
Expand Down Expand Up @@ -224,3 +238,17 @@ def test_cost_and_headers():
assert telemetry.parse_headers("") == {}
with pytest.raises(ValueError):
telemetry.parse_headers("bad header=1")


def test_a_new_workers_first_job_is_traced(client, db, cfg, folder, collector):
"""A worker reads the saved settings when it starts, so telemetry is on before its first job's span."""
from app.domain import settings

rid = seed(db, cfg, folder)[0]
settings.save(db, cfg, "telemetry", {"enabled": True, "endpoint": collector})
jobs.enqueue(db, rid, ["analyze"])
telemetry.shutdown() # as in a fresh worker process
worker = jobs.Worker(db, settings.Settings(db, cfg).current, name="w1", steps=["analyze"]) # lens worker --steps
assert worker.drain() == 1
telemetry.flush()
assert "job" in {s["name"] for s in fake_otlp.Handler.spans}
9 changes: 9 additions & 0 deletions nextjs-frontend/__tests__/settings-admin-models.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,15 @@ describe("settings fields", () => {
const on = { id: "telemetry.enabled", field: spec("telemetry.enabled"), before: false, after: true };
expect(why(on)).toMatch(/start sending/);
expect(why({ ...on, before: true, after: false })).toMatch(/Nothing more is sent/);
const moved = {
id: "telemetry.endpoint",
field: spec("telemetry.endpoint"),
before: "http://a:4318",
after: "http://b:4318",
};
expect(why(moved)).toMatch(/headers are removed/);
expect(why({ ...moved, after: "http://a:4318/otlp" })).toBeNull();
expect(why({ ...moved, before: null })).toBeNull();
});
});

Expand Down
11 changes: 11 additions & 0 deletions nextjs-frontend/components/settings/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1436,6 +1436,17 @@ export function why(c: Change): string | null {
return c.after
? "The server and its workers start sending traces and metrics to the endpoint within a few seconds."
: "Nothing more is sent; what was sent stays with your collector.";
if (id === "telemetry.endpoint" && c.before && c.after !== c.before) {
const origin = (u: unknown) => {
try {
return new URL(String(u)).origin;
} catch {
return String(u);
}
};
if (origin(c.before) !== origin(c.after))
return "Saved headers are removed, since they were for the old collector: enter them again if the new one needs them.";
}
if (id === "telemetry.headers")
return c.after === "" ? "The headers are removed." : "Sent with every export; stored encrypted.";
if (id === "llm.api_key")
Expand Down
Loading