diff --git a/CHANGELOG.md b/CHANGELOG.md index 90116938e..a94986c05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,6 +143,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version - bump the python-minor group ([#253](https://github.com/adeelahmad/lense/pull/253)) - bump the javascript-minor group ([#316](https://github.com/adeelahmad/lense/pull/316)) +- **Logs in the web app, and to a syslog server.** The API and every worker now write structured log lines with a + level set in **Settings → Logs** (and per logger, like `httpx warning`), to the console as text or JSON and to a file + per process under `data_dir/logs`. The same page shows the latest lines from all of them, with search and filters. + Turned on there, lines go to a syslog server too (RFC 5424 over UDP, TCP or TLS), from a background queue so a slow + server never holds Lens up; off by default. Keys, tokens and passwords are replaced with *** before any line is + written or sent ([Logging](docs/logging.md)). + ## 0.7.0 October 10, 2026 {id="0.7.0"} - **Prometheus metrics.** Lens can serve `/metrics` for Prometheus and Grafana: jobs by status and pipeline, imports, diff --git a/docs/configuration.md b/docs/configuration.md index ca5c1769e..198da1c5d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,6 +34,7 @@ with AES-GCM and are write-only: the API reports whether one is set, never its v | `LENS_SETUP_SETTINGS` | | settings to start with, as JSON `{"section": {"key": value}}` (or that JSON base64url-encoded): saved once at startup as if made in the app, so unlike the variables above they stay editable there. The [web installer](get-started.md#the-web-installer) writes its answers here; a `budget` section (`hard_usd`, `soft_usd`, `period`, `on_over`, `price_in`, `price_out`) sets [the overall budget](budgets.md#one-budget-for-everything) | | `LENS_SETUP_WIZARD` | | `off`: never show the setup wizard | | `LENS_TELEMETRY` / `LENS_TELEMETRY_ENDPOINT` / `LENS_TELEMETRY_HEADERS` | off | opt-in OpenTelemetry traces and metrics, sent only to this OTLP/HTTP endpoint ([Telemetry](telemetry.md)); `LENS_TELEMETRY=off` keeps it off whatever Settings say | +| `LENS_LOG_LEVEL` / `LENS_LOG_FORMAT` | info / text | the log level and console format for the API and every worker ([Logging](logging.md)); win over Settings → Logs | | `FRONTEND_URL` | `http://localhost:3000` | the web app's address as people use it: links in emails, and where apps send people to sign in ([OAuth](authentication.md#oauth)) | | `CORS_ORIGINS` | `["http://localhost:3000"]` | origins allowed to call the API from a browser | | `OPENAPI_URL` | `/openapi.json` | `""` disables `/docs` and the schema | diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 000000000..21ff6088b --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,60 @@ +# Logging + +Every Lens process (the API, each `lens worker`, `lens watch`) writes one structured record per log line, and sends it +to up to three places: + +- **The console**, which is what `docker logs` and `docker compose logs` show: as text, or as one JSON object per line + for a log shipper (**Format**). +- **A file per process** under `data_dir/logs` (`api-.jsonl`, `worker-.jsonl`, …), one JSON object per + line, rotated at **File size** and kept **Files kept** times. **Settings → Logs** reads these, so its viewer shows the + workers' lines as well as the API's. A file no process has written for 14 days (an old container's) is removed. +- **A syslog server**, off by default (below). + +Change any of it in **Settings → Logs**; the API and every worker follow within a few seconds, without a restart. +`LENS_LOG_LEVEL` and `LENS_LOG_FORMAT` in .env win over Settings, which then show those fields locked. + +## What a record holds + +```json +{"ts": "2026-10-10T20:02:19.809Z", "level": "warning", "logger": "lens.email", "msg": "…", + "process": "worker", "pid": 42, "host": "lens-worker-1", "exc": "Traceback …", "fields": {"job": 7}} +``` + +`exc` is there when the line carries an error's traceback, and `fields` when the code logged extra values. + +**No secrets.** Before a line goes anywhere it is redacted: bearer and basic credentials, `key=value` pairs whose key +names a key, token, secret, password, cookie or authorization, the password in a URL (`https://user:***@host`), and +well-known key shapes (`sk-…`, `ghp_…`, `xox…-`, JSON web tokens) become `***`. Extra fields whose name says key, token, +secret or password are dropped to `***` too. + +## Levels + +**Level** (info by default) is what is kept on the console and in the files. **Logger levels** gives single loggers +their own, one per line, like `httpx warning` to quiet every outgoing request, or `lens.bridge debug` to follow the +chat-room bridge closely. Lens's own loggers start with `lens`; the web server's access log is `uvicorn.access`. + +## Sending to a syslog server + +Turn on **Send to syslog** and set the server's **Host** and **Port**. Lens speaks +[RFC 5424](https://www.rfc-editor.org/rfc/rfc5424) over: + +| Transport | Standard | Framing | +| --- | --- | --- | +| UDP (default, port 514) | RFC 5426 | one line per datagram | +| TCP | RFC 6587 | octet counting (` `), or one line per newline for servers that want that | +| TLS (usually port 6514) | RFC 5425 | as TCP; the server's certificate is checked unless **Verify the certificate** is off | + +Each message is `1 TIMESTAMP HOST APP-NAME PROCESS.PID LOGGER - MESSAGE`, where PRI combines **Facility** +(local0 by default) with the line's level, APP-NAME is **App name** (`lens`), and MESSAGE is the line (and its +traceback) or, with **Message format** JSON, the whole record above. Only lines at **Syslog level** or above are sent. +Messages longer than 8 KB are cut. + +Lines are queued and sent from a background thread, so a slow or unreachable server never holds up Lens. While the +server can't be reached Lens retries, waiting up to 30 seconds between tries, and once 10,000 lines are waiting new +ones are dropped and counted. **Send a test line** sends one message to the saved server, even while sending is off, +and Settings → Logs shows how many lines the API has sent and dropped and the last error. + +rsyslog, syslog-ng, Graylog, Grafana Loki (through promtail or Alloy's syslog receiver), Splunk, Papertrail and most +log tools take RFC 5424 syslog. + +Not yet built: sending straight to Loki or another HTTP endpoint, and client certificates for TLS. diff --git a/fastapi_backend/.env.example b/fastapi_backend/.env.example index fb9b937fd..0a85ee2c9 100644 --- a/fastapi_backend/.env.example +++ b/fastapi_backend/.env.example @@ -57,5 +57,9 @@ CORS_ORIGINS=["http://localhost:3000"] # LENS_TELEMETRY_ENDPOINT=http://localhost:4318 # your OpenTelemetry collector's OTLP/HTTP address # LENS_TELEMETRY_HEADERS= # optional: key=value pairs, comma-separated, URL-encoded +# --- Logging (docs/logging.md): level and console format for the API and every worker; win over Settings → Logs --- +# LENS_LOG_LEVEL=info # debug, info, warning, error or critical +# LENS_LOG_FORMAT=text # text, or json for one JSON object per line (for a log shipper) + # Uncomment to disable /docs and /openapi.json # OPENAPI_URL="" diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index 5da9e1c3c..8fab4347f 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -6,7 +6,7 @@ import pathlib import shutil import time -from typing import Any +from typing import Any, Literal from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Request @@ -24,6 +24,7 @@ keyring, llm, local_llm, + logs, machine, mailbox, matterbridge, @@ -57,6 +58,9 @@ IndexQueued, LlmTestResult, LocalLlmStatus, + LogLine, + LogStatus, + LogTestResult, MailboxStatus, MailTestResult, NfsStatus, @@ -363,6 +367,36 @@ def test_telemetry(user: AdminWriter, cfg: Cfg) -> TelemetryTestResult: return TelemetryTestResult(ok=ok, error=error, ms=ms) +@router.get("/settings/logs/status") +def log_status(user: AdminReader, cfg: Cfg) -> LogStatus: + """The API process's log level, its log file, and how sending to syslog is going (off by default).""" + return LogStatus.model_validate(logs.status(cfg)) + + +@router.post("/settings/logs/test") +def test_logs(user: AdminWriter, cfg: Cfg) -> LogTestResult: + """Send one line to the saved syslog server now (on or off), to check the address, port and transport.""" + ok, error, ms = logs.test_send(cfg) + return LogTestResult(ok=ok, error=error, ms=ms) + + +@router.get("/admin/logs") +def read_logs( + user: AdminReader, + cfg: Cfg, + limit: int = Query(200, ge=1, le=2000), + level: Literal["debug", "info", "warning", "error", "critical"] | None = Query(None, description="this level and above"), + q: str | None = Query(None, max_length=200, description="text the message or error contains"), + process: str | None = Query(None, max_length=40, description="api, worker or watcher"), + logger: str | None = Query(None, max_length=100, description="a logger name, or the start of one (lens.email)"), + before: str | None = Query(None, max_length=40, description="a line's ts: only older lines (the next page)"), +) -> list[LogLine]: + """The latest log lines from the API and every worker (their files under data_dir/logs), newest first. Lines are + redacted as they're written: no keys, tokens or passwords.""" + rows = logs.read(cfg, limit, level, q, process, logger, before) + return [LogLine.model_validate(r) for r in rows] + + @router.get("/audit") def list_audit(user: AdminReader, db: Db, limit: int = Query(200, ge=1, le=1000)) -> list[AuditEntry]: """Who changed what, newest first.""" diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index 629b2df12..6f74804da 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -266,6 +266,7 @@ def platform_main(argv, config): https, jobs, local_llm, + logs, nfs_share, notify, passkeys, @@ -372,24 +373,26 @@ def password(): print(f"password changed for {a.email}") elif a.cmd == "worker": telemetry.set_role("worker") - wk = jobs.Worker(db, C, a.name, a.steps.split(",") if a.steps else None, log=print) + logs.setup("worker") # every line as a record: the console, the log file and syslog (docs/logging.md) + say = logs.printer("lens.worker") + wk = jobs.Worker(db, C, a.name, a.steps.split(",") if a.steps else None, log=say) if a.once: print(f"ran {wk.drain()} job(s)") else: print(f"worker {wk.name} runs {', '.join(sorted(wk.can))}; Ctrl-C to stop") stop = threading.Event() - notify.start(db, C, stop, name=wk.name, log=print) # sends notifications too (docs/notifications.md) + notify.start(db, C, stop, name=wk.name, log=say) # sends notifications too (docs/notifications.md) if not (a.steps or a.no_schedule): # and scans watched folders and runs routines, as `lens watch` does - routines.start(db, C, stop, log=print) - sensors.start(db, C, stop, log=print, name=wk.name) # and the sensor hub, while sensors are on - smb_share.start(db, C, stop, name=wk.name, log=print) # and the SMB share of the file tree, while it's on - shares.start(db, C, stop, log=print, name=wk.name) # and the file shares (S3, WebDAV), while they're on - channels.start_all(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat channels, while they're on - tunnel.start(db, C, stop, name=wk.name, log=print) # and the Cloudflare tunnel, while it's on - https.start(db, C, stop, name=wk.name, log=print) # and HTTPS, where the ports are (LENS_HTTPS_LISTEN) - local_llm.start(db, C, stop, name=wk.name, log=print) # and the local chat model, while it's on - nfs_share.start(db, C, stop, name=wk.name, log=print) # and the file tree over NFS, while it's on - updates.start(db, C, stop, log=print) # and the daily check for a new release (docs/updates.md) + routines.start(db, C, stop, log=say) + sensors.start(db, C, stop, log=say, name=wk.name) # and the sensor hub, while sensors are on + smb_share.start(db, C, stop, name=wk.name, log=say) # and the SMB share of the file tree, while it's on + shares.start(db, C, stop, log=say, name=wk.name) # and the file shares (S3, WebDAV), while they're on + channels.start_all(db, C, stop, lambda: cfg, name=wk.name, log_fn=say) # and the chat channels, while they're on + tunnel.start(db, C, stop, name=wk.name, log=say) # and the Cloudflare tunnel, while it's on + https.start(db, C, stop, name=wk.name, log=say) # and HTTPS, where the ports are (LENS_HTTPS_LISTEN) + local_llm.start(db, C, stop, name=wk.name, log=say) # and the local chat model, while it's on + nfs_share.start(db, C, stop, name=wk.name, log=say) # and the file tree over NFS, while it's on + updates.start(db, C, stop, log=say) # and the daily check for a new release (docs/updates.md) _stop_on_term() try: wk.loop(stop) @@ -401,13 +404,15 @@ def password(): print(f"ran {routines.run_due(db, C(), print)} routine(s)") else: telemetry.set_role("watcher") + logs.setup("watcher") + say = logs.printer("lens.watch") print("watching storage sources and running routines; Ctrl-C to stop") _stop_on_term() - sensors.start(db, C, threading.Event(), log=print, name=f"watch-{os.getpid()}") + sensors.start(db, C, threading.Event(), log=say, name=f"watch-{os.getpid()}") try: while True: - sources.poll_due(db, C(), print) - routines.run_due(db, C(), print) + sources.poll_due(db, C(), say) + routines.run_due(db, C(), say) time.sleep(max(5, min(C()["sources"]["check_seconds"], routines.CHECK_SECONDS))) except KeyboardInterrupt: pass diff --git a/fastapi_backend/app/domain/logs.py b/fastapi_backend/app/domain/logs.py new file mode 100644 index 000000000..5032b9641 --- /dev/null +++ b/fastapi_backend/app/domain/logs.py @@ -0,0 +1,577 @@ +"""Logging: one structured record for every line the API and each worker write, kept on this machine, shown in the web +app, and sent to a syslog server only when an admin turns that on (docs/logging.md). + +Every process (the API, each `lens worker`, `lens watch`) logs through Python's logging with three outputs: + +* the console (docker logs), as text or as one JSON object per line (logs.format); +* a file per process under data_dir/logs, one JSON object per line, rotated at logs.file_mb and kept logs.file_keep + times; the web app's log viewer reads these (`read`), so it shows the workers' lines too; +* a syslog server (RFC 5424 over UDP, TCP or TLS), off by default: lines go through a queue to a background thread, so + a slow or missing server never holds up a request; when the queue is full new lines are dropped and counted. + +Before any line leaves the process it is redacted: bearer and basic credentials, key=value pairs naming a key, token, +secret or password, the password in a URL, and well-known key shapes (sk-…, ghp_…, xox…-, JWTs) become ***. + +Settings are re-read when someone saves them (settings.Settings.current calls `apply`), so a change applies to the API +and every worker without a restart. LENS_LOG_LEVEL and LENS_LOG_FORMAT in .env win over the app. +""" + +from __future__ import annotations + +import contextlib +import datetime +import ipaddress +import json +import logging +import logging.handlers +import os +import pathlib +import queue +import re +import socket +import ssl +import sys +import threading +import time + +from . import __version__ + +LEVELS = {"debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR, "critical": logging.CRITICAL} +NAMES = {v: k for k, v in LEVELS.items()} +# RFC 5424 severities +SEVERITY = {logging.DEBUG: 7, logging.INFO: 6, logging.WARNING: 4, logging.ERROR: 3, logging.CRITICAL: 2} +FACILITIES = {"user": 1, "daemon": 3, "auth": 4, "syslog": 5, **{f"local{i}": 16 + i for i in range(8)}} +TRANSPORTS = ("udp", "tcp", "tls") +MAX_BYTES = 8192 # the most of one line sent to syslog (rsyslog's default maxMessageSize) +QUEUE = 10_000 # lines waiting for the syslog server before new ones are dropped +PRUNE_DAYS = 14 # a log file no process has written for this long (an old container's) is removed +LOGGER_RX = re.compile(r"^[A-Za-z0-9_.-]{1,100}$") +_STD = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime", "lens"} + +# what is never written out: the match keeps its first group and the rest becomes *** +_SECRETS = [ + re.compile(r"(?i)\b((?:bearer|basic)\s+)[A-Za-z0-9._~+/=-]{6,}"), + re.compile( + r"(?i)\b((?:api[_-]?key|apikey|access[_-]?token|refresh[_-]?token|id[_-]?token|auth[_-]?token|token|secret|" + r"client[_-]?secret|password|passwd|pwd|authorization|cookie|set-cookie|x-api-key)[\"']?\s*[:=]\s*[\"']?)" + r"(?!\*\*\*)[^\s\"'&,;}]+" + ), + re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://[^/\s:@]+:)[^/\s@]+(?=@)"), + re.compile(r"()\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{10,})"), + re.compile(r"()\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), +] +_SECRET_KEY = re.compile(r"(?i)(key|token|secret|password|passwd|authorization|cookie)") + + +def redact(text): + """The text with credentials replaced by ***.""" + if not text: + return text + for rx in _SECRETS: + text = rx.sub(lambda m: m.group(1) + "***", text) + return text + + +_role = ["api"] +_lock = threading.Lock() +_state = {"key": None, "cfg": None, "console": None, "file": None, "syslog": None, "levels": {}} +_host = socket.gethostname() + + +def _clean(value, limit=4000): + if isinstance(value, (bool, int, float)) or value is None: + return value + return redact(str(value))[:limit] + + +def record(r): + """A log record as the dict written to the file and sent as JSON: redacted once, then shared by every output.""" + got = getattr(r, "lens", None) + if got is not None: + return got + try: + msg = r.getMessage() + except Exception: # noqa: BLE001 - a bad format string still logs something + msg = f"{r.msg} {r.args}" + out = { + "ts": datetime.datetime.fromtimestamp(r.created, datetime.UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z"), + "level": NAMES.get(r.levelno, r.levelname.lower()), + "logger": r.name, + "msg": redact(msg), + "process": _role[0], + "pid": r.process, + "host": _host, + } + if r.exc_info and r.exc_info[0] is not None: + out["exc"] = redact(logging.Formatter().formatException(r.exc_info)) + elif r.exc_text: + out["exc"] = redact(r.exc_text) + fields = {k: ("***" if _SECRET_KEY.search(k) else _clean(v)) for k, v in r.__dict__.items() if k not in _STD and not k.startswith("_")} + if fields: + out["fields"] = fields + r.lens = out + return out + + +class JsonFormatter(logging.Formatter): + def format(self, r): + return json.dumps(record(r), ensure_ascii=False, default=str) + + +class TextFormatter(logging.Formatter): + def format(self, r): + d = record(r) + line = f"{d['ts']} {d['level'].upper()} {d['process']} {d['logger']}: {d['msg']}" + if d.get("fields"): + line += " " + " ".join(f"{k}={v}" for k, v in d["fields"].items()) + return line + ("\n" + d["exc"] if d.get("exc") else "") + + +_gate = {"level": logging.INFO, "levels": {}} + + +def _override(name): + """The level an admin gave this logger or the nearest of its parents (logs.levels), or None.""" + levels = _gate["levels"] + while name: + if name in levels: + return levels[name] + name = name.rpartition(".")[0] + return None + + +class Gate(logging.Filter): + """What an output keeps: the logger's own level when it has one, else logs.level (the console and the file); for + syslog, syslog_level too.""" + + def __init__(self, floor=None): + super().__init__() + self.floor = floor + + def filter(self, r): + own = _override(r.name) + if self.floor is not None: + return r.levelno >= self.floor and (own is None or r.levelno >= own) + return r.levelno >= (own if own is not None else _gate["level"]) + + +def _ours(h): + return getattr(h, "_lens", False) + + +def _tag(h): + h._lens = True + return h + + +def set_role(role): + """What this process is (api, worker, watcher): each record says it, and it names the process's log file.""" + _role[0] = role + + +def _env_level(): + return LEVELS.get((os.environ.get("LENS_LOG_LEVEL") or "").strip().lower(), logging.INFO) + + +def _env_format(): + f = (os.environ.get("LENS_LOG_FORMAT") or "").strip().lower() + return f if f in ("text", "json") else "text" + + +def setup(role=None): + """Log to the console now, before the settings are read (called once per process as it starts).""" + again = bool(role) and role != _role[0] and _state["cfg"] is not None + if role: + set_role(role) + root = logging.getLogger() + with _lock: + if _state["console"] is None: + h = _tag(logging.StreamHandler(sys.stderr)) + h.addFilter(Gate()) + root.addHandler(h) + _state["console"] = h + _state["console"].setFormatter(JsonFormatter() if _env_format() == "json" else TextFormatter()) + if _state["key"] is None: + _gate["level"] = _env_level() + root.setLevel(_env_level()) + # uvicorn's own lines (the access log, startup and errors) go through these outputs too, in the same form + for name in ("uvicorn", "uvicorn.error", "uvicorn.access"): + lg = logging.getLogger(name) + lg.handlers = [h for h in lg.handlers if _ours(h)] + lg.propagate = True + if again: # the settings were read before this process knew what it is: its file is named for it now + _close("file") + _state["key"] = None + apply(_state["cfg"]) + + +def printer(name="lens"): + """A `log=print` replacement for the long-running loops (the worker, the watcher): their lines become records.""" + lg = logging.getLogger(name) + + def log(*args, **_): + lg.info(" ".join(str(a) for a in args)) + + return log + + +def section(cfg): + return (cfg or {}).get("logs") or {} + + +def level(name, default=logging.INFO): + return LEVELS.get(str(name or "").lower(), default) + + +def _wanted(cfg): + s = section(cfg) + data_dir = (cfg or {}).get("data_dir") + ship = None + if s.get("syslog") and s.get("syslog_host"): + ship = ( + str(s["syslog_host"]), + int(s.get("syslog_port") or 514), + s.get("syslog_transport") if s.get("syslog_transport") in TRANSPORTS else "udp", + level(s.get("syslog_level")), + FACILITIES.get(s.get("syslog_facility"), FACILITIES["local0"]), + s.get("syslog_format") if s.get("syslog_format") in ("text", "json") else "text", + s.get("syslog_framing") if s.get("syslog_framing") in ("octet", "newline") else "octet", + s.get("syslog_verify") is not False, + s.get("syslog_app_name") or "lens", + ) + levels = s.get("levels") if isinstance(s.get("levels"), dict) else {} + return ( + level(s.get("level")), + s.get("format") if s.get("format") in ("text", "json") else "text", + tuple(sorted((k, level(v)) for k, v in levels.items() if isinstance(k, str) and LOGGER_RX.match(k))), + (str(pathlib.Path(data_dir) / "logs"), int(s.get("file_mb") or 5), int(s.get("file_keep") or 2)) + if data_dir and s.get("file", True) + else None, + ship, + ) + + +def file_name(role=None): + host = re.sub(r"[^A-Za-z0-9_.-]", "-", _host)[:64] or "host" + return f"{role or _role[0]}-{host}.jsonl" + + +def apply(cfg): + """Make this process's logging match the settings. Cheap when nothing changed.""" + want = _wanted(cfg) + _state["cfg"] = cfg + if want == _state["key"]: + return + with _lock: + if want == _state["key"]: + return + lvl, fmt, levels, file, ship = want + root = logging.getLogger() + if _state["console"] is None: + _state["console"] = _tag(logging.StreamHandler(sys.stderr)) + _state["console"].addFilter(Gate()) + root.addHandler(_state["console"]) + _state["console"].setFormatter(JsonFormatter() if fmt == "json" else TextFormatter()) + _gate["level"], _gate["levels"] = lvl, dict(levels) + old = _state["key"] + if file != (old[3] if old else None): + _close("file") + if file is not None: + folder, mb, keep = file + try: + pathlib.Path(folder).mkdir(parents=True, exist_ok=True) + _prune(folder) + h = _tag( + logging.handlers.RotatingFileHandler( + pathlib.Path(folder) / file_name(), maxBytes=mb * 1024 * 1024, backupCount=keep, encoding="utf-8" + ) + ) + h.setFormatter(JsonFormatter()) + h.addFilter(Gate()) + root.addHandler(h) + _state["file"] = h + except OSError as e: + sys.stderr.write(f"logs: can't write to {folder} ({e}); the log viewer won't show this process\n") + if ship != (old[4] if old else None): + _close("syslog") + if ship is not None: + h = _tag(SyslogHandler(*ship)) + root.addHandler(h) + _state["syslog"] = h + for name in set(_state["levels"]) - {k for k, _ in levels}: + logging.getLogger(name).setLevel(logging.NOTSET) # an override taken away: back to the root's level + for name, v in levels: + logging.getLogger(name).setLevel(v) + _state["levels"] = dict(levels) + lowest = min([lvl, *(v for _, v in levels), *([ship[3]] if ship else [])]) + root.setLevel(lowest) + _state["key"] = want + + +def _close(which): + h = _state[which] + _state[which] = None + if h is not None: + logging.getLogger().removeHandler(h) + with contextlib.suppress(Exception): + h.close() + + +def shutdown(): + """Stop the syslog thread and close the file (tests; atexit does it otherwise).""" + with _lock: + _close("syslog") + _close("file") + _state["key"] = None + + +def _prune(folder): + cutoff = time.time() - PRUNE_DAYS * 86400 + for p in pathlib.Path(folder).glob("*.jsonl*"): + with contextlib.suppress(OSError): + if p.stat().st_mtime < cutoff: + p.unlink() + + +# --- syslog (RFC 5424, sent over UDP per RFC 5426, TCP per RFC 6587 or TLS per RFC 5425) --- + + +def _printable(s, n): + s = "".join(c if 33 <= ord(c) <= 126 else "_" for c in str(s)) + return s[:n] or "-" + + +def rfc5424(r, facility, fmt, app_name, host=None): + """One record as an RFC 5424 message (bytes, without framing).""" + d = record(r) + pri = facility * 8 + SEVERITY.get(r.levelno, 6) + head = " ".join( + [ + f"<{pri}>1", + d["ts"], + _printable(host or _host, 255), + _printable(app_name, 48), + _printable(f"{d['process']}.{d['pid']}", 128), + _printable(d["logger"], 32), + "-", + ] + ) + if fmt == "json": + body = json.dumps(d, ensure_ascii=False, default=str) + else: + body = d["msg"] + ("\n" + d["exc"] if d.get("exc") else "") + return (head + " " + body).encode("utf-8", "replace")[:MAX_BYTES] + + +class Sender: + """A connection to a syslog server; `send` raises OSError when the line couldn't go.""" + + def __init__(self, host, port, transport, framing="octet", verify=True, timeout=5.0): + self.host, self.port, self.transport, self.framing, self.verify, self.timeout = host, port, transport, framing, verify, timeout + self.sock = None + + def _connect(self): + infos = socket.getaddrinfo(self.host, self.port, type=socket.SOCK_DGRAM if self.transport == "udp" else socket.SOCK_STREAM) + err = None + for family, kind, proto, _, addr in infos: + if ipaddress.ip_address(addr[0].split("%")[0]).is_link_local: + err = OSError(f"{self.host} is a link-local address; Lens doesn't send logs there") + continue + s = socket.socket(family, kind, proto) + s.settimeout(self.timeout) + try: + s.connect(addr) + if self.transport == "tls": + ctx = ssl.create_default_context() + if not self.verify: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + s = ctx.wrap_socket(s, server_hostname=self.host) + return s + except OSError as e: + err = e + s.close() + raise err or OSError(f"couldn't find {self.host}") + + def send(self, msg): + if self.sock is None: + self.sock = self._connect() + try: + if self.transport == "udp": + self.sock.send(msg) + elif self.framing == "newline": + self.sock.sendall(msg.replace(b"\n", b" ") + b"\n") + else: + self.sock.sendall(str(len(msg)).encode() + b" " + msg) + except OSError: + self.close() + raise + + def close(self): + if self.sock is not None: + with contextlib.suppress(OSError): + self.sock.close() + self.sock = None + + +class SyslogHandler(logging.Handler): + """Queues each record and sends it from a background thread; never blocks the caller.""" + + def __init__(self, host, port, transport, lvl, facility, fmt, framing, verify, app_name): + super().__init__() + self.addFilter(Gate(lvl)) + self.facility, self.fmt, self.app_name = facility, fmt, app_name + self.target = f"{transport}://{host}:{port}" + self.sender = Sender(host, port, transport, framing, verify) + self.q = queue.Queue(QUEUE) + self.stats = {"sent": 0, "dropped": 0, "last_sent": None, "last_error": None, "error_at": None} + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, name="lens-syslog", daemon=True) + self._thread.start() + + def emit(self, r): + try: + self.q.put_nowait(rfc5424(r, self.facility, self.fmt, self.app_name)) + except queue.Full: + self.stats["dropped"] += 1 + except Exception: # noqa: BLE001 - logging never raises into the caller + self.handleError(r) + + def _run(self): + wait = 1.0 + while not self._stop.is_set(): + try: + msg = self.q.get(timeout=0.5) + except queue.Empty: + continue + if msg is None: + break + while not self._stop.is_set(): + try: + self.sender.send(msg) + self.stats["sent"] += 1 + self.stats["last_sent"] = time.time() + wait = 1.0 + break + except OSError as e: + # never logged (that would come back here): the status in Settings → Logs shows it + self.stats["last_error"] = f"{type(e).__name__}: {e}"[:300] + self.stats["error_at"] = time.time() + self._stop.wait(wait) + wait = min(wait * 2, 30.0) + self.sender.close() + + def close(self): + self._stop.set() + with contextlib.suppress(queue.Full): + self.q.put_nowait(None) + self._thread.join(timeout=2) + super().close() + + +def status(cfg): + """What this process does with its logs now.""" + s = section(cfg) + h = _state["syslog"] + f = _state["file"] + return { + "level": NAMES.get(level(s.get("level")), "info"), + "file": f.baseFilename if f is not None else None, + "syslog": bool(h), + "target": h.target if h else None, + "sent": h.stats["sent"] if h else 0, + "dropped": h.stats["dropped"] if h else 0, + "queued": h.q.qsize() if h else 0, + "last_sent": h.stats["last_sent"] if h else None, + "last_error": h.stats["last_error"] if h else None, + "error_at": h.stats["error_at"] if h else None, + } + + +def test_send(cfg): + """Send one line to the saved syslog server now (on or off): (ok, error, ms).""" + s = section(cfg) + if not s.get("syslog_host"): + return False, "set the syslog server's address first", None + transport = s.get("syslog_transport") if s.get("syslog_transport") in TRANSPORTS else "udp" + sender = Sender( + str(s["syslog_host"]), + int(s.get("syslog_port") or 514), + transport, + s.get("syslog_framing") or "octet", + s.get("syslog_verify") is not False, + ) + r = logging.LogRecord("lens.logs", logging.INFO, __file__, 0, "Lens %s: test message from Settings → Logs", (__version__,), None) + t = time.monotonic() + try: + sender.send( + rfc5424(r, FACILITIES.get(s.get("syslog_facility"), 16), s.get("syslog_format") or "text", s.get("syslog_app_name") or "lens") + ) + return True, None, int((time.monotonic() - t) * 1000) + except (OSError, ValueError) as e: + return False, f"{type(e).__name__}: {e}"[:300], None + finally: + sender.close() + + +# --- reading the files back, for the log viewer --- + + +def _lines_backwards(path, chunk=65536): + with open(path, "rb") as f: + f.seek(0, os.SEEK_END) + end = f.tell() + rest = b"" + while end > 0: + n = min(chunk, end) + end -= n + f.seek(end) + buf = f.read(n) + rest + parts = buf.split(b"\n") + rest = parts[0] + for line in reversed(parts[1:]): + if line.strip(): + yield line + if rest.strip(): + yield rest + + +def files(cfg): + folder = pathlib.Path((cfg or {}).get("data_dir") or ".") / "logs" + return sorted(folder.glob("*.jsonl*")) if folder.is_dir() else [] + + +def read(cfg, limit=200, min_level=None, q=None, process=None, logger=None, before=None): + """The newest log records from every process's file, newest first, matching the filters. `before` is a record's + ts: only older ones (the next page).""" + lo = level(min_level, logging.DEBUG) if min_level else logging.DEBUG + needle = (q or "").lower() + found = [] + for p in files(cfg): + got = 0 + try: + for raw in _lines_backwards(p): + try: + d = json.loads(raw) + except ValueError: + continue + if not isinstance(d, dict) or not isinstance(d.get("ts"), str): + continue + if before and d["ts"] >= before: + continue + if LEVELS.get(d.get("level"), logging.INFO) < lo: + continue + if process and d.get("process") != process: + continue + if logger and not str(d.get("logger", "")).startswith(logger): + continue + if needle and needle not in (str(d.get("msg", "")) + " " + str(d.get("exc", ""))).lower(): + continue + found.append(d) + got += 1 + if got >= limit: + break + except OSError: + continue + found.sort(key=lambda d: d["ts"], reverse=True) + return found[:limit] diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index eb67539fe..9ed4a99cd 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -19,7 +19,7 @@ import threading import urllib.parse -from . import convert, ipgroups, mail_routes, objects, store, telemetry +from . import convert, ipgroups, logs, mail_routes, objects, store, telemetry R = store.R EDITABLE = { @@ -50,6 +50,7 @@ "matterbridge": None, "notifications": None, "telemetry": None, + "logs": None, # repo is a startup setting only (what gets installed isn't chosen in the web app) "updates": ("check", "check_hours", "auto_install", "email"), "activity": None, @@ -172,6 +173,13 @@ ("https", "challenge"): {"http", "tls-alpn", "dns"}, ("files", "store"): {"local", "connection"}, ("documents", "converter"): {"auto", "anytopdf", "lens"}, + ("logs", "level"): set(logs.LEVELS), + ("logs", "syslog_level"): set(logs.LEVELS), + ("logs", "format"): {"text", "json"}, + ("logs", "syslog_format"): {"text", "json"}, + ("logs", "syslog_transport"): set(logs.TRANSPORTS), + ("logs", "syslog_facility"): set(logs.FACILITIES), + ("logs", "syslog_framing"): {"octet", "newline"}, } # Settings the environment (.env) sets, which win over archive.yaml and the app and show as locked there: the # break-glass allowed hosts, the model provider so an install can be configured without the setup wizard, and @@ -195,6 +203,8 @@ ("telemetry", "enabled"): "LENS_TELEMETRY", ("telemetry", "endpoint"): "LENS_TELEMETRY_ENDPOINT", ("telemetry", "headers"): "LENS_TELEMETRY_HEADERS", + ("logs", "level"): "LENS_LOG_LEVEL", + ("logs", "format"): "LENS_LOG_FORMAT", ("telemetry", "prometheus"): "LENS_PROMETHEUS", ("telemetry", "prometheus_token"): "LENS_PROMETHEUS_TOKEN", ("telemetry", "grafana_url"): "LENS_GRAFANA_URL", @@ -334,6 +344,7 @@ def current(self): if self._cfg is None or v != self._ver: self._cfg, self._ver = effective(self.db, self.base), v telemetry.apply(self._cfg) # turned on, changed or off in the app: this process follows + logs.apply(self._cfg) # and its logging (level, file, syslog) return self._cfg @@ -439,6 +450,8 @@ def _check(section, key, value, default): return _notify_setting(key, value) if section == "telemetry": return _telemetry_setting(key, value) + if section == "logs": + return _logs_setting(key, value) if section == "updates": return _updates_setting(key, value) if section == "activity": @@ -1043,6 +1056,54 @@ def _sensor_setting(key, value): return value +LOG_RANGES = {"file_mb": (1, 100), "file_keep": (1, 10), "syslog_port": (1, 65535)} +APP_NAME_RX = re.compile(r"^[!-~]{1,48}$") +HOST_RX = re.compile(r"^[A-Za-z0-9.:_-]{1,253}$") + + +def _logs_setting(key, value): + if ("logs", key) in ENUMS: # checked against ENUMS already + return value + if key in ("file", "syslog", "syslog_verify"): + if not isinstance(value, bool): + raise ValueError(f"logs.{key} is true or false") + return value + if key == "levels": + if value is None: + return {} + if not isinstance(value, dict) or len(value) > 100: + raise ValueError('logs.levels gives loggers their own level, like {"httpx": "warning"}') + out = {} + for name, lvl in value.items(): + if not logs.LOGGER_RX.match(str(name)) or str(lvl).lower() not in logs.LEVELS: + raise ValueError(f"logs.levels: {name} needs a logger name and one of: {', '.join(logs.LEVELS)}") + out[str(name)] = str(lvl).lower() + return out + if key == "syslog_host": + if value in (None, ""): + return None + v = str(value).strip().strip("[]") + if not HOST_RX.match(v): + raise ValueError("logs.syslog_host is the syslog server's host name or IP address, without a port") + try: + link_local = ipaddress.ip_address(v).is_link_local + except ValueError: + if ":" in v: # only an IPv6 address has colons; the port is logs.syslog_port + raise ValueError("logs.syslog_host is the syslog server's host name or IP address, without a port") from None + link_local = False + if link_local: + raise ValueError("logs.syslog_host can't be a link-local address (169.254.0.0/16, fe80::/10)") + return v + if key == "syslog_app_name": + if not (isinstance(value, str) and APP_NAME_RX.match(value.strip())): + raise ValueError("logs.syslog_app_name is up to 48 printable characters, without spaces") + return value.strip() + lo, hi = LOG_RANGES[key] + if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi): + raise ValueError(f"logs.{key} is a whole number from {lo} to {hi}") + return value + + def _nfs_setting(key, value): if key == "enabled": if not isinstance(value, bool): @@ -1267,6 +1328,8 @@ def save(db, base, section, changes, user=None): # the saved headers (an auth token) were for the old collector: they don't go to a new host if before: sealed.pop("headers", None) + if section == "logs" and data.get("syslog") and not data.get("syslog_host", (base.get("logs") or {}).get("syslog_host")): + raise ValueError("set the syslog server's address (logs.syslog_host) to send logs to it") if section == "speakers": m = data.get("match_threshold", defaults["match_threshold"]) r = data.get("review_threshold", defaults["review_threshold"]) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 6062f68ae..5cb28a45b 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -613,6 +613,27 @@ def labels(): # where Grafana shows Lens's charts (docs/prometheus.md), for a link on Admin → System health; none: no link "grafana_url": None, }, + # logging (docs/logging.md): level and format for the console (docker logs) and a file per process under + # data_dir/logs (what Settings → Logs shows); levels sets single loggers, like {"httpx": "warning"}. syslog sends + # every line at syslog_level or above to a syslog server (RFC 5424 over udp, tcp or tls); off by default + "logs": { + "level": "info", + "format": "text", + "levels": {}, + "file": True, + "file_mb": 5, + "file_keep": 2, + "syslog": False, + "syslog_host": None, + "syslog_port": 514, + "syslog_transport": "udp", + "syslog_level": "info", + "syslog_facility": "local0", + "syslog_format": "text", + "syslog_framing": "octet", + "syslog_verify": True, + "syslog_app_name": "lens", + }, # new releases (docs/updates.md): check asks GitHub for repo's latest release every check_hours, for the web app's # banner; auto_install installs one as soon as it's seen, through the machine's upgrade agent (install.sh sets it up) # email: tell admins by email when a new release is seen (once per release; needs Settings › Email). Namespaces' diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py index ba386c867..cbb7c692e 100644 --- a/fastapi_backend/app/main.py +++ b/fastapi_backend/app/main.py @@ -7,7 +7,6 @@ from __future__ import annotations import contextlib -import logging from collections.abc import AsyncIterator from typing import Any @@ -27,11 +26,11 @@ from app.config import settings from app.core import middleware from app.core.runtime import Archive -from app.domain import __version__, activity, store, telemetry +from app.domain import __version__, activity, logs, store, telemetry from app.domain.render import WEB_DIR from app.utils import simple_generate_unique_route_id -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logs.setup("api") # the console now; the level, the log file and syslog follow the settings once they're read def _attach(app: FastAPI, archive: Archive) -> None: @@ -60,6 +59,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: finally: archive.close() telemetry.shutdown() + logs.shutdown() app = FastAPI( title=settings.PROJECT_NAME, diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py index e8272dfc3..b3835ed40 100644 --- a/fastapi_backend/app/schemas/admin.py +++ b/fastapi_backend/app/schemas/admin.py @@ -346,6 +346,41 @@ class TelemetryTestResult(ResponseModel): ms: int | None = None +class LogStatus(ResponseModel): + """What the API process does with its logs now; each worker follows the same settings on its own.""" + + level: str + file: str | None = Field(None, description="the API's log file, under data_dir/logs") + syslog: bool = Field(description="whether lines are being sent to a syslog server") + target: str | None = Field(None, description="where, as transport://host:port") + sent: int + dropped: int = Field(description="lines dropped because the server couldn't keep up or was unreachable") + queued: int + last_sent: float | None = None + last_error: str | None = None + error_at: float | None = None + + +class LogTestResult(ResponseModel): + ok: bool + error: str | None = None + ms: int | None = None + + +class LogLine(ResponseModel): + """One log record, redacted as it was written.""" + + ts: str + level: str + logger: str + msg: str + process: str + pid: int | None = None + host: str | None = None + exc: str | None = None + fields: dict[str, Any] | None = None + + class FileStoreTry(RequestModel): """Where to try keeping files, before saving it (Settings → Storage); nothing given tries what is saved.""" diff --git a/fastapi_backend/tests/api/test_logs.py b/fastapi_backend/tests/api/test_logs.py new file mode 100644 index 000000000..69ee7edb0 --- /dev/null +++ b/fastapi_backend/tests/api/test_logs.py @@ -0,0 +1,184 @@ +"""Logging: records in a file per process that the log viewer reads, redacted, and sent to syslog only when turned on.""" + +from __future__ import annotations + +import logging +import socket +import socketserver +import threading +import time + +import pytest + +from app.domain import logs +from tests.helpers import login, make_user + + +@pytest.fixture(autouse=True) +def _reset(): + yield + logs.shutdown() + + +def admin(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +class _Udp(socketserver.BaseRequestHandler): + got: list[bytes] = [] + + def handle(self): + _Udp.got.append(self.request[0]) + + +class _Tcp(socketserver.StreamRequestHandler): + got: list[bytes] = [] + + def handle(self): + _Tcp.got.append(self.rfile.read()) + + +def _wait(cond, seconds=5): + end = time.time() + seconds + while time.time() < end: + if cond(): + return True + time.sleep(0.05) + return False + + +def test_redacts_credentials(): + text = ( + "GET https://bob:hunter2@example.com/x?api_key=abc123def&token=zzz Authorization: Bearer abcdefgh123 " + "key sk-abcdefghijklmnopqrstu password=letmein prompt_tokens=1000" + ) + out = logs.redact(text) + for secret in ("hunter2", "abc123def", "zzz", "abcdefgh123", "sk-abcdefghijklmnopqrstu", "letmein"): + assert secret not in out + assert "prompt_tokens=1000" in out and "bob:***@example.com" in out + + +def test_viewer_reads_the_file_newest_first_and_filters(client, db, cfg): + h = admin(client, db) + lg = logging.getLogger("lens.test") + lg.info("first line") + lg.warning("second line with password=swordfish", extra={"job": 7, "api_key": "k"}) + try: + raise ValueError("boom") + except ValueError: + lg.exception("third line") + rows = client.get("/api/v1/admin/logs", params={"logger": "lens.test"}, headers=h).json() + assert [r["msg"] for r in rows[:3]] == ["third line", "second line with password=***", "first line"] + assert rows[0]["level"] == "error" and "ValueError: boom" in rows[0]["exc"] and rows[0]["process"] == "api" + assert rows[1]["fields"] == {"job": 7, "api_key": "***"} + warn = client.get("/api/v1/admin/logs", params={"logger": "lens.test", "level": "warning"}, headers=h).json() + assert [r["msg"] for r in warn] == ["third line", "second line with password=***"] + found = client.get("/api/v1/admin/logs", params={"q": "FIRST LINE"}, headers=h).json() + assert [r["msg"] for r in found] == ["first line"] + older = client.get("/api/v1/admin/logs", params={"logger": "lens.test", "before": rows[1]["ts"]}, headers=h).json() + assert all(r["ts"] < rows[1]["ts"] for r in older) + # admins only + make_user(db, "a@x.io", "a password 123") + assert client.get("/api/v1/admin/logs", headers=login(client, "a@x.io", "a password 123")).status_code == 403 + + +def test_level_and_per_logger_levels_follow_the_settings(client, db, cfg): + h = admin(client, db) + body = {"level": "warning", "levels": {"lens.chatty": "error", "lens.loud": "debug"}} + assert client.put("/api/v1/settings/logs", json=body, headers=h).status_code == 200 + client.get("/api/v1/settings/logs/status", headers=h) # reads the settings: this process follows + logging.getLogger("lens.quiet").info("not kept") + logging.getLogger("lens.quiet").warning("kept") + logging.getLogger("lens.chatty.sub").warning("not kept either") + logging.getLogger("lens.loud").debug("kept: its own level") + rows = client.get("/api/v1/admin/logs", params={"q": "kept"}, headers=h).json() + assert [r["msg"] for r in rows] == ["kept: its own level", "kept"] + assert client.get("/api/v1/settings/logs/status", headers=h).json()["level"] == "warning" + + +def test_settings_are_checked(client, db, cfg): + h = admin(client, db) + bad = [ + {"level": "loud"}, + {"syslog": True}, # nowhere to send to + {"syslog_host": "169.254.169.254"}, + {"syslog_host": "host:514"}, + {"syslog_port": 0}, + {"syslog_transport": "carrier-pigeon"}, + {"levels": {"bad name!": "info"}}, + {"file_mb": 1000}, + ] + for body in bad: + assert client.put("/api/v1/settings/logs", json=body, headers=h).status_code == 400, body + v = client.get("/api/v1/settings", headers=h).json()["logs"]["values"] + assert v["syslog"] is False and v["level"] == "info" and v["file"] is True + + +def test_syslog_off_by_default_then_sent_over_udp(client, db, cfg): + _Udp.got = [] + srv = socketserver.UDPServer(("127.0.0.1", 0), _Udp) + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + h = admin(client, db) + s = client.get("/api/v1/settings/logs/status", headers=h).json() + assert s["syslog"] is False and s["target"] is None + logging.getLogger("lens.test").warning("while off") + port = srv.server_address[1] + r = client.put( + "/api/v1/settings/logs", + json={"syslog_host": "127.0.0.1", "syslog_port": port, "syslog_facility": "local3"}, + headers=h, + ) + assert r.status_code == 200 + t = client.post("/api/v1/settings/logs/test", headers=h).json() + assert t["ok"], t + assert _wait(lambda: any(b"test message" in g for g in _Udp.got)) + assert not any(b"while off" in g for g in _Udp.got) + client.put("/api/v1/settings/logs", json={"syslog": True}, headers=h) + s = client.get("/api/v1/settings/logs/status", headers=h).json() + assert s["syslog"] is True and s["target"] == f"udp://127.0.0.1:{port}" + logging.getLogger("lens.test").warning("shipped with token=abcdef") + assert _wait(lambda: any(b"shipped" in g for g in _Udp.got)) + line = next(g for g in _Udp.got if b"shipped" in g).decode() + # 1 ts host lens api.pid lens.test - msg + assert line.startswith("<156>1 ") and " lens api." in line and " lens.test - shipped with token=***" in line + assert "abcdef" not in line + finally: + srv.shutdown() + + +def test_syslog_over_tcp_uses_octet_counting(): + _Tcp.got = [] + srv = socketserver.TCPServer(("127.0.0.1", 0), _Tcp) + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + cfg = { + "logs": {"syslog_host": "127.0.0.1", "syslog_port": srv.server_address[1], "syslog_transport": "tcp", "syslog_format": "json"} + } + ok, error, _ = logs.test_send(cfg) + assert ok, error + assert _wait(lambda: _Tcp.got) + n, _, msg = _Tcp.got[0].partition(b" ") + assert int(n) == len(msg) and msg.startswith(b"<134>1 ") and b'"msg": "Lens ' in msg + finally: + srv.shutdown() + + +def test_unreachable_server_never_blocks(): + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() # nothing listens there + h = logs.SyslogHandler("127.0.0.1", port, "tcp", logging.INFO, 16, "text", "octet", True, "lens") + try: + lg = logging.getLogger("lens.unreachable") + lg.addHandler(h) + t = time.time() + for i in range(100): + lg.warning("line %d", i) + assert time.time() - t < 1 + assert _wait(lambda: h.stats["last_error"]) + finally: + logging.getLogger("lens.unreachable").removeHandler(h) + h.close() diff --git a/mkdocs.yml b/mkdocs.yml index 09dd1474e..b9bdc218b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,6 +107,7 @@ nav: - Storage: storage.md - Database: database.md - Telemetry: telemetry.md + - Logging: logging.md - Prometheus: prometheus.md - Build: - Architecture: architecture.md diff --git a/nextjs-frontend/__tests__/settings-logs.test.ts b/nextjs-frontend/__tests__/settings-logs.test.ts new file mode 100644 index 000000000..00c4ba86f --- /dev/null +++ b/nextjs-frontend/__tests__/settings-logs.test.ts @@ -0,0 +1,16 @@ +import { levelsToLines, linesToLevels } from "@/components/settings/model"; + +describe("logger levels", () => { + it("round-trips lines of logger and level", () => { + const parsed = linesToLevels("httpx warning\n\n lens.bridge DEBUG "); + expect(parsed).toEqual({ value: { httpx: "warning", "lens.bridge": "debug" } }); + expect(levelsToLines({ httpx: "warning" })).toBe("httpx warning"); + }); + + it("names the line that's wrong", () => { + expect(linesToLevels("httpx warning\nhttpx loud")).toEqual({ + error: expect.stringContaining("Line 2"), + }); + expect("error" in linesToLevels("bad name! info")).toBe(true); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 7b9770738..1c104626d 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1820,11 +1820,17 @@ export type { LoginResponse, LoginResponses, LoginTicket, + LogLine, LogoutData, LogoutError, LogoutErrors, LogoutResponse, LogoutResponses, + LogStatus, + LogStatusData, + LogStatusResponse, + LogStatusResponses, + LogTestResult, LostPasskeyData, LostPasskeyError, LostPasskeyErrors, @@ -2183,6 +2189,11 @@ export type { ReadFileResponse, ReadFileResponses, Reading, + ReadLogsData, + ReadLogsError, + ReadLogsErrors, + ReadLogsResponse, + ReadLogsResponses, Recording, RecordingAccess, RecordingAccessUpdate, @@ -2799,6 +2810,9 @@ export type { TestLlmData, TestLlmResponse, TestLlmResponses, + TestLogsData, + TestLogsResponse, + TestLogsResponses, TestMailboxData, TestMailboxResponse, TestMailboxResponses, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 3414ef23f..89b06d7ac 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -939,6 +939,8 @@ import type { LogoutData, LogoutErrors, LogoutResponses, + LogStatusData, + LogStatusResponses, LostPasskeyData, LostPasskeyErrors, LostPasskeyResponses, @@ -1069,6 +1071,9 @@ import type { ReadFileData, ReadFileErrors, ReadFileResponses, + ReadLogsData, + ReadLogsErrors, + ReadLogsResponses, RecordingTopicsData, RecordingTopicsErrors, RecordingTopicsResponses, @@ -1351,6 +1356,8 @@ import type { TestFileStoreResponses, TestLlmData, TestLlmResponses, + TestLogsData, + TestLogsResponses, TestMailboxData, TestMailboxResponses, TestMailData, @@ -2885,6 +2892,49 @@ export class Admin { }); } + /** + * Log Status + * + * The API process's log level, its log file, and how sending to syslog is going (off by default). + */ + public static logStatus( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/settings/logs/status", + ...options, + }); + } + + /** + * Test Logs + * + * Send one line to the saved syslog server now (on or off), to check the address, port and transport. + */ + public static testLogs( + options?: Options, + ): RequestResult { + return (options?.client ?? client).post({ + url: "/api/v1/settings/logs/test", + ...options, + }); + } + + /** + * Read Logs + * + * The latest log lines from the API and every worker (their files under data_dir/logs), newest first. Lines are + * redacted as they're written: no keys, tokens or passwords. + */ + public static readLogs( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/admin/logs", + ...options, + }); + } + /** * List Audit * diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index ced6d01d0..4ce6281d7 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -7624,6 +7624,129 @@ export type LocalModelServer = { [key: string]: unknown; }; +/** + * LogLine + * + * One log record, redacted as it was written. + */ +export type LogLine = { + /** + * Ts + */ + ts: string; + /** + * Level + */ + level: string; + /** + * Logger + */ + logger: string; + /** + * Msg + */ + msg: string; + /** + * Process + */ + process: string; + /** + * Pid + */ + pid?: number | null; + /** + * Host + */ + host?: string | null; + /** + * Exc + */ + exc?: string | null; + /** + * Fields + */ + fields?: { + [key: string]: unknown; + } | null; + [key: string]: unknown; +}; + +/** + * LogStatus + * + * What the API process does with its logs now; each worker follows the same settings on its own. + */ +export type LogStatus = { + /** + * Level + */ + level: string; + /** + * File + * + * the API's log file, under data_dir/logs + */ + file?: string | null; + /** + * Syslog + * + * whether lines are being sent to a syslog server + */ + syslog: boolean; + /** + * Target + * + * where, as transport://host:port + */ + target?: string | null; + /** + * Sent + */ + sent: number; + /** + * Dropped + * + * lines dropped because the server couldn't keep up or was unreachable + */ + dropped: number; + /** + * Queued + */ + queued: number; + /** + * Last Sent + */ + last_sent?: number | null; + /** + * Last Error + */ + last_error?: string | null; + /** + * Error At + */ + error_at?: number | null; + [key: string]: unknown; +}; + +/** + * LogTestResult + */ +export type LogTestResult = { + /** + * Ok + */ + ok: boolean; + /** + * Error + */ + error?: string | null; + /** + * Ms + */ + ms?: number | null; + [key: string]: unknown; +}; + /** * LoginRequest */ @@ -21249,6 +21372,100 @@ export type TestTelemetryResponses = { export type TestTelemetryResponse = TestTelemetryResponses[keyof TestTelemetryResponses]; +export type LogStatusData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/logs/status"; +}; + +export type LogStatusResponses = { + /** + * Successful Response + */ + 200: LogStatus; +}; + +export type LogStatusResponse = LogStatusResponses[keyof LogStatusResponses]; + +export type TestLogsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/settings/logs/test"; +}; + +export type TestLogsResponses = { + /** + * Successful Response + */ + 200: LogTestResult; +}; + +export type TestLogsResponse = TestLogsResponses[keyof TestLogsResponses]; + +export type ReadLogsData = { + body?: never; + path?: never; + query?: { + /** + * Limit + */ + limit?: number; + /** + * Level + * + * this level and above + */ + level?: "debug" | "info" | "warning" | "error" | "critical" | null; + /** + * Q + * + * text the message or error contains + */ + q?: string | null; + /** + * Process + * + * api, worker or watcher + */ + process?: string | null; + /** + * Logger + * + * a logger name, or the start of one (lens.email) + */ + logger?: string | null; + /** + * Before + * + * a line's ts: only older lines (the next page) + */ + before?: string | null; + }; + url: "/api/v1/admin/logs"; +}; + +export type ReadLogsErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ReadLogsError = ReadLogsErrors[keyof ReadLogsErrors]; + +export type ReadLogsResponses = { + /** + * Response Admin-Read Logs + * + * Successful Response + */ + 200: Array; +}; + +export type ReadLogsResponse = ReadLogsResponses[keyof ReadLogsResponses]; + export type ListAuditData = { body?: never; path?: never; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index f1577dd4c..2bad54dfa 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -10,6 +10,7 @@ import type { MatterbridgeStatus } from "@/app/openapi-client/types.gen"; import { MonitoringSwitch } from "@/components/admin/monitoring-switch"; import { BackupsPanel } from "@/components/settings/backups-panel"; import { ComponentsStatus } from "@/components/settings/components-status"; +import { LogViewer } from "@/components/settings/log-viewer"; import { AllTokens } from "@/components/account/all-tokens"; import { ACCESS } from "@/components/iiif/metadata-model"; import { RIGHTS } from "@/components/iiif/rights"; @@ -401,6 +402,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { return ; case "telemetry": return ; + case "logs": + return ; case "fedora": return ; case "sensors": @@ -685,6 +688,91 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) { ); } +function LogsBody({ ctx }: { ctx: BodyCtx }) { + const client = useApiClient(); + const l = ctx.view.logs; + const saved = l?.values ?? {}; + const status = useQuery({ + queryKey: ["log-status", l?.updated_at ?? null], + queryFn: () => data(Admin.logStatus({ client })), + refetchInterval: 15_000, + }); + const test = useMutation({ mutationFn: () => data(Admin.testLogs({ client })) }); + const s = status.data; + const host = (saved.syslog_host as string | null | undefined) ?? null; + const on = Boolean(saved.syslog); + const failing = Boolean(s?.last_error && (!s.last_sent || (s.error_at ?? 0) > s.last_sent)); + return ( + <> + {on && s?.syslog ? ( + + {failing + ? `Can’t reach it: ${s.last_error}. Lines wait and are tried again${s.dropped ? `; ${s.dropped} dropped so far` : ""}.` + : `${s.sent} line${s.sent === 1 ? "" : "s"} sent from the server${s.dropped ? `, ${s.dropped} dropped` : ""}. Each worker sends its own.`} + + ) : ( + + Lines go to the console and to a file per process. Turn on syslog below to send them to a log server too. + + )} +
+ + +
+ + +
+ + +
+ +
+ + + +
+
+ + + +
+
+ + + +
+

+ RFC 5424 syslog, as rsyslog, syslog-ng, Graylog, Loki’s syslog receiver and most log tools take it. Keys, tokens + and passwords are replaced with *** in every line before it is written or sent. +

+
+ + + {ctx.dirty + ? "Uses the saved settings, not your unsaved changes" + : "Sends one line, even while sending is off"} + +
+ {test.data && + (test.data.ok ? ( + + {saved.syslog_transport === "udp" + ? "UDP doesn’t confirm delivery: look for it on the server." + : `The server took it in ${test.data.ms} ms.`} + + ) : ( + + {test.data.error} + + ))} + {test.isError && {test.error.message}} + + + ); +} + function EncryptionBody({ ctx }: { ctx: BodyCtx }) { const client = useApiClient(); const progress = useQuery({ diff --git a/nextjs-frontend/components/settings/log-viewer.tsx b/nextjs-frontend/components/settings/log-viewer.tsx new file mode 100644 index 000000000..de6c7813a --- /dev/null +++ b/nextjs-frontend/components/settings/log-viewer.tsx @@ -0,0 +1,164 @@ +"use client"; + +import { useInfiniteQuery } from "@tanstack/react-query"; +import { RefreshCw } from "lucide-react"; +import { useState } from "react"; + +import { Admin } from "@/app/openapi-client"; +import type { LogLine } from "@/app/openapi-client/types.gen"; +import { Banner } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { SearchInput, Select, Switch } from "@/components/ui/field"; +import { Skeleton } from "@/components/ui/states"; +import { data, useApiClient } from "@/lib/api/browser"; +import { cn } from "@/lib/utils"; + +const PAGE = 200; +type Level = "debug" | "info" | "warning" | "error" | "critical"; + +const TONE: Record = { + debug: "text-fg-muted", + info: "text-fg-secondary", + warning: "text-gold-dark", + error: "text-red-dark", + critical: "text-red-dark font-bold", +}; + +function Line({ l }: { l: LogLine }) { + const [open, setOpen] = useState(false); + const more = Boolean(l.exc || (l.fields && Object.keys(l.fields).length)); + return ( +
  • + + {open && ( +
    +          {[
    +            l.fields
    +              ? Object.entries(l.fields)
    +                  .map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`)
    +                  .join(" ")
    +              : "",
    +            l.exc ?? "",
    +          ]
    +            .filter(Boolean)
    +            .join("\n")}
    +        
    + )} +
  • + ); +} + +/** The latest lines from the server and every worker (their log files), newest first, with filters. */ +export function LogViewer() { + const client = useApiClient(); + const [level, setLevel] = useState<"" | Level>(""); + const [q, setQ] = useState(""); + const [process, setProcess] = useState(""); + const [follow, setFollow] = useState(true); + const lines = useInfiniteQuery({ + queryKey: ["logs", level, q, process], + queryFn: ({ pageParam }) => + data( + Admin.readLogs({ + client, + query: { + limit: PAGE, + level: level || undefined, + q: q.trim() || undefined, + process: process || undefined, + before: pageParam || undefined, + }, + }), + ), + initialPageParam: "", + getNextPageParam: (last) => (last.length === PAGE ? last[last.length - 1].ts : undefined), + refetchInterval: follow ? 5000 : false, + }); + const rows = lines.data?.pages.flat() ?? []; + return ( +
    +
    +

    Recent lines

    +
    + + +
    +
    +
    + setQ(e.target.value)} + aria-label="Search the log" + /> + setProcess(e.target.value)} + options={[ + { value: "", label: "Every process" }, + { value: "api", label: "Server" }, + { value: "worker", label: "Workers" }, + { value: "watcher", label: "Watcher" }, + ]} + /> +
    + {lines.isError && {lines.error.message}} + {lines.isPending ? ( + + ) : rows.length ? ( +
      + {rows.map((l, i) => ( + + ))} +
    + ) : ( +

    + {q || level || process + ? "No lines match." + : "No lines yet. With log files off, nothing is kept to show here."} +

    + )} + {lines.hasNextPage && ( + + )} +
    + ); +} diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index e5c37d00c..3afa73f1b 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -85,6 +85,7 @@ export type SectionId = | "slack" | "mailbox" | "telemetry" + | "logs" | "fedora" | "sensors" | "nfs" @@ -275,6 +276,13 @@ export const SECTIONS: SectionSpec[] = [ description: "Off unless you turn it on. Traces and metrics from the server and its workers, sent only to an OpenTelemetry collector you run or choose; nothing goes anywhere else.", }, + { + id: "logs", + label: "Logs", + backend: ["logs"], + description: + "What the server and its workers log, the latest lines from all of them, and sending them to a syslog server (off unless you turn it on). Keys, tokens and passwords are taken out of every line.", + }, { id: "fedora", label: "Fedora repository", @@ -2486,6 +2494,134 @@ export const FIELDS: FieldSpec[] = [ placeholder: "gpt-4o-mini 0.15 0.60", hint: "One model per line: its name, then the input and output price in dollars per million tokens. The AI assistant’s prices count for the configured model", }, + // Logs + { + section: "logs", + key: "level", + label: "Level", + kind: "select", + options: [ + { value: "debug", label: "Debug" }, + { value: "info", label: "Info" }, + { value: "warning", label: "Warning" }, + { value: "error", label: "Error" }, + { value: "critical", label: "Critical" }, + ], + hint: "What is kept on the console (docker logs) and in the log files", + }, + { + section: "logs", + key: "format", + label: "Console format", + kind: "select", + options: [ + { value: "text", label: "Text" }, + { value: "json", label: "JSON, one object per line" }, + ], + }, + { + section: "logs", + key: "levels", + label: "Logger levels", + kind: "lines", + mono: true, + placeholder: "httpx warning", + hint: "One logger per line with its own level, like httpx warning or lens.bridge debug", + }, + { + section: "logs", + key: "file", + label: "Keep log files", + kind: "switch", + hint: "A file per process under data_dir/logs; the lines below come from these", + }, + { section: "logs", key: "file_mb", label: "File size", kind: "int", min: 1, max: 100, unit: "MB" }, + { section: "logs", key: "file_keep", label: "Files kept", kind: "int", min: 1, max: 10 }, + { + section: "logs", + key: "syslog", + label: "Send to syslog", + kind: "switch", + hint: "Off by default; while off, nothing leaves this machine", + }, + { + section: "logs", + key: "syslog_host", + label: "Host", + kind: "text", + nullable: true, + mono: true, + placeholder: "logs.home.lan", + hint: "The syslog server’s host name or IP address", + }, + { section: "logs", key: "syslog_port", label: "Port", kind: "int", min: 1, max: 65535 }, + { + section: "logs", + key: "syslog_transport", + label: "Transport", + kind: "select", + options: [ + { value: "udp", label: "UDP" }, + { value: "tcp", label: "TCP" }, + { value: "tls", label: "TLS" }, + ], + }, + { + section: "logs", + key: "syslog_level", + label: "Syslog level", + kind: "select", + options: [ + { value: "debug", label: "Debug" }, + { value: "info", label: "Info" }, + { value: "warning", label: "Warning" }, + { value: "error", label: "Error" }, + { value: "critical", label: "Critical" }, + ], + hint: "Only lines at this level or above are sent", + }, + { + section: "logs", + key: "syslog_facility", + label: "Facility", + kind: "select", + options: [ + "user", + "daemon", + "auth", + "syslog", + "local0", + "local1", + "local2", + "local3", + "local4", + "local5", + "local6", + "local7", + ].map((v) => ({ value: v, label: v })), + }, + { + section: "logs", + key: "syslog_format", + label: "Message format", + kind: "select", + options: [ + { value: "text", label: "Text" }, + { value: "json", label: "JSON record" }, + ], + }, + { + section: "logs", + key: "syslog_framing", + label: "TCP framing", + kind: "select", + options: [ + { value: "octet", label: "Octet counting (RFC 6587)" }, + { value: "newline", label: "One line per newline" }, + ], + }, + { section: "logs", key: "syslog_verify", label: "Verify the certificate", kind: "switch", hint: "For TLS" }, + { section: "logs", key: "syslog_app_name", label: "App name", kind: "text", mono: true }, { section: "tokens", key: "oauth_enabled", @@ -2772,6 +2908,7 @@ export function toUi(f: FieldSpec, v: unknown): unknown { return Array.isArray(v) ? v.map(String) : []; case "lines": if (f.section === "telemetry" && f.key === "prices") return pricesToLines(v); + if (f.section === "logs" && f.key === "levels") return levelsToLines(v); return Array.isArray(v) ? v.join("\n") : ""; case "days": return typeof v === "number" ? String(Math.round((v / 24) * 100) / 100) : ""; @@ -2811,6 +2948,34 @@ export function linesToPrices(text: string): Parsed { return { value: out }; } +export const LOG_LEVELS = ["debug", "info", "warning", "error", "critical"]; + +/** logs.levels as lines: "logger level". */ +export function levelsToLines(v: unknown): string { + if (!v || typeof v !== "object") return ""; + return Object.entries(v as Record) + .map(([name, level]) => `${name} ${level}`) + .join("\n"); +} + +/** Lines of "logger level" back to logs.levels. */ +export function linesToLevels(text: string): Parsed { + const out: Record = {}; + const lines = text + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + for (let i = 0; i < lines.length; i++) { + const [name, level, ...rest] = lines[i].split(/\s+/); + if (rest.length || !/^[A-Za-z0-9_.-]{1,100}$/.test(name) || !LOG_LEVELS.includes((level ?? "").toLowerCase())) + return { + error: `Line ${i + 1}: write the logger, then its level (${LOG_LEVELS.join(", ")}), like httpx warning`, + }; + out[name] = level.toLowerCase(); + } + return { value: out }; +} + /** Parses a control's value back to what the backend stores, checking it like the backend does. */ export function parse(f: FieldSpec, ui: unknown): Parsed { switch (f.kind) { @@ -2822,6 +2987,7 @@ export function parse(f: FieldSpec, ui: unknown): Parsed { return { value: ui as string[] }; case "lines": if (f.section === "telemetry" && f.key === "prices") return linesToPrices(String(ui ?? "")); + if (f.section === "logs" && f.key === "levels") return linesToLevels(String(ui ?? "")); return { value: String(ui ?? "") .split("\n") @@ -2965,6 +3131,14 @@ export function crossErrors(values: Record): Record) ?? {}).length; + return n ? `${n} logger${n === 1 ? "" : "s"}` : "none"; + } if (Array.isArray(v)) { if (f.kind === "checks" && f.section === "ai") return `${v.length} off`; return v.length > 3 ? `${v.length} ${f.kind === "lines" ? "lines" : "items"}` : v.join(", ") || "none"; @@ -3066,6 +3244,12 @@ export function why(c: Change): string | null { : "Scrapers must send the new token; stored encrypted."; if (id === "telemetry.headers") return c.after === "" ? "The headers are removed." : "Sent with every export; stored encrypted."; + if (id === "logs.syslog") + return c.after + ? "The server and its workers start sending their log lines to the syslog server within a few seconds." + : "Nothing more is sent; what was sent stays on the syslog server."; + if (id === "logs.level" || id === "logs.levels") + return "The server and its workers follow within a few seconds, without a restart."; if (id === "llm.api_key") return c.after === "" ? "The key is removed; LLM steps and chat stop until a new one is set." diff --git a/nextjs-frontend/components/settings/section.tsx b/nextjs-frontend/components/settings/section.tsx index 59efaf404..d4a9ae0d5 100644 --- a/nextjs-frontend/components/settings/section.tsx +++ b/nextjs-frontend/components/settings/section.tsx @@ -45,6 +45,8 @@ const ENV: Record = { "telemetry.enabled": "LENS_TELEMETRY", "telemetry.endpoint": "LENS_TELEMETRY_ENDPOINT", "telemetry.headers": "LENS_TELEMETRY_HEADERS", + "logs.level": "LENS_LOG_LEVEL", + "logs.format": "LENS_LOG_FORMAT", "telemetry.prometheus": "LENS_PROMETHEUS", "telemetry.prometheus_token": "LENS_PROMETHEUS_TOKEN", "telemetry.grafana_url": "LENS_GRAFANA_URL", diff --git a/nextjs-frontend/components/settings/settings-shell.tsx b/nextjs-frontend/components/settings/settings-shell.tsx index 53b91ce35..9aa3317e6 100644 --- a/nextjs-frontend/components/settings/settings-shell.tsx +++ b/nextjs-frontend/components/settings/settings-shell.tsx @@ -40,6 +40,7 @@ import { PackageCheck, RadioTower, Network, + ScrollText, } from "lucide-react"; import Link from "next/link"; import { useCallback, useState } from "react"; @@ -89,6 +90,7 @@ const ICON: Record = { slack: Hash, mailbox: AtSign, telemetry: Activity, + logs: ScrollText, fedora: Archive, sensors: RadioTower, nfs: FolderTree, diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index f40766bad..b010d26d3 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2956,6 +2956,201 @@ } } }, + "/api/v1/settings/logs/status": { + "get": { + "tags": [ + "admin" + ], + "summary": "Log Status", + "description": "The API process's log level, its log file, and how sending to syslog is going (off by default).", + "operationId": "log_status", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LogStatus" + } + } + } + } + } + } + }, + "/api/v1/settings/logs/test": { + "post": { + "tags": [ + "admin" + ], + "summary": "Test Logs", + "description": "Send one line to the saved syslog server now (on or off), to check the address, port and transport.", + "operationId": "test_logs", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LogTestResult" + } + } + } + } + } + } + }, + "/api/v1/admin/logs": { + "get": { + "tags": [ + "admin" + ], + "summary": "Read Logs", + "description": "The latest log lines from the API and every worker (their files under data_dir/logs), newest first. Lines are\nredacted as they're written: no keys, tokens or passwords.", + "operationId": "read_logs", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 2000, + "minimum": 1, + "default": 200, + "title": "Limit" + } + }, + { + "name": "level", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "debug", + "info", + "warning", + "error", + "critical" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "this level and above", + "title": "Level" + }, + "description": "this level and above" + }, + { + "name": "q", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 200 + }, + { + "type": "null" + } + ], + "description": "text the message or error contains", + "title": "Q" + }, + "description": "text the message or error contains" + }, + { + "name": "process", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 40 + }, + { + "type": "null" + } + ], + "description": "api, worker or watcher", + "title": "Process" + }, + "description": "api, worker or watcher" + }, + { + "name": "logger", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 100 + }, + { + "type": "null" + } + ], + "description": "a logger name, or the start of one (lens.email)", + "title": "Logger" + }, + "description": "a logger name, or the start of one (lens.email)" + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 40 + }, + { + "type": "null" + } + ], + "description": "a line's ts: only older lines (the next page)", + "title": "Before" + }, + "description": "a line's ts: only older lines (the next page)" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LogLine" + }, + "title": "Response Admin-Read Logs" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/audit": { "get": { "tags": [ @@ -39315,6 +39510,216 @@ ], "title": "LocalModelServer" }, + "LogLine": { + "properties": { + "ts": { + "type": "string", + "title": "Ts" + }, + "level": { + "type": "string", + "title": "Level" + }, + "logger": { + "type": "string", + "title": "Logger" + }, + "msg": { + "type": "string", + "title": "Msg" + }, + "process": { + "type": "string", + "title": "Process" + }, + "pid": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Pid" + }, + "host": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Host" + }, + "exc": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Exc" + }, + "fields": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Fields" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ts", + "level", + "logger", + "msg", + "process" + ], + "title": "LogLine", + "description": "One log record, redacted as it was written." + }, + "LogStatus": { + "properties": { + "level": { + "type": "string", + "title": "Level" + }, + "file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "File", + "description": "the API's log file, under data_dir/logs" + }, + "syslog": { + "type": "boolean", + "title": "Syslog", + "description": "whether lines are being sent to a syslog server" + }, + "target": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Target", + "description": "where, as transport://host:port" + }, + "sent": { + "type": "integer", + "title": "Sent" + }, + "dropped": { + "type": "integer", + "title": "Dropped", + "description": "lines dropped because the server couldn't keep up or was unreachable" + }, + "queued": { + "type": "integer", + "title": "Queued" + }, + "last_sent": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Last Sent" + }, + "last_error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Error" + }, + "error_at": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Error At" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "level", + "syslog", + "sent", + "dropped", + "queued" + ], + "title": "LogStatus", + "description": "What the API process does with its logs now; each worker follows the same settings on its own." + }, + "LogTestResult": { + "properties": { + "ok": { + "type": "boolean", + "title": "Ok" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ms" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ok" + ], + "title": "LogTestResult" + }, "LoginRequest": { "properties": { "email": {