diff --git a/CHANGELOG.md b/CHANGELOG.md
index 90116938e..a94986c05 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -143,6 +143,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
- bump the python-minor group ([#253](https://github.com/adeelahmad/lense/pull/253))
- bump the javascript-minor group ([#316](https://github.com/adeelahmad/lense/pull/316))
+- **Logs in the web app, and to a syslog server.** The API and every worker now write structured log lines with a
+ level set in **Settings → Logs** (and per logger, like `httpx warning`), to the console as text or JSON and to a file
+ per process under `data_dir/logs`. The same page shows the latest lines from all of them, with search and filters.
+ Turned on there, lines go to a syslog server too (RFC 5424 over UDP, TCP or TLS), from a background queue so a slow
+ server never holds Lens up; off by default. Keys, tokens and passwords are replaced with *** before any line is
+ written or sent ([Logging](docs/logging.md)).
+
## 0.7.0 October 10, 2026 {id="0.7.0"}
- **Prometheus metrics.** Lens can serve `/metrics` for Prometheus and Grafana: jobs by status and pipeline, imports,
diff --git a/docs/configuration.md b/docs/configuration.md
index ca5c1769e..198da1c5d 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -34,6 +34,7 @@ with AES-GCM and are write-only: the API reports whether one is set, never its v
| `LENS_SETUP_SETTINGS` | | settings to start with, as JSON `{"section": {"key": value}}` (or that JSON base64url-encoded): saved once at startup as if made in the app, so unlike the variables above they stay editable there. The [web installer](get-started.md#the-web-installer) writes its answers here; a `budget` section (`hard_usd`, `soft_usd`, `period`, `on_over`, `price_in`, `price_out`) sets [the overall budget](budgets.md#one-budget-for-everything) |
| `LENS_SETUP_WIZARD` | | `off`: never show the setup wizard |
| `LENS_TELEMETRY` / `LENS_TELEMETRY_ENDPOINT` / `LENS_TELEMETRY_HEADERS` | off | opt-in OpenTelemetry traces and metrics, sent only to this OTLP/HTTP endpoint ([Telemetry](telemetry.md)); `LENS_TELEMETRY=off` keeps it off whatever Settings say |
+| `LENS_LOG_LEVEL` / `LENS_LOG_FORMAT` | info / text | the log level and console format for the API and every worker ([Logging](logging.md)); win over Settings → Logs |
| `FRONTEND_URL` | `http://localhost:3000` | the web app's address as people use it: links in emails, and where apps send people to sign in ([OAuth](authentication.md#oauth)) |
| `CORS_ORIGINS` | `["http://localhost:3000"]` | origins allowed to call the API from a browser |
| `OPENAPI_URL` | `/openapi.json` | `""` disables `/docs` and the schema |
diff --git a/docs/logging.md b/docs/logging.md
new file mode 100644
index 000000000..21ff6088b
--- /dev/null
+++ b/docs/logging.md
@@ -0,0 +1,60 @@
+# Logging
+
+Every Lens process (the API, each `lens worker`, `lens watch`) writes one structured record per log line, and sends it
+to up to three places:
+
+- **The console**, which is what `docker logs` and `docker compose logs` show: as text, or as one JSON object per line
+ for a log shipper (**Format**).
+- **A file per process** under `data_dir/logs` (`api-.jsonl`, `worker-.jsonl`, …), one JSON object per
+ line, rotated at **File size** and kept **Files kept** times. **Settings → Logs** reads these, so its viewer shows the
+ workers' lines as well as the API's. A file no process has written for 14 days (an old container's) is removed.
+- **A syslog server**, off by default (below).
+
+Change any of it in **Settings → Logs**; the API and every worker follow within a few seconds, without a restart.
+`LENS_LOG_LEVEL` and `LENS_LOG_FORMAT` in .env win over Settings, which then show those fields locked.
+
+## What a record holds
+
+```json
+{"ts": "2026-10-10T20:02:19.809Z", "level": "warning", "logger": "lens.email", "msg": "…",
+ "process": "worker", "pid": 42, "host": "lens-worker-1", "exc": "Traceback …", "fields": {"job": 7}}
+```
+
+`exc` is there when the line carries an error's traceback, and `fields` when the code logged extra values.
+
+**No secrets.** Before a line goes anywhere it is redacted: bearer and basic credentials, `key=value` pairs whose key
+names a key, token, secret, password, cookie or authorization, the password in a URL (`https://user:***@host`), and
+well-known key shapes (`sk-…`, `ghp_…`, `xox…-`, JSON web tokens) become `***`. Extra fields whose name says key, token,
+secret or password are dropped to `***` too.
+
+## Levels
+
+**Level** (info by default) is what is kept on the console and in the files. **Logger levels** gives single loggers
+their own, one per line, like `httpx warning` to quiet every outgoing request, or `lens.bridge debug` to follow the
+chat-room bridge closely. Lens's own loggers start with `lens`; the web server's access log is `uvicorn.access`.
+
+## Sending to a syslog server
+
+Turn on **Send to syslog** and set the server's **Host** and **Port**. Lens speaks
+[RFC 5424](https://www.rfc-editor.org/rfc/rfc5424) over:
+
+| Transport | Standard | Framing |
+| --- | --- | --- |
+| UDP (default, port 514) | RFC 5426 | one line per datagram |
+| TCP | RFC 6587 | octet counting (``), or one line per newline for servers that want that |
+| TLS (usually port 6514) | RFC 5425 | as TCP; the server's certificate is checked unless **Verify the certificate** is off |
+
+Each message is `1 TIMESTAMP HOST APP-NAME PROCESS.PID LOGGER - MESSAGE`, where PRI combines **Facility**
+(local0 by default) with the line's level, APP-NAME is **App name** (`lens`), and MESSAGE is the line (and its
+traceback) or, with **Message format** JSON, the whole record above. Only lines at **Syslog level** or above are sent.
+Messages longer than 8 KB are cut.
+
+Lines are queued and sent from a background thread, so a slow or unreachable server never holds up Lens. While the
+server can't be reached Lens retries, waiting up to 30 seconds between tries, and once 10,000 lines are waiting new
+ones are dropped and counted. **Send a test line** sends one message to the saved server, even while sending is off,
+and Settings → Logs shows how many lines the API has sent and dropped and the last error.
+
+rsyslog, syslog-ng, Graylog, Grafana Loki (through promtail or Alloy's syslog receiver), Splunk, Papertrail and most
+log tools take RFC 5424 syslog.
+
+Not yet built: sending straight to Loki or another HTTP endpoint, and client certificates for TLS.
diff --git a/fastapi_backend/.env.example b/fastapi_backend/.env.example
index fb9b937fd..0a85ee2c9 100644
--- a/fastapi_backend/.env.example
+++ b/fastapi_backend/.env.example
@@ -57,5 +57,9 @@ CORS_ORIGINS=["http://localhost:3000"]
# LENS_TELEMETRY_ENDPOINT=http://localhost:4318 # your OpenTelemetry collector's OTLP/HTTP address
# LENS_TELEMETRY_HEADERS= # optional: key=value pairs, comma-separated, URL-encoded
+# --- Logging (docs/logging.md): level and console format for the API and every worker; win over Settings → Logs ---
+# LENS_LOG_LEVEL=info # debug, info, warning, error or critical
+# LENS_LOG_FORMAT=text # text, or json for one JSON object per line (for a log shipper)
+
# Uncomment to disable /docs and /openapi.json
# OPENAPI_URL=""
diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py
index 5da9e1c3c..8fab4347f 100644
--- a/fastapi_backend/app/api/v1/routes/admin.py
+++ b/fastapi_backend/app/api/v1/routes/admin.py
@@ -6,7 +6,7 @@
import pathlib
import shutil
import time
-from typing import Any
+from typing import Any, Literal
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Request
@@ -24,6 +24,7 @@
keyring,
llm,
local_llm,
+ logs,
machine,
mailbox,
matterbridge,
@@ -57,6 +58,9 @@
IndexQueued,
LlmTestResult,
LocalLlmStatus,
+ LogLine,
+ LogStatus,
+ LogTestResult,
MailboxStatus,
MailTestResult,
NfsStatus,
@@ -363,6 +367,36 @@ def test_telemetry(user: AdminWriter, cfg: Cfg) -> TelemetryTestResult:
return TelemetryTestResult(ok=ok, error=error, ms=ms)
+@router.get("/settings/logs/status")
+def log_status(user: AdminReader, cfg: Cfg) -> LogStatus:
+ """The API process's log level, its log file, and how sending to syslog is going (off by default)."""
+ return LogStatus.model_validate(logs.status(cfg))
+
+
+@router.post("/settings/logs/test")
+def test_logs(user: AdminWriter, cfg: Cfg) -> LogTestResult:
+ """Send one line to the saved syslog server now (on or off), to check the address, port and transport."""
+ ok, error, ms = logs.test_send(cfg)
+ return LogTestResult(ok=ok, error=error, ms=ms)
+
+
+@router.get("/admin/logs")
+def read_logs(
+ user: AdminReader,
+ cfg: Cfg,
+ limit: int = Query(200, ge=1, le=2000),
+ level: Literal["debug", "info", "warning", "error", "critical"] | None = Query(None, description="this level and above"),
+ q: str | None = Query(None, max_length=200, description="text the message or error contains"),
+ process: str | None = Query(None, max_length=40, description="api, worker or watcher"),
+ logger: str | None = Query(None, max_length=100, description="a logger name, or the start of one (lens.email)"),
+ before: str | None = Query(None, max_length=40, description="a line's ts: only older lines (the next page)"),
+) -> list[LogLine]:
+ """The latest log lines from the API and every worker (their files under data_dir/logs), newest first. Lines are
+ redacted as they're written: no keys, tokens or passwords."""
+ rows = logs.read(cfg, limit, level, q, process, logger, before)
+ return [LogLine.model_validate(r) for r in rows]
+
+
@router.get("/audit")
def list_audit(user: AdminReader, db: Db, limit: int = Query(200, ge=1, le=1000)) -> list[AuditEntry]:
"""Who changed what, newest first."""
diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py
index 629b2df12..6f74804da 100644
--- a/fastapi_backend/app/cli.py
+++ b/fastapi_backend/app/cli.py
@@ -266,6 +266,7 @@ def platform_main(argv, config):
https,
jobs,
local_llm,
+ logs,
nfs_share,
notify,
passkeys,
@@ -372,24 +373,26 @@ def password():
print(f"password changed for {a.email}")
elif a.cmd == "worker":
telemetry.set_role("worker")
- wk = jobs.Worker(db, C, a.name, a.steps.split(",") if a.steps else None, log=print)
+ logs.setup("worker") # every line as a record: the console, the log file and syslog (docs/logging.md)
+ say = logs.printer("lens.worker")
+ wk = jobs.Worker(db, C, a.name, a.steps.split(",") if a.steps else None, log=say)
if a.once:
print(f"ran {wk.drain()} job(s)")
else:
print(f"worker {wk.name} runs {', '.join(sorted(wk.can))}; Ctrl-C to stop")
stop = threading.Event()
- notify.start(db, C, stop, name=wk.name, log=print) # sends notifications too (docs/notifications.md)
+ notify.start(db, C, stop, name=wk.name, log=say) # sends notifications too (docs/notifications.md)
if not (a.steps or a.no_schedule): # and scans watched folders and runs routines, as `lens watch` does
- routines.start(db, C, stop, log=print)
- sensors.start(db, C, stop, log=print, name=wk.name) # and the sensor hub, while sensors are on
- smb_share.start(db, C, stop, name=wk.name, log=print) # and the SMB share of the file tree, while it's on
- shares.start(db, C, stop, log=print, name=wk.name) # and the file shares (S3, WebDAV), while they're on
- channels.start_all(db, C, stop, lambda: cfg, name=wk.name, log_fn=print) # and the chat channels, while they're on
- tunnel.start(db, C, stop, name=wk.name, log=print) # and the Cloudflare tunnel, while it's on
- https.start(db, C, stop, name=wk.name, log=print) # and HTTPS, where the ports are (LENS_HTTPS_LISTEN)
- local_llm.start(db, C, stop, name=wk.name, log=print) # and the local chat model, while it's on
- nfs_share.start(db, C, stop, name=wk.name, log=print) # and the file tree over NFS, while it's on
- updates.start(db, C, stop, log=print) # and the daily check for a new release (docs/updates.md)
+ routines.start(db, C, stop, log=say)
+ sensors.start(db, C, stop, log=say, name=wk.name) # and the sensor hub, while sensors are on
+ smb_share.start(db, C, stop, name=wk.name, log=say) # and the SMB share of the file tree, while it's on
+ shares.start(db, C, stop, log=say, name=wk.name) # and the file shares (S3, WebDAV), while they're on
+ channels.start_all(db, C, stop, lambda: cfg, name=wk.name, log_fn=say) # and the chat channels, while they're on
+ tunnel.start(db, C, stop, name=wk.name, log=say) # and the Cloudflare tunnel, while it's on
+ https.start(db, C, stop, name=wk.name, log=say) # and HTTPS, where the ports are (LENS_HTTPS_LISTEN)
+ local_llm.start(db, C, stop, name=wk.name, log=say) # and the local chat model, while it's on
+ nfs_share.start(db, C, stop, name=wk.name, log=say) # and the file tree over NFS, while it's on
+ updates.start(db, C, stop, log=say) # and the daily check for a new release (docs/updates.md)
_stop_on_term()
try:
wk.loop(stop)
@@ -401,13 +404,15 @@ def password():
print(f"ran {routines.run_due(db, C(), print)} routine(s)")
else:
telemetry.set_role("watcher")
+ logs.setup("watcher")
+ say = logs.printer("lens.watch")
print("watching storage sources and running routines; Ctrl-C to stop")
_stop_on_term()
- sensors.start(db, C, threading.Event(), log=print, name=f"watch-{os.getpid()}")
+ sensors.start(db, C, threading.Event(), log=say, name=f"watch-{os.getpid()}")
try:
while True:
- sources.poll_due(db, C(), print)
- routines.run_due(db, C(), print)
+ sources.poll_due(db, C(), say)
+ routines.run_due(db, C(), say)
time.sleep(max(5, min(C()["sources"]["check_seconds"], routines.CHECK_SECONDS)))
except KeyboardInterrupt:
pass
diff --git a/fastapi_backend/app/domain/logs.py b/fastapi_backend/app/domain/logs.py
new file mode 100644
index 000000000..5032b9641
--- /dev/null
+++ b/fastapi_backend/app/domain/logs.py
@@ -0,0 +1,577 @@
+"""Logging: one structured record for every line the API and each worker write, kept on this machine, shown in the web
+app, and sent to a syslog server only when an admin turns that on (docs/logging.md).
+
+Every process (the API, each `lens worker`, `lens watch`) logs through Python's logging with three outputs:
+
+* the console (docker logs), as text or as one JSON object per line (logs.format);
+* a file per process under data_dir/logs, one JSON object per line, rotated at logs.file_mb and kept logs.file_keep
+ times; the web app's log viewer reads these (`read`), so it shows the workers' lines too;
+* a syslog server (RFC 5424 over UDP, TCP or TLS), off by default: lines go through a queue to a background thread, so
+ a slow or missing server never holds up a request; when the queue is full new lines are dropped and counted.
+
+Before any line leaves the process it is redacted: bearer and basic credentials, key=value pairs naming a key, token,
+secret or password, the password in a URL, and well-known key shapes (sk-…, ghp_…, xox…-, JWTs) become ***.
+
+Settings are re-read when someone saves them (settings.Settings.current calls `apply`), so a change applies to the API
+and every worker without a restart. LENS_LOG_LEVEL and LENS_LOG_FORMAT in .env win over the app.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import datetime
+import ipaddress
+import json
+import logging
+import logging.handlers
+import os
+import pathlib
+import queue
+import re
+import socket
+import ssl
+import sys
+import threading
+import time
+
+from . import __version__
+
+LEVELS = {"debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR, "critical": logging.CRITICAL}
+NAMES = {v: k for k, v in LEVELS.items()}
+# RFC 5424 severities
+SEVERITY = {logging.DEBUG: 7, logging.INFO: 6, logging.WARNING: 4, logging.ERROR: 3, logging.CRITICAL: 2}
+FACILITIES = {"user": 1, "daemon": 3, "auth": 4, "syslog": 5, **{f"local{i}": 16 + i for i in range(8)}}
+TRANSPORTS = ("udp", "tcp", "tls")
+MAX_BYTES = 8192 # the most of one line sent to syslog (rsyslog's default maxMessageSize)
+QUEUE = 10_000 # lines waiting for the syslog server before new ones are dropped
+PRUNE_DAYS = 14 # a log file no process has written for this long (an old container's) is removed
+LOGGER_RX = re.compile(r"^[A-Za-z0-9_.-]{1,100}$")
+_STD = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime", "lens"}
+
+# what is never written out: the match keeps its first group and the rest becomes ***
+_SECRETS = [
+ re.compile(r"(?i)\b((?:bearer|basic)\s+)[A-Za-z0-9._~+/=-]{6,}"),
+ re.compile(
+ r"(?i)\b((?:api[_-]?key|apikey|access[_-]?token|refresh[_-]?token|id[_-]?token|auth[_-]?token|token|secret|"
+ r"client[_-]?secret|password|passwd|pwd|authorization|cookie|set-cookie|x-api-key)[\"']?\s*[:=]\s*[\"']?)"
+ r"(?!\*\*\*)[^\s\"'&,;}]+"
+ ),
+ re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://[^/\s:@]+:)[^/\s@]+(?=@)"),
+ re.compile(r"()\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{10,})"),
+ re.compile(r"()\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"),
+]
+_SECRET_KEY = re.compile(r"(?i)(key|token|secret|password|passwd|authorization|cookie)")
+
+
+def redact(text):
+ """The text with credentials replaced by ***."""
+ if not text:
+ return text
+ for rx in _SECRETS:
+ text = rx.sub(lambda m: m.group(1) + "***", text)
+ return text
+
+
+_role = ["api"]
+_lock = threading.Lock()
+_state = {"key": None, "cfg": None, "console": None, "file": None, "syslog": None, "levels": {}}
+_host = socket.gethostname()
+
+
+def _clean(value, limit=4000):
+ if isinstance(value, (bool, int, float)) or value is None:
+ return value
+ return redact(str(value))[:limit]
+
+
+def record(r):
+ """A log record as the dict written to the file and sent as JSON: redacted once, then shared by every output."""
+ got = getattr(r, "lens", None)
+ if got is not None:
+ return got
+ try:
+ msg = r.getMessage()
+ except Exception: # noqa: BLE001 - a bad format string still logs something
+ msg = f"{r.msg} {r.args}"
+ out = {
+ "ts": datetime.datetime.fromtimestamp(r.created, datetime.UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
+ "level": NAMES.get(r.levelno, r.levelname.lower()),
+ "logger": r.name,
+ "msg": redact(msg),
+ "process": _role[0],
+ "pid": r.process,
+ "host": _host,
+ }
+ if r.exc_info and r.exc_info[0] is not None:
+ out["exc"] = redact(logging.Formatter().formatException(r.exc_info))
+ elif r.exc_text:
+ out["exc"] = redact(r.exc_text)
+ fields = {k: ("***" if _SECRET_KEY.search(k) else _clean(v)) for k, v in r.__dict__.items() if k not in _STD and not k.startswith("_")}
+ if fields:
+ out["fields"] = fields
+ r.lens = out
+ return out
+
+
+class JsonFormatter(logging.Formatter):
+ def format(self, r):
+ return json.dumps(record(r), ensure_ascii=False, default=str)
+
+
+class TextFormatter(logging.Formatter):
+ def format(self, r):
+ d = record(r)
+ line = f"{d['ts']} {d['level'].upper()} {d['process']} {d['logger']}: {d['msg']}"
+ if d.get("fields"):
+ line += " " + " ".join(f"{k}={v}" for k, v in d["fields"].items())
+ return line + ("\n" + d["exc"] if d.get("exc") else "")
+
+
+_gate = {"level": logging.INFO, "levels": {}}
+
+
+def _override(name):
+ """The level an admin gave this logger or the nearest of its parents (logs.levels), or None."""
+ levels = _gate["levels"]
+ while name:
+ if name in levels:
+ return levels[name]
+ name = name.rpartition(".")[0]
+ return None
+
+
+class Gate(logging.Filter):
+ """What an output keeps: the logger's own level when it has one, else logs.level (the console and the file); for
+ syslog, syslog_level too."""
+
+ def __init__(self, floor=None):
+ super().__init__()
+ self.floor = floor
+
+ def filter(self, r):
+ own = _override(r.name)
+ if self.floor is not None:
+ return r.levelno >= self.floor and (own is None or r.levelno >= own)
+ return r.levelno >= (own if own is not None else _gate["level"])
+
+
+def _ours(h):
+ return getattr(h, "_lens", False)
+
+
+def _tag(h):
+ h._lens = True
+ return h
+
+
+def set_role(role):
+ """What this process is (api, worker, watcher): each record says it, and it names the process's log file."""
+ _role[0] = role
+
+
+def _env_level():
+ return LEVELS.get((os.environ.get("LENS_LOG_LEVEL") or "").strip().lower(), logging.INFO)
+
+
+def _env_format():
+ f = (os.environ.get("LENS_LOG_FORMAT") or "").strip().lower()
+ return f if f in ("text", "json") else "text"
+
+
+def setup(role=None):
+ """Log to the console now, before the settings are read (called once per process as it starts)."""
+ again = bool(role) and role != _role[0] and _state["cfg"] is not None
+ if role:
+ set_role(role)
+ root = logging.getLogger()
+ with _lock:
+ if _state["console"] is None:
+ h = _tag(logging.StreamHandler(sys.stderr))
+ h.addFilter(Gate())
+ root.addHandler(h)
+ _state["console"] = h
+ _state["console"].setFormatter(JsonFormatter() if _env_format() == "json" else TextFormatter())
+ if _state["key"] is None:
+ _gate["level"] = _env_level()
+ root.setLevel(_env_level())
+ # uvicorn's own lines (the access log, startup and errors) go through these outputs too, in the same form
+ for name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
+ lg = logging.getLogger(name)
+ lg.handlers = [h for h in lg.handlers if _ours(h)]
+ lg.propagate = True
+ if again: # the settings were read before this process knew what it is: its file is named for it now
+ _close("file")
+ _state["key"] = None
+ apply(_state["cfg"])
+
+
+def printer(name="lens"):
+ """A `log=print` replacement for the long-running loops (the worker, the watcher): their lines become records."""
+ lg = logging.getLogger(name)
+
+ def log(*args, **_):
+ lg.info(" ".join(str(a) for a in args))
+
+ return log
+
+
+def section(cfg):
+ return (cfg or {}).get("logs") or {}
+
+
+def level(name, default=logging.INFO):
+ return LEVELS.get(str(name or "").lower(), default)
+
+
+def _wanted(cfg):
+ s = section(cfg)
+ data_dir = (cfg or {}).get("data_dir")
+ ship = None
+ if s.get("syslog") and s.get("syslog_host"):
+ ship = (
+ str(s["syslog_host"]),
+ int(s.get("syslog_port") or 514),
+ s.get("syslog_transport") if s.get("syslog_transport") in TRANSPORTS else "udp",
+ level(s.get("syslog_level")),
+ FACILITIES.get(s.get("syslog_facility"), FACILITIES["local0"]),
+ s.get("syslog_format") if s.get("syslog_format") in ("text", "json") else "text",
+ s.get("syslog_framing") if s.get("syslog_framing") in ("octet", "newline") else "octet",
+ s.get("syslog_verify") is not False,
+ s.get("syslog_app_name") or "lens",
+ )
+ levels = s.get("levels") if isinstance(s.get("levels"), dict) else {}
+ return (
+ level(s.get("level")),
+ s.get("format") if s.get("format") in ("text", "json") else "text",
+ tuple(sorted((k, level(v)) for k, v in levels.items() if isinstance(k, str) and LOGGER_RX.match(k))),
+ (str(pathlib.Path(data_dir) / "logs"), int(s.get("file_mb") or 5), int(s.get("file_keep") or 2))
+ if data_dir and s.get("file", True)
+ else None,
+ ship,
+ )
+
+
+def file_name(role=None):
+ host = re.sub(r"[^A-Za-z0-9_.-]", "-", _host)[:64] or "host"
+ return f"{role or _role[0]}-{host}.jsonl"
+
+
+def apply(cfg):
+ """Make this process's logging match the settings. Cheap when nothing changed."""
+ want = _wanted(cfg)
+ _state["cfg"] = cfg
+ if want == _state["key"]:
+ return
+ with _lock:
+ if want == _state["key"]:
+ return
+ lvl, fmt, levels, file, ship = want
+ root = logging.getLogger()
+ if _state["console"] is None:
+ _state["console"] = _tag(logging.StreamHandler(sys.stderr))
+ _state["console"].addFilter(Gate())
+ root.addHandler(_state["console"])
+ _state["console"].setFormatter(JsonFormatter() if fmt == "json" else TextFormatter())
+ _gate["level"], _gate["levels"] = lvl, dict(levels)
+ old = _state["key"]
+ if file != (old[3] if old else None):
+ _close("file")
+ if file is not None:
+ folder, mb, keep = file
+ try:
+ pathlib.Path(folder).mkdir(parents=True, exist_ok=True)
+ _prune(folder)
+ h = _tag(
+ logging.handlers.RotatingFileHandler(
+ pathlib.Path(folder) / file_name(), maxBytes=mb * 1024 * 1024, backupCount=keep, encoding="utf-8"
+ )
+ )
+ h.setFormatter(JsonFormatter())
+ h.addFilter(Gate())
+ root.addHandler(h)
+ _state["file"] = h
+ except OSError as e:
+ sys.stderr.write(f"logs: can't write to {folder} ({e}); the log viewer won't show this process\n")
+ if ship != (old[4] if old else None):
+ _close("syslog")
+ if ship is not None:
+ h = _tag(SyslogHandler(*ship))
+ root.addHandler(h)
+ _state["syslog"] = h
+ for name in set(_state["levels"]) - {k for k, _ in levels}:
+ logging.getLogger(name).setLevel(logging.NOTSET) # an override taken away: back to the root's level
+ for name, v in levels:
+ logging.getLogger(name).setLevel(v)
+ _state["levels"] = dict(levels)
+ lowest = min([lvl, *(v for _, v in levels), *([ship[3]] if ship else [])])
+ root.setLevel(lowest)
+ _state["key"] = want
+
+
+def _close(which):
+ h = _state[which]
+ _state[which] = None
+ if h is not None:
+ logging.getLogger().removeHandler(h)
+ with contextlib.suppress(Exception):
+ h.close()
+
+
+def shutdown():
+ """Stop the syslog thread and close the file (tests; atexit does it otherwise)."""
+ with _lock:
+ _close("syslog")
+ _close("file")
+ _state["key"] = None
+
+
+def _prune(folder):
+ cutoff = time.time() - PRUNE_DAYS * 86400
+ for p in pathlib.Path(folder).glob("*.jsonl*"):
+ with contextlib.suppress(OSError):
+ if p.stat().st_mtime < cutoff:
+ p.unlink()
+
+
+# --- syslog (RFC 5424, sent over UDP per RFC 5426, TCP per RFC 6587 or TLS per RFC 5425) ---
+
+
+def _printable(s, n):
+ s = "".join(c if 33 <= ord(c) <= 126 else "_" for c in str(s))
+ return s[:n] or "-"
+
+
+def rfc5424(r, facility, fmt, app_name, host=None):
+ """One record as an RFC 5424 message (bytes, without framing)."""
+ d = record(r)
+ pri = facility * 8 + SEVERITY.get(r.levelno, 6)
+ head = " ".join(
+ [
+ f"<{pri}>1",
+ d["ts"],
+ _printable(host or _host, 255),
+ _printable(app_name, 48),
+ _printable(f"{d['process']}.{d['pid']}", 128),
+ _printable(d["logger"], 32),
+ "-",
+ ]
+ )
+ if fmt == "json":
+ body = json.dumps(d, ensure_ascii=False, default=str)
+ else:
+ body = d["msg"] + ("\n" + d["exc"] if d.get("exc") else "")
+ return (head + " " + body).encode("utf-8", "replace")[:MAX_BYTES]
+
+
+class Sender:
+ """A connection to a syslog server; `send` raises OSError when the line couldn't go."""
+
+ def __init__(self, host, port, transport, framing="octet", verify=True, timeout=5.0):
+ self.host, self.port, self.transport, self.framing, self.verify, self.timeout = host, port, transport, framing, verify, timeout
+ self.sock = None
+
+ def _connect(self):
+ infos = socket.getaddrinfo(self.host, self.port, type=socket.SOCK_DGRAM if self.transport == "udp" else socket.SOCK_STREAM)
+ err = None
+ for family, kind, proto, _, addr in infos:
+ if ipaddress.ip_address(addr[0].split("%")[0]).is_link_local:
+ err = OSError(f"{self.host} is a link-local address; Lens doesn't send logs there")
+ continue
+ s = socket.socket(family, kind, proto)
+ s.settimeout(self.timeout)
+ try:
+ s.connect(addr)
+ if self.transport == "tls":
+ ctx = ssl.create_default_context()
+ if not self.verify:
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ s = ctx.wrap_socket(s, server_hostname=self.host)
+ return s
+ except OSError as e:
+ err = e
+ s.close()
+ raise err or OSError(f"couldn't find {self.host}")
+
+ def send(self, msg):
+ if self.sock is None:
+ self.sock = self._connect()
+ try:
+ if self.transport == "udp":
+ self.sock.send(msg)
+ elif self.framing == "newline":
+ self.sock.sendall(msg.replace(b"\n", b" ") + b"\n")
+ else:
+ self.sock.sendall(str(len(msg)).encode() + b" " + msg)
+ except OSError:
+ self.close()
+ raise
+
+ def close(self):
+ if self.sock is not None:
+ with contextlib.suppress(OSError):
+ self.sock.close()
+ self.sock = None
+
+
+class SyslogHandler(logging.Handler):
+ """Queues each record and sends it from a background thread; never blocks the caller."""
+
+ def __init__(self, host, port, transport, lvl, facility, fmt, framing, verify, app_name):
+ super().__init__()
+ self.addFilter(Gate(lvl))
+ self.facility, self.fmt, self.app_name = facility, fmt, app_name
+ self.target = f"{transport}://{host}:{port}"
+ self.sender = Sender(host, port, transport, framing, verify)
+ self.q = queue.Queue(QUEUE)
+ self.stats = {"sent": 0, "dropped": 0, "last_sent": None, "last_error": None, "error_at": None}
+ self._stop = threading.Event()
+ self._thread = threading.Thread(target=self._run, name="lens-syslog", daemon=True)
+ self._thread.start()
+
+ def emit(self, r):
+ try:
+ self.q.put_nowait(rfc5424(r, self.facility, self.fmt, self.app_name))
+ except queue.Full:
+ self.stats["dropped"] += 1
+ except Exception: # noqa: BLE001 - logging never raises into the caller
+ self.handleError(r)
+
+ def _run(self):
+ wait = 1.0
+ while not self._stop.is_set():
+ try:
+ msg = self.q.get(timeout=0.5)
+ except queue.Empty:
+ continue
+ if msg is None:
+ break
+ while not self._stop.is_set():
+ try:
+ self.sender.send(msg)
+ self.stats["sent"] += 1
+ self.stats["last_sent"] = time.time()
+ wait = 1.0
+ break
+ except OSError as e:
+ # never logged (that would come back here): the status in Settings → Logs shows it
+ self.stats["last_error"] = f"{type(e).__name__}: {e}"[:300]
+ self.stats["error_at"] = time.time()
+ self._stop.wait(wait)
+ wait = min(wait * 2, 30.0)
+ self.sender.close()
+
+ def close(self):
+ self._stop.set()
+ with contextlib.suppress(queue.Full):
+ self.q.put_nowait(None)
+ self._thread.join(timeout=2)
+ super().close()
+
+
+def status(cfg):
+ """What this process does with its logs now."""
+ s = section(cfg)
+ h = _state["syslog"]
+ f = _state["file"]
+ return {
+ "level": NAMES.get(level(s.get("level")), "info"),
+ "file": f.baseFilename if f is not None else None,
+ "syslog": bool(h),
+ "target": h.target if h else None,
+ "sent": h.stats["sent"] if h else 0,
+ "dropped": h.stats["dropped"] if h else 0,
+ "queued": h.q.qsize() if h else 0,
+ "last_sent": h.stats["last_sent"] if h else None,
+ "last_error": h.stats["last_error"] if h else None,
+ "error_at": h.stats["error_at"] if h else None,
+ }
+
+
+def test_send(cfg):
+ """Send one line to the saved syslog server now (on or off): (ok, error, ms)."""
+ s = section(cfg)
+ if not s.get("syslog_host"):
+ return False, "set the syslog server's address first", None
+ transport = s.get("syslog_transport") if s.get("syslog_transport") in TRANSPORTS else "udp"
+ sender = Sender(
+ str(s["syslog_host"]),
+ int(s.get("syslog_port") or 514),
+ transport,
+ s.get("syslog_framing") or "octet",
+ s.get("syslog_verify") is not False,
+ )
+ r = logging.LogRecord("lens.logs", logging.INFO, __file__, 0, "Lens %s: test message from Settings → Logs", (__version__,), None)
+ t = time.monotonic()
+ try:
+ sender.send(
+ rfc5424(r, FACILITIES.get(s.get("syslog_facility"), 16), s.get("syslog_format") or "text", s.get("syslog_app_name") or "lens")
+ )
+ return True, None, int((time.monotonic() - t) * 1000)
+ except (OSError, ValueError) as e:
+ return False, f"{type(e).__name__}: {e}"[:300], None
+ finally:
+ sender.close()
+
+
+# --- reading the files back, for the log viewer ---
+
+
+def _lines_backwards(path, chunk=65536):
+ with open(path, "rb") as f:
+ f.seek(0, os.SEEK_END)
+ end = f.tell()
+ rest = b""
+ while end > 0:
+ n = min(chunk, end)
+ end -= n
+ f.seek(end)
+ buf = f.read(n) + rest
+ parts = buf.split(b"\n")
+ rest = parts[0]
+ for line in reversed(parts[1:]):
+ if line.strip():
+ yield line
+ if rest.strip():
+ yield rest
+
+
+def files(cfg):
+ folder = pathlib.Path((cfg or {}).get("data_dir") or ".") / "logs"
+ return sorted(folder.glob("*.jsonl*")) if folder.is_dir() else []
+
+
+def read(cfg, limit=200, min_level=None, q=None, process=None, logger=None, before=None):
+ """The newest log records from every process's file, newest first, matching the filters. `before` is a record's
+ ts: only older ones (the next page)."""
+ lo = level(min_level, logging.DEBUG) if min_level else logging.DEBUG
+ needle = (q or "").lower()
+ found = []
+ for p in files(cfg):
+ got = 0
+ try:
+ for raw in _lines_backwards(p):
+ try:
+ d = json.loads(raw)
+ except ValueError:
+ continue
+ if not isinstance(d, dict) or not isinstance(d.get("ts"), str):
+ continue
+ if before and d["ts"] >= before:
+ continue
+ if LEVELS.get(d.get("level"), logging.INFO) < lo:
+ continue
+ if process and d.get("process") != process:
+ continue
+ if logger and not str(d.get("logger", "")).startswith(logger):
+ continue
+ if needle and needle not in (str(d.get("msg", "")) + " " + str(d.get("exc", ""))).lower():
+ continue
+ found.append(d)
+ got += 1
+ if got >= limit:
+ break
+ except OSError:
+ continue
+ found.sort(key=lambda d: d["ts"], reverse=True)
+ return found[:limit]
diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py
index eb67539fe..9ed4a99cd 100644
--- a/fastapi_backend/app/domain/settings.py
+++ b/fastapi_backend/app/domain/settings.py
@@ -19,7 +19,7 @@
import threading
import urllib.parse
-from . import convert, ipgroups, mail_routes, objects, store, telemetry
+from . import convert, ipgroups, logs, mail_routes, objects, store, telemetry
R = store.R
EDITABLE = {
@@ -50,6 +50,7 @@
"matterbridge": None,
"notifications": None,
"telemetry": None,
+ "logs": None,
# repo is a startup setting only (what gets installed isn't chosen in the web app)
"updates": ("check", "check_hours", "auto_install", "email"),
"activity": None,
@@ -172,6 +173,13 @@
("https", "challenge"): {"http", "tls-alpn", "dns"},
("files", "store"): {"local", "connection"},
("documents", "converter"): {"auto", "anytopdf", "lens"},
+ ("logs", "level"): set(logs.LEVELS),
+ ("logs", "syslog_level"): set(logs.LEVELS),
+ ("logs", "format"): {"text", "json"},
+ ("logs", "syslog_format"): {"text", "json"},
+ ("logs", "syslog_transport"): set(logs.TRANSPORTS),
+ ("logs", "syslog_facility"): set(logs.FACILITIES),
+ ("logs", "syslog_framing"): {"octet", "newline"},
}
# Settings the environment (.env) sets, which win over archive.yaml and the app and show as locked there: the
# break-glass allowed hosts, the model provider so an install can be configured without the setup wizard, and
@@ -195,6 +203,8 @@
("telemetry", "enabled"): "LENS_TELEMETRY",
("telemetry", "endpoint"): "LENS_TELEMETRY_ENDPOINT",
("telemetry", "headers"): "LENS_TELEMETRY_HEADERS",
+ ("logs", "level"): "LENS_LOG_LEVEL",
+ ("logs", "format"): "LENS_LOG_FORMAT",
("telemetry", "prometheus"): "LENS_PROMETHEUS",
("telemetry", "prometheus_token"): "LENS_PROMETHEUS_TOKEN",
("telemetry", "grafana_url"): "LENS_GRAFANA_URL",
@@ -334,6 +344,7 @@ def current(self):
if self._cfg is None or v != self._ver:
self._cfg, self._ver = effective(self.db, self.base), v
telemetry.apply(self._cfg) # turned on, changed or off in the app: this process follows
+ logs.apply(self._cfg) # and its logging (level, file, syslog)
return self._cfg
@@ -439,6 +450,8 @@ def _check(section, key, value, default):
return _notify_setting(key, value)
if section == "telemetry":
return _telemetry_setting(key, value)
+ if section == "logs":
+ return _logs_setting(key, value)
if section == "updates":
return _updates_setting(key, value)
if section == "activity":
@@ -1043,6 +1056,54 @@ def _sensor_setting(key, value):
return value
+LOG_RANGES = {"file_mb": (1, 100), "file_keep": (1, 10), "syslog_port": (1, 65535)}
+APP_NAME_RX = re.compile(r"^[!-~]{1,48}$")
+HOST_RX = re.compile(r"^[A-Za-z0-9.:_-]{1,253}$")
+
+
+def _logs_setting(key, value):
+ if ("logs", key) in ENUMS: # checked against ENUMS already
+ return value
+ if key in ("file", "syslog", "syslog_verify"):
+ if not isinstance(value, bool):
+ raise ValueError(f"logs.{key} is true or false")
+ return value
+ if key == "levels":
+ if value is None:
+ return {}
+ if not isinstance(value, dict) or len(value) > 100:
+ raise ValueError('logs.levels gives loggers their own level, like {"httpx": "warning"}')
+ out = {}
+ for name, lvl in value.items():
+ if not logs.LOGGER_RX.match(str(name)) or str(lvl).lower() not in logs.LEVELS:
+ raise ValueError(f"logs.levels: {name} needs a logger name and one of: {', '.join(logs.LEVELS)}")
+ out[str(name)] = str(lvl).lower()
+ return out
+ if key == "syslog_host":
+ if value in (None, ""):
+ return None
+ v = str(value).strip().strip("[]")
+ if not HOST_RX.match(v):
+ raise ValueError("logs.syslog_host is the syslog server's host name or IP address, without a port")
+ try:
+ link_local = ipaddress.ip_address(v).is_link_local
+ except ValueError:
+ if ":" in v: # only an IPv6 address has colons; the port is logs.syslog_port
+ raise ValueError("logs.syslog_host is the syslog server's host name or IP address, without a port") from None
+ link_local = False
+ if link_local:
+ raise ValueError("logs.syslog_host can't be a link-local address (169.254.0.0/16, fe80::/10)")
+ return v
+ if key == "syslog_app_name":
+ if not (isinstance(value, str) and APP_NAME_RX.match(value.strip())):
+ raise ValueError("logs.syslog_app_name is up to 48 printable characters, without spaces")
+ return value.strip()
+ lo, hi = LOG_RANGES[key]
+ if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi):
+ raise ValueError(f"logs.{key} is a whole number from {lo} to {hi}")
+ return value
+
+
def _nfs_setting(key, value):
if key == "enabled":
if not isinstance(value, bool):
@@ -1267,6 +1328,8 @@ def save(db, base, section, changes, user=None):
# the saved headers (an auth token) were for the old collector: they don't go to a new host
if before:
sealed.pop("headers", None)
+ if section == "logs" and data.get("syslog") and not data.get("syslog_host", (base.get("logs") or {}).get("syslog_host")):
+ raise ValueError("set the syslog server's address (logs.syslog_host) to send logs to it")
if section == "speakers":
m = data.get("match_threshold", defaults["match_threshold"])
r = data.get("review_threshold", defaults["review_threshold"])
diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
index 6062f68ae..5cb28a45b 100644
--- a/fastapi_backend/app/domain/store.py
+++ b/fastapi_backend/app/domain/store.py
@@ -613,6 +613,27 @@ def labels():
# where Grafana shows Lens's charts (docs/prometheus.md), for a link on Admin → System health; none: no link
"grafana_url": None,
},
+ # logging (docs/logging.md): level and format for the console (docker logs) and a file per process under
+ # data_dir/logs (what Settings → Logs shows); levels sets single loggers, like {"httpx": "warning"}. syslog sends
+ # every line at syslog_level or above to a syslog server (RFC 5424 over udp, tcp or tls); off by default
+ "logs": {
+ "level": "info",
+ "format": "text",
+ "levels": {},
+ "file": True,
+ "file_mb": 5,
+ "file_keep": 2,
+ "syslog": False,
+ "syslog_host": None,
+ "syslog_port": 514,
+ "syslog_transport": "udp",
+ "syslog_level": "info",
+ "syslog_facility": "local0",
+ "syslog_format": "text",
+ "syslog_framing": "octet",
+ "syslog_verify": True,
+ "syslog_app_name": "lens",
+ },
# new releases (docs/updates.md): check asks GitHub for repo's latest release every check_hours, for the web app's
# banner; auto_install installs one as soon as it's seen, through the machine's upgrade agent (install.sh sets it up)
# email: tell admins by email when a new release is seen (once per release; needs Settings › Email). Namespaces'
diff --git a/fastapi_backend/app/main.py b/fastapi_backend/app/main.py
index ba386c867..cbb7c692e 100644
--- a/fastapi_backend/app/main.py
+++ b/fastapi_backend/app/main.py
@@ -7,7 +7,6 @@
from __future__ import annotations
import contextlib
-import logging
from collections.abc import AsyncIterator
from typing import Any
@@ -27,11 +26,11 @@
from app.config import settings
from app.core import middleware
from app.core.runtime import Archive
-from app.domain import __version__, activity, store, telemetry
+from app.domain import __version__, activity, logs, store, telemetry
from app.domain.render import WEB_DIR
from app.utils import simple_generate_unique_route_id
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logs.setup("api") # the console now; the level, the log file and syslog follow the settings once they're read
def _attach(app: FastAPI, archive: Archive) -> None:
@@ -60,6 +59,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
finally:
archive.close()
telemetry.shutdown()
+ logs.shutdown()
app = FastAPI(
title=settings.PROJECT_NAME,
diff --git a/fastapi_backend/app/schemas/admin.py b/fastapi_backend/app/schemas/admin.py
index e8272dfc3..b3835ed40 100644
--- a/fastapi_backend/app/schemas/admin.py
+++ b/fastapi_backend/app/schemas/admin.py
@@ -346,6 +346,41 @@ class TelemetryTestResult(ResponseModel):
ms: int | None = None
+class LogStatus(ResponseModel):
+ """What the API process does with its logs now; each worker follows the same settings on its own."""
+
+ level: str
+ file: str | None = Field(None, description="the API's log file, under data_dir/logs")
+ syslog: bool = Field(description="whether lines are being sent to a syslog server")
+ target: str | None = Field(None, description="where, as transport://host:port")
+ sent: int
+ dropped: int = Field(description="lines dropped because the server couldn't keep up or was unreachable")
+ queued: int
+ last_sent: float | None = None
+ last_error: str | None = None
+ error_at: float | None = None
+
+
+class LogTestResult(ResponseModel):
+ ok: bool
+ error: str | None = None
+ ms: int | None = None
+
+
+class LogLine(ResponseModel):
+ """One log record, redacted as it was written."""
+
+ ts: str
+ level: str
+ logger: str
+ msg: str
+ process: str
+ pid: int | None = None
+ host: str | None = None
+ exc: str | None = None
+ fields: dict[str, Any] | None = None
+
+
class FileStoreTry(RequestModel):
"""Where to try keeping files, before saving it (Settings → Storage); nothing given tries what is saved."""
diff --git a/fastapi_backend/tests/api/test_logs.py b/fastapi_backend/tests/api/test_logs.py
new file mode 100644
index 000000000..69ee7edb0
--- /dev/null
+++ b/fastapi_backend/tests/api/test_logs.py
@@ -0,0 +1,184 @@
+"""Logging: records in a file per process that the log viewer reads, redacted, and sent to syslog only when turned on."""
+
+from __future__ import annotations
+
+import logging
+import socket
+import socketserver
+import threading
+import time
+
+import pytest
+
+from app.domain import logs
+from tests.helpers import login, make_user
+
+
+@pytest.fixture(autouse=True)
+def _reset():
+ yield
+ logs.shutdown()
+
+
+def admin(client, db):
+ make_user(db, "root@x.io", "root password 1", admin=True)
+ return login(client, "root@x.io", "root password 1")
+
+
+class _Udp(socketserver.BaseRequestHandler):
+ got: list[bytes] = []
+
+ def handle(self):
+ _Udp.got.append(self.request[0])
+
+
+class _Tcp(socketserver.StreamRequestHandler):
+ got: list[bytes] = []
+
+ def handle(self):
+ _Tcp.got.append(self.rfile.read())
+
+
+def _wait(cond, seconds=5):
+ end = time.time() + seconds
+ while time.time() < end:
+ if cond():
+ return True
+ time.sleep(0.05)
+ return False
+
+
+def test_redacts_credentials():
+ text = (
+ "GET https://bob:hunter2@example.com/x?api_key=abc123def&token=zzz Authorization: Bearer abcdefgh123 "
+ "key sk-abcdefghijklmnopqrstu password=letmein prompt_tokens=1000"
+ )
+ out = logs.redact(text)
+ for secret in ("hunter2", "abc123def", "zzz", "abcdefgh123", "sk-abcdefghijklmnopqrstu", "letmein"):
+ assert secret not in out
+ assert "prompt_tokens=1000" in out and "bob:***@example.com" in out
+
+
+def test_viewer_reads_the_file_newest_first_and_filters(client, db, cfg):
+ h = admin(client, db)
+ lg = logging.getLogger("lens.test")
+ lg.info("first line")
+ lg.warning("second line with password=swordfish", extra={"job": 7, "api_key": "k"})
+ try:
+ raise ValueError("boom")
+ except ValueError:
+ lg.exception("third line")
+ rows = client.get("/api/v1/admin/logs", params={"logger": "lens.test"}, headers=h).json()
+ assert [r["msg"] for r in rows[:3]] == ["third line", "second line with password=***", "first line"]
+ assert rows[0]["level"] == "error" and "ValueError: boom" in rows[0]["exc"] and rows[0]["process"] == "api"
+ assert rows[1]["fields"] == {"job": 7, "api_key": "***"}
+ warn = client.get("/api/v1/admin/logs", params={"logger": "lens.test", "level": "warning"}, headers=h).json()
+ assert [r["msg"] for r in warn] == ["third line", "second line with password=***"]
+ found = client.get("/api/v1/admin/logs", params={"q": "FIRST LINE"}, headers=h).json()
+ assert [r["msg"] for r in found] == ["first line"]
+ older = client.get("/api/v1/admin/logs", params={"logger": "lens.test", "before": rows[1]["ts"]}, headers=h).json()
+ assert all(r["ts"] < rows[1]["ts"] for r in older)
+ # admins only
+ make_user(db, "a@x.io", "a password 123")
+ assert client.get("/api/v1/admin/logs", headers=login(client, "a@x.io", "a password 123")).status_code == 403
+
+
+def test_level_and_per_logger_levels_follow_the_settings(client, db, cfg):
+ h = admin(client, db)
+ body = {"level": "warning", "levels": {"lens.chatty": "error", "lens.loud": "debug"}}
+ assert client.put("/api/v1/settings/logs", json=body, headers=h).status_code == 200
+ client.get("/api/v1/settings/logs/status", headers=h) # reads the settings: this process follows
+ logging.getLogger("lens.quiet").info("not kept")
+ logging.getLogger("lens.quiet").warning("kept")
+ logging.getLogger("lens.chatty.sub").warning("not kept either")
+ logging.getLogger("lens.loud").debug("kept: its own level")
+ rows = client.get("/api/v1/admin/logs", params={"q": "kept"}, headers=h).json()
+ assert [r["msg"] for r in rows] == ["kept: its own level", "kept"]
+ assert client.get("/api/v1/settings/logs/status", headers=h).json()["level"] == "warning"
+
+
+def test_settings_are_checked(client, db, cfg):
+ h = admin(client, db)
+ bad = [
+ {"level": "loud"},
+ {"syslog": True}, # nowhere to send to
+ {"syslog_host": "169.254.169.254"},
+ {"syslog_host": "host:514"},
+ {"syslog_port": 0},
+ {"syslog_transport": "carrier-pigeon"},
+ {"levels": {"bad name!": "info"}},
+ {"file_mb": 1000},
+ ]
+ for body in bad:
+ assert client.put("/api/v1/settings/logs", json=body, headers=h).status_code == 400, body
+ v = client.get("/api/v1/settings", headers=h).json()["logs"]["values"]
+ assert v["syslog"] is False and v["level"] == "info" and v["file"] is True
+
+
+def test_syslog_off_by_default_then_sent_over_udp(client, db, cfg):
+ _Udp.got = []
+ srv = socketserver.UDPServer(("127.0.0.1", 0), _Udp)
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ try:
+ h = admin(client, db)
+ s = client.get("/api/v1/settings/logs/status", headers=h).json()
+ assert s["syslog"] is False and s["target"] is None
+ logging.getLogger("lens.test").warning("while off")
+ port = srv.server_address[1]
+ r = client.put(
+ "/api/v1/settings/logs",
+ json={"syslog_host": "127.0.0.1", "syslog_port": port, "syslog_facility": "local3"},
+ headers=h,
+ )
+ assert r.status_code == 200
+ t = client.post("/api/v1/settings/logs/test", headers=h).json()
+ assert t["ok"], t
+ assert _wait(lambda: any(b"test message" in g for g in _Udp.got))
+ assert not any(b"while off" in g for g in _Udp.got)
+ client.put("/api/v1/settings/logs", json={"syslog": True}, headers=h)
+ s = client.get("/api/v1/settings/logs/status", headers=h).json()
+ assert s["syslog"] is True and s["target"] == f"udp://127.0.0.1:{port}"
+ logging.getLogger("lens.test").warning("shipped with token=abcdef")
+ assert _wait(lambda: any(b"shipped" in g for g in _Udp.got))
+ line = next(g for g in _Udp.got if b"shipped" in g).decode()
+ # 1 ts host lens api.pid lens.test - msg
+ assert line.startswith("<156>1 ") and " lens api." in line and " lens.test - shipped with token=***" in line
+ assert "abcdef" not in line
+ finally:
+ srv.shutdown()
+
+
+def test_syslog_over_tcp_uses_octet_counting():
+ _Tcp.got = []
+ srv = socketserver.TCPServer(("127.0.0.1", 0), _Tcp)
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ try:
+ cfg = {
+ "logs": {"syslog_host": "127.0.0.1", "syslog_port": srv.server_address[1], "syslog_transport": "tcp", "syslog_format": "json"}
+ }
+ ok, error, _ = logs.test_send(cfg)
+ assert ok, error
+ assert _wait(lambda: _Tcp.got)
+ n, _, msg = _Tcp.got[0].partition(b" ")
+ assert int(n) == len(msg) and msg.startswith(b"<134>1 ") and b'"msg": "Lens ' in msg
+ finally:
+ srv.shutdown()
+
+
+def test_unreachable_server_never_blocks():
+ s = socket.socket()
+ s.bind(("127.0.0.1", 0))
+ port = s.getsockname()[1]
+ s.close() # nothing listens there
+ h = logs.SyslogHandler("127.0.0.1", port, "tcp", logging.INFO, 16, "text", "octet", True, "lens")
+ try:
+ lg = logging.getLogger("lens.unreachable")
+ lg.addHandler(h)
+ t = time.time()
+ for i in range(100):
+ lg.warning("line %d", i)
+ assert time.time() - t < 1
+ assert _wait(lambda: h.stats["last_error"])
+ finally:
+ logging.getLogger("lens.unreachable").removeHandler(h)
+ h.close()
diff --git a/mkdocs.yml b/mkdocs.yml
index 09dd1474e..b9bdc218b 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -107,6 +107,7 @@ nav:
- Storage: storage.md
- Database: database.md
- Telemetry: telemetry.md
+ - Logging: logging.md
- Prometheus: prometheus.md
- Build:
- Architecture: architecture.md
diff --git a/nextjs-frontend/__tests__/settings-logs.test.ts b/nextjs-frontend/__tests__/settings-logs.test.ts
new file mode 100644
index 000000000..00c4ba86f
--- /dev/null
+++ b/nextjs-frontend/__tests__/settings-logs.test.ts
@@ -0,0 +1,16 @@
+import { levelsToLines, linesToLevels } from "@/components/settings/model";
+
+describe("logger levels", () => {
+ it("round-trips lines of logger and level", () => {
+ const parsed = linesToLevels("httpx warning\n\n lens.bridge DEBUG ");
+ expect(parsed).toEqual({ value: { httpx: "warning", "lens.bridge": "debug" } });
+ expect(levelsToLines({ httpx: "warning" })).toBe("httpx warning");
+ });
+
+ it("names the line that's wrong", () => {
+ expect(linesToLevels("httpx warning\nhttpx loud")).toEqual({
+ error: expect.stringContaining("Line 2"),
+ });
+ expect("error" in linesToLevels("bad name! info")).toBe(true);
+ });
+});
diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts
index 7b9770738..1c104626d 100644
--- a/nextjs-frontend/app/openapi-client/index.ts
+++ b/nextjs-frontend/app/openapi-client/index.ts
@@ -1820,11 +1820,17 @@ export type {
LoginResponse,
LoginResponses,
LoginTicket,
+ LogLine,
LogoutData,
LogoutError,
LogoutErrors,
LogoutResponse,
LogoutResponses,
+ LogStatus,
+ LogStatusData,
+ LogStatusResponse,
+ LogStatusResponses,
+ LogTestResult,
LostPasskeyData,
LostPasskeyError,
LostPasskeyErrors,
@@ -2183,6 +2189,11 @@ export type {
ReadFileResponse,
ReadFileResponses,
Reading,
+ ReadLogsData,
+ ReadLogsError,
+ ReadLogsErrors,
+ ReadLogsResponse,
+ ReadLogsResponses,
Recording,
RecordingAccess,
RecordingAccessUpdate,
@@ -2799,6 +2810,9 @@ export type {
TestLlmData,
TestLlmResponse,
TestLlmResponses,
+ TestLogsData,
+ TestLogsResponse,
+ TestLogsResponses,
TestMailboxData,
TestMailboxResponse,
TestMailboxResponses,
diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts
index 3414ef23f..89b06d7ac 100644
--- a/nextjs-frontend/app/openapi-client/sdk.gen.ts
+++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts
@@ -939,6 +939,8 @@ import type {
LogoutData,
LogoutErrors,
LogoutResponses,
+ LogStatusData,
+ LogStatusResponses,
LostPasskeyData,
LostPasskeyErrors,
LostPasskeyResponses,
@@ -1069,6 +1071,9 @@ import type {
ReadFileData,
ReadFileErrors,
ReadFileResponses,
+ ReadLogsData,
+ ReadLogsErrors,
+ ReadLogsResponses,
RecordingTopicsData,
RecordingTopicsErrors,
RecordingTopicsResponses,
@@ -1351,6 +1356,8 @@ import type {
TestFileStoreResponses,
TestLlmData,
TestLlmResponses,
+ TestLogsData,
+ TestLogsResponses,
TestMailboxData,
TestMailboxResponses,
TestMailData,
@@ -2885,6 +2892,49 @@ export class Admin {
});
}
+ /**
+ * Log Status
+ *
+ * The API process's log level, its log file, and how sending to syslog is going (off by default).
+ */
+ public static logStatus(
+ options?: Options,
+ ): RequestResult {
+ return (options?.client ?? client).get({
+ url: "/api/v1/settings/logs/status",
+ ...options,
+ });
+ }
+
+ /**
+ * Test Logs
+ *
+ * Send one line to the saved syslog server now (on or off), to check the address, port and transport.
+ */
+ public static testLogs(
+ options?: Options,
+ ): RequestResult {
+ return (options?.client ?? client).post({
+ url: "/api/v1/settings/logs/test",
+ ...options,
+ });
+ }
+
+ /**
+ * Read Logs
+ *
+ * The latest log lines from the API and every worker (their files under data_dir/logs), newest first. Lines are
+ * redacted as they're written: no keys, tokens or passwords.
+ */
+ public static readLogs(
+ options?: Options,
+ ): RequestResult {
+ return (options?.client ?? client).get({
+ url: "/api/v1/admin/logs",
+ ...options,
+ });
+ }
+
/**
* List Audit
*
diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts
index ced6d01d0..4ce6281d7 100644
--- a/nextjs-frontend/app/openapi-client/types.gen.ts
+++ b/nextjs-frontend/app/openapi-client/types.gen.ts
@@ -7624,6 +7624,129 @@ export type LocalModelServer = {
[key: string]: unknown;
};
+/**
+ * LogLine
+ *
+ * One log record, redacted as it was written.
+ */
+export type LogLine = {
+ /**
+ * Ts
+ */
+ ts: string;
+ /**
+ * Level
+ */
+ level: string;
+ /**
+ * Logger
+ */
+ logger: string;
+ /**
+ * Msg
+ */
+ msg: string;
+ /**
+ * Process
+ */
+ process: string;
+ /**
+ * Pid
+ */
+ pid?: number | null;
+ /**
+ * Host
+ */
+ host?: string | null;
+ /**
+ * Exc
+ */
+ exc?: string | null;
+ /**
+ * Fields
+ */
+ fields?: {
+ [key: string]: unknown;
+ } | null;
+ [key: string]: unknown;
+};
+
+/**
+ * LogStatus
+ *
+ * What the API process does with its logs now; each worker follows the same settings on its own.
+ */
+export type LogStatus = {
+ /**
+ * Level
+ */
+ level: string;
+ /**
+ * File
+ *
+ * the API's log file, under data_dir/logs
+ */
+ file?: string | null;
+ /**
+ * Syslog
+ *
+ * whether lines are being sent to a syslog server
+ */
+ syslog: boolean;
+ /**
+ * Target
+ *
+ * where, as transport://host:port
+ */
+ target?: string | null;
+ /**
+ * Sent
+ */
+ sent: number;
+ /**
+ * Dropped
+ *
+ * lines dropped because the server couldn't keep up or was unreachable
+ */
+ dropped: number;
+ /**
+ * Queued
+ */
+ queued: number;
+ /**
+ * Last Sent
+ */
+ last_sent?: number | null;
+ /**
+ * Last Error
+ */
+ last_error?: string | null;
+ /**
+ * Error At
+ */
+ error_at?: number | null;
+ [key: string]: unknown;
+};
+
+/**
+ * LogTestResult
+ */
+export type LogTestResult = {
+ /**
+ * Ok
+ */
+ ok: boolean;
+ /**
+ * Error
+ */
+ error?: string | null;
+ /**
+ * Ms
+ */
+ ms?: number | null;
+ [key: string]: unknown;
+};
+
/**
* LoginRequest
*/
@@ -21249,6 +21372,100 @@ export type TestTelemetryResponses = {
export type TestTelemetryResponse = TestTelemetryResponses[keyof TestTelemetryResponses];
+export type LogStatusData = {
+ body?: never;
+ path?: never;
+ query?: never;
+ url: "/api/v1/settings/logs/status";
+};
+
+export type LogStatusResponses = {
+ /**
+ * Successful Response
+ */
+ 200: LogStatus;
+};
+
+export type LogStatusResponse = LogStatusResponses[keyof LogStatusResponses];
+
+export type TestLogsData = {
+ body?: never;
+ path?: never;
+ query?: never;
+ url: "/api/v1/settings/logs/test";
+};
+
+export type TestLogsResponses = {
+ /**
+ * Successful Response
+ */
+ 200: LogTestResult;
+};
+
+export type TestLogsResponse = TestLogsResponses[keyof TestLogsResponses];
+
+export type ReadLogsData = {
+ body?: never;
+ path?: never;
+ query?: {
+ /**
+ * Limit
+ */
+ limit?: number;
+ /**
+ * Level
+ *
+ * this level and above
+ */
+ level?: "debug" | "info" | "warning" | "error" | "critical" | null;
+ /**
+ * Q
+ *
+ * text the message or error contains
+ */
+ q?: string | null;
+ /**
+ * Process
+ *
+ * api, worker or watcher
+ */
+ process?: string | null;
+ /**
+ * Logger
+ *
+ * a logger name, or the start of one (lens.email)
+ */
+ logger?: string | null;
+ /**
+ * Before
+ *
+ * a line's ts: only older lines (the next page)
+ */
+ before?: string | null;
+ };
+ url: "/api/v1/admin/logs";
+};
+
+export type ReadLogsErrors = {
+ /**
+ * Validation Error
+ */
+ 422: HttpValidationError;
+};
+
+export type ReadLogsError = ReadLogsErrors[keyof ReadLogsErrors];
+
+export type ReadLogsResponses = {
+ /**
+ * Response Admin-Read Logs
+ *
+ * Successful Response
+ */
+ 200: Array;
+};
+
+export type ReadLogsResponse = ReadLogsResponses[keyof ReadLogsResponses];
+
export type ListAuditData = {
body?: never;
path?: never;
diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx
index f1577dd4c..2bad54dfa 100644
--- a/nextjs-frontend/components/settings/bodies.tsx
+++ b/nextjs-frontend/components/settings/bodies.tsx
@@ -10,6 +10,7 @@ import type { MatterbridgeStatus } from "@/app/openapi-client/types.gen";
import { MonitoringSwitch } from "@/components/admin/monitoring-switch";
import { BackupsPanel } from "@/components/settings/backups-panel";
import { ComponentsStatus } from "@/components/settings/components-status";
+import { LogViewer } from "@/components/settings/log-viewer";
import { AllTokens } from "@/components/account/all-tokens";
import { ACCESS } from "@/components/iiif/metadata-model";
import { RIGHTS } from "@/components/iiif/rights";
@@ -401,6 +402,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) {
return ;
case "telemetry":
return ;
+ case "logs":
+ return ;
case "fedora":
return ;
case "sensors":
@@ -685,6 +688,91 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) {
);
}
+function LogsBody({ ctx }: { ctx: BodyCtx }) {
+ const client = useApiClient();
+ const l = ctx.view.logs;
+ const saved = l?.values ?? {};
+ const status = useQuery({
+ queryKey: ["log-status", l?.updated_at ?? null],
+ queryFn: () => data(Admin.logStatus({ client })),
+ refetchInterval: 15_000,
+ });
+ const test = useMutation({ mutationFn: () => data(Admin.testLogs({ client })) });
+ const s = status.data;
+ const host = (saved.syslog_host as string | null | undefined) ?? null;
+ const on = Boolean(saved.syslog);
+ const failing = Boolean(s?.last_error && (!s.last_sent || (s.error_at ?? 0) > s.last_sent));
+ return (
+ <>
+ {on && s?.syslog ? (
+
+ {failing
+ ? `Can’t reach it: ${s.last_error}. Lines wait and are tried again${s.dropped ? `; ${s.dropped} dropped so far` : ""}.`
+ : `${s.sent} line${s.sent === 1 ? "" : "s"} sent from the server${s.dropped ? `, ${s.dropped} dropped` : ""}. Each worker sends its own.`}
+
+ ) : (
+
+ Lines go to the console and to a file per process. Turn on syslog below to send them to a log server too.
+
+ )}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ RFC 5424 syslog, as rsyslog, syslog-ng, Graylog, Loki’s syslog receiver and most log tools take it. Keys, tokens
+ and passwords are replaced with *** in every line before it is written or sent.
+
+
+ } onClick={() => test.mutate()} disabled={test.isPending || !host}>
+ {test.isPending ? "Sending…" : "Send a test line"}
+
+
+ {ctx.dirty
+ ? "Uses the saved settings, not your unsaved changes"
+ : "Sends one line, even while sending is off"}
+
+
+ {test.data &&
+ (test.data.ok ? (
+
+ {saved.syslog_transport === "udp"
+ ? "UDP doesn’t confirm delivery: look for it on the server."
+ : `The server took it in ${test.data.ms} ms.`}
+
+ ) : (
+
+ {test.data.error}
+
+ ))}
+ {test.isError && {test.error.message}}
+
+ >
+ );
+}
+
function EncryptionBody({ ctx }: { ctx: BodyCtx }) {
const client = useApiClient();
const progress = useQuery({
diff --git a/nextjs-frontend/components/settings/log-viewer.tsx b/nextjs-frontend/components/settings/log-viewer.tsx
new file mode 100644
index 000000000..de6c7813a
--- /dev/null
+++ b/nextjs-frontend/components/settings/log-viewer.tsx
@@ -0,0 +1,164 @@
+"use client";
+
+import { useInfiniteQuery } from "@tanstack/react-query";
+import { RefreshCw } from "lucide-react";
+import { useState } from "react";
+
+import { Admin } from "@/app/openapi-client";
+import type { LogLine } from "@/app/openapi-client/types.gen";
+import { Banner } from "@/components/ui/banner";
+import { Button } from "@/components/ui/button";
+import { SearchInput, Select, Switch } from "@/components/ui/field";
+import { Skeleton } from "@/components/ui/states";
+import { data, useApiClient } from "@/lib/api/browser";
+import { cn } from "@/lib/utils";
+
+const PAGE = 200;
+type Level = "debug" | "info" | "warning" | "error" | "critical";
+
+const TONE: Record = {
+ debug: "text-fg-muted",
+ info: "text-fg-secondary",
+ warning: "text-gold-dark",
+ error: "text-red-dark",
+ critical: "text-red-dark font-bold",
+};
+
+function Line({ l }: { l: LogLine }) {
+ const [open, setOpen] = useState(false);
+ const more = Boolean(l.exc || (l.fields && Object.keys(l.fields).length));
+ return (
+