From 175206373b878cfbcb39a94872ea31d602034625 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 23:49:53 +0000 Subject: [PATCH] feat(automations): routine runs, budget warnings, backups and releases as activities routine.ran reads routine_run rows that ended by finished_at (new index); budget.warned, backup.finished and lens.release are emitted with keys where budgets.sweep alerts, where a manual or scheduled backup ends and where updates.announce tells people, so each is logged once. They are for admins unless about one namespace. The name cache now keys on the fields asked for. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01QewNs6jxSB2vRoag2EvdyX --- docs/automations.md | 26 +- docs/budgets.md | 3 +- docs/database.md | 3 + docs/updates.md | 1 + fastapi_backend/app/domain/backups.py | 20 +- fastapi_backend/app/domain/budgets.py | 6 + fastapi_backend/app/domain/events_content.py | 6 +- fastapi_backend/app/domain/events_streams.py | 426 +++++++++++++++++- fastapi_backend/app/domain/store.py | 2 + fastapi_backend/app/domain/updates.py | 6 + .../tests/api/test_system_activities.py | 172 +++++++ 11 files changed, 652 insertions(+), 19 deletions(-) create mode 100644 fastapi_backend/tests/api/test_system_activities.py diff --git a/docs/automations.md b/docs/automations.md index b76b13cdd..57be95bc7 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -17,7 +17,7 @@ merged is called built. | Comments, notes written and saved, and changes to the entity graph | built | | Sensors: readings a sensor keeps, alert and notable log patterns, devices that report for the first time ([Sensors](#sensors-as-activities)) | built | | What people say to Lens, in the web app and chat apps, as `chat.message` ([Chat](#chat-as-activities)) | built | -| More kinds: system events | planned | +| Lens itself: routine runs, budget warnings, backups and new releases ([Lens itself](#lens-itself-as-activities)) | built | | The timeline API (`GET /api/v1/activities`), the MCP tools `timeline` and `event_kinds`, the assistant's `timeline` | built | | Review marks on the timeline (reviewed, important, hidden), their filters and `mark_activities` ([Timeline](timeline.md)) | built | | The shapes of triggers, joins, an If, policy and intents, checked (`app/domain/automation_spec.py`) | built | @@ -112,6 +112,10 @@ and the paths holding outside text are in `GET /api/v1/activities/kinds` (`event | `sensor.alert` | | `log` | `{type: Note, id: lens:sensor_pattern/}`, `label` (alert or notable), `pattern`, `example` (a line it fits), `count` (lines in all), `lines` (in the batch that raised it), `hour` | 192.168.1.1 logged an alert: wlan: authentication failed for | | `sensor.joined` | | `sensors` | `{type: Service, id: lens:sensor/, url: /sensors/}`, `sensor_type` (mqtt, syslog, ...), `device`, `status` | A new MQTT device reported: zigbee2mqtt/kitchen | | `chat.message` | | `chat_messages` | `{type: Note, id: lens:chat_message/, name: the start of it, url: /chat/}`, `text`, `channel` (web, telegram, slack, bridge, mailbox), `room`, `chat`, `direct` (a private chat or the web app), `attachments` (how many files) | sam wrote in Telegram: “groceries: milk, eggs” | +| `routine.ran` | | `routine_runs` | `{type: Note, id: lens:routine_run/, url: /routines/}`, `status` (done, partly, error, skipped), `trigger` (schedule, manual), `startTime`, `endTime`, `actions`, `failed`, `error`, `changes` (graph changes applied and proposed), `cost_usd`, `tokens` | Nightly tidy ran | +| `budget.warned` | | `log` | `{type: Note, id: lens:budget/, url: /costs}`, `resource` (all, space:, routine:, ...), `state` (near, over), `share`, `usd`, `tokens`, `estimate`, `cap_usd`, `cap_tokens`, `period`, `since` | The pods namespace has used 82% of its budget | +| `backup.finished` | | `log` | `{type: Document, id: lens:backup/, url: /settings/backups}`, `label` (manual, scheduled), `ok`, `error`, `kind` (export, folder), `bytes`, `files`, `copied` (to the storage connection) | A scheduled backup finished: scheduled-20261015-020000.surql.gz | +| `lens.release` | | `log` | `{type: Note, id: lens:release/, url: its release notes}`, `tag`, `title`, `current` (what this server runs), `published_at` | Lens v0.7.0 is out (this server runs 0.6.0) | | `webhook.received` | | `log` | `{type: Note, hook: the webhook's id, payload: what was sent (JSON, a form's fields or text), content_type, query}`; the actor is the webhook (`Service`, `lens:automation_hook/`, "Webhook for ") | Webhook for Front door was called | A new asset is one activity, of the most specific kind its row tells, decided by its content type and where it came @@ -180,6 +184,22 @@ said to Lens in a room is never kept, so it is never an activity. Its text and n - A question an automation's `ask_agent` asks (`origin` on the message) isn't a person's, so it is never a `chat.message`, and an automation can't set itself off that way. +#### Lens itself as activities + +Built (`app/domain/events_streams.py`). What Lens does on its own is for admins, unless it is about one namespace. +Each is logged once, however many processes see it happen. + +- `routine.ran` is a routine run that ended: done, partly (some actions failed), error, or skipped over budget once + someone picked to skip it. Its actor is the routine (`Application`, `lens:routine/`). A run held over budget is + one when it ends; one someone let run anyway runs again as a new run, and that run is the activity. A routine of + one namespace makes activities in it. +- `budget.warned` is a budget crossing its warning share (`warn_at`) or its cap, once per period and state, logged by + the same check that alerts on it ([Budgets](budgets.md#warnings)). A namespace's budget makes it in that namespace. +- `backup.finished` is a backup taken in Settings or through the API, or on the schedule, which says whether it + worked (a scheduled one that failed says why) and whether it was copied to the storage connection. +- `lens.release` is a newer Lens release, once per release, when Lens tells the admins and notification targets + ([Updates](updates.md#being-told)). + ### Sources and the event log Built. A source (`events.Source`, added with `events.register_source`) reads rows Lens already keeps as activities, @@ -212,6 +232,8 @@ A kind belongs to the source named in its `source`, and a source may list more i - `sensor_readings` (`app/domain/events_streams.py`) reads `sensor_reading` by `at` (microseconds), the id breaking ties, one sensor at a time, as `sensor.reading`; `sensors` reads the `storage_source` rows discovery made by `created_at`, as `sensor.joined`. +- `routine_runs` (`app/domain/events_streams.py`) reads `routine_run` rows that ended by `finished_at`, the id + breaking ties, as `routine.ran`. - `chat_messages` (`app/domain/events_streams.py`) reads `chat_message` rows with role user by `created_at` over the index on (role, created_at), the id breaking ties, as `chat.message`. - `log` reads `event_log`: activities no row of their own records (a deletion, a webhook call), written by @@ -632,3 +654,5 @@ def _(db, cfg, payload): ... # payload + late (true - Chat: recording every message in a room as an activity (opt-in per channel, off by default); a chat user who is also a Lens account as that person; a timeline page of others' private messages is read and left out, up to five source pages, so narrow it by account if that proves slow. +- Lens itself: a run held over budget, waiting for someone's pick, as an activity of its own; backups taken before + upgrades or with `lens backup` (only those from Settings, the API and the schedule are activities today). diff --git a/docs/budgets.md b/docs/budgets.md index 1d4028d31..a552369e2 100644 --- a/docs/budgets.md +++ b/docs/budgets.md @@ -65,7 +65,8 @@ Asking a routine to run from the app or the API when it's over budget is refused ## Warnings Every five minutes the scheduler looks at every budget. One that crosses `warn_at` or its cap gets an alert, once per -period: on the budget (`alert`) and as a `budget.near` or `budget.over` row in its activity history. +period: on the budget (`alert`) and as a `budget.near` or `budget.over` row in its activity history. Automations +see it too, as a `budget.warned` activity ([Automations](automations.md#lens-itself-as-activities)). ## The meter and the Costs page diff --git a/docs/database.md b/docs/database.md index 2fb49e085..14c383f7a 100644 --- a/docs/database.md +++ b/docs/database.md @@ -170,6 +170,9 @@ first ([Updates and backups](updates.md)). Each is one entry in `/back downloads as one `.tar.gz` holding `` and `.files/`. Backups from before files were kept show *database only* in Settings. +A backup taken in Settings, through the API or on the schedule is also a `backup.finished` activity for automations, +and a scheduled one that failed says why ([Automations](automations.md#lens-itself-as-activities)). + **Restoring** replaces the whole database, and the files under `data_dir`, with the backup's (a *database only* backup leaves the files as they are). What was there is backed up first (`before-restore`) and put back if the restore fails; a backup from an older Lens is brought up to date (schema and upgrade steps) afterwards. diff --git a/docs/updates.md b/docs/updates.md index ae967015f..c6502e91c 100644 --- a/docs/updates.md +++ b/docs/updates.md @@ -22,6 +22,7 @@ When a check finds a release newer than the one running, Lens says so once per r * **by email** to every admin (`updates.email`, on by default; needs an email server in Settings › Email); * **to notification targets** (webhooks, Matterbridge, Slack, Discord) whose namespace owners chose the **A new Lens release is out** event (`lens.release`, [Notifications](notifications.md)); +* to automations, as a `lens.release` activity ([Automations](automations.md#lens-itself-as-activities)); * in the server log. ## Upgrading diff --git a/fastapi_backend/app/domain/backups.py b/fastapi_backend/app/domain/backups.py index e98b9989e..47de09c9f 100644 --- a/fastapi_backend/app/domain/backups.py +++ b/fastapi_backend/app/domain/backups.py @@ -97,7 +97,20 @@ def take(db): dst = migrations.backup(db, "manual") if dst is None: return None - return next((b for b in listing(db) if b["name"] == dst.name), None) + got = next((b for b in listing(db) if b["name"] == dst.name), None) + if got: + _finished(db, "manual", got) + return got + + +def _finished(db, label, got=None, error=None, copied=None, at=None): + """backup.finished for automations (events_streams.py); a backup is never undone for want of it.""" + try: + from . import events_streams + + events_streams.backup_finished(db, label, got, error, copied, at) + except Exception as e: # noqa: BLE001 - the backup stands without it + log.info("backups: couldn't log %s: %s", (got or {}).get("name") or label, e) def delete(db, name): @@ -341,9 +354,14 @@ def run_due(db, cfg, log=None, now=None): ) if log: log(f"scheduled backup: {type(e).__name__}: {e}") + _finished(db, "scheduled", error=f"{type(e).__name__}: {e}", at=st["at"]) return True + copied = None if got and remote(cfg): send_quietly(db, cfg, got["name"], log) + copied = not remote_state(db).get("error") + if got: + _finished(db, "scheduled", got, copied=copied) return True diff --git a/fastapi_backend/app/domain/budgets.py b/fastapi_backend/app/domain/budgets.py index cb9249c7e..b29cc46f8 100644 --- a/fastapi_backend/app/domain/budgets.py +++ b/fastapi_backend/app/domain/budgets.py @@ -345,6 +345,12 @@ def sweep(db, cfg, now=None, every=None): activity.record( "run", f"budget.{st['state']}", [res], cfg, db, detail={"share": st["share"], "estimate": st["spent"]["estimate"] or None} ) + try: # for automations: budget.warned (events_streams.py) + from . import events_streams + + events_streams.budget_warned(db, b, alert, st) + except Exception as e: # noqa: BLE001 - the alert stands without it + log.info("budgets: couldn't log the warning for %s: %s", res, e) made.append({"resource": res, **alert}) return made diff --git a/fastapi_backend/app/domain/events_content.py b/fastapi_backend/app/domain/events_content.py index 4dad95f0e..a4874407f 100644 --- a/fastapi_backend/app/domain/events_content.py +++ b/fastapi_backend/app/domain/events_content.py @@ -60,13 +60,13 @@ def _lookup(db, table, keys, fields="name", by="id"): """{key: row} for these keys of a table (record ids, or the values of the indexed field `by`), one query for the - ones not looked up in the last NAME_SECONDS.""" + ones not looked up (with these `fields`) in the last NAME_SECONDS.""" keys = [k for k in dict.fromkeys(keys) if k is not None] now, out, todo = time.monotonic(), {}, [] with _names_lock: mine = _names.setdefault(db, {}) for k in keys: - hit = mine.get((table, by, k)) + hit = mine.get((table, by, fields, k)) if hit and now - hit[0] < NAME_SECONDS: if hit[1] is not None: out[k] = hit[1] @@ -83,7 +83,7 @@ def _lookup(db, table, keys, fields="name", by="id"): if len(mine) > 20000: mine.clear() for k in todo: - mine[(table, by, k)] = (now, got.get(k)) + mine[(table, by, fields, k)] = (now, got.get(k)) out.update(got) return out diff --git a/fastapi_backend/app/domain/events_streams.py b/fastapi_backend/app/domain/events_streams.py index e9b215682..1da991e59 100644 --- a/fastapi_backend/app/domain/events_streams.py +++ b/fastapi_backend/app/domain/events_streams.py @@ -33,6 +33,7 @@ READING_RX = re.compile(r"^reading-([a-z0-9]{1,100})$") JOINED_RX = re.compile(r"^joined-sensor-(\d+)$") SAID_RX = re.compile(r"^said-(\d+)$") +RAN_RX = re.compile(r"^ran-routine_run-(\d+)$") REF_UNSAFE = re.compile(r"[^A-Za-z0-9_.:@+-]") TEXT_MOST = 1000 # characters of a reading's text an activity carries @@ -474,35 +475,38 @@ def _said(db, rows): return out -def _said_rows(db, pos, op, limit, since=None, until=None): - """Messages past `pos` ([created_at, id]; op ">" oldest first, "<" newest first): those at its time with a later - id, then those after it, so the (role, created_at) index's range and order do the work.""" - order = "created_at, id" if op == ">" else "created_at DESC, id DESC" +def _rows_past(db, table, fields, where, col, pos, op, limit, since=None, until=None): + """Rows of `table` matching `where` past `pos` ([col, id]; op ">" oldest first, "<" newest first): those at its + time with a later id, then those after it, so an index on (..., col) does the range and the order. `since` and + `until` bound `col` (to the second).""" + order = f"{col}, id" if op == ">" else f"{col} DESC, id DESC" bounds, p = "", {} if since: - bounds += " AND created_at > $since" + bounds += f" AND {col} > $since" p["since"] = events_content._sec(since) if until: - bounds += " AND created_at <= $until" + bounds += f" AND {col} <= $until" p["until"] = events_content._sec(until) rows = [] if pos is not None and pos[1] is not None: rows += db.rows( - f"SELECT {SAID_FIELDS} FROM chat_message WHERE {SAID_WHERE} AND created_at = $pt AND id {op} $pi{bounds} " - f"ORDER BY {order} LIMIT {int(limit)}", + f"SELECT {fields} FROM {table} WHERE {where} AND {col} = $pt AND id {op} $pi{bounds} ORDER BY {order} LIMIT {int(limit)}", pt=pos[0], - pi=R("chat_message", pos[1]), + pi=R(table, pos[1]), **p, ) - where = "true" if pos is None else f"created_at {op} $pt" if pos is not None: p["pt"] = pos[0] - rows += db.rows( - f"SELECT {SAID_FIELDS} FROM chat_message WHERE {SAID_WHERE} AND {where}{bounds} ORDER BY {order} LIMIT {int(limit)}", **p - ) + after = "true" if pos is None else f"{col} {op} $pt" + rows += db.rows(f"SELECT {fields} FROM {table} WHERE {where} AND {after}{bounds} ORDER BY {order} LIMIT {int(limit)}", **p) return rows[: int(limit)] +def _said_rows(db, pos, op, limit, since=None, until=None): + """Messages past `pos`, over the (role, created_at) index.""" + return _rows_past(db, "chat_message", SAID_FIELDS, SAID_WHERE, "created_at", pos, op, limit, since, until) + + class ChatSource(events.Source): """What people said to Lens (chat_message rows with role user) as chat.message, read by (created_at, id).""" @@ -530,6 +534,217 @@ def get_many(self, db, cfg, ids): events.register_source(ChatSource()) +# ---------- system ---------- +RAN_FIELDS = "id, record::id(id) AS n, routine, trigger, status, started_at, finished_at, error, cost_usd, tokens, changes, results" +RAN_WHERE = "finished_at != NONE AND status != 'released'" # released: a held run someone let run, which runs anew +RAN_SAYS = {"done": "ran", "partly": "ran with errors", "error": "failed", "skipped": "was skipped (over budget)"} + + +def _system(spaces): + """`space` and `audience` for something about Lens itself: the one namespace it is about, else admins only.""" + spaces = [s for s in spaces or [] if isinstance(s, int) and not isinstance(s, bool)] + return _where(spaces[0] if len(spaces) == 1 else None) + + +def _ran(db, rows): + rows = [r for r in rows if r and isinstance(r.get("n"), int)] + if not rows: + return [] + labels = [(f"routine:{r['routine']}", None) for r in rows if isinstance(r.get("routine"), int)] + who = events_content.actors(db, labels) + routines = events_content._lookup(db, "routine", {r.get("routine") for r in rows}, "name, namespaces") + out = [] + for r in rows: + rid = r.get("routine") + try: + published = events.stamp(r["finished_at"]) + except (TypeError, ValueError, KeyError): + continue + actor = who.get((f"routine:{rid}", None)) or {"type": "Application", "id": f"lens:routine/{rid}", "name": f"Routine {rid}"} + results = [x for x in r.get("results") or [] if isinstance(x, dict)] + changes = r.get("changes") if isinstance(r.get("changes"), dict) else None + obj = store.clean( + { + "type": "Note", + "id": f"lens:routine_run/{r['n']}", + "name": f"{actor['name']}, run {r['n']}", + "url": f"/routines/{rid}", + "status": r.get("status"), + "trigger": r.get("trigger"), + "startTime": events.stamp(r["started_at"]) if r.get("started_at") else None, + "endTime": published, + "actions": len(results), + "failed": sum(1 for x in results if x.get("status") == "error"), + "error": str(r["error"])[:300] if r.get("error") else None, + "changes": changes, + "cost_usd": r.get("cost_usd") if isinstance(r.get("cost_usd"), (int, float)) else None, + "tokens": r.get("tokens") if isinstance(r.get("tokens"), int) else None, + } + ) + out.append( + events.build( + f"ran-routine_run-{r['n']}", + "routine.ran", + actor=actor, + obj=obj, + summary=f"{actor['name']} {RAN_SAYS.get(r.get('status'), 'finished')}", + source="routine_runs", + pos=[r["finished_at"], r["n"]], + published=published, + **_system((routines.get(rid) or {}).get("namespaces")), + ) + ) + return out + + +class RoutineRunSource(events.Source): + """routine_run rows that finished (done, partly, error, skipped) as routine.ran, read by (finished_at, id).""" + + name = "routine_runs" + + def read(self, db, cfg, pos, spaces, limit, hint=None): + return _ran(db, _rows_past(db, "routine_run", RAN_FIELDS, RAN_WHERE, "finished_at", pos, ">", limit)) + + def page(self, db, cfg, before, who, flt, limit): + p: dict = {} + where = [RAN_WHERE] + if before is not None: + where.append(events.after_sql("finished_at", before, p, "<", table="routine_run")) + # bounds that can't be an index range: newest first over one hits SurrealDB's range-order bug (docs/automations.md) + if flt.get("since"): + p["since"] = events_content._sec(flt["since"]) + where.append("(finished_at > $since OR finished_at = NONE)") + if flt.get("until"): + p["until"] = events_content._sec(flt["until"]) + where.append("(finished_at <= $until OR finished_at = NONE)") + rows = db.rows( + f"SELECT {RAN_FIELDS} FROM routine_run WHERE {' AND '.join(where)} ORDER BY finished_at DESC, id DESC LIMIT {int(limit)}", **p + ) + return _ran(db, rows) + + def get(self, db, cfg, activity_id): + m = RAN_RX.match(activity_id) + if not m: + return None + got = _ran(db, db.rows(f"SELECT {RAN_FIELDS} FROM $r WHERE {RAN_WHERE}", r=R("routine_run", int(m.group(1))))) + return got[0] if got else None + + +events.register_source(RoutineRunSource()) + + +def _budget_name(db, resource): + """A budget's resource in words: all of Lens, a namespace, a routine, a pipeline or a workflow.""" + if resource == "all": + return "Lens", None + table, _, n = str(resource).partition(":") + n = int(n) if n.isdigit() else None + if table == "space" and n is not None: + return f"The {store.space_names(db).get(n) or f'namespace {n}'} namespace", n + row = events_content._lookup(db, table, [n], "name").get(n) if table in ("routine", "pipeline", "workflow") and n is not None else None + return (str((row or {}).get("name") or "").strip() or f"{table.capitalize()} {n}"), None + + +def budget_warned(db, budget, alert, st): + """A budget.warned for a budget that crossed its warning share or its cap this period (budgets.sweep, once per + period and state). Its id.""" + res = budget["resource"] + name, space = _budget_name(db, res) + over = alert["state"] == "over" + spent = st.get("spent") or {} + obj = store.clean( + { + "type": "Note", + "id": f"lens:budget/{REF_UNSAFE.sub('_', res)}", + "name": f"Budget for {name[:1].lower() + name[1:] if name.startswith('The ') else name}", + "url": "/costs", + "resource": res, + "state": alert["state"], + "share": alert.get("share"), + "usd": spent.get("usd"), + "tokens": spent.get("tokens"), + "estimate": bool(spent.get("estimate")) or None, + "cap_usd": budget.get("usd"), + "cap_tokens": budget.get("tokens"), + "period": budget.get("period") or "month", + "since": st.get("since"), + } + ) + share = alert.get("share") or 0 + where = _system([space] if space is not None else None) + return events.emit( + db, + "budget.warned", + actor=dict(events.LENS), + obj=obj, + space=where["space"], + audience=where["audience"], + summary=f"{name} is over its budget" if over else f"{name} has used {share:.0%} of its budget", + key=f"budget.warned:{res}:{alert['state']}:{st.get('since')}", + ) + + +def backup_finished(db, label, got=None, error=None, copied=None, at=None): + """A backup.finished for a backup taken by hand or on the schedule (backups.py): `got` its entry, else `error` for + one that failed. Its id.""" + name = (got or {}).get("name") + ok = got is not None and not error + obj = store.clean( + { + "type": "Document", + "id": f"lens:backup/{name}" if name else "lens:backup", + "name": name or f"{label} backup", + "url": "/settings/backups", + "label": label, + "ok": ok, + "error": str(error)[:500] if error else None, + "kind": (got or {}).get("kind"), + "bytes": (got or {}).get("bytes"), + "files": (got or {}).get("files"), + "copied": copied, + } + ) + what = "A scheduled backup" if label == "scheduled" else "A backup" + return events.emit( + db, + "backup.finished", + actor=dict(events.LENS), + obj=obj, + space=None, + audience="admins", + summary=f"{what} finished: {name}" if ok else f"{what} failed: {str(error)[:200]}", + key=f"backup.finished:{name or f'{label}:{at}'}", + ) + + +def lens_release(db, rel, current): + """A lens.release for a newer Lens release (updates.announce, once per release). Its id.""" + tag = str(rel["tag"]) + url = rel.get("url") if isinstance(rel.get("url"), str) and rel["url"].startswith("https://") else None + obj = store.clean( + { + "type": "Note", + "id": f"lens:release/{REF_UNSAFE.sub('_', tag)[:200]}", + "name": f"Lens {tag}", + "url": url, + "tag": tag, + "title": " ".join(str(rel.get("name") or "").split())[:200] or None, + "current": current, + "published_at": rel.get("published_at"), + } + ) + return events.emit( + db, + "lens.release", + actor=dict(events.LENS), + obj=obj, + space=None, + audience="admins", + summary=f"Lens {tag} is out (this server runs {current})", + key=f"lens.release:{tag}", + ) + + _SENSOR_OBJECT = { "stream": _s("Stream"), "value": _s("Value", "number"), @@ -721,3 +936,188 @@ def _sample(aid, kind, as_type, actor, obj, summary, source, pos, untrusted, spa space=3, ), ) +LENS = dict(events.LENS) +RAN_UNTRUSTED = ("actor.name", "object.name") +events.kind( + "routine.ran", + label="Routine ran", + source="routine_runs", + as_type="Update", + object_type="Note", + fields={ + "object": { + "type": "object", + "properties": { + "status": _s("Status", enum=["done", "partly", "error", "skipped"]), + "trigger": _s("Started by", enum=["schedule", "manual"]), + "startTime": _s("Started", format="date-time"), + "endTime": _s("Finished", format="date-time"), + "actions": _s("Actions", "integer"), + "failed": _s("Actions that failed", "integer"), + "error": _s("Error"), + "changes": {"type": "object", "title": "Graph changes (applied, proposed)"}, + "cost_usd": _s("Cost (USD)", "number"), + "tokens": _s("Tokens", "integer"), + }, + } + }, + untrusted=RAN_UNTRUSTED, + sample=_sample( + "ran-routine_run-51", + "routine.ran", + "Update", + {"type": "Application", "id": "lens:routine/4", "name": "Nightly tidy"}, + { + "type": "Note", + "id": "lens:routine_run/51", + "name": "Nightly tidy, run 51", + "url": "/routines/4", + "status": "done", + "trigger": "schedule", + "startTime": "2026-10-15T02:00:00.000Z", + "endTime": "2026-10-15T02:03:12.000Z", + "actions": 2, + "failed": 0, + "changes": {"applied": 3, "proposed": 1}, + }, + "Nightly tidy ran", + "routine_runs", + ["2026-10-15T02:03:12+00:00", 51], + RAN_UNTRUSTED, + ), +) +SYSTEM_UNTRUSTED = ("object.name",) +events.kind( + "budget.warned", + label="Budget warning", + source="log", + as_type="Announce", + object_type="Note", + fields={ + "object": { + "type": "object", + "properties": { + "resource": _s("What the budget is for (all, space:, routine:, pipeline:, workflow:)"), + "state": _s("State", enum=["near", "over"]), + "share": _s("Share of the budget used", "number"), + "usd": _s("Spent (USD)", "number"), + "tokens": _s("Spent (tokens)", "integer"), + "estimate": _s("The USD is an estimate", "boolean"), + "cap_usd": _s("Budget (USD)", "number"), + "cap_tokens": _s("Budget (tokens)", "integer"), + "period": _s("Period", enum=["day", "week", "month"]), + "since": _s("Period started", format="date-time"), + }, + } + }, + untrusted=SYSTEM_UNTRUSTED, + sample=_sample( + "log-52", + "budget.warned", + "Announce", + LENS, + { + "type": "Note", + "id": "lens:budget/space_3", + "name": "Budget for the pods namespace", + "url": "/costs", + "resource": "space:3", + "state": "near", + "share": 0.82, + "usd": 4.1, + "tokens": 812000, + "cap_usd": 5.0, + "period": "month", + "since": "2026-10-01T00:00:00+00:00", + }, + "The pods namespace has used 82% of its budget", + "log", + ["2026-10-15T16:21:58+00:00", 52], + SYSTEM_UNTRUSTED, + space=3, + ), +) +events.kind( + "backup.finished", + label="Backup finished", + source="log", + as_type="Create", + object_type="Document", + fields={ + "object": { + "type": "object", + "properties": { + "label": _s("Taken", enum=["manual", "scheduled"]), + "ok": _s("It worked", "boolean"), + "error": _s("Why it failed"), + "kind": _s("Kind", enum=["export", "folder"]), + "bytes": _s("Size (bytes)", "integer"), + "files": _s("Has the data folder's files", "boolean"), + "copied": _s("Copied to the storage connection", "boolean"), + }, + } + }, + untrusted=(), + sample=_sample( + "log-53", + "backup.finished", + "Create", + LENS, + { + "type": "Document", + "id": "lens:backup/scheduled-20261015-020000.surql.gz", + "name": "scheduled-20261015-020000.surql.gz", + "url": "/settings/backups", + "label": "scheduled", + "ok": True, + "kind": "export", + "bytes": 48213004, + "files": True, + "copied": True, + }, + "A scheduled backup finished: scheduled-20261015-020000.surql.gz", + "log", + ["2026-10-15T02:00:41+00:00", 53], + (), + ), +) +RELEASE_UNTRUSTED = ("object.title",) +events.kind( + "lens.release", + label="New Lens release", + source="log", + as_type="Announce", + object_type="Note", + fields={ + "object": { + "type": "object", + "properties": { + "tag": _s("Version"), + "title": _s("Release name"), + "current": _s("This server runs"), + "published_at": _s("Published", format="date-time"), + }, + } + }, + untrusted=RELEASE_UNTRUSTED, + sample=_sample( + "log-54", + "lens.release", + "Announce", + LENS, + { + "type": "Note", + "id": "lens:release/v0.7.0", + "name": "Lens v0.7.0", + "url": "https://github.com/adeelahmad/lense/releases/tag/v0.7.0", + "tag": "v0.7.0", + "title": "v0.7.0", + "current": "0.6.0", + "published_at": "2026-10-15T09:00:00Z", + }, + "Lens v0.7.0 is out (this server runs 0.6.0)", + "log", + ["2026-10-15T16:21:58+00:00", 54], + RELEASE_UNTRUSTED, + ), +) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 8eec3f5c6..54ea82b5b 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -1360,6 +1360,8 @@ def close(self): "DEFINE TABLE IF NOT EXISTS fedora_status SCHEMALESS", "DEFINE TABLE IF NOT EXISTS routine_run SCHEMALESS", "DEFINE INDEX IF NOT EXISTS routine_run_r ON routine_run FIELDS routine", + # finished runs, by when, as routine.ran (events_streams.py) + "DEFINE INDEX IF NOT EXISTS routine_run_finished ON routine_run FIELDS finished_at", "DEFINE TABLE IF NOT EXISTS graph_change SCHEMALESS", "DEFINE INDEX IF NOT EXISTS graph_change_run ON graph_change FIELDS run", "DEFINE INDEX IF NOT EXISTS graph_change_status ON graph_change FIELDS status", diff --git a/fastapi_backend/app/domain/updates.py b/fastapi_backend/app/domain/updates.py index 944a6a7d2..32008f073 100644 --- a/fastapi_backend/app/domain/updates.py +++ b/fastapi_backend/app/domain/updates.py @@ -450,6 +450,12 @@ def announce(db, cfg, log_fn=None): notify.release(db, cfg, rel, current()) except Exception: # noqa: BLE001 - the email still goes log.exception("couldn't queue the release notifications") + try: # for automations: lens.release (events_streams.py) + from . import events_streams + + events_streams.lens_release(db, rel, current()) + except Exception: # noqa: BLE001 - the email still goes + log.exception("couldn't log the release for automations") if cfg["updates"].get("email", True): admins = db.values("SELECT VALUE email FROM account WHERE admin = true AND disabled != true") try: diff --git a/fastapi_backend/tests/api/test_system_activities.py b/fastapi_backend/tests/api/test_system_activities.py new file mode 100644 index 000000000..e75971cdf --- /dev/null +++ b/fastapi_backend/tests/api/test_system_activities.py @@ -0,0 +1,172 @@ +"""Lens's own happenings as activities (docs/automations.md, Kinds): a routine run that finished as `routine.ran`, a +budget crossing its warning or its cap as `budget.warned`, a backup taken by hand or on the schedule as +`backup.finished`, and a newer Lens release as `lens.release`. Each is for admins unless it is about one namespace, +and each is logged once however often the thing that makes it runs.""" + +from __future__ import annotations + +import datetime as dt +import time + +import pytest + +from app.domain import activity, backups, budgets, events, events_streams, migrations, routines, store, updates + +ADMIN = events.viewer(1, None, admin=True) + + +@pytest.fixture(autouse=True) +def fresh(): + events._seen.clear() + events._claimed.clear() + yield + + +def _start(db, cfg, consumer, kinds): + """A follower's first look marks where it starts: what happens in that second isn't seen, so wait for the next.""" + assert events.follow(db, cfg, consumer, kinds=kinds)[0] == [] + t = store.now() + while store.now() == t: + time.sleep(0.05) + + +def _logged(db, kind): + return sorted(db.values("SELECT VALUE record::id(id) FROM event_log WHERE kind = $k", k=kind)) + + +def test_a_finished_routine_run_is_routine_ran(db, cfg): + pods = store.ns_id(db, "pods") + one = routines.create(db, "Tidy pods", [{"type": "sync"}], namespaces=[pods]) + every = routines.create(db, "Tidy all", [{"type": "sync"}]) + _start(db, cfg, "test.runs", ["routine.ran"]) + first = routines.run(db, cfg, one) + second = routines.run(db, cfg, every) + held = db.next_id("routine_run") + db.q( + "CREATE $r CONTENT $d", + r=store.R("routine_run", held), + d={"routine": one, "trigger": "schedule", "status": "held", "started_at": store.now(), "results": []}, + ) + + got, commit = events.follow(db, cfg, "test.runs", kinds=["routine.ran"]) + commit() + assert [a["id"] for a in got] == [f"ran-routine_run-{first}", f"ran-routine_run-{second}"] # held: not finished + for a in got: + events.check_activity(a) + a = got[0] + assert a["kind"] == "routine.ran" and a["type"] == "Update" and a["space"] == pods and "audience" not in a["lens"] + assert a["actor"] == {"type": "Application", "id": f"lens:routine/{one}", "name": "Tidy pods"} + assert a["summary"] == "Tidy pods ran" + o = a["object"] + assert o["id"] == f"lens:routine_run/{first}" and o["url"] == f"/routines/{one}" and o["name"] == f"Tidy pods, run {first}" + assert (o["status"], o["trigger"], o["actions"], o["failed"]) == ("done", "manual", 1, 0) + assert o["startTime"].endswith("Z") and o["endTime"] == a["published"] + assert got[1]["space"] is None and got[1]["lens"]["audience"] == "admins" # every namespace: admins + + # a held run picked: skipped is a run that ended; released runs again as a new run, so it isn't one + routines.decide_held(db, held, run=False) + got, commit = events.follow(db, cfg, "test.runs", kinds=["routine.ran"]) + commit() + assert [(a["id"], a["summary"]) for a in got] == [(f"ran-routine_run-{held}", "Tidy pods was skipped (over budget)")] + db.q("UPDATE $r SET status = 'released'", r=store.R("routine_run", held)) + assert events.get(db, cfg, f"ran-routine_run-{held}", ADMIN) is None + + page = events.page(db, cfg, ADMIN, flt={"kinds": ["routine.ran"]})["items"] + assert [x["id"] for x in page] == [f"ran-routine_run-{second}", f"ran-routine_run-{first}"] + reader = events.viewer(2, [pods]) + assert [x["id"] for x in events.page(db, cfg, reader, flt={"kinds": ["routine.ran"]})["items"]] == [f"ran-routine_run-{first}"] + assert events.get(db, cfg, f"ran-routine_run-{second}", reader) is None + + +def test_a_budget_crossing_is_budget_warned_once(db, cfg): + pods = store.ns_id(db, "pods") + res = f"space:{pods}" + budgets.put(db, res, usd=1) + budgets.put(db, "all", usd=100) + activity.record("out", "model.chat", [res], db=db, model="m", cost_usd=0.9) + assert [a["state"] for a in budgets.sweep(db, cfg)] == ["near"] + assert budgets.sweep(db, cfg) == [] + [n] = _logged(db, "budget.warned") + a = events.check_activity(events.get(db, cfg, f"log-{n}", ADMIN)) + assert a["kind"] == "budget.warned" and a["type"] == "Announce" and a["space"] == pods and "audience" not in a["lens"] + assert a["actor"] == events.LENS and a["summary"] == "The pods namespace has used 90% of its budget" + o = a["object"] + assert (o["resource"], o["state"], o["share"], o["usd"], o["cap_usd"], o["period"]) == (res, "near", 0.9, 0.9, 1, "month") + assert o["name"] == "Budget for the pods namespace" and o["url"] == "/costs" + + activity.record("out", "model.chat", [res], db=db, model="m", cost_usd=0.2) + assert [a["state"] for a in budgets.sweep(db, cfg)] == ["over"] + over = events.get(db, cfg, f"log-{_logged(db, 'budget.warned')[-1]}", ADMIN) + assert over["summary"] == "The pods namespace is over its budget" and over["object"]["state"] == "over" + # emitted again (another process sweeping): the same activity + b, st = budgets.get(db, res), budgets.status(db, res) + alert = {"state": "over", "since": st["since"], "share": st["share"]} + assert events_streams.budget_warned(db, b, alert, st) == over["id"] and len(_logged(db, "budget.warned")) == 2 + + +def test_a_backup_taken_is_backup_finished(db, cfg, monkeypatch): + folder = backups.folder(db) + + def fake(db_, label="manual"): + folder.mkdir(parents=True, exist_ok=True) + p = folder / f"{label}-20261015-020000.surql.gz" + p.write_bytes(b"backup") + return p + + monkeypatch.setattr(migrations, "backup", fake) + got = backups.take(db) + [n] = _logged(db, "backup.finished") + a = events.check_activity(events.get(db, cfg, f"log-{n}", ADMIN)) + assert a["kind"] == "backup.finished" and a["type"] == "Create" and a["space"] is None and a["lens"]["audience"] == "admins" + assert a["summary"] == f"A backup finished: {got['name']}" + assert {k: a["object"][k] for k in ("id", "label", "ok", "kind", "bytes")} == { + "id": f"lens:backup/{got['name']}", + "label": "manual", + "ok": True, + "kind": "export", + "bytes": 6, + } + assert events.get(db, cfg, f"log-{n}", events.viewer(2, [store.ns_id(db, "pods")])) is None + + # on the schedule: when it's due, once, and a failure says why + cfg["backups"] = {**cfg["backups"], "schedule": "0 2 * * *", "timezone": "UTC"} + t0 = dt.datetime(2026, 10, 15, 12, 0, tzinfo=dt.UTC) + assert backups.run_due(db, cfg, now=t0) is False # works out the next time + assert backups.run_due(db, cfg, now=t0 + dt.timedelta(days=1)) is True + scheduled = events.get(db, cfg, f"log-{_logged(db, 'backup.finished')[-1]}", ADMIN) + assert scheduled["summary"] == "A scheduled backup finished: scheduled-20261015-020000.surql.gz" + assert scheduled["object"]["label"] == "scheduled" and "copied" not in scheduled["object"] # no storage connection + + def broken(db_, label="manual"): + raise OSError("disk full") + + monkeypatch.setattr(migrations, "backup", broken) + assert backups.run_due(db, cfg, now=t0 + dt.timedelta(days=2)) is True + failed = events.get(db, cfg, f"log-{_logged(db, 'backup.finished')[-1]}", ADMIN) + assert failed["summary"] == "A scheduled backup failed: OSError: disk full" + assert failed["object"]["ok"] is False and failed["object"]["error"] == "OSError: disk full" + assert len(_logged(db, "backup.finished")) == 3 + + +def test_a_new_release_is_lens_release_once(db, cfg, monkeypatch): + from app import email + + async def quiet(*a): + return None + + monkeypatch.setattr(email, "send_release_email", quiet) + rel = { + "tag": "v99.0.0", + "name": "v99.0.0", + "url": "https://github.com/adeelahmad/lense/releases/tag/v99.0.0", + "published_at": "2026-10-15T09:00:00Z", + } + updates._put(db, latest=rel) + assert updates.announce(db, cfg) is True + assert updates.announce(db, cfg) is False + [n] = _logged(db, "lens.release") + a = events.check_activity(events.get(db, cfg, f"log-{n}", ADMIN)) + assert a["kind"] == "lens.release" and a["lens"]["audience"] == "admins" and a["actor"] == events.LENS + assert a["summary"] == f"Lens v99.0.0 is out (this server runs {updates.current()})" + assert a["object"]["url"] == rel["url"] and a["object"]["tag"] == "v99.0.0" and a["object"]["current"] == updates.current() + assert events_streams.lens_release(db, rel, updates.current()) == a["id"] and len(_logged(db, "lens.release")) == 1