diff --git a/docs/automations.md b/docs/automations.md
index 79f1c3be2..999d25e67 100644
--- a/docs/automations.md
+++ b/docs/automations.md
@@ -10,7 +10,10 @@ merged is called built.
| Part | Status |
| --- | --- |
| Match trees: one condition language for triggers, an automation's If and workflow nodes | built |
-| Activities, the kinds catalog, the event log, following the stream, the timeline API | planned |
+| Activities, the kinds catalog, the event log, following the stream, timeline reads (`app/domain/events.py`) | built |
+| New assets as `asset.added` (the `recordings` source) | built |
+| More kinds: emails, files, calendar events, jobs, deletions, comments, notes, graph changes, sensors, chat | planned |
+| The timeline API, MCP and assistant tools | planned |
| Automations, triggers, joins, schedules, actions, approvals, variables, the timeline page, the studio | planned |
## Contract
@@ -18,6 +21,120 @@ merged is called built.
Every part of the studio builds on the shapes below. A change to one of them comes in a small pull request of its
own that changes the code, this section and the tests together.
+### Activities
+
+Built (`app/domain/events.py`). Everything that happens in Lens, from every source, is an activity: a plain dict in
+the shape of a [W3C Activity Streams 2.0](https://www.w3.org/TR/activitystreams-core/) activity, with a Lens `kind`
+that triggers subscribe to. `events.check_activity(a)` checks it, naming the first problem.
+
+```json
+{"id": "create-recording-7", "type": "Create", "kind": "asset.added", "published": "2026-10-15T08:12:03.000Z",
+ "actor": {"type": "Person", "id": "lens:account/1", "name": "Sam"},
+ "object": {"type": "Audio", "id": "lens:recording/7", "name": "Call with the school", "url": "/resources/7",
+ "media": "audio"},
+ "target": null, "tag": [], "summary": "Sam added “Call with the school”", "space": 1,
+ "lens": {"source": "recordings", "pos": ["2026-10-15T08:12:03+00:00", 7], "untrusted": ["object.name"],
+ "cause": null}}
+```
+
+- `id`: stable, made from what the source reads (`create-recording-7`, `log-12`), so an activity is the same every
+ time it is read: 1 to 120 lowercase letters, digits and `_ . : -`.
+- `type`: the AS2 verb (Create, Update, Delete, Add, Remove, Join, Leave, Arrive, Announce, Like, Accept, Reject),
+ which the kind decides.
+- `published`: when it happened, ISO 8601 in UTC with milliseconds, so times from every source sort as text.
+- `actor`: who did it, `{type, id, name}`. Sources and storage are `Service`, people `Person`, Lens itself, the agent
+ and automations `Application` (`lens:automation/4`; Lens is `lens:app`). Entities are never actors; they go in
+ `tag`.
+- `object`: what it is about, with the kind's own fields. Ids are `lens:
/` references, so actions can act
+ on them; `events.ref_id(ref, "recording")` reads the id back.
+- `target`: where to, when the verb has one (Add to a collection), else null. `tag`: up to 50 objects such as
+ `{"type": "Mention", "id": "lens:entity/9", "name": "St Mary's"}`.
+- `summary`: one line of at most 300 characters, made by fixed rules, never by a model.
+- `space`: the namespace id, or null for activities of no namespace.
+- `lens`: Lens's own fields. `source` and `pos` (the source's position, `[time, id]`) are for the stream machinery.
+ `untrusted` lists the dotted paths holding outside text (titles, email bodies, webhook payloads): prompts wrap
+ them with `untrusted.wrap`. `cause` is `{automation, run, depth}` when an automation's action caused it. `spaces`
+ lists every namespace when there are several. `audience` narrows who may see it: `space` (the default: people who
+ read one of its namespaces), `owner` (only the account in `lens.owner`) or `admins`.
+
+### Kinds
+
+Built. `events.kind(name, *, label, source, as_type, object_type, parent=None, fields=None, untrusted=(),
+sample=None, noisy=False)` registers a kind: a name like `noun.past_verb`, a label for people, the source that makes
+it, its AS2 verb and object type, a parent (a trigger on the parent matches it too), JSON Schemas for its `object`
+(and `actor` or `target`) in `fields`, the paths that hold outside text, a realistic sample (checked when it is
+registered), and whether it is `noisy` (left out of the timeline unless named). `events.kinds()` lists them,
+`events.kind_info(name)` gives one with its children and its whole `schema` (every activity's fields, its parents',
+then its own), `events.descendants(name)` the kind and every kind under it, and `events.field_at(kind, path)` the
+schema of one field, whose `title` names it in words (`match.describe(tree, kind)` says "the title contains “call”").
+
+| Kind | Parent | Source | Object |
+| --- | --- | --- | --- |
+| `asset.added` | | `recordings` | `{type: Audio, Video, Image or Document, id: lens:recording/, name: its title, url: /resources/, media: audio, video, image, document or transcript, content_type}` |
+
+### Sources and the event log
+
+Built. A source (`events.Source`, added with `events.register_source`) reads rows Lens already keeps as activities,
+so nothing is stored twice:
+
+- `read(db, cfg, pos, spaces, limit, hint)`: activities strictly after `pos` (`[time, id]`, or `[time, null]` for
+ after every row at that time), oldest first, at most `limit`, of the namespaces in `spaces` (null: all). `hint`
+ is narrowing a source may use.
+- `page(db, cfg, before, viewer, flt, limit)`: newest first, strictly before `before` (`key` of the last one shown),
+ with what the viewer may see and the filters it can apply cheaply (the stream checks every filter again).
+- `get(db, cfg, activity_id)`: one activity by its id, or null; `now(db)`, `back(db, seconds)`: positions of now and of
+ `seconds` ago; `key(a)`: the position `page` orders by.
+
+A kind belongs to the source named in its `source`, and a source may list more in `kinds`. Two are built:
+
+- `recordings` (`app/domain/events_content.py`) reads `recording` by `created_at`, the id breaking ties, as
+ `asset.added`. The actor is whoever added it: an automation (its first job was queued by `automation:`), the
+ storage source a watched file came from (`Service`, `lens:source/`), the person whose upload it was (`Person`,
+ `lens:account/`), or Lens (a folder scan).
+- `log` reads `event_log`: activities no row of their own records (a deletion, a webhook call), written by
+ `events.emit(db, kind, *, actor, obj, space, summary, target=None, tag=None, key=None, published=None, spaces=None,
+ audience=None, owner=None, untrusted=None)`, which gives the id (`log-`). Emitting a `key` again changes
+ nothing and gives the first id, so a retry never doubles an activity. Inside `with events.caused_by({"automation":
+ 4, "run": 51, "depth": 1}):` what is emitted carries that cause. It is read by `(logged_at, id)` and paged by
+ `published`, and its rows are kept for `events.keep_days` (30).
+
+### Following the stream
+
+Built. The API and `lens worker` are separate processes on one database, so following works the way notifications
+do: `events.follow(db, cfg, consumer, kinds=None, spaces=None, limit=500, hint=None)` gives `(activities, commit)`,
+the activities since the consumer's last commit, oldest first. Handle them in order, then call `commit()` to save
+where the next look starts (`event_cursor:`, a position per source).
+
+- A consumer seen for the first time starts at now: nothing that happened before replays, not even through the
+ lookback. What happens in the second it starts in isn't seen.
+- Only `kinds` (and their descendants; null: every kind) in `spaces` (null: every namespace and none). A source that
+ makes none of the kinds wanted isn't read, and its cursor moves to now, so a kind listened to later doesn't replay
+ a backlog. `events.idle(db, consumer)` moves every cursor to now.
+- A source with more waiting gives `limit` at a time. Once caught up, a look also reads the last 60 seconds again,
+ for a row written as the last look ran, minus what this process already handled. So claim each activity before
+ acting on it: `events.claim(db, consumer, key)` is true for the one process, and the one time, that claims `key`
+ (an activity's id, say), with a `CREATE` of `event_claim:` only one process wins. `events.unclaimed(db,
+ consumer, keys)` lists the keys nobody has claimed yet, in one query. Claims go after a day.
+- `events.emit` wakes the followers in its process; a follower waits between looks with `g = events.generation()`
+ before the look and `events.wait(events.poll_seconds(cfg), stop, since=g)` after it (`events.poll_seconds`, 5).
+
+### Timeline reads
+
+Built. `events.page(db, cfg, viewer, before=None, flt=None, limit=50)` reads the stream newest first for a viewer
+(`events.viewer(account, spaces, admin)`, spaces null for every one): `{"items": [...], "next": ...}`, where `next`
+(an opaque cursor, null at the end) goes in `before` for the page after. It merges every source's `page`, applies
+who may see what, and when filters thin a page out it reads on, up to five pages from each source.
+`events.get(db, cfg, id, viewer)` reads one activity, null when there is none or it isn't the viewer's to see.
+
+- Who sees what: `lens.audience` `owner` is its owner alone; `admins` is admins; `space` is everyone who reads one of
+ its namespaces (`space` or `lens.spaces`); admins read every namespace, and an activity of no namespace is for
+ admins. People who were given only some collections of a namespace don't see its activities yet.
+- Filters: `kinds` (with their descendants; a noisy kind only when named), `actor` (an actor's id, or several),
+ `spaces` (namespace ids), `tag` (an entity's id or name), `since` (published after) and `until` (at or before),
+ `q` (words in the summary) and `where` (a match tree on the activity, below).
+- `events.tidy(db, cfg)` drops event log rows past `events.keep_days` and claims past a day; the routine scheduler
+ runs it once an hour.
+
### Match trees
Built (`app/domain/match.py`). One condition language for a trigger's `where`, an automation's `if`, and the
@@ -52,3 +169,7 @@ tree, `path` (when given) is the part of the value the tree tests. Graphs saved
- More operators: `in`, `matches` (bounded regular expressions), `between`, times against now, `changed`.
- The canvas shows and edits match trees on condition, filter and switch nodes (today it edits one test; a tree
saved through the API is kept).
+- A `?format=as2` JSON-LD view and per-actor outboxes for ActivityPub; SurrealDB `DEFINE EVENT` for Update verbs that
+ no row records; pushing instead of polling.
+- People given only some collections of a namespace see that namespace's activities about those collections.
+- Name an automation actor by its automation's name once automations are stored.
diff --git a/docs/database.md b/docs/database.md
index 7bc0c15d9..4382d9bd5 100644
--- a/docs/database.md
+++ b/docs/database.md
@@ -68,6 +68,13 @@ cosine; servers only) defined when the first vector is stored, since its `DIMENS
which model and dimension that is. A server searches it with `embedding <|k,ef|> $vec AND …filters`, which 3.2.4
applies inside the index scan (a filtered search still returns k rows).
+The stream of activities automations follow ([Automations](automations.md#contract)) is mostly a view over rows
+kept anyway, such as `recording.created_at`. What no row records goes in `event_log:` (by `logged_at` and
+`published`, kept for `events.keep_days`), with the keys it was emitted with in `event_log_key:`. Each consumer
+of the stream keeps where its next look starts in `event_cursor:` (a position per source), and claims what
+it handles in `event_claim:`; a claim is a `CREATE` only one process wins, and claims go
+after a day.
+
## Upgrades
Installs keep their data across updates (running `install.sh` again, a new image, `git pull`). New tables, fields and
diff --git a/fastapi_backend/app/domain/events.py b/fastapi_backend/app/domain/events.py
new file mode 100644
index 000000000..44283cc69
--- /dev/null
+++ b/fastapi_backend/app/domain/events.py
@@ -0,0 +1,1091 @@
+"""The stream of activities (docs/automations.md, Contract): what happens in Lens, from every source, as time-coded
+activities in the W3C Activity Streams 2.0 shape, for automations to follow and the timeline to show.
+
+An activity is a plain dict (`check_activity` says what it holds): an AS2 verb (`type`), a Lens `kind` that triggers
+subscribe to (`email.received`), when it happened, who did it (`actor`), what it is about (`object`), a one-line
+summary made by fixed rules, its namespace, and Lens's own fields under `lens`.
+
+Most activities aren't stored twice. A source (`Source`, `register_source`) reads rows Lens already keeps, such as
+recordings, and makes activities of them, with ids built from what it reads (`create-recording-7`), so they are the
+same every time they're read. What has no row of its own (a deletion, a webhook call) is written to a small log,
+`event_log`, by `emit`; the `log` source reads it back. Kinds (`kind`) are registered in a catalog with the source
+that makes them, the fields of their objects, which of those hold outside text, and a sample.
+
+Following the stream works across processes the way notify.py does it, since the API and `lens worker` are separate
+processes on one database: each consumer has a cursor (`event_cursor:`, a position per source), reads past
+it in (time, id) order, and claims what it handles (`claim`: a CREATE only one process wins). A consumer seen for the
+first time starts at now, so nothing that happened before replays. Once caught up, a read also reads the last
+LOOKBACK seconds again, for a row written as the last look ran; claims keep those from being handled twice. A source
+that can't make a kind the consumer wants isn't read, and its cursor moves to now, so a kind someone starts listening
+to later doesn't replay a backlog.
+
+The timeline reads the stream newest first (`page`), merging the sources, applying who may see what (`lens.audience`
+and namespaces), and reading on while filters thin a page out. Nothing here runs on its own: until something follows
+the stream, an install behaves as before. `tidy` drops old log rows and claims.
+"""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import contextvars
+import datetime as dt
+import hashlib
+import json
+import re
+import threading
+import time
+from dataclasses import dataclass
+
+from . import store
+
+R = store.R
+ID_RX = re.compile(r"^[a-z0-9_.:-]{1,120}$")
+KIND_RX = re.compile(r"^[a-z][a-z0-9_]{0,40}(\.[a-z][a-z0-9_]{0,40}){1,3}$")
+NAME_RX = re.compile(r"^[a-z][a-z0-9_.:-]{0,79}$")
+REF_RX = re.compile(r"^lens:[a-z][a-z0-9_]{0,40}(/[A-Za-z0-9_.:@+-]{1,200})?$")
+PATH_RX = re.compile(r"^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*$")
+TYPES = ("Create", "Update", "Delete", "Add", "Remove", "Join", "Leave", "Arrive", "Announce", "Like", "Accept", "Reject")
+ACTOR_TYPES = ("Service", "Person", "Application")
+AUDIENCES = ("space", "owner", "admins")
+LENS = {"type": "Application", "id": "lens:app", "name": "Lens"}
+LOOKBACK = 60 # seconds a caught-up read goes back over, for rows written as the last look ran
+PAGE = 500 # rows a follow reads per source
+PAGE_ROUNDS = 5 # source pages a timeline page reads at most, per source, when filters thin it out
+SUMMARY_MAX = 300
+MAX_TAGS = 50
+FILTERS = ("kinds", "actor", "spaces", "tag", "since", "until", "q", "where")
+SWEEP_SECONDS = 3600
+CLAIM_DAYS = 1
+
+_swept = {"at": 0.0}
+_cause: contextvars.ContextVar[dict | None] = contextvars.ContextVar("lens_event_cause", default=None)
+_wakes = threading.Condition()
+_generation = [0]
+_seen: dict[str, dict[str, None]] = {} # per consumer: activities this process handled lately (not read again)
+_claimed: dict[str, None] = {} # claims this process knows are taken, so it doesn't ask the database again
+_lock = threading.Lock()
+
+
+# ---------- times ----------
+def _parse(t):
+ """An aware UTC datetime from an ISO 8601 string (naive is UTC) or a datetime."""
+ if isinstance(t, dt.datetime):
+ d = t
+ else:
+ s = str(t).strip()
+ d = dt.datetime.fromisoformat(s[:-1] + "+00:00" if s.endswith("Z") else s)
+ return (d if d.tzinfo else d.replace(tzinfo=dt.timezone.utc)).astimezone(dt.timezone.utc)
+
+
+def stamp(t=None):
+ """`published` as activities have it: ISO 8601 in UTC with milliseconds (2026-10-15T16:21:58.000Z), so stamps
+ from every source sort as text."""
+ d = _parse(t) if t is not None else dt.datetime.now(dt.timezone.utc)
+ return d.strftime("%Y-%m-%dT%H:%M:%S.") + f"{d.microsecond // 1000:03d}Z"
+
+
+def ago(seconds, now=None):
+ """The time `seconds` ago, as store.now() writes times (to the second, +00:00)."""
+ base = _parse(now) if now is not None else dt.datetime.now(dt.timezone.utc)
+ return (base - dt.timedelta(seconds=seconds)).isoformat(timespec="seconds")
+
+
+def _tkey(t):
+ if isinstance(t, str):
+ try:
+ return _parse(t).timestamp()
+ except ValueError:
+ return t
+ return t
+
+
+def _ikey(i):
+ """A row id, for ordering: numbers before text, each in its own order."""
+ return (0, i, "") if isinstance(i, (int, float)) and not isinstance(i, bool) else (1, 0, str(i))
+
+
+def pos_key(pos):
+ """A cursor position [time, id] for ordering. An id of None means every row at that time was read, so it comes
+ after every [time, id] at the same time."""
+ t, i = (list(pos) + [None, None])[:2]
+ return (_tkey(t), 1, (0, 0, "")) if i is None else (_tkey(t), 0, _ikey(i))
+
+
+# ---------- the shape ----------
+def _ref(v, what):
+ if not isinstance(v, str) or not REF_RX.match(v):
+ raise ValueError(f"{what} is a Lens reference like lens:recording/7")
+
+
+def check_activity(a):
+ """The activity, if it has the shape every part of Lens reads (docs/automations.md, Activity): ValueError naming
+ the first problem otherwise."""
+ if not isinstance(a, dict):
+ raise ValueError("an activity is an object")
+ if not isinstance(a.get("id"), str) or not ID_RX.match(a["id"]):
+ raise ValueError("an activity's id is 1 to 120 lowercase letters, digits and _ . : -")
+ if a.get("type") not in TYPES:
+ raise ValueError(f"an activity's type is one of {', '.join(TYPES)}")
+ if a.get("kind") not in _KINDS:
+ raise ValueError(f"no kind {a.get('kind')}; register it with events.kind first")
+ try:
+ if not isinstance(a.get("published"), str) or not a["published"].endswith("Z"):
+ raise ValueError
+ _parse(a["published"])
+ except ValueError:
+ raise ValueError("published is an ISO 8601 time in UTC, like 2026-10-15T16:21:58.000Z") from None
+ actor = a.get("actor")
+ if not isinstance(actor, dict) or actor.get("type") not in ACTOR_TYPES:
+ raise ValueError(f"the actor is an object whose type is {', '.join(ACTOR_TYPES)}")
+ _ref(actor.get("id"), "the actor's id")
+ if not isinstance(actor.get("name"), str) or not actor["name"].strip():
+ raise ValueError("the actor has a name")
+ obj = a.get("object")
+ if not isinstance(obj, dict) or not isinstance(obj.get("type"), str) or not obj["type"]:
+ raise ValueError("the object is an object with a type")
+ if obj.get("id") is not None:
+ _ref(obj["id"], "the object's id")
+ if a.get("target") is not None and not isinstance(a["target"], dict):
+ raise ValueError("the target is an object, or null")
+ tags = a.get("tag") or []
+ if not isinstance(tags, list) or len(tags) > MAX_TAGS or not all(isinstance(t, dict) and t.get("type") for t in tags):
+ raise ValueError(f"tag is a list of at most {MAX_TAGS} objects, each with a type")
+ if not isinstance(a.get("summary"), str) or not a["summary"].strip() or len(a["summary"]) > SUMMARY_MAX:
+ raise ValueError(f"the summary is one line of at most {SUMMARY_MAX} characters")
+ if a.get("space") is not None and (not isinstance(a["space"], int) or isinstance(a["space"], bool)):
+ raise ValueError("space is a namespace id, or null")
+ lens = a.get("lens")
+ if not isinstance(lens, dict) or not isinstance(lens.get("source"), str):
+ raise ValueError("lens is an object naming the source that made the activity")
+ if not isinstance(lens.get("pos"), list) or len(lens["pos"]) != 2:
+ raise ValueError("lens.pos is the source's position, [time, id]")
+ paths = lens.get("untrusted") or []
+ if not isinstance(paths, list) or not all(isinstance(p, str) and PATH_RX.match(p) for p in paths):
+ raise ValueError("lens.untrusted is a list of dotted paths, like object.subject")
+ cause = lens.get("cause")
+ if cause is not None and (
+ not isinstance(cause, dict) or not all(isinstance(cause.get(k), int) for k in ("automation", "run", "depth"))
+ ):
+ raise ValueError("lens.cause is {automation, run, depth}, or null")
+ spaces = lens.get("spaces")
+ if spaces is not None and (not isinstance(spaces, list) or not all(isinstance(s, int) for s in spaces)):
+ raise ValueError("lens.spaces is a list of namespace ids")
+ if lens.get("audience", "space") not in AUDIENCES:
+ raise ValueError(f"lens.audience is one of {', '.join(AUDIENCES)}")
+ if lens.get("audience") == "owner" and not isinstance(lens.get("owner"), int):
+ raise ValueError("an activity only its owner sees names the owner's account (lens.owner)")
+ return a
+
+
+def build(
+ aid,
+ kind,
+ *,
+ actor,
+ obj,
+ summary,
+ space,
+ source,
+ pos,
+ published,
+ target=None,
+ tag=None,
+ untrusted=None,
+ cause=None,
+ spaces=None,
+ audience=None,
+ owner=None,
+):
+ """An activity in the shape check_activity wants, for sources to build theirs with. `untrusted` defaults to the
+ kind's outside-text paths; `type` is the kind's AS2 verb."""
+ k = _KINDS[kind]
+ lens = {"source": source, "pos": list(pos), "untrusted": list(k.untrusted if untrusted is None else untrusted), "cause": cause}
+ if spaces:
+ lens["spaces"] = sorted(set(spaces))
+ if audience and audience != "space":
+ lens["audience"] = audience
+ if owner is not None:
+ lens["owner"] = owner
+ return {
+ "id": aid,
+ "type": k.as_type,
+ "kind": kind,
+ "published": stamp(published),
+ "actor": actor,
+ "object": obj,
+ "target": target,
+ "tag": list(tag or []),
+ "summary": " ".join(str(summary).split())[:SUMMARY_MAX],
+ "space": space,
+ "lens": lens,
+ }
+
+
+def ref_id(ref, table):
+ """The id in a `lens:/` reference (an object's id, say), a number where it is one; None for a reference
+ to another table."""
+ m = re.match(rf"^lens:{re.escape(table)}/(.+)$", ref if isinstance(ref, str) else "")
+ if not m:
+ return None
+ return int(m.group(1)) if m.group(1).isdigit() else m.group(1)
+
+
+# ---------- the catalog of kinds ----------
+@dataclass(frozen=True)
+class Kind:
+ name: str
+ label: str
+ source: str
+ as_type: str
+ object_type: str
+ parent: str | None
+ fields: dict
+ untrusted: tuple[str, ...]
+ sample: dict | None
+ noisy: bool
+
+ def info(self):
+ return {
+ "name": self.name,
+ "label": self.label,
+ "source": self.source,
+ "type": self.as_type,
+ "object_type": self.object_type,
+ "parent": self.parent,
+ "children": sorted(k for k, v in _KINDS.items() if v.parent == self.name),
+ "fields": self.fields,
+ "schema": schema(self.name),
+ "untrusted": list(self.untrusted),
+ "noisy": self.noisy,
+ "sample": self.sample,
+ }
+
+
+_KINDS: dict[str, Kind] = {}
+BASE_FIELDS = {
+ "actor": {
+ "type": "object",
+ "title": "Who",
+ "properties": {
+ "type": {"type": "string", "title": "Kind of actor", "enum": list(ACTOR_TYPES)},
+ "id": {"type": "string", "title": "Actor"},
+ "name": {"type": "string", "title": "Actor's name"},
+ },
+ },
+ "object": {
+ "type": "object",
+ "title": "What",
+ "properties": {
+ "type": {"type": "string", "title": "Object type"},
+ "id": {"type": "string", "title": "Object"},
+ "name": {"type": "string", "title": "Name"},
+ "url": {"type": "string", "title": "Link"},
+ },
+ },
+}
+
+
+def kind(name, *, label, source, as_type, object_type, parent=None, fields=None, untrusted=(), sample=None, noisy=False):
+ """Register a kind of activity: its name (`.`), a label for people, the source that makes it, its
+ AS2 verb and object type, a parent kind (a trigger on the parent matches it too), JSON Schemas for its `object`
+ (and `actor` or `target`) in `fields`, the dotted paths that hold outside text, a realistic sample, and whether
+ it is `noisy` (left out of the timeline unless asked for). Registering a name again replaces it."""
+ if not KIND_RX.match(name or ""):
+ raise ValueError(f"a kind is named like noun.past_verb (email.received), not {name}")
+ if not NAME_RX.match(source or ""):
+ raise ValueError(f"kind {name}: name the source that makes it")
+ if as_type not in TYPES:
+ raise ValueError(f"kind {name}: its type is one of {', '.join(TYPES)}")
+ if parent is not None and (parent not in _KINDS or parent == name):
+ raise ValueError(f"kind {name}: register its parent {parent} first")
+ fields = dict(fields or {})
+ if set(fields) - {"object", "actor", "target"} or not all(isinstance(v, dict) for v in fields.values()):
+ raise ValueError(f"kind {name}: fields are JSON Schemas for object, actor or target")
+ if not all(isinstance(p, str) and PATH_RX.match(p) for p in untrusted):
+ raise ValueError(f"kind {name}: untrusted paths are dotted, like object.subject")
+ old = _KINDS.get(name)
+ _KINDS[name] = Kind(name, label, source, as_type, object_type, parent, fields, tuple(untrusted), sample, bool(noisy))
+ if sample is not None:
+ try:
+ check_activity(sample)
+ if sample["kind"] != name:
+ raise ValueError(f"its sample is a {sample['kind']}")
+ except ValueError as e:
+ if old:
+ _KINDS[name] = old
+ else:
+ del _KINDS[name]
+ raise ValueError(f"kind {name}: {e}") from None
+ return _KINDS[name]
+
+
+def kinds():
+ """Every kind, as the API and agents list them."""
+ return [k.info() for k in _KINDS.values()]
+
+
+def kind_info(name):
+ """One kind; KeyError when there's none."""
+ return _KINDS[name].info()
+
+
+def descendants(name):
+ """The kind and every kind under it (a trigger on asset.added matches email.received)."""
+ out, todo = set(), [name]
+ while todo:
+ k = todo.pop()
+ if k not in out:
+ out.add(k)
+ todo += [c for c, v in _KINDS.items() if v.parent == k]
+ return out
+
+
+def expand(names):
+ """The kinds named and their descendants; None (every kind) stays None. ValueError for a kind that isn't there."""
+ if names is None:
+ return None
+ unknown = [n for n in names if n not in _KINDS]
+ if unknown:
+ raise ValueError(f"no kind {unknown[0]}; GET /api/v1/activities/kinds lists them")
+ return set().union(*(descendants(n) for n in names)) if names else set()
+
+
+def _merged(base, extra):
+ if not extra:
+ return base
+ out = {**base, **{k: v for k, v in extra.items() if k != "properties"}}
+ out["properties"] = {**(base.get("properties") or {}), **(extra.get("properties") or {})}
+ return out
+
+
+def schema(name):
+ """A JSON Schema of the kind's activities: every activity's own fields, its parents' fields, then its own."""
+ chain, k = [], _KINDS.get(name)
+ while k:
+ chain.insert(0, k)
+ k = _KINDS.get(k.parent) if k.parent else None
+ parts = {p: dict(v) for p, v in BASE_FIELDS.items()}
+ for k in chain:
+ for p, v in k.fields.items():
+ parts[p] = _merged(parts.get(p) or {"type": "object"}, v)
+ return {
+ "type": "object",
+ "properties": {
+ "id": {"type": "string", "title": "Activity"},
+ "kind": {"type": "string", "title": "Kind"},
+ "published": {"type": "string", "format": "date-time", "title": "When"},
+ "summary": {"type": "string", "title": "Summary"},
+ "space": {"type": ["integer", "null"], "title": "Namespace"},
+ "tag": {"type": "array", "title": "Tags", "items": {"type": "object"}},
+ **parts,
+ },
+ }
+
+
+def field_at(name, path):
+ """The schema of the field at a dotted path in a kind's activities (list items by number), or None."""
+ node = schema(name) if name in _KINDS else None
+ for part in (path or "").split("."):
+ if not isinstance(node, dict):
+ return None
+ props = node.get("properties")
+ if props and part in props:
+ node = props[part]
+ elif node.get("type") == "array" and part.isdigit():
+ node = node.get("items")
+ else:
+ return None
+ return node if isinstance(node, dict) else None
+
+
+# Assets: the kind every new recording, document, image, email or web page is, and the parent of M2's kinds for each
+# (email.received, file.added, ...). The recordings source makes it (events_content.py).
+kind(
+ "asset.added",
+ label="Asset added",
+ source="recordings",
+ as_type="Create",
+ object_type="Document",
+ fields={
+ "object": {
+ "type": "object",
+ "properties": {
+ "type": {"type": "string", "title": "Object type", "enum": ["Audio", "Video", "Image", "Document"]},
+ "id": {"type": "string", "title": "Asset"},
+ "name": {"type": "string", "title": "Title"},
+ "url": {"type": "string", "title": "Link"},
+ "media": {"type": "string", "title": "Media", "enum": ["audio", "video", "image", "document", "transcript"]},
+ "content_type": {"type": "string", "title": "Content type"},
+ },
+ }
+ },
+ untrusted=("object.name",),
+ sample={
+ "id": "create-recording-7",
+ "type": "Create",
+ "kind": "asset.added",
+ "published": "2026-10-15T08:12:03.000Z",
+ "actor": {"type": "Person", "id": "lens:account/1", "name": "Sam"},
+ "object": {"type": "Audio", "id": "lens:recording/7", "name": "Call with the school", "url": "/resources/7", "media": "audio"},
+ "target": None,
+ "tag": [],
+ "summary": "Sam added “Call with the school”",
+ "space": 1,
+ "lens": {"source": "recordings", "pos": ["2026-10-15T08:12:03+00:00", 7], "untrusted": ["object.name"], "cause": None},
+ },
+)
+
+
+# ---------- sources ----------
+class Source:
+ """One family of rows read as activities. A source names itself, may list the kinds it makes (the catalog's kinds
+ whose `source` is its name count too), and reads:
+
+ - `read(db, cfg, pos, spaces, limit, hint)`: activities strictly after `pos` ([time, id], or [time, None] for
+ after every row at `time`), oldest first, at most `limit`; only namespaces in `spaces` unless it is None. Each
+ activity's `lens.pos` is its row's position. `hint` is narrowing it may use, like {"actors": [...]}.
+ - `page(db, cfg, before, viewer, flt, limit)`: newest first, strictly before `before` (`key` of the last one
+ shown) or from the newest, with what `viewer` may see and the filters it can apply cheaply; the core checks
+ every filter again.
+ - `get(db, cfg, activity_id)`: one activity by its id, or None (also for ids that aren't its own).
+ - `now(db)` and `back(db, seconds)`: the position of now, and of `seconds` ago, in its own terms.
+ - `key(a)`: the position `page` orders by, [time, id]; its `lens.pos` unless it pages by another column.
+ """
+
+ name = ""
+ kinds: tuple[str, ...] = ()
+
+ def read(self, db, cfg, pos, spaces, limit, hint=None):
+ raise NotImplementedError
+
+ def page(self, db, cfg, before, viewer, flt, limit):
+ raise NotImplementedError
+
+ def get(self, db, cfg, activity_id):
+ return None
+
+ def now(self, db):
+ return [store.now(), None]
+
+ def back(self, db, seconds):
+ return [ago(seconds), None]
+
+ def key(self, a):
+ return a["lens"]["pos"]
+
+
+_SOURCES: dict[str, Source] = {}
+
+
+def register_source(src):
+ """Add a source to the stream (registering a name again replaces it)."""
+ if not NAME_RX.match(getattr(src, "name", "") or ""):
+ raise ValueError("a source has a name of lowercase letters, digits and _ . : -")
+ _SOURCES[src.name] = src
+ return src
+
+
+def sources():
+ return list(_SOURCES.values())
+
+
+def produces(src):
+ """The kinds a source can make: those it lists and the catalog's kinds that name it."""
+ return set(src.kinds) | {k for k, v in _KINDS.items() if v.source == src.name}
+
+
+def after_sql(col, pos, params, op=">", table=None):
+ """The SQL for "after `pos`" ([time, id], id None for after every row at time) on a table read by (`col`, id),
+ with its parameters added to `params`; op "<" for "before" when paging newest first. With `table`, the id is a
+ key of that table's records, so it compares as SurrealDB orders record ids."""
+ params["pt"] = pos[0]
+ if pos[1] is None:
+ return f"{col} {op} $pt"
+ params["pi"] = R(table, pos[1]) if table else pos[1]
+ return f"({col} {op} $pt OR ({col} = $pt AND id {op} $pi))"
+
+
+# ---------- who sees what ----------
+def viewer(account=None, spaces=None, admin=False):
+ """Who reads the timeline: their account id, the namespaces they read (None: every one, for admins) and whether
+ they are an admin."""
+ return {"account": account, "spaces": None if spaces is None else set(spaces), "admin": bool(admin)}
+
+
+def spaces_of(a):
+ """Every namespace an activity belongs to."""
+ out = [a.get("space")] + list((a.get("lens") or {}).get("spaces") or [])
+ return [s for s in dict.fromkeys(out) if s is not None]
+
+
+def visible(a, who):
+ """Whether `who` (viewer) may see an activity: `lens.audience` owner is its owner alone, admins is admins, and
+ space (the default) is everyone who reads one of its namespaces (admins read all; one with no namespace is for
+ admins)."""
+ lens = a.get("lens") or {}
+ aud = lens.get("audience") or "space"
+ if aud == "owner":
+ return who.get("account") is not None and lens.get("owner") == who["account"]
+ if aud == "admins" or who.get("admin"):
+ return bool(who.get("admin"))
+ readable = who.get("spaces") or set()
+ return any(s in readable for s in spaces_of(a))
+
+
+# ---------- the event log ----------
+@contextlib.contextmanager
+def caused_by(run):
+ """Activities emitted inside are caused by an automation's run: {"automation": 4, "run": 51, "depth": 1}."""
+ if not (isinstance(run, dict) and all(isinstance(run.get(k), int) for k in ("automation", "run", "depth"))):
+ raise ValueError("a cause is {automation, run, depth}")
+ token = _cause.set({k: run[k] for k in ("automation", "run", "depth")})
+ try:
+ yield
+ finally:
+ _cause.reset(token)
+
+
+def cause():
+ """The run whose actions are running now ({automation, run, depth}), or None."""
+ c = _cause.get()
+ return dict(c) if c else None
+
+
+def _key_ref(key):
+ return R("event_log_key", hashlib.sha1(str(key).encode()).hexdigest())
+
+
+def emit(
+ db,
+ kind,
+ *,
+ actor,
+ obj,
+ space,
+ summary,
+ target=None,
+ tag=None,
+ key=None,
+ published=None,
+ spaces=None,
+ audience=None,
+ owner=None,
+ untrusted=None,
+):
+ """Log an activity that no row of its own records (a deletion, a webhook call) and wake the followers in this
+ process. Its id (`log-`). With `key`, emitting the same key again changes nothing and gives the first id, so a
+ retry never doubles it. Inside `caused_by`, the activity carries that cause."""
+ k = _KINDS.get(kind)
+ if not k:
+ raise ValueError(f"no kind {kind}; register it with events.kind first")
+ if k.source != "log":
+ raise ValueError(f"{kind} is read from its own rows ({k.source}), not emitted")
+ if key is not None:
+ known = db.one("SELECT log FROM $r", r=_key_ref(key))
+ if known:
+ return f"log-{known['log']}"
+ n = db.next_id("event_log")
+ at = store.now()
+ a = build(
+ f"log-{n}",
+ kind,
+ actor=actor,
+ obj=obj,
+ summary=summary,
+ space=space,
+ source="log",
+ pos=[at, n],
+ published=published or at,
+ target=target,
+ tag=tag,
+ untrusted=untrusted,
+ cause=cause(),
+ spaces=spaces,
+ audience=audience,
+ owner=owner,
+ )
+ check_activity(a)
+ row = store.clean(
+ {
+ "kind": kind,
+ "type": a["type"],
+ "published": a["published"],
+ "logged_at": at,
+ "actor": a["actor"],
+ "object": a["object"],
+ "target": a["target"],
+ "tag": a["tag"],
+ "summary": a["summary"],
+ "space": space,
+ "spaces": a["lens"].get("spaces"),
+ "audience": a["lens"].get("audience"),
+ "owner": owner,
+ "untrusted": a["lens"]["untrusted"],
+ "cause": a["lens"]["cause"],
+ "key": key,
+ }
+ )
+ st, p = ["CREATE $e CONTENT $d"], {"e": R("event_log", n), "d": row}
+ if key is not None:
+ st.insert(0, "CREATE $k CONTENT $kd")
+ p.update(k=_key_ref(key), kd={"key": str(key)[:500], "log": n, "at": at})
+ try:
+ db.run(st, **p)
+ except Exception as e: # noqa: BLE001 - only a key that's already there means someone else logged it first
+ if key is None or "already exists" not in str(e):
+ raise
+ return f"log-{db.one('SELECT log FROM $r', r=_key_ref(key))['log']}"
+ wake()
+ return a["id"]
+
+
+LOG_FIELDS = (
+ "record::id(id) AS n, kind, published, logged_at, actor, object, target, tag, summary, space, spaces, audience, owner, untrusted, cause"
+)
+
+
+def _from_log(r):
+ return build(
+ f"log-{r['n']}",
+ r["kind"],
+ actor=r["actor"],
+ obj=r["object"],
+ summary=r["summary"],
+ space=r.get("space"),
+ source="log",
+ pos=[r["logged_at"], r["n"]],
+ published=r["published"],
+ target=r.get("target"),
+ tag=r.get("tag"),
+ untrusted=r.get("untrusted") or [],
+ cause=r.get("cause"),
+ spaces=r.get("spaces"),
+ audience=r.get("audience"),
+ owner=r.get("owner"),
+ )
+
+
+def space_sql(spaces, params, col="space", many=None):
+ """SQL keeping rows of the namespaces in `spaces` (None: all), its parameters added to `params`."""
+ if spaces is None:
+ return "true"
+ params["sp"] = sorted(spaces)
+ return f"({col} IN $sp" + (f" OR {many} CONTAINSANY $sp)" if many else ")")
+
+
+class LogSource(Source):
+ """event_log: activities that no row of their own records, each with the kind it was emitted as. Followed by
+ (logged_at, id): ids are given out in order, and a row written late is caught by the lookback. Paged by
+ `published`."""
+
+ name = "log"
+
+ def _kinds_sql(self, flt, params):
+ ks = (flt or {}).get("kinds")
+ if ks is None:
+ return "true"
+ params["ks"] = sorted(ks)
+ return "kind IN $ks"
+
+ def read(self, db, cfg, pos, spaces, limit, hint=None):
+ p = {}
+ where = f"{after_sql('logged_at', pos, p, table='event_log')} AND {space_sql(spaces, p, many='spaces')}"
+ rows = db.rows(f"SELECT {LOG_FIELDS}, id AS rid FROM event_log WHERE {where} ORDER BY logged_at, rid LIMIT {int(limit)}", **p)
+ return [_from_log(r) for r in rows if r.get("kind") in _KINDS]
+
+ def page(self, db, cfg, before, who, flt, limit):
+ p = {}
+ where = [self._kinds_sql(flt, p)]
+ if before is not None:
+ where.append(after_sql("published", before, p, "<", table="event_log"))
+ if not who.get("admin"):
+ p["me"] = who.get("account")
+ readable = space_sql(who.get("spaces") or set(), p, many="spaces")
+ where.append(f"(((audience = NONE OR audience = 'space') AND {readable}) OR (audience = 'owner' AND owner = $me))")
+ else:
+ p["me"] = who.get("account")
+ where.append("(audience != 'owner' OR owner = $me)")
+ for k, op in (("since", ">"), ("until", "<=")):
+ if (flt or {}).get(k):
+ p[k] = flt[k]
+ where.append(f"published {op} ${k}")
+ rows = db.rows(
+ f"SELECT {LOG_FIELDS}, id AS rid FROM event_log WHERE {' AND '.join(where)} ORDER BY published DESC, rid DESC LIMIT {int(limit)}",
+ **p,
+ )
+ return [_from_log(r) for r in rows if r.get("kind") in _KINDS]
+
+ def get(self, db, cfg, activity_id):
+ m = re.match(r"^log-(\d+)$", activity_id)
+ if not m:
+ return None
+ r = db.one(f"SELECT {LOG_FIELDS} FROM $r", r=R("event_log", int(m.group(1))))
+ return _from_log(r) if r and r.get("kind") in _KINDS else None
+
+ def key(self, a):
+ return [a["published"], a["lens"]["pos"][1]]
+
+
+register_source(LogSource())
+
+
+# ---------- following ----------
+def _check_name(consumer):
+ if not NAME_RX.match(consumer or ""):
+ raise ValueError("a consumer is named with lowercase letters, digits and _ . : - (automations, say)")
+
+
+def _cursor_ref(consumer):
+ return R("event_cursor", consumer)
+
+
+def _later_of(a, b):
+ return a if b is None or pos_key(a) >= pos_key(b) else b
+
+
+def _start(db, consumer, at=None):
+ """Every source's cursor at now: what happened before isn't read, not even by the lookback."""
+ pos = {s.name: s.now(db) for s in sources()}
+ db.q("UPSERT $c SET pos = $p, floor = $p, at = $t", c=_cursor_ref(consumer), p=pos, t=at or store.now())
+ return pos
+
+
+def idle(db, consumer):
+ """A consumer with nothing to do moves every cursor to now, so nothing that happens meanwhile replays later."""
+ _check_name(consumer)
+ _start(db, consumer)
+
+
+def _in(a, want, spaces):
+ if want is not None and a.get("kind") not in want:
+ return False
+ return spaces is None or any(s in spaces for s in spaces_of(a))
+
+
+def _order(a):
+ """Oldest first across sources: by the time of each one's position (its published time when the position isn't a
+ time), then source and id."""
+ p = a["lens"]["pos"]
+ t = _tkey(p[0])
+ if not isinstance(t, float):
+ t = _parse(a["published"]).timestamp()
+ return (t, a["lens"]["source"], _ikey(p[1]))
+
+
+def follow(db, cfg, consumer, kinds=None, spaces=None, limit=PAGE, hint=None):
+ """What happened since this consumer's last commit: (activities oldest first, commit). Handle them in order, then
+ call commit() to save where the next look starts. Only `kinds` (and their descendants; None: every kind) in
+ `spaces` (None: every namespace and none). A source with more waiting gives `limit` at a time; once caught up, a
+ read gives the last LOOKBACK seconds again (minus what this process already handled), so claim each before acting
+ on it. The first look only marks where this consumer starts."""
+ _check_name(consumer)
+ want = expand(kinds)
+ spaces = None if spaces is None else set(spaces)
+ st = db.one("SELECT pos, floor FROM $c", c=_cursor_ref(consumer))
+ if not st or not isinstance(st.get("pos"), dict):
+ _start(db, consumer)
+ return [], lambda: None
+ pos, floor = dict(st["pos"]), dict(st.get("floor") or {})
+ moved, floors, out = {}, {}, []
+ with _lock:
+ seen = dict(_seen.get(consumer) or {})
+ for s in sources():
+ here = pos.get(s.name)
+ if here is None or (want is not None and not produces(s) & want):
+ moved[s.name] = floors[s.name] = s.now(db) # nobody wants these: when someone does, they start from then
+ continue
+ rows = s.read(db, cfg, here, spaces, limit, hint)
+ if rows:
+ moved[s.name] = rows[-1]["lens"]["pos"]
+ if len(rows) < limit: # caught up: the last LOOKBACK seconds again, not before where this consumer started
+ have = {a["id"] for a in rows}
+ since = s.back(db, LOOKBACK)
+ if floor.get(s.name) is not None and pos_key(floor[s.name]) > pos_key(since):
+ since = floor[s.name]
+ for _ in range(PAGE_ROUNDS):
+ if pos_key(since) >= pos_key(here):
+ break
+ back = s.read(db, cfg, since, spaces, limit, hint)
+ rows += [a for a in back if pos_key(a["lens"]["pos"]) <= pos_key(here) and a["id"] not in have and a["id"] not in seen]
+ if len(back) < limit:
+ break
+ since = back[-1]["lens"]["pos"]
+ out += [a for a in rows if _in(a, want, spaces)]
+ out.sort(key=_order)
+
+ def commit():
+ cur = db.one("SELECT pos, floor FROM $c", c=_cursor_ref(consumer)) or {}
+ p, f = dict(cur.get("pos") or pos), dict(cur.get("floor") or floor)
+ for name, at in moved.items():
+ p[name] = _later_of(at, p.get(name))
+ for name, at in floors.items():
+ f[name] = _later_of(at, f.get(name))
+ db.q("UPSERT $c SET pos = $p, floor = $f, at = $t", c=_cursor_ref(consumer), p=p, f=f, t=store.now())
+ with _lock:
+ mine = _seen.setdefault(consumer, {})
+ for a in out:
+ mine[a["id"]] = None
+ while len(mine) > 20000:
+ mine.pop(next(iter(mine)))
+
+ return out, commit
+
+
+def _claim_ref(consumer, key):
+ return R("event_claim", hashlib.sha1(f"{consumer}|{key}".encode()).hexdigest())
+
+
+def _known(ref_id):
+ with _lock:
+ _claimed[ref_id] = None
+ while len(_claimed) > 50000:
+ _claimed.pop(next(iter(_claimed)))
+
+
+def claim(db, consumer, key):
+ """True for the one process (and the one time) that claims `key` for `consumer`, such as an activity's id."""
+ _check_name(consumer)
+ ref = _claim_ref(consumer, key)
+ with _lock:
+ if ref.id in _claimed:
+ return False
+ try:
+ db.run(["CREATE $r CONTENT $d"], r=ref, d={"consumer": consumer, "key": str(key)[:500], "at": store.now()})
+ except Exception as e: # noqa: BLE001 - only a claim that's already there means another process has it
+ if "already exists" not in str(e):
+ raise
+ _known(ref.id)
+ return False
+ _known(ref.id)
+ return True
+
+
+def unclaimed(db, consumer, keys):
+ """The keys nobody has claimed yet, in one query: for skipping what is taken before claiming one by one (it claims
+ nothing)."""
+ refs_ = {k: _claim_ref(consumer, k) for k in keys}
+ with _lock:
+ todo = {k: r for k, r in refs_.items() if r.id not in _claimed}
+ if not todo:
+ return []
+ taken = set(db.values("SELECT VALUE record::id(id) FROM event_claim WHERE id IN $ids", ids=list(todo.values())))
+ for r in todo.values():
+ if r.id in taken:
+ _known(r.id)
+ return [k for k, r in todo.items() if r.id not in taken]
+
+
+def wake():
+ """Have the followers in this process look at once (emit calls it)."""
+ with _wakes:
+ _generation[0] += 1
+ _wakes.notify_all()
+
+
+def generation():
+ """How many wakes this process has had: take it before a look, and give it to `wait` after."""
+ with _wakes:
+ return _generation[0]
+
+
+def wait(seconds, stop=None, since=None):
+ """Sleep until a wake after `since` (a `generation()`; default: now), `stop` (a threading.Event) is set, or
+ `seconds` pass; True when woken. Every follower waiting is woken, and a wake that came during the look counts."""
+ until = time.monotonic() + max(0.0, float(seconds))
+ with _wakes:
+ seen = _generation[0] if since is None else since
+ while _generation[0] == seen:
+ left = until - time.monotonic()
+ if left <= 0 or (stop is not None and stop.is_set()):
+ return False
+ _wakes.wait(min(0.5, left))
+ return True
+
+
+def section(cfg):
+ return {"keep_days": 30, "poll_seconds": 5, **(cfg.get("events") or {})}
+
+
+def poll_seconds(cfg):
+ """How often followers look when nothing wakes them (events.poll_seconds)."""
+ return max(1, int(section(cfg)["poll_seconds"] or 5))
+
+
+# ---------- the timeline ----------
+def check_filter(flt):
+ """Timeline filters, cleaned: kinds (names; their descendants count too), actor (an actor's id, or several),
+ spaces (namespace ids), tag (an entity's id or name), since (published after) and until (at or before), q (words in
+ the summary) and where (a match tree on the activity). ValueError naming the first problem."""
+ from . import match
+
+ flt = dict(flt or {})
+ extra = sorted(set(flt) - set(FILTERS))
+ if extra:
+ raise ValueError(f"no filter {extra[0]}; filters are {', '.join(FILTERS)}")
+ out = {}
+ if flt.get("kinds"):
+ ks = [flt["kinds"]] if isinstance(flt["kinds"], str) else list(flt["kinds"])
+ expand(ks)
+ out["kinds"] = ks
+ if flt.get("actor"):
+ a = [flt["actor"]] if isinstance(flt["actor"], str) else list(flt["actor"])
+ for x in a:
+ _ref(x, "actor")
+ out["actor"] = a
+ if flt.get("spaces") is not None:
+ if not isinstance(flt["spaces"], (list, set, tuple)) or not all(isinstance(s, int) for s in flt["spaces"]):
+ raise ValueError("spaces is a list of namespace ids")
+ out["spaces"] = set(flt["spaces"])
+ if flt.get("tag"):
+ out["tag"] = str(flt["tag"]).strip()[:200]
+ for k in ("since", "until"):
+ if flt.get(k):
+ try:
+ out[k] = stamp(flt[k])
+ except ValueError:
+ raise ValueError(f"{k} is a time like 2026-10-15T16:21:58Z") from None
+ if flt.get("q"):
+ out["q"] = [w for w in str(flt["q"]).casefold().split() if w][:20]
+ if flt.get("where"):
+ out["where"] = match.check(flt["where"], "where")
+ return out
+
+
+def _matches(a, flt):
+ from . import match
+
+ if flt.get("actor") and (a.get("actor") or {}).get("id") not in flt["actor"]:
+ return False
+ if flt.get("spaces") is not None and not any(s in flt["spaces"] for s in spaces_of(a)):
+ return False
+ if flt.get("tag"):
+ t = flt["tag"].casefold()
+ if not any(str(x.get("id") or "").casefold() == t or str(x.get("name") or "").casefold() == t for x in a.get("tag") or []):
+ return False
+ if flt.get("since") and not a["published"] > flt["since"]:
+ return False
+ if flt.get("until") and not a["published"] <= flt["until"]:
+ return False
+ if flt.get("q"):
+ text = a.get("summary", "").casefold()
+ if not all(w in text for w in flt["q"]):
+ return False
+ return not flt.get("where") or match.test(flt["where"], a)
+
+
+def _wanted(flt):
+ """The kinds a timeline page shows: those asked for (with descendants, noisy ones only when named), else every
+ kind that isn't noisy."""
+ if flt.get("kinds"):
+ named = set(flt["kinds"])
+ return {k for k in expand(flt["kinds"]) if not _KINDS[k].noisy or k in named}
+ return {k for k, v in _KINDS.items() if not v.noisy}
+
+
+def _encode(cur):
+ raw = json.dumps(cur, separators=(",", ":"), sort_keys=True).encode()
+ return base64.urlsafe_b64encode(raw).decode().rstrip("=")
+
+
+def _decode(before):
+ try:
+ cur = json.loads(base64.urlsafe_b64decode(before + "=" * (-len(before) % 4)))
+ if not isinstance(cur, dict) or not all(v is None or (isinstance(v, list) and len(v) == 2) for v in cur.values()):
+ raise ValueError
+ return cur
+ except (ValueError, TypeError):
+ raise ValueError("before is the `next` of an earlier page") from None
+
+
+def page(db, cfg, who, before=None, flt=None, limit=50):
+ """A page of the timeline for `who` (viewer), newest first: {items, next}. `next` goes in `before` for the page
+ after (None at the end). Filters (check_filter) thin a page out; it reads on, up to PAGE_ROUNDS source pages each,
+ so a filtered page isn't short while there is more."""
+ flt = check_filter(flt)
+ limit = max(1, min(int(limit), 200))
+ want = _wanted(flt)
+ sflt = {**flt, "kinds": want}
+ cur = _decode(before) if before else None
+ srcs = [s for s in sources() if produces(s) & want and (cur is None or s.name in cur)]
+ at = {s.name: (cur or {}).get(s.name) for s in srcs}
+ got: dict[str, list] = {s.name: [] for s in srcs}
+ reads = dict.fromkeys(got, 0)
+ done: set[str] = set()
+
+ def ok(a):
+ return a.get("kind") in want and visible(a, who) and _matches(a, flt)
+
+ def k(s, a):
+ return (a["published"], s.name, _ikey(s.key(a)[1]))
+
+ def fetch(s):
+ rows = s.page(db, cfg, at[s.name], who, sflt, limit)[:limit]
+ reads[s.name] += 1
+ got[s.name] += rows
+ if rows:
+ at[s.name] = s.key(rows[-1])
+ if len(rows) < limit:
+ done.add(s.name)
+
+ for s in srcs:
+ fetch(s)
+ while True:
+ # past the oldest row read from a source with more to read, that source may have something newer unread
+ open_ = [s for s in srcs if s.name not in done]
+ horizon = max((k(s, got[s.name][-1]) for s in open_), default=None)
+ fit = sorted(
+ ((k(s, a), s.name, a) for s in srcs for a in got[s.name] if ok(a) and (horizon is None or k(s, a) >= horizon)),
+ key=lambda x: x[0],
+ reverse=True,
+ )
+ if len(fit) >= limit or not open_:
+ break
+ more = [s for s in open_ if k(s, got[s.name][-1]) == horizon and reads[s.name] < PAGE_ROUNDS]
+ if not more:
+ break
+ for s in more:
+ fetch(s)
+ items = [a for _, _, a in fit[:limit]]
+ shown = {a["id"] for a in items}
+ nxt = {}
+ for s in srcs:
+ last, stopped = (cur or {}).get(s.name), False
+ for a in got[s.name]:
+ if ok(a) and a["id"] not in shown:
+ stopped = True
+ break
+ last = s.key(a)
+ if stopped or s.name not in done:
+ nxt[s.name] = last
+ return {"items": items, "next": _encode(nxt) if nxt else None}
+
+
+def get(db, cfg, activity_id, who):
+ """One activity `who` may see, by its id; None when there's none or it isn't theirs to see."""
+ if not isinstance(activity_id, str) or not ID_RX.match(activity_id):
+ return None
+ for s in sources():
+ a = s.get(db, cfg, activity_id)
+ if a:
+ return a if visible(a, who) else None
+ return None
+
+
+# ---------- keeping it small ----------
+def tidy(db, cfg, now=None):
+ """Drop event_log rows older than events.keep_days and claims older than a day, at most once an hour per process
+ (routines.run_due calls it every 30 seconds); the cut-off, or 0 when it didn't look."""
+ t = time.monotonic()
+ if now is None and _swept["at"] and t - _swept["at"] < SWEEP_SECONDS:
+ return 0
+ _swept["at"] = t
+ days = int(section(cfg)["keep_days"] or 30)
+ cut = ago(days * 86400, now)
+ db.q("DELETE event_log WHERE logged_at < $c", c=cut)
+ db.q("DELETE event_log_key WHERE at < $c", c=cut)
+ db.q("DELETE event_claim WHERE at < $c", c=ago(CLAIM_DAYS * 86400, now))
+ return cut
+
+
+from . import events_content # noqa: E402, F401 - the recordings source, registered with the stream once it is loaded
diff --git a/fastapi_backend/app/domain/events_content.py b/fastapi_backend/app/domain/events_content.py
new file mode 100644
index 000000000..39e8acbaa
--- /dev/null
+++ b/fastapi_backend/app/domain/events_content.py
@@ -0,0 +1,154 @@
+"""Content as activities (docs/automations.md, Sources): the `recordings` source reads the assets Lens keeps (the
+`recording` table: audio, video, documents, images, emails, web pages) as `asset.added` activities, by `created_at`
+with the id breaking ties, so nothing is stored twice.
+
+Each one's id is `create-recording-`, so it is the same every time it's read. Its actor is whoever added it: an
+automation (the asset's first job was queued by `automation:`), the storage source a watched file came from, the
+person whose upload it was, or Lens itself (a folder scan). Its object is the asset; its title is outside text.
+"""
+
+from __future__ import annotations
+
+import re
+
+from . import events, store
+
+R = store.R
+FIELDS = "id AS rid, record::id(id) AS id, space, title, source, media, remote, content_type, created_at"
+TYPES = {"audio": "Audio", "video": "Video", "image": "Image", "document": "Document", "transcript": "Document"}
+AUTOMATION_RX = re.compile(r"^automation:(\d+)$")
+
+
+def _sec(t):
+ """A time as `created_at` is written (store.now(): to the second, +00:00), cut down to the second."""
+ return events._parse(t).replace(microsecond=0).isoformat(timespec="seconds")
+
+
+def _actors(db, rows):
+ """{recording id: actor} for these rows, in a few queries for all of them."""
+ ids = [r["id"] for r in rows]
+ if not ids:
+ return {}
+ first_job: dict = {}
+ for j in db.rows("SELECT recording, created_by, created_at FROM job WHERE recording IN $ids ORDER BY created_at", ids=ids):
+ first_job.setdefault(j["recording"], j.get("created_by"))
+ uploads = {u["recording"]: u for u in db.rows("SELECT recording, account, email FROM upload WHERE recording IN $ids", ids=ids)}
+ accounts = {u["account"] for u in uploads.values() if isinstance(u.get("account"), int)}
+ names = {
+ a["id"]: a.get("name") or a.get("email")
+ for a in (
+ db.rows("SELECT record::id(id) AS id, name, email FROM account WHERE id IN $a", a=[R("account", n) for n in accounts])
+ if accounts
+ else []
+ )
+ }
+ srcs = {(r.get("remote") or {}).get("source") for r in rows} - {None}
+ sources = {
+ s["id"]: s.get("name")
+ for s in (
+ db.rows("SELECT record::id(id) AS id, name FROM storage_source WHERE id IN $s", s=[R("storage_source", n) for n in srcs])
+ if srcs
+ else []
+ )
+ }
+ out = {}
+ for r in rows:
+ m = AUTOMATION_RX.match(str(first_job.get(r["id"]) or ""))
+ sid = (r.get("remote") or {}).get("source")
+ up = uploads.get(r["id"])
+ if m:
+ out[r["id"]] = {"type": "Application", "id": f"lens:automation/{m.group(1)}", "name": f"Automation {m.group(1)}"}
+ elif sid is not None and re.match(r"^[A-Za-z0-9_.:@+-]{1,200}$", str(sid)):
+ out[r["id"]] = {"type": "Service", "id": f"lens:source/{sid}", "name": sources.get(sid) or f"Source {sid}"}
+ elif up and isinstance(up.get("account"), int):
+ out[r["id"]] = {
+ "type": "Person",
+ "id": f"lens:account/{up['account']}",
+ "name": names.get(up["account"]) or up.get("email") or "Someone",
+ }
+ else:
+ out[r["id"]] = dict(events.LENS)
+ return out
+
+
+def _activities(db, rows):
+ """The rows as asset.added activities (a row whose created_at isn't a time is skipped)."""
+ from . import render
+
+ who = _actors(db, rows)
+ out = []
+ for r in rows:
+ try:
+ published = events.stamp(r["created_at"])
+ except (TypeError, ValueError):
+ continue
+ media = render.kind(r)
+ title = " ".join(str(r.get("title") or "").split()) or "Untitled"
+ actor = who[r["id"]]
+ obj = store.clean(
+ {
+ "type": TYPES.get(media, "Document"),
+ "id": f"lens:recording/{r['id']}",
+ "name": title,
+ "url": f"/resources/{r['id']}",
+ "media": media,
+ "content_type": r.get("content_type") if isinstance(r.get("content_type"), str) else None,
+ }
+ )
+ out.append(
+ events.build(
+ f"create-recording-{r['id']}",
+ "asset.added",
+ actor=actor,
+ obj=obj,
+ summary=f"{actor['name']} added “{title[:200]}”",
+ space=r.get("space"),
+ source="recordings",
+ pos=[r["created_at"], r["id"]],
+ published=published,
+ )
+ )
+ return out
+
+
+class RecordingsSource(events.Source):
+ """recording rows as asset.added, read by (created_at, id)."""
+
+ name = "recordings"
+ kinds = ("asset.added",)
+
+ def read(self, db, cfg, pos, spaces, limit, hint=None):
+ p: dict = {}
+ where = f"{events.after_sql('created_at', pos, p, table='recording')} AND {events.space_sql(spaces, p)}"
+ rows = db.rows(f"SELECT {FIELDS} FROM recording WHERE {where} ORDER BY created_at, rid LIMIT {int(limit)}", **p)
+ return _activities(db, rows)
+
+ def page(self, db, cfg, before, who, flt, limit):
+ p: dict = {}
+ where = [events.space_sql(None if who.get("admin") else (who.get("spaces") or set()), p), "created_at != NONE"]
+ if before is not None:
+ where.append(events.after_sql("created_at", before, p, "<", table="recording"))
+ if flt.get("spaces") is not None:
+ p["fs"] = sorted(flt["spaces"])
+ where.append("space IN $fs")
+ if flt.get("since"): # a looser bound, to the second; the stream checks each filter exactly again
+ p["since"] = _sec(flt["since"])
+ where.append("created_at > $since")
+ if flt.get("until"):
+ p["until"] = _sec(flt["until"])
+ where.append("created_at <= $until")
+ rows = db.rows(
+ f"SELECT {FIELDS} FROM recording WHERE {' AND '.join(where)} ORDER BY created_at DESC, rid DESC LIMIT {int(limit)}", **p
+ )
+ return _activities(db, rows)
+
+ def get(self, db, cfg, activity_id):
+ m = re.match(r"^create-recording-(\d+)$", activity_id)
+ if not m:
+ return None
+ r = db.one(f"SELECT {FIELDS} FROM $r", r=R("recording", int(m.group(1))))
+ got = _activities(db, [r]) if r else []
+ return got[0] if got else None
+
+
+events.register_source(RecordingsSource())
diff --git a/fastapi_backend/app/domain/match.py b/fastapi_backend/app/domain/match.py
index 793d86059..75b7008a0 100644
--- a/fastapi_backend/app/domain/match.py
+++ b/fastapi_backend/app/domain/match.py
@@ -100,7 +100,13 @@ def paths(tree):
def _label(path, kind):
- """A path in words (`kind` is for naming fields by their titles, once kinds have schemas)."""
+ """A path in words: with a kind, the title its schema gives the field there ("the title"), else the path."""
+ if kind and path:
+ from . import events
+
+ title = (events.field_at(kind, path) or {}).get("title")
+ if title:
+ return f"the {title[0].lower()}{title[1:]}"
return path or "the value"
diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py
index 681498a27..397b5d81b 100644
--- a/fastapi_backend/app/domain/routines.py
+++ b/fastapi_backend/app/domain/routines.py
@@ -27,6 +27,7 @@
analytics,
backups,
budgets,
+ events,
fedora,
jobs,
notebook,
@@ -603,6 +604,11 @@ def run_due(db, cfg, log=print, now=None):
stamp = _iso(now)
sweep(db, now)
activity.tidy(db, cfg) # the activity ledger past activity.keep_days, once an hour
+ try:
+ events.tidy(db, cfg) # the event log past events.keep_days and claims past a day, once an hour
+ except Exception as e: # noqa: BLE001 - the schedule carries on
+ if log:
+ log(f"events: {type(e).__name__}: {e}")
try:
analytics.tick(db, cfg, now) # once a day, while Google Analytics is on (docs/analytics.md)
except Exception as e: # noqa: BLE001 - the schedule carries on
diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
index a1105be18..66826a91b 100644
--- a/fastapi_backend/app/domain/store.py
+++ b/fastapi_backend/app/domain/store.py
@@ -619,6 +619,9 @@ def labels():
# the activity ledger (docs/activity.md): every change and call in or out, with its cost, per resource. On by
# default; reads: log API reads (GET) too; keep_days: how long rows are kept
"activity": {"enabled": True, "reads": False, "keep_days": 365},
+ # the stream of activities automations follow (docs/automations.md): keep_days: how long activities with no row
+ # of their own (the event log) are kept; poll_seconds: how often followers look when nothing wakes them
+ "events": {"keep_days": 30, "poll_seconds": 5},
# database backups (docs/database.md, Backups): schedule is a cron expression in timezone, or none (the default)
# for none; the newest `keep` scheduled backups are kept. connection: a storage connection (sources.py) every new
# backup is also copied to, under folder, encrypted with a key from secret.key first (none: kept here only)
@@ -1250,6 +1253,18 @@ def close(self):
"DEFINE INDEX IF NOT EXISTS notify_delivery_status ON notify_delivery FIELDS status",
"DEFINE TABLE IF NOT EXISTS notify_event SCHEMALESS",
"DEFINE TABLE IF NOT EXISTS notify_state SCHEMALESS",
+ # the stream of activities (domain/events.py, docs/automations.md): activities no row of their own records
+ # (event_log:), the keys they were emitted with (event_log_key:), where each consumer's next look starts
+ # (event_cursor:) and what each consumer claimed (event_claim:)
+ "DEFINE TABLE IF NOT EXISTS event_log SCHEMALESS",
+ "DEFINE INDEX IF NOT EXISTS event_log_logged ON event_log FIELDS logged_at",
+ "DEFINE INDEX IF NOT EXISTS event_log_published ON event_log FIELDS published",
+ "DEFINE TABLE IF NOT EXISTS event_log_key SCHEMALESS",
+ "DEFINE INDEX IF NOT EXISTS event_log_key_at ON event_log_key FIELDS at",
+ "DEFINE TABLE IF NOT EXISTS event_cursor SCHEMALESS",
+ "DEFINE TABLE IF NOT EXISTS event_claim SCHEMALESS",
+ "DEFINE INDEX IF NOT EXISTS event_claim_at ON event_claim FIELDS at",
+ "DEFINE INDEX IF NOT EXISTS upload_recording ON upload FIELDS recording",
# every line of a run's log, in chunks (jobs.RunLog): job_log:
"DEFINE TABLE IF NOT EXISTS job_log SCHEMALESS",
"DEFINE INDEX IF NOT EXISTS job_log_job ON job_log FIELDS job",
diff --git a/fastapi_backend/tests/domain/test_events.py b/fastapi_backend/tests/domain/test_events.py
new file mode 100644
index 000000000..720d78c82
--- /dev/null
+++ b/fastapi_backend/tests/domain/test_events.py
@@ -0,0 +1,387 @@
+"""The stream of activities (docs/automations.md, Contract): the activity shape, the kinds catalog, the event log,
+following the stream once across processes, the timeline read newest first with who may see what, and tidying."""
+
+from __future__ import annotations
+
+import datetime as dt
+import threading
+import time
+
+import pytest
+
+from app.domain import events, match, store
+
+R = store.R
+
+
+@pytest.fixture(autouse=True)
+def kinds():
+ """Test kinds, gone again after each test; nothing this process handled or claimed carries over."""
+ before = dict(events._KINDS)
+ events.kind(
+ "test_thing.happened",
+ label="Thing happened",
+ source="log",
+ as_type="Announce",
+ object_type="Note",
+ fields={"object": {"type": "object", "properties": {"size": {"type": "integer", "title": "Size"}}}},
+ untrusted=("object.name",),
+ )
+ events.kind("test_thing.ticked", label="Thing ticked", source="log", as_type="Update", object_type="Note", noisy=True)
+ events._seen.clear()
+ events._claimed.clear()
+ yield
+ events._KINDS.clear()
+ events._KINDS.update(before)
+ events._seen.clear()
+ events._claimed.clear()
+
+
+def at(seconds=0):
+ """A time as store.now() writes it, `seconds` from now."""
+ return (dt.datetime.now(dt.UTC) + dt.timedelta(seconds=seconds)).isoformat(timespec="seconds")
+
+
+def next_second():
+ start = store.now()
+ while store.now() == start:
+ time.sleep(0.05)
+
+
+def recording(db, sid, title, when, **extra):
+ rid = db.next_id("recording")
+ db.q("CREATE $r CONTENT $d", r=R("recording", rid), d={"space": sid, "title": title, "source": "audio", "created_at": when, **extra})
+ return rid
+
+
+def thing(db, name, space, **kw):
+ return events.emit(
+ db, "test_thing.happened", actor=events.LENS, obj={"type": "Note", "name": name}, space=space, summary=f"Lens noted {name}", **kw
+ )
+
+
+ADMIN = events.viewer(1, None, admin=True)
+
+
+# ---------- the shape ----------
+def test_activities_have_one_shape():
+ sample = events.kind_info("asset.added")["sample"]
+ assert events.check_activity(sample) is sample
+ bad = [
+ ({"id": "Create-7"}, "an activity's id"),
+ ({"type": "Created"}, "type is one of Create"),
+ ({"kind": "email.nope"}, "no kind email.nope"),
+ ({"published": "2026-10-15T08:12:03+00:00"}, "published is an ISO 8601 time in UTC"),
+ ({"actor": {"type": "Entity", "id": "lens:entity/1", "name": "x"}}, "the actor is an object whose type"),
+ ({"actor": {"type": "Person", "id": "account/1", "name": "x"}}, "the actor's id is a Lens reference"),
+ ({"actor": {"type": "Person", "id": "lens:account/1", "name": " "}}, "the actor has a name"),
+ ({"object": {"id": "lens:recording/7"}}, "the object is an object with a type"),
+ ({"tag": [{"id": "lens:entity/1"}]}, "tag is a list of at most 50 objects"),
+ ({"summary": "x" * 301}, "the summary is one line"),
+ ({"space": True}, "space is a namespace id"),
+ ({"lens": {"pos": [1, 2]}}, "lens is an object naming the source"),
+ ({"lens": {**sample["lens"], "untrusted": ["object subject"]}}, "lens.untrusted is a list of dotted paths"),
+ ({"lens": {**sample["lens"], "cause": {"automation": 4}}}, "lens.cause is"),
+ ({"lens": {**sample["lens"], "audience": "owner"}}, "names the owner's account"),
+ ({"lens": {**sample["lens"], "audience": "friends"}}, "lens.audience is one of space, owner, admins"),
+ ]
+ for change, said in bad:
+ with pytest.raises(ValueError, match=said):
+ events.check_activity({**sample, **change})
+
+
+def test_the_catalog_knows_each_kind_its_parents_and_fields():
+ events.kind(
+ "test_mail.received",
+ label="Mail received",
+ source="recordings",
+ as_type="Create",
+ object_type="Document",
+ parent="asset.added",
+ untrusted=("object.name", "object.subject"),
+ fields={
+ "object": {
+ "type": "object",
+ "properties": {
+ "subject": {"type": "string", "title": "Subject"},
+ "from": {"type": "array", "title": "Sender", "items": {"type": "string", "title": "Address"}},
+ },
+ }
+ },
+ )
+ events.kind(
+ "test_mail.flagged", label="Flagged", source="recordings", as_type="Create", object_type="Document", parent="test_mail.received"
+ )
+ assert events.descendants("asset.added") == {"asset.added", "test_mail.received", "test_mail.flagged"}
+ assert events.expand(["test_mail.received"]) == {"test_mail.received", "test_mail.flagged"} and events.expand(None) is None
+ with pytest.raises(ValueError, match="no kind test_nope.x"):
+ events.expand(["test_nope.x"])
+ info = events.kind_info("asset.added")
+ assert info["children"] == ["test_mail.received"] and info["source"] == "recordings" and info["type"] == "Create"
+ assert info["untrusted"] == ["object.name"] and info["sample"]["kind"] == "asset.added"
+ # a child's schema has every activity's fields, its parents' and its own
+ assert events.field_at("test_mail.flagged", "object.subject")["title"] == "Subject"
+ assert events.field_at("test_mail.flagged", "object.name")["title"] == "Title"
+ assert events.field_at("test_mail.flagged", "object.from.0")["title"] == "Address"
+ assert events.field_at("test_mail.flagged", "actor.name")["title"] == "Actor's name"
+ assert events.field_at("test_mail.flagged", "object.nope") is None and events.field_at("test_nope.x", "object") is None
+ assert {"asset.added", "test_thing.happened"} <= {k["name"] for k in events.kinds()}
+ tree = {"all": [{"path": "object.subject", "op": "contains", "value": "trip"}, {"path": "object.weird", "op": "exists"}]}
+ assert match.describe(tree, "test_mail.received") == "the subject contains “trip” and object.weird is set"
+ bad = [
+ (dict(name="Email.Received"), "named like noun.past_verb"),
+ (dict(source=""), "name the source"),
+ (dict(as_type="Sent"), "its type is one of"),
+ (dict(parent="test_nope.x"), "register its parent test_nope.x first"),
+ (dict(fields={"body": {}}), "fields are JSON Schemas"),
+ (dict(untrusted=("object subject",)), "untrusted paths are dotted"),
+ (dict(sample={**info["sample"], "kind": "asset.added"}), "its sample is a asset.added"),
+ (dict(sample={**info["sample"], "kind": "test_bad.added", "summary": ""}), "the summary is one line"),
+ ]
+ for change, said in bad:
+ args = {"name": "test_bad.added", "label": "Bad", "source": "log", "as_type": "Create", "object_type": "Note", **change}
+ with pytest.raises(ValueError, match=said):
+ events.kind(args.pop("name"), **args)
+ assert "test_bad.added" not in events._KINDS # a kind whose sample is wrong isn't left registered
+
+
+# ---------- the event log ----------
+def test_emit_logs_once_per_key_and_carries_its_cause(db, cfg):
+ sid = store.ns_id(db, "pods")
+ a = thing(db, "one", sid, key="hook-17")
+ assert a.startswith("log-") and thing(db, "one again", sid, key="hook-17") == a # a retry never doubles it
+ assert len(db.values("SELECT VALUE id FROM event_log")) == 1
+ with events.caused_by({"automation": 4, "run": 51, "depth": 1}):
+ assert events.cause() == {"automation": 4, "run": 51, "depth": 1}
+ b = thing(db, "two", sid, tag=[{"type": "Mention", "id": "lens:entity/9", "name": "St Mary's"}])
+ assert events.cause() is None
+ got = events.get(db, cfg, b, ADMIN)
+ assert got["kind"] == "test_thing.happened" and got["type"] == "Announce" and got["actor"] == events.LENS
+ assert got["lens"]["cause"] == {"automation": 4, "run": 51, "depth": 1} and got["lens"]["untrusted"] == ["object.name"]
+ assert got["tag"][0]["id"] == "lens:entity/9" and got["summary"] == "Lens noted two" and got["published"].endswith("Z")
+ assert events.check_activity(got) and events.get(db, cfg, a, ADMIN)["lens"]["cause"] is None
+ with pytest.raises(ValueError, match="asset.added is read from its own rows"):
+ events.emit(db, "asset.added", actor=events.LENS, obj={"type": "Audio"}, space=sid, summary="x")
+ with pytest.raises(ValueError, match="no kind test_nope.x"):
+ events.emit(db, "test_nope.x", actor=events.LENS, obj={"type": "Note"}, space=sid, summary="x")
+ with pytest.raises(ValueError, match="a cause is"):
+ with events.caused_by({"automation": 4}):
+ pass
+
+
+# ---------- following ----------
+def test_following_reads_each_source_in_order_once(db, cfg):
+ pods, calls = store.ns_id(db, "pods"), store.ns_id(db, "calls")
+ recording(db, pods, "Before", at())
+ assert events.follow(db, cfg, "test.a")[0] == [] # the first look only marks where it starts
+ next_second()
+ a1, a2 = thing(db, "one", pods), thing(db, "two", calls)
+ late = db.next_id("recording") # given out now, written after the looks below (a slow transaction)
+ now = store.now()
+ r1, r2 = recording(db, pods, "Call one", now), recording(db, calls, "Call two", now) # equal times: the id breaks the tie
+ got, commit = events.follow(db, cfg, "test.a", limit=1) # one at a time from each source
+ assert [a["id"] for a in got] == [a1, f"create-recording-{r1}"]
+ commit()
+ got, commit = events.follow(db, cfg, "test.a", limit=1)
+ assert [a["id"] for a in got] == [a2, f"create-recording-{r2}"]
+ commit()
+ got, commit = events.follow(db, cfg, "test.a")
+ assert got == [] # caught up: the lookback reads them again, but this process handled them already
+ commit()
+ events._seen.clear() # another process: the lookback gives them again, and claims keep them from being done twice
+ again, _ = events.follow(db, cfg, "test.a")
+ assert {a["id"] for a in again} == {a1, a2, f"create-recording-{r1}", f"create-recording-{r2}"}
+ assert [a["id"] for a in again if a["lens"]["source"] == "log"] == [a1, a2] # oldest first, in each source's order
+ # a row written as the last look ran (before the cursor, inside the lookback) is still seen
+ db.q("CREATE $r CONTENT $d", r=R("recording", late), d={"space": pods, "title": "Late", "source": "audio", "created_at": now})
+ got, _ = events.follow(db, cfg, "test.a", kinds=["asset.added"], spaces=[pods])
+ assert f"create-recording-{late}" in {a["id"] for a in got} and {a["space"] for a in got} == {pods}
+ assert all(a["kind"] == "asset.added" for a in got)
+ # a consumer seen for the first time starts at now: nothing before replays, not even what the lookback would see
+ assert events.follow(db, cfg, "test.b")[0] == []
+ got, _ = events.follow(db, cfg, "test.b")
+ assert got == []
+ with pytest.raises(ValueError, match="a consumer is named"):
+ events.follow(db, cfg, "Test B")
+
+
+def test_a_kind_listened_to_later_does_not_replay_a_backlog(db, cfg):
+ pods = store.ns_id(db, "pods")
+ events.follow(db, cfg, "test.c", kinds=["test_thing.happened"])
+ next_second()
+ recording(db, pods, "Unwanted", store.now())
+ got, commit = events.follow(db, cfg, "test.c", kinds=["test_thing.happened"])
+ assert got == [] # recordings make no wanted kind: not read, and their cursor moves to now
+ commit()
+ next_second()
+ events._seen.clear()
+ got, _ = events.follow(db, cfg, "test.c", kinds=["asset.added"])
+ assert got == [] # not the recording from before, not even through the lookback
+ rid = recording(db, pods, "Wanted", at(1))
+ got, commit = events.follow(db, cfg, "test.c", kinds=["asset.added"])
+ assert [a["id"] for a in got] == [f"create-recording-{rid}"]
+ events.idle(db, "test.c") # nothing to do: every cursor at now
+ assert db.one("SELECT pos FROM $c", c=R("event_cursor", "test.c"))["pos"]["recordings"][1] is None
+
+
+def test_a_claim_is_won_once(db):
+ wins = []
+
+ def go():
+ wins.append(events.claim(db, "test.d", "create-recording-7"))
+
+ threads = [threading.Thread(target=go) for _ in range(6)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert sorted(wins) == [False] * 5 + [True]
+ events._claimed.clear() # another process, with nothing claimed in memory
+ assert events.claim(db, "test.d", "create-recording-7") is False
+ assert events.claim(db, "test.other", "create-recording-7") is True # claims are per consumer
+ events._claimed.clear()
+ assert events.unclaimed(db, "test.d", ["create-recording-7", "create-recording-8"]) == ["create-recording-8"]
+
+
+def test_wake_reaches_every_follower_waiting():
+ g = events.generation()
+ events.wake() # a wake during the look still counts
+ assert events.wait(5, since=g) is True
+ woke = []
+ waiters = [threading.Thread(target=lambda: woke.append(events.wait(5))) for _ in range(3)]
+ for t in waiters:
+ t.start()
+ time.sleep(0.2)
+ events.wake()
+ for t in waiters:
+ t.join()
+ assert woke == [True, True, True]
+ stop = threading.Event()
+ stop.set()
+ assert events.wait(5, stop) is False and events.wait(0.1) is False
+
+
+# ---------- the timeline ----------
+@pytest.fixture
+def feed(db):
+ pods, calls = store.ns_id(db, "pods"), store.ns_id(db, "calls")
+ db.q("CREATE $r CONTENT $d", r=R("account", 7), d={"email": "sam@example.org", "name": "Sam"})
+ db.q("CREATE $r CONTENT $d", r=R("storage_source", 3), d={"name": "Office share"})
+ up = recording(db, pods, "Parents' evening", at(-50))
+ db.q("CREATE upload CONTENT $d", d={"recording": up, "account": 7, "email": "sam@example.org"})
+ share = recording(db, pods, "Minutes", at(-40), remote={"source": 3, "path": "minutes.docx"}, source="document")
+ auto = recording(db, calls, "Call with the school", at(-30))
+ db.q("CREATE job CONTENT $d", d={"recording": auto, "created_by": "automation:4", "created_at": at(-30)})
+ ids = {
+ "up": f"create-recording-{up}",
+ "share": f"create-recording-{share}",
+ "auto": f"create-recording-{auto}",
+ "pods": thing(
+ db, "school trip", pods, published=events.stamp(at(-45)), tag=[{"type": "Mention", "id": "lens:entity/9", "name": "St Mary's"}]
+ ),
+ "both": thing(db, "merge", calls, spaces=[pods, calls], published=events.stamp(at(-35))),
+ "admins": thing(db, "backup", None, audience="admins", published=events.stamp(at(-20))),
+ "mine": thing(db, "chat", None, audience="owner", owner=7, published=events.stamp(at(-10))),
+ "nobody": thing(db, "system", None, published=events.stamp(at(-5))),
+ }
+ events.emit(db, "test_thing.ticked", actor=events.LENS, obj={"type": "Note"}, space=pods, summary="tick")
+ return pods, calls, ids
+
+
+def everything(db, cfg, who, flt=None, limit=2):
+ out, before, pages = [], None, 0
+ while True:
+ p = events.page(db, cfg, who, before, flt, limit)
+ out += [a["id"] for a in p["items"]]
+ pages += 1
+ if not p["next"]:
+ return out, pages
+ before = p["next"]
+ assert pages < 50
+
+
+def test_the_timeline_is_newest_first_and_shows_each_person_what_they_may_see(db, cfg, feed):
+ pods, calls, ids = feed
+ newest = ["nobody", "mine", "admins", "auto", "both", "share", "pods", "up"]
+ got, pages = everything(db, cfg, ADMIN)
+ assert got == [ids[k] for k in newest if k != "mine"] and pages >= 4 # an admin sees all but someone's own
+ member = events.viewer(7, {pods})
+ assert everything(db, cfg, member, limit=3)[0] == [ids[k] for k in ["mine", "both", "share", "pods", "up"]]
+ other = events.viewer(8, {calls})
+ assert everything(db, cfg, other, limit=50)[0] == [ids["auto"], ids["both"]]
+ assert events.page(db, cfg, events.viewer(9, set()))["items"] == []
+ # one at a time comes to the same, without repeats
+ assert everything(db, cfg, ADMIN, limit=1)[0] == got
+ assert events.get(db, cfg, ids["auto"], other)["object"]["name"] == "Call with the school"
+ assert events.get(db, cfg, ids["auto"], member) is None and events.get(db, cfg, ids["admins"], member) is None
+ assert events.get(db, cfg, "create-recording-999", ADMIN) is None and events.get(db, cfg, "Bad id", ADMIN) is None
+ with pytest.raises(ValueError, match="before is the `next` of an earlier page"):
+ events.page(db, cfg, ADMIN, "nope")
+
+
+def test_recordings_are_asset_added_by_whoever_added_them(db, cfg, feed):
+ _, _, ids = feed
+ up, share, auto = (events.get(db, cfg, ids[k], ADMIN) for k in ("up", "share", "auto"))
+ assert up["actor"] == {"type": "Person", "id": "lens:account/7", "name": "Sam"} and up["summary"] == "Sam added “Parents' evening”"
+ assert share["actor"] == {"type": "Service", "id": "lens:source/3", "name": "Office share"}
+ assert share["object"]["type"] == "Document" and share["object"]["media"] == "document"
+ assert auto["actor"]["id"] == "lens:automation/4" and auto["actor"]["type"] == "Application"
+ assert up["object"] == {
+ "type": "Audio",
+ "id": "lens:recording/" + ids["up"].split("-")[-1],
+ "name": "Parents' evening",
+ "url": "/resources/" + ids["up"].split("-")[-1],
+ "media": "audio",
+ }
+ assert up["type"] == "Create" and up["kind"] == "asset.added" and up["lens"]["untrusted"] == ["object.name"]
+ for a in (up, share, auto):
+ events.check_activity(a)
+ plain = recording(db, store.ns_id(db, "pods"), "Scanned", at(-1))
+ assert events.get(db, cfg, f"create-recording-{plain}", ADMIN)["actor"] == events.LENS
+
+
+def test_timeline_filters(db, cfg, feed):
+ pods, calls, ids = feed
+
+ def ids_of(flt, who=ADMIN, limit=50):
+ return everything(db, cfg, who, flt, limit)[0]
+
+ assert ids_of({"kinds": ["asset.added"]}) == [ids["auto"], ids["share"], ids["up"]]
+ assert len(ids_of({"kinds": ["test_thing.ticked"]})) == 1 # a noisy kind only when named
+ assert ids_of({"actor": "lens:automation/4"}) == [ids["auto"]]
+ assert ids_of({"spaces": [pods]}) == [ids["both"], ids["share"], ids["pods"], ids["up"]]
+ assert ids_of({"tag": "st mary's"}) == [ids["pods"]] and ids_of({"tag": "lens:entity/9"}) == [ids["pods"]]
+ assert ids_of({"q": "school"}) == [ids["auto"], ids["pods"]]
+ assert ids_of({"where": {"all": [{"path": "object.media", "op": "equals", "value": "document"}]}}) == [ids["share"]]
+ a = events.get(db, cfg, ids["share"], ADMIN)["published"]
+ b = events.get(db, cfg, ids["auto"], ADMIN)["published"]
+ assert ids_of({"since": a, "until": b}) == [ids["auto"], ids["both"]] # after since, up to and including until
+ # filters that thin pages out read on, so a page isn't short while there is more
+ assert ids_of({"q": "evening"}, limit=1) == [ids["up"]]
+ bad = [
+ ({"colour": "red"}, "no filter colour"),
+ ({"kinds": ["test_nope.x"]}, "no kind test_nope.x"),
+ ({"actor": "sam"}, "actor is a Lens reference"),
+ ({"spaces": "pods"}, "spaces is a list of namespace ids"),
+ ({"since": "last week"}, "since is a time"),
+ ({"where": {"any": []}}, "where: any needs at least one condition"),
+ ]
+ for flt, said in bad:
+ with pytest.raises(ValueError, match=said):
+ events.page(db, cfg, ADMIN, flt=flt)
+
+
+# ---------- keeping it small ----------
+def test_tidy_drops_old_log_rows_and_claims(db, cfg):
+ sid = store.ns_id(db, "pods")
+ keep = thing(db, "recent", sid, key="k-recent")
+ old = db.next_id("event_log")
+ db.q("CREATE $r CONTENT $d", r=R("event_log", old), d={"kind": "test_thing.happened", "logged_at": at(-31 * 86400)})
+ db.q("CREATE $r CONTENT $d", r=R("event_log_key", "old"), d={"log": old, "at": at(-31 * 86400)})
+ events.claim(db, "test.e", "new")
+ db.q("CREATE $r CONTENT $d", r=R("event_claim", "old"), d={"consumer": "test.e", "key": "old", "at": at(-2 * 86400)})
+ assert events.tidy(db, cfg, now=at())
+ assert db.values("SELECT VALUE record::id(id) FROM event_log") == [int(keep.split("-")[1])]
+ assert len(db.values("SELECT VALUE id FROM event_log_key")) == 1
+ assert db.values("SELECT VALUE key FROM event_claim") == ["new"]
+ assert events.tidy(db, {**cfg, "events": {"keep_days": 1}}, now=at(2 * 86400)) and db.values("SELECT VALUE id FROM event_log") == []