diff --git a/docs/automations.md b/docs/automations.md index 79f1c3be2..999d25e67 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -10,7 +10,10 @@ merged is called built. | Part | Status | | --- | --- | | Match trees: one condition language for triggers, an automation's If and workflow nodes | built | -| Activities, the kinds catalog, the event log, following the stream, the timeline API | planned | +| Activities, the kinds catalog, the event log, following the stream, timeline reads (`app/domain/events.py`) | built | +| New assets as `asset.added` (the `recordings` source) | built | +| More kinds: emails, files, calendar events, jobs, deletions, comments, notes, graph changes, sensors, chat | planned | +| The timeline API, MCP and assistant tools | planned | | Automations, triggers, joins, schedules, actions, approvals, variables, the timeline page, the studio | planned | ## Contract @@ -18,6 +21,120 @@ merged is called built. Every part of the studio builds on the shapes below. A change to one of them comes in a small pull request of its own that changes the code, this section and the tests together. +### Activities + +Built (`app/domain/events.py`). Everything that happens in Lens, from every source, is an activity: a plain dict in +the shape of a [W3C Activity Streams 2.0](https://www.w3.org/TR/activitystreams-core/) activity, with a Lens `kind` +that triggers subscribe to. `events.check_activity(a)` checks it, naming the first problem. + +```json +{"id": "create-recording-7", "type": "Create", "kind": "asset.added", "published": "2026-10-15T08:12:03.000Z", + "actor": {"type": "Person", "id": "lens:account/1", "name": "Sam"}, + "object": {"type": "Audio", "id": "lens:recording/7", "name": "Call with the school", "url": "/resources/7", + "media": "audio"}, + "target": null, "tag": [], "summary": "Sam added “Call with the school”", "space": 1, + "lens": {"source": "recordings", "pos": ["2026-10-15T08:12:03+00:00", 7], "untrusted": ["object.name"], + "cause": null}} +``` + +- `id`: stable, made from what the source reads (`create-recording-7`, `log-12`), so an activity is the same every + time it is read: 1 to 120 lowercase letters, digits and `_ . : -`. +- `type`: the AS2 verb (Create, Update, Delete, Add, Remove, Join, Leave, Arrive, Announce, Like, Accept, Reject), + which the kind decides. +- `published`: when it happened, ISO 8601 in UTC with milliseconds, so times from every source sort as text. +- `actor`: who did it, `{type, id, name}`. Sources and storage are `Service`, people `Person`, Lens itself, the agent + and automations `Application` (`lens:automation/4`; Lens is `lens:app`). Entities are never actors; they go in + `tag`. +- `object`: what it is about, with the kind's own fields. Ids are `lens:/` references, so actions can act + on them; `events.ref_id(ref, "recording")` reads the id back. +- `target`: where to, when the verb has one (Add to a collection), else null. `tag`: up to 50 objects such as + `{"type": "Mention", "id": "lens:entity/9", "name": "St Mary's"}`. +- `summary`: one line of at most 300 characters, made by fixed rules, never by a model. +- `space`: the namespace id, or null for activities of no namespace. +- `lens`: Lens's own fields. `source` and `pos` (the source's position, `[time, id]`) are for the stream machinery. + `untrusted` lists the dotted paths holding outside text (titles, email bodies, webhook payloads): prompts wrap + them with `untrusted.wrap`. `cause` is `{automation, run, depth}` when an automation's action caused it. `spaces` + lists every namespace when there are several. `audience` narrows who may see it: `space` (the default: people who + read one of its namespaces), `owner` (only the account in `lens.owner`) or `admins`. + +### Kinds + +Built. `events.kind(name, *, label, source, as_type, object_type, parent=None, fields=None, untrusted=(), +sample=None, noisy=False)` registers a kind: a name like `noun.past_verb`, a label for people, the source that makes +it, its AS2 verb and object type, a parent (a trigger on the parent matches it too), JSON Schemas for its `object` +(and `actor` or `target`) in `fields`, the paths that hold outside text, a realistic sample (checked when it is +registered), and whether it is `noisy` (left out of the timeline unless named). `events.kinds()` lists them, +`events.kind_info(name)` gives one with its children and its whole `schema` (every activity's fields, its parents', +then its own), `events.descendants(name)` the kind and every kind under it, and `events.field_at(kind, path)` the +schema of one field, whose `title` names it in words (`match.describe(tree, kind)` says "the title contains “call”"). + +| Kind | Parent | Source | Object | +| --- | --- | --- | --- | +| `asset.added` | | `recordings` | `{type: Audio, Video, Image or Document, id: lens:recording/, name: its title, url: /resources/, media: audio, video, image, document or transcript, content_type}` | + +### Sources and the event log + +Built. A source (`events.Source`, added with `events.register_source`) reads rows Lens already keeps as activities, +so nothing is stored twice: + +- `read(db, cfg, pos, spaces, limit, hint)`: activities strictly after `pos` (`[time, id]`, or `[time, null]` for + after every row at that time), oldest first, at most `limit`, of the namespaces in `spaces` (null: all). `hint` + is narrowing a source may use. +- `page(db, cfg, before, viewer, flt, limit)`: newest first, strictly before `before` (`key` of the last one shown), + with what the viewer may see and the filters it can apply cheaply (the stream checks every filter again). +- `get(db, cfg, activity_id)`: one activity by its id, or null; `now(db)`, `back(db, seconds)`: positions of now and of + `seconds` ago; `key(a)`: the position `page` orders by. + +A kind belongs to the source named in its `source`, and a source may list more in `kinds`. Two are built: + +- `recordings` (`app/domain/events_content.py`) reads `recording` by `created_at`, the id breaking ties, as + `asset.added`. The actor is whoever added it: an automation (its first job was queued by `automation:`), the + storage source a watched file came from (`Service`, `lens:source/`), the person whose upload it was (`Person`, + `lens:account/`), or Lens (a folder scan). +- `log` reads `event_log`: activities no row of their own records (a deletion, a webhook call), written by + `events.emit(db, kind, *, actor, obj, space, summary, target=None, tag=None, key=None, published=None, spaces=None, + audience=None, owner=None, untrusted=None)`, which gives the id (`log-`). Emitting a `key` again changes + nothing and gives the first id, so a retry never doubles an activity. Inside `with events.caused_by({"automation": + 4, "run": 51, "depth": 1}):` what is emitted carries that cause. It is read by `(logged_at, id)` and paged by + `published`, and its rows are kept for `events.keep_days` (30). + +### Following the stream + +Built. The API and `lens worker` are separate processes on one database, so following works the way notifications +do: `events.follow(db, cfg, consumer, kinds=None, spaces=None, limit=500, hint=None)` gives `(activities, commit)`, +the activities since the consumer's last commit, oldest first. Handle them in order, then call `commit()` to save +where the next look starts (`event_cursor:`, a position per source). + +- A consumer seen for the first time starts at now: nothing that happened before replays, not even through the + lookback. What happens in the second it starts in isn't seen. +- Only `kinds` (and their descendants; null: every kind) in `spaces` (null: every namespace and none). A source that + makes none of the kinds wanted isn't read, and its cursor moves to now, so a kind listened to later doesn't replay + a backlog. `events.idle(db, consumer)` moves every cursor to now. +- A source with more waiting gives `limit` at a time. Once caught up, a look also reads the last 60 seconds again, + for a row written as the last look ran, minus what this process already handled. So claim each activity before + acting on it: `events.claim(db, consumer, key)` is true for the one process, and the one time, that claims `key` + (an activity's id, say), with a `CREATE` of `event_claim:` only one process wins. `events.unclaimed(db, + consumer, keys)` lists the keys nobody has claimed yet, in one query. Claims go after a day. +- `events.emit` wakes the followers in its process; a follower waits between looks with `g = events.generation()` + before the look and `events.wait(events.poll_seconds(cfg), stop, since=g)` after it (`events.poll_seconds`, 5). + +### Timeline reads + +Built. `events.page(db, cfg, viewer, before=None, flt=None, limit=50)` reads the stream newest first for a viewer +(`events.viewer(account, spaces, admin)`, spaces null for every one): `{"items": [...], "next": ...}`, where `next` +(an opaque cursor, null at the end) goes in `before` for the page after. It merges every source's `page`, applies +who may see what, and when filters thin a page out it reads on, up to five pages from each source. +`events.get(db, cfg, id, viewer)` reads one activity, null when there is none or it isn't the viewer's to see. + +- Who sees what: `lens.audience` `owner` is its owner alone; `admins` is admins; `space` is everyone who reads one of + its namespaces (`space` or `lens.spaces`); admins read every namespace, and an activity of no namespace is for + admins. People who were given only some collections of a namespace don't see its activities yet. +- Filters: `kinds` (with their descendants; a noisy kind only when named), `actor` (an actor's id, or several), + `spaces` (namespace ids), `tag` (an entity's id or name), `since` (published after) and `until` (at or before), + `q` (words in the summary) and `where` (a match tree on the activity, below). +- `events.tidy(db, cfg)` drops event log rows past `events.keep_days` and claims past a day; the routine scheduler + runs it once an hour. + ### Match trees Built (`app/domain/match.py`). One condition language for a trigger's `where`, an automation's `if`, and the @@ -52,3 +169,7 @@ tree, `path` (when given) is the part of the value the tree tests. Graphs saved - More operators: `in`, `matches` (bounded regular expressions), `between`, times against now, `changed`. - The canvas shows and edits match trees on condition, filter and switch nodes (today it edits one test; a tree saved through the API is kept). +- A `?format=as2` JSON-LD view and per-actor outboxes for ActivityPub; SurrealDB `DEFINE EVENT` for Update verbs that + no row records; pushing instead of polling. +- People given only some collections of a namespace see that namespace's activities about those collections. +- Name an automation actor by its automation's name once automations are stored. diff --git a/docs/database.md b/docs/database.md index 7bc0c15d9..4382d9bd5 100644 --- a/docs/database.md +++ b/docs/database.md @@ -68,6 +68,13 @@ cosine; servers only) defined when the first vector is stored, since its `DIMENS which model and dimension that is. A server searches it with `embedding <|k,ef|> $vec AND …filters`, which 3.2.4 applies inside the index scan (a filtered search still returns k rows). +The stream of activities automations follow ([Automations](automations.md#contract)) is mostly a view over rows +kept anyway, such as `recording.created_at`. What no row records goes in `event_log:` (by `logged_at` and +`published`, kept for `events.keep_days`), with the keys it was emitted with in `event_log_key:`. Each consumer +of the stream keeps where its next look starts in `event_cursor:` (a position per source), and claims what +it handles in `event_claim:`; a claim is a `CREATE` only one process wins, and claims go +after a day. + ## Upgrades Installs keep their data across updates (running `install.sh` again, a new image, `git pull`). New tables, fields and diff --git a/fastapi_backend/app/domain/events.py b/fastapi_backend/app/domain/events.py new file mode 100644 index 000000000..44283cc69 --- /dev/null +++ b/fastapi_backend/app/domain/events.py @@ -0,0 +1,1091 @@ +"""The stream of activities (docs/automations.md, Contract): what happens in Lens, from every source, as time-coded +activities in the W3C Activity Streams 2.0 shape, for automations to follow and the timeline to show. + +An activity is a plain dict (`check_activity` says what it holds): an AS2 verb (`type`), a Lens `kind` that triggers +subscribe to (`email.received`), when it happened, who did it (`actor`), what it is about (`object`), a one-line +summary made by fixed rules, its namespace, and Lens's own fields under `lens`. + +Most activities aren't stored twice. A source (`Source`, `register_source`) reads rows Lens already keeps, such as +recordings, and makes activities of them, with ids built from what it reads (`create-recording-7`), so they are the +same every time they're read. What has no row of its own (a deletion, a webhook call) is written to a small log, +`event_log`, by `emit`; the `log` source reads it back. Kinds (`kind`) are registered in a catalog with the source +that makes them, the fields of their objects, which of those hold outside text, and a sample. + +Following the stream works across processes the way notify.py does it, since the API and `lens worker` are separate +processes on one database: each consumer has a cursor (`event_cursor:`, a position per source), reads past +it in (time, id) order, and claims what it handles (`claim`: a CREATE only one process wins). A consumer seen for the +first time starts at now, so nothing that happened before replays. Once caught up, a read also reads the last +LOOKBACK seconds again, for a row written as the last look ran; claims keep those from being handled twice. A source +that can't make a kind the consumer wants isn't read, and its cursor moves to now, so a kind someone starts listening +to later doesn't replay a backlog. + +The timeline reads the stream newest first (`page`), merging the sources, applying who may see what (`lens.audience` +and namespaces), and reading on while filters thin a page out. Nothing here runs on its own: until something follows +the stream, an install behaves as before. `tidy` drops old log rows and claims. +""" + +from __future__ import annotations + +import base64 +import contextlib +import contextvars +import datetime as dt +import hashlib +import json +import re +import threading +import time +from dataclasses import dataclass + +from . import store + +R = store.R +ID_RX = re.compile(r"^[a-z0-9_.:-]{1,120}$") +KIND_RX = re.compile(r"^[a-z][a-z0-9_]{0,40}(\.[a-z][a-z0-9_]{0,40}){1,3}$") +NAME_RX = re.compile(r"^[a-z][a-z0-9_.:-]{0,79}$") +REF_RX = re.compile(r"^lens:[a-z][a-z0-9_]{0,40}(/[A-Za-z0-9_.:@+-]{1,200})?$") +PATH_RX = re.compile(r"^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*$") +TYPES = ("Create", "Update", "Delete", "Add", "Remove", "Join", "Leave", "Arrive", "Announce", "Like", "Accept", "Reject") +ACTOR_TYPES = ("Service", "Person", "Application") +AUDIENCES = ("space", "owner", "admins") +LENS = {"type": "Application", "id": "lens:app", "name": "Lens"} +LOOKBACK = 60 # seconds a caught-up read goes back over, for rows written as the last look ran +PAGE = 500 # rows a follow reads per source +PAGE_ROUNDS = 5 # source pages a timeline page reads at most, per source, when filters thin it out +SUMMARY_MAX = 300 +MAX_TAGS = 50 +FILTERS = ("kinds", "actor", "spaces", "tag", "since", "until", "q", "where") +SWEEP_SECONDS = 3600 +CLAIM_DAYS = 1 + +_swept = {"at": 0.0} +_cause: contextvars.ContextVar[dict | None] = contextvars.ContextVar("lens_event_cause", default=None) +_wakes = threading.Condition() +_generation = [0] +_seen: dict[str, dict[str, None]] = {} # per consumer: activities this process handled lately (not read again) +_claimed: dict[str, None] = {} # claims this process knows are taken, so it doesn't ask the database again +_lock = threading.Lock() + + +# ---------- times ---------- +def _parse(t): + """An aware UTC datetime from an ISO 8601 string (naive is UTC) or a datetime.""" + if isinstance(t, dt.datetime): + d = t + else: + s = str(t).strip() + d = dt.datetime.fromisoformat(s[:-1] + "+00:00" if s.endswith("Z") else s) + return (d if d.tzinfo else d.replace(tzinfo=dt.timezone.utc)).astimezone(dt.timezone.utc) + + +def stamp(t=None): + """`published` as activities have it: ISO 8601 in UTC with milliseconds (2026-10-15T16:21:58.000Z), so stamps + from every source sort as text.""" + d = _parse(t) if t is not None else dt.datetime.now(dt.timezone.utc) + return d.strftime("%Y-%m-%dT%H:%M:%S.") + f"{d.microsecond // 1000:03d}Z" + + +def ago(seconds, now=None): + """The time `seconds` ago, as store.now() writes times (to the second, +00:00).""" + base = _parse(now) if now is not None else dt.datetime.now(dt.timezone.utc) + return (base - dt.timedelta(seconds=seconds)).isoformat(timespec="seconds") + + +def _tkey(t): + if isinstance(t, str): + try: + return _parse(t).timestamp() + except ValueError: + return t + return t + + +def _ikey(i): + """A row id, for ordering: numbers before text, each in its own order.""" + return (0, i, "") if isinstance(i, (int, float)) and not isinstance(i, bool) else (1, 0, str(i)) + + +def pos_key(pos): + """A cursor position [time, id] for ordering. An id of None means every row at that time was read, so it comes + after every [time, id] at the same time.""" + t, i = (list(pos) + [None, None])[:2] + return (_tkey(t), 1, (0, 0, "")) if i is None else (_tkey(t), 0, _ikey(i)) + + +# ---------- the shape ---------- +def _ref(v, what): + if not isinstance(v, str) or not REF_RX.match(v): + raise ValueError(f"{what} is a Lens reference like lens:recording/7") + + +def check_activity(a): + """The activity, if it has the shape every part of Lens reads (docs/automations.md, Activity): ValueError naming + the first problem otherwise.""" + if not isinstance(a, dict): + raise ValueError("an activity is an object") + if not isinstance(a.get("id"), str) or not ID_RX.match(a["id"]): + raise ValueError("an activity's id is 1 to 120 lowercase letters, digits and _ . : -") + if a.get("type") not in TYPES: + raise ValueError(f"an activity's type is one of {', '.join(TYPES)}") + if a.get("kind") not in _KINDS: + raise ValueError(f"no kind {a.get('kind')}; register it with events.kind first") + try: + if not isinstance(a.get("published"), str) or not a["published"].endswith("Z"): + raise ValueError + _parse(a["published"]) + except ValueError: + raise ValueError("published is an ISO 8601 time in UTC, like 2026-10-15T16:21:58.000Z") from None + actor = a.get("actor") + if not isinstance(actor, dict) or actor.get("type") not in ACTOR_TYPES: + raise ValueError(f"the actor is an object whose type is {', '.join(ACTOR_TYPES)}") + _ref(actor.get("id"), "the actor's id") + if not isinstance(actor.get("name"), str) or not actor["name"].strip(): + raise ValueError("the actor has a name") + obj = a.get("object") + if not isinstance(obj, dict) or not isinstance(obj.get("type"), str) or not obj["type"]: + raise ValueError("the object is an object with a type") + if obj.get("id") is not None: + _ref(obj["id"], "the object's id") + if a.get("target") is not None and not isinstance(a["target"], dict): + raise ValueError("the target is an object, or null") + tags = a.get("tag") or [] + if not isinstance(tags, list) or len(tags) > MAX_TAGS or not all(isinstance(t, dict) and t.get("type") for t in tags): + raise ValueError(f"tag is a list of at most {MAX_TAGS} objects, each with a type") + if not isinstance(a.get("summary"), str) or not a["summary"].strip() or len(a["summary"]) > SUMMARY_MAX: + raise ValueError(f"the summary is one line of at most {SUMMARY_MAX} characters") + if a.get("space") is not None and (not isinstance(a["space"], int) or isinstance(a["space"], bool)): + raise ValueError("space is a namespace id, or null") + lens = a.get("lens") + if not isinstance(lens, dict) or not isinstance(lens.get("source"), str): + raise ValueError("lens is an object naming the source that made the activity") + if not isinstance(lens.get("pos"), list) or len(lens["pos"]) != 2: + raise ValueError("lens.pos is the source's position, [time, id]") + paths = lens.get("untrusted") or [] + if not isinstance(paths, list) or not all(isinstance(p, str) and PATH_RX.match(p) for p in paths): + raise ValueError("lens.untrusted is a list of dotted paths, like object.subject") + cause = lens.get("cause") + if cause is not None and ( + not isinstance(cause, dict) or not all(isinstance(cause.get(k), int) for k in ("automation", "run", "depth")) + ): + raise ValueError("lens.cause is {automation, run, depth}, or null") + spaces = lens.get("spaces") + if spaces is not None and (not isinstance(spaces, list) or not all(isinstance(s, int) for s in spaces)): + raise ValueError("lens.spaces is a list of namespace ids") + if lens.get("audience", "space") not in AUDIENCES: + raise ValueError(f"lens.audience is one of {', '.join(AUDIENCES)}") + if lens.get("audience") == "owner" and not isinstance(lens.get("owner"), int): + raise ValueError("an activity only its owner sees names the owner's account (lens.owner)") + return a + + +def build( + aid, + kind, + *, + actor, + obj, + summary, + space, + source, + pos, + published, + target=None, + tag=None, + untrusted=None, + cause=None, + spaces=None, + audience=None, + owner=None, +): + """An activity in the shape check_activity wants, for sources to build theirs with. `untrusted` defaults to the + kind's outside-text paths; `type` is the kind's AS2 verb.""" + k = _KINDS[kind] + lens = {"source": source, "pos": list(pos), "untrusted": list(k.untrusted if untrusted is None else untrusted), "cause": cause} + if spaces: + lens["spaces"] = sorted(set(spaces)) + if audience and audience != "space": + lens["audience"] = audience + if owner is not None: + lens["owner"] = owner + return { + "id": aid, + "type": k.as_type, + "kind": kind, + "published": stamp(published), + "actor": actor, + "object": obj, + "target": target, + "tag": list(tag or []), + "summary": " ".join(str(summary).split())[:SUMMARY_MAX], + "space": space, + "lens": lens, + } + + +def ref_id(ref, table): + """The id in a `lens:
/` reference (an object's id, say), a number where it is one; None for a reference + to another table.""" + m = re.match(rf"^lens:{re.escape(table)}/(.+)$", ref if isinstance(ref, str) else "") + if not m: + return None + return int(m.group(1)) if m.group(1).isdigit() else m.group(1) + + +# ---------- the catalog of kinds ---------- +@dataclass(frozen=True) +class Kind: + name: str + label: str + source: str + as_type: str + object_type: str + parent: str | None + fields: dict + untrusted: tuple[str, ...] + sample: dict | None + noisy: bool + + def info(self): + return { + "name": self.name, + "label": self.label, + "source": self.source, + "type": self.as_type, + "object_type": self.object_type, + "parent": self.parent, + "children": sorted(k for k, v in _KINDS.items() if v.parent == self.name), + "fields": self.fields, + "schema": schema(self.name), + "untrusted": list(self.untrusted), + "noisy": self.noisy, + "sample": self.sample, + } + + +_KINDS: dict[str, Kind] = {} +BASE_FIELDS = { + "actor": { + "type": "object", + "title": "Who", + "properties": { + "type": {"type": "string", "title": "Kind of actor", "enum": list(ACTOR_TYPES)}, + "id": {"type": "string", "title": "Actor"}, + "name": {"type": "string", "title": "Actor's name"}, + }, + }, + "object": { + "type": "object", + "title": "What", + "properties": { + "type": {"type": "string", "title": "Object type"}, + "id": {"type": "string", "title": "Object"}, + "name": {"type": "string", "title": "Name"}, + "url": {"type": "string", "title": "Link"}, + }, + }, +} + + +def kind(name, *, label, source, as_type, object_type, parent=None, fields=None, untrusted=(), sample=None, noisy=False): + """Register a kind of activity: its name (`.`), a label for people, the source that makes it, its + AS2 verb and object type, a parent kind (a trigger on the parent matches it too), JSON Schemas for its `object` + (and `actor` or `target`) in `fields`, the dotted paths that hold outside text, a realistic sample, and whether + it is `noisy` (left out of the timeline unless asked for). Registering a name again replaces it.""" + if not KIND_RX.match(name or ""): + raise ValueError(f"a kind is named like noun.past_verb (email.received), not {name}") + if not NAME_RX.match(source or ""): + raise ValueError(f"kind {name}: name the source that makes it") + if as_type not in TYPES: + raise ValueError(f"kind {name}: its type is one of {', '.join(TYPES)}") + if parent is not None and (parent not in _KINDS or parent == name): + raise ValueError(f"kind {name}: register its parent {parent} first") + fields = dict(fields or {}) + if set(fields) - {"object", "actor", "target"} or not all(isinstance(v, dict) for v in fields.values()): + raise ValueError(f"kind {name}: fields are JSON Schemas for object, actor or target") + if not all(isinstance(p, str) and PATH_RX.match(p) for p in untrusted): + raise ValueError(f"kind {name}: untrusted paths are dotted, like object.subject") + old = _KINDS.get(name) + _KINDS[name] = Kind(name, label, source, as_type, object_type, parent, fields, tuple(untrusted), sample, bool(noisy)) + if sample is not None: + try: + check_activity(sample) + if sample["kind"] != name: + raise ValueError(f"its sample is a {sample['kind']}") + except ValueError as e: + if old: + _KINDS[name] = old + else: + del _KINDS[name] + raise ValueError(f"kind {name}: {e}") from None + return _KINDS[name] + + +def kinds(): + """Every kind, as the API and agents list them.""" + return [k.info() for k in _KINDS.values()] + + +def kind_info(name): + """One kind; KeyError when there's none.""" + return _KINDS[name].info() + + +def descendants(name): + """The kind and every kind under it (a trigger on asset.added matches email.received).""" + out, todo = set(), [name] + while todo: + k = todo.pop() + if k not in out: + out.add(k) + todo += [c for c, v in _KINDS.items() if v.parent == k] + return out + + +def expand(names): + """The kinds named and their descendants; None (every kind) stays None. ValueError for a kind that isn't there.""" + if names is None: + return None + unknown = [n for n in names if n not in _KINDS] + if unknown: + raise ValueError(f"no kind {unknown[0]}; GET /api/v1/activities/kinds lists them") + return set().union(*(descendants(n) for n in names)) if names else set() + + +def _merged(base, extra): + if not extra: + return base + out = {**base, **{k: v for k, v in extra.items() if k != "properties"}} + out["properties"] = {**(base.get("properties") or {}), **(extra.get("properties") or {})} + return out + + +def schema(name): + """A JSON Schema of the kind's activities: every activity's own fields, its parents' fields, then its own.""" + chain, k = [], _KINDS.get(name) + while k: + chain.insert(0, k) + k = _KINDS.get(k.parent) if k.parent else None + parts = {p: dict(v) for p, v in BASE_FIELDS.items()} + for k in chain: + for p, v in k.fields.items(): + parts[p] = _merged(parts.get(p) or {"type": "object"}, v) + return { + "type": "object", + "properties": { + "id": {"type": "string", "title": "Activity"}, + "kind": {"type": "string", "title": "Kind"}, + "published": {"type": "string", "format": "date-time", "title": "When"}, + "summary": {"type": "string", "title": "Summary"}, + "space": {"type": ["integer", "null"], "title": "Namespace"}, + "tag": {"type": "array", "title": "Tags", "items": {"type": "object"}}, + **parts, + }, + } + + +def field_at(name, path): + """The schema of the field at a dotted path in a kind's activities (list items by number), or None.""" + node = schema(name) if name in _KINDS else None + for part in (path or "").split("."): + if not isinstance(node, dict): + return None + props = node.get("properties") + if props and part in props: + node = props[part] + elif node.get("type") == "array" and part.isdigit(): + node = node.get("items") + else: + return None + return node if isinstance(node, dict) else None + + +# Assets: the kind every new recording, document, image, email or web page is, and the parent of M2's kinds for each +# (email.received, file.added, ...). The recordings source makes it (events_content.py). +kind( + "asset.added", + label="Asset added", + source="recordings", + as_type="Create", + object_type="Document", + fields={ + "object": { + "type": "object", + "properties": { + "type": {"type": "string", "title": "Object type", "enum": ["Audio", "Video", "Image", "Document"]}, + "id": {"type": "string", "title": "Asset"}, + "name": {"type": "string", "title": "Title"}, + "url": {"type": "string", "title": "Link"}, + "media": {"type": "string", "title": "Media", "enum": ["audio", "video", "image", "document", "transcript"]}, + "content_type": {"type": "string", "title": "Content type"}, + }, + } + }, + untrusted=("object.name",), + sample={ + "id": "create-recording-7", + "type": "Create", + "kind": "asset.added", + "published": "2026-10-15T08:12:03.000Z", + "actor": {"type": "Person", "id": "lens:account/1", "name": "Sam"}, + "object": {"type": "Audio", "id": "lens:recording/7", "name": "Call with the school", "url": "/resources/7", "media": "audio"}, + "target": None, + "tag": [], + "summary": "Sam added “Call with the school”", + "space": 1, + "lens": {"source": "recordings", "pos": ["2026-10-15T08:12:03+00:00", 7], "untrusted": ["object.name"], "cause": None}, + }, +) + + +# ---------- sources ---------- +class Source: + """One family of rows read as activities. A source names itself, may list the kinds it makes (the catalog's kinds + whose `source` is its name count too), and reads: + + - `read(db, cfg, pos, spaces, limit, hint)`: activities strictly after `pos` ([time, id], or [time, None] for + after every row at `time`), oldest first, at most `limit`; only namespaces in `spaces` unless it is None. Each + activity's `lens.pos` is its row's position. `hint` is narrowing it may use, like {"actors": [...]}. + - `page(db, cfg, before, viewer, flt, limit)`: newest first, strictly before `before` (`key` of the last one + shown) or from the newest, with what `viewer` may see and the filters it can apply cheaply; the core checks + every filter again. + - `get(db, cfg, activity_id)`: one activity by its id, or None (also for ids that aren't its own). + - `now(db)` and `back(db, seconds)`: the position of now, and of `seconds` ago, in its own terms. + - `key(a)`: the position `page` orders by, [time, id]; its `lens.pos` unless it pages by another column. + """ + + name = "" + kinds: tuple[str, ...] = () + + def read(self, db, cfg, pos, spaces, limit, hint=None): + raise NotImplementedError + + def page(self, db, cfg, before, viewer, flt, limit): + raise NotImplementedError + + def get(self, db, cfg, activity_id): + return None + + def now(self, db): + return [store.now(), None] + + def back(self, db, seconds): + return [ago(seconds), None] + + def key(self, a): + return a["lens"]["pos"] + + +_SOURCES: dict[str, Source] = {} + + +def register_source(src): + """Add a source to the stream (registering a name again replaces it).""" + if not NAME_RX.match(getattr(src, "name", "") or ""): + raise ValueError("a source has a name of lowercase letters, digits and _ . : -") + _SOURCES[src.name] = src + return src + + +def sources(): + return list(_SOURCES.values()) + + +def produces(src): + """The kinds a source can make: those it lists and the catalog's kinds that name it.""" + return set(src.kinds) | {k for k, v in _KINDS.items() if v.source == src.name} + + +def after_sql(col, pos, params, op=">", table=None): + """The SQL for "after `pos`" ([time, id], id None for after every row at time) on a table read by (`col`, id), + with its parameters added to `params`; op "<" for "before" when paging newest first. With `table`, the id is a + key of that table's records, so it compares as SurrealDB orders record ids.""" + params["pt"] = pos[0] + if pos[1] is None: + return f"{col} {op} $pt" + params["pi"] = R(table, pos[1]) if table else pos[1] + return f"({col} {op} $pt OR ({col} = $pt AND id {op} $pi))" + + +# ---------- who sees what ---------- +def viewer(account=None, spaces=None, admin=False): + """Who reads the timeline: their account id, the namespaces they read (None: every one, for admins) and whether + they are an admin.""" + return {"account": account, "spaces": None if spaces is None else set(spaces), "admin": bool(admin)} + + +def spaces_of(a): + """Every namespace an activity belongs to.""" + out = [a.get("space")] + list((a.get("lens") or {}).get("spaces") or []) + return [s for s in dict.fromkeys(out) if s is not None] + + +def visible(a, who): + """Whether `who` (viewer) may see an activity: `lens.audience` owner is its owner alone, admins is admins, and + space (the default) is everyone who reads one of its namespaces (admins read all; one with no namespace is for + admins).""" + lens = a.get("lens") or {} + aud = lens.get("audience") or "space" + if aud == "owner": + return who.get("account") is not None and lens.get("owner") == who["account"] + if aud == "admins" or who.get("admin"): + return bool(who.get("admin")) + readable = who.get("spaces") or set() + return any(s in readable for s in spaces_of(a)) + + +# ---------- the event log ---------- +@contextlib.contextmanager +def caused_by(run): + """Activities emitted inside are caused by an automation's run: {"automation": 4, "run": 51, "depth": 1}.""" + if not (isinstance(run, dict) and all(isinstance(run.get(k), int) for k in ("automation", "run", "depth"))): + raise ValueError("a cause is {automation, run, depth}") + token = _cause.set({k: run[k] for k in ("automation", "run", "depth")}) + try: + yield + finally: + _cause.reset(token) + + +def cause(): + """The run whose actions are running now ({automation, run, depth}), or None.""" + c = _cause.get() + return dict(c) if c else None + + +def _key_ref(key): + return R("event_log_key", hashlib.sha1(str(key).encode()).hexdigest()) + + +def emit( + db, + kind, + *, + actor, + obj, + space, + summary, + target=None, + tag=None, + key=None, + published=None, + spaces=None, + audience=None, + owner=None, + untrusted=None, +): + """Log an activity that no row of its own records (a deletion, a webhook call) and wake the followers in this + process. Its id (`log-`). With `key`, emitting the same key again changes nothing and gives the first id, so a + retry never doubles it. Inside `caused_by`, the activity carries that cause.""" + k = _KINDS.get(kind) + if not k: + raise ValueError(f"no kind {kind}; register it with events.kind first") + if k.source != "log": + raise ValueError(f"{kind} is read from its own rows ({k.source}), not emitted") + if key is not None: + known = db.one("SELECT log FROM $r", r=_key_ref(key)) + if known: + return f"log-{known['log']}" + n = db.next_id("event_log") + at = store.now() + a = build( + f"log-{n}", + kind, + actor=actor, + obj=obj, + summary=summary, + space=space, + source="log", + pos=[at, n], + published=published or at, + target=target, + tag=tag, + untrusted=untrusted, + cause=cause(), + spaces=spaces, + audience=audience, + owner=owner, + ) + check_activity(a) + row = store.clean( + { + "kind": kind, + "type": a["type"], + "published": a["published"], + "logged_at": at, + "actor": a["actor"], + "object": a["object"], + "target": a["target"], + "tag": a["tag"], + "summary": a["summary"], + "space": space, + "spaces": a["lens"].get("spaces"), + "audience": a["lens"].get("audience"), + "owner": owner, + "untrusted": a["lens"]["untrusted"], + "cause": a["lens"]["cause"], + "key": key, + } + ) + st, p = ["CREATE $e CONTENT $d"], {"e": R("event_log", n), "d": row} + if key is not None: + st.insert(0, "CREATE $k CONTENT $kd") + p.update(k=_key_ref(key), kd={"key": str(key)[:500], "log": n, "at": at}) + try: + db.run(st, **p) + except Exception as e: # noqa: BLE001 - only a key that's already there means someone else logged it first + if key is None or "already exists" not in str(e): + raise + return f"log-{db.one('SELECT log FROM $r', r=_key_ref(key))['log']}" + wake() + return a["id"] + + +LOG_FIELDS = ( + "record::id(id) AS n, kind, published, logged_at, actor, object, target, tag, summary, space, spaces, audience, owner, untrusted, cause" +) + + +def _from_log(r): + return build( + f"log-{r['n']}", + r["kind"], + actor=r["actor"], + obj=r["object"], + summary=r["summary"], + space=r.get("space"), + source="log", + pos=[r["logged_at"], r["n"]], + published=r["published"], + target=r.get("target"), + tag=r.get("tag"), + untrusted=r.get("untrusted") or [], + cause=r.get("cause"), + spaces=r.get("spaces"), + audience=r.get("audience"), + owner=r.get("owner"), + ) + + +def space_sql(spaces, params, col="space", many=None): + """SQL keeping rows of the namespaces in `spaces` (None: all), its parameters added to `params`.""" + if spaces is None: + return "true" + params["sp"] = sorted(spaces) + return f"({col} IN $sp" + (f" OR {many} CONTAINSANY $sp)" if many else ")") + + +class LogSource(Source): + """event_log: activities that no row of their own records, each with the kind it was emitted as. Followed by + (logged_at, id): ids are given out in order, and a row written late is caught by the lookback. Paged by + `published`.""" + + name = "log" + + def _kinds_sql(self, flt, params): + ks = (flt or {}).get("kinds") + if ks is None: + return "true" + params["ks"] = sorted(ks) + return "kind IN $ks" + + def read(self, db, cfg, pos, spaces, limit, hint=None): + p = {} + where = f"{after_sql('logged_at', pos, p, table='event_log')} AND {space_sql(spaces, p, many='spaces')}" + rows = db.rows(f"SELECT {LOG_FIELDS}, id AS rid FROM event_log WHERE {where} ORDER BY logged_at, rid LIMIT {int(limit)}", **p) + return [_from_log(r) for r in rows if r.get("kind") in _KINDS] + + def page(self, db, cfg, before, who, flt, limit): + p = {} + where = [self._kinds_sql(flt, p)] + if before is not None: + where.append(after_sql("published", before, p, "<", table="event_log")) + if not who.get("admin"): + p["me"] = who.get("account") + readable = space_sql(who.get("spaces") or set(), p, many="spaces") + where.append(f"(((audience = NONE OR audience = 'space') AND {readable}) OR (audience = 'owner' AND owner = $me))") + else: + p["me"] = who.get("account") + where.append("(audience != 'owner' OR owner = $me)") + for k, op in (("since", ">"), ("until", "<=")): + if (flt or {}).get(k): + p[k] = flt[k] + where.append(f"published {op} ${k}") + rows = db.rows( + f"SELECT {LOG_FIELDS}, id AS rid FROM event_log WHERE {' AND '.join(where)} ORDER BY published DESC, rid DESC LIMIT {int(limit)}", + **p, + ) + return [_from_log(r) for r in rows if r.get("kind") in _KINDS] + + def get(self, db, cfg, activity_id): + m = re.match(r"^log-(\d+)$", activity_id) + if not m: + return None + r = db.one(f"SELECT {LOG_FIELDS} FROM $r", r=R("event_log", int(m.group(1)))) + return _from_log(r) if r and r.get("kind") in _KINDS else None + + def key(self, a): + return [a["published"], a["lens"]["pos"][1]] + + +register_source(LogSource()) + + +# ---------- following ---------- +def _check_name(consumer): + if not NAME_RX.match(consumer or ""): + raise ValueError("a consumer is named with lowercase letters, digits and _ . : - (automations, say)") + + +def _cursor_ref(consumer): + return R("event_cursor", consumer) + + +def _later_of(a, b): + return a if b is None or pos_key(a) >= pos_key(b) else b + + +def _start(db, consumer, at=None): + """Every source's cursor at now: what happened before isn't read, not even by the lookback.""" + pos = {s.name: s.now(db) for s in sources()} + db.q("UPSERT $c SET pos = $p, floor = $p, at = $t", c=_cursor_ref(consumer), p=pos, t=at or store.now()) + return pos + + +def idle(db, consumer): + """A consumer with nothing to do moves every cursor to now, so nothing that happens meanwhile replays later.""" + _check_name(consumer) + _start(db, consumer) + + +def _in(a, want, spaces): + if want is not None and a.get("kind") not in want: + return False + return spaces is None or any(s in spaces for s in spaces_of(a)) + + +def _order(a): + """Oldest first across sources: by the time of each one's position (its published time when the position isn't a + time), then source and id.""" + p = a["lens"]["pos"] + t = _tkey(p[0]) + if not isinstance(t, float): + t = _parse(a["published"]).timestamp() + return (t, a["lens"]["source"], _ikey(p[1])) + + +def follow(db, cfg, consumer, kinds=None, spaces=None, limit=PAGE, hint=None): + """What happened since this consumer's last commit: (activities oldest first, commit). Handle them in order, then + call commit() to save where the next look starts. Only `kinds` (and their descendants; None: every kind) in + `spaces` (None: every namespace and none). A source with more waiting gives `limit` at a time; once caught up, a + read gives the last LOOKBACK seconds again (minus what this process already handled), so claim each before acting + on it. The first look only marks where this consumer starts.""" + _check_name(consumer) + want = expand(kinds) + spaces = None if spaces is None else set(spaces) + st = db.one("SELECT pos, floor FROM $c", c=_cursor_ref(consumer)) + if not st or not isinstance(st.get("pos"), dict): + _start(db, consumer) + return [], lambda: None + pos, floor = dict(st["pos"]), dict(st.get("floor") or {}) + moved, floors, out = {}, {}, [] + with _lock: + seen = dict(_seen.get(consumer) or {}) + for s in sources(): + here = pos.get(s.name) + if here is None or (want is not None and not produces(s) & want): + moved[s.name] = floors[s.name] = s.now(db) # nobody wants these: when someone does, they start from then + continue + rows = s.read(db, cfg, here, spaces, limit, hint) + if rows: + moved[s.name] = rows[-1]["lens"]["pos"] + if len(rows) < limit: # caught up: the last LOOKBACK seconds again, not before where this consumer started + have = {a["id"] for a in rows} + since = s.back(db, LOOKBACK) + if floor.get(s.name) is not None and pos_key(floor[s.name]) > pos_key(since): + since = floor[s.name] + for _ in range(PAGE_ROUNDS): + if pos_key(since) >= pos_key(here): + break + back = s.read(db, cfg, since, spaces, limit, hint) + rows += [a for a in back if pos_key(a["lens"]["pos"]) <= pos_key(here) and a["id"] not in have and a["id"] not in seen] + if len(back) < limit: + break + since = back[-1]["lens"]["pos"] + out += [a for a in rows if _in(a, want, spaces)] + out.sort(key=_order) + + def commit(): + cur = db.one("SELECT pos, floor FROM $c", c=_cursor_ref(consumer)) or {} + p, f = dict(cur.get("pos") or pos), dict(cur.get("floor") or floor) + for name, at in moved.items(): + p[name] = _later_of(at, p.get(name)) + for name, at in floors.items(): + f[name] = _later_of(at, f.get(name)) + db.q("UPSERT $c SET pos = $p, floor = $f, at = $t", c=_cursor_ref(consumer), p=p, f=f, t=store.now()) + with _lock: + mine = _seen.setdefault(consumer, {}) + for a in out: + mine[a["id"]] = None + while len(mine) > 20000: + mine.pop(next(iter(mine))) + + return out, commit + + +def _claim_ref(consumer, key): + return R("event_claim", hashlib.sha1(f"{consumer}|{key}".encode()).hexdigest()) + + +def _known(ref_id): + with _lock: + _claimed[ref_id] = None + while len(_claimed) > 50000: + _claimed.pop(next(iter(_claimed))) + + +def claim(db, consumer, key): + """True for the one process (and the one time) that claims `key` for `consumer`, such as an activity's id.""" + _check_name(consumer) + ref = _claim_ref(consumer, key) + with _lock: + if ref.id in _claimed: + return False + try: + db.run(["CREATE $r CONTENT $d"], r=ref, d={"consumer": consumer, "key": str(key)[:500], "at": store.now()}) + except Exception as e: # noqa: BLE001 - only a claim that's already there means another process has it + if "already exists" not in str(e): + raise + _known(ref.id) + return False + _known(ref.id) + return True + + +def unclaimed(db, consumer, keys): + """The keys nobody has claimed yet, in one query: for skipping what is taken before claiming one by one (it claims + nothing).""" + refs_ = {k: _claim_ref(consumer, k) for k in keys} + with _lock: + todo = {k: r for k, r in refs_.items() if r.id not in _claimed} + if not todo: + return [] + taken = set(db.values("SELECT VALUE record::id(id) FROM event_claim WHERE id IN $ids", ids=list(todo.values()))) + for r in todo.values(): + if r.id in taken: + _known(r.id) + return [k for k, r in todo.items() if r.id not in taken] + + +def wake(): + """Have the followers in this process look at once (emit calls it).""" + with _wakes: + _generation[0] += 1 + _wakes.notify_all() + + +def generation(): + """How many wakes this process has had: take it before a look, and give it to `wait` after.""" + with _wakes: + return _generation[0] + + +def wait(seconds, stop=None, since=None): + """Sleep until a wake after `since` (a `generation()`; default: now), `stop` (a threading.Event) is set, or + `seconds` pass; True when woken. Every follower waiting is woken, and a wake that came during the look counts.""" + until = time.monotonic() + max(0.0, float(seconds)) + with _wakes: + seen = _generation[0] if since is None else since + while _generation[0] == seen: + left = until - time.monotonic() + if left <= 0 or (stop is not None and stop.is_set()): + return False + _wakes.wait(min(0.5, left)) + return True + + +def section(cfg): + return {"keep_days": 30, "poll_seconds": 5, **(cfg.get("events") or {})} + + +def poll_seconds(cfg): + """How often followers look when nothing wakes them (events.poll_seconds).""" + return max(1, int(section(cfg)["poll_seconds"] or 5)) + + +# ---------- the timeline ---------- +def check_filter(flt): + """Timeline filters, cleaned: kinds (names; their descendants count too), actor (an actor's id, or several), + spaces (namespace ids), tag (an entity's id or name), since (published after) and until (at or before), q (words in + the summary) and where (a match tree on the activity). ValueError naming the first problem.""" + from . import match + + flt = dict(flt or {}) + extra = sorted(set(flt) - set(FILTERS)) + if extra: + raise ValueError(f"no filter {extra[0]}; filters are {', '.join(FILTERS)}") + out = {} + if flt.get("kinds"): + ks = [flt["kinds"]] if isinstance(flt["kinds"], str) else list(flt["kinds"]) + expand(ks) + out["kinds"] = ks + if flt.get("actor"): + a = [flt["actor"]] if isinstance(flt["actor"], str) else list(flt["actor"]) + for x in a: + _ref(x, "actor") + out["actor"] = a + if flt.get("spaces") is not None: + if not isinstance(flt["spaces"], (list, set, tuple)) or not all(isinstance(s, int) for s in flt["spaces"]): + raise ValueError("spaces is a list of namespace ids") + out["spaces"] = set(flt["spaces"]) + if flt.get("tag"): + out["tag"] = str(flt["tag"]).strip()[:200] + for k in ("since", "until"): + if flt.get(k): + try: + out[k] = stamp(flt[k]) + except ValueError: + raise ValueError(f"{k} is a time like 2026-10-15T16:21:58Z") from None + if flt.get("q"): + out["q"] = [w for w in str(flt["q"]).casefold().split() if w][:20] + if flt.get("where"): + out["where"] = match.check(flt["where"], "where") + return out + + +def _matches(a, flt): + from . import match + + if flt.get("actor") and (a.get("actor") or {}).get("id") not in flt["actor"]: + return False + if flt.get("spaces") is not None and not any(s in flt["spaces"] for s in spaces_of(a)): + return False + if flt.get("tag"): + t = flt["tag"].casefold() + if not any(str(x.get("id") or "").casefold() == t or str(x.get("name") or "").casefold() == t for x in a.get("tag") or []): + return False + if flt.get("since") and not a["published"] > flt["since"]: + return False + if flt.get("until") and not a["published"] <= flt["until"]: + return False + if flt.get("q"): + text = a.get("summary", "").casefold() + if not all(w in text for w in flt["q"]): + return False + return not flt.get("where") or match.test(flt["where"], a) + + +def _wanted(flt): + """The kinds a timeline page shows: those asked for (with descendants, noisy ones only when named), else every + kind that isn't noisy.""" + if flt.get("kinds"): + named = set(flt["kinds"]) + return {k for k in expand(flt["kinds"]) if not _KINDS[k].noisy or k in named} + return {k for k, v in _KINDS.items() if not v.noisy} + + +def _encode(cur): + raw = json.dumps(cur, separators=(",", ":"), sort_keys=True).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def _decode(before): + try: + cur = json.loads(base64.urlsafe_b64decode(before + "=" * (-len(before) % 4))) + if not isinstance(cur, dict) or not all(v is None or (isinstance(v, list) and len(v) == 2) for v in cur.values()): + raise ValueError + return cur + except (ValueError, TypeError): + raise ValueError("before is the `next` of an earlier page") from None + + +def page(db, cfg, who, before=None, flt=None, limit=50): + """A page of the timeline for `who` (viewer), newest first: {items, next}. `next` goes in `before` for the page + after (None at the end). Filters (check_filter) thin a page out; it reads on, up to PAGE_ROUNDS source pages each, + so a filtered page isn't short while there is more.""" + flt = check_filter(flt) + limit = max(1, min(int(limit), 200)) + want = _wanted(flt) + sflt = {**flt, "kinds": want} + cur = _decode(before) if before else None + srcs = [s for s in sources() if produces(s) & want and (cur is None or s.name in cur)] + at = {s.name: (cur or {}).get(s.name) for s in srcs} + got: dict[str, list] = {s.name: [] for s in srcs} + reads = dict.fromkeys(got, 0) + done: set[str] = set() + + def ok(a): + return a.get("kind") in want and visible(a, who) and _matches(a, flt) + + def k(s, a): + return (a["published"], s.name, _ikey(s.key(a)[1])) + + def fetch(s): + rows = s.page(db, cfg, at[s.name], who, sflt, limit)[:limit] + reads[s.name] += 1 + got[s.name] += rows + if rows: + at[s.name] = s.key(rows[-1]) + if len(rows) < limit: + done.add(s.name) + + for s in srcs: + fetch(s) + while True: + # past the oldest row read from a source with more to read, that source may have something newer unread + open_ = [s for s in srcs if s.name not in done] + horizon = max((k(s, got[s.name][-1]) for s in open_), default=None) + fit = sorted( + ((k(s, a), s.name, a) for s in srcs for a in got[s.name] if ok(a) and (horizon is None or k(s, a) >= horizon)), + key=lambda x: x[0], + reverse=True, + ) + if len(fit) >= limit or not open_: + break + more = [s for s in open_ if k(s, got[s.name][-1]) == horizon and reads[s.name] < PAGE_ROUNDS] + if not more: + break + for s in more: + fetch(s) + items = [a for _, _, a in fit[:limit]] + shown = {a["id"] for a in items} + nxt = {} + for s in srcs: + last, stopped = (cur or {}).get(s.name), False + for a in got[s.name]: + if ok(a) and a["id"] not in shown: + stopped = True + break + last = s.key(a) + if stopped or s.name not in done: + nxt[s.name] = last + return {"items": items, "next": _encode(nxt) if nxt else None} + + +def get(db, cfg, activity_id, who): + """One activity `who` may see, by its id; None when there's none or it isn't theirs to see.""" + if not isinstance(activity_id, str) or not ID_RX.match(activity_id): + return None + for s in sources(): + a = s.get(db, cfg, activity_id) + if a: + return a if visible(a, who) else None + return None + + +# ---------- keeping it small ---------- +def tidy(db, cfg, now=None): + """Drop event_log rows older than events.keep_days and claims older than a day, at most once an hour per process + (routines.run_due calls it every 30 seconds); the cut-off, or 0 when it didn't look.""" + t = time.monotonic() + if now is None and _swept["at"] and t - _swept["at"] < SWEEP_SECONDS: + return 0 + _swept["at"] = t + days = int(section(cfg)["keep_days"] or 30) + cut = ago(days * 86400, now) + db.q("DELETE event_log WHERE logged_at < $c", c=cut) + db.q("DELETE event_log_key WHERE at < $c", c=cut) + db.q("DELETE event_claim WHERE at < $c", c=ago(CLAIM_DAYS * 86400, now)) + return cut + + +from . import events_content # noqa: E402, F401 - the recordings source, registered with the stream once it is loaded diff --git a/fastapi_backend/app/domain/events_content.py b/fastapi_backend/app/domain/events_content.py new file mode 100644 index 000000000..39e8acbaa --- /dev/null +++ b/fastapi_backend/app/domain/events_content.py @@ -0,0 +1,154 @@ +"""Content as activities (docs/automations.md, Sources): the `recordings` source reads the assets Lens keeps (the +`recording` table: audio, video, documents, images, emails, web pages) as `asset.added` activities, by `created_at` +with the id breaking ties, so nothing is stored twice. + +Each one's id is `create-recording-`, so it is the same every time it's read. Its actor is whoever added it: an +automation (the asset's first job was queued by `automation:`), the storage source a watched file came from, the +person whose upload it was, or Lens itself (a folder scan). Its object is the asset; its title is outside text. +""" + +from __future__ import annotations + +import re + +from . import events, store + +R = store.R +FIELDS = "id AS rid, record::id(id) AS id, space, title, source, media, remote, content_type, created_at" +TYPES = {"audio": "Audio", "video": "Video", "image": "Image", "document": "Document", "transcript": "Document"} +AUTOMATION_RX = re.compile(r"^automation:(\d+)$") + + +def _sec(t): + """A time as `created_at` is written (store.now(): to the second, +00:00), cut down to the second.""" + return events._parse(t).replace(microsecond=0).isoformat(timespec="seconds") + + +def _actors(db, rows): + """{recording id: actor} for these rows, in a few queries for all of them.""" + ids = [r["id"] for r in rows] + if not ids: + return {} + first_job: dict = {} + for j in db.rows("SELECT recording, created_by, created_at FROM job WHERE recording IN $ids ORDER BY created_at", ids=ids): + first_job.setdefault(j["recording"], j.get("created_by")) + uploads = {u["recording"]: u for u in db.rows("SELECT recording, account, email FROM upload WHERE recording IN $ids", ids=ids)} + accounts = {u["account"] for u in uploads.values() if isinstance(u.get("account"), int)} + names = { + a["id"]: a.get("name") or a.get("email") + for a in ( + db.rows("SELECT record::id(id) AS id, name, email FROM account WHERE id IN $a", a=[R("account", n) for n in accounts]) + if accounts + else [] + ) + } + srcs = {(r.get("remote") or {}).get("source") for r in rows} - {None} + sources = { + s["id"]: s.get("name") + for s in ( + db.rows("SELECT record::id(id) AS id, name FROM storage_source WHERE id IN $s", s=[R("storage_source", n) for n in srcs]) + if srcs + else [] + ) + } + out = {} + for r in rows: + m = AUTOMATION_RX.match(str(first_job.get(r["id"]) or "")) + sid = (r.get("remote") or {}).get("source") + up = uploads.get(r["id"]) + if m: + out[r["id"]] = {"type": "Application", "id": f"lens:automation/{m.group(1)}", "name": f"Automation {m.group(1)}"} + elif sid is not None and re.match(r"^[A-Za-z0-9_.:@+-]{1,200}$", str(sid)): + out[r["id"]] = {"type": "Service", "id": f"lens:source/{sid}", "name": sources.get(sid) or f"Source {sid}"} + elif up and isinstance(up.get("account"), int): + out[r["id"]] = { + "type": "Person", + "id": f"lens:account/{up['account']}", + "name": names.get(up["account"]) or up.get("email") or "Someone", + } + else: + out[r["id"]] = dict(events.LENS) + return out + + +def _activities(db, rows): + """The rows as asset.added activities (a row whose created_at isn't a time is skipped).""" + from . import render + + who = _actors(db, rows) + out = [] + for r in rows: + try: + published = events.stamp(r["created_at"]) + except (TypeError, ValueError): + continue + media = render.kind(r) + title = " ".join(str(r.get("title") or "").split()) or "Untitled" + actor = who[r["id"]] + obj = store.clean( + { + "type": TYPES.get(media, "Document"), + "id": f"lens:recording/{r['id']}", + "name": title, + "url": f"/resources/{r['id']}", + "media": media, + "content_type": r.get("content_type") if isinstance(r.get("content_type"), str) else None, + } + ) + out.append( + events.build( + f"create-recording-{r['id']}", + "asset.added", + actor=actor, + obj=obj, + summary=f"{actor['name']} added “{title[:200]}”", + space=r.get("space"), + source="recordings", + pos=[r["created_at"], r["id"]], + published=published, + ) + ) + return out + + +class RecordingsSource(events.Source): + """recording rows as asset.added, read by (created_at, id).""" + + name = "recordings" + kinds = ("asset.added",) + + def read(self, db, cfg, pos, spaces, limit, hint=None): + p: dict = {} + where = f"{events.after_sql('created_at', pos, p, table='recording')} AND {events.space_sql(spaces, p)}" + rows = db.rows(f"SELECT {FIELDS} FROM recording WHERE {where} ORDER BY created_at, rid LIMIT {int(limit)}", **p) + return _activities(db, rows) + + def page(self, db, cfg, before, who, flt, limit): + p: dict = {} + where = [events.space_sql(None if who.get("admin") else (who.get("spaces") or set()), p), "created_at != NONE"] + if before is not None: + where.append(events.after_sql("created_at", before, p, "<", table="recording")) + if flt.get("spaces") is not None: + p["fs"] = sorted(flt["spaces"]) + where.append("space IN $fs") + if flt.get("since"): # a looser bound, to the second; the stream checks each filter exactly again + p["since"] = _sec(flt["since"]) + where.append("created_at > $since") + if flt.get("until"): + p["until"] = _sec(flt["until"]) + where.append("created_at <= $until") + rows = db.rows( + f"SELECT {FIELDS} FROM recording WHERE {' AND '.join(where)} ORDER BY created_at DESC, rid DESC LIMIT {int(limit)}", **p + ) + return _activities(db, rows) + + def get(self, db, cfg, activity_id): + m = re.match(r"^create-recording-(\d+)$", activity_id) + if not m: + return None + r = db.one(f"SELECT {FIELDS} FROM $r", r=R("recording", int(m.group(1)))) + got = _activities(db, [r]) if r else [] + return got[0] if got else None + + +events.register_source(RecordingsSource()) diff --git a/fastapi_backend/app/domain/match.py b/fastapi_backend/app/domain/match.py index 793d86059..75b7008a0 100644 --- a/fastapi_backend/app/domain/match.py +++ b/fastapi_backend/app/domain/match.py @@ -100,7 +100,13 @@ def paths(tree): def _label(path, kind): - """A path in words (`kind` is for naming fields by their titles, once kinds have schemas).""" + """A path in words: with a kind, the title its schema gives the field there ("the title"), else the path.""" + if kind and path: + from . import events + + title = (events.field_at(kind, path) or {}).get("title") + if title: + return f"the {title[0].lower()}{title[1:]}" return path or "the value" diff --git a/fastapi_backend/app/domain/routines.py b/fastapi_backend/app/domain/routines.py index 681498a27..397b5d81b 100644 --- a/fastapi_backend/app/domain/routines.py +++ b/fastapi_backend/app/domain/routines.py @@ -27,6 +27,7 @@ analytics, backups, budgets, + events, fedora, jobs, notebook, @@ -603,6 +604,11 @@ def run_due(db, cfg, log=print, now=None): stamp = _iso(now) sweep(db, now) activity.tidy(db, cfg) # the activity ledger past activity.keep_days, once an hour + try: + events.tidy(db, cfg) # the event log past events.keep_days and claims past a day, once an hour + except Exception as e: # noqa: BLE001 - the schedule carries on + if log: + log(f"events: {type(e).__name__}: {e}") try: analytics.tick(db, cfg, now) # once a day, while Google Analytics is on (docs/analytics.md) except Exception as e: # noqa: BLE001 - the schedule carries on diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index a1105be18..66826a91b 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -619,6 +619,9 @@ def labels(): # the activity ledger (docs/activity.md): every change and call in or out, with its cost, per resource. On by # default; reads: log API reads (GET) too; keep_days: how long rows are kept "activity": {"enabled": True, "reads": False, "keep_days": 365}, + # the stream of activities automations follow (docs/automations.md): keep_days: how long activities with no row + # of their own (the event log) are kept; poll_seconds: how often followers look when nothing wakes them + "events": {"keep_days": 30, "poll_seconds": 5}, # database backups (docs/database.md, Backups): schedule is a cron expression in timezone, or none (the default) # for none; the newest `keep` scheduled backups are kept. connection: a storage connection (sources.py) every new # backup is also copied to, under folder, encrypted with a key from secret.key first (none: kept here only) @@ -1250,6 +1253,18 @@ def close(self): "DEFINE INDEX IF NOT EXISTS notify_delivery_status ON notify_delivery FIELDS status", "DEFINE TABLE IF NOT EXISTS notify_event SCHEMALESS", "DEFINE TABLE IF NOT EXISTS notify_state SCHEMALESS", + # the stream of activities (domain/events.py, docs/automations.md): activities no row of their own records + # (event_log:), the keys they were emitted with (event_log_key:), where each consumer's next look starts + # (event_cursor:) and what each consumer claimed (event_claim:) + "DEFINE TABLE IF NOT EXISTS event_log SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS event_log_logged ON event_log FIELDS logged_at", + "DEFINE INDEX IF NOT EXISTS event_log_published ON event_log FIELDS published", + "DEFINE TABLE IF NOT EXISTS event_log_key SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS event_log_key_at ON event_log_key FIELDS at", + "DEFINE TABLE IF NOT EXISTS event_cursor SCHEMALESS", + "DEFINE TABLE IF NOT EXISTS event_claim SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS event_claim_at ON event_claim FIELDS at", + "DEFINE INDEX IF NOT EXISTS upload_recording ON upload FIELDS recording", # every line of a run's log, in chunks (jobs.RunLog): job_log: "DEFINE TABLE IF NOT EXISTS job_log SCHEMALESS", "DEFINE INDEX IF NOT EXISTS job_log_job ON job_log FIELDS job", diff --git a/fastapi_backend/tests/domain/test_events.py b/fastapi_backend/tests/domain/test_events.py new file mode 100644 index 000000000..720d78c82 --- /dev/null +++ b/fastapi_backend/tests/domain/test_events.py @@ -0,0 +1,387 @@ +"""The stream of activities (docs/automations.md, Contract): the activity shape, the kinds catalog, the event log, +following the stream once across processes, the timeline read newest first with who may see what, and tidying.""" + +from __future__ import annotations + +import datetime as dt +import threading +import time + +import pytest + +from app.domain import events, match, store + +R = store.R + + +@pytest.fixture(autouse=True) +def kinds(): + """Test kinds, gone again after each test; nothing this process handled or claimed carries over.""" + before = dict(events._KINDS) + events.kind( + "test_thing.happened", + label="Thing happened", + source="log", + as_type="Announce", + object_type="Note", + fields={"object": {"type": "object", "properties": {"size": {"type": "integer", "title": "Size"}}}}, + untrusted=("object.name",), + ) + events.kind("test_thing.ticked", label="Thing ticked", source="log", as_type="Update", object_type="Note", noisy=True) + events._seen.clear() + events._claimed.clear() + yield + events._KINDS.clear() + events._KINDS.update(before) + events._seen.clear() + events._claimed.clear() + + +def at(seconds=0): + """A time as store.now() writes it, `seconds` from now.""" + return (dt.datetime.now(dt.UTC) + dt.timedelta(seconds=seconds)).isoformat(timespec="seconds") + + +def next_second(): + start = store.now() + while store.now() == start: + time.sleep(0.05) + + +def recording(db, sid, title, when, **extra): + rid = db.next_id("recording") + db.q("CREATE $r CONTENT $d", r=R("recording", rid), d={"space": sid, "title": title, "source": "audio", "created_at": when, **extra}) + return rid + + +def thing(db, name, space, **kw): + return events.emit( + db, "test_thing.happened", actor=events.LENS, obj={"type": "Note", "name": name}, space=space, summary=f"Lens noted {name}", **kw + ) + + +ADMIN = events.viewer(1, None, admin=True) + + +# ---------- the shape ---------- +def test_activities_have_one_shape(): + sample = events.kind_info("asset.added")["sample"] + assert events.check_activity(sample) is sample + bad = [ + ({"id": "Create-7"}, "an activity's id"), + ({"type": "Created"}, "type is one of Create"), + ({"kind": "email.nope"}, "no kind email.nope"), + ({"published": "2026-10-15T08:12:03+00:00"}, "published is an ISO 8601 time in UTC"), + ({"actor": {"type": "Entity", "id": "lens:entity/1", "name": "x"}}, "the actor is an object whose type"), + ({"actor": {"type": "Person", "id": "account/1", "name": "x"}}, "the actor's id is a Lens reference"), + ({"actor": {"type": "Person", "id": "lens:account/1", "name": " "}}, "the actor has a name"), + ({"object": {"id": "lens:recording/7"}}, "the object is an object with a type"), + ({"tag": [{"id": "lens:entity/1"}]}, "tag is a list of at most 50 objects"), + ({"summary": "x" * 301}, "the summary is one line"), + ({"space": True}, "space is a namespace id"), + ({"lens": {"pos": [1, 2]}}, "lens is an object naming the source"), + ({"lens": {**sample["lens"], "untrusted": ["object subject"]}}, "lens.untrusted is a list of dotted paths"), + ({"lens": {**sample["lens"], "cause": {"automation": 4}}}, "lens.cause is"), + ({"lens": {**sample["lens"], "audience": "owner"}}, "names the owner's account"), + ({"lens": {**sample["lens"], "audience": "friends"}}, "lens.audience is one of space, owner, admins"), + ] + for change, said in bad: + with pytest.raises(ValueError, match=said): + events.check_activity({**sample, **change}) + + +def test_the_catalog_knows_each_kind_its_parents_and_fields(): + events.kind( + "test_mail.received", + label="Mail received", + source="recordings", + as_type="Create", + object_type="Document", + parent="asset.added", + untrusted=("object.name", "object.subject"), + fields={ + "object": { + "type": "object", + "properties": { + "subject": {"type": "string", "title": "Subject"}, + "from": {"type": "array", "title": "Sender", "items": {"type": "string", "title": "Address"}}, + }, + } + }, + ) + events.kind( + "test_mail.flagged", label="Flagged", source="recordings", as_type="Create", object_type="Document", parent="test_mail.received" + ) + assert events.descendants("asset.added") == {"asset.added", "test_mail.received", "test_mail.flagged"} + assert events.expand(["test_mail.received"]) == {"test_mail.received", "test_mail.flagged"} and events.expand(None) is None + with pytest.raises(ValueError, match="no kind test_nope.x"): + events.expand(["test_nope.x"]) + info = events.kind_info("asset.added") + assert info["children"] == ["test_mail.received"] and info["source"] == "recordings" and info["type"] == "Create" + assert info["untrusted"] == ["object.name"] and info["sample"]["kind"] == "asset.added" + # a child's schema has every activity's fields, its parents' and its own + assert events.field_at("test_mail.flagged", "object.subject")["title"] == "Subject" + assert events.field_at("test_mail.flagged", "object.name")["title"] == "Title" + assert events.field_at("test_mail.flagged", "object.from.0")["title"] == "Address" + assert events.field_at("test_mail.flagged", "actor.name")["title"] == "Actor's name" + assert events.field_at("test_mail.flagged", "object.nope") is None and events.field_at("test_nope.x", "object") is None + assert {"asset.added", "test_thing.happened"} <= {k["name"] for k in events.kinds()} + tree = {"all": [{"path": "object.subject", "op": "contains", "value": "trip"}, {"path": "object.weird", "op": "exists"}]} + assert match.describe(tree, "test_mail.received") == "the subject contains “trip” and object.weird is set" + bad = [ + (dict(name="Email.Received"), "named like noun.past_verb"), + (dict(source=""), "name the source"), + (dict(as_type="Sent"), "its type is one of"), + (dict(parent="test_nope.x"), "register its parent test_nope.x first"), + (dict(fields={"body": {}}), "fields are JSON Schemas"), + (dict(untrusted=("object subject",)), "untrusted paths are dotted"), + (dict(sample={**info["sample"], "kind": "asset.added"}), "its sample is a asset.added"), + (dict(sample={**info["sample"], "kind": "test_bad.added", "summary": ""}), "the summary is one line"), + ] + for change, said in bad: + args = {"name": "test_bad.added", "label": "Bad", "source": "log", "as_type": "Create", "object_type": "Note", **change} + with pytest.raises(ValueError, match=said): + events.kind(args.pop("name"), **args) + assert "test_bad.added" not in events._KINDS # a kind whose sample is wrong isn't left registered + + +# ---------- the event log ---------- +def test_emit_logs_once_per_key_and_carries_its_cause(db, cfg): + sid = store.ns_id(db, "pods") + a = thing(db, "one", sid, key="hook-17") + assert a.startswith("log-") and thing(db, "one again", sid, key="hook-17") == a # a retry never doubles it + assert len(db.values("SELECT VALUE id FROM event_log")) == 1 + with events.caused_by({"automation": 4, "run": 51, "depth": 1}): + assert events.cause() == {"automation": 4, "run": 51, "depth": 1} + b = thing(db, "two", sid, tag=[{"type": "Mention", "id": "lens:entity/9", "name": "St Mary's"}]) + assert events.cause() is None + got = events.get(db, cfg, b, ADMIN) + assert got["kind"] == "test_thing.happened" and got["type"] == "Announce" and got["actor"] == events.LENS + assert got["lens"]["cause"] == {"automation": 4, "run": 51, "depth": 1} and got["lens"]["untrusted"] == ["object.name"] + assert got["tag"][0]["id"] == "lens:entity/9" and got["summary"] == "Lens noted two" and got["published"].endswith("Z") + assert events.check_activity(got) and events.get(db, cfg, a, ADMIN)["lens"]["cause"] is None + with pytest.raises(ValueError, match="asset.added is read from its own rows"): + events.emit(db, "asset.added", actor=events.LENS, obj={"type": "Audio"}, space=sid, summary="x") + with pytest.raises(ValueError, match="no kind test_nope.x"): + events.emit(db, "test_nope.x", actor=events.LENS, obj={"type": "Note"}, space=sid, summary="x") + with pytest.raises(ValueError, match="a cause is"): + with events.caused_by({"automation": 4}): + pass + + +# ---------- following ---------- +def test_following_reads_each_source_in_order_once(db, cfg): + pods, calls = store.ns_id(db, "pods"), store.ns_id(db, "calls") + recording(db, pods, "Before", at()) + assert events.follow(db, cfg, "test.a")[0] == [] # the first look only marks where it starts + next_second() + a1, a2 = thing(db, "one", pods), thing(db, "two", calls) + late = db.next_id("recording") # given out now, written after the looks below (a slow transaction) + now = store.now() + r1, r2 = recording(db, pods, "Call one", now), recording(db, calls, "Call two", now) # equal times: the id breaks the tie + got, commit = events.follow(db, cfg, "test.a", limit=1) # one at a time from each source + assert [a["id"] for a in got] == [a1, f"create-recording-{r1}"] + commit() + got, commit = events.follow(db, cfg, "test.a", limit=1) + assert [a["id"] for a in got] == [a2, f"create-recording-{r2}"] + commit() + got, commit = events.follow(db, cfg, "test.a") + assert got == [] # caught up: the lookback reads them again, but this process handled them already + commit() + events._seen.clear() # another process: the lookback gives them again, and claims keep them from being done twice + again, _ = events.follow(db, cfg, "test.a") + assert {a["id"] for a in again} == {a1, a2, f"create-recording-{r1}", f"create-recording-{r2}"} + assert [a["id"] for a in again if a["lens"]["source"] == "log"] == [a1, a2] # oldest first, in each source's order + # a row written as the last look ran (before the cursor, inside the lookback) is still seen + db.q("CREATE $r CONTENT $d", r=R("recording", late), d={"space": pods, "title": "Late", "source": "audio", "created_at": now}) + got, _ = events.follow(db, cfg, "test.a", kinds=["asset.added"], spaces=[pods]) + assert f"create-recording-{late}" in {a["id"] for a in got} and {a["space"] for a in got} == {pods} + assert all(a["kind"] == "asset.added" for a in got) + # a consumer seen for the first time starts at now: nothing before replays, not even what the lookback would see + assert events.follow(db, cfg, "test.b")[0] == [] + got, _ = events.follow(db, cfg, "test.b") + assert got == [] + with pytest.raises(ValueError, match="a consumer is named"): + events.follow(db, cfg, "Test B") + + +def test_a_kind_listened_to_later_does_not_replay_a_backlog(db, cfg): + pods = store.ns_id(db, "pods") + events.follow(db, cfg, "test.c", kinds=["test_thing.happened"]) + next_second() + recording(db, pods, "Unwanted", store.now()) + got, commit = events.follow(db, cfg, "test.c", kinds=["test_thing.happened"]) + assert got == [] # recordings make no wanted kind: not read, and their cursor moves to now + commit() + next_second() + events._seen.clear() + got, _ = events.follow(db, cfg, "test.c", kinds=["asset.added"]) + assert got == [] # not the recording from before, not even through the lookback + rid = recording(db, pods, "Wanted", at(1)) + got, commit = events.follow(db, cfg, "test.c", kinds=["asset.added"]) + assert [a["id"] for a in got] == [f"create-recording-{rid}"] + events.idle(db, "test.c") # nothing to do: every cursor at now + assert db.one("SELECT pos FROM $c", c=R("event_cursor", "test.c"))["pos"]["recordings"][1] is None + + +def test_a_claim_is_won_once(db): + wins = [] + + def go(): + wins.append(events.claim(db, "test.d", "create-recording-7")) + + threads = [threading.Thread(target=go) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + assert sorted(wins) == [False] * 5 + [True] + events._claimed.clear() # another process, with nothing claimed in memory + assert events.claim(db, "test.d", "create-recording-7") is False + assert events.claim(db, "test.other", "create-recording-7") is True # claims are per consumer + events._claimed.clear() + assert events.unclaimed(db, "test.d", ["create-recording-7", "create-recording-8"]) == ["create-recording-8"] + + +def test_wake_reaches_every_follower_waiting(): + g = events.generation() + events.wake() # a wake during the look still counts + assert events.wait(5, since=g) is True + woke = [] + waiters = [threading.Thread(target=lambda: woke.append(events.wait(5))) for _ in range(3)] + for t in waiters: + t.start() + time.sleep(0.2) + events.wake() + for t in waiters: + t.join() + assert woke == [True, True, True] + stop = threading.Event() + stop.set() + assert events.wait(5, stop) is False and events.wait(0.1) is False + + +# ---------- the timeline ---------- +@pytest.fixture +def feed(db): + pods, calls = store.ns_id(db, "pods"), store.ns_id(db, "calls") + db.q("CREATE $r CONTENT $d", r=R("account", 7), d={"email": "sam@example.org", "name": "Sam"}) + db.q("CREATE $r CONTENT $d", r=R("storage_source", 3), d={"name": "Office share"}) + up = recording(db, pods, "Parents' evening", at(-50)) + db.q("CREATE upload CONTENT $d", d={"recording": up, "account": 7, "email": "sam@example.org"}) + share = recording(db, pods, "Minutes", at(-40), remote={"source": 3, "path": "minutes.docx"}, source="document") + auto = recording(db, calls, "Call with the school", at(-30)) + db.q("CREATE job CONTENT $d", d={"recording": auto, "created_by": "automation:4", "created_at": at(-30)}) + ids = { + "up": f"create-recording-{up}", + "share": f"create-recording-{share}", + "auto": f"create-recording-{auto}", + "pods": thing( + db, "school trip", pods, published=events.stamp(at(-45)), tag=[{"type": "Mention", "id": "lens:entity/9", "name": "St Mary's"}] + ), + "both": thing(db, "merge", calls, spaces=[pods, calls], published=events.stamp(at(-35))), + "admins": thing(db, "backup", None, audience="admins", published=events.stamp(at(-20))), + "mine": thing(db, "chat", None, audience="owner", owner=7, published=events.stamp(at(-10))), + "nobody": thing(db, "system", None, published=events.stamp(at(-5))), + } + events.emit(db, "test_thing.ticked", actor=events.LENS, obj={"type": "Note"}, space=pods, summary="tick") + return pods, calls, ids + + +def everything(db, cfg, who, flt=None, limit=2): + out, before, pages = [], None, 0 + while True: + p = events.page(db, cfg, who, before, flt, limit) + out += [a["id"] for a in p["items"]] + pages += 1 + if not p["next"]: + return out, pages + before = p["next"] + assert pages < 50 + + +def test_the_timeline_is_newest_first_and_shows_each_person_what_they_may_see(db, cfg, feed): + pods, calls, ids = feed + newest = ["nobody", "mine", "admins", "auto", "both", "share", "pods", "up"] + got, pages = everything(db, cfg, ADMIN) + assert got == [ids[k] for k in newest if k != "mine"] and pages >= 4 # an admin sees all but someone's own + member = events.viewer(7, {pods}) + assert everything(db, cfg, member, limit=3)[0] == [ids[k] for k in ["mine", "both", "share", "pods", "up"]] + other = events.viewer(8, {calls}) + assert everything(db, cfg, other, limit=50)[0] == [ids["auto"], ids["both"]] + assert events.page(db, cfg, events.viewer(9, set()))["items"] == [] + # one at a time comes to the same, without repeats + assert everything(db, cfg, ADMIN, limit=1)[0] == got + assert events.get(db, cfg, ids["auto"], other)["object"]["name"] == "Call with the school" + assert events.get(db, cfg, ids["auto"], member) is None and events.get(db, cfg, ids["admins"], member) is None + assert events.get(db, cfg, "create-recording-999", ADMIN) is None and events.get(db, cfg, "Bad id", ADMIN) is None + with pytest.raises(ValueError, match="before is the `next` of an earlier page"): + events.page(db, cfg, ADMIN, "nope") + + +def test_recordings_are_asset_added_by_whoever_added_them(db, cfg, feed): + _, _, ids = feed + up, share, auto = (events.get(db, cfg, ids[k], ADMIN) for k in ("up", "share", "auto")) + assert up["actor"] == {"type": "Person", "id": "lens:account/7", "name": "Sam"} and up["summary"] == "Sam added “Parents' evening”" + assert share["actor"] == {"type": "Service", "id": "lens:source/3", "name": "Office share"} + assert share["object"]["type"] == "Document" and share["object"]["media"] == "document" + assert auto["actor"]["id"] == "lens:automation/4" and auto["actor"]["type"] == "Application" + assert up["object"] == { + "type": "Audio", + "id": "lens:recording/" + ids["up"].split("-")[-1], + "name": "Parents' evening", + "url": "/resources/" + ids["up"].split("-")[-1], + "media": "audio", + } + assert up["type"] == "Create" and up["kind"] == "asset.added" and up["lens"]["untrusted"] == ["object.name"] + for a in (up, share, auto): + events.check_activity(a) + plain = recording(db, store.ns_id(db, "pods"), "Scanned", at(-1)) + assert events.get(db, cfg, f"create-recording-{plain}", ADMIN)["actor"] == events.LENS + + +def test_timeline_filters(db, cfg, feed): + pods, calls, ids = feed + + def ids_of(flt, who=ADMIN, limit=50): + return everything(db, cfg, who, flt, limit)[0] + + assert ids_of({"kinds": ["asset.added"]}) == [ids["auto"], ids["share"], ids["up"]] + assert len(ids_of({"kinds": ["test_thing.ticked"]})) == 1 # a noisy kind only when named + assert ids_of({"actor": "lens:automation/4"}) == [ids["auto"]] + assert ids_of({"spaces": [pods]}) == [ids["both"], ids["share"], ids["pods"], ids["up"]] + assert ids_of({"tag": "st mary's"}) == [ids["pods"]] and ids_of({"tag": "lens:entity/9"}) == [ids["pods"]] + assert ids_of({"q": "school"}) == [ids["auto"], ids["pods"]] + assert ids_of({"where": {"all": [{"path": "object.media", "op": "equals", "value": "document"}]}}) == [ids["share"]] + a = events.get(db, cfg, ids["share"], ADMIN)["published"] + b = events.get(db, cfg, ids["auto"], ADMIN)["published"] + assert ids_of({"since": a, "until": b}) == [ids["auto"], ids["both"]] # after since, up to and including until + # filters that thin pages out read on, so a page isn't short while there is more + assert ids_of({"q": "evening"}, limit=1) == [ids["up"]] + bad = [ + ({"colour": "red"}, "no filter colour"), + ({"kinds": ["test_nope.x"]}, "no kind test_nope.x"), + ({"actor": "sam"}, "actor is a Lens reference"), + ({"spaces": "pods"}, "spaces is a list of namespace ids"), + ({"since": "last week"}, "since is a time"), + ({"where": {"any": []}}, "where: any needs at least one condition"), + ] + for flt, said in bad: + with pytest.raises(ValueError, match=said): + events.page(db, cfg, ADMIN, flt=flt) + + +# ---------- keeping it small ---------- +def test_tidy_drops_old_log_rows_and_claims(db, cfg): + sid = store.ns_id(db, "pods") + keep = thing(db, "recent", sid, key="k-recent") + old = db.next_id("event_log") + db.q("CREATE $r CONTENT $d", r=R("event_log", old), d={"kind": "test_thing.happened", "logged_at": at(-31 * 86400)}) + db.q("CREATE $r CONTENT $d", r=R("event_log_key", "old"), d={"log": old, "at": at(-31 * 86400)}) + events.claim(db, "test.e", "new") + db.q("CREATE $r CONTENT $d", r=R("event_claim", "old"), d={"consumer": "test.e", "key": "old", "at": at(-2 * 86400)}) + assert events.tidy(db, cfg, now=at()) + assert db.values("SELECT VALUE record::id(id) FROM event_log") == [int(keep.split("-")[1])] + assert len(db.values("SELECT VALUE id FROM event_log_key")) == 1 + assert db.values("SELECT VALUE key FROM event_claim") == ["new"] + assert events.tidy(db, {**cfg, "events": {"keep_days": 1}}, now=at(2 * 86400)) and db.values("SELECT VALUE id FROM event_log") == []