From 2cb29f80c28d5e9ac5dabb65aa1a82d02fa2f19d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 22:33:39 +0000 Subject: [PATCH] feat(automations): finished runs, deletions and calendar moves as activities Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_019nbuG2q5wEXeUxWb8vr5HJ --- docs/automations.md | 19 +- fastapi_backend/app/domain/deletion.py | 10 +- fastapi_backend/app/domain/events_content.py | 398 +++++++++++++++++- fastapi_backend/app/domain/sources.py | 13 +- .../tests/domain/test_events_content.py | 104 ++++- 5 files changed, 523 insertions(+), 21 deletions(-) diff --git a/docs/automations.md b/docs/automations.md index eafc09c5a..392f3371e 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -13,7 +13,8 @@ merged is called built. | Activities, the kinds catalog, the event log, following the stream, timeline reads (`app/domain/events.py`) | built | | New assets as `asset.added` (the `recordings` source) | built | | New assets by kind: emails received, calendar events, files from storage, uploads, pages saved from the web, imports | built | -| More kinds: jobs, deletions, calendar moves, comments, notes, graph changes, sensors, chat | planned | +| Runs that finished or failed, deletions and calendar events that moved | built | +| More kinds: comments, notes, graph changes, sensors, chat | planned | | The timeline API (`GET /api/v1/activities`), the MCP tools `timeline` and `event_kinds`, the assistant's `timeline` | built | | Review marks on the timeline (reviewed, important, hidden), their filters and `mark_activities` ([Timeline](timeline.md)) | built | | The shapes of triggers, joins, an If, policy and intents, checked (`app/domain/automation_spec.py`) | built | @@ -87,6 +88,10 @@ and the paths holding outside text are in `GET /api/v1/activities/kinds` (`event | `upload.added` | `asset.added` | `recordings` | `path` (the file's name), `mediaType`, `size`, `duration` | Sam uploaded “Call with the school” | | `web_page.saved` | `asset.added` | `recordings` | `href` (the page's address), `site` | Sam saved “Museum opening hours” from museum.example.org | | `import.received` | `asset.added` | `recordings` | `via`: hook (an import hook), paste (pasted text) or file (a file read as text) | Phone shortcut imported “Shopping list” | +| `asset.processed` | | `jobs` | the asset's fields, `job` (`lens:job/`), `steps`, `batch`; tagged with its entities | “School trip on Friday” finished processing (pages, analyze, embed) | +| `asset.failed` | | `jobs` | as `asset.processed`, with `step` (where it failed) and `error` | Processing “Call with the school” failed at transcribe | +| `asset.deleted` | | `log` | `type`, `id`, `name`, `media` | Sam deleted “Old minutes” | +| `calendar_event.changed` | | `log` | `startTime`, `endTime`, `location`, `calendar`, and `before` (`startTime`, `endTime`) | Family calendar moved “Parents' evening” to 2026-10-21 18:00 | A new asset is one activity, of the most specific kind its row tells, decided by its content type and where it came from, never by reading its file: an email (content type email, or from an IMAP or Gmail mailbox), a calendar event @@ -95,6 +100,13 @@ from, never by reading its file: an email (content type email, or from an IMAP o imported), else `asset.added`. A file an email or archive held, made a resource of its own, stays `asset.added`: the email is what arrived. A trigger on `asset.added` matches every one of them. +A run of an asset's pipeline that finishes is `asset.processed` (it succeeded) or `asset.failed`, one activity per +run, batch runs included. Its actor is whoever queued it, and its tags are the asset's entities (the ten most +mentioned), which exist only once it has been processed: `asset.processed` is the kind for "when an email about St +Mary's is understood". A run tried again finishes again, as another activity. A deletion is logged as +`asset.deleted` where the asset is deleted (once per asset); a calendar event read again that starts at another time +is logged as `calendar_event.changed` (once per event and new start), with the times it had before. + Fields come from what the row keeps, so reading them costs nothing on the ingest path. A message's addressing is kept as it is listed (`recording.mail`: who sent it to whom, its subject, list, mailbox, and Gmail's snippet), an event's times, place and people as it is imported (`recording.event`), and an email's attachments and the start of @@ -125,6 +137,9 @@ A kind belongs to the source named in its `source`, and a source may list more i `lens:sensor/`), or Lens (a folder scan). `events_content.actors(db, labels)` names them the way Lens writes them down (an account id, an email, `watch:`, `routine:`, `import_hook:`, `automation:`, ...), in a few queries for a whole page, and keeps names for 30 seconds. +- `jobs` (`app/domain/events_content.py`) reads `job` by `finished_at`, the id breaking ties, as `asset.processed` + and `asset.failed`; their ids carry the time they finished (`processed-job-51-1792081450`), so a run tried again + is a new activity. - `log` reads `event_log`: activities no row of their own records (a deletion, a webhook call), written by `events.emit(db, kind, *, actor, obj, space, summary, target=None, tag=None, key=None, published=None, spaces=None, audience=None, owner=None, untrusted=None)`, which gives the id (`log-`). Emitting a `key` again changes @@ -337,6 +352,8 @@ written. `ask_agent` passes the activity it is about the same way. - Name an automation actor by its automation's name once automations are stored. - A timeline page filtered to one child of `asset.added` (only emails, say) reads every new asset and keeps those of that kind, up to five source pages; narrow it in the query (path, `remote`, `web`) if that proves slow. +- Moves of Google Calendar and iCal events only: events in a calendar file on storage are read as new files. An + event that was cancelled or deleted from its calendar isn't an activity yet. - Folder scans of a namespace's own folders as `file.added`; uploaded `.ics` files with their event's fields; a snippet for IMAP messages before they are read (it needs the message body). - Per-recipient threading of email conversations; entity tags on new assets once extraction runs at ingest. diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py index c41057188..ea6020d80 100644 --- a/fastapi_backend/app/domain/deletion.py +++ b/fastapi_backend/app/domain/deletion.py @@ -16,12 +16,14 @@ from __future__ import annotations +import logging import pathlib import shutil from . import access as acc, files as filemod, notebook, render, store, video R = store.R +log = logging.getLogger("lens") # rows that belong to the recording OWN = ( "mentions", @@ -149,7 +151,7 @@ def orphans(db, speakers, faces): def delete(db, cfg, rid, by=None): """Delete a recording (see the module docstring). Returns what the audit log keeps of it. Raises KeyError when there is no such recording and Running when a job is working on it.""" - rec = db.one("SELECT title, space, path, fingerprint, remote FROM $r", r=R("recording", rid)) + rec = db.one("SELECT title, space, path, fingerprint, remote, source, media FROM $r", r=R("recording", rid)) if not rec: raise KeyError(rid) stop_jobs(db, rid) @@ -187,6 +189,12 @@ def delete(db, cfg, rid, by=None): orphans(db, speakers, faces) if acc.published(a): acc.activity(db, rid, "Delete") # harvesters drop it + try: + from . import events_content + + events_content.deleted(db, rid, rec, by) # asset.deleted, for automations and the timeline + except Exception: # noqa: BLE001 - the asset is gone either way; a missed activity mustn't undo that + log.exception("couldn't log the deletion of recording %s", rid) return store.clean({"title": rec.get("title"), "namespace": ns, "path": rec.get("path"), "access": a["access"]}) diff --git a/fastapi_backend/app/domain/events_content.py b/fastapi_backend/app/domain/events_content.py index 79b2d4532..0eedb69a8 100644 --- a/fastapi_backend/app/domain/events_content.py +++ b/fastapi_backend/app/domain/events_content.py @@ -364,10 +364,23 @@ def _label_of(r, first, upload): return first -def _activities(db, rows): - """The rows as activities of their kinds (a row whose created_at isn't a time is skipped).""" +def _asset(db, r, types): + """What every activity about an asset says of it: its type, id, title, link, media and content type.""" from . import render + media = render.kind(r) + return { + "type": TYPES.get(media, "Document"), + "id": f"lens:recording/{r['id']}", + "name": " ".join(str(r.get("title") or "").split()) or "Untitled", + "url": f"/resources/{r['id']}", + "media": media, + "content_type": _content_type(db, r, types), + } + + +def _activities(db, rows): + """The rows as activities of their kinds (a row whose created_at isn't a time is skipped).""" rows = [r for r in rows if r] ids = [r["id"] for r in rows] if not ids: @@ -389,19 +402,9 @@ def _activities(db, rows): published = events.stamp(r["created_at"]) except (TypeError, ValueError): continue - media = render.kind(r) - title = " ".join(str(r.get("title") or "").split()) or "Untitled" actor = who[labels[r["id"]]] - ctype = _content_type(db, r, types) - kind = _kind(r, ctype, (srcs.get((r.get("remote") or {}).get("source")) or {}).get("type"), actor) - obj = { - "type": TYPES.get(media, "Document"), - "id": f"lens:recording/{r['id']}", - "name": title, - "url": f"/resources/{r['id']}", - "media": media, - "content_type": ctype, - } + obj = _asset(db, r, types) + kind = _kind(r, obj["content_type"], (srcs.get((r.get("remote") or {}).get("source")) or {}).get("type"), actor) if kind == "email.received": obj.update(_mail(r)) elif kind == "calendar_event.added": @@ -643,3 +646,370 @@ def _child(name, label, props, untrusted, sample_actor, sample_obj, summary): {"type": "Document", "name": "Shopping list", "media": "transcript", "content_type": "text", "via": "hook"}, "Phone shortcut imported “Shopping list”", ) + + +# ---------- runs that finished ---------- +JOB_FIELDS = "id AS rid, record::id(id) AS id, recording, space, steps, step_index, status, error, created_by, finished_at, batch" +ASSET_FIELDS = "record::id(id) AS id, title, source, media, path, content_type, duration_ms, form" +JOB_KINDS = {"succeeded": "asset.processed", "failed": "asset.failed"} +JOB_ID_RX = re.compile(r"^(processed|failed)-job-(\d+)-(\d+)$") +TAGS = 10 # entities a processed asset is tagged with, the most mentioned first + + +def entity_tags(db, rids, most=TAGS): + """{recording: [Mention tags]}: each asset's entities, the most mentioned first, at most `most`, in two queries.""" + counts: dict = {} + if rids: + for m in db.rows( + "SELECT recording, entity, count() AS n FROM mentions WHERE recording IN $r GROUP BY recording, entity", r=list(rids) + ): + if _num(m.get("entity")) is not None: + counts.setdefault(m["recording"], []).append((-int(m.get("n") or 0), m["entity"])) + top = {rid: [e for _, e in sorted(c)[:most]] for rid, c in counts.items()} + names = _lookup(db, "entity", {e for es in top.values() for e in es}) + return { + rid: [ + {"type": "Mention", "id": f"lens:entity/{e}", "name": str(names[e].get("name"))[:200]} + for e in es + if names.get(e, {}).get("name") + ] + for rid, es in top.items() + } + + +def _step_names(steps): + return [s if isinstance(s, str) else str((s or {}).get("type") or "") for s in steps or []] + + +def _epoch(t): + return int(events._parse(t).timestamp()) + + +def _job_activities(db, rows): + """Finished runs as asset.processed (succeeded) and asset.failed, one per run, about their asset; a row whose + finished_at isn't a time is skipped.""" + rows = [r for r in rows if r and r.get("status") in JOB_KINDS] + if not rows: + return [] + rids = list({r["recording"] for r in rows if _num(r.get("recording")) is not None}) + assets = {a["id"]: a for a in db.rows(f"SELECT {ASSET_FIELDS} FROM recording WHERE id IN $r", r=[R("recording", i) for i in rids])} + who = actors(db, [(r.get("created_by"), r.get("space")) for r in rows]) + tags = entity_tags(db, [i for i in rids if i in assets]) + types = _content_types(db) + out = [] + for r in rows: + try: + published, n = events.stamp(r["finished_at"]), _epoch(r["finished_at"]) + except (TypeError, ValueError): + continue + kind, rec = JOB_KINDS[r["status"]], assets.get(r.get("recording")) + if rec: + obj = _asset(db, rec, types) + else: # deleted since + obj = {"type": "Document", "id": f"lens:recording/{r.get('recording')}", "name": "A deleted asset"} + steps = _step_names(r.get("steps")) + obj.update(job=f"lens:job/{r['id']}", steps=steps) + if r.get("batch") is not None: + obj["batch"] = f"lens:batch/{r['batch']}" + if kind == "asset.failed": + i = r.get("step_index") or 0 + obj.update( + step=steps[i] if isinstance(i, int) and 0 <= i < len(steps) else None, error=_text(r.get("error")) or "no reason given" + ) + obj = store.clean(obj) + title = obj["name"][:200] + if kind == "asset.processed": + summary = f"“{title}” finished processing" + (f" ({', '.join(steps)})" if steps else "") + else: + summary = f"Processing “{title}” failed" + (f" at {obj['step']}" if obj.get("step") else "") + out.append( + events.build( + f"{'processed' if kind == 'asset.processed' else 'failed'}-job-{r['id']}-{n}", + kind, + actor=who[(r.get("created_by"), r.get("space"))], + obj=obj, + summary=summary, + space=r.get("space"), + source="jobs", + pos=[r["finished_at"], r["id"]], + published=published, + tag=tags.get(r.get("recording")) or [], + ) + ) + return out + + +class JobsSource(events.Source): + """job rows that finished (succeeded or failed) as asset.processed and asset.failed, read by (finished_at, id). + A run tried again finishes again, as another activity: its id has the time it finished.""" + + name = "jobs" + + def _statuses(self, kinds, p): + want = sorted(st for st, k in JOB_KINDS.items() if kinds is None or k in kinds) + p["st"] = want + return "status IN $st" + + def read(self, db, cfg, pos, spaces, limit, hint=None): + p: dict = {} + where = f"{self._statuses(None, p)} AND {events.after_sql('finished_at', pos, p, table='job')} AND {events.space_sql(spaces, p)}" + rows = db.rows(f"SELECT {JOB_FIELDS} FROM job WHERE {where} ORDER BY finished_at, rid LIMIT {int(limit)}", **p) + return _job_activities(db, rows) + + def page(self, db, cfg, before, who, flt, limit): + p: dict = {} + where = [ + self._statuses(flt.get("kinds"), p), + events.space_sql(None if who.get("admin") else (who.get("spaces") or set()), p), + "finished_at != NONE", + ] + if before is not None: + where.append(events.after_sql("finished_at", before, p, "<", table="job")) + if flt.get("spaces") is not None: + p["fs"] = sorted(flt["spaces"]) + where.append("space IN $fs") + if flt.get("since"): + p["since"] = _sec(flt["since"]) + where.append("finished_at > $since") + if flt.get("until"): + p["until"] = _sec(flt["until"]) + where.append("finished_at <= $until") + rows = db.rows( + f"SELECT {JOB_FIELDS} FROM job WHERE {' AND '.join(where)} ORDER BY finished_at DESC, rid DESC LIMIT {int(limit)}", **p + ) + return _job_activities(db, rows) + + def get(self, db, cfg, activity_id): + m = JOB_ID_RX.match(activity_id) + if not m: + return None + r = db.one(f"SELECT {JOB_FIELDS} FROM $r", r=R("job", int(m.group(2)))) + got = _job_activities(db, [r]) if r else [] + return got[0] if got and got[0]["id"] == activity_id else None # an earlier try of a run tried again is gone + + def get_many(self, db, cfg, ids): + want = set(ids) + ns = {int(m.group(2)) for m in (JOB_ID_RX.match(i) for i in ids) if m} + if not ns: + return [] + rows = db.rows(f"SELECT {JOB_FIELDS} FROM job WHERE id IN $rs", rs=[R("job", n) for n in sorted(ns)]) + return [a for a in _job_activities(db, rows) if a["id"] in want] + + +events.register_source(JobsSource()) + + +# ---------- what no row of its own records: deletions, calendar moves ---------- +def deleted(db, rid, rec, by=None): + """Log that an asset was deleted (deletion.py calls it once the asset is gone): asset.deleted, by `by` (an email, + `automation:`, ...). Logged once per asset, whatever retries.""" + from . import render + + actor = actors(db, [(by, rec.get("space"))])[(by, rec.get("space"))] + title = " ".join(str(rec.get("title") or "").split()) or "Untitled" + media = render.kind(rec) + obj = {"type": TYPES.get(media, "Document"), "id": f"lens:recording/{int(rid)}", "name": title, "media": media} + return events.emit( + db, + "asset.deleted", + actor=actor, + obj=obj, + space=rec.get("space"), + summary=f"{actor['name']} deleted “{title[:200]}”", + key=f"deleted-{int(rid)}", + ) + + +def calendar_moved(db, rid, space, src, title, before, after): + """Log that a calendar event's time changed (sources.py calls it when an event it reads again starts at another + time than it did): calendar_event.changed, by the calendar it came from. Logged once per event and new start.""" + start = after.get("startTime") + if not start or not before.get("startTime") or before.get("startTime") == start: + return None + actor = {"type": "Service", "id": f"lens:source/{src['id']}", "name": src.get("name") or f"Source {src['id']}"} + title = " ".join(str(title or "").split()) or "Untitled" + obj = store.clean( + { + "type": "Document", + "id": f"lens:recording/{int(rid)}", + "name": title, + "url": f"/resources/{int(rid)}", + "startTime": start, + "endTime": after.get("endTime"), + "location": after.get("location"), + "calendar": after.get("calendar"), + "before": store.clean({"startTime": before.get("startTime"), "endTime": before.get("endTime")}), + } + ) + return events.emit( + db, + "calendar_event.changed", + actor=actor, + obj=obj, + space=space, + summary=f"{actor['name']} moved “{title[:200]}” to {str(start)[:16].replace('T', ' ')}", + key=f"cal-{after.get('uid') or rid}-{start}", + ) + + +# ---------- their kinds ---------- +JOB_OBJECT = { + "job": _s("Run"), + "steps": {"type": "array", "title": "Steps", "items": {"type": "string", "title": "Step"}}, + "batch": _s("Batch run"), +} +_ASSET = { + "type": _s("Object type", enum=["Audio", "Video", "Image", "Document"]), + "id": _s("Asset"), + "name": _s("Title"), + "url": _s("Link"), + "media": _s("Media", enum=["audio", "video", "image", "document", "transcript"]), + "content_type": _s("Content type"), +} + + +def _sample(aid, kind, typ, source, pos, actor, obj, summary, untrusted, tag=()): + return { + "id": aid, + "type": typ, + "kind": kind, + "published": "2026-10-15T16:24:10.000Z", + "actor": actor, + "object": obj, + "target": None, + "tag": list(tag), + "summary": summary, + "space": 3, + "lens": {"source": source, "pos": pos, "untrusted": list(untrusted), "cause": None}, + } + + +WORK_GMAIL = {"type": "Service", "id": "lens:source/12", "name": "Work Gmail"} +SCHOOL = {"type": "Mention", "id": "lens:entity/44", "name": "St Mary's School"} +events.kind( + "asset.processed", + label="Asset processed", + source="jobs", + as_type="Update", + object_type="Document", + fields={"object": {"type": "object", "properties": {**_ASSET, **JOB_OBJECT}}}, + untrusted=("object.name",), + sample=_sample( + "processed-job-51-1792081450", + "asset.processed", + "Update", + "jobs", + ["2026-10-15T16:24:10+00:00", 51], + WORK_GMAIL, + { + "type": "Document", + "id": "lens:recording/7", + "name": "School trip on Friday", + "url": "/resources/7", + "media": "document", + "content_type": "email", + "job": "lens:job/51", + "steps": ["pages", "analyze", "embed"], + }, + "“School trip on Friday” finished processing (pages, analyze, embed)", + ("object.name",), + [SCHOOL], + ), +) +events.kind( + "asset.failed", + label="Asset failed to process", + source="jobs", + as_type="Announce", + object_type="Document", + fields={"object": {"type": "object", "properties": {**_ASSET, **JOB_OBJECT, "step": _s("Failed at"), "error": _s("Error")}}}, + untrusted=("object.name", "object.error"), + sample=_sample( + "failed-job-52-1792081450", + "asset.failed", + "Announce", + "jobs", + ["2026-10-15T16:24:10+00:00", 52], + {"type": "Person", "id": "lens:account/1", "name": "Sam"}, + { + "type": "Audio", + "id": "lens:recording/8", + "name": "Call with the school", + "url": "/resources/8", + "media": "audio", + "job": "lens:job/52", + "steps": ["transcribe", "analyze"], + "step": "transcribe", + "error": "the transcription model isn't reachable", + }, + "Processing “Call with the school” failed at transcribe", + ("object.name", "object.error"), + ), +) +events.kind( + "asset.deleted", + label="Asset deleted", + source="log", + as_type="Delete", + object_type="Document", + fields={"object": {"type": "object", "properties": {k: _ASSET[k] for k in ("type", "id", "name", "media")}}}, + untrusted=("object.name",), + sample=_sample( + "log-12", + "asset.deleted", + "Delete", + "log", + ["2026-10-15T16:24:10+00:00", 12], + {"type": "Person", "id": "lens:account/1", "name": "Sam"}, + {"type": "Document", "id": "lens:recording/7", "name": "Old minutes", "media": "document"}, + "Sam deleted “Old minutes”", + ("object.name",), + ), +) +events.kind( + "calendar_event.changed", + label="Calendar event moved", + source="log", + as_type="Update", + object_type="Document", + fields={ + "object": { + "type": "object", + "properties": { + "id": _s("Asset"), + "name": _s("Title"), + "url": _s("Link"), + "startTime": _s("Starts", format="date-time"), + "endTime": _s("Ends", format="date-time"), + "location": _s("Where"), + "calendar": _s("Calendar"), + "before": { + "type": "object", + "title": "Before", + "properties": {"startTime": _s("Started", format="date-time"), "endTime": _s("Ended", format="date-time")}, + }, + }, + } + }, + untrusted=("object.name", "object.location"), + sample=_sample( + "log-13", + "calendar_event.changed", + "Update", + "log", + ["2026-10-15T16:24:10+00:00", 13], + {"type": "Service", "id": "lens:source/14", "name": "Family calendar"}, + { + "type": "Document", + "id": "lens:recording/9", + "name": "Parents' evening", + "url": "/resources/9", + "startTime": "2026-10-21T18:00:00+01:00", + "endTime": "2026-10-21T19:30:00+01:00", + "calendar": "Family calendar", + "before": {"startTime": "2026-10-20T18:00:00+01:00", "endTime": "2026-10-20T19:30:00+01:00"}, + }, + "Family calendar moved “Parents' evening” to 2026-10-21 18:00", + ("object.name", "object.location"), + ), +) diff --git a/fastapi_backend/app/domain/sources.py b/fastapi_backend/app/domain/sources.py index b7a19b0f3..1b433fd15 100644 --- a/fastapi_backend/app/domain/sources.py +++ b/fastapi_backend/app/domain/sources.py @@ -592,11 +592,16 @@ def _ingest(db, cfg, src, f, kind, space, by, steps=None, pipeline=None, collect ns = (db.one("SELECT name FROM $s", s=R("space", space)) or {})["name"] from . import keyring + old = None with keyring.plain_path(db, cfg, cached_copy(db, cfg, src["id"], f["path"], space)) as local: if feeds.handles(src) and not feeds.immutable(src): - _same_resource(db, space, src["id"], f["path"], ingest.fingerprint(local)) + old = _same_resource(db, space, src["id"], f["path"], ingest.fingerprint(local)) rid = ingest.import_transcript(db, cfg, ns, local, title=title, log=lambda *a: None, collection=collection) event = _event_of(local, f, src) + if old and event and isinstance(old.get("event"), dict): + from . import events_content + + events_content.calendar_moved(db, rid, space, src, title, old["event"], event) # calendar_event.changed db.q( "UPDATE $r MERGE $d", r=R("recording", rid), @@ -678,15 +683,17 @@ def _event_of(local, f, src): def _same_resource(db, space, sid, path, fp): """A calendar event that changed is read again into the resource it already is (found by where it came from), not - made a second one: that resource takes the new file's fingerprint, which the import then finds.""" + made a second one: that resource takes the new file's fingerprint, which the import then finds. Returns that + resource as it was (its id, fingerprint and event), or None.""" old = db.one( - "SELECT record::id(id) AS id, fingerprint FROM recording WHERE space = $sp AND remote.source = $s AND remote.path = $p LIMIT 1", + "SELECT record::id(id) AS id, fingerprint, event FROM recording WHERE space = $sp AND remote.source = $s AND remote.path = $p LIMIT 1", sp=space, s=sid, p=path, ) if old and old.get("fingerprint") != fp: db.q("UPDATE $r SET fingerprint = $f, fp_key = $k", r=R("recording", old["id"]), f=fp, k=f"{space}:{fp}") + return old def file_kind(cfg, name, documents_as="document"): diff --git a/fastapi_backend/tests/domain/test_events_content.py b/fastapi_backend/tests/domain/test_events_content.py index 05d9b9d0e..12f5820a0 100644 --- a/fastapi_backend/tests/domain/test_events_content.py +++ b/fastapi_backend/tests/domain/test_events_content.py @@ -10,11 +10,26 @@ import pytest -from app.domain import convert, events, events_content, feeds, gmail, import_hooks, ingest, sources, store, uploads, webcapture -from tests.helpers import make_user, write_wav +from app.domain import ( + convert, + deletion, + events, + events_content, + feeds, + gmail, + import_hooks, + ingest, + jobs, + sources, + store, + uploads, + webcapture, +) +from tests.helpers import PODS1, drain, make_user, write_wav R = store.R ADMIN = events.viewer(1, None, admin=True) +PODS = "\n".join(f"[00:{i:02d}] {line.split('|')[0]}: {line.split('|')[2]}" for i, line in enumerate(PODS1.splitlines())) ICS = """BEGIN:VCALENDAR\r VERSION:2.0\r BEGIN:VEVENT\r @@ -357,3 +372,88 @@ def test_paging_six_hundred_assets_of_one_second(db, cfg): commit() got = [a["id"] for a in first + rest] assert len(first) == 500 and len(got) == 605 and len(set(got)) == 605 + + +# ---------- runs, deletions, calendar moves ---------- +def test_finished_runs_are_processed_or_failed_with_the_assets_entities(db, cfg, monkeypatch): + sam = make_user(db, "sam@example.org", "sam password 1") + start(db, cfg, "test.runs") + rid = ingest.import_text(db, cfg, "pods", PODS, title="Capsid episode") + jid = jobs.enqueue(db, rid, ["analyze"], by="sam@example.org") + drain(db, cfg) + got = [a for a in new(db, cfg, "test.runs") if a["kind"] == "asset.processed"] + assert len(got) == 1 + a = got[0] + finished = db.one("SELECT finished_at FROM $r", r=R("job", jid))["finished_at"] + assert a["id"] == f"processed-job-{jid}-{int(events._parse(finished).timestamp())}" and a["type"] == "Update" + assert a["actor"] == {"type": "Person", "id": f"lens:account/{sam}", "name": "sam"} and a["space"] == store.ns_id(db, "pods") + assert a["object"]["id"] == f"lens:recording/{rid}" and a["object"]["job"] == f"lens:job/{jid}" and a["object"]["steps"] == ["analyze"] + assert a["summary"] == "“Capsid episode” finished processing (analyze)" + names = [t["name"] for t in a["tag"]] + assert "Dyno Therapeutics" in names and len(names) <= 10 and all(t["type"] == "Mention" for t in a["tag"]) + assert names[0] == "Dyno Therapeutics" # the most mentioned first + assert events.get(db, cfg, a["id"], ADMIN) == a + # a run that fails says where and why; tried again, it finishes again as another activity + monkeypatch.setitem(jobs.STEPS, "report", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("the report model isn't reachable"))) + bad = jobs.enqueue(db, rid, ["report"], by="automation:4") + drain(db, cfg) + fail = [x for x in new(db, cfg, "test.runs") if x["kind"] == "asset.failed"] + assert len(fail) == 1 and fail[0]["actor"]["id"] == "lens:automation/4" and fail[0]["type"] == "Announce" + assert fail[0]["object"]["step"] == "report" and "isn't reachable" in fail[0]["object"]["error"] + assert fail[0]["summary"] == "Processing “Capsid episode” failed at report" and "object.error" in fail[0]["lens"]["untrusted"] + first = fail[0]["id"] + time.sleep(1.05) # a second later, so the second try finishes at another time + jobs.retry(db, bad) + drain(db, cfg) + again = [x for x in new(db, cfg, "test.runs") if x["kind"] == "asset.failed"] + assert len(again) == 1 and again[0]["id"] != first and events.get(db, cfg, first, ADMIN) is None + assert events.get_many(db, cfg, [first, again[0]["id"], a["id"]], ADMIN) == {again[0]["id"]: again[0], a["id"]: a} + # the timeline narrows by kind in its query; a viewer of another namespace sees none of it + ids = [x["id"] for x in events.page(db, cfg, ADMIN, flt={"kinds": ["asset.failed"]})["items"]] + assert ids == [again[0]["id"]] + assert events.page(db, cfg, events.viewer(9, {store.ns_id(db, "calls")}), flt={"kinds": ["asset.processed"]})["items"] == [] + + +def test_a_deletion_is_logged_once_as_asset_deleted(db, cfg): + sam = make_user(db, "sam@example.org", "sam password 1") + pods = store.ns_id(db, "pods") + rid = ingest.import_text(db, cfg, "pods", "[00:00] Sam: Old minutes.", title="Old minutes") + start(db, cfg, "test.gone") + deletion.delete(db, cfg, rid, "sam@example.org") + got = new(db, cfg, "test.gone", kinds=["asset.deleted"]) + assert len(got) == 1 + a = got[0] + assert a["kind"] == "asset.deleted" and a["type"] == "Delete" and a["space"] == pods and a["id"].startswith("log-") + assert a["actor"] == {"type": "Person", "id": f"lens:account/{sam}", "name": "sam"} + assert a["object"] == {"type": "Document", "id": f"lens:recording/{rid}", "name": "Old minutes", "media": "transcript"} + assert a["summary"] == "sam deleted “Old minutes”" + # logged once per asset, whatever retries + assert events_content.deleted(db, rid, {"title": "Old minutes", "space": pods, "source": "transcript"}, "sam@example.org") == a["id"] + assert len(db.values("SELECT VALUE id FROM event_log WHERE kind = 'asset.deleted'")) == 1 + + +def test_an_event_that_moves_is_calendar_event_changed(db, cfg, monkeypatch): + pods = store.ns_id(db, "pods") + src = source(db, "Family calendar", "ical", user="") + body = {"now": ICS.encode()} + monkeypatch.setattr(feeds, "fetch", lambda cfg, src, path: body["now"]) + f = {"path": "evening.ics", "title": "Parents' evening", "size": len(ICS), "modified": store.now(), "when": None} + rid, _ = sources._ingest(db, cfg, src, f, "transcript", pods, "watch:1") + start(db, cfg, "test.moves") + # the same event, read again, starts a day later + body["now"] = ICS.replace("20261020T17", "20261021T17").replace("20261020T18", "20261021T18").encode() + again, _ = sources._ingest(db, cfg, src, {**f, "modified": "2026-10-16T08:00:00+00:00"}, "transcript", pods, "watch:1") + assert again == rid and db.one("SELECT event FROM $r", r=R("recording", rid))["event"]["startTime"] == "2026-10-21T17:00:00+00:00" + got = new(db, cfg, "test.moves", kinds=["calendar_event.changed"]) + assert len(got) == 1 + a = got[0] + assert a["actor"] == {"type": "Service", "id": f"lens:source/{src['id']}", "name": "Family calendar"} and a["type"] == "Update" + assert a["object"]["startTime"] == "2026-10-21T17:00:00+00:00" and a["object"]["endTime"] == "2026-10-21T18:30:00+00:00" + assert a["object"]["before"] == {"startTime": "2026-10-20T17:00:00+00:00", "endTime": "2026-10-20T18:30:00+00:00"} + assert a["summary"] == "Family calendar moved “Parents' evening” to 2026-10-21 17:00" and a["object"]["id"] == f"lens:recording/{rid}" + # read again unchanged, or the same move noticed twice: nothing more + sources._ingest(db, cfg, src, {**f, "modified": "2026-10-16T09:00:00+00:00"}, "transcript", pods, "watch:1") + old = {"startTime": "2026-10-20T17:00:00+00:00"} + new_ = db.one("SELECT event FROM $r", r=R("recording", rid))["event"] + assert events_content.calendar_moved(db, rid, pods, src, "Parents' evening", old, new_) == a["id"] + assert len(db.values("SELECT VALUE id FROM event_log WHERE kind = 'calendar_event.changed'")) == 1