From ecb3cc65c3ad1e9cf963c035e28a409a8a41690b Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 11:14:58 +0000 Subject: [PATCH 1/2] feat(playlists): keep moments as clips and play them back to back Playlists hold clips: a moment of any resource (t0 to t1, with a title, a note and the words picked), or a highlight's passage and label. A playlist is its maker's, private until shared; each reader gets only the clips of resources they can read, each with the resource's signed media link as the recording player gets it. The API adds, reorders and removes clips (Undo adds one back at its position), and agents list playlists and add moments over MCP, to the one used last by default. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- docs/api.md | 40 +++ docs/mcp.md | 7 +- fastapi_backend/app/api/mcp_tools.py | 120 ++++++++ fastapi_backend/app/api/v1/router.py | 2 + .../app/api/v1/routes/playlists.py | 238 ++++++++++++++++ fastapi_backend/app/domain/auth.py | 9 +- fastapi_backend/app/domain/deletion.py | 3 +- fastapi_backend/app/domain/playlists.py | 262 +++++++++++++++++ fastapi_backend/app/domain/store.py | 6 + fastapi_backend/app/schemas/playlists.py | 80 ++++++ fastapi_backend/tests/api/test_mcp.py | 1 + fastapi_backend/tests/api/test_playlists.py | 268 ++++++++++++++++++ nextjs-frontend/app/openapi-client/index.ts | 52 ++++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 180 ++++++++++++ threat-model.md | 1 + threat-model.yaml | 8 + 16 files changed, 1270 insertions(+), 7 deletions(-) create mode 100644 fastapi_backend/app/api/v1/routes/playlists.py create mode 100644 fastapi_backend/app/domain/playlists.py create mode 100644 fastapi_backend/app/schemas/playlists.py create mode 100644 fastapi_backend/tests/api/test_playlists.py diff --git a/docs/api.md b/docs/api.md index 9597585c..e3d0a897 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1845,6 +1845,46 @@ assets with audio or video that you can read) and when an app last read it, made `POST /{cid}/feed/token` gives it a new address and `DELETE /{cid}/feed` stops it ([A collection's podcast feed](feeds.md#a-collections-podcast-feed)). +## playlists + +Clips of moments of any resources, kept in order to play back to back. + +``` +GET /api/v1/playlists +POST /api/v1/playlists +GET /api/v1/playlists/{pid} +PATCH /api/v1/playlists/{pid} +DELETE /api/v1/playlists/{pid} +POST /api/v1/playlists/{pid}/items +PATCH /api/v1/playlists/{pid}/items/{iid} +DELETE /api/v1/playlists/{pid}/items/{iid} +PUT /api/v1/playlists/{pid}/order +``` + +A playlist is its maker's: a `name` unique among theirs (up to 120 characters), a `description`, and `shared`. A +private one is theirs alone; a shared one is read by everyone signed in, and each reader gets only the clips of +resources they have a role on (in the resource's namespace, or on its collection), whoever made it. Only its maker +changes it. Up to 200 each. Sharing and unsharing are audited (`playlist.share`, `playlist.unshare`). + +A clip is a moment of a resource you can read: `POST /{pid}/items` with the `recording` and `t0` and `t1` (ms from +the start; a document's pages have a reading-pace clock; without `t1`, or with one not after `t0`, it lasts 30 +seconds, and it stops at the end of the resource), a `title` (up to 200 characters), a `note` (up to 2,000) and the +`quote` picked in the text, at `position` (from 0; default the end). With `highlight` it clips that +[highlight](#highlights): its resource, passage and label, unless given; the clip stays when the highlight goes. +Clips aren't highlights: anyone who can read a resource keeps a clip of it, and only for themselves, where an editor +makes a highlight for everyone. Up to 500 in a playlist. `PATCH` changes a clip's title, note (empty clears them) or +times; `DELETE` takes it out, and Undo adds it again with its `position`. `PUT /{pid}/order` with `items`, clip ids +in their new order, puts those first, as named, and the clips not named after them, as they were. + +`GET /playlists` lists yours, the latest used first (the first is where a clip goes when none is named; adding, +changing, removing or reordering clips uses a playlist), then the ones others shared, each with `count` and +`duration_ms` of the clips you can read in it. `GET /{pid}` is everything a player needs: the clips in order, each +with its resource's `recording_title`, `namespace` and `media` kind, its signed `audio` link (the resource's audio or +video, as the recording player gets it; `null` when it has none, so play its text), its times and `link`, its page in +the web app at the clip's start. `hidden` counts the clips of your own playlist whose resources you can no longer +read. Clips go when their resource is deleted; private playlists go when their maker's account is removed, shared ones +stay. Agents use them through [MCP](mcp.md) (`list_playlists`, `add_to_playlist`). + ## views ``` diff --git a/docs/mcp.md b/docs/mcp.md index ae5cf9a5..f7a677f7 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -47,8 +47,9 @@ has the same tools, but sees only that namespace, whatever else the person who s that take a namespace use this one when none is named, `list_namespaces` and `list_files` show only it, and the tools about the server itself (`file_storage`, `monitoring`, `services`, `mcp_servers`, `sandbox_gateway`, `install_settings`, `mesh` and the ones that change them) aren't there, even for admins. Nor are the person's own conversations (`list_chats`, -`read_chat`, `rename_chat`, `export_chat`) and projects (`list_projects`, `read_project`, `create_project`, -`update_project`, `schedule_project`), which belong to no namespace. An admin counts as an owner of the namespace there. +`read_chat`, `rename_chat`, `export_chat`), projects (`list_projects`, `read_project`, `create_project`, +`update_project`, `schedule_project`) and playlists (`list_playlists`, `add_to_playlist`), which belong to no namespace. +An admin counts as an owner of the namespace there. It is off until an owner of the namespace turns it on: **Admin → Namespaces →** the namespace **→ Connect apps**, which also copies its address, or `PATCH /api/v1/namespaces//integrations` with `{"mcp": true}`, or an agent's @@ -97,6 +98,8 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `find_notes` | [notes](notes.md) by words in their title, summary or text, or by where they're filed (PARA), with the words around the match and links | | `read_note` | a note's Markdown with its links and backlinks, by id or as the page of a thing (`recording:12`) | | `write_note` | write a note or a thing's page, marked as written by an assistant. Needs a write-scope token and editor access | +| `list_playlists` | the person's [playlists](api.md#playlists), the latest used first, then the ones others shared, each with how many clips they can read in it and its link; with `playlist_id`, its clips in order, each with its resource, times, title, note and a link to that moment | +| `add_to_playlist` | keep a moment of a resource (or a highlight) as a clip with a title and a note, at the end of a playlist: by id, by name (made when there's none of that name) or the one used last (a new one called Clips when there's none). Needs a write-scope token | | `list_chats` | the person's conversations with Lens's [assistant](assistant.md#chat), newest first, by words in their title, each with its link | | `read_chat` | one of them: what it draws on, and the version showing (or the one ending at a `message`), each message with its versions, the tools an answer used, its citations with links and its record; `tree` adds every message's parent and role | | `rename_chat` | name one of them, or have Lens [title](assistant.md#titles) it from how it starts (no `title`). Needs a write-scope token | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index a324853e..9e29b310 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -36,6 +36,7 @@ from app.api.v1.routes import jobs as job_routes from app.api.v1.routes import namespaces as namespace_routes from app.api.v1.routes import notebook as note_routes +from app.api.v1.routes import playlists as playlist_routes from app.api.v1.routes import projects as project_routes from app.api.v1.routes import recordings as recording_routes from app.api.v1.routes import research as research_routes @@ -74,6 +75,7 @@ moondream, notebook, nsfw, + playlists, rdf, render, research, @@ -114,6 +116,7 @@ from app.schemas.entities import EntityHide, EntityList, EntityRename, EntityRetype, EntityUpdate from app.schemas.jobs import JobMove, JobsPick from app.schemas.notebook import NotePage +from app.schemas.playlists import ClipCreate from app.schemas.projects import ( ProjectCreate, ProjectScheduleCreate, @@ -1922,6 +1925,123 @@ def write_note( return {"note_id": pid, "url": _note_url(ctx, pid)} +# ---------- playlists: clips of moments, played back to back (docs/api.md#playlists) ---------- +def _playlist_url(ctx: Context, pid: int) -> str: + return f"{ctx.web}/playlists/{pid}" + + +@tool( + "list_playlists", + "List playlists", + "Your playlists, the latest used first (the first is where add_to_playlist puts a clip by default), then the ones " + "others shared: each one's id, name, how many clips you can read in it and how long they play, and its link. With " + "playlist_id, one playlist and its clips in order: each clip's resource, times (ms), title, note and a link to that " + "moment. Only clips of resources you can read.", + Arg("playlist_id", "integer", "one playlist, with its clips"), + personal=True, +) +def list_playlists(ctx: Context, playlist_id: int | None) -> dict[str, Any]: + def head(p: Any) -> dict[str, Any]: + return store.clean( + { + "playlist_id": p.id, + "name": p.name, + "description": p.description, + "shared": p.shared or None, + "mine": p.mine, + "made_by": None if p.mine else p.created_by_name or p.created_by, + "clips": p.count, + "duration_ms": p.duration_ms, + "url": _playlist_url(ctx, p.id), + } + ) + + if playlist_id is None: + return {"playlists": [head(p) for p in playlist_routes.list_playlists(ctx.user, ctx.acl, ctx.db)]} + d = playlist_routes.get_playlist(playlist_id, ctx.user, ctx.acl, ctx.db, ctx.cfg) + return { + **head(d), + "items": [ + store.clean( + { + "clip_id": c.id, + "recording_id": c.recording, + "recording_title": c.recording_title, + "namespace": c.namespace, + "t0": c.t0, + "t1": c.t1, + "title": c.title, + "note": c.note, + "url": ctx.link(c.recording, c.t0), + } + ) + for c in d.items + ], + } + + +@tool( + "add_to_playlist", + "Add a clip to a playlist", + "Keep a moment of a resource the person can read as a clip (from t0 to t1 in ms, with a title and a note) at the " + "end of one of their playlists: playlist_id, or a playlist by its name (made when they have none of that name), " + "or by default the one they used last (a new one called Clips when they have none). Give recording_id with t0, " + "or a highlight_id to clip that highlight. Needs a token with the write scope.", + Arg("recording_id", "integer", "the resource (from search or list_recordings)"), + Arg("t0", "integer", "where the clip starts, in ms from the start", minimum=0), + Arg("t1", "integer", "where it ends, in ms (default: 30 seconds on)", minimum=0), + Arg("title", "string", "what it is, in a few words", max_length=200), + Arg("note", "string", "a note on it", max_length=2000), + Arg("highlight_id", "integer", "clip this highlight instead: its resource, passage and label"), + Arg("playlist_id", "integer", "the playlist (from list_playlists)"), + Arg("playlist", "string", "or the playlist's name", max_length=120), + writes=True, + personal=True, +) +def add_to_playlist( + ctx: Context, + recording_id: int | None, + t0: int | None, + t1: int | None, + title: str | None, + note: str | None, + highlight_id: int | None, + playlist_id: int | None, + playlist: str | None, +) -> dict[str, Any]: + if not ctx.user.can_write: + raise ToolError("this token is read-only: ask for a token with the write scope to add clips") + if recording_id is None and highlight_id is None: + raise ToolError("give recording_id with t0, or highlight_id") + if recording_id is not None and highlight_id is None and t0 is None: + raise ToolError("give t0, where the clip starts") + if recording_id is not None: + ctx.acl.recording(recording_id) # one the person can read, before any playlist is made + if playlist_id is None: + mine = [p for p in playlists.visible(ctx.db, ctx.user.id) if p["account"] == ctx.user.id] + if playlist: + same = next((p for p in mine if p["name"].lower() == " ".join(playlist.split()).lower()), None) + playlist_id = same["id"] if same else playlists.create(ctx.db, ctx.user.id, playlist) + else: + playlist_id = playlists.default_for(ctx.db, ctx.user.id) + body = ClipCreate(recording=recording_id, highlight=highlight_id, t0=t0, t1=t1, title=title, note=note) + c = playlist_routes.add_clip(playlist_id, body, ctx.user, ctx.acl, ctx.db, ctx.cfg) + name = playlists.get(ctx.db, playlist_id)["name"] + return store.clean( + { + "clip_id": c.id, + "playlist_id": playlist_id, + "playlist": name, + "position": c.position, + "recording_id": c.recording, + "t0": c.t0, + "t1": c.t1, + "title": c.title, + "url": _playlist_url(ctx, playlist_id), + } + ) + + # ---------- the person's conversations with Lens's assistant (docs/assistant.md, "Copy and export") ---------- @tool( "list_chats", diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index a97bf809..6fe96f04 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -54,6 +54,7 @@ oauth, passkeys, pipelines, + playlists, podcasts, projects, public, @@ -152,6 +153,7 @@ research, browser, collections, + playlists, batches, public, requests, diff --git a/fastapi_backend/app/api/v1/routes/playlists.py b/fastapi_backend/app/api/v1/routes/playlists.py new file mode 100644 index 00000000..38580446 --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/playlists.py @@ -0,0 +1,238 @@ +"""Playlists: clips of resources (a moment from t0 to t1, with a title and a note) in order, to play back to back +(docs/api.md#playlists). + +A playlist is its maker's, private until shared. Whoever reads it gets only the clips of resources they have a role +on (in the resource's namespace, or on its collection): share links and signed links don't reach them. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, HTTPException + +from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Principal, Writer, domain_errors +from app.api.media import sign_urls +from app.api.v1.routes.recordings import audio_link +from app.domain import auth, highlights, playlists, store +from app.domain.store import DB +from app.schemas.common import Ok +from app.schemas.playlists import Clip, ClipCreate, ClipUpdate, Playlist, PlaylistCreate, PlaylistDetail, PlaylistOrder, PlaylistUpdate + +R = store.R +router = APIRouter(prefix="/playlists", tags=["playlists"]) + + +def _playlist(db: DB, pid: int, user: Principal, write: bool = False) -> dict[str, Any]: + """A playlist this person may read (theirs, or shared) or, with write, change (theirs only); else 404.""" + try: + p = playlists.get(db, pid) + except KeyError: + raise HTTPException(404, "not found") from None + if p["account"] != user.id and (write or not p["shared"]): + raise HTTPException(404, "not found") + return p + + +def _clip(db: DB, pid: int, iid: int) -> dict[str, Any]: + try: + i = playlists.get_item(db, iid) + except KeyError: + raise HTTPException(404, "not found") from None + if i["playlist"] != pid: + raise HTTPException(404, "not found") + return i + + +def _readable(acl: Access, db: DB, rids: set[int]) -> dict[int, dict[str, Any]]: + """The resources among these this person has a role on, by id.""" + if not rids: + return {} + rows = db.rows( + "SELECT record::id(id) AS id, space, collection, title, duration_ms, media, source FROM recording WHERE id IN $ids", + ids=[R("recording", r) for r in sorted(rids)], + ) + return {r["id"]: r for r in rows if acl.rank_in(r["space"], r.get("collection"))} + + +def _resource(acl: Access, db: DB, rid: int) -> dict[str, Any]: + """A resource to clip: one this person has a role on, else 404.""" + rec = db.one("SELECT space, collection, duration_ms FROM $r", r=R("recording", rid)) + if not rec: + raise HTTPException(404, "not found") + acl.need_in(rec["space"], rec.get("collection")) + return rec + + +def _people(db: DB, ids: set[int]) -> dict[int, dict[str, Any]]: + if not ids: + return {} + rows = db.rows("SELECT record::id(id) AS id, email, name FROM account WHERE id IN $ids", ids=[R("account", a) for a in ids]) + return {a["id"]: a for a in rows} + + +def _kind(rec: dict[str, Any]) -> str: + return (rec.get("media") or {}).get("kind") or ("audio" if rec.get("source") == "audio" else "transcript") + + +def _summaries(acl: Access, db: DB, rows: list[dict[str, Any]], user: Principal) -> list[Playlist]: + held = playlists.items_of(db, [p["id"] for p in rows]) + recs = _readable(acl, db, {i["recording"] for items in held.values() for i in items}) + people = _people(db, {p["account"] for p in rows}) + out = [] + for p in rows: + mine = [i for i in held[p["id"]] if i["recording"] in recs] + by = people.get(p["account"], {}) + out.append( + Playlist( + id=p["id"], + name=p["name"], + description=p.get("description"), + shared=p["shared"], + mine=p["account"] == user.id, + created_by=by.get("email"), + created_by_name=by.get("name") or None, + count=len(mine), + duration_ms=sum(i["t1"] - i["t0"] for i in mine), + created_at=p.get("created_at"), + updated_at=p.get("updated_at"), + ) + ) + return out + + +def _detail(acl: Access, db: DB, cfg: dict[str, Any], p: dict[str, Any], user: Principal) -> PlaylistDetail: + """The playlist with its clips the reader may read, each with the resource's signed media link, as the player gets it.""" + held = playlists.items(db, p["id"]) + recs = _readable(acl, db, {i["recording"] for i in held}) + names = store.space_names(db) + media: dict[int, str | None] = {rid: audio_link(db, cfg, rid) for rid in recs} + items = [] + for i in (i for i in held if i["recording"] in recs): + rec = recs[i["recording"]] + items.append( + { + **{ + k: i.get(k) + for k in ("id", "playlist", "position", "recording", "t0", "t1", "title", "note", "quote", "highlight", "created_at") + }, + "recording_title": rec.get("title"), + "namespace": names.get(rec["space"]), + "media": _kind(rec), + "audio": media[i["recording"]], + "link": f"/resources/{i['recording']}" + (f"?t={i['t0'] // 1000}" if i["t0"] >= 1000 else ""), + } + ) + sign_urls(items, full=True) # each one's resource is one the reader has a role on + summary = _summaries(acl, db, [p], user)[0] + mine = p["account"] == user.id + return PlaylistDetail( + **summary.model_dump(), items=[Clip.model_validate(x) for x in items], hidden=len(held) - len(items) if mine else 0 + ) + + +@router.get("") +def list_playlists(user: CurrentUser, acl: Acl, db: Db) -> list[Playlist]: + """Your playlists, the latest used first (the first is where a clip goes when none is named), then the ones others + shared; each counts only the clips of resources you can read.""" + return _summaries(acl, db, playlists.visible(db, user.id), user) + + +@router.post("") +def create_playlist(body: PlaylistCreate, user: Writer, acl: Acl, db: Db) -> Playlist: + """Make a playlist, with a name unique among yours. Up to 200 each.""" + with domain_errors(): + pid = playlists.create(db, user.id, body.name, body.description, body.shared) + if body.shared: + auth.audit(db, user.as_audit(), "playlist.share", f"playlist:{pid}", {"name": body.name}) + return _summaries(acl, db, [playlists.get(db, pid)], user)[0] + + +@router.get("/{pid}") +def get_playlist(pid: int, user: CurrentUser, acl: Acl, db: Db, cfg: Cfg) -> PlaylistDetail: + """The playlist and its clips in order, each with what a player needs: the resource's kind, its signed audio or + video link (null when it has none: play its text), the clip's times and its page's link. Only clips of + resources you can read; `hidden` counts the others in your own playlist.""" + return _detail(acl, db, cfg, _playlist(db, pid, user), user) + + +@router.patch("/{pid}") +def update_playlist(pid: int, body: PlaylistUpdate, user: Writer, acl: Acl, db: Db) -> Playlist: + """Rename your playlist, change its description, or share or unshare it. Sharing and unsharing are audited + (`playlist.share`, `playlist.unshare`).""" + p = _playlist(db, pid, user, write=True) + with domain_errors(): + playlists.update(db, pid, body.name, body.description, body.shared) + if body.shared is not None and body.shared != p["shared"]: + auth.audit(db, user.as_audit(), "playlist.share" if body.shared else "playlist.unshare", f"playlist:{pid}", {"name": p["name"]}) + return _summaries(acl, db, [playlists.get(db, pid)], user)[0] + + +@router.delete("/{pid}") +def delete_playlist(pid: int, user: Writer, db: Db) -> Ok: + """Delete your playlist and its clips.""" + _playlist(db, pid, user, write=True) + playlists.delete(db, pid) + return Ok() + + +@router.post("/{pid}/items") +def add_clip(pid: int, body: ClipCreate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Clip: + """Add a clip to your playlist: a resource you can read (`recording`) from `t0` to `t1`, or a `highlight` on one + (its passage and label, unless given), with a `title` and a `note`, at `position` (default: the end). Up to 500 + in a playlist.""" + _playlist(db, pid, user, write=True) + rid, t0, t1, title, quote = body.recording, body.t0, body.t1, body.title, body.quote + if body.highlight is not None: + try: + h = highlights.get(db, body.highlight) + except KeyError: + raise HTTPException(404, "not found") from None + _resource(acl, db, h["recording"]) + if rid is not None and rid != h["recording"]: + raise HTTPException(400, "that highlight is on another resource") + rid = h["recording"] + if t0 is None: + t0, t1 = h["t0"], h["t1"] if t1 is None else t1 + title, quote = title if title is not None else h.get("label") or h.get("quote"), quote or h.get("quote") + if rid is None: + raise HTTPException(400, "give the resource (recording) or a highlight") + rec = _resource(acl, db, rid) + with domain_errors(): + iid = playlists.add( + db, pid, user.id, rid, t0, t1, (title or "")[:200], body.note, quote, body.highlight, body.position, rec.get("duration_ms") + ) + return _one(acl, db, cfg, pid, iid, user) + + +def _one(acl: Access, db: DB, cfg: dict[str, Any], pid: int, iid: int, user: Principal) -> Clip: + return next(c for c in _detail(acl, db, cfg, playlists.get(db, pid), user).items if c.id == iid) + + +@router.patch("/{pid}/items/{iid}") +def update_clip(pid: int, iid: int, body: ClipUpdate, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> Clip: + """Change a clip's title or note (empty clears it), or its times.""" + _playlist(db, pid, user, write=True) + i = _clip(db, pid, iid) + rec = _resource(acl, db, i["recording"]) + with domain_errors(): + playlists.update_item(db, iid, body.title, body.note, body.t0, body.t1, rec.get("duration_ms")) + return _one(acl, db, cfg, pid, iid, user) + + +@router.delete("/{pid}/items/{iid}") +def remove_clip(pid: int, iid: int, user: Writer, db: Db) -> Ok: + """Take a clip out of your playlist. To undo, add it again with its `position`.""" + _playlist(db, pid, user, write=True) + _clip(db, pid, iid) + playlists.remove_item(db, iid) + return Ok() + + +@router.put("/{pid}/order") +def reorder_playlist(pid: int, body: PlaylistOrder, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> PlaylistDetail: + """Put your playlist's clips in this order (their ids); clips not named follow, as they were.""" + p = _playlist(db, pid, user, write=True) + with domain_errors(): + playlists.reorder(db, pid, body.items) + return _detail(acl, db, cfg, p, user) diff --git a/fastapi_backend/app/domain/auth.py b/fastapi_backend/app/domain/auth.py index 2ac29e5c..b8ad2f9a 100644 --- a/fastapi_backend/app/domain/auth.py +++ b/fastapi_backend/app/domain/auth.py @@ -159,17 +159,18 @@ def update_account(db, uid, name=None, admin=None, disabled=None, password=None) def remove_account(db, uid, lose_vaults=False): """Remove someone for good: every way they sign in (sessions, passkeys, sign-in links, connected accounts, API tokens, apps they gave access to, their collections' podcast feed addresses), every role and permission they had, - and what only they could see (their chats, their own notes, highlights, saved views and collections, circles, what - was shared with them and what agents asked, unfinished uploads). What they shared stays (notes and views for - everyone, comments, shares with circles), under their name: the account is kept as a marker without an email, so their + and what only they could see (their chats, their own notes, highlights, saved views, collections and playlists, + circles, what was shared with them and what agents asked, unfinished uploads). What they shared stays (notes, views + and playlists for everyone, comments, shares with circles), under their name: the account is kept as a marker without an email, so their address can be invited again. Refused (ValueError) when their passkeys are the only way into a vault, unless `lose_vaults`.""" - from . import passkeys + from . import passkeys, playlists u = get_account(db, uid) if not u or u.get("removed_at"): raise KeyError(uid) passkeys.remove_all(db, uid, lose_vaults) + playlists.remove_account(db, uid) for g in db.values("SELECT VALUE record::id(id) FROM oauth_grant WHERE account = $a", a=uid): db.run(["DELETE oauth_token WHERE gid = $g", "DELETE $r"], g=g, r=R("oauth_grant", g)) chats = db.values("SELECT VALUE record::id(id) FROM chat WHERE account = $a", a=uid) diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py index 4846cc37..15c5253d 100644 --- a/fastapi_backend/app/domain/deletion.py +++ b/fastapi_backend/app/domain/deletion.py @@ -4,7 +4,7 @@ What goes: its transcript and everything derived from it (segments, speakers' appearances, chapters, entities' mentions, terms, edits), video shots, text on screen and face tracks with their frames, the pages drawn of documents and images, outputs (and the words of them search finds) and reports, shares, -notes, comments, highlights, supplementary files, permissions, requests for access, and its place in IP groups, fixed collections, chat +notes, comments, highlights, clips in playlists, supplementary files, permissions, requests for access, and its place in IP groups, fixed collections, chat scopes and batch runs that haven't started it. Speakers and faces that only it had, and that nobody named, go too; named ones stay. Its jobs are cancelled (a job that is running has to stop first), and harvesters hear a Delete when it was public. @@ -47,6 +47,7 @@ "note", "comment", "highlight", + "playlist_item", "permission", "access_request", "resource_file", diff --git a/fastapi_backend/app/domain/playlists.py b/fastapi_backend/app/domain/playlists.py new file mode 100644 index 00000000..b5f25847 --- /dev/null +++ b/fastapi_backend/app/domain/playlists.py @@ -0,0 +1,262 @@ +"""Playlists (docs/api.md#playlists): clips, moments of resources kept under a title, gathered in order to play back to +back. + +A clip is an item of a playlist: a resource and a time range of it in ms (a document's pages have a reading-pace +clock), with a title and a note. It may come from a highlight, whose passage and label it copies (and keeps the +highlight's id); it stays when the highlight goes. Clips are not highlights: anyone who can read a resource keeps a +clip of it, and a clip is its maker's, where a highlight is made by an editor for everyone. + +A playlist is its maker's, private until shared; a shared one is read by everyone signed in, and each reader gets +only the clips of resources they can read, whoever made it. Only its maker changes it. Clips go when their resource +is deleted; the playlist stays. +""" + +from __future__ import annotations + +from . import notes, store + +R = store.R +MAX_PLAYLISTS = 200 # per person +MAX_ITEMS = 500 # in one playlist +NAME_MAX = 120 +DESCRIPTION_MAX = 2000 +TITLE_MAX = 200 +NOTE_MAX = 2000 +DEFAULT_NAME = "Clips" # the playlist a clip goes in when you have none yet +DEFAULT_LENGTH_MS = 30_000 # a clip of a moment (no end given) lasts this long, or to the end of the resource +FIELDS = "record::id(id) AS id, account, name, description, shared ?? false AS shared, used ?? 0 AS used, created_at, updated_at" +ITEM_FIELDS = "record::id(id) AS id, playlist, account, recording, t0, t1, title, note, quote, highlight, position, created_at, updated_at" + + +def _text(value, most, what): + value = " ".join(str(value or "").split()) + if len(value) > most: + raise ValueError(f"A {what} can have up to {most} characters.") + return value or None + + +def _name(name): + name = _text(name, NAME_MAX, "playlist's name") + if not name: + raise ValueError("Give the playlist a name.") + return name + + +def _note(note): + note = str(note or "").strip() + if len(note) > NOTE_MAX: + raise ValueError(f"A clip's note can have up to {NOTE_MAX} characters.") + return note or None + + +def span(t0, t1=None, duration_ms=None): + """A clip's time range (t0, t1) in ms: t1 is DEFAULT_LENGTH_MS after t0 when not given or not after it, and stops + at the end of the resource when its length is known. ValueError for one that ends before it starts or starts past + the end.""" + if t0 is None: + raise ValueError("A clip needs the moment it starts (t0).") + t0 = int(t0) + if t1 is not None and int(t1) < t0: + raise ValueError("A clip has to end after it starts.") + if t0 < 0: + raise ValueError("A clip starts at 0 or later.") + if duration_ms and t0 > duration_ms: + raise ValueError("That moment is past the end of the resource.") + t1 = int(t1) if t1 is not None and int(t1) > t0 else t0 + DEFAULT_LENGTH_MS + return t0, min(t1, int(duration_ms)) if duration_ms else t1 + + +# ---------- playlists ---------- +def get(db, pid): + """The playlist, or KeyError.""" + p = db.one(f"SELECT {FIELDS} FROM $r", r=R("playlist", int(pid))) + if not p: + raise KeyError(pid) + return p + + +def visible(db, account): + """Your playlists, the latest used first (the first is the one a clip goes in by default), then the ones others + shared, the latest used first.""" + rows = db.rows(f"SELECT {FIELDS} FROM playlist WHERE account = $a OR shared = true", a=account) + rows.sort(key=lambda p: (p["used"], p["id"]), reverse=True) + return sorted(rows, key=lambda p: p["account"] != account) + + +def last_used(db, account): + """Your latest used playlist, or None.""" + mine = [p for p in visible(db, account) if p["account"] == account] + return mine[0] if mine else None + + +def create(db, account, name, description=None, shared=False): + """A new playlist; its id. ValueError for no name, a name you use already, or too many playlists.""" + name = _name(name) + mine = db.rows("SELECT name FROM playlist WHERE account = $a", a=account) + if len(mine) >= MAX_PLAYLISTS: + raise ValueError(f"You have {MAX_PLAYLISTS} playlists already; delete one first.") + if any(p["name"].lower() == name.lower() for p in mine): + raise ValueError(f"You have a playlist called {name} already.") + pid = db.next_id("playlist") + t = store.now() + row = { + "account": account, + "name": name, + "description": _text(description, DESCRIPTION_MAX, "playlist's description"), + "shared": bool(shared), + "used": db.next_id("playlist_used"), + "created_at": t, + "updated_at": t, + } + db.q("CREATE $r CONTENT $d", r=R("playlist", pid), d=store.clean(row)) + return pid + + +def default_for(db, account): + """The playlist a clip goes in when none is named: your latest used one, or a new one called Clips.""" + p = last_used(db, account) + return p["id"] if p else create(db, account, DEFAULT_NAME) + + +def update(db, pid, name=None, description=None, shared=None): + """Rename it, change its description (an empty one clears it), or share or unshare it.""" + p = get(db, pid) + sets, v = ["updated_at = $t"], {"t": store.now()} + if name is not None: + name = _name(name) + others = db.values("SELECT VALUE name FROM playlist WHERE account = $a AND id != $r", a=p["account"], r=R("playlist", int(pid))) + if any(n.lower() == name.lower() for n in others): + raise ValueError(f"You have a playlist called {name} already.") + sets.append("name = $n") + v["n"] = name + if description is not None: + sets.append("description = $d") + v["d"] = _text(description, DESCRIPTION_MAX, "playlist's description") + if shared is not None: + sets.append("shared = $s") + v["s"] = bool(shared) + db.q(f"UPDATE $r SET {', '.join(sets)}", r=R("playlist", int(pid)), **v) + + +def delete(db, pid): + """The playlist and its clips.""" + db.run(["DELETE playlist_item WHERE playlist = $p", "DELETE $r"], p=int(pid), r=R("playlist", int(pid))) + + +def _touch(db, pid): + """It was just used: the next clip goes in it by default (`used` counts, so two changes in a second keep their order).""" + db.q("UPDATE $r SET updated_at = $t, used = $u", r=R("playlist", int(pid)), t=store.now(), u=db.next_id("playlist_used")) + + +# ---------- clips ---------- +def items(db, pid): + """The playlist's clips, in order.""" + rows = db.rows(f"SELECT {ITEM_FIELDS} FROM playlist_item WHERE playlist = $p", p=int(pid)) + return sorted(rows, key=lambda i: (i.get("position") or 0, i["id"])) + + +def items_of(db, pids): + """{playlist: its clips, in order} for several playlists at once.""" + out: dict[int, list] = {int(p): [] for p in pids} + if not out: + return out + for i in sorted( + db.rows(f"SELECT {ITEM_FIELDS} FROM playlist_item WHERE playlist IN $p", p=sorted(out)), + key=lambda i: (i.get("position") or 0, i["id"]), + ): + out[i["playlist"]].append(i) + return out + + +def get_item(db, iid): + """The clip, or KeyError.""" + i = db.one(f"SELECT {ITEM_FIELDS} FROM $r", r=R("playlist_item", int(iid))) + if not i: + raise KeyError(iid) + return i + + +def _renumber(db, ordered): + """Positions 0, 1, 2… in this order, writing only the ones that change.""" + for n, i in enumerate(ordered): + if i.get("position") != n: + db.q("UPDATE $r SET position = $n", r=R("playlist_item", i["id"]), n=n) + + +def add(db, pid, account, rid, t0, t1=None, title=None, note=None, quote=None, highlight=None, position=None, duration_ms=None): + """Add a clip of resource `rid` from t0 to t1 (see span) at `position` (from 0; default the end); its id. + ValueError for a moment that doesn't fit the resource, a title or note too long, or a full playlist.""" + t0, t1 = span(t0, t1, duration_ms) + title, note = _text(title, TITLE_MAX, "clip's title"), _note(note) + now = items(db, pid) + if len(now) >= MAX_ITEMS: + raise ValueError(f"This playlist has {MAX_ITEMS} clips already; remove one first.") + at = len(now) if position is None else max(0, min(int(position), len(now))) + iid = db.next_id("playlist_item") + t = store.now() + row = { + "playlist": int(pid), + "account": account, + "recording": int(rid), + "t0": t0, + "t1": t1, + "title": title, + "note": note, + "quote": " ".join(str(quote or "").split())[: notes.QUOTE_MAX] or None, + "highlight": int(highlight) if highlight is not None else None, + "position": at, + "created_at": t, + "updated_at": t, + } + db.q("CREATE $r CONTENT $d", r=R("playlist_item", iid), d=store.clean(row)) + _renumber(db, [*now[:at], {"id": iid, "position": at}, *now[at:]]) + _touch(db, pid) + return iid + + +def update_item(db, iid, title=None, note=None, t0=None, t1=None, duration_ms=None): + """Change its title or note (an empty one clears it), or its time range.""" + i = get_item(db, iid) + sets, v = ["updated_at = $t"], {"t": store.now()} + if title is not None: + sets.append("title = $ti") + v["ti"] = _text(title, TITLE_MAX, "clip's title") + if note is not None: + sets.append("note = $no") + v["no"] = _note(note) + if t0 is not None or t1 is not None: + a, b = span(i["t0"] if t0 is None else t0, i["t1"] if t1 is None else t1, duration_ms) + sets += ["t0 = $a", "t1 = $b"] + v.update(a=a, b=b) + db.q(f"UPDATE $r SET {', '.join(sets)}", r=R("playlist_item", int(iid)), **v) + _touch(db, i["playlist"]) + + +def remove_item(db, iid): + i = get_item(db, iid) + db.q("DELETE $r", r=R("playlist_item", int(iid))) + _renumber(db, items(db, i["playlist"])) + _touch(db, i["playlist"]) + + +def reorder(db, pid, order): + """Put the clips in this order (their ids): the ones named first, as named, then any others as they were. + ValueError for an id that isn't one of the playlist's clips, or one named twice.""" + now = items(db, pid) + have = {i["id"]: i for i in now} + order = [int(x) for x in order] + if len(set(order)) != len(order): + raise ValueError("Name each clip once.") + stray = [x for x in order if x not in have] + if stray: + raise ValueError(f"Clip {stray[0]} isn't in this playlist.") + named = set(order) + _renumber(db, [have[x] for x in order] + [i for i in now if i["id"] not in named]) + _touch(db, pid) + + +def remove_account(db, account): + """What goes when someone is removed: their playlists that weren't shared, with the clips in them.""" + gone = db.values("SELECT VALUE record::id(id) FROM playlist WHERE account = $a AND (shared ?? false) = false", a=account) + if gone: + db.run(["DELETE playlist_item WHERE playlist IN $p", "DELETE playlist WHERE id IN $r"], p=gone, r=[R("playlist", p) for p in gone]) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index db550def..6dd5a337 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -1800,6 +1800,12 @@ def close(self): # passages of resources marked in colour by their editors (app/domain/highlights.py) "DEFINE TABLE IF NOT EXISTS highlight SCHEMALESS", "DEFINE INDEX IF NOT EXISTS highlight_rec ON highlight FIELDS recording", + # people's playlists, and the clips in them: moments of resources in order (app/domain/playlists.py) + "DEFINE TABLE IF NOT EXISTS playlist SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS playlist_owner ON playlist FIELDS account", + "DEFINE TABLE IF NOT EXISTS playlist_item SCHEMALESS", + "DEFINE INDEX IF NOT EXISTS playlist_item_list ON playlist_item FIELDS playlist", + "DEFINE INDEX IF NOT EXISTS playlist_item_rec ON playlist_item FIELDS recording", # custom metadata fields defined on namespaces and collections (app/domain/fields.py); key: ":::" "DEFINE TABLE IF NOT EXISTS field SCHEMALESS", "DEFINE INDEX IF NOT EXISTS field_space ON field FIELDS space", diff --git a/fastapi_backend/app/schemas/playlists.py b/fastapi_backend/app/schemas/playlists.py new file mode 100644 index 00000000..14928382 --- /dev/null +++ b/fastapi_backend/app/schemas/playlists.py @@ -0,0 +1,80 @@ +"""Playlists: clips (moments of resources, with a title and a note) in order, to play back to back. Private unless +shared; a reader gets only the clips of resources they can read.""" + +from __future__ import annotations + +from pydantic import Field + +from app.schemas.common import RequestModel, ResponseModel + + +class PlaylistCreate(RequestModel): + name: str = Field(min_length=1, max_length=120, description="unique among your playlists") + description: str | None = Field(None, max_length=2000) + shared: bool = Field(False, description="everyone signed in reads it, each seeing only the clips of resources they can read") + + +class PlaylistUpdate(RequestModel): + name: str | None = Field(None, min_length=1, max_length=120) + description: str | None = Field(None, max_length=2000, description="empty clears it") + shared: bool | None = None + + +class Playlist(ResponseModel): + id: int + name: str + description: str | None = None + shared: bool = False + mine: bool = Field(description="you made it: only you change it") + created_by: str | None = Field(None, description="who made it: their email") + created_by_name: str | None = None + count: int = Field(description="its clips of resources you can read") + duration_ms: int = Field(description="how long those clips play, back to back") + created_at: str | None = None + updated_at: str | None = None + + +class ClipCreate(RequestModel): + recording: int | None = Field(None, description="the resource the clip is of (or give highlight)") + highlight: int | None = Field(None, description="clip a highlight: its resource, passage and label, unless given here") + t0: int | None = Field(None, ge=0, description="where the clip starts, in ms from the start (required without highlight)") + t1: int | None = Field(None, ge=0, description="where it ends, in ms (default: 30 seconds on, or the end of the resource)") + title: str | None = Field(None, max_length=200, description="what it is, in a few words") + note: str | None = Field(None, max_length=2000) + quote: str | None = Field(None, max_length=1000, description="the words picked in the text") + position: int | None = Field(None, ge=0, description="where it goes, from 0 (default: the end)") + + +class ClipUpdate(RequestModel): + title: str | None = Field(None, max_length=200, description="empty clears it") + note: str | None = Field(None, max_length=2000, description="empty clears it") + t0: int | None = Field(None, ge=0) + t1: int | None = Field(None, ge=0) + + +class PlaylistOrder(RequestModel): + items: list[int] = Field(max_length=500, description="clip ids in their new order; clips not named follow, as they were") + + +class Clip(ResponseModel): + id: int + playlist: int + position: int = Field(description="its place in the playlist, from 0") + recording: int + t0: int = Field(description="where it starts, in ms from the start") + t1: int = Field(description="where it ends, in ms") + title: str | None = None + note: str | None = None + quote: str | None = Field(None, description="the words picked in the text") + highlight: int | None = Field(None, description="the highlight it was clipped from") + recording_title: str | None = None + namespace: str | None = None + media: str = Field(description="the resource's kind: audio, video, document, image or transcript") + audio: str | None = Field(None, description="a signed link to the resource's audio or video, or null when it has none") + link: str = Field(description="the resource's page in the web app at the clip's start") + created_at: str | None = None + + +class PlaylistDetail(Playlist): + items: list[Clip] = Field(description="its clips of resources you can read, in order, each with what a player needs") + hidden: int = Field(0, description="clips of resources you can't read, left out") diff --git a/fastapi_backend/tests/api/test_mcp.py b/fastapi_backend/tests/api/test_mcp.py index b0e42aca..3571ee1e 100644 --- a/fastapi_backend/tests/api/test_mcp.py +++ b/fastapi_backend/tests/api/test_mcp.py @@ -82,6 +82,7 @@ def test_clients_are_sent_to_sign_in(client, app, env): "delete_topic", "undo_topic_change", "write_note", + "add_to_playlist", "rename_chat", "create_project", "update_project", diff --git a/fastapi_backend/tests/api/test_playlists.py b/fastapi_backend/tests/api/test_playlists.py new file mode 100644 index 00000000..afea4ffd --- /dev/null +++ b/fastapi_backend/tests/api/test_playlists.py @@ -0,0 +1,268 @@ +"""Playlists (docs/api.md#playlists): clips of moments of any resources, kept in order to play back to back. A playlist +is its maker's; a shared one is read by everyone signed in, each reader getting only the clips of resources they can +read, with what a player needs; clips go with their resource, and private playlists with their maker.""" + +from __future__ import annotations + +import pytest + +from app.domain import analyze, auth, ingest, playlists, store +from tests.api.test_iiif import CLIP +from tests.helpers import login, make_user, quiet, seed, write_wav + +R = store.R +P = "/api/v1/playlists" + + +@pytest.fixture +def env(client, db, cfg, folder): + a, b, call = seed(db, cfg, folder) + wav, tr = folder / "clip.wav", folder / "clip.txt" + write_wav(wav, seconds=3.0) + tr.write_text(CLIP) + sound = ingest.import_transcript(db, cfg, "pods", tr, audio=wav, log=quiet) + analyze.analyze_pending(db, cfg, log=quiet) + make_user(db, "both@x.io", "both password 1", roles={"pods": "viewer", "calls": "viewer"}) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + make_user(db, "out@x.io", "outsider password 1", roles={"calls": "viewer"}) + return { + "a": a, + "b": b, + "call": call, + "sound": sound, + "hb": login(client, "both@x.io", "both password 1"), + "hv": login(client, "vi@x.io", "viewer password 1"), + "he": login(client, "ed@x.io", "editor password 1"), + "hx": login(client, "out@x.io", "outsider password 1"), + } + + +def _items(client, pid, h): + r = client.get(f"{P}/{pid}", headers=h) + assert r.status_code == 200, r.text + return r.json()["items"] + + +def test_playlist_crud_and_clips(client, env, db): + a, sound, hv = env["a"], env["sound"], env["hv"] + # nothing yet; a playlist with a unique name + assert client.get(P, headers=hv).json() == [] + r = client.post(P, headers=hv, json={"name": " Best bits ", "description": "for the team"}) + assert r.status_code == 200, r.text + pl = r.json() + assert (pl["name"], pl["description"], pl["shared"], pl["mine"], pl["count"], pl["duration_ms"]) == ( + "Best bits", + "for the team", + False, + True, + 0, + 0, + ) + pid = pl["id"] + assert client.post(P, headers=hv, json={"name": "best BITS"}).status_code == 400 + + # a viewer clips a moment of a resource they can read, with a title, a note and the words picked + url = f"{P}/{pid}/items" + r = client.post(url, headers=hv, json={"recording": sound, "t0": 500, "t1": 2000, "title": "The intro", "note": "start here"}) + assert r.status_code == 200, r.text + c = r.json() + assert {k: c[k] for k in ("playlist", "position", "recording", "t0", "t1", "title", "note", "media", "namespace")} == { + "playlist": pid, + "position": 0, + "recording": sound, + "t0": 500, + "t1": 2000, + "title": "The intro", + "note": "start here", + "media": "audio", + "namespace": "pods", + } + # what a player needs: the resource's signed audio link (as the recording player gets it), and its page there + assert c["audio"].startswith(f"/api/v1/recordings/{sound}/audio?") and "sig=" in c["audio"] + assert client.get(c["audio"]).status_code in (200, 206) + assert c["link"] == f"/resources/{sound}" + # a moment without an end lasts 30 seconds, or stops at the end of the resource; a text has no audio + a_end = db.one("SELECT duration_ms FROM $r", r=R("recording", a))["duration_ms"] + t = client.post(url, headers=hv, json={"recording": a, "t0": 4000, "quote": "the capsid\n model"}).json() + assert (t["t0"], t["t1"], t["quote"], t["audio"], t["media"], t["link"]) == ( + 4000, + min(34000, a_end), + "the capsid model", + None, + "transcript", + f"/resources/{a}?t=4", + ) + end = db.one("SELECT duration_ms FROM $r", r=R("recording", sound))["duration_ms"] + assert client.post(url, headers=hv, json={"recording": sound, "t0": 1000}).json()["t1"] == end + assert client.post(url, headers=hv, json={"recording": sound, "t0": end + 5000}).status_code == 400 + assert client.post(url, headers=hv, json={"recording": sound, "t0": 2000, "t1": 1000}).status_code == 400 + assert client.post(url, headers=hv, json={"t0": 0}).status_code == 400 + assert client.post(url, headers=hv, json={"recording": env["call"], "t0": 0}).status_code == 404 # not one they read + + # change one; an empty note clears it + r = client.patch(f"{url}/{c['id']}", headers=hv, json={"title": "Intro", "note": "", "t1": 2500}) + assert r.status_code == 200, r.text + assert (r.json()["title"], r.json()["note"], r.json()["t0"], r.json()["t1"]) == ("Intro", None, 500, 2500) + + # the list counts them and how long they play + listed = client.get(P, headers=hv).json() + assert [(p["id"], p["count"]) for p in listed] == [(pid, 3)] + assert listed[0]["duration_ms"] == 2000 + min(30000, a_end - 4000) + (end - 1000) + # rename, describe; delete takes the clips with it + r = client.patch(f"{P}/{pid}", headers=hv, json={"name": "Keepers", "description": ""}) + assert (r.json()["name"], r.json()["description"]) == ("Keepers", None) + assert client.delete(f"{P}/{pid}", headers=hv).status_code == 200 + assert client.get(f"{P}/{pid}", headers=hv).status_code == 404 + assert db.values("SELECT VALUE id FROM playlist_item WHERE playlist = $p", p=pid) == [] + + +def test_order_reorder_remove_and_undo(client, env): + a, b, sound, hv = env["a"], env["b"], env["sound"], env["hv"] + pid = client.post(P, headers=hv, json={"name": "Order"}).json()["id"] + url = f"{P}/{pid}/items" + ids = [ + client.post(url, headers=hv, json={"recording": r, "t0": 0, "title": n}).json()["id"] + for r, n in ((a, "one"), (b, "two"), (sound, "three")) + ] + one, two, three = ids + titles = lambda: [i["title"] for i in _items(client, pid, hv)] # noqa: E731 + assert titles() == ["one", "two", "three"] + # added at a place, the others make room + four = client.post(url, headers=hv, json={"recording": a, "t0": 1000, "title": "four", "position": 1}).json() + assert four["position"] == 1 and titles() == ["one", "four", "two", "three"] + # reorder: the ones named first, as named, then the others as they were + r = client.put(f"{P}/{pid}/order", headers=hv, json={"items": [three, one]}) + assert r.status_code == 200, r.text + assert [i["title"] for i in r.json()["items"]] == ["three", "one", "four", "two"] + assert [i["position"] for i in r.json()["items"]] == [0, 1, 2, 3] + assert client.put(f"{P}/{pid}/order", headers=hv, json={"items": [one, one]}).status_code == 400 + assert client.put(f"{P}/{pid}/order", headers=hv, json={"items": [99999]}).status_code == 400 + # remove one; Undo adds it back where it was + gone = next(i for i in _items(client, pid, hv) if i["id"] == one) + assert client.delete(f"{url}/{one}", headers=hv).status_code == 200 + assert titles() == ["three", "four", "two"] + assert client.delete(f"{url}/{one}", headers=hv).status_code == 404 + back = {k: gone[k] for k in ("recording", "t0", "t1", "title", "note", "quote", "position")} + assert client.post(url, headers=hv, json={k: v for k, v in back.items() if v is not None}).json()["position"] == 1 + assert titles() == ["three", "one", "four", "two"] + + +def test_last_used_is_the_default(client, env, db): + hv, a = env["hv"], env["a"] + uid = auth.find_account(db, "vi@x.io")["id"] + # none yet: a clip goes in a new one called Clips + first = playlists.default_for(db, uid) + assert playlists.get(db, first)["name"] == "Clips" + second = client.post(P, headers=hv, json={"name": "Second"}).json()["id"] + assert client.get(P, headers=hv).json()[0]["id"] == second # the newest is the latest used + client.post(f"{P}/{first}/items", headers=hv, json={"recording": a, "t0": 0}) + assert [p["id"] for p in client.get(P, headers=hv).json()] == [first, second] # adding to it uses it + assert playlists.default_for(db, uid) == first + + +def test_readers_get_only_clips_of_resources_they_can_read(client, env, db): + a, call, hb, hv, he, hx = env["a"], env["call"], env["hb"], env["hv"], env["he"], env["hx"] + # someone who reads pods and calls keeps clips of both + pid = client.post(P, headers=hb, json={"name": "Mixed"}).json()["id"] + url = f"{P}/{pid}/items" + client.post(url, headers=hb, json={"recording": a, "t0": 0, "title": "pods clip"}) + client.post(url, headers=hb, json={"recording": call, "t0": 0, "title": "calls clip", "note": "secret plan"}) + assert [i["title"] for i in _items(client, pid, hb)] == ["pods clip", "calls clip"] + # private: nobody else sees it, or changes it + for h in (hv, hx): + assert client.get(f"{P}/{pid}", headers=h).status_code == 404 + assert all(p["id"] != pid for p in client.get(P, headers=h).json()) + # shared: everyone signed in reads it, each getting only the clips of resources they can read + r = client.patch(f"{P}/{pid}", headers=hb, json={"shared": True}) + assert r.status_code == 200 and r.json()["shared"] is True + assert db.rows("SELECT action, target FROM audit_log WHERE action = 'playlist.share'") == [ + {"action": "playlist.share", "target": f"playlist:{pid}"} + ] + seen = client.get(f"{P}/{pid}", headers=hv).json() + assert [i["title"] for i in seen["items"]] == ["pods clip"] and seen["hidden"] == 0 and seen["mine"] is False + assert "secret plan" not in str(seen) + assert next(p for p in client.get(P, headers=hv).json() if p["id"] == pid)["count"] == 1 + assert [i["title"] for i in _items(client, pid, hx)] == ["calls clip"] + # only its maker changes it + item = _items(client, pid, hv)[0]["id"] + assert client.post(url, headers=he, json={"recording": a, "t0": 0}).status_code == 404 + assert client.patch(f"{P}/{pid}", headers=he, json={"name": "Mine"}).status_code == 404 + assert client.delete(f"{url}/{item}", headers=hv).status_code == 404 + assert client.put(f"{P}/{pid}/order", headers=hv, json={"items": [item]}).status_code == 404 + assert client.delete(f"{P}/{pid}", headers=hv).status_code == 404 + # its maker losing access to calls: the clip is left out for them too, counted as hidden + auth.set_role(db, auth.find_account(db, "both@x.io")["id"], store.ns_id(db, "calls"), None) + mine = client.get(f"{P}/{pid}", headers=login(client, "both@x.io", "both password 1")).json() + assert [i["title"] for i in mine["items"]] == ["pods clip"] and mine["hidden"] == 1 + + +def test_clips_of_highlights(client, env): + a, he, hv = env["a"], env["he"], env["hv"] + h = client.post( + f"/api/v1/resources/{a}/highlights", headers=he, json={"t0": 3000, "t1": 7000, "quote": "the capsid", "label": "Key claim"} + ).json() + pid = client.post(P, headers=hv, json={"name": "From highlights"}).json()["id"] + r = client.post(f"{P}/{pid}/items", headers=hv, json={"highlight": h["id"]}) + assert r.status_code == 200, r.text + c = r.json() + assert (c["recording"], c["t0"], c["t1"], c["title"], c["quote"], c["highlight"]) == (a, 3000, 7000, "Key claim", "the capsid", h["id"]) + assert client.post(f"{P}/{pid}/items", headers=hv, json={"highlight": h["id"], "recording": env["b"]}).status_code == 400 + # a reader without access to the highlight's resource can't clip it + hx = env["hx"] + other = client.post(P, headers=hx, json={"name": "Elsewhere"}).json()["id"] + assert client.post(f"{P}/{other}/items", headers=hx, json={"highlight": h["id"]}).status_code == 404 + # the clip stays when the highlight goes + client.delete(f"/api/v1/resources/{a}/highlights/{h['id']}", headers=he) + assert [i["title"] for i in _items(client, pid, hv)] == ["Key claim"] + + +def test_clips_go_with_their_resource_and_private_playlists_with_their_maker(client, env, db, cfg): + from app.domain import deletion + + a, b, hv = env["a"], env["b"], env["hv"] + pid = client.post(P, headers=hv, json={"name": "Going"}).json()["id"] + keep = client.post(P, headers=hv, json={"name": "Kept", "shared": True}).json()["id"] + for p in (pid, keep): + client.post(f"{P}/{p}/items", headers=hv, json={"recording": a, "t0": 0}) + client.post(f"{P}/{p}/items", headers=hv, json={"recording": b, "t0": 0}) + deletion.delete(db, cfg, b) + assert [i["recording"] for i in _items(client, pid, hv)] == [a] + assert db.values("SELECT VALUE id FROM playlist_item WHERE recording = $r", r=b) == [] + auth.remove_account(db, auth.find_account(db, "vi@x.io")["id"]) + assert db.values("SELECT VALUE record::id(id) FROM playlist") == [keep] + assert db.values("SELECT VALUE playlist FROM playlist_item") == [keep] + + +def test_agents_list_playlists_and_add_moments(client, env): + from tests.api.test_mcp import tool, tool_error + from tests.api.test_oauth import _bearer, _grant + + a, call, hv = env["a"], env["call"], env["hv"] + # a read-only app lists but doesn't add + _, tokens = _grant(client, hv, scope="read") + ro = _bearer(tokens) + assert tool(client, ro, "list_playlists") == {"playlists": []} + assert "read-only" in tool_error(client, ro, "add_to_playlist", recording_id=a, t0=1000) + # with none yet, a moment goes in a new playlist called Clips; then in the one used last + got = tool(client, hv, "add_to_playlist", recording_id=a, t0=1000, t1=4000, title="Opening", note="from an agent") + assert (got["playlist"], got["position"], got["t0"], got["t1"], got["title"]) == ("Clips", 0, 1000, 4000, "Opening") + assert got["url"].endswith(f"/playlists/{got['playlist_id']}") + other = client.post(P, headers=hv, json={"name": "Talks"}).json()["id"] + assert tool(client, hv, "add_to_playlist", recording_id=a, t0=2000)["playlist_id"] == other + # by name, made when there's none of that name; by id + named = tool(client, hv, "add_to_playlist", recording_id=a, t0=0, playlist="clips") + assert named["playlist_id"] == got["playlist_id"] and named["position"] == 1 + fresh = tool(client, hv, "add_to_playlist", recording_id=a, t0=0, playlist="Quotes") + assert fresh["playlist"] == "Quotes" and fresh["playlist_id"] not in (other, got["playlist_id"]) + # only resources the person can read, and nothing is made for one they can't + assert "not found" in tool_error(client, hv, "add_to_playlist", recording_id=call, t0=0, playlist="Nope") + assert all(p["name"] != "Nope" for p in tool(client, hv, "list_playlists")["playlists"]) + assert "t0" in tool_error(client, hv, "add_to_playlist", recording_id=a) + # one playlist, its clips in order with links to their moments + one = tool(client, hv, "list_playlists", playlist_id=got["playlist_id"]) + assert (one["name"], one["clips"]) == ("Clips", 2) + assert [(c.get("title"), c["t0"]) for c in one["items"]] == [("Opening", 1000), (None, 0)] + assert one["items"][0]["url"].endswith(f"/resources/{a}?t=1") and one["items"][0]["note"] == "from an agent" + listed = tool(client, hv, "list_playlists")["playlists"] + assert [p["name"] for p in listed] == ["Quotes", "Clips", "Talks"] diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index baf7bef2..ae8379fc 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -36,6 +36,7 @@ export { Oauth, type Options, Pipelines, + Playlists, Podcasts, Projects, Public, @@ -101,6 +102,11 @@ export type { AddAssistantMemoryErrors, AddAssistantMemoryResponse, AddAssistantMemoryResponses, + AddClipData, + AddClipError, + AddClipErrors, + AddClipResponse, + AddClipResponses, AddFileData, AddFileError, AddFileErrors, @@ -444,6 +450,9 @@ export type { ClientOptions, ClientRegistered, ClientRegistration, + Clip, + ClipCreate, + ClipUpdate, CloseRecordingToIpGroupData, CloseRecordingToIpGroupError, CloseRecordingToIpGroupErrors, @@ -678,6 +687,11 @@ export type { CreatePipelineVersionErrors, CreatePipelineVersionResponse, CreatePipelineVersionResponses, + CreatePlaylistData, + CreatePlaylistError, + CreatePlaylistErrors, + CreatePlaylistResponse, + CreatePlaylistResponses, CreatePodcastData, CreatePodcastError, CreatePodcastErrors, @@ -960,6 +974,11 @@ export type { DeletePageFileResponses, DeletePageResponse, DeletePageResponses, + DeletePlaylistData, + DeletePlaylistError, + DeletePlaylistErrors, + DeletePlaylistResponse, + DeletePlaylistResponses, DeletePodcastData, DeletePodcastError, DeletePodcastErrors, @@ -1566,6 +1585,11 @@ export type { GetPlayerErrors, GetPlayerResponse, GetPlayerResponses, + GetPlaylistData, + GetPlaylistError, + GetPlaylistErrors, + GetPlaylistResponse, + GetPlaylistResponses, GetPodcastData, GetPodcastError, GetPodcastErrors, @@ -2290,6 +2314,9 @@ export type { ListPipelinesData, ListPipelinesResponse, ListPipelinesResponses, + ListPlaylistsData, + ListPlaylistsResponse, + ListPlaylistsResponses, ListPodcastsData, ListPodcastsError, ListPodcastsErrors, @@ -2906,6 +2933,11 @@ export type { PlanTranslationResponse, PlanTranslationResponses, Player, + Playlist, + PlaylistCreate, + PlaylistDetail, + PlaylistOrder, + PlaylistUpdate, Podcast, PodcastCheck, PodcastCitation, @@ -3184,6 +3216,11 @@ export type { RemoveBudgetErrors, RemoveBudgetResponse, RemoveBudgetResponses, + RemoveClipData, + RemoveClipError, + RemoveClipErrors, + RemoveClipResponse, + RemoveClipResponses, Removed, RemoveFeedItemData, RemoveFeedItemError, @@ -3271,6 +3308,11 @@ export type { RenewNodeErrors, RenewNodeResponse, RenewNodeResponses, + ReorderPlaylistData, + ReorderPlaylistError, + ReorderPlaylistErrors, + ReorderPlaylistResponse, + ReorderPlaylistResponses, Reply, ReplyBefore, ReplyDraft, @@ -4239,6 +4281,11 @@ export type { UpdateCircleErrors, UpdateCircleResponse, UpdateCircleResponses, + UpdateClipData, + UpdateClipError, + UpdateClipErrors, + UpdateClipResponse, + UpdateClipResponses, UpdateCollectionData, UpdateCollectionError, UpdateCollectionErrors, @@ -4364,6 +4411,11 @@ export type { UpdatePipelineErrors, UpdatePipelineResponse, UpdatePipelineResponses, + UpdatePlaylistData, + UpdatePlaylistError, + UpdatePlaylistErrors, + UpdatePlaylistResponse, + UpdatePlaylistResponses, UpdateProjectData, UpdateProjectError, UpdateProjectErrors, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index a3c978ab..af193e2d 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -23,6 +23,9 @@ import type { AddAssistantMemoryData, AddAssistantMemoryErrors, AddAssistantMemoryResponses, + AddClipData, + AddClipErrors, + AddClipResponses, AddFileData, AddFileErrors, AddFileResponses, @@ -276,6 +279,9 @@ import type { CreatePipelineVersionData, CreatePipelineVersionErrors, CreatePipelineVersionResponses, + CreatePlaylistData, + CreatePlaylistErrors, + CreatePlaylistResponses, CreatePodcastData, CreatePodcastErrors, CreatePodcastResponses, @@ -440,6 +446,9 @@ import type { DeletePageFileErrors, DeletePageFileResponses, DeletePageResponses, + DeletePlaylistData, + DeletePlaylistErrors, + DeletePlaylistResponses, DeletePodcastData, DeletePodcastErrors, DeletePodcastResponses, @@ -752,6 +761,9 @@ import type { GetPlayerData, GetPlayerErrors, GetPlayerResponses, + GetPlaylistData, + GetPlaylistErrors, + GetPlaylistResponses, GetPodcastData, GetPodcastErrors, GetPodcastResponses, @@ -1143,6 +1155,8 @@ import type { ListPendingAccessRequestsResponses, ListPipelinesData, ListPipelinesResponses, + ListPlaylistsData, + ListPlaylistsResponses, ListPodcastsData, ListPodcastsErrors, ListPodcastsResponses, @@ -1530,6 +1544,9 @@ import type { RemoveBudgetData, RemoveBudgetErrors, RemoveBudgetResponses, + RemoveClipData, + RemoveClipErrors, + RemoveClipResponses, RemoveFeedItemData, RemoveFeedItemErrors, RemoveFeedItemResponses, @@ -1581,6 +1598,9 @@ import type { RenewNodeData, RenewNodeErrors, RenewNodeResponses, + ReorderPlaylistData, + ReorderPlaylistErrors, + ReorderPlaylistResponses, ReprocessRecordingData, ReprocessRecordingErrors, ReprocessRecordingResponses, @@ -2029,6 +2049,9 @@ import type { UpdateCircleData, UpdateCircleErrors, UpdateCircleResponses, + UpdateClipData, + UpdateClipErrors, + UpdateClipResponses, UpdateCollectionData, UpdateCollectionErrors, UpdateCollectionResponses, @@ -2104,6 +2127,9 @@ import type { UpdatePipelineData, UpdatePipelineErrors, UpdatePipelineResponses, + UpdatePlaylistData, + UpdatePlaylistErrors, + UpdatePlaylistResponses, UpdateProjectData, UpdateProjectErrors, UpdateProjectResponses, @@ -12149,6 +12175,160 @@ export class Collections { } } +export class Playlists { + /** + * List Playlists + * + * Your playlists, the latest used first (the first is where a clip goes when none is named), then the ones others + * shared; each counts only the clips of resources you can read. + */ + public static listPlaylists( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/playlists", + ...options, + }); + } + + /** + * Create Playlist + * + * Make a playlist, with a name unique among yours. Up to 200 each. + */ + public static createPlaylist( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/playlists", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Delete Playlist + * + * Delete your playlist and its clips. + */ + public static deletePlaylist( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/playlists/{pid}", + ...options, + }); + } + + /** + * Get Playlist + * + * The playlist and its clips in order, each with what a player needs: the resource's kind, its signed audio or + * video link (null when it has none: play its text), the clip's times and its page's link. Only clips of + * resources you can read; `hidden` counts the others in your own playlist. + */ + public static getPlaylist( + options: Options, + ): RequestResult { + return (options.client ?? client).get({ + url: "/api/v1/playlists/{pid}", + ...options, + }); + } + + /** + * Update Playlist + * + * Rename your playlist, change its description, or share or unshare it. Sharing and unsharing are audited + * (`playlist.share`, `playlist.unshare`). + */ + public static updatePlaylist( + options: Options, + ): RequestResult { + return (options.client ?? client).patch({ + url: "/api/v1/playlists/{pid}", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Add Clip + * + * Add a clip to your playlist: a resource you can read (`recording`) from `t0` to `t1`, or a `highlight` on one + * (its passage and label, unless given), with a `title` and a `note`, at `position` (default: the end). Up to 500 + * in a playlist. + */ + public static addClip( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/playlists/{pid}/items", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Remove Clip + * + * Take a clip out of your playlist. To undo, add it again with its `position`. + */ + public static removeClip( + options: Options, + ): RequestResult { + return (options.client ?? client).delete({ + url: "/api/v1/playlists/{pid}/items/{iid}", + ...options, + }); + } + + /** + * Update Clip + * + * Change a clip's title or note (empty clears it), or its times. + */ + public static updateClip( + options: Options, + ): RequestResult { + return (options.client ?? client).patch({ + url: "/api/v1/playlists/{pid}/items/{iid}", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } + + /** + * Reorder Playlist + * + * Put your playlist's clips in this order (their ids); clips not named follow, as they were. + */ + public static reorderPlaylist( + options: Options, + ): RequestResult { + return (options.client ?? client).put({ + url: "/api/v1/playlists/{pid}/order", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } +} + export class Batches { /** * Estimate Batch diff --git a/threat-model.md b/threat-model.md index 7fb98cda..8a3786fb 100644 --- a/threat-model.md +++ b/threat-model.md @@ -460,6 +460,7 @@ stored text for the consumer's format. | Signed links in responses | `api/media.py` | server-written for allow-listed fields only; text that looks like a link is never signed | that a link is private: it is a bearer credential until expiry | *(documented, docs/architecture.md "is never signed")* | | Exports, RDF, JSON-LD, IIIF manifests, WebVTT, Dublin Core | linked data, IIIF | values as typed; RDF responses send `Access-Control-Allow-Origin: *` | escaping for the consumer's format | *(inferred, Q24)* | | Evidence PDFs | `evidence.py`, `evidence_report.py` | the resource's file and what Lens holds about it (metadata, summary, speakers, emotion, entities, what is seen), for anyone who can read it; places only for members of its namespace; shared notes, highlights and comments only when picked, never private notes; what a model wrote is labelled | that a PDF handed on holds less than its reader could see (pick its parts) | *(documented, docs/api.md "never anyone's private notes")* | +| Playlists | `domain/playlists.py`, `routes/playlists.py` | a shared playlist's clips (titles, notes, quotes, signed media links) only where its reader has a role on the clip's resource; its name and description to everyone signed in | that a shared playlist's name and description stay among the people who can read its clips | *(documented, docs/api.md "whoever made it")* | | Asset description | IIIF, public pages | "never a closed part"; anyone who may see the page sees it | that closing parts hides the description | *(documented, docs/access.md "never a closed part")* | | Embed player and `/s/` pages | `api/pages.py` | HTML built from DB values under a strict CSP; framable only per `server.embed_frame_ancestors` | that titles are inert outside this page | *(documented, docs/configuration.md "Content-Security-Policy")* | | Report pages | `api/pages.py` | links re-signed on read for every asset of the namespace in the page | that a report link opens only the report | *(inferred, Q10)* | diff --git a/threat-model.yaml b/threat-model.yaml index 7e917851..19977178 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -969,6 +969,14 @@ outputs: invariants: [] downstream_must_not_assume: - {id: dmna-evidence-pdf-less, statement: "that a PDF handed on holds less than its reader could see (pick its parts)", provenance: {kind: documented, source: 'docs/api.md "never anyone''s private notes"'}} + - component: authz + channel: "Playlists (domain/playlists.py, routes/playlists.py)" + taint: same-as-input + x-taint-note: "a shared playlist's clips (titles, notes, quotes, signed media links) only where its reader has a role on the clip's resource; its name and description to everyone signed in" + taint_provenance: {kind: documented, source: 'docs/api.md "whoever made it"'} + invariants: [] + downstream_must_not_assume: + - {id: dmna-playlist-name-private, statement: "that a shared playlist's name and description stay among the people who can read its clips", provenance: {kind: documented, source: 'docs/api.md "whoever made it"'}} - component: authz channel: "Recording description (IIIF, public pages)" taint: same-as-input From d47d2360802043a0e4075bad6b83b7472a73325c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 11:24:42 +0000 Subject: [PATCH 2/2] feat(playlists): add moments to a playlist from the transcript, and play it back to back Add to playlist in the selection toolbar and beside each highlight keeps the moment as a clip in the playlist used last (a new one called Clips when there is none), with Undo. The Playlists page lists yours and the shared ones; a playlist's page plays its clips back to back on each resource's own audio or video, reorders them by dragging or with Up and Down, and takes one out with Undo. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 6 +- changelog.d/clips.md | 7 + docs/api.md | 3 +- docs/playlists.md | 48 + docs/requirements.md | 1 + mkdocs.yml | 1 + nextjs-frontend/__tests__/nav-groups.test.tsx | 12 + .../__tests__/playlist-page.test.tsx | 144 +++ .../__tests__/playlists-model.test.ts | 136 +++ .../__tests__/recording-comments.test.tsx | 49 + .../app/(app)/playlists/[id]/page.tsx | 12 + nextjs-frontend/app/(app)/playlists/page.tsx | 10 + .../app/openapi-client/types.gen.ts | 610 ++++++++++ .../components/app-shell/app-shell.tsx | 3 + .../components/app-shell/nav-config.ts | 12 +- nextjs-frontend/components/app-shell/nav.tsx | 9 +- .../components/playlists/hooks.tsx | 82 ++ nextjs-frontend/components/playlists/model.ts | 106 ++ .../components/playlists/player.tsx | 151 +++ .../components/playlists/playlist-page.tsx | 283 +++++ .../components/playlists/playlists-page.tsx | 120 ++ .../components/recording/highlights-tab.tsx | 14 +- .../recording/selection-toolbar.tsx | 27 +- nextjs-frontend/openapi.json | 1038 +++++++++++++++++ 24 files changed, 2875 insertions(+), 9 deletions(-) create mode 100644 changelog.d/clips.md create mode 100644 docs/playlists.md create mode 100644 nextjs-frontend/__tests__/playlist-page.test.tsx create mode 100644 nextjs-frontend/__tests__/playlists-model.test.ts create mode 100644 nextjs-frontend/app/(app)/playlists/[id]/page.tsx create mode 100644 nextjs-frontend/app/(app)/playlists/page.tsx create mode 100644 nextjs-frontend/components/playlists/hooks.tsx create mode 100644 nextjs-frontend/components/playlists/model.ts create mode 100644 nextjs-frontend/components/playlists/player.tsx create mode 100644 nextjs-frontend/components/playlists/playlist-page.tsx create mode 100644 nextjs-frontend/components/playlists/playlists-page.tsx diff --git a/README.md b/README.md index f44bbaf2..f3f08822 100644 --- a/README.md +++ b/README.md @@ -199,14 +199,16 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p entity and topic, written by you or the assistant, in a tree where notes are renamed, moved, copied and pinned (pins are your own) and each note shows the notes it's inside and the ones inside it, with today's note one click away, templates to start from, a search by their words, a download as Markdown, and Undo and Recently deleted instead of a - confirm. Export an asset or a collection as an evidence PDF with everything Lens found about it, or the parts you + confirm. Keep a moment of any recording, video or document as a clip with a title and a note, in one click from the + transcript or a highlight, and play clips from any resources back to back in a playlist, reordered by dragging and + taken out with Undo. Export an asset or a collection as an evidence PDF with everything Lens found about it, or the parts you pick. Publish a namespace's resources, episodes, timeline activities or text as a private RSS feed or a podcast feed that any feed reader or podcast app follows (Feeds, from the Podcasts page), and let a routine fill it on a schedule with what's new, or make an episode about the week's new resources. Follow a saved collection in your podcast app: its page gives your own private address for it, listing its assets with audio or video that you can read. ([Podcasts](docs/podcasts.md), [specs from transcripts](docs/notes.md#specs-from-transcripts), [notes](docs/notes.md), - [evidence PDFs](docs/api.md#evidence-pdfs), [published feeds](docs/feeds.md); + [playlists](docs/playlists.md), [evidence PDFs](docs/api.md#evidence-pdfs), [published feeds](docs/feeds.md); [checks](docs/requirements.md#learn-from-it)) - **Automate it.** Pipelines and workflows on a canvas (with Convert steps that make a video's soundtrack an audio asset and a document's pages image assets, and Read steps that pick their own converter, such as anytopdf; what their model steps write about diff --git a/changelog.d/clips.md b/changelog.d/clips.md new file mode 100644 index 00000000..efb54ff7 --- /dev/null +++ b/changelog.d/clips.md @@ -0,0 +1,7 @@ +- **Clips and playlists.** Keep a moment of any recording, video or document as a clip, with a title and a note: select + words in a transcript and choose Add to playlist, or add a highlight from its tab. One click puts it at the end of + the playlist you used last (a new one called Clips when you have none), with Undo. The new Playlists page lists yours + and the ones others shared; a playlist's page plays its clips back to back on each resource's own audio or video, + reorders them by dragging or with Up and Down, and takes one out with Undo. A shared playlist shows each reader only + the clips of resources they can read. Agents use `/api/v1/playlists` and the MCP tools `list_playlists` and + `add_to_playlist`. New tables only: nothing to do when upgrading ([Playlists](docs/playlists.md)). diff --git a/docs/api.md b/docs/api.md index e3d0a897..a1ccc895 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1883,7 +1883,8 @@ with its resource's `recording_title`, `namespace` and `media` kind, its signed video, as the recording player gets it; `null` when it has none, so play its text), its times and `link`, its page in the web app at the clip's start. `hidden` counts the clips of your own playlist whose resources you can no longer read. Clips go when their resource is deleted; private playlists go when their maker's account is removed, shared ones -stay. Agents use them through [MCP](mcp.md) (`list_playlists`, `add_to_playlist`). +stay. Agents use them through [MCP](mcp.md) (`list_playlists`, `add_to_playlist`); the web app's side is in +[Playlists](playlists.md). ## views diff --git a/docs/playlists.md b/docs/playlists.md new file mode 100644 index 00000000..3e942d9f --- /dev/null +++ b/docs/playlists.md @@ -0,0 +1,48 @@ +# Playlists + +Keep a moment of a recording, a video or a document as a clip, with a title and a note, and gather clips from any +resources into a playlist that plays them back to back. + +## Keeping a clip + +Select words in a transcript and choose **Add to playlist**, or choose the **Add to playlist** button beside a +highlight in its tab. One click: the clip goes at the end of the playlist you used last (a new one called Clips when +you have none), titled with the words you picked or the highlight's label, and the message that says where it went +has **Undo** and a link to the playlist. A clip of a selection runs from the first word to the last; a clip of a +single moment lasts 30 seconds, or to the end of the resource. + +Anyone who can read a resource keeps clips of it, for themselves. That's how a clip differs from a +[highlight](api.md#highlights), which an editor makes for everyone who reads the resource. A clip of a highlight +copies its passage and label and stays when the highlight goes. + +## Playing a playlist + +**Playlists** in the menu (there once you have one; the toast links there before that) lists yours, the latest used first, then the ones others shared. Type a name and press Enter +to make one. A playlist's page plays its clips back to back: **Play all**, or the play button of any clip, then +Previous and Next. Each clip plays from its start to its end on its resource's own audio or video (the same signed link +the recording player uses) and then the next one starts; clips of text without audio are skipped, and **Open here** +opens the resource at the moment playing. + +Drag a clip to reorder it, or use its Up and Down buttons; the order is saved at once. **Remove from playlist** takes a +clip out with Undo, which puts it back in its place. Rename a playlist by editing its name, share it with the +**Shared** switch, and delete it with Undo, which makes it again with its clips. + +## Who sees what + +A playlist is yours and private until you share it. A shared playlist is read by everyone signed in, and each person +gets only the clips of resources they have a role on, whoever made the playlist: a clip of a resource they can't read +isn't shown, nor are its title and note. Only you change your playlists. Clips go when their resource is deleted; +your private playlists go when your account is removed, and shared ones stay. + +## Agents + +Agents use playlists through the [API](api.md#playlists) and [MCP](mcp.md): `list_playlists` lists them (and one's +clips, with links to their moments), and `add_to_playlist` keeps a moment or a highlight as a clip, in a playlist +named by id or name, or by default the one used last. + +## Refine later + +- Play clips of text (documents, transcripts without audio) by reading them aloud instead of skipping them. +- Share a playlist with chosen people or a namespace, not only everyone signed in, and give it a public link. +- Export a playlist as one audio or video file, or as a podcast feed. +- Let the assistant in chat list playlists and add clips, as MCP does. diff --git a/docs/requirements.md b/docs/requirements.md index eb401cbc..23b71a53 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -249,6 +249,7 @@ request, to another check that proves the same thing; loosening a requirement or | A saved collection is a private podcast feed of its assets with audio or video: one address per person who may read it, listing only what they can read, from the collection's page in one click with Copy, rotated or stopped like a namespace's feed; through the API and MCP | [test_feeds.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_feeds.py) `test_a_collection_feed_lists_its_assets_with_media`, `test_a_collection_feed_token_rotates_and_stops`, `test_agents_get_a_collection_feed`; [collection-feed.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/collection-feed.test.ts) | | A routine fills a feed on a schedule, or makes a podcast episode about what's new: graph workflow nodes pick the resources or timeline activities since the last run, add them to a feed, or make an episode about them; Try says what they would do | [test_feed_nodes.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_feed_nodes.py) `test_a_routine_publishes_new_resources_to_a_feed`, `test_a_routine_publishes_the_timeline_and_text`, `test_a_routine_makes_an_episode_and_the_feed_lists_it_once_made`, `test_try_says_what_it_would_do_and_bad_settings_are_refused` | | Feeds in the web app: a Feeds page, reached from the Podcasts page and under Explore once there is a feed; New feed with the title focused and Enter making it in the current namespace; a feed's page copies the address in one click, gets a new one, edits what readers show, and takes items out or deletes the feed with Undo instead of a confirm; the publishing nodes on the canvas, checked before Publish | [feeds-ui.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/feeds-ui.test.tsx) `the Feeds page lists each feed with its kind, namespace, items and whether an app reads it`, `New feed opens on the title, defaults to the current namespace, and Enter makes it and opens its page`, `a feed's page copies the address, gets a new one, and takes an item out with Undo`, `deleting a feed is undone by making it again with its items, at a new address`, `editors change what readers show, and Enter in a field saves`; [workflow-flow.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-flow.test.ts) `a Select node into Publish to feed or Podcast episode is a whole graph workflow`, `says what to fix: a feed to choose, the limits, and a graph that publishes nothing`, `names the selection and the feed on the canvas`; [nav-groups.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/nav-groups.test.tsx) `feeds are under Explore, after Podcasts, for everyone`, `Feeds is in the nav only once there is a feed, so a phone's nav stays as short as it was` | +| A moment of any resource kept as a clip (a title, a note) in one click from the transcript or a highlight, in the playlist used last; playlists play clips back to back, reorder and remove with Undo, and each reader gets only the clips of resources they can read; through the API and MCP | [test_playlists.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_playlists.py) `test_playlist_crud_and_clips`, `test_order_reorder_remove_and_undo`, `test_last_used_is_the_default`, `test_readers_get_only_clips_of_resources_they_can_read`, `test_clips_of_highlights`, `test_clips_go_with_their_resource_and_private_playlists_with_their_maker`, `test_agents_list_playlists_and_add_moments`; [playlists-model.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/playlists-model.test.ts); [playlist-page.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/playlist-page.test.tsx) `reorders with Up and Down and saves the new order`, `removes a clip at once, and Undo puts it back in its place`; [recording-comments.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/recording-comments.test.tsx) `keeps a highlight as a clip in the latest playlist in one click, with Undo`, `makes a first playlist called Clips when there is none`; [nav-groups.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/nav-groups.test.tsx) `Playlists is in the nav only once there is one, under Explore after Feeds` | | An asset or a collection exported as an evidence PDF, whole or the parts you pick | [test_evidence.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_evidence.py) `test_a_collection_as_one_evidence_pdf`, `test_everything_lens_found_goes_in`, `test_picking_the_parts`; [evidence-parts.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/evidence-parts.test.tsx) | ## Automate it diff --git a/mkdocs.yml b/mkdocs.yml index 0e35006e..5a21f270 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,6 +85,7 @@ nav: - Podcasts: podcasts.md - Published feeds: feeds.md - Notes: notes.md + - Playlists: playlists.md - Web clipper: web-clipper.md - The assistant in your browser: browser-assistant.md - Sensors: sensors.md diff --git a/nextjs-frontend/__tests__/nav-groups.test.tsx b/nextjs-frontend/__tests__/nav-groups.test.tsx index 5791ed80..27e81c09 100644 --- a/nextjs-frontend/__tests__/nav-groups.test.tsx +++ b/nextjs-frontend/__tests__/nav-groups.test.tsx @@ -58,6 +58,18 @@ test("Feeds is in the nav only once there is a feed, so a phone's nav stays as s expect(screen.getByRole("link", { name: "Feeds 1" })).toHaveAttribute("href", "/feeds"); }); +test("Playlists is in the nav only once there is one, under Explore after Feeds", () => { + const { unmount } = nav(false); + expect(screen.queryByRole("link", { name: /^Playlists/ })).not.toBeInTheDocument(); + unmount(); + nav(false, { playlists: 2 }); + expect(screen.getByRole("link", { name: "Playlists 2" })).toHaveAttribute("href", "/playlists"); + const items = navFor(false); + const i = items.findIndex((it) => it.href === "/playlists"); + expect(items[i]).toMatchObject({ group: "Explore" }); + expect(items[i - 1].href).toBe("/feeds"); +}); + test("Automations is under Automate for everyone, right after Approvals", () => { for (const admin of [true, false]) { const items = navFor(admin); diff --git a/nextjs-frontend/__tests__/playlist-page.test.tsx b/nextjs-frontend/__tests__/playlist-page.test.tsx new file mode 100644 index 00000000..548537dd --- /dev/null +++ b/nextjs-frontend/__tests__/playlist-page.test.tsx @@ -0,0 +1,144 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import type { Clip, PlaylistDetail } from "@/app/openapi-client/types.gen"; +import { PlaylistPage } from "@/components/playlists/playlist-page"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); +const api = { + getPlaylist: jest.fn, [unknown]>(), + reorderPlaylist: jest.fn, [unknown]>(), + removeClip: jest.fn, [unknown]>(), + addClip: jest.fn, [unknown]>(), + updatePlaylist: jest.fn, [unknown]>(), + deletePlaylist: jest.fn, [unknown]>(), + createPlaylist: jest.fn, [unknown]>(), +}; +jest.mock("@/app/openapi-client", () => ({ + Playlists: Object.fromEntries( + [ + "getPlaylist", + "reorderPlaylist", + "removeClip", + "addClip", + "updatePlaylist", + "deletePlaylist", + "createPlaylist", + ].map((k) => [k, (a: unknown) => api[k as keyof typeof api](a)]), + ), +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +const push = jest.fn(); +jest.mock("next/navigation", () => ({ useRouter: () => ({ push }) })); +const toast = jest.fn(); +jest.mock("@/components/ui/toast", () => ({ useToast: () => toast })); + +const clip = (id: number, over: Partial = {}): Clip => ({ + id, + playlist: 5, + position: id - 1, + recording: 10 + id, + t0: 1000, + t1: 6000, + title: `Clip ${id}`, + recording_title: `Episode ${id}`, + namespace: "pods", + media: "audio", + audio: `/api/v1/recordings/${10 + id}/audio?sig=x`, + link: `/resources/${10 + id}?t=1`, + ...over, +}); +const playlist = (over: Partial = {}): PlaylistDetail => ({ + id: 5, + name: "Talks", + shared: false, + mine: true, + count: 3, + duration_ms: 15000, + items: [clip(1), clip(2), clip(3, { audio: null, media: "transcript" })], + hidden: 0, + ...over, +}); + +function show() { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + return render( + + + + + , + ); +} + +const titles = () => + within(screen.getByRole("list", { name: "Clips" })) + .getAllByRole("listitem") + .map((li) => within(li).getByText(/^Clip \d$/).textContent); + +beforeAll(() => { + HTMLMediaElement.prototype.pause = jest.fn(); + HTMLMediaElement.prototype.load = jest.fn(); + HTMLMediaElement.prototype.play = jest.fn(() => Promise.resolve()); +}); +beforeEach(() => { + jest.clearAllMocks(); + api.getPlaylist.mockImplementation(() => ok(playlist())); + api.reorderPlaylist.mockImplementation(() => new Promise(() => undefined)); + api.removeClip.mockImplementation(() => ok({ ok: true })); + api.addClip.mockImplementation(() => ok(clip(9))); +}); + +describe("a playlist's page", () => { + it("lists the clips in order with their resource and times; text clips can't play", async () => { + show(); + await screen.findByRole("list", { name: "Clips" }); + expect(titles()).toEqual(["Clip 1", "Clip 2", "Clip 3"]); + expect(screen.getByRole("textbox", { name: "Playlist name" })).toHaveValue("Talks"); + expect(screen.getByRole("button", { name: "Play Clip 1" })).toBeEnabled(); + expect(screen.getByRole("button", { name: /Text only/ })).toBeDisabled(); + expect(screen.getByText("Episode 1")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Play all" })).toBeEnabled(); + }); + + it("reorders with Up and Down and saves the new order", async () => { + show(); + await screen.findByRole("list", { name: "Clips" }); + const rows = within(screen.getByRole("list", { name: "Clips" })).getAllByRole("listitem"); + expect(within(rows[0]).getByRole("button", { name: "Move up" })).toBeDisabled(); + fireEvent.click(within(rows[0]).getByRole("button", { name: "Move down" })); + expect(titles()).toEqual(["Clip 2", "Clip 1", "Clip 3"]); + await waitFor(() => expect(api.reorderPlaylist).toHaveBeenCalledTimes(1)); + expect(api.reorderPlaylist.mock.calls[0][0]).toMatchObject({ path: { pid: 5 }, body: { items: [2, 1, 3] } }); + }); + + it("removes a clip at once, and Undo puts it back in its place", async () => { + show(); + await screen.findByRole("list", { name: "Clips" }); + const rows = within(screen.getByRole("list", { name: "Clips" })).getAllByRole("listitem"); + api.getPlaylist.mockImplementation(() => ok(playlist({ items: [clip(1), clip(3)] }))); // as the server has it now + fireEvent.click(within(rows[1]).getByRole("button", { name: "Remove from playlist" })); + expect(titles()).toEqual(["Clip 1", "Clip 3"]); + await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Removed Clip 2" }))); + expect(api.removeClip.mock.calls[0][0]).toMatchObject({ path: { pid: 5, iid: 2 } }); + toast.mock.calls[0][0].action.onClick(); + await waitFor(() => expect(titles()).toEqual(["Clip 1", "Clip 2", "Clip 3"])); + await waitFor(() => expect(api.addClip).toHaveBeenCalledTimes(1)); + expect(api.addClip.mock.calls[0][0]).toMatchObject({ + path: { pid: 5 }, + body: { recording: 12, t0: 1000, t1: 6000, title: "Clip 2", position: 1 }, + }); + }); + + it("shows someone else's shared playlist without ways to change it", async () => { + api.getPlaylist.mockImplementation(() => ok(playlist({ mine: false, shared: true, created_by_name: "Ana" }))); + show(); + await screen.findByRole("list", { name: "Clips" }); + expect(screen.queryByRole("textbox", { name: "Playlist name" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Move down" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Remove from playlist" })).not.toBeInTheDocument(); + expect(screen.getByText(/by Ana/)).toBeInTheDocument(); + }); +}); diff --git a/nextjs-frontend/__tests__/playlists-model.test.ts b/nextjs-frontend/__tests__/playlists-model.test.ts new file mode 100644 index 00000000..f1ad2521 --- /dev/null +++ b/nextjs-frontend/__tests__/playlists-model.test.ts @@ -0,0 +1,136 @@ +import type { Clip } from "@/app/openapi-client/types.gen"; +import { + clipTitle, + defaultPlaylist, + firstPlayable, + followCurrent, + moveBy, + moveTo, + nextPlayable, + reachedEnd, + readdBody, + removeItem, + restoreItem, + titleFromQuote, + totalMs, +} from "@/components/playlists/model"; + +function clip(id: number, over: Partial = {}): Clip { + return { + id, + playlist: 1, + position: id - 1, + recording: 10 + id, + t0: 1000 * id, + t1: 1000 * id + 5000, + media: "audio", + audio: `/api/v1/recordings/${10 + id}/audio?sig=x`, + link: `/resources/${10 + id}`, + ...over, + }; +} + +const ids = (xs: { id: number }[]) => xs.map((x) => x.id); + +describe("playing a playlist back to back", () => { + it("goes on to the next clip with media, skipping clips of text, and stops at the end", () => { + const items = [clip(1), clip(2, { audio: null, media: "transcript" }), clip(3), clip(4, { audio: null })]; + expect(nextPlayable(items, 0)).toBe(2); + expect(nextPlayable(items, 2)).toBeNull(); + expect(nextPlayable(items, -1)).toBe(0); + expect(firstPlayable(items)).toBe(0); + expect(firstPlayable([clip(1, { audio: null }), clip(2)])).toBe(1); + expect(firstPlayable([clip(1, { audio: null })])).toBeNull(); + }); + + it("moves on when a clip reaches its end", () => { + const c = clip(1); // 1000 to 6000 + expect(reachedEnd(c, 5000)).toBe(false); + expect(reachedEnd(c, 5960)).toBe(true); + expect(reachedEnd(c, 7000)).toBe(true); + }); + + it("adds up how long the clips play", () => { + expect(totalMs([clip(1), clip(2)])).toBe(10000); + expect(totalMs([])).toBe(0); + }); + + it("keeps the clip playing when the list changes around it", () => { + const before = [clip(1), clip(2), clip(3)]; + expect(followCurrent(before, moveTo(before, 0, 2), 0)).toBe(2); + expect(followCurrent(before, removeItem(before, 2).items, 1)).toBe(1); // gone: the one now in its place + expect(followCurrent(before, removeItem(before, 3).items, 2)).toBe(1); + expect(followCurrent(before, [], 0)).toBeNull(); + expect(followCurrent(before, before, null)).toBeNull(); + }); +}); + +describe("reordering", () => { + const items = [clip(1), clip(2), clip(3), clip(4)]; + + it("moves a clip by dragging it to another place", () => { + expect(ids(moveTo(items, 0, 2))).toEqual([2, 3, 1, 4]); + expect(ids(moveTo(items, 3, 0))).toEqual([4, 1, 2, 3]); + expect(ids(moveTo(items, 1, 99))).toEqual([1, 3, 4, 2]); + expect(moveTo(items, 1, 1)).toBe(items); // nothing to save + expect(moveTo(items, 9, 0)).toBe(items); + }); + + it("moves a clip up or down one place, staying put at either end", () => { + expect(ids(moveBy(items, 3, -1))).toEqual([1, 3, 2, 4]); + expect(ids(moveBy(items, 3, 1))).toEqual([1, 2, 4, 3]); + expect(moveBy(items, 1, -1)).toBe(items); + expect(moveBy(items, 4, 1)).toBe(items); + expect(moveBy(items, 99, 1)).toBe(items); + }); +}); + +describe("removing with Undo", () => { + it("takes a clip out and puts it back where it was", () => { + const items = [clip(1), clip(2), clip(3)]; + const { items: left, removed } = removeItem(items, 2); + expect(ids(left)).toEqual([1, 3]); + expect(removed).toEqual({ item: items[1], index: 1 }); + expect(ids(restoreItem(left, removed!))).toEqual([1, 2, 3]); + // the list got shorter meanwhile: it goes at the end + expect(ids(restoreItem([clip(1)], { item: clip(5), index: 3 }))).toEqual([1, 5]); + expect(removeItem(items, 99)).toEqual({ items, removed: null }); + }); + + it("adds it again through the API at its old place, with all it had", () => { + const c = clip(2, { title: "Intro", note: "start here", quote: "hello", highlight: 7 }); + expect(readdBody({ item: c, index: 1 })).toEqual({ + recording: 12, + t0: 2000, + t1: 7000, + title: "Intro", + note: "start here", + quote: "hello", + highlight: 7, + position: 1, + }); + expect(readdBody({ item: clip(1, { title: null }), index: 0 }).title).toBeUndefined(); + }); +}); + +describe("keeping a clip", () => { + it("goes in your latest used playlist, never one someone else shared", () => { + expect( + defaultPlaylist([ + { id: 3, mine: false }, + { id: 5, mine: true }, + { id: 2, mine: true }, + ])?.id, + ).toBe(5); + expect(defaultPlaylist([{ id: 3, mine: false }])).toBeNull(); + expect(defaultPlaylist(undefined)).toBeNull(); + }); + + it("is titled by the words picked, cut to a line, or by its resource and moment", () => { + expect(titleFromQuote(" the capsid\n model beat ")).toBe("the capsid model beat"); + expect(titleFromQuote("x".repeat(100), 10)).toBe(`${"x".repeat(9)}…`); + expect(clipTitle(clip(1, { title: "Intro" }))).toBe("Intro"); + expect(clipTitle(clip(1, { recording_title: "Episode 1", t0: 65000 }))).toBe("Episode 1 at 1:05"); + expect(clipTitle(clip(1, { t0: 0 }))).toBe("Asset 11 at 0:00"); + }); +}); diff --git a/nextjs-frontend/__tests__/recording-comments.test.tsx b/nextjs-frontend/__tests__/recording-comments.test.tsx index 0841668e..ab70b8a6 100644 --- a/nextjs-frontend/__tests__/recording-comments.test.tsx +++ b/nextjs-frontend/__tests__/recording-comments.test.tsx @@ -20,6 +20,10 @@ const api = { createHighlight: jest.fn, [unknown]>(), updateHighlight: jest.fn, [unknown]>(), deleteHighlight: jest.fn, [unknown]>(), + listPlaylists: jest.fn, [unknown]>(), + createPlaylist: jest.fn, [unknown]>(), + addClip: jest.fn, [unknown]>(), + removeClip: jest.fn, [unknown]>(), }; jest.mock("@/app/openapi-client", () => ({ Comments: { @@ -32,10 +36,20 @@ jest.mock("@/app/openapi-client", () => ({ updateHighlight: (a: unknown) => api.updateHighlight(a), deleteHighlight: (a: unknown) => api.deleteHighlight(a), }, + Playlists: { + listPlaylists: (a: unknown) => api.listPlaylists(a), + createPlaylist: (a: unknown) => api.createPlaylist(a), + addClip: (a: unknown) => api.addClip(a), + removeClip: (a: unknown) => api.removeClip(a), + }, })); jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); const toast = jest.fn(); jest.mock("@/components/ui/toast", () => ({ useToast: () => toast })); +jest.mock("next/link", () => ({ + __esModule: true, + default: ({ children }: { children: React.ReactNode }) => children, +})); const comment = (over: Partial): Comment => ({ id: 1, @@ -270,4 +284,39 @@ describe("the Highlights tab", () => { expect(within(first).getByRole("button", { name: "Remove highlight" })).toHaveAttribute("aria-disabled", "true"); expect(screen.getByText(/Editors of pods make them/)).toBeInTheDocument(); }); + it("keeps a highlight as a clip in the latest playlist in one click, with Undo", async () => { + api.listPlaylists.mockImplementation(() => + ok([ + { id: 4, name: "Shared one", mine: false, shared: true, count: 1, duration_ms: 1 }, + { id: 7, name: "Talks", mine: true, shared: false, count: 0, duration_ms: 0 }, + ]), + ); + api.addClip.mockImplementation(() => ok({ id: 30, playlist: 7 })); + api.removeClip.mockImplementation(() => ok({ ok: true })); + show("highlights", { canEdit: false }); + const list = await screen.findByRole("list", { name: "Highlights" }); + const [first] = within(list).getAllByRole("listitem"); + fireEvent.click(within(first).getByRole("button", { name: "Add to playlist" })); + await waitFor(() => expect(api.addClip).toHaveBeenCalledTimes(1)); + expect(api.addClip.mock.calls[0][0]).toMatchObject({ path: { pid: 7 }, body: { highlight: 1 } }); + expect(api.createPlaylist).not.toHaveBeenCalled(); + await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Added to Talks" }))); + const shown = toast.mock.calls.find(([t]) => t.title === "Added to Talks")![0]; + shown.action.onClick(); + await waitFor(() => + expect(api.removeClip).toHaveBeenCalledWith(expect.objectContaining({ path: { pid: 7, iid: 30 } })), + ); + }); + + it("makes a first playlist called Clips when there is none", async () => { + api.listPlaylists.mockImplementation(() => ok([])); + api.createPlaylist.mockImplementation(() => ok({ id: 8, name: "Clips", mine: true })); + api.addClip.mockImplementation(() => ok({ id: 31, playlist: 8 })); + show("highlights"); + const list = await screen.findByRole("list", { name: "Highlights" }); + fireEvent.click(within(within(list).getAllByRole("listitem")[0]).getByRole("button", { name: "Add to playlist" })); + await waitFor(() => expect(api.addClip).toHaveBeenCalledTimes(1)); + expect(api.createPlaylist.mock.calls[0][0]).toMatchObject({ body: { name: "Clips" } }); + expect(api.addClip.mock.calls[0][0]).toMatchObject({ path: { pid: 8 } }); + }); }); diff --git a/nextjs-frontend/app/(app)/playlists/[id]/page.tsx b/nextjs-frontend/app/(app)/playlists/[id]/page.tsx new file mode 100644 index 00000000..621d2046 --- /dev/null +++ b/nextjs-frontend/app/(app)/playlists/[id]/page.tsx @@ -0,0 +1,12 @@ +import type { Metadata } from "next"; +import { notFound } from "next/navigation"; + +import { PlaylistPage } from "@/components/playlists/playlist-page"; + +export const metadata: Metadata = { title: "Playlist" }; + +export default async function Page({ params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + if (!/^\d+$/.test(id)) notFound(); + return ; +} diff --git a/nextjs-frontend/app/(app)/playlists/page.tsx b/nextjs-frontend/app/(app)/playlists/page.tsx new file mode 100644 index 00000000..b0f94db0 --- /dev/null +++ b/nextjs-frontend/app/(app)/playlists/page.tsx @@ -0,0 +1,10 @@ +import type { Metadata } from "next"; + +import { PlaylistsPage } from "@/components/playlists/playlists-page"; + +export const metadata: Metadata = { title: "Playlists" }; + +/** Playlists: clips of moments, played back to back. */ +export default function Page() { + return ; +} diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index d1e03e22..bd1716bd 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -4757,6 +4757,171 @@ export type ClientRegistration = { scope?: string | null; }; +/** + * Clip + */ +export type Clip = { + /** + * Id + */ + id: number; + /** + * Playlist + */ + playlist: number; + /** + * Position + * + * its place in the playlist, from 0 + */ + position: number; + /** + * Recording + */ + recording: number; + /** + * T0 + * + * where it starts, in ms from the start + */ + t0: number; + /** + * T1 + * + * where it ends, in ms + */ + t1: number; + /** + * Title + */ + title?: string | null; + /** + * Note + */ + note?: string | null; + /** + * Quote + * + * the words picked in the text + */ + quote?: string | null; + /** + * Highlight + * + * the highlight it was clipped from + */ + highlight?: number | null; + /** + * Recording Title + */ + recording_title?: string | null; + /** + * Namespace + */ + namespace?: string | null; + /** + * Media + * + * the resource's kind: audio, video, document, image or transcript + */ + media: string; + /** + * Audio + * + * a signed link to the resource's audio or video, or null when it has none + */ + audio?: string | null; + /** + * Link + * + * the resource's page in the web app at the clip's start + */ + link: string; + /** + * Created At + */ + created_at?: string | null; + [key: string]: unknown; +}; + +/** + * ClipCreate + */ +export type ClipCreate = { + /** + * Recording + * + * the resource the clip is of (or give highlight) + */ + recording?: number | null; + /** + * Highlight + * + * clip a highlight: its resource, passage and label, unless given here + */ + highlight?: number | null; + /** + * T0 + * + * where the clip starts, in ms from the start (required without highlight) + */ + t0?: number | null; + /** + * T1 + * + * where it ends, in ms (default: 30 seconds on, or the end of the resource) + */ + t1?: number | null; + /** + * Title + * + * what it is, in a few words + */ + title?: string | null; + /** + * Note + */ + note?: string | null; + /** + * Quote + * + * the words picked in the text + */ + quote?: string | null; + /** + * Position + * + * where it goes, from 0 (default: the end) + */ + position?: number | null; +}; + +/** + * ClipUpdate + */ +export type ClipUpdate = { + /** + * Title + * + * empty clears it + */ + title?: string | null; + /** + * Note + * + * empty clears it + */ + note?: string | null; + /** + * T0 + */ + t0?: number | null; + /** + * T1 + */ + t1?: number | null; +}; + /** * CloudModelProvider */ @@ -18670,6 +18835,190 @@ export type Player = { [key: string]: unknown; }; +/** + * Playlist + */ +export type Playlist = { + /** + * Id + */ + id: number; + /** + * Name + */ + name: string; + /** + * Description + */ + description?: string | null; + /** + * Shared + */ + shared?: boolean; + /** + * Mine + * + * you made it: only you change it + */ + mine: boolean; + /** + * Created By + * + * who made it: their email + */ + created_by?: string | null; + /** + * Created By Name + */ + created_by_name?: string | null; + /** + * Count + * + * its clips of resources you can read + */ + count: number; + /** + * Duration Ms + * + * how long those clips play, back to back + */ + duration_ms: number; + /** + * Created At + */ + created_at?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + [key: string]: unknown; +}; + +/** + * PlaylistCreate + */ +export type PlaylistCreate = { + /** + * Name + * + * unique among your playlists + */ + name: string; + /** + * Description + */ + description?: string | null; + /** + * Shared + * + * everyone signed in reads it, each seeing only the clips of resources they can read + */ + shared?: boolean; +}; + +/** + * PlaylistDetail + */ +export type PlaylistDetail = { + /** + * Id + */ + id: number; + /** + * Name + */ + name: string; + /** + * Description + */ + description?: string | null; + /** + * Shared + */ + shared?: boolean; + /** + * Mine + * + * you made it: only you change it + */ + mine: boolean; + /** + * Created By + * + * who made it: their email + */ + created_by?: string | null; + /** + * Created By Name + */ + created_by_name?: string | null; + /** + * Count + * + * its clips of resources you can read + */ + count: number; + /** + * Duration Ms + * + * how long those clips play, back to back + */ + duration_ms: number; + /** + * Created At + */ + created_at?: string | null; + /** + * Updated At + */ + updated_at?: string | null; + /** + * Items + * + * its clips of resources you can read, in order, each with what a player needs + */ + items: Array; + /** + * Hidden + * + * clips of resources you can't read, left out + */ + hidden?: number; + [key: string]: unknown; +}; + +/** + * PlaylistOrder + */ +export type PlaylistOrder = { + /** + * Items + * + * clip ids in their new order; clips not named follow, as they were + */ + items: Array; +}; + +/** + * PlaylistUpdate + */ +export type PlaylistUpdate = { + /** + * Name + */ + name?: string | null; + /** + * Description + * + * empty clears it + */ + description?: string | null; + /** + * Shared + */ + shared?: boolean | null; +}; + /** * Podcast */ @@ -49374,6 +49723,267 @@ export type RotateCollectionFeedResponses = { export type RotateCollectionFeedResponse = RotateCollectionFeedResponses[keyof RotateCollectionFeedResponses]; +export type ListPlaylistsData = { + body?: never; + path?: never; + query?: never; + url: "/api/v1/playlists"; +}; + +export type ListPlaylistsResponses = { + /** + * Response Playlists-List Playlists + * + * Successful Response + */ + 200: Array; +}; + +export type ListPlaylistsResponse = ListPlaylistsResponses[keyof ListPlaylistsResponses]; + +export type CreatePlaylistData = { + body: PlaylistCreate; + path?: never; + query?: never; + url: "/api/v1/playlists"; +}; + +export type CreatePlaylistErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type CreatePlaylistError = CreatePlaylistErrors[keyof CreatePlaylistErrors]; + +export type CreatePlaylistResponses = { + /** + * Successful Response + */ + 200: Playlist; +}; + +export type CreatePlaylistResponse = CreatePlaylistResponses[keyof CreatePlaylistResponses]; + +export type DeletePlaylistData = { + body?: never; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}"; +}; + +export type DeletePlaylistErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type DeletePlaylistError = DeletePlaylistErrors[keyof DeletePlaylistErrors]; + +export type DeletePlaylistResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type DeletePlaylistResponse = DeletePlaylistResponses[keyof DeletePlaylistResponses]; + +export type GetPlaylistData = { + body?: never; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}"; +}; + +export type GetPlaylistErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type GetPlaylistError = GetPlaylistErrors[keyof GetPlaylistErrors]; + +export type GetPlaylistResponses = { + /** + * Successful Response + */ + 200: PlaylistDetail; +}; + +export type GetPlaylistResponse = GetPlaylistResponses[keyof GetPlaylistResponses]; + +export type UpdatePlaylistData = { + body: PlaylistUpdate; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}"; +}; + +export type UpdatePlaylistErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type UpdatePlaylistError = UpdatePlaylistErrors[keyof UpdatePlaylistErrors]; + +export type UpdatePlaylistResponses = { + /** + * Successful Response + */ + 200: Playlist; +}; + +export type UpdatePlaylistResponse = UpdatePlaylistResponses[keyof UpdatePlaylistResponses]; + +export type AddClipData = { + body: ClipCreate; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}/items"; +}; + +export type AddClipErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type AddClipError = AddClipErrors[keyof AddClipErrors]; + +export type AddClipResponses = { + /** + * Successful Response + */ + 200: Clip; +}; + +export type AddClipResponse = AddClipResponses[keyof AddClipResponses]; + +export type RemoveClipData = { + body?: never; + path: { + /** + * Pid + */ + pid: number; + /** + * Iid + */ + iid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}/items/{iid}"; +}; + +export type RemoveClipErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type RemoveClipError = RemoveClipErrors[keyof RemoveClipErrors]; + +export type RemoveClipResponses = { + /** + * Successful Response + */ + 200: Ok; +}; + +export type RemoveClipResponse = RemoveClipResponses[keyof RemoveClipResponses]; + +export type UpdateClipData = { + body: ClipUpdate; + path: { + /** + * Pid + */ + pid: number; + /** + * Iid + */ + iid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}/items/{iid}"; +}; + +export type UpdateClipErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type UpdateClipError = UpdateClipErrors[keyof UpdateClipErrors]; + +export type UpdateClipResponses = { + /** + * Successful Response + */ + 200: Clip; +}; + +export type UpdateClipResponse = UpdateClipResponses[keyof UpdateClipResponses]; + +export type ReorderPlaylistData = { + body: PlaylistOrder; + path: { + /** + * Pid + */ + pid: number; + }; + query?: never; + url: "/api/v1/playlists/{pid}/order"; +}; + +export type ReorderPlaylistErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ReorderPlaylistError = ReorderPlaylistErrors[keyof ReorderPlaylistErrors]; + +export type ReorderPlaylistResponses = { + /** + * Successful Response + */ + 200: PlaylistDetail; +}; + +export type ReorderPlaylistResponse = ReorderPlaylistResponses[keyof ReorderPlaylistResponses]; + export type EstimateBatchData = { body: BatchPlan; path?: never; diff --git a/nextjs-frontend/components/app-shell/app-shell.tsx b/nextjs-frontend/components/app-shell/app-shell.tsx index f1796cdf..afcb711a 100644 --- a/nextjs-frontend/components/app-shell/app-shell.tsx +++ b/nextjs-frontend/components/app-shell/app-shell.tsx @@ -19,6 +19,7 @@ import { useApprovalCount } from "@/components/approvals/use-approvals"; import { BudgetMeter } from "@/components/costs/budget-meter"; import { WelcomeTour } from "@/components/home/welcome-tour"; import { PageChat } from "@/components/page-chat/page-chat"; +import { usePlaylists } from "@/components/playlists/hooks"; import { useReplyCount } from "@/components/replies/use-replies"; import { SimilarOnSelect } from "@/components/similar/selection-popover"; import { Button, IconButton } from "@/components/ui/button"; @@ -123,12 +124,14 @@ export function AppShell({ user, children }: { user: SessionUser; children: Reac queryFn: () => data(Feeds.listFeeds({ client })), staleTime: 60_000, }); + const playlists = usePlaylists(); const replies = useReplyCount(); const counts = { recordings, reviews: reviews.data, approvals: approvals.data?.waiting || undefined, feeds: feeds.data?.length || undefined, + playlists: playlists.data?.length || undefined, replies: replies.data || undefined, }; const canImport = can("editor", namespace); diff --git a/nextjs-frontend/components/app-shell/nav-config.ts b/nextjs-frontend/components/app-shell/nav-config.ts index bb89d143..ad41c802 100644 --- a/nextjs-frontend/components/app-shell/nav-config.ts +++ b/nextjs-frontend/components/app-shell/nav-config.ts @@ -7,6 +7,7 @@ import { House, Inbox, LibraryBig, + ListVideo, MessagesSquare, NotebookPen, Podcast, @@ -34,7 +35,7 @@ export type NavItem = { /** Only shown to people who aren't administrators (admins have it under another item). */ membersOnly?: boolean; /** A count shown next to the label (library size, speakers to review in Settings, approvals and email waiting). */ - countKey?: "recordings" | "reviews" | "approvals" | "feeds" | "replies"; + countKey?: "recordings" | "reviews" | "approvals" | "feeds" | "playlists" | "replies"; /** Shown only while its count is above zero, or while its page is open, so the nav of a Lens with nothing there * stays as short as it was (on a phone, every item more pushes the ones below off the screen). */ whileCounted?: boolean; @@ -67,6 +68,15 @@ export const NAV: NavItem[] = [ { label: "Podcasts", href: "/podcasts", icon: Podcast, group: "Explore" }, // in the nav once there's a feed (Podcasts' Feeds button and Jump to lead there before that) { label: "Feeds", href: "/feeds", icon: Rss, countKey: "feeds", whileCounted: true, group: "Explore" }, + // in the nav once there's a playlist (Add to playlist's toast links there before that) + { + label: "Playlists", + href: "/playlists", + icon: ListVideo, + countKey: "playlists", + whileCounted: true, + group: "Explore", + }, { label: "Reports", href: "/reports", icon: ChartNoAxesColumn, group: "Explore" }, // what waits for your pick: in the nav while anything does; the top bar's badge and Home's row lead there too { diff --git a/nextjs-frontend/components/app-shell/nav.tsx b/nextjs-frontend/components/app-shell/nav.tsx index d3609038..1beea4f5 100644 --- a/nextjs-frontend/components/app-shell/nav.tsx +++ b/nextjs-frontend/components/app-shell/nav.tsx @@ -26,7 +26,14 @@ export function Nav({ admin: boolean; collapsed: boolean; onToggle?: () => void; - counts: { recordings?: number; reviews?: number; approvals?: number; feeds?: number; replies?: number }; + counts: { + recordings?: number; + reviews?: number; + approvals?: number; + feeds?: number; + playlists?: number; + replies?: number; + }; onNavigate?: () => void; className?: string; top?: ReactNode; diff --git a/nextjs-frontend/components/playlists/hooks.tsx b/nextjs-frontend/components/playlists/hooks.tsx new file mode 100644 index 00000000..ece87d8b --- /dev/null +++ b/nextjs-frontend/components/playlists/hooks.tsx @@ -0,0 +1,82 @@ +"use client"; + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import Link from "next/link"; +import { useCallback } from "react"; + +import { Playlists } from "@/app/openapi-client"; +import type { ClipCreate } from "@/app/openapi-client/types.gen"; +import { defaultPlaylist, FIRST_PLAYLIST } from "@/components/playlists/model"; +import { useToast } from "@/components/ui/toast"; +import { ApiError, data, useApiClient } from "@/lib/api/browser"; + +export const pk = { + all: ["playlists"] as const, + one: (id: number) => ["playlists", id] as const, +}; + +export function usePlaylists() { + const client = useApiClient(); + return useQuery({ queryKey: pk.all, queryFn: () => data(Playlists.listPlaylists({ client })), staleTime: 30_000 }); +} + +export function usePlaylist(id: number) { + const client = useApiClient(); + // the media links in it are signed for a while; changes refresh it + return useQuery({ + queryKey: pk.one(id), + queryFn: () => data(Playlists.getPlaylist({ client, path: { pid: id } })), + staleTime: 60_000, + }); +} + +/** + * Keep a moment as a clip in one click: it goes at the end of your latest used playlist (a new one called Clips when + * you have none), and the toast says where, with Undo. + */ +export function useKeepClip() { + const client = useApiClient(); + const qc = useQueryClient(); + const toast = useToast(); + const add = useMutation({ + mutationFn: async (body: Omit) => { + const list = await qc.fetchQuery({ + queryKey: pk.all, + queryFn: () => data(Playlists.listPlaylists({ client })), + staleTime: 30_000, + }); + const target = + defaultPlaylist(list) ?? (await data(Playlists.createPlaylist({ client, body: { name: FIRST_PLAYLIST } }))); + const clip = await data(Playlists.addClip({ client, path: { pid: target.id }, body })); + return { clip, playlist: target }; + }, + onSuccess: ({ clip, playlist }) => { + void qc.invalidateQueries({ queryKey: pk.all }); + toast({ + title: `Added to ${playlist.name}`, + tone: "green", + body: ( + + Open the playlist + + ), + action: { + label: "Undo", + onClick: () => + void data(Playlists.removeClip({ client, path: { pid: playlist.id, iid: clip.id } })).then( + () => qc.invalidateQueries({ queryKey: pk.all }), + () => toast({ title: "Couldn't undo it", tone: "red" }), + ), + }, + }); + }, + onError: (e: unknown) => + toast({ + title: "Couldn't add it to a playlist", + body: e instanceof ApiError ? e.message : "Please try again.", + tone: "red", + }), + }); + const keep = useCallback((body: Omit) => add.mutate(body), [add]); + return { keep, pending: add.isPending }; +} diff --git a/nextjs-frontend/components/playlists/model.ts b/nextjs-frontend/components/playlists/model.ts new file mode 100644 index 00000000..0aa9f5c9 --- /dev/null +++ b/nextjs-frontend/components/playlists/model.ts @@ -0,0 +1,106 @@ +import type { Clip, Playlist } from "@/app/openapi-client/types.gen"; +import { tc } from "@/lib/format"; + +/** The playlist a clip goes in by default: your latest used one (the list has yours first, the latest used first). */ +export function defaultPlaylist

>(list: P[] | undefined): P | null { + return list?.find((p) => p.mine) ?? null; +} + +/** The name a first playlist gets when you keep a clip and have none (as the API's agents get). */ +export const FIRST_PLAYLIST = "Clips"; + +/** A clip's title: its own, else the resource's with where it starts. */ +export function clipTitle(c: Pick): string { + return c.title || `${c.recording_title || `Asset ${c.recording}`} at ${tc(c.t0)}`; +} + +/** A title for a clip of selected words: the words, cut to a short line. */ +export function titleFromQuote(quote: string, most = 80): string { + const q = quote.replace(/\s+/g, " ").trim(); + return q.length > most ? `${q.slice(0, most - 1).trimEnd()}…` : q; +} + +/** How long clips play back to back, in ms. */ +export function totalMs(items: Pick[]): number { + return items.reduce((n, c) => n + Math.max(0, c.t1 - c.t0), 0); +} + +/** The next clip to play after `index` that has media; null at the end. Clips of text (no audio) are skipped. */ +export function nextPlayable(items: Pick[], index: number): number | null { + for (let i = index + 1; i < items.length; i++) if (items[i].audio) return i; + return null; +} + +/** The first clip with media at or after `index`; null when none is. */ +export function firstPlayable(items: Pick[], index = 0): number | null { + return index < items.length && items[index]?.audio ? index : nextPlayable(items, index); +} + +/** Whether the clip playing has reached its end (ms into the resource), so the player goes on to the next. */ +export function reachedEnd(c: Pick, ms: number): boolean { + return ms >= c.t1 - 50; // a frame's slack: timeupdate fires a few times a second +} + +/** The list with the item at `from` moved to `to` (both clamped). */ +export function moveTo(items: T[], from: number, to: number): T[] { + if (from < 0 || from >= items.length) return items; + const at = Math.max(0, Math.min(to, items.length - 1)); + if (at === from) return items; + const next = items.slice(); + const [it] = next.splice(from, 1); + next.splice(at, 0, it); + return next; +} + +/** The list with item `id` moved up (-1) or down (+1). */ +export function moveBy(items: T[], id: number, delta: -1 | 1): T[] { + const i = items.findIndex((x) => x.id === id); + return i < 0 ? items : moveTo(items, i, i + delta); +} + +export type Removed = { item: T; index: number }; + +/** The list without item `id`, and what Undo needs to put it back. */ +export function removeItem( + items: T[], + id: number, +): { items: T[]; removed: Removed | null } { + const index = items.findIndex((x) => x.id === id); + if (index < 0) return { items, removed: null }; + return { items: items.filter((x) => x.id !== id), removed: { item: items[index], index } }; +} + +/** Undo: the removed item back where it was (or at the end, when the list got shorter). */ +export function restoreItem(items: T[], removed: Removed): T[] { + const next = items.slice(); + next.splice(Math.min(removed.index, next.length), 0, removed.item); + return next; +} + +/** The body that adds a removed clip again in its place (Undo, through the API). */ +export function readdBody(removed: Removed) { + const c = removed.item; + return { + recording: c.recording, + t0: c.t0, + t1: c.t1, + title: c.title ?? undefined, + note: c.note ?? undefined, + quote: c.quote ?? undefined, + highlight: c.highlight ?? undefined, + position: removed.index, + }; +} + +/** Where the clip playing is after a reorder or a removal: the same clip's new place, else the one now there. */ +export function followCurrent( + before: { id: number }[], + after: { id: number }[], + current: number | null, +): number | null { + if (current == null) return null; + const id = before[current]?.id; + const at = after.findIndex((x) => x.id === id); + if (at >= 0) return at; + return after.length ? Math.min(current, after.length - 1) : null; +} diff --git a/nextjs-frontend/components/playlists/player.tsx b/nextjs-frontend/components/playlists/player.tsx new file mode 100644 index 00000000..13e778ce --- /dev/null +++ b/nextjs-frontend/components/playlists/player.tsx @@ -0,0 +1,151 @@ +"use client"; + +import { ExternalLink, Pause, Play, SkipBack, SkipForward } from "lucide-react"; +import Link from "next/link"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import type { Clip } from "@/app/openapi-client/types.gen"; +import { clipTitle, firstPlayable, nextPlayable, reachedEnd } from "@/components/playlists/model"; +import { recordingHref } from "@/components/search/links"; +import { IconButton } from "@/components/ui/button"; +import { tc } from "@/lib/format"; +import { cn } from "@/lib/utils"; + +/** + * Plays a playlist's clips back to back on one media element: each clip from t0 to t1 of its resource's signed link + * (as the recording player gets it), then the next with media; clips of text are skipped. Clips of one resource in a + * row play without loading it again. + */ +export function usePlaylistPlayer(items: Clip[]) { + const el = useRef(null); + const loaded = useRef(null); + const list = useRef(items); + const at = useRef(null); + const [index, setIndexState] = useState(null); + const [playing, setPlaying] = useState(false); + const [time, setTime] = useState(0); + useEffect(() => { + list.current = items; + }, [items]); + const setIndex = useCallback((i: number | null) => { + at.current = i; + setIndexState(i); + }, []); + + const start = useCallback( + (i: number | null) => { + const a = el.current; + const c = i == null ? null : list.current[i]; + if (!a || !c?.audio) { + a?.pause(); + return; + } + setIndex(i); + setTime(c.t0); + const go = () => { + a.currentTime = c.t0 / 1000; + a.play().catch(() => setPlaying(false)); + }; + if (loaded.current !== c.audio) { + loaded.current = c.audio; + a.src = c.audio; + a.addEventListener("loadedmetadata", go, { once: true }); + a.load(); + } else go(); + }, + [setIndex], + ); + + const playAll = useCallback(() => start(firstPlayable(list.current)), [start]); + const next = useCallback(() => start(nextPlayable(list.current, at.current ?? -1)), [start]); + const previous = useCallback(() => { + const cur = at.current ?? 0; + for (let i = cur - 1; i >= 0; i--) if (list.current[i]?.audio) return start(i); + start(cur); + }, [start]); + const toggle = useCallback(() => { + const a = el.current; + if (!a) return; + if (at.current == null) return playAll(); + if (a.paused) a.play().catch(() => setPlaying(false)); + else a.pause(); + }, [playAll]); + + const bind = { + ref: el, + playsInline: true, + preload: "none" as const, + onPlay: () => setPlaying(true), + onPause: () => setPlaying(false), + onEnded: () => next(), + onTimeUpdate: (e: React.SyntheticEvent) => { + const ms = e.currentTarget.currentTime * 1000; + setTime(ms); + const c = at.current == null ? null : list.current[at.current]; + if (c && reachedEnd(c, ms)) { + const n = nextPlayable(list.current, at.current ?? -1); + if (n == null) { + e.currentTarget.pause(); + setIndex(null); + } else start(n); + } + }, + }; + const current = index == null ? null : (items[index] ?? null); + return { index, current, playing, time, start, playAll, next, previous, toggle, setIndex, bind }; +} + +export type PlaylistPlayer = ReturnType; + +/** The player: the video when the clip is of one, and the controls: back, play or pause, next, the clip and where it is. */ +export function PlaylistPlayerBar({ player, count }: { player: PlaylistPlayer; count: number }) { + const { current, index, playing, time, toggle, next, previous, bind } = player; + const video = current?.media === "video"; + return ( +

+
+ ); +} diff --git a/nextjs-frontend/components/playlists/playlist-page.tsx b/nextjs-frontend/components/playlists/playlist-page.tsx new file mode 100644 index 00000000..ada54075 --- /dev/null +++ b/nextjs-frontend/components/playlists/playlist-page.tsx @@ -0,0 +1,283 @@ +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { ChevronDown, ChevronUp, GripVertical, ListVideo, Play, Trash2 } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect, useState } from "react"; + +import { Playlists } from "@/app/openapi-client"; +import type { Clip, PlaylistDetail } from "@/app/openapi-client/types.gen"; +import { pk, usePlaylist } from "@/components/playlists/hooks"; +import { + clipTitle, + followCurrent, + moveBy, + moveTo, + readdBody, + removeItem, + restoreItem, + totalMs, +} from "@/components/playlists/model"; +import { PlaylistPlayerBar, usePlaylistPlayer } from "@/components/playlists/player"; +import { recordingHref } from "@/components/search/links"; +import { Banner } from "@/components/ui/banner"; +import { Button, IconButton } from "@/components/ui/button"; +import { Input, Switch } from "@/components/ui/field"; +import { EmptyState, PageHeader, SkeletonRows } from "@/components/ui/states"; +import { useToast } from "@/components/ui/toast"; +import { ApiError, data, useApiClient } from "@/lib/api/browser"; +import { tc } from "@/lib/format"; +import { cn } from "@/lib/utils"; + +/** + * A playlist: its clips played back to back, reordered by dragging or with Up and Down, and taken out with Undo. Its + * maker renames, shares and deletes it (Undo makes it again with its clips). + */ +export function PlaylistPage({ id }: { id: number }) { + const q = usePlaylist(id); + if (q.isLoading) return ; + if (q.isError || !q.data) + return ( +
+ {q.error?.message ?? "Couldn’t load the playlist"} +
+ ); + return ; +} + +function Loaded({ p }: { p: PlaylistDetail }) { + const client = useApiClient(); + const qc = useQueryClient(); + const router = useRouter(); + const toast = useToast(); + const [items, setItems] = useState(p.items); + const [name, setName] = useState(p.name); + const [dragging, setDragging] = useState(null); + const player = usePlaylistPlayer(items); + useEffect(() => setItems(p.items), [p.items]); + useEffect(() => setName(p.name), [p.name]); + const fail = (title: string) => (e: unknown) => + toast({ title, body: e instanceof ApiError ? e.message : "Please try again.", tone: "red" }); + const refresh = () => { + void qc.invalidateQueries({ queryKey: pk.one(p.id) }); + void qc.invalidateQueries({ queryKey: pk.all }); + }; + + /** Show the new order at once, keep the clip playing where it went, then save it. */ + const apply = (next: Clip[]) => { + player.setIndex(followCurrent(items, next, player.index)); + setItems(next); + }; + const order = useMutation({ + mutationFn: (next: Clip[]) => + data(Playlists.reorderPlaylist({ client, path: { pid: p.id }, body: { items: next.map((c) => c.id) } })), + onSuccess: (d) => qc.setQueryData(pk.one(p.id), d), + onError: (e) => { + fail("Couldn't save the order")(e); + refresh(); + }, + }); + const reorder = (next: Clip[]) => { + if (next === items) return; + apply(next); + order.mutate(next); + }; + const remove = (clip: Clip) => { + const { items: next, removed } = removeItem(items, clip.id); + if (!removed) return; + apply(next); + void data(Playlists.removeClip({ client, path: { pid: p.id, iid: clip.id } })).then(() => { + refresh(); + toast({ + title: `Removed ${clipTitle(clip)}`, + tone: "green", + action: { + label: "Undo", + onClick: () => { + setItems((now) => restoreItem(now, removed)); + void data(Playlists.addClip({ client, path: { pid: p.id }, body: readdBody(removed) })).then( + refresh, + fail("Couldn't put it back"), + ); + }, + }, + }); + }, fail("Couldn't remove it")); + }; + const update = useMutation({ + mutationFn: (body: { name?: string; shared?: boolean }) => + data(Playlists.updatePlaylist({ client, path: { pid: p.id }, body })), + onSuccess: refresh, + onError: (e) => { + fail("Couldn't change the playlist")(e); + setName(p.name); + }, + }); + const rename = () => { + const n = name.trim(); + if (n && n !== p.name) update.mutate({ name: n }); + else setName(p.name); + }; + const del = useMutation({ + mutationFn: () => data(Playlists.deletePlaylist({ client, path: { pid: p.id } })), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: pk.all }); + router.push("/playlists"); + toast({ + title: `Deleted ${p.name}`, + tone: "green", + action: { + label: "Undo", + onClick: () => + void (async () => { + const made = await data( + Playlists.createPlaylist({ + client, + body: { name: p.name, description: p.description ?? undefined, shared: p.shared }, + }), + ); + for (const [index, item] of items.entries()) + await data(Playlists.addClip({ client, path: { pid: made.id }, body: readdBody({ item, index }) })); + void qc.invalidateQueries({ queryKey: pk.all }); + router.push(`/playlists/${made.id}`); + })().catch(fail("Couldn't make it again")), + }, + }); + }, + onError: fail("Couldn't delete the playlist"), + }); + + return ( +
+ setName(e.target.value)} + onBlur={rename} + onKeyDown={(e) => { + if (e.key === "Enter") e.currentTarget.blur(); + if (e.key === "Escape") setName(p.name); + }} + className="h-11 border-transparent px-1 text-[24px] font-bold hover:border-border" + /> + ) : ( + p.name + ) + } + meta={`${items.length === 1 ? "1 clip" : `${items.length} clips`} · ${tc(totalMs(items))}${ + p.mine ? "" : ` · by ${p.created_by_name || p.created_by || "someone"}` + }`} + actions={ + p.mine && ( + <> + update.mutate({ shared })} + label="Shared" + aria-label="Share with everyone signed in" + /> + + + ) + } + > + + All playlists + + + {p.description &&

{p.description}

} + {(p.hidden ?? 0) > 0 && ( + + {p.hidden === 1 ? "1 clip is" : `${p.hidden ?? 0} clips are`} of resources you can no longer read, so they’re + left out. + + )} +
+ +
+ {items.length === 0 ? ( + } title="No clips yet"> + Select words in a transcript and choose Add to playlist, or add a highlight from its tab. + + ) : ( +
    + {items.map((c, i) => ( +
  1. setDragging(i)} + onDragEnd={() => setDragging(null)} + onDragOver={(e) => p.mine && dragging != null && e.preventDefault()} + onDrop={(e) => { + e.preventDefault(); + if (dragging != null) reorder(moveTo(items, dragging, i)); + setDragging(null); + }} + aria-current={player.index === i || undefined} + className={cn( + "flex items-start gap-2 border-t border-border px-1 py-3 first:border-t-0", + player.index === i && "rounded-md bg-blue-surface", + dragging === i && "opacity-50", + )} + > + {p.mine && } + {i + 1} + player.start(i)} + size={32} + > + + +
    + {clipTitle(c)} + + + {c.recording_title || `Asset ${c.recording}`} + + {c.namespace ? ` · ${c.namespace}` : ""} · {tc(c.t0)}–{tc(c.t1)} + + {c.quote && c.quote !== c.title && ( +
    {c.quote}
    + )} + {c.note &&

    {c.note}

    } +
    + {p.mine && ( + + reorder(moveBy(items, c.id, -1))} + > + + + reorder(moveBy(items, c.id, 1))} + > + + + remove(c)}> + + + + )} +
  2. + ))} +
+ )} +
+ ); +} diff --git a/nextjs-frontend/components/playlists/playlists-page.tsx b/nextjs-frontend/components/playlists/playlists-page.tsx new file mode 100644 index 00000000..175ff0f6 --- /dev/null +++ b/nextjs-frontend/components/playlists/playlists-page.tsx @@ -0,0 +1,120 @@ +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { ListVideo, Plus, Users } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; + +import { Playlists } from "@/app/openapi-client"; +import { pk, usePlaylists } from "@/components/playlists/hooks"; +import { Banner } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/field"; +import { EmptyState, PageHeader, SkeletonRows } from "@/components/ui/states"; +import { useToast } from "@/components/ui/toast"; +import { ApiError, data, useApiClient } from "@/lib/api/browser"; +import { tc } from "@/lib/format"; + +/** Playlists: yours (the latest used first) and the ones others shared, with a name field that makes one on Enter. */ +export function PlaylistsPage() { + const client = useApiClient(); + const qc = useQueryClient(); + const router = useRouter(); + const toast = useToast(); + const list = usePlaylists(); + const [name, setName] = useState(""); + const create = useMutation({ + mutationFn: (n: string) => data(Playlists.createPlaylist({ client, body: { name: n } })), + onSuccess: (p) => { + void qc.invalidateQueries({ queryKey: pk.all }); + router.push(`/playlists/${p.id}`); + }, + onError: (e: unknown) => + toast({ + title: "Couldn't make the playlist", + body: e instanceof ApiError ? e.message : "Please try again.", + tone: "red", + }), + }); + const submit = (e: React.FormEvent) => { + e.preventDefault(); + if (name.trim()) create.mutate(name.trim()); + }; + const mine = list.data?.filter((p) => p.mine) ?? []; + const shared = list.data?.filter((p) => !p.mine) ?? []; + return ( +
+ +
+ setName(e.target.value)} + placeholder="New playlist name" + aria-label="New playlist name" + maxLength={120} + className="flex-1" + /> + +
+ {list.isLoading && } + {list.isError && ( + list.refetch()}> + Try again + + } + > + {list.error.message} + + )} + {list.data?.length === 0 && ( + } title="No playlists yet"> + Select words in a transcript and choose Add to playlist, or add a highlight from its tab: the moment goes in + your latest playlist, or a new one called Clips. + + )} + {mine.length > 0 &&
} + {shared.length > 0 &&
} +
+ ); +} + +function Section({ title, rows }: { title: string; rows: NonNullable["data"]> }) { + const id = `playlists-${title.replace(/\W+/g, "-").toLowerCase()}`; + return ( +
+

+ {title} +

+
    + {rows.map((p) => ( +
  • + + {p.shared ? ( + + ) : ( + + )} + + {p.name} + + {p.mine ? (p.shared ? "Shared" : "Only you") : `By ${p.created_by_name || p.created_by || "someone"}`} + {p.description ? ` · ${p.description}` : ""} + + + + {p.count === 1 ? "1 clip" : `${p.count} clips`} · {tc(p.duration_ms)} + + +
  • + ))} +
+
+ ); +} diff --git a/nextjs-frontend/components/recording/highlights-tab.tsx b/nextjs-frontend/components/recording/highlights-tab.tsx index ac563a58..1e8dea9d 100644 --- a/nextjs-frontend/components/recording/highlights-tab.tsx +++ b/nextjs-frontend/components/recording/highlights-tab.tsx @@ -1,10 +1,11 @@ "use client"; -import { Highlighter, Trash2 } from "lucide-react"; +import { Highlighter, ListPlus, Trash2 } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import type { Highlight } from "@/app/openapi-client/types.gen"; import { usePlayerApi } from "@/components/player/media"; +import { useKeepClip } from "@/components/playlists/hooks"; import { COLOURS, LABEL_MAX, colourOf, highlightTitle } from "@/components/recording/comments-model"; import { useRec } from "@/components/recording/context"; import { useHighlightActions, useHighlights } from "@/components/recording/hooks"; @@ -64,6 +65,7 @@ function HighlightRow({ h, focused }: { h: Highlight; focused: boolean }) { const { id, ns, paged, where } = useRec(); const api = usePlayerApi(); const { update, remove } = useHighlightActions(id); + const { keep, pending } = useKeepClip(); const [label, setLabel] = useState(h.label ?? ""); const row = useRef(null); useEffect(() => setLabel(h.label ?? ""), [h.label]); @@ -121,6 +123,16 @@ function HighlightRow({ h, focused }: { h: Highlight; focused: boolean }) { {writer(h)} · + +