From ed001e17b2e260bfe13e64dd28e3276383943cf5 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:14:47 +0000 Subject: [PATCH 1/2] feat(pipelines): convert steps that make a video's soundtrack an audio asset and a document's pages image assets Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141Zzcr9fDE5qRnKhFUVATi --- DEVLOG.md | 2 +- README.md | 6 +- docs/processing.md | 22 ++ docs/requirements.md | 1 + .../app/api/v1/routes/recordings.py | 6 +- fastapi_backend/app/domain/conversions.py | 206 ++++++++++++++++++ fastapi_backend/app/domain/deletion.py | 9 +- fastapi_backend/app/domain/events_content.py | 6 +- fastapi_backend/app/domain/jobs.py | 10 + fastapi_backend/app/domain/pipelines.py | 47 +++- fastapi_backend/app/domain/store.py | 1 + fastapi_backend/app/schemas/recordings.py | 10 + fastapi_backend/tests/api/test_conversions.py | 150 +++++++++++++ .../__tests__/pipeline-convert.test.ts | 38 ++++ nextjs-frontend/app/openapi-client/index.ts | 1 + .../app/openapi-client/types.gen.ts | 31 +++ .../components/pipelines/pipeline-canvas.tsx | 8 +- .../components/pipelines/pipeline-editor.tsx | 6 +- .../components/pipelines/pipeline-model.ts | 30 +++ .../components/pipelines/step-settings.tsx | 74 ++++++- .../components/recording/header.tsx | 16 ++ nextjs-frontend/components/ui/loop.tsx | 1 + nextjs-frontend/openapi.json | 58 +++++ 23 files changed, 718 insertions(+), 21 deletions(-) create mode 100644 fastapi_backend/app/domain/conversions.py create mode 100644 fastapi_backend/tests/api/test_conversions.py create mode 100644 nextjs-frontend/__tests__/pipeline-convert.test.ts diff --git a/DEVLOG.md b/DEVLOG.md index 718447286..b73ff9987 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -619,7 +619,7 @@ Todo: - [x] Canvas editor in the web app (pipelines and workflows): drag nodes, connect ports, node settings, I/O, save versions - [x] Attach workflows from the pipeline editor - [x] Content-type mapping on the Pipelines page (By content type) -- [ ] Asset converter steps (video → audio, document → images) as pipeline steps +- [x] Asset converter steps (video → audio, document → images) as pipeline steps (the convert step, conversions.py) - [ ] Workflow node types: HTTP call, template render, entity filter - [x] Docs: processing.md section on workflows diff --git a/README.md b/README.md index 04f55891d..a8b5fe84a 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p [specs from transcripts](docs/notes.md#specs-from-transcripts), [notes](docs/notes.md), [evidence PDFs](docs/api.md#evidence-pdfs), [published feeds](docs/feeds.md); [checks](docs/requirements.md#learn-from-it)) -- **Automate it.** Pipelines and workflows on a canvas, routines on a schedule, batch runs over a collection, and +- **Automate it.** Pipelines and workflows on a canvas (with Convert steps that make a video's soundtrack an audio + asset and a document's pages image assets), routines on a schedule, batch runs over a collection, and notifications to webhooks, Slack, Discord and Matterbridge. Everything that happens in Lens (new assets, emails, calendar events, files, feed posts, what people say to Lens, sensor readings, notes, graph changes, runs, budget warnings) is kept as a stream of activities in W3C Activity Streams form, which agents read through the API, MCP and @@ -161,7 +162,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p in the chat it told you in; agents can decline an approval but never give one. Each automation has limits and budgets, and agents make, try and run them over MCP and in the assistant, or draft one from a sentence; the studio to draw them in the web app is planned. ([Processing](docs/processing.md#workflows-and-the-canvas), - [notifications](docs/notifications.md), [automations](docs/automations.md), + [converting assets](docs/processing.md#converting-assets), [notifications](docs/notifications.md), + [automations](docs/automations.md), [approvals](docs/automations.md#approvals), [timeline](docs/timeline.md); [checks](docs/requirements.md#automate-it)) - **See what it costs.** An activity ledger of every model call, with reports, charts and budgets that cap spending. Setup asks for one overall budget after a cloud model, with a soft limit that warns and a hard limit that stops paid diff --git a/docs/processing.md b/docs/processing.md index b9f469e35..ab3cef1f6 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -671,6 +671,8 @@ audio skip themselves for imported text. Steps: template unless the step names another prompt template ([Specs from transcripts](notes.md#specs-from-transcripts)). - `report` with a template: renders an HTML page into the namespace's reports. These pages may not run scripts. - `export`: renders a template to a file (Markdown, text, HTML...) and can copy it to a storage source. +- `convert`: makes new assets from one, a video's soundtrack as audio or a document's pages as images + ([Converting assets](#converting-assets)). Any step can carry a condition: `min_minutes`, `max_minutes`, `source` (audio or transcript), `languages`. @@ -688,6 +690,26 @@ New pipelines, workflows, templates and routines don't need a name typed: the we steps, what a workflow runs on, a template's kind, a routine's schedule and actions), numbered when it's taken ("Prompt template 2"). The pencil next to the name on its page renames it later, as does `PATCH` on its address. +### Converting assets + +A Convert step makes new assets from the one it runs on. Default pipelines have none; add one in the pipeline editor +or on the canvas (Step library → Convert). + +- **To audio** (`{"type": "convert", "to": "audio"}`): a video's soundtrack becomes an audio asset, titled + " (audio)". It is m4a unless `format` says `mp3`, `opus` or `flac`; AAC sound in m4a (or MP3 + sound in mp3) is copied as it is, not encoded again. Audio, documents and videos with no sound are skipped. +- **To images** (`{"type": "convert", "to": "images"}`): each page the Read step drew becomes an image asset, + titled ", page N", the same JPEG the asset's page shows (documents.page_pixels on its longest side). Put a + Read step before it; without drawn pages it skips. A multi-page TIFF's frames work too; at most 200 pages are made + into images from one document. + +New assets are kept in the same namespace and collection, encrypted where its files are, and each says which asset +(and page) it was made from: `converted_from` on `GET /api/v1/recordings/{id}`, and a link on its page. They then run +the pipeline their content type gives them, as an upload does; `pipeline` names another pipeline by its id, or +`"none"` keeps them as they are until a pipeline is run on them. Running the step again keeps what it made before; +`force` makes their files again. An asset a Convert step made isn't converted again, so a pipeline can't feed itself, +and deleting it deletes its file. + ## Workflows and the canvas A pipeline's steps make derived files (a transcript or a page's text, shots, text on screen, faces); workflows turn them into metadata. A diff --git a/docs/requirements.md b/docs/requirements.md index 7a4214b31..a54d307b1 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -177,6 +177,7 @@ request, to another check that proves the same thing; loosening a requirement or | Requirement | Checks | |---|---| | Pipelines and workflows on a canvas | [test_workflows.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_workflows.py) `test_workflows_run_in_pipelines_drawn_as_graphs`; [workflow-canvas.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-canvas.test.ts); [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `pipeline-create`, `workflow-create` | +| Convert steps make new assets: a video's soundtrack as audio, a document's pages as images, each run through its own pipeline | [test_conversions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_conversions.py) `test_a_videos_soundtrack_becomes_an_audio_asset`, `test_a_documents_pages_become_image_assets`, `test_settings_are_checked`; [pipeline-convert.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/pipeline-convert.test.ts) `is added from the library ready to run: to audio unless images are picked` | | A Classify node in workflows chooses from labels, content types or topics, and acting on a sure answer keeps something | [test_classify.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_classify.py) `test_a_workflow_classifies`; [workflow-flow.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/workflow-flow.test.ts) `chooses from labels, content types or topics, and acting on a sure answer keeps something` | | Routines on a schedule | [test_routines.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_routines.py) `test_routines_are_checked_and_run_when_due`, `test_schedules`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `routine-create`; [baseline.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline.json) `routine-create` ≤ 2 clicks | | Batch runs over a collection | [test_batches.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_batches.py) `test_collections_and_batches` | diff --git a/fastapi_backend/app/api/v1/routes/recordings.py b/fastapi_backend/app/api/v1/routes/recordings.py index 50a5c9b50..671729655 100644 --- a/fastapi_backend/app/api/v1/routes/recordings.py +++ b/fastapi_backend/app/api/v1/routes/recordings.py @@ -324,6 +324,7 @@ def get_recording(rid: int, acl: Acl, db: Db, cfg: Cfg) -> Recording: ] d["jobs"] = jobs.list_jobs(db, recording=rid, limit=5) d["attached_to"] = _attached_to(acl, db, r.get("attached_to")) + d["converted_from"] = _attached_to(acl, db, r.get("converted_from")) if r.get("enrichment") and acl.member(r): # where a photo was taken: members only d["enrichment"] = {k: v for k, v in r["enrichment"].items() if k != "photo_fp"} d["nsfw"] = _nsfw(db, cfg, r, rid) @@ -350,14 +351,15 @@ def mark_recording_nsfw(rid: int, body: RecordingNsfwUpdate, acl: Acl, user: Wri def _attached_to(acl: Access, db: DB, link: dict[str, Any] | None) -> dict[str, Any] | None: - """The email an attachment came from, with its title, when the caller may see that email.""" + """The email an attachment came from (or the asset a convert step made this one from), with its title, when the + caller may see it.""" if not link or not link.get("resource"): return None try: email = acl.recording(int(link["resource"])) except HTTPException: return None # deleted since, or not theirs to see - return {"resource": int(link["resource"]), "file": link.get("file"), "title": email.get("title")} + return {**link, "resource": int(link["resource"]), "title": email.get("title")} @router.patch("/{rid}") diff --git a/fastapi_backend/app/domain/conversions.py b/fastapi_backend/app/domain/conversions.py new file mode 100644 index 000000000..7561399ca --- /dev/null +++ b/fastapi_backend/app/domain/conversions.py @@ -0,0 +1,206 @@ +"""The convert step: new assets made from one (docs/processing.md#converting-assets). + +`{type: convert, to: audio}` makes a video's soundtrack an audio asset of its own, as m4a (AAC, copied as it is when +the video's sound is AAC already), mp3, opus or flac (`format`). `{type: convert, to: images}` makes each page a +document's Read step drew (or each frame of a TIFF) an image asset of its own, the JPEG Lens shows for that page. + +A new asset lives in the namespace and collection of the one it was made from, says which asset and page it was made +from (converted_from), and is kept encrypted like an upload. It then runs the pipeline its content type gives it, as an +upload does, or the one the step names (`pipeline`), or none (`pipeline: "none"`): it's kept as it is until a pipeline +is run on it. Running the step again keeps the assets it made before (forced, it makes their files again). An asset +made by a convert step isn't converted again, so a pipeline that converts can't feed itself. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import subprocess + +from . import documents, ingest, jobs, keyring, store, video + +R = store.R +TO = ("audio", "images") +# format: (codec that is copied as it is, ffmpeg's encoder and its options) +AUDIO = { + "m4a": ("aac", ["-c:a", "aac", "-b:a", "128k"]), + "mp3": ("mp3", ["-c:a", "libmp3lame", "-q:a", "4"]), + "opus": ("opus", ["-c:a", "libopus", "-b:a", "64k"]), + "flac": ("flac", ["-c:a", "flac"]), +} +MAX_PAGES = 200 # images made from one document at most + + +def check(spec): + """A convert step's settings (pipelines.validate_steps): ValueError naming the first problem.""" + if spec.get("to") not in TO: + raise ValueError("convert step: convert to audio or images") + fmt = spec.get("format") + if fmt is not None and (spec["to"] != "audio" or fmt not in AUDIO): + if spec["to"] != "audio": + raise ValueError("convert step: images are JPEGs, with no format to choose") + raise ValueError(f"convert step: the audio format is one of {', '.join(AUDIO)}") + p = spec.get("pipeline") + if p is not None and p != "none" and (isinstance(p, bool) or not isinstance(p, int)): + raise ValueError('convert step: pipeline is a pipeline\'s id, or "none" to keep the new assets as they are') + + +def made_from(db, rid, to): + """The assets a convert step made from this one before: {page or None: id}.""" + rows = db.rows( + "SELECT record::id(id) AS id, converted_from FROM recording WHERE converted_from.resource = $r AND converted_from.to = $t", + r=int(rid), + t=to, + ) + return {(r["converted_from"] or {}).get("page"): r["id"] for r in rows} + + +def step(db, cfg, rid, say, spec): + rec = db.one( + "SELECT space, collection, path, remote, source, title, recorded_at, duration_ms, converted_from FROM $r", + r=R("recording", rid), + ) + if not rec: + raise KeyError(rid) + if rec.get("converted_from"): + raise jobs.Skip("it was made by a convert step itself, so it isn't converted again") + if spec["to"] == "audio": + return _audio(db, cfg, rid, rec, say, spec) + return _images(db, cfg, rid, rec, say, spec) + + +def _stem(rec, rid): + stem = pathlib.PurePosixPath(str(rec.get("path") or "")).stem or str(rec.get("title") or "") + return re.sub(r"[^\w .()-]+", "_", stem).strip(" .")[:100] or f"asset-{int(rid)}" + + +def _folder(db, cfg, rec, rid): + ns = store.space_names(db).get(rec["space"]) or str(rec["space"]) + d = pathlib.Path(cfg["data_dir"]) / "uploads" / ns / f"converted-{int(rid)}" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _audio_codec(path): + out = subprocess.run( + ["ffprobe", "-v", "error", "-format_whitelist", ingest.MEDIA_DEMUXERS, "-select_streams", "a:0"] + + ["-show_entries", "stream=codec_name", "-of", "json", str(path)], + capture_output=True, + text=True, + timeout=120, + ) + streams = json.loads(out.stdout or "{}").get("streams") or [] + return (streams[0] if streams else {}).get("codec_name") + + +def _audio(db, cfg, rid, rec, say, spec): + if rec.get("source") != "audio": + raise jobs.Skip("only a video has a soundtrack to make into audio") + path = ingest.audio_path(db, cfg, rec) + if not path or not os.path.exists(path): + raise jobs.Skip("there is no media file") + media = video.probe_media(path) + if media["kind"] != "video": + raise jobs.Skip("it's audio already") + if not media.get("has_audio"): + raise jobs.Skip("the video has no sound") + have = made_from(db, rid, "audio").get(None) + if have and not spec.get("force"): + return say(f"kept the audio made from it before (asset {have}); force the step to make it again") + fmt = spec.get("format") or "m4a" + same, encode = AUDIO[fmt] + codec = ["-c:a", "copy"] if _audio_codec(path) == same else encode + dest = _folder(db, cfg, rec, rid) / f"{_stem(rec, rid)}.{fmt}" + tmp = dest.with_name(f".part-{dest.name}") + argv = ["ffmpeg", "-nostdin", "-v", "error", "-y", "-format_whitelist", ingest.MEDIA_DEMUXERS, "-i", str(path)] + # no picture, no tags: the same video makes the same file, byte for byte + argv += ["-map", "0:a:0", "-vn", "-sn", "-dn", "-map_metadata", "-1", "-fflags", "+bitexact", "-flags:a", "+bitexact", *codec, str(tmp)] + try: + out = subprocess.run(argv, capture_output=True, timeout=ingest.DECODE_SECONDS) + if out.returncode != 0: + raise RuntimeError("ffmpeg: " + out.stderr.decode(errors="replace")[-300:]) + os.replace(tmp, dest) + finally: + tmp.unlink(missing_ok=True) + keyring.protect(db, cfg, rec["space"], dest) # encrypted like an upload, where its namespace's files are + title = f"{rec.get('title') or _stem(rec, rid)} (audio)" + more = {"duration_ms": media["duration_ms"] or rec.get("duration_ms")} + new = _keep(db, cfg, rid, rec, dest, "audio", title, {"to": "audio"}, have, spec, more) + how = "copied" if codec[-1] == "copy" else "encoded" + say(f"made its soundtrack an audio asset ({fmt}, {how}): asset {new}") + + +def _images(db, cfg, rid, rec, say, spec): + if rec.get("source") not in documents.KINDS: + raise jobs.Skip("only a document's pages (or a TIFF's frames) are made into images") + pages = [p for p in documents.pages(db, rid) if p.get("image")] + if not pages: + raise jobs.Skip("its pages haven't been drawn: put a Read step before this one") + if rec["source"] == "image" and len(pages) < 2: + raise jobs.Skip("it's one image already") + if len(pages) > MAX_PAGES: + say(f"only its first {MAX_PAGES} of {len(pages)} pages are made into images") + pages = pages[:MAX_PAGES] + have = made_from(db, rid, "images") + d, frames, stem = _folder(db, cfg, rec, rid), video.frames_dir(cfg, rid), _stem(rec, rid) + made = kept = 0 + for p in pages: + old = have.get(p["idx"]) + if old and not spec.get("force"): + kept += 1 + continue + src = frames / p["image"] + if not src.exists(): + continue + dest = d / f"{stem}-page-{p['idx'] + 1:04d}.jpg" + keyring.keep(db, cfg, rec["space"], dest, keyring.read_plain(db, cfg, src)) # encrypted as it's written, if wanted + name = p.get("label") or str(p["idx"] + 1) + title = f"{rec.get('title') or stem}, page {name}" + _keep(db, cfg, rid, rec, dest, "image", title, {"to": "images", "page": p["idx"]}, old, spec, {"media": {"kind": "image"}}) + made += 1 + say(f"made {made} page(s) into image assets" + (f"; kept the {kept} made before (force the step to make them again)" if kept else "")) + + +def _keep(db, cfg, rid, rec, dest, source, title, link, old, spec, more): + """The converted file (kept as it's to be, encrypted or not) as an asset: a new one, or the one made before when + forced; queued for its pipeline.""" + st, fp = dest.stat(), ingest.fingerprint(dest, db=db, cfg=cfg) + row = { + "path": str(dest), + "source": source, + "size": keyring.plain_size(db, cfg, dest), + "mtime": st.st_mtime, + "fingerprint": fp, + "fp_key": f"{rec['space']}:{fp}", + **more, + } + if old: + db.q("UPDATE $r MERGE $d", r=R("recording", old), d={**row, "status": "new"}) + new = old + else: + known = db.one("SELECT record::id(id) AS id FROM recording WHERE fp_key = $k", k=row["fp_key"]) + if known: # the same file is an asset here already + return known["id"] + new = db.next_id("recording") + db.q( + "CREATE $r CONTENT $d", + r=R("recording", new), + d=store.clean( + { + **row, + "space": rec["space"], + "collection": rec.get("collection"), + "title": title[:200], + "recorded_at": rec.get("recorded_at") or store.now(), + "converted_from": {"resource": int(rid), **link}, + "status": "new", + "created_at": store.now(), + } + ), + ) + then = spec.get("pipeline") + if then != "none": + jobs.enqueue(db, new, None, by=f"convert:{int(rid)}", pipeline=then) + return new diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py index 1264304d1..771071884 100644 --- a/fastapi_backend/app/domain/deletion.py +++ b/fastapi_backend/app/domain/deletion.py @@ -20,7 +20,7 @@ import pathlib import shutil -from . import access as acc, files as filemod, notebook, render, store, video +from . import access as acc, files as filemod, keyring, notebook, render, store, video R = store.R log = logging.getLogger("lens") @@ -83,11 +83,14 @@ def stop_jobs(db, rid, verb="delete"): def _files(db, cfg, rec, rid, ns): """The files Lens made from the recording: its frames, its report pages, the PDF made of a document, the exports - only it wrote, and a cached copy of a remote file that no other recording uses. Never the media file.""" + only it wrote, and a cached copy of a remote file that no other recording uses. Never the media file, unless a + convert step made it (conversions.py).""" data = pathlib.Path(cfg["data_dir"]) reports = data / "reports" / (ns or "_") name = f"{render.slug(rec.get('title'))}-{rid}" out = [reports / f"{name}.html", *reports.glob(f"{name}--*.html"), data / "renditions" / f"{int(rid)}.pdf"] + if rec.get("converted_from") and rec.get("path") and keyring.owned(cfg, rec["path"]): + out.append(pathlib.Path(rec["path"])) exports = data / "exports" / (ns or "_") mine = { (o.get("value") or {}).get("file") @@ -152,7 +155,7 @@ def orphans(db, speakers, faces): def delete(db, cfg, rid, by=None): """Delete a recording (see the module docstring). Returns what the audit log keeps of it. Raises KeyError when there is no such recording and Running when a job is working on it.""" - rec = db.one("SELECT title, space, path, fingerprint, remote, source, media FROM $r", r=R("recording", rid)) + rec = db.one("SELECT title, space, path, fingerprint, remote, source, media, converted_from FROM $r", r=R("recording", rid)) if not rec: raise KeyError(rid) stop_jobs(db, rid) diff --git a/fastapi_backend/app/domain/events_content.py b/fastapi_backend/app/domain/events_content.py index b69f27868..4517301f9 100644 --- a/fastapi_backend/app/domain/events_content.py +++ b/fastapi_backend/app/domain/events_content.py @@ -36,7 +36,7 @@ R = store.R FIELDS = ( "id AS rid, record::id(id) AS id, space, title, source, media, remote, content_type, created_at, path, web, engine, " - "created_by, mail, event, post, email, size, duration_ms, attached_to, sensor, form" + "created_by, mail, event, post, email, size, duration_ms, attached_to, converted_from, sensor, form" ) TYPES = {"audio": "Audio", "video": "Video", "image": "Image", "document": "Document", "transcript": "Document"} AUTOMATION_RX = re.compile(r"^automation:(\d+)$") @@ -335,8 +335,8 @@ def _imported(r, actor): def _kind(r, ctype, source_type, actor): """The most specific kind the row tells (module docstring).""" - if r.get("attached_to"): - return "asset.added" # a file an email or archive held: the email is what arrived + if r.get("attached_to") or r.get("converted_from"): + return "asset.added" # a file an email or archive held (the email is what arrived), or one a convert step made if ctype == "email" or source_type in MAIL_SOURCES: return "email.received" if ctype == "calendar_event" or source_type in CALENDAR_SOURCES: diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 242bda891..1a0bb904e 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -199,6 +199,13 @@ def _nsfw(db, cfg, rid, say, spec=None): nsfw.step(db, cfg, rid, say) +def _convert(db, cfg, rid, say, spec=None): + """New assets made from this one: a video's soundtrack as audio, a document's pages as images (conversions.py).""" + from . import conversions + + conversions.step(db, cfg, rid, say, spec) + + def _describe(db, cfg, rid, say, spec=None): from . import descriptions @@ -216,6 +223,7 @@ def _describe(db, cfg, rid, say, spec=None): "objects": _objects, "nsfw": _nsfw, "describe": _describe, + "convert": _convert, "clean": _clean, "analyze": _analyze, "embed": _embed, @@ -939,6 +947,8 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None): self.can.add("embed") if "analyze" in self.can: # cleaning up a transcript needs no more than analysing it; older lists run it too self.can.add("clean") + if not steps and self.can & {"transcribe", "read"}: # converting needs what they need; older lists run it too + self.can.add("convert") if not steps and "objects" in self.can: # NSFW classification runs on ONNX Runtime too; older lists run it as well self.can.add("nsfw") if not steps and log and (missing := sorted(set(STEPS) - self.can)): diff --git a/fastapi_backend/app/domain/pipelines.py b/fastapi_backend/app/domain/pipelines.py index 1b52c5198..d0666141e 100644 --- a/fastapi_backend/app/domain/pipelines.py +++ b/fastapi_backend/app/domain/pipelines.py @@ -1,6 +1,7 @@ """Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without one, recordings get the standard pipeline. Steps: transcribe (speech to text), read (a document's or an -image's pages and their text), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), describe, +image's pages and their text), convert (new assets made from one: a video's soundtrack as audio, a document's pages +as images, conversions.py), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), describe, clean (a transcript tidied before analysis, cleanup.py), analyze, embed (passages for search by meaning, semantic.py), summarize, llm (a prompt template whose structured result is saved as a named output), spec (a technical spec from the transcript, written to a note linked to the recording, specs.py), report (built in, or from a report template) and @@ -29,14 +30,42 @@ ] # fmt: skip TYPES = { "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", + "convert", "llm", "spec", "report", "export", "workflow", } # fmt: skip -ASSET_STEPS = ("transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe") # they make something of the media -KEYS = {"type", "name", "when", "template", "version", "key", "filename", "destination", "model", "force", "workflow"} +ASSET_STEPS = ( + "transcribe", + "read", + "diarize", + "shots", + "ocr", + "faces", + "objects", + "nsfw", + "describe", + "convert", +) # they make something of the media +KEYS = { + "type", + "name", + "when", + "template", + "version", + "key", + "filename", + "destination", + "model", + "force", + "workflow", + "to", + "format", + "pipeline", +} +ONLY = {"to": "convert", "format": "convert", "pipeline": "convert"} # settings only one type of step has NODE_RX = re.compile(r"^[A-Za-z0-9_-]{1,40}$") WHEN = {"min_minutes": (int, float), "max_minutes": (int, float), "source": str, "languages": list} KEY_RX = re.compile(r"^[a-z][a-z0-9_]{0,40}$") @@ -52,6 +81,9 @@ def validate_steps(db, steps): extra = sorted(set(s) - KEYS) if extra: raise ValueError(f"{t} step has no setting {extra[0]}") + other = next((k for k in s if ONLY.get(k, t) != t), None) + if other: + raise ValueError(f"{t} step has no setting {other}") for k, v in (s.get("when") or {}).items(): if k not in WHEN or not isinstance(v, WHEN[k]): raise ValueError(f"conditions are {', '.join(WHEN)}") @@ -64,6 +96,15 @@ def validate_steps(db, steps): raise ValueError("workflow step: choose a workflow") from None if w["scope"] != "recording": raise ValueError(f"workflow step: {w['name']} {workflows.RUN_BY.get(w['scope'], 'runs elsewhere')}") + if t == "convert": + from . import conversions + + conversions.check(s) + if s.get("pipeline") not in (None, "none"): + try: + get(db, s["pipeline"]) + except KeyError: + raise ValueError("convert step: choose a pipeline for the new assets") from None need = {"llm": "prompt", "spec": "prompt", "export": "export", "report": "report"}.get(t) if need and (s.get("template") or t not in ("report", "spec")): # a report or a spec has a built-in one try: diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 362812c12..f31bceca1 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -283,6 +283,7 @@ def labels(): "objects", "nsfw", "describe", + "convert", "clean", "analyze", "embed", diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index 81642fe82..6c26192b3 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -255,6 +255,15 @@ class AttachedTo(ResponseModel): title: str | None = None +class ConvertedFrom(ResponseModel): + """The asset a convert step made this one from (docs/processing.md#converting-assets).""" + + resource: int + to: Literal["audio", "images"] + page: int | None = Field(None, description="the page it is, from 0, for images made of a document's pages") + title: str | None = None + + class Recording(ResponseModel): """The asset row (less its envelope) plus what the asset page needs.""" @@ -282,6 +291,7 @@ class Recording(ResponseModel): rendition: Rendition | None = Field(None, description="a document that isn't a PDF: how the PDF it's read from was made") described: Described | None = Field(None, description="a document: what it says about itself (title, authors, dates), read by Tika") attached_to: AttachedTo | None = Field(None, description="an email's attachment made a resource of its own: that email") + converted_from: ConvertedFrom | None = Field(None, description="made by a convert step: the asset it was made from") web: WebPage | None = Field(None, description="a web page captured as a document") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") nsfw: NsfwState | None = Field(None, description="NSFW classification, where it's on in its namespace and it was classified") diff --git a/fastapi_backend/tests/api/test_conversions.py b/fastapi_backend/tests/api/test_conversions.py new file mode 100644 index 000000000..b39866db9 --- /dev/null +++ b/fastapi_backend/tests/api/test_conversions.py @@ -0,0 +1,150 @@ +"""The convert step: a video's soundtrack as an audio asset, a document's pages as image assets.""" + +from __future__ import annotations + +import pathlib +import shutil + +import pytest + +from app.domain import ingest, keyring, settings, store, video +from tests.helpers import drain, login, make_user, quiet, text_pdf, write_wav +from tests.video_helpers import make_video, needs_ffmpeg + +POPPLER = pytest.mark.skipif(not shutil.which("pdftoppm"), reason="needs poppler-utils") + + +@pytest.fixture +def h(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + return login(client, "root@x.io", "root password 1") + + +def _pipeline(client, h, steps, name="Convert"): + r = client.post("/api/v1/pipelines", headers=h, json={"name": name, "steps": steps}) + assert r.status_code == 200, r.text + return r.json()["id"] + + +def _run(client, h, db, cfg, pid, rid): + r = client.post(f"/api/v1/pipelines/{pid}/run", headers=h, json={"recording": rid}) + assert r.status_code == 200, r.text + drain(db, cfg) + job = db.one("SELECT status, log FROM $j", j=store.R("job", r.json()["job"])) + assert job["status"] == "succeeded", job["log"][-5:] + return " ".join(job["log"]) + + +def _made(db, rid): + return db.rows( + "SELECT record::id(id) AS id, title, source, path, status, converted_from, space, collection FROM recording " + "WHERE converted_from.resource = $r ORDER BY id", + r=rid, + ) + + +def _video(db, cfg, folder): + mp4, tr = folder / "review.mp4", folder / "review.txt" + make_video(mp4) + tr.write_text("[00:00] Mara: Here is the invoice.\n[00:03] Tom: And this is Atlas.") + return ingest.import_transcript(db, cfg, "pods", tr, audio=mp4, log=quiet) + + +def test_settings_are_checked(client, h): + bad = [ + {"type": "convert"}, + {"type": "convert", "to": "pdf"}, + {"type": "convert", "to": "images", "format": "png"}, + {"type": "convert", "to": "audio", "format": "ogg"}, + {"type": "convert", "to": "audio", "pipeline": 999}, + {"type": "convert", "to": "audio", "pipeline": "standard"}, + {"type": "analyze", "to": "audio"}, # only a convert step converts + ] + for step in bad: + r = client.post("/api/v1/pipelines", headers=h, json={"name": "x", "steps": [step]}) + assert r.status_code == 400, step + other = _pipeline(client, h, ["summarize"], "After") + ok = {"type": "convert", "to": "audio", "format": "mp3", "pipeline": other} + pid = _pipeline(client, h, [ok, {"type": "convert", "to": "images", "pipeline": "none"}]) + assert client.get(f"/api/v1/pipelines/{pid}", headers=h).json()["steps"][0] == ok + assert "convert" in client.get("/api/v1/pipelines", headers=h).json()["asset_steps"] + + +@needs_ffmpeg +def test_a_videos_soundtrack_becomes_an_audio_asset(client, h, db, cfg, folder): + rid = _video(db, cfg, folder) + pid = _pipeline(client, h, [{"type": "convert", "to": "audio", "pipeline": "none"}]) + log = _run(client, h, db, cfg, pid, rid) + assert "audio asset (m4a, copied)" in log # the video's sound is AAC: copied, not encoded again + [a] = _made(db, rid) + assert (a["source"], a["status"], a["converted_from"], a["title"]) == ( + "audio", + "new", + {"resource": rid, "to": "audio"}, + "review (audio)", + ) + assert video.probe_media(a["path"])["kind"] == "audio" and a["path"].endswith(f"/uploads/pods/converted-{rid}/review.m4a") + assert not db.values("SELECT VALUE id FROM job WHERE recording = $r", r=a["id"]) # "none": kept as it is + + d = client.get(f"/api/v1/recordings/{a['id']}", headers=h).json() + assert d["converted_from"] == {"resource": rid, "to": "audio", "page": None, "title": "review"} + + again = _run(client, h, db, cfg, pid, rid) + assert f"kept the audio made from it before (asset {a['id']})" in again and len(_made(db, rid)) == 1 + + mp3 = _pipeline(client, h, [{"type": "convert", "to": "audio", "format": "mp3", "force": True}], "MP3") + log = _run(client, h, db, cfg, mp3, rid) + assert "audio asset (mp3, encoded)" in log + [b] = _made(db, rid) + assert b["id"] == a["id"] and b["path"].endswith(".mp3") # forced: the same asset, its file made again + # with no pipeline named, the new asset runs the one its content type gives it, as an upload does + assert db.values("SELECT VALUE id FROM job WHERE recording = $r", r=b["id"]) + + # an asset a convert step made isn't converted again, and audio has no soundtrack to take + log = _run(client, h, db, cfg, pid, b["id"]) + assert "isn't converted again" in log + + r = client.delete(f"/api/v1/recordings/{b['id']}", headers=h) + assert r.status_code in (200, 204), r.text + assert not pathlib.Path(b["path"]).exists() # the file the step made goes with it; the video stays + assert pathlib.Path(db.one("SELECT path FROM $r", r=store.R("recording", rid))["path"]).exists() + + +@needs_ffmpeg +def test_a_video_without_sound_or_audio_alone_is_skipped(client, h, db, cfg, folder): + wav, tr = folder / "talk.wav", folder / "talk.txt" + write_wav(wav) + tr.write_text("[00:00] Ann: Hello.") + rid = ingest.import_transcript(db, cfg, "pods", tr, audio=wav, log=quiet) + pid = _pipeline(client, h, [{"type": "convert", "to": "audio"}, {"type": "convert", "to": "images"}]) + log = _run(client, h, db, cfg, pid, rid) + assert "it's audio already" in log and "only a document's pages" in log and not _made(db, rid) + + +@POPPLER +def test_a_documents_pages_become_image_assets(client, h, db, cfg): + settings.save(db, cfg, "encryption", {"files": True}) + pid = _pipeline(client, h, ["read", {"type": "convert", "to": "images", "pipeline": "none"}]) + pages = [["Harbour report"], ["Second page"], ["Third page"]] + data = text_pdf(pages) + r = client.post("/api/v1/uploads", headers=h, json={"namespace": "pods", "filename": "harbour.pdf", "size": len(data), "pipeline": pid}) + assert r.status_code == 201, r.text + r = client.put(f"/api/v1/uploads/{r.json()['id']}?offset=0", headers={**h, "Content-Type": "application/octet-stream"}, content=data) + rid = r.json()["recording"] + drain(db, settings.effective(db, cfg)) + made = _made(db, rid) + assert [m["title"] for m in made] == ["harbour, page 1", "harbour, page 2", "harbour, page 3"] + assert [m["converted_from"]["page"] for m in made] == [0, 1, 2] and {m["source"] for m in made} == {"image"} + first = pathlib.Path(made[0]["path"]) + assert keyring.is_encrypted(first) # kept encrypted, like an upload + assert keyring.read_plain(db, cfg, first)[:3] == b"\xff\xd8\xff" # a JPEG + + # without its pages drawn there's nothing to make images of + bare = _pipeline(client, h, [{"type": "convert", "to": "images"}, {"type": "convert", "to": "audio"}], "Bare") + data = text_pdf([["Alone"]]) + r = client.post("/api/v1/uploads", headers=h, json={"namespace": "pods", "filename": "alone.pdf", "size": len(data), "pipeline": bare}) + r = client.put(f"/api/v1/uploads/{r.json()['id']}?offset=0", headers={**h, "Content-Type": "application/octet-stream"}, content=data) + other = r.json()["recording"] + drain(db, settings.effective(db, cfg)) + log = " ".join(db.one("SELECT log FROM job WHERE recording = $r", r=other)["log"]) + assert "put a Read step before this one" in log and "only a video has a soundtrack" in log and not _made(db, other) diff --git a/nextjs-frontend/__tests__/pipeline-convert.test.ts b/nextjs-frontend/__tests__/pipeline-convert.test.ts new file mode 100644 index 000000000..60127d0bf --- /dev/null +++ b/nextjs-frontend/__tests__/pipeline-convert.test.ts @@ -0,0 +1,38 @@ +import { cleanSpec, moveStep, newStep, specProblems, stepSummary } from "@/components/pipelines/pipeline-model"; +import { stepLabel } from "@/components/recording/jobs"; + +describe("the convert step in a pipeline", () => { + it("is added from the library ready to run: to audio unless images are picked", () => { + expect(newStep("convert")).toEqual({ type: "convert", to: "audio" }); + expect(newStep("convert:images")).toEqual({ type: "convert", to: "images" }); + expect(newStep("llm")).toEqual({ type: "llm", key: "" }); + expect(newStep("workflow:3")).toEqual({ type: "workflow", workflow: 3 }); + expect(stepLabel("convert")).toBe("Convert"); + }); + it("keeps only the settings it has, the format only for audio", () => { + expect(cleanSpec({ type: "convert", to: "audio", format: "mp3", pipeline: "none" })).toEqual({ + type: "convert", + to: "audio", + format: "mp3", + pipeline: "none", + }); + expect(cleanSpec({ type: "convert", to: "images", format: "mp3", pipeline: 4 })).toEqual({ + type: "convert", + to: "images", + pipeline: 4, + }); + expect(cleanSpec({ type: "analyze", to: "audio" })).toBe("analyze"); + }); + it("says what it makes, and asks for what to convert to", () => { + expect(stepSummary({ type: "convert", to: "audio" })).toBe("to audio · m4a"); + expect(stepSummary({ type: "convert", to: "images", pipeline: "none" })).toBe("pages to images · not processed"); + expect(specProblems([{ type: "convert" }], () => undefined)).toEqual({ + 0: "Choose what to convert to: audio or images.", + }); + expect(specProblems([newStep("convert")], () => undefined)).toEqual({}); + }); + it("comes after the pages are read", () => { + const r = moveStep([{ type: "read" }, { type: "convert", to: "images" }], 1, 0); + expect(r.error).toBe("Convert needs Read pages’s output, so it has to come after it."); + }); +}); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 123eda165..356f089a8 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -437,6 +437,7 @@ export type { ControlWorkerResponse, ControlWorkerResponses, ConversionNodeTestResult, + ConvertedFrom, Converters, CopyPageData, CopyPageError, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 51de46f24..a24a91be7 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -5286,6 +5286,33 @@ export type ConversionNodeTestResult = { [key: string]: unknown; }; +/** + * ConvertedFrom + * + * The asset a convert step made this one from (docs/processing.md#converting-assets). + */ +export type ConvertedFrom = { + /** + * Resource + */ + resource: number; + /** + * To + */ + to: "audio" | "images"; + /** + * Page + * + * the page it is, from 0, for images made of a document's pages + */ + page?: number | null; + /** + * Title + */ + title?: string | null; + [key: string]: unknown; +}; + /** * Converters * @@ -17571,6 +17598,10 @@ export type Recording = { * an email's attachment made a resource of its own: that email */ attached_to?: AttachedTo | null; + /** + * made by a convert step: the asset it was made from + */ + converted_from?: ConvertedFrom | null; /** * a web page captured as a document */ diff --git a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx index a7592ea30..a40f4591a 100644 --- a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx @@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { + ArrowRightLeft, AudioLines, Captions, Clapperboard, @@ -36,6 +37,7 @@ import { PROVIDES, cleanPipelineGraph, graphOrder, + newStep, orderProblem, specProblems, stepSummary, @@ -63,6 +65,7 @@ const ICON: Record<string, LucideIcon> = { objects: Shapes, nsfw: EyeOff, describe: Eye, + convert: ArrowRightLeft, clean: Eraser, analyze: TextSearch, embed: ScanSearch, @@ -74,7 +77,7 @@ const ICON: Record<string, LucideIcon> = { workflow: Workflow, }; const TONE = { intent: "blue", red: "red", green: "green", gate: "gold", neutral: "neutral" } as const; -const ASSETS = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"]; +const ASSETS = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "convert"]; const METADATA = ["clean", "analyze", "embed", "summarize", "llm", "spec", "report", "export"]; /** A pipeline drawn as a graph: asset steps, metadata steps and workflows, an edge saying what runs after what. */ @@ -173,7 +176,6 @@ export function PipelineCanvas({ id }: { id: number }) { }); const add = (payload: string, x?: number, y?: number) => { - const [type, wid] = payload.split(":"); const at = graph.nodes.find((n) => n.id === sel); const nid = freshId( graph.nodes.map((n) => n.id), @@ -181,7 +183,7 @@ export function PipelineCanvas({ id }: { id: number }) { ); const node: PlNode = { id: nid, - step: { type, ...(type === "llm" ? { key: "" } : {}), ...(wid ? { workflow: Number(wid) } : {}) }, + step: newStep(payload), x: x ?? (at?.x ?? 0) + 240, y: y ?? at?.y ?? 80, }; diff --git a/nextjs-frontend/components/pipelines/pipeline-editor.tsx b/nextjs-frontend/components/pipelines/pipeline-editor.tsx index 4c0eca48a..05a060d0d 100644 --- a/nextjs-frontend/components/pipelines/pipeline-editor.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-editor.tsx @@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { + ArrowRightLeft, AudioLines, Captions, ChevronDown, @@ -37,6 +38,7 @@ import { PROVIDES, cleanSpec, moveStep, + newStep, sameSteps, specProblems, stepSummary, @@ -70,6 +72,7 @@ const ICON: Record<string, LucideIcon> = { objects: Shapes, nsfw: EyeOff, describe: Eye, + convert: ArrowRightLeft, clean: Eraser, analyze: TextSearch, embed: ScanSearch, @@ -88,6 +91,7 @@ const LIB_ORDER = [ "objects", "nsfw", "describe", + "convert", "clean", "analyze", "embed", @@ -340,7 +344,7 @@ export function PipelineEditor({ id }: { id?: number }) { const add = (type: string) => { const at = steps.length ? Math.min(sel + 1, steps.length) : 0; const next = steps.slice(); - next.splice(at, 0, { type, ...(type === "llm" ? { key: "" } : {}) }); + next.splice(at, 0, newStep(type)); setSteps(next); setSel(at); setMoveMsg(null); diff --git a/nextjs-frontend/components/pipelines/pipeline-model.ts b/nextjs-frontend/components/pipelines/pipeline-model.ts index 31bdedc85..73455d272 100644 --- a/nextjs-frontend/components/pipelines/pipeline-model.ts +++ b/nextjs-frontend/components/pipelines/pipeline-model.ts @@ -22,8 +22,24 @@ export type StepSpec = { model?: string; force?: boolean; workflow?: number; + /** A convert step: what the new assets are, their audio format, and the pipeline they run ("none": kept as they are). */ + to?: "audio" | "images"; + format?: string; + pipeline?: number | "none"; }; +/** The audio formats a convert step makes (the first is the default). */ +export const AUDIO_FORMATS = ["m4a", "mp3", "opus", "flac"]; + +/** A new step from the library: "llm", "convert", "workflow:3" (that workflow). */ +export function newStep(payload: string): StepSpec { + const [type, arg] = payload.split(":"); + if (type === "llm") return { type, key: "" }; + if (type === "convert") return { type, to: arg === "images" ? "images" : "audio" }; + if (type === "workflow" && arg) return { type, workflow: Number(arg) }; + return { type }; +} + /** A name for a pipeline nobody named, from its steps ("Transcribe to report", "Analyze and summarize"), told apart * from the pipelines there are. */ export function suggestedPipelineName(steps: StepSpec[], taken: Iterable<string>): string { @@ -65,6 +81,11 @@ export function cleanSpec(s: StepSpec): StepSpec | string { }; if (s.model?.trim()) out.model = s.model.trim(); if (s.force) out.force = true; + if (s.type === "convert") { + if (s.to) out.to = s.to; + if (s.to === "audio" && s.format) out.format = s.format; + if (s.pipeline != null) out.pipeline = s.pipeline; + } return Object.keys(out).length === 1 ? s.type : (out as StepSpec); } @@ -76,6 +97,7 @@ export const NEEDS: Record<string, string[]> = { objects: ["shots"], nsfw: ["read", "shots"], describe: ["shots"], + convert: ["transcribe", "read"], clean: ["transcribe", "diarize"], analyze: ["transcribe", "read", "diarize", "clean"], embed: ["transcribe", "read", "describe", "clean"], @@ -98,6 +120,7 @@ export const PROVIDES: Record<string, string> = { objects: "objects[]", nsfw: "nsfw (flagged · score)", describe: "descriptions[]", + convert: "new assets: the soundtrack as audio, or each page as an image", clean: "tidied transcript lines", analyze: "sections[] · entities[] · keywords[] · stats", embed: "passages searchable by meaning", @@ -119,6 +142,8 @@ export const DESCRIBE: Record<string, string> = { objects: "Finds objects (people, cars, animals …) in keyframes and pages (Settings → Video)", nsfw: "Flags NSFW pictures in keyframes, pages and images with a local model, where it’s on (Settings → NSFW)", describe: "Describes each shot and page with a model that can see images (Settings → LLM provider)", + convert: + "Makes new assets from this one: a video’s soundtrack as audio, or a document’s pages as images, each run through its own pipeline", clean: "Tidies a transcript before analysis: fillers, repeated words, spacing and capitals, where it’s on (Settings → Analysis)", analyze: "Chapters, entities, keywords and talk-time stats", @@ -183,6 +208,10 @@ export function stepSummary(s: StepSpec, templateName?: (id: number) => string | else if (s.type === "spec") parts.push("Spec from transcript"); if (s.type === "llm" && s.key) parts.push(`outputs.${s.key}`); if (s.type === "export" && s.filename) parts.push(s.filename); + if (s.type === "convert") { + parts.push(s.to === "images" ? "pages to images" : `to audio · ${s.format ?? AUDIO_FORMATS[0]}`); + if (s.pipeline === "none") parts.push("not processed"); + } if (s.model) parts.push(s.model); if (s.force) parts.push("forced"); const w = whenText(s.when); @@ -201,6 +230,7 @@ export function specProblems( if (s.type === "workflow" && s.workflow == null) out[i] = "Choose a workflow."; else if (s.type === "llm" && s.template == null) out[i] = "Choose a prompt template."; else if (s.type === "export" && s.template == null) out[i] = "Choose an export template."; + else if (s.type === "convert" && !s.to) out[i] = "Choose what to convert to: audio or images."; else if (need && s.template != null && templateKind(s.template) && templateKind(s.template) !== need) out[i] = `Needs a ${need} template.`; else if (s.type === "llm" && !/^[a-z][a-z0-9_]{0,40}$/.test(s.key ?? "")) diff --git a/nextjs-frontend/components/pipelines/step-settings.tsx b/nextjs-frontend/components/pipelines/step-settings.tsx index e0d07107c..f8867b44a 100644 --- a/nextjs-frontend/components/pipelines/step-settings.tsx +++ b/nextjs-frontend/components/pipelines/step-settings.tsx @@ -6,7 +6,8 @@ import Link from "next/link"; import { Sources, Templates, Workflows } from "@/app/openapi-client"; import type { TemplateSummary } from "@/app/openapi-client/types.gen"; import { stepLabel } from "@/components/activity/job-model"; -import { DESCRIBE, PROVIDES, type StepSpec } from "@/components/pipelines/pipeline-model"; +import { usePipelineCatalog } from "@/components/pipelines/catalog-header"; +import { AUDIO_FORMATS, DESCRIBE, PROVIDES, type StepSpec } from "@/components/pipelines/pipeline-model"; import { Checkbox, Field, Input, Select } from "@/components/ui/field"; import { data, useApiClient } from "@/lib/api/browser"; import { useArchive } from "@/lib/hooks/session"; @@ -84,6 +85,7 @@ export function StepSettings({ enabled: step.type === "workflow", staleTime: 30_000, }); + const pipelines = usePipelineCatalog(); const set = (patch: Partial<StepSpec>) => onChange({ ...step, ...patch }); const setWhen = (patch: Partial<NonNullable<StepSpec["when"]>>) => set({ when: { ...(step.when ?? {}), ...patch } }); const kind = { llm: "prompt", spec: "prompt", report: "report", export: "export" }[step.type]; @@ -149,6 +151,67 @@ export function StepSettings({ )} </Field> )} + {step.type === "convert" && ( + <> + <div className="grid grid-cols-2 gap-2.5"> + <Field label="Convert to"> + {({ id }) => ( + <Select + id={id} + value={step.to ?? "audio"} + onChange={(e) => set({ to: e.target.value === "images" ? "images" : "audio", format: undefined })} + options={[ + { value: "audio", label: "Audio, from a video" }, + { value: "images", label: "Images, one per page" }, + ]} + /> + )} + </Field> + {step.to !== "images" && ( + <Field label="Format" hint="m4a copies AAC sound as it is"> + {({ id, describedBy }) => ( + <Select + id={id} + aria-describedby={describedBy} + value={step.format ?? AUDIO_FORMATS[0]} + onChange={(e) => + set({ format: e.target.value === AUDIO_FORMATS[0] ? undefined : e.target.value }) + } + options={AUDIO_FORMATS.map((f) => ({ value: f, label: f }))} + /> + )} + </Field> + )} + </div> + <Field + label="Then run on the new assets" + hint={ + step.to === "images" + ? "Put a Read step before this one: its pages become the images" + : "The new asset is kept in the same namespace and collection" + } + > + {({ id, describedBy }) => ( + <Select + id={id} + aria-describedby={describedBy} + value={step.pipeline == null ? "" : String(step.pipeline)} + onChange={(e) => + set({ + pipeline: + e.target.value === "" ? undefined : e.target.value === "none" ? "none" : Number(e.target.value), + }) + } + options={[ + { value: "", label: "Their usual pipeline, as an upload" }, + { value: "none", label: "Nothing: keep them as they are" }, + ...(pipelines.data?.pipelines ?? []).map((p) => ({ value: String(p.id), label: p.name })), + ]} + /> + )} + </Field> + </> + )} {kind && ( <> <Field @@ -287,14 +350,19 @@ export function StepSettings({ with a prompt template. </p> )} - {(step.type === "transcribe" || step.type === "diarize" || step.type === "clean") && ( + {(step.type === "transcribe" || + step.type === "diarize" || + step.type === "clean" || + step.type === "convert") && ( <Checkbox label={ step.type === "transcribe" ? "Transcribe again even if the file came with a transcript" : step.type === "diarize" ? "Find speakers again even if the transcript labelled them" - : "Clean up even while Settings → Analysis has it off" + : step.type === "convert" + ? "Make the files again even if this asset was converted before" + : "Clean up even while Settings → Analysis has it off" } checked={Boolean(step.force)} onCheckedChange={(v) => set({ force: v || undefined })} diff --git a/nextjs-frontend/components/recording/header.tsx b/nextjs-frontend/components/recording/header.tsx index f2f5a23ff..610149f77 100644 --- a/nextjs-frontend/components/recording/header.tsx +++ b/nextjs-frontend/components/recording/header.tsx @@ -1,6 +1,7 @@ "use client"; import { + ArrowRightLeft, ChevronDown, ChevronRight, Clock3, @@ -146,6 +147,21 @@ export function RecordingHeader() { </span> </Meta> )} + {rec.converted_from && ( + <Meta icon={<ArrowRightLeft />}> + <span> + {rec.converted_from.to === "images" && rec.converted_from.page != null + ? `Page ${rec.converted_from.page + 1} of ` + : "Soundtrack of "} + <Link + href={`/resources/${rec.converted_from.resource}`} + className="font-semibold text-fg-accent hover:underline" + > + {rec.converted_from.title || "another asset"} + </Link> + </span> + </Meta> + )} {source && ( <Meta icon={source.href ? <Globe /> : source.remote ? <Cloud /> : <FileText />}> {source.href ? ( diff --git a/nextjs-frontend/components/ui/loop.tsx b/nextjs-frontend/components/ui/loop.tsx index 57a6db46b..f50ce7707 100644 --- a/nextjs-frontend/components/ui/loop.tsx +++ b/nextjs-frontend/components/ui/loop.tsx @@ -39,6 +39,7 @@ export const STEP_LABEL: Record<string, string> = { objects: "Objects", nsfw: "NSFW", describe: "Describe", + convert: "Convert", clean: "Clean up", analyze: "Analyze", embed: "Index for meaning", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index dcfb349c5..9e29a0c00 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -41712,6 +41712,53 @@ ], "title": "ConversionNodeTestResult" }, + "ConvertedFrom": { + "properties": { + "resource": { + "type": "integer", + "title": "Resource" + }, + "to": { + "type": "string", + "enum": [ + "audio", + "images" + ], + "title": "To" + }, + "page": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Page", + "description": "the page it is, from 0, for images made of a document's pages" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "resource", + "to" + ], + "title": "ConvertedFrom", + "description": "The asset a convert step made this one from (docs/processing.md#converting-assets)." + }, "Converters": { "properties": { "office": { @@ -62864,6 +62911,17 @@ ], "description": "an email's attachment made a resource of its own: that email" }, + "converted_from": { + "anyOf": [ + { + "$ref": "#/components/schemas/ConvertedFrom" + }, + { + "type": "null" + } + ], + "description": "made by a convert step: the asset it was made from" + }, "web": { "anyOf": [ { From d656da4831821fd01ca8be818d356da3409f4e7e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad <noreply@anthropic.com> Date: Sun, 18 Oct 2026 06:15:36 +0000 Subject: [PATCH 2/2] fix(pipelines): a convert step made again drops the file it replaced Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0141Zzcr9fDE5qRnKhFUVATi --- fastapi_backend/app/domain/conversions.py | 9 +++++++-- fastapi_backend/tests/api/test_conversions.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fastapi_backend/app/domain/conversions.py b/fastapi_backend/app/domain/conversions.py index 7561399ca..a50714c1c 100644 --- a/fastapi_backend/app/domain/conversions.py +++ b/fastapi_backend/app/domain/conversions.py @@ -177,11 +177,16 @@ def _keep(db, cfg, rid, rec, dest, source, title, link, old, spec, more): **more, } if old: + was = (db.one("SELECT path FROM $r", r=R("recording", old)) or {}).get("path") db.q("UPDATE $r MERGE $d", r=R("recording", old), d={**row, "status": "new"}) + if was and was != str(dest) and keyring.owned(cfg, was): # made again in another format: the old file goes + pathlib.Path(was).unlink(missing_ok=True) new = old else: - known = db.one("SELECT record::id(id) AS id FROM recording WHERE fp_key = $k", k=row["fp_key"]) - if known: # the same file is an asset here already + known = db.one("SELECT record::id(id) AS id, path FROM recording WHERE fp_key = $k", k=row["fp_key"]) + if known: # the same file is an asset here already: the copy just made isn't kept + if known.get("path") != str(dest): + dest.unlink(missing_ok=True) return known["id"] new = db.next_id("recording") db.q( diff --git a/fastapi_backend/tests/api/test_conversions.py b/fastapi_backend/tests/api/test_conversions.py index b39866db9..8044cfb07 100644 --- a/fastapi_backend/tests/api/test_conversions.py +++ b/fastapi_backend/tests/api/test_conversions.py @@ -97,6 +97,7 @@ def test_a_videos_soundtrack_becomes_an_audio_asset(client, h, db, cfg, folder): assert "audio asset (mp3, encoded)" in log [b] = _made(db, rid) assert b["id"] == a["id"] and b["path"].endswith(".mp3") # forced: the same asset, its file made again + assert not pathlib.Path(a["path"]).exists() # the m4a it replaced is gone # with no pipeline named, the new asset runs the one its content type gives it, as an upload does assert db.values("SELECT VALUE id FROM job WHERE recording = $r", r=b["id"])