From 4a1a8bd1a426292bcd08e3736869462e39bb410c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:50:11 +0000 Subject: [PATCH 1/3] fix(jobs): read documents and images instead of transcribing them Documents and images ran the pipeline's transcribe step, so a worker held them until the speech model (and, for diarize, the voice model) was fetched. They now get a read step of their own: transcribe becomes read and diarize is left out for them, in any pipeline. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01R52DQopnWr6KKw8MiuQ7bi --- docs/api.md | 6 +-- fastapi_backend/app/domain/components.py | 2 +- fastapi_backend/app/domain/jobs.py | 40 +++++++++++++++---- fastapi_backend/app/domain/pipelines.py | 8 ++-- fastapi_backend/app/domain/store.py | 1 + fastapi_backend/app/domain/video.py | 2 +- fastapi_backend/tests/api/test_documents.py | 10 ++--- fastapi_backend/tests/api/test_jobs.py | 23 +++++++++++ .../components/pipelines/pipeline-canvas.tsx | 1 + .../components/pipelines/pipeline-editor.tsx | 1 + .../components/pipelines/pipeline-model.ts | 10 +++-- nextjs-frontend/components/ui/loop.tsx | 2 + 12 files changed, 82 insertions(+), 24 deletions(-) diff --git a/docs/api.md b/docs/api.md index 8d862bc80..c0c6afdbc 100644 --- a/docs/api.md +++ b/docs/api.md @@ -509,11 +509,11 @@ title when the caller may see it); an attachment file's `resource` is the resour in the namespace (`documents.attachment_resources`). Transcribing an email again keeps and makes nothing twice. `GET /uploads/limits` says what the server can convert (`convert: {office, pages, msg}`); an upload it can't read is refused, saying what it needs, and the files that can be transcripts too (PDFs, Word, text and Markdown) can be imported -as transcripts instead (`POST /import`). Its pipeline's transcribe step draws its -pages and reads their text: a PDF's own text, block by block with where each block is on its page (poppler's +as transcripts instead (`POST /import`). Its pipeline's read step (its transcribe step, for a document or an image: +no speech model is fetched for one) draws its pages and reads their text: a PDF's own text, block by block with where each block is on its page (poppler's pdftotext; pypdf without poppler, when there are no pages to look at), and by OCR (`video.ocr_engine`) the pages with fewer than `documents.ocr_below_chars` characters of text, drawn sharper for it. An image is one page (a TIFF one per -frame), read by OCR. Diarize, shots and text on screen skip them; analyze, summarize and report run as for any text. +frame), read by OCR. Diarize is left out for them, shots and text on screen skip them; analyze, summarize and report run as for any text. [Configuration](configuration.md#documents-and-images) has the settings. The text becomes the resource's segments, one per block, so search, chat, entities, notes and corrections diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index 52cd510b6..6f53c352b 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -423,7 +423,7 @@ def _serves(self, cfg): detects = cfg["video"].get("object_engine") == "anytopdf" # its objects plugin runs here, node or not faces = cfg["video"].get("face_engine") == "anytopdf" enriches = bool((cfg.get("analysis") or {}).get("anytopdf")) - steps = {"transcribe": converts, "objects": detects, "faces": faces, "analyze": enriches} + steps = {"read": converts, "objects": detects, "faces": faces, "analyze": enriches} return {step for step, wanted in steps.items() if wanted} def needed(self, cfg, m): diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 81773d8af..92ce00179 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -41,12 +41,20 @@ def _spec(s): return {"type": s} if isinstance(s, str) else dict(s) +def _read(db, cfg, rid, say, spec=None): + """A document's or an image's step: draw its pages and read their text (documents.py). It needs no speech model, + so a document never waits for one to be fetched.""" + if _source(db, rid) not in ("document", "image"): + raise Skip("only documents and images are read; audio and video are transcribed") + from . import documents + + return documents.transcribe(db, cfg, rid, say) + + def _transcribe(db, cfg, rid, say, spec=None): rec = db.one("SELECT source, engine, status, envelope, web, path, space, title FROM $r", r=R("recording", rid)) or {} - if rec.get("source") in ("document", "image"): - from . import documents - - return documents.transcribe(db, cfg, rid, say) + if rec.get("source") in ("document", "image"): # a run queued before documents had a step of their own + return _read(db, cfg, rid, say, spec) if rec.get("source") != "audio": raise Skip("an imported transcript has nothing to transcribe") if str(rec.get("engine") or "").startswith("import:") and rec.get("status") != "new" and not (spec or {}).get("force"): @@ -170,6 +178,7 @@ def _describe(db, cfg, rid, say, spec=None): VIDEO_STEPS = {"shots", "ocr", "faces", "objects", "describe"} # added after workers.steps lists were first written STEPS = { "transcribe": _transcribe, + "read": _read, "diarize": _diarize, "shots": _shots, "ocr": _ocr, @@ -187,9 +196,24 @@ def _describe(db, cfg, rid, say, spec=None): } +def for_source(steps, source): + """A pipeline's steps as they run on a resource of this source: a document or an image is read, not transcribed, + and has no voices to tell apart, so a speech or voice model is never fetched for it.""" + if source not in ("document", "image"): + return steps + out = [] + for s in steps: + if s.get("type") == "transcribe": + s = {**s, "type": "read"} + if s.get("type") == "diarize" or (s.get("type") == "read" and any(o.get("type") == "read" for o in out)): + continue + out.append(s) + return out + + def enqueue(db, rid, steps=None, by=None, priority=0, pipeline=None, batch=None): """Queue a recording for these steps (default: its namespace's pipeline); one already queued or running keeps its job.""" - rec = db.one("SELECT space FROM $r", r=R("recording", rid)) + rec = db.one("SELECT space, source FROM $r", r=R("recording", rid)) if not rec: raise KeyError(rid) ref = None @@ -197,7 +221,7 @@ def enqueue(db, rid, steps=None, by=None, priority=0, pipeline=None, batch=None) from . import content_types steps, ref = pipelines.resolve(db, rec["space"], pipeline, None if pipeline else content_types.of_recording(db, rid)[0]) - steps = [_spec(s) for s in steps] + steps = for_source([_spec(s) for s in steps], rec.get("source")) if not steps or any(s.get("type") not in STEPS for s in steps): raise ValueError(f"steps are {', '.join(STEPS)}") live = db.values("SELECT VALUE record::id(id) FROM job WHERE recording = $r AND status IN $a", r=rid, a=ACTIVE) @@ -233,7 +257,7 @@ def add_steps(db, rid, steps, by=None): """Run these steps on a recording after what its queued or running job still has to do, or as a new job when it has none. Workers read a job's steps again before each step, and only finish a job whose steps are all done, so steps added while it runs aren't missed.""" - specs = [_spec(s) for s in steps] + specs = for_source([_spec(s) for s in steps], _source(db, rid)) for jid in db.values("SELECT VALUE record::id(id) FROM job WHERE recording = $r AND status IN $a", r=rid, a=ACTIVE): if db.rows( "UPDATE $j SET steps = array::concat(steps, $m), updated_at = $t WHERE status IN $a RETURN AFTER", @@ -839,6 +863,8 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None): cfg_fn() # reads the saved settings now, so telemetry (if on) is set up before the first job's span self.name = name or f"{socket.gethostname()}-{os.getpid()}" self.can = set(steps or cfg_fn()["workers"]["steps"]) & set(STEPS) + if not steps and "transcribe" in self.can: # reading documents moved out of transcribe; older lists run it too + self.can.add("read") if "llm" in self.can: # a workflow or a podcast needs what an llm step needs; older lists run them too self.can |= {"workflow", "podcast"} if not steps and "transcribe" in self.can and not self.can & VIDEO_STEPS: diff --git a/fastapi_backend/app/domain/pipelines.py b/fastapi_backend/app/domain/pipelines.py index 063a7c350..ba96b416d 100644 --- a/fastapi_backend/app/domain/pipelines.py +++ b/fastapi_backend/app/domain/pipelines.py @@ -1,5 +1,6 @@ """Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without -one, recordings get the standard pipeline. Steps: transcribe, diarize, analyze, embed (passages for search by meaning, +one, recordings get the standard pipeline. Steps: transcribe (speech to text), read (a document's or an +image's pages and their text), diarize, analyze, embed (passages for search by meaning, semantic.py), summarize, llm (a prompt template whose structured result is saved as a named output), report (built in, or from a report template) and export (a template rendered to a file, optionally copied to a storage source) and workflow (a workflow graph, workflows.py, that turns what the steps made into metadata). Any step can carry a @@ -24,10 +25,11 @@ R = store.R STANDARD = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "report"] TYPES = { - "transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "llm", "report", "export", + "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "llm", "report", + "export", "workflow", } # fmt: skip -ASSET_STEPS = ("transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe") # they make something of the media +ASSET_STEPS = ("transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "describe") # they make something of the media KEYS = {"type", "name", "when", "template", "version", "key", "filename", "destination", "model", "force", "workflow"} NODE_RX = re.compile(r"^[A-Za-z0-9_-]{1,40}$") WHEN = {"min_minutes": (int, float), "max_minutes": (int, float), "source": str, "languages": list} diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index de3e3f873..19d8b5738 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -253,6 +253,7 @@ def labels(): "max_attempts": 3, "steps": [ "transcribe", + "read", "diarize", "shots", "ocr", diff --git a/fastapi_backend/app/domain/video.py b/fastapi_backend/app/domain/video.py index 50f758802..f8f083c8d 100644 --- a/fastapi_backend/app/domain/video.py +++ b/fastapi_backend/app/domain/video.py @@ -537,7 +537,7 @@ def _norm(text): def step_ocr(db, cfg, rid, say): rec = db.one("SELECT space, source, media, samples, sample_ms FROM $r", r=R("recording", rid)) or {} if rec.get("source") in ("document", "image"): - raise jobs.Skip("its pages were read when it was transcribed") + raise jobs.Skip("its pages were read by the read step") if (rec.get("media") or {}).get("kind") != "video": raise jobs.Skip("it isn't a video") engine, why = ocr_engine_why(cfg) diff --git a/fastapi_backend/tests/api/test_documents.py b/fastapi_backend/tests/api/test_documents.py index f8427bb5f..82de1daad 100644 --- a/fastapi_backend/tests/api/test_documents.py +++ b/fastapi_backend/tests/api/test_documents.py @@ -1,5 +1,5 @@ """Documents and images as resources (docs/api.md, Documents and images): uploaded like audio and video, their pages -drawn and read by the transcribe step (a PDF's own text, OCR for scans and images), then shown, listed, searched and +drawn and read by the read step (a PDF's own text, OCR for scans and images), then shown, listed, searched and summarised by page.""" from __future__ import annotations @@ -89,10 +89,10 @@ def test_a_pdf_becomes_a_document_with_its_pages_and_text(client, env, db, cfg): assert (rec["status"], rec["engine"], rec.get("duration_ms")) == ("analyzed", "pdf", None) assert rec["media"]["kind"] == "document" and rec["media"]["pages"] == 2 and rec["media"]["height"] == 2000 runs = _runs(client, he, up["job"]) - assert runs["transcribe"] == ("done", "2 page(s), 3 block(s) of text") - assert runs["diarize"] == ("skipped", "a document has no voices") + assert runs["read"] == ("done", "2 page(s), 3 block(s) of text") + assert "transcribe" not in runs and "diarize" not in runs # read, not transcribed: no speech model is fetched assert runs["shots"] == ("skipped", "it isn't a video") - assert runs["ocr"] == ("skipped", "its pages were read when it was transcribed") + assert runs["ocr"] == ("skipped", "its pages were read by the read step") # summaries cite pages: the first point is on page 1, the follow-up on page 2 assert rec["summary"]["key_points"][0]["page"] == 0 and rec["summary"]["action_items"][0]["page"] == 1 @@ -192,7 +192,7 @@ def test_without_poppler_or_an_ocr_engine(client, env, db, cfg, monkeypatch): assert [x["image"] for x in documents.pages(db, img["recording"])] == ["page-0001.jpg"] assert db.rows("SELECT * FROM segment WHERE recording = $r", r=img["recording"]) == [] runs = _runs(client, he, img["job"]) - assert runs["transcribe"] == ("done", "1 page(s), 0 block(s) of text") + assert runs["read"] == ("done", "1 page(s), 0 block(s) of text") assert runs["summarize"] == ("skipped", "there's no text to summarise") log = client.get(f"/api/v1/jobs/{img['job']}/log", headers=he).json()["lines"] assert any("its text wasn't read: OCR is off (video.ocr_engine)" in x for x in log) diff --git a/fastapi_backend/tests/api/test_jobs.py b/fastapi_backend/tests/api/test_jobs.py index 7bb63f52a..8cbd22e8e 100644 --- a/fastapi_backend/tests/api/test_jobs.py +++ b/fastapi_backend/tests/api/test_jobs.py @@ -377,3 +377,26 @@ def test_a_steps_list_from_before_the_video_steps_runs_them_too(db, cfg): w = jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": [*old, "shots"]}}, name="chosen", log=said.append) assert "ocr" not in w.can and "leaves out describe, faces, objects, ocr" in said[0] assert jobs.Worker(db, lambda: cfg, name="mac", steps=["transcribe"], log=said.append).can == {"transcribe"} and len(said) == 1 + + +def test_a_document_is_read_not_transcribed(db, cfg, folder): + """The standard pipeline (or any pipeline) on a document or an image reads its pages: no transcribe or diarize + step is queued for it, so no speech or voice model is fetched and it never waits for one.""" + _, _, _ = seed(db, cfg, folder) + rid = db.next_id("recording") + db.q( + "CREATE $r CONTENT {source: 'document', status: 'new', space: (SELECT VALUE space FROM recording LIMIT 1)[0]}", + r=store.R("recording", rid), + ) + jid = jobs.enqueue(db, rid) + steps = [s["type"] for s in db.one("SELECT steps FROM $j", j=store.R("job", jid))["steps"]] + assert steps[0] == "read" and "transcribe" not in steps and "diarize" not in steps + assert db.values("SELECT VALUE next_step FROM job WHERE status = 'queued'") == ["read"] + # audio keeps transcribe; an explicit read on it has nothing to do + assert jobs.for_source([{"type": "transcribe"}, {"type": "diarize"}], "audio") == [{"type": "transcribe"}, {"type": "diarize"}] + assert jobs.for_source([{"type": "transcribe"}, {"type": "read"}, {"type": "report"}], "image") == [ + {"type": "read"}, + {"type": "report"}, + ] + # a worker whose steps list predates the read step runs it too + assert "read" in jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": ["transcribe"]}}, name="old").can diff --git a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx index ffc4889fe..205e47133 100644 --- a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx @@ -52,6 +52,7 @@ import { cn } from "@/lib/utils"; const ICON: Record = { transcribe: Captions, + read: ScanText, diarize: AudioLines, shots: Clapperboard, ocr: ScanText, diff --git a/nextjs-frontend/components/pipelines/pipeline-editor.tsx b/nextjs-frontend/components/pipelines/pipeline-editor.tsx index e5a448acb..bbf9823b4 100644 --- a/nextjs-frontend/components/pipelines/pipeline-editor.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-editor.tsx @@ -58,6 +58,7 @@ import { cn } from "@/lib/utils"; const ICON: Record = { transcribe: Captions, + read: ScanText, diarize: AudioLines, shots: Clapperboard, ocr: ScanText, diff --git a/nextjs-frontend/components/pipelines/pipeline-model.ts b/nextjs-frontend/components/pipelines/pipeline-model.ts index aa40fccf8..1766338eb 100644 --- a/nextjs-frontend/components/pipelines/pipeline-model.ts +++ b/nextjs-frontend/components/pipelines/pipeline-model.ts @@ -61,18 +61,19 @@ export const NEEDS: Record = { faces: ["shots"], objects: ["shots"], describe: ["shots"], - analyze: ["transcribe", "diarize"], - embed: ["transcribe", "describe"], + analyze: ["transcribe", "read", "diarize"], + embed: ["transcribe", "read", "describe"], summarize: ["analyze"], - llm: ["transcribe", "diarize", "analyze"], + llm: ["transcribe", "read", "diarize", "analyze"], report: ["analyze", "summarize", "llm"], export: ["analyze", "summarize", "llm"], - workflow: ["transcribe", "diarize"], + workflow: ["transcribe", "read", "diarize"], }; /** What a step makes available to later steps and templates. */ export const PROVIDES: Record = { transcribe: "transcript · segments[] · language", + read: "pages[] · segments[]", diarize: "speakers[] · turns", shots: "shots[] · frames", ocr: "text on screen", @@ -90,6 +91,7 @@ export const PROVIDES: Record = { export const DESCRIBE: Record = { transcribe: "Speech to text with word timings (Settings → Transcription)", + read: "Draws a document’s or an image’s pages and reads their text (Settings → Documents)", diarize: "Who spoke when (Settings → Diarization)", shots: "Cuts a video into shots and keyframes", ocr: "Reads text on screen (Settings → Video)", diff --git a/nextjs-frontend/components/ui/loop.tsx b/nextjs-frontend/components/ui/loop.tsx index 1821bc432..f2e52e2d0 100644 --- a/nextjs-frontend/components/ui/loop.tsx +++ b/nextjs-frontend/components/ui/loop.tsx @@ -10,6 +10,7 @@ import { cn } from "@/lib/utils"; */ export const STEP_TONE: Record = { transcribe: "intent", + read: "intent", shots: "intent", diarize: "red", ocr: "red", @@ -27,6 +28,7 @@ export const STEP_TONE: Record = { export const STEP_LABEL: Record = { transcribe: "Transcribe", + read: "Read pages", diarize: "Diarize", shots: "Shots", ocr: "Text on screen", From 39310b3a570d2e0bc28e75c20c003e63e6e52509 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:53:27 +0000 Subject: [PATCH 2/3] test(enrich): anytopdf conversion serves the read step Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01R52DQopnWr6KKw8MiuQ7bi --- fastapi_backend/tests/api/test_enrich.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastapi_backend/tests/api/test_enrich.py b/fastapi_backend/tests/api/test_enrich.py index c3653a4e6..2f6b8456b 100644 --- a/fastapi_backend/tests/api/test_enrich.py +++ b/fastapi_backend/tests/api/test_enrich.py @@ -139,7 +139,7 @@ def test_fetched_for_analysis_and_only_waits_analysis(cfg): assert c.needed(cfg, None) is bool(enrich.anytopdf.archive()) assert c.serves(cfg, None) == {"analyze"} cfg["documents"]["converter"] = "anytopdf" - assert c.serves(cfg, None) == {"transcribe", "analyze"} + assert c.serves(cfg, None) == {"read", "analyze"} @pytest.mark.skipif(not REAL, reason="LENS_TEST_ANYTOPDF names a real anytopdf 0.4.0") From e6d3fd03a2ea1ab97a881633a5c181fc6dd75508 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:54:40 +0000 Subject: [PATCH 3/3] test(documents): a document's entities reach the graph Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01R52DQopnWr6KKw8MiuQ7bi --- fastapi_backend/tests/api/test_documents.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fastapi_backend/tests/api/test_documents.py b/fastapi_backend/tests/api/test_documents.py index 82de1daad..9c48169ae 100644 --- a/fastapi_backend/tests/api/test_documents.py +++ b/fastapi_backend/tests/api/test_documents.py @@ -435,3 +435,13 @@ def watch(ns, **more): assert watch("old", kinds="both") == [("call", "transcript"), ("ep1", "audio"), ("harbour", "transcript")] assert watch("new") == [("call", "document"), ("ep1", "audio"), ("harbour", "document"), ("photo", "image")] assert client.get("/api/v1/watches", headers=ha).json()[-1]["kinds"] == "all" # the default now + + +@POPPLER +def test_a_document_adds_to_the_graph(client, env, db, cfg): + """A document's text is analysed like a transcript's: the people and places in it become entities in the graph.""" + pdf = text_pdf([["Alice Johnson met Bob Smith at the harbour.", "Alice Johnson wrote to Microsoft."]]) + up = _upload(client, env["he"], pdf, "minutes.pdf") + drain(db, cfg) + said = {m["text"] for m in db.rows("SELECT text FROM mentions WHERE recording = $r", r=up["recording"])} + assert {"Alice Johnson", "Bob Smith", "Microsoft"} <= said