From 0b87651f33396649d6b78251548422e250ccdd40 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 23:11:38 +0000 Subject: [PATCH] fix(jobs): skip transcribe and diarize for imported transcripts A pasted or imported transcript (no audio) queued the standard pipeline from transcribe, so a fresh install fetched PyTorch and SenseVoice (about 1 GB) before analysing it, only for transcribe to skip. for_source now leaves both steps out for a transcript source, as it does for documents and images. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TKh8zrmfKCvPJhRiv6u9mY --- DEVLOG.md | 10 ++++++++++ docs/api.md | 4 +++- docs/components.md | 4 +++- fastapi_backend/app/domain/jobs.py | 6 +++++- fastapi_backend/tests/api/test_imports.py | 5 +++-- fastapi_backend/tests/api/test_jobs.py | 18 ++++++++++++++++++ 6 files changed, 42 insertions(+), 5 deletions(-) diff --git a/DEVLOG.md b/DEVLOG.md index 4d4256604..b6e4739e9 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,16 @@ Plans and progress for work in flight. Newest first. +## 2026-10-10 · Pasted transcripts fetch no speech model + +Found by the clicks survey: on a fresh install, importing a pasted transcript queued the standard pipeline from +transcribe, so the worker fetched PyTorch and SenseVoice (about 1 GB) before analysing it, only for transcribe to skip. +`jobs.for_source` now leaves transcribe and diarize out for a `transcript` source, as it does for documents and images. + +Refine later: transcript jobs queued before the upgrade still wait for that fetch once (a migration step would back up +the whole database for a queue that drains on its own); a run of only transcribe or diarize on a transcript still +queues them, to say why they skip, and may fetch the model; a batch of only diarize over documents fails its enqueue. + ## 2026-10-10 · The assistant in your browser Goal (Adeel): chat with the assistant from the browser extension, and let it do things by using the browser, like the diff --git a/docs/api.md b/docs/api.md index b7839a1c1..26e1f94d5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -700,7 +700,9 @@ POST /api/v1/import/web/links `POST /import` queues the namespace's pipeline after the import, or the saved pipeline named by `pipeline` (any of `GET /pipelines`; 400 for one that doesn't exist, before anything is saved). Every import takes a `collection` of the namespace to put the recordings in (default: its default collection; 404 for one of another namespace): `POST /import`, -`POST /import/source`, `POST /uploads` and `POST /import/iiif`. +`POST /import/source`, `POST /uploads` and `POST /import/iiif`. A transcript imported without its audio has nothing to +transcribe and its speakers came with it, so its pipeline's transcribe and diarize steps are left out and no speech or +voice model is fetched for it. `POST /api/v1/import/source {source, paths, namespace, pipeline?, collection?, documents_as?}` imports chosen files of a storage source now, rather than watching their folder (admins, like sources; up to 500 paths, as diff --git a/docs/components.md b/docs/components.md index e0f961e44..8c4102540 100644 --- a/docs/components.md +++ b/docs/components.md @@ -6,7 +6,9 @@ nothing to install by hand. Each worker looks after its own machine. It compares what the settings ask for with what's installed, and fetches what's missing in the background, one thing at a time. Engines and models for job steps are fetched **on first use**: the first time a recording is waiting for the step that needs them. Until then they show as *On first use*, and an -archive that only uses cloud models never downloads them, which keeps a small machine (a Raspberry Pi) small. +archive that only uses cloud models never downloads them, which keeps a small machine (a Raspberry Pi) small. Documents, +images and transcripts imported without their audio (pasted ones too) skip the transcribe and diarize steps, so they +never fetch a speech or voice model. | Component | Fetched when | What | |---|---|---| diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index b44a9fcef..1a61594f2 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -198,7 +198,11 @@ def _describe(db, cfg, rid, say, spec=None): def for_source(steps, source): """A pipeline's steps as they run on a resource of this source: a document or an image is read, not transcribed, - and has no voices to tell apart, so a speech or voice model is never fetched for it.""" + and has no voices to tell apart, so a speech or voice model is never fetched for it. An imported transcript has + nothing to transcribe and its speakers came with it, so it skips both steps too (unless they're all that was asked + for: those still run, to say why there's nothing to do).""" + if source == "transcript": + return [s for s in steps if s.get("type") not in ("transcribe", "diarize")] or steps if source not in ("document", "image"): return steps out = [] diff --git a/fastapi_backend/tests/api/test_imports.py b/fastapi_backend/tests/api/test_imports.py index 0189c86b6..516388354 100644 --- a/fastapi_backend/tests/api/test_imports.py +++ b/fastapi_backend/tests/api/test_imports.py @@ -76,6 +76,7 @@ def test_choosing_the_pipeline_that_runs_after(client, db, cfg, folder): job = client.post("/api/v1/import", headers=h, json={"namespace": "pods", "text": text, "pipeline": quick}).json()["job"] row = db.one("SELECT steps, pipeline FROM $j", j=store.R("job", job)) assert [s["type"] for s in row["steps"]] == ["analyze", "report"] and row["pipeline"]["name"] == "Quick look" - # without one, the namespace's (here the standard one) + # without one, the namespace's (here the standard one), less what a transcript has nothing to do for job = client.post("/api/v1/import", headers=h, json={"namespace": "pods", "text": text + "\n[00:09] Ben: Later."}).json()["job"] - assert [s["type"] for s in db.one("SELECT steps FROM $j", j=store.R("job", job))["steps"]] == pipelines.STANDARD + standard = [t for t in pipelines.STANDARD if t not in ("transcribe", "diarize")] + assert [s["type"] for s in db.one("SELECT steps FROM $j", j=store.R("job", job))["steps"]] == standard diff --git a/fastapi_backend/tests/api/test_jobs.py b/fastapi_backend/tests/api/test_jobs.py index 8cbd22e8e..2270e3896 100644 --- a/fastapi_backend/tests/api/test_jobs.py +++ b/fastapi_backend/tests/api/test_jobs.py @@ -400,3 +400,21 @@ def test_a_document_is_read_not_transcribed(db, cfg, folder): ] # a worker whose steps list predates the read step runs it too assert "read" in jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": ["transcribe"]}}, name="old").can + + +def test_a_pasted_transcript_never_waits_for_a_speech_model(db, cfg): + """A pasted or imported transcript (no audio) has nothing to transcribe and its speakers came with it: the standard + pipeline queues no transcribe or diarize step for it, so no worker fetches PyTorch, SenseVoice or voice IDs (about + a gigabyte) before analysing it.""" + rid = ingest.import_text(db, cfg, "pods", "[00:00] Ann: One.\n[00:02] Ben: Two.", title="Pasted") + jid = jobs.enqueue(db, rid) + steps = [s["type"] for s in db.one("SELECT steps FROM $j", j=store.R("job", jid))["steps"]] + assert "transcribe" not in steps and "diarize" not in steps and "analyze" in steps + assert "transcribe" not in db.values("SELECT VALUE next_step FROM job WHERE status = 'queued'") + # steps added later leave them out too; asked for on their own, they still run (and say why they skip) + assert jobs.for_source([{"type": "diarize"}, {"type": "report"}], "transcript") == [{"type": "report"}] + assert jobs.for_source([{"type": "transcribe"}], "transcript") == [{"type": "transcribe"}] + # a worker whose speech and voice steps wait for their models to be fetched still runs it + w = jobs.Worker(db, lambda: cfg, name="small") + w.keeper = type("Fetching", (), {"blocked": lambda self: {"transcribe", "diarize"}})() + assert w.run_once() and db.one("SELECT status FROM $j", j=store.R("job", jid))["status"] == "succeeded"