From 9c6e17c75514c9d07bb06e82d647d7e76b82e625 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 07:18:32 +0000 Subject: [PATCH] feat(imports): take a video's own captions as its transcript Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 9 ++- changelog.d/captions.md | 3 + docs/api.md | 7 ++ docs/requirements.md | 1 + fastapi_backend/app/domain/jobs.py | 2 + fastapi_backend/app/domain/mediaurl.py | 76 ++++++++++++++++++- fastapi_backend/app/domain/settings.py | 2 + fastapi_backend/app/domain/store.py | 1 + fastapi_backend/tests/api/test_video_urls.py | 34 ++++++++- .../components/settings/bodies.tsx | 1 + nextjs-frontend/components/settings/model.ts | 7 ++ 11 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 changelog.d/captions.md diff --git a/README.md b/README.md index 4b718fed1..5ad3b99fe 100644 --- a/README.md +++ b/README.md @@ -146,10 +146,11 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p Office files, photos, JSON and zip or tar archives, and, with an Apache Tika server (an opt-in extra), documents nothing else here can read; capture web pages one at a time, from a link list or a bookmarks file, by API or by asking the assistant, from private networks you allow in Settings too; and import videos and podcasts by their address (off - by default). Say what a file or a page is as it comes in (a podcast, an interview, a scan), in Import, by API or - to the assistant, and its content type's pipeline runs from the start. A file, page or video the namespace already - has links to the one there instead of being imported again. Media browsers can't play, such as an AVI or HEVC from a - phone, can get a copy they can (off by default). + by default), taking a video's own captions as its transcript when people wrote some (off by default too). Say what a + file or a page is as it comes in (a podcast, an interview, a scan), in Import, by API or to the assistant, and its + content type's pipeline runs from the start. A file, page or video the namespace already has links to the one there + instead of being imported again. Media browsers can't play, such as an AVI or HEVC from a phone, can get a copy they + can (off by default). ([Sensors](docs/sensors.md), [processing](docs/processing.md), [Apache Tika](docs/configuration.md#apache-tika), [import webhooks](docs/api.md#import-webhooks), [playable copies](docs/video.md#playable-copies); [checks](docs/requirements.md#bring-in-everything)) diff --git a/changelog.d/captions.md b/changelog.d/captions.md new file mode 100644 index 000000000..08eef2dab --- /dev/null +++ b/changelog.d/captions.md @@ -0,0 +1,3 @@ +- **A video's own captions.** With Settings › Documents › Use a video's own captions on (off by default), a video + imported by its address whose captions people wrote takes them as its transcript, in the transcription language or + the video's own, so nothing is transcribed ([Videos and podcasts](docs/api.md#videos-and-podcasts)). diff --git a/docs/api.md b/docs/api.md index ceec23f14..f4fa98204 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1068,6 +1068,13 @@ the video steps then see its picture. Its `web` is `{url, media: true, final, si the video's own unless `title` gave one. Downloading happens once: transcribing it again keeps the file, which is the resource's media (`GET /resources/{rid}/media`). Audited as `import.video`. +**Its own captions.** With `documents.video_captions` on too (Settings › Documents › Use a video's own captions, off +by default), captions people wrote for the video come down with it (a site's subtitles, never its automatic ones) and +become its transcript, so nothing is transcribed and no speech model or provider is used. Of the languages it has, +Lens takes the one set for transcription, else the video's own, else the first; the file is kept next to the media +(`web.captions` names it and its language), encrypted like the rest when the namespace is, and the transcript's +engine is `import:captions`. A video without such captions is transcribed as before. + ### Import webhooks An import webhook is an address other services push files, web addresses or text to (a scanner, a phone shortcut, diff --git a/docs/requirements.md b/docs/requirements.md index c7106df0f..57c571670 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -174,6 +174,7 @@ request, to another check that proves the same thing; loosening a requirement or | Web pages captured one at a time, from a link list or a bookmarks file, by API or by asking; private networks only when allowed | [test_webcapture.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_webcapture.py) `test_a_web_page_becomes_a_document`, `test_an_admin_allows_a_private_network_in_settings`, `test_private_addresses_are_refused`; [test_web_links.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_links.py) `test_one_or_several_addresses_as_a_list`, `test_a_bookmarks_file_becomes_pages_to_capture`; [test_setup_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_setup_assistant.py) `test_the_assistant_captures_web_pages` | | A content type chosen as a file or a page comes in, in Import, by API or to the assistant, runs that content type's pipeline from the start | [import-content-type.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-content-type.test.tsx) `offers the content types of the file's base type, and none chosen means recognised from the file`; [import-web.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-web.test.tsx) `captures pages as the content type chosen for them`; [test_uploads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_uploads.py) `test_a_content_type_chosen_at_upload`, `test_a_held_file_is_placed_as_a_content_type`; [test_imports.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_imports.py) `test_choosing_the_content_type_at_import`; [test_web_links.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_links.py) `test_pages_captured_as_a_content_type`; [test_setup_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_setup_assistant.py) `test_the_assistant_imports_files_as_the_content_type_said` | | A file, page or video the namespace already has links to the one there instead of being imported again: a copy in a watched folder on this machine, a page or video address imported twice | [test_uploads.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_uploads.py) `test_a_copy_in_a_folder_on_this_machine_links_to_the_one_there`; [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_an_address_imported_again_links_to_what_the_namespace_has` | +| A video's own captions, written by people, become its transcript instead of transcribing it, off by default | [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_a_videos_own_captions_become_its_transcript` | | Videos and podcasts imported by their address, off by default | [test_video_urls.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video_urls.py) `test_off_by_default`, `test_a_video_by_its_address_is_downloaded_then_transcribed`; [import-web.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/import-web.test.tsx) `imports a video or podcast by its address, when asked to` | ## Clip from the browser diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index e4cc3beaa..3d6b142bc 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -118,6 +118,8 @@ def _transcribe_media(db, cfg, rid, say, spec=None): from . import mediaurl mediaurl.ensure(db, cfg, rid, rec, say) + if mediaurl.use_captions(db, cfg, rid, say): # captions people wrote came with it: no need to transcribe + return None ingest.transcribe_one(db, cfg, rid, say) diff --git a/fastapi_backend/app/domain/mediaurl.py b/fastapi_backend/app/domain/mediaurl.py index 61cff4db2..35e271d58 100644 --- a/fastapi_backend/app/domain/mediaurl.py +++ b/fastapi_backend/app/domain/mediaurl.py @@ -10,6 +10,10 @@ Downloading happens in the resource's transcribe step, once: the media is the resource's file, data_dir/media//.<ext>, and a resource transcribed again keeps it. + +With documents.video_captions on (off by default), captions people wrote for the video (a site's subtitles, never its +automatic ones) come down with it, in the language set for transcription when there is one, else the video's own, +else the first there is: they become its transcript, so nothing is transcribed. A video without them is transcribed. """ from __future__ import annotations @@ -142,9 +146,28 @@ def _why(err): return last.removeprefix("ERROR:").strip()[:300] or "it ended without saying why" +def captions_on(cfg): + return bool((cfg.get("documents") or {}).get("video_captions")) + + +def pick_captions(into, info, cfg): + """The captions file yt-dlp wrote next to the media (media.<language>.vtt) to use: the transcription language's, + else the video's own language's, else the first; None when there are none.""" + got = {p.name[len("media.") : -len(".vtt")]: p for p in sorted(into.glob("media.*.vtt")) if p.is_file()} + if not got: + return None + want = [(cfg.get("transcribe") or {}).get("language"), info.get("language")] + for w in (str(x).lower() for x in want if x and x != "auto"): + for lang, p in got.items(): + if lang.lower() == w or lang.lower().split("-")[0] == w.split("-")[0]: + return lang, p + lang = next(iter(got)) + return lang, got[lang] + + def download(cfg, url, into, say=None): """The media at `url`, downloaded into the folder `into`: (its path, what yt-dlp said of it). ValueError when it - can't be had.""" + can't be had. With documents.video_captions on, the captions people wrote for it come too (media.<lang>.vtt).""" argv = ready(cfg, say) most = cfg["uploads"]["max_mb"] into.mkdir(parents=True, exist_ok=True) @@ -164,6 +187,7 @@ def download(cfg, url, into, say=None): "--hls-prefer-native", "--write-info-json", "--no-write-comments", + *(["--write-subs", "--no-write-auto-subs", "--sub-langs", "all,-live_chat", "--sub-format", "vtt"] if captions_on(cfg) else []), "-o", "media.%(ext)s", "--", @@ -174,7 +198,7 @@ def download(cfg, url, into, say=None): except subprocess.TimeoutExpired: raise ValueError(f"downloading it took longer than {SECONDS // 3600} hours") from None refused = f"; refused: {guard.refused[0]}" if guard.refused else "" - got = [p for p in into.glob("media.*") if p.suffix not in (".json", ".part", ".ytdl") and p.is_file()] + got = [p for p in into.glob("media.*") if p.suffix not in (".json", ".part", ".ytdl", ".vtt") and p.is_file()] if run.returncode or not got: raise ValueError(f"its media couldn't be downloaded ({_why(run.stderr)}){refused}") info = {} @@ -202,6 +226,10 @@ def ensure(db, cfg, rid, rec, say): d.mkdir(parents=True, exist_ok=True) out = d / f"{render.slug(title or webcapture.placeholder(web['url']))}{path.suffix.lower()}" shutil.move(str(path), out) + picked = pick_captions(pathlib.Path(tmp), info, cfg) if captions_on(cfg) else None + if picked: + subs = d / f"captions.{render.slug(picked[0]) or 'und'}.vtt" + shutil.move(str(picked[1]), subs) st = out.stat() dur, ch = ingest.probe(out) patch = store.clean( @@ -219,6 +247,7 @@ def ensure(db, cfg, rid, rec, say): "site": str(info.get("extractor_key") or "")[:100] or None, "by": str(info.get("uploader") or info.get("channel") or "")[:200] or None, "captured_at": store.now(), + "captions": {"file": subs.name, "language": picked[0][:20]} if picked else None, } ), } @@ -226,6 +255,47 @@ def ensure(db, cfg, rid, rec, say): if title and rec.get("title") == webcapture.placeholder(web["url"]): # still called by its address: its own title patch["title"] = title keyring.protect(db, cfg, rec["space"], out) + if picked: + keyring.protect(db, cfg, rec["space"], subs) db.q("UPDATE $r MERGE $d", r=R("recording", rid), d=patch) - say(f"downloaded {patch['web']['final']}") + say(f"downloaded {patch['web']['final']}" + (f", with its {picked[0]} captions" if picked else "")) return str(out) + + +def use_captions(db, cfg, rid, say): + """The captions that came with a video, as its transcript (documents.video_captions): True when they were used; + False when it has none, so it is transcribed.""" + rec = db.one("SELECT space, path, remote, web FROM $r", r=R("recording", rid)) or {} + cap = ((rec.get("web") or {}).get("captions")) or {} + f = folder(cfg, rid) / str(cap.get("file") or "") + if not (captions_on(cfg) and cap.get("file") and f.is_file()): + return False + with keyring.open_plain(db, cfg, f) as fh: + text = fh.read().decode("utf-8-sig", errors="replace") + segs = ingest.read_text_transcript(text, "vtt", f.name)["segments"] + if not segs: + return False + try: + env = ingest.envelope(ingest.decode(ingest.audio_path(db, cfg, rec))) + except (RuntimeError, OSError): + env = None + lang = str(cap.get("language") or "").lower() or None + for s in segs: + s.setdefault("lang", lang) + ingest.write_transcript( + db, + rid, + rec["space"], + segs, + store.clean( + { + "status": "transcribed", + "engine": "import:captions", + "language": lang, + "envelope": env, + "transcribed_at": store.now(), + } + ), + ) + say(f"used the {cap.get('language')} captions that came with it: {len(segs)} lines, nothing transcribed") + return True diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index aa98d4fd2..76ae0b4ed 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -128,6 +128,7 @@ "convert_seconds", "attachment_resources", "video_urls", + "video_captions", "converter", "anytopdf_url", "anytopdf_token", @@ -632,6 +633,7 @@ def _check(section, key, value, default): if (section, key) in ( ("documents", "attachment_resources"), ("documents", "video_urls"), + ("documents", "video_captions"), ("documents", "tika_text"), ("documents", "tika_metadata"), ): diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 01146bc30..166285009 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -275,6 +275,7 @@ def labels(): "chromium": None, "web_networks": [], "video_urls": False, # mediaurl.py: videos and podcasts imported by their address, downloaded by yt-dlp + "video_captions": False, # and their own captions, written by people, used as the transcript instead of transcribing "converter": "auto", # anytopdf.py: auto (where Lens can't convert), anytopdf (every document and image), lens "anytopdf": None, # the anytopdf program; None: the one downloaded on first use, else one on PATH "anytopdf_url": None, # a conversion node: `anytopdf queue serve` on another machine diff --git a/fastapi_backend/tests/api/test_video_urls.py b/fastapi_backend/tests/api/test_video_urls.py index 615599867..aa7c30cfc 100644 --- a/fastapi_backend/tests/api/test_video_urls.py +++ b/fastapi_backend/tests/api/test_video_urls.py @@ -27,7 +27,10 @@ sys.exit(1) with wave.open("media.wav", "wb") as w: w.setnchannels(1), w.setsampwidth(2), w.setframerate(16000), w.writeframes(b"\0\0" * 16000) -json.dump({"title": "Harbour talk: episode 4", "webpage_url": url, "extractor_key": "Generic", "uploader": "Harbour FM"}, open("media.info.json", "w")) +json.dump({"title": "Harbour talk: episode 4", "webpage_url": url, "extractor_key": "Generic", "uploader": "Harbour FM", "language": "cy"}, open("media.info.json", "w")) +if "--write-subs" in args and "nocaptions" not in url: + for lang, line in (("en", "Welcome to the harbour."), ("cy", "Croeso i'r harbwr.")): + open(f"media.{lang}.vtt", "w").write(f"WEBVTT\n\n00:00:00.000 --> 00:00:00.900\n{line}\n") """ @@ -103,6 +106,7 @@ def test_a_video_by_its_address_is_downloaded_then_transcribed(client, env, cfg, args = run["args"] for flag in ("--ignore-config", "--no-plugin-dirs", "--no-playlist", "--no-cache-dir", "--hls-prefer-native"): assert flag in args + assert "--write-subs" not in args # captions only when asked for proxy = args[args.index("--proxy") + 1] assert proxy.startswith("http://127.0.0.1:") and run["https_proxy"] == proxy assert args[args.index("--max-filesize") + 1] == f"{cfg['uploads']['max_mb']}M" and args[-2:] == ["--", "https://8.8.8.8/watch?v=1"] @@ -111,6 +115,34 @@ def test_a_video_by_its_address_is_downloaded_then_transcribed(client, env, cfg, assert len(log.read_text().splitlines()) == 1 and heard == [rid, rid] +def test_a_videos_own_captions_become_its_transcript(client, env, cfg, db, tmp_path, monkeypatch): + cfg["documents"]["video_urls"] = cfg["documents"]["video_captions"] = True + log = fake(cfg, tmp_path) + heard = [] + monkeypatch.setattr(ingest, "transcribe_one", lambda db, cfg, rid, say: heard.append(rid)) + sid = store.ns_id(db, "pods") + rid = mediaurl.create(db, sid, "https://8.8.8.8/watch?v=2") + said = [] + jobs._transcribe(db, cfg, rid, said.append) + args = json.loads(log.read_text().splitlines()[-1])["args"] + assert "--write-subs" in args and "--no-write-auto-subs" in args # people's captions, never automatic ones + assert heard == [] # nothing transcribed + rec = db.one("SELECT engine, language, status, web FROM $r", r=R("recording", rid)) + assert (rec["engine"], rec["language"], rec["status"]) == ("import:captions", "cy", "transcribed") # the video's own + assert rec["web"]["captions"]["language"] == "cy" + assert [s["text"] for s in db.rows("SELECT text FROM segment WHERE recording = $r", r=rid)] == ["Croeso i'r harbwr."] + assert "used the cy captions" in said[-1] + + cfg["transcribe"]["language"] = "en" # the transcription language wins + other = mediaurl.create(db, sid, "https://8.8.8.8/watch?v=3") + jobs._transcribe(db, cfg, other, said.append) + assert db.one("SELECT language FROM $r", r=R("recording", other))["language"] == "en" + + none = mediaurl.create(db, sid, "https://8.8.8.8/watch?v=nocaptions") + jobs._transcribe(db, cfg, none, said.append) + assert heard == [none] # no captions: transcribed as before + + def test_what_it_cant_download_says_why(cfg, db, tmp_path, monkeypatch): monkeypatch.setattr(mediaurl, "build", lambda *a: ("yt-dlp_linux", "0" * 64)) cfg["documents"]["video_urls"] = True diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 074b72c89..615b1321b 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -731,6 +731,7 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { </More> <F ctx={ctx} id="documents.attachment_resources" /> <F ctx={ctx} id="documents.video_urls" /> + {v("documents.video_urls") ? <F ctx={ctx} id="documents.video_captions" /> : null} <F ctx={ctx} id="documents.web_networks" /> <F ctx={ctx} id="documents.converter" /> <F ctx={ctx} id="documents.anytopdf_url" /> diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 6d3cb933f..d6744e81f 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -3948,6 +3948,13 @@ export const FIELDS: FieldSpec[] = [ kind: "switch", hint: "Downloads the media with yt-dlp (fetched on first use) from public sites, then transcribes it; off by default", }, + { + section: "documents", + key: "video_captions", + label: "Use a video’s own captions", + kind: "switch", + hint: "When people wrote captions for it, they become its transcript and nothing is transcribed; else it’s transcribed", + }, { section: "documents", key: "converter",