From 33eea428acadf1d5addf9c123794b077b6ca774b Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:29:17 +0000 Subject: [PATCH 1/3] feat(pipelines): clean up a recording's sound before it's transcribed A new pipeline step, enhance (Clean up audio), runs a recording through WebRTC's audio processing with pywebrtc-audio (Apache-2.0, ~0.5 MB, fetched the first time the step runs): noise suppression, automatic gain, a high-pass filter, echo cancellation for two-channel call recordings, and voice detection that leaves pauses out of what's transcribed. The transcript's times are mapped back onto the recording's own timeline. The step keeps its options and what it measured on the recording (audio_cleanup); transcribing then hears the cleaned audio. The file is never changed and no copy is stored. No pipeline has the step until one adds it, so defaults are unchanged. Settings > Transcription > Audio cleanup holds the options (transcribe.cleanup). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014WcBxU5ARBoLWMYZU46St6 --- CHANGELOG.md | 7 + README.md | 4 +- docs/components.md | 1 + docs/processing.md | 36 +- docs/requirements.md | 1 + fastapi_backend/app/domain/audio_cleanup.py | 316 ++++++++++++++++++ fastapi_backend/app/domain/components.py | 21 ++ fastapi_backend/app/domain/ingest.py | 13 +- fastapi_backend/app/domain/jobs.py | 22 +- fastapi_backend/app/domain/pipelines.py | 5 +- fastapi_backend/app/domain/settings.py | 4 + fastapi_backend/app/domain/store.py | 11 + fastapi_backend/app/schemas/recordings.py | 19 ++ fastapi_backend/pyproject.toml | 4 + .../tests/domain/test_audio_cleanup.py | 247 ++++++++++++++ fastapi_backend/uv.lock | 33 +- .../__tests__/recording-jobs.test.ts | 11 +- nextjs-frontend/app/openapi-client/index.ts | 1 + .../app/openapi-client/types.gen.ts | 91 +++++ nextjs-frontend/components/batches/run-on.tsx | 1 + .../components/pipelines/pipeline-canvas.tsx | 4 +- .../components/pipelines/pipeline-editor.tsx | 3 + .../components/pipelines/pipeline-model.ts | 4 + nextjs-frontend/components/recording/jobs.ts | 12 +- .../components/settings/bodies.tsx | 27 ++ nextjs-frontend/components/settings/model.ts | 54 +++ nextjs-frontend/components/ui/loop.tsx | 2 + nextjs-frontend/openapi.json | 123 +++++++ 28 files changed, 1059 insertions(+), 18 deletions(-) create mode 100644 fastapi_backend/app/domain/audio_cleanup.py create mode 100644 fastapi_backend/tests/domain/test_audio_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cedf013e..8e5b63864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Clean up a recording's sound before it's transcribed.** A new pipeline step, Clean up audio (`enhance`), runs a + recording through WebRTC's audio processing (pywebrtc-audio, Apache-2.0, about 0.5 MB, fetched the first time it + runs): noise suppression, automatic gain, a high-pass filter, echo cancellation for two-channel call recordings, and + pauses left out of what's transcribed, with the transcript's times put back on the recording's timeline. Settings → + Transcription → Audio cleanup holds what it does; no pipeline has it until you add it, the file is never changed, and + `GET /api/v1/recordings/{id}` says what it found (`audio_cleanup`) + ([Cleaning up audio](docs/processing.md#cleaning-up-audio)). - **The upgrade agent starts the code sandbox's gateway.** Settings → Code sandbox → Run the gateway on this machine now works on installs with an upgrade agent: the agent knows the `sandbox` profile and starts or stops its three containers, as it does the MCP runner, the browser and monitoring. An older agent still says to run the line by hand diff --git a/README.md b/README.md index 04f55891d..461f3d2fe 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,15 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p rules, without a model, and a decision model types the ones the rules can't. Find what is most like a passage, an asset, a topic or an entity, in a Similar tab on every asset or by selecting text anywhere, and, with anytopdf, photos by what they show. Pages and shots are described in words by the model provider's vision model or, off by default, by - Florence-2 or Moondream on your own server. Transcripts can be cleaned up before analysis (stutters, "um"s, spacing + Florence-2 or Moondream on your own server. A recording's sound can be cleaned up before it's transcribed (noise, + volume, echo, and silence left out, with the transcript's times kept), and transcripts cleaned up before analysis (stutters, "um"s, spacing and, with a local model, punctuation put back), entities found by spaCy instead of the rules, and the dates people say read as days on the calendar, each off by default. ([Graph](docs/graph.md), [entity pages](docs/notes.md#entity-pages), [similar content](docs/processing.md#similar-content), [graph history](docs/graph-history.md), [topics](docs/topics.md), [video](docs/video.md), [NSFW classification](docs/nsfw.md), [long recordings](docs/speech-providers.md#long-recordings), + [cleaning up audio](docs/processing.md#cleaning-up-audio), [cleaning up transcripts](docs/processing.md#cleaning-up-transcripts), [dates](docs/processing.md#dates); [checks](docs/requirements.md#understand-it)) - **Learn from it.** Make a two-host podcast episode about what you pick, every claim cited and checked against its diff --git a/docs/components.md b/docs/components.md index 89aca762e..07cd2d43d 100644 --- a/docs/components.md +++ b/docs/components.md @@ -25,6 +25,7 @@ never fetch a speech or voice model. | Prompt-injection classifier | `ai.injection_check` is `classifier` | ProtectAI's deberta-v3-base-prompt-injection-v2 (Apache-2.0, about 740 MB), the tokenizers library and ONNX Runtime ([third-party content](assistant.md#third-party-content-and-prompt-injection)) | | Punctuation model | `analysis.cleanup` is on and `analysis.cleanup_punctuation` isn't `off` ([Cleaning up transcripts](processing.md#cleaning-up-transcripts)) | kredor/punctuate-all at a pinned revision (MIT, about 1.1 GB), with Transformers and PyTorch's CPU build | | spaCy | `analysis.entities` is `spacy` ([Entities](processing.md#entities-and-the-graph-explorer)) | spaCy (MIT) and the model in `analysis.spacy_model`: `en_core_web_sm`, `de_core_news_sm` or `xx_ent_wiki_sm` (MIT, 11 to 15 MB each), about 60 MB in all | +| WebRTC audio processing | a job first reaches a pipeline's `enhance` step ([Cleaning up audio](processing.md#cleaning-up-audio)) | pywebrtc-audio (Apache-2.0, about 0.5 MB, numpy only): WebRTC's noise suppression, gain control, echo cancellation and voice detection | | DuckDuckGo search | `web.search` is `duckduckgo` | ddgs (MIT, about 30 MB with lxml and primp), the client the assistant's [web search](assistant.md#web-search-and-reading-pages) reads DuckDuckGo's results with | | yt-dlp | `documents.video_urls` is on | the yt-dlp program, release 2026.08.19 (Unlicense), to download videos and podcasts imported by address | | Chat model, embedding model | the LLM provider or embeddings server is Ollama | pulled on that server (`/api/pull`) | diff --git a/docs/processing.md b/docs/processing.md index b9f469e35..dbbd50209 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -7,6 +7,8 @@ How assets move through Lens: where they come from, what each step does, and how - `scan` finds audio under each namespace's paths and fingerprints it: moved files keep their history, duplicates are skipped, and so are files whose asset someone deleted or moved to another namespace (at the same path, or a copy of the same file). +- `enhance` (only in a pipeline that has it) cleans up the sound before it's transcribed: noise, volume, rumble, + echo, and silence left out of what's transcribed ([Cleaning up audio](#cleaning-up-audio)). - `transcribe` uses SenseVoice, faster-whisper or mlx-whisper. When the configured engine isn't installed on a worker (or doesn't import there), it uses the next one that is and says so in the job log; with none at all the step fails, saying how to add one. A file that fails is marked and the batch carries on. For a @@ -23,6 +25,36 @@ How assets move through Lens: where they come from, what each step does, and how Every step takes `--ns`, `--limit` and `--force`; `run` does them all, and a lock stops two runs overlapping. +### Cleaning up audio + +A phone on a table, a fan, a quiet voice across the room or a call with the other side coming back through the +speakers: speech-to-text hears all of it. The `enhance` step (Clean up audio in the pipeline editor and Reprocess) +runs a recording through WebRTC's audio processing, the code a browser runs on a call, with +[pywebrtc-audio](https://github.com/strands-labs/pywebrtc-audio) (Apache-2.0, about 0.5 MB; fetched the first time +the step runs, [Components](components.md)). No pipeline has it until you add it before Transcribe, so nothing changes +until then. What it does is in Settings → Transcription → Audio cleanup (`transcribe.cleanup`): + +| Setting | Default | What it does | +|---|---|---| +| `noise` | `moderate` | Noise suppression: `off`, `low`, `moderate`, `high` or `very_high` (6, 12, 18 or 21 dB) | +| `gain` | on | Automatic gain control, so a quiet voice is as loud as a near one | +| `high_pass` | on | Takes out the rumble below speech: traffic, handling noise, a knocked desk | +| `echo` | `off` | Echo cancellation for a two-channel recording with what the speakers played on one channel (`left` or `right`) and the microphone on the other, as call recorders keep them; the far side's voice is taken out of the microphone, then both sides are mixed | +| `trim_silence` | on | Pauses of `min_silence_ms` (2000) or longer, by WebRTC's voice detection, are left out of what's transcribed | + +Leaving silence out means a cloud engine bills fewer minutes (the activity ledger's seconds are what was sent), a +local one finishes sooner and Whisper has no quiet stretch to imagine words in. The transcript's times are put back on +the recording's own timeline, so the player and every citation stay where the words were said. + +The step measures how much of the recording is speech and keeps what it ran with on the asset (`audio_cleanup` in +`GET /api/v1/recordings/{id}`: the options, `speech_ms`, `kept_ms`, the gain it ended on). From then on transcribing +that asset hears it cleaned up that way, also on a later Reprocess. The file is never changed and no second copy is +stored: cleaning takes a fraction of a second per minute of audio, so it's done again each time. Speaker separation and +voice IDs still listen to the original. Run the step with every option off to have the asset transcribed as it is +again. A worker that can't fetch pywebrtc-audio transcribes the original and says so in the job log. Run it on one +asset with Reprocess → Clean up audio, or `POST /api/v1/recordings/{id}/reprocess` with +`{"steps": ["enhance", "transcribe"]}`; workers whose `workers.steps` list has `transcribe` run it too. + ### Cleaning up transcripts Speech comes out of transcription as it was said, and the extractor, chapters, keywords and search all read it. With @@ -663,8 +695,8 @@ A pipeline is a named, versioned list of steps; each namespace can choose its de analyze, summarize, report). Imports, watched folders and reprocessing use the namespace's pipeline; steps that need audio skip themselves for imported text. Steps: -- `transcribe`, `diarize`, `clean` ([Cleaning up transcripts](#cleaning-up-transcripts)), `analyze`, `summarize`, and - `report` (the built-in report). +- `enhance` ([Cleaning up audio](#cleaning-up-audio)), `transcribe`, `diarize`, `clean` + ([Cleaning up transcripts](#cleaning-up-transcripts)), `analyze`, `summarize`, and `report` (the built-in report). - `llm`: renders a prompt template, sends it to the configured model, checks the reply against the template's JSON Schema and saves it as a named output (e.g. `meeting_notes`). - `spec`: writes a technical spec from the transcript into a note linked to the asset, with the Spec from transcript diff --git a/docs/requirements.md b/docs/requirements.md index 7a4214b31..d795e2aa9 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -152,6 +152,7 @@ request, to another check that proves the same thing; loosening a requirement or | With anytopdf, photos found by what they show | [test_photos.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_photos.py) `test_photos_found_by_what_they_show`, `test_off_by_default` | | Pages and shots described by the provider's vision model or, off by default, by Florence-2 on this server, fetched pinned and checked when picked | [test_florence.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_florence.py) `test_florence_describes_pictures_without_a_model_provider`, `test_florence_is_fetched_pinned_and_checked`, `test_admins_and_agents_choose_what_describes_pictures`, `test_a_picture_is_read_as_the_model_was_trained` | | Or by Moondream on this server, off by default, fetched at a pinned revision only where the chosen model can run, and saying why where it can't | [test_moondream.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_moondream.py) `test_moondream_describes_pictures_without_a_model_provider`, `test_where_each_moondream_can_run`, `test_on_a_cpu_moondream_3_is_never_fetched_and_the_step_says_why`, `test_moondream_is_fetched_pinned`, `test_admins_and_agents_choose_moondream` | +| A recording's sound cleaned up before transcription in a pipeline that has the step (noise, gain, echo), with silence left out of what's transcribed and the transcript's times kept on the recording's timeline | [test_audio_cleanup.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_audio_cleanup.py) `test_speech_is_found_and_silence_left_out_of_transcription`, `test_echo_of_the_far_side_is_taken_out`, `test_without_the_step_or_with_every_option_off_it_is_transcribed_as_it_is`, `test_an_agent_reads_what_the_step_did` | | Transcripts cleaned up before analysis (stutters, "um"s, spacing), off by default, each change in the edit history with Revert and people's corrections left alone | [test_cleanup.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_cleanup.py) `test_off_by_default_changes_nothing`, `test_clean_step_tidies_lines_and_keeps_them_in_the_history`, `test_people_s_corrections_outrank_the_rules` | | Punctuation put back in transcripts that lack it by a local model, fetched only when picked | [test_cleanup.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_cleanup.py) `test_punctuation_put_back_where_it_is_missing`, `test_without_the_model_the_rules_run_alone`, `test_the_model_is_fetched_only_when_it_is_asked_for` | | Entities found by spaCy, fetched the first time it's picked, with the defined entities first and the rules until it's there | [test_spacy_entities.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_spacy_entities.py) `test_defined_entities_come_first_then_spacy_s_names`, `test_the_analyze_step_and_workflows_use_it`, `test_without_spacy_the_rules_find_them`, `test_spacy_is_fetched_only_when_it_is_picked` | diff --git a/fastapi_backend/app/domain/audio_cleanup.py b/fastapi_backend/app/domain/audio_cleanup.py new file mode 100644 index 000000000..d68e4e336 --- /dev/null +++ b/fastapi_backend/app/domain/audio_cleanup.py @@ -0,0 +1,316 @@ +"""Cleaning up a recording's sound before it's transcribed: the enhance step (Settings → Transcription → Audio cleanup). + +A phone on a table, a fan, a quiet speaker across the room, a call recorded with the other side's voice coming back +through the speakers: speech-to-text hears all of it. The enhance step runs the recording through WebRTC's audio +processing, the same code a browser runs on a call, through pywebrtc-audio (Apache-2.0, about 0.5 MB, numpy only, +fetched the first time the step runs). Every option is in the settings: + +- noise: noise suppression, off or low, moderate, high or very high (6, 12, 18 or 21 dB). +- gain: automatic gain control, so a quiet voice is as loud as a near one. +- high_pass: takes out the rumble below speech (traffic, handling noise, a desk being knocked). +- echo: echo cancellation, for a two-channel recording with what the speakers played on one channel (left or right) + and the microphone on the other, as call recorders keep them: the far side's voice is taken out of the microphone, + then both sides are mixed for transcription. +- trim_silence: the stretches with nobody speaking (min_silence_ms or longer, by WebRTC's voice detection) are left + out of what's transcribed, so a cloud engine bills fewer minutes, a local one finishes sooner and Whisper has no + silence to imagine words in. The transcript's times are put back on the recording's own timeline. + +The step itself measures how much of the recording is speech and keeps the options it ran with on the recording +(`audio_cleanup`); from then on transcribing it hears the audio cleaned up that way. The recording's file is never +changed and no second copy is stored: the cleaning takes a fraction of a second per minute, so it's done again each +time. Speaker separation and voice IDs still listen to the original. Running the step with every option off puts the +recording back to being transcribed as it is. +""" + +from __future__ import annotations + +import importlib.metadata + +import numpy as np + +from . import components, ingest, store + +R = store.R +SR = ingest.SR +FRAME = SR // 100 # WebRTC works in 10 ms frames +WINDOW = 10 # frames: voice detection is read per tenth of a second +THRESHOLD = 0.4 # a tenth of a second whose mean speech probability reaches this is speech +PAD_MS = 300 # kept either side of speech, so a word's first and last sounds aren't cut +NOISE = {"off": None, "low": 0, "moderate": 1, "high": 2, "very_high": 3} # WebRTC's levels: 6, 12, 18 and 21 dB +ECHO = ("off", "left", "right") # which channel holds what the speakers played +DEFAULTS = store.DEFAULTS["transcribe"]["cleanup"] +KEYS = ("noise", "gain", "high_pass", "echo", "trim_silence", "min_silence_ms") +PACKAGE = "pywebrtc-audio" + + +def check(value): + """transcribe.cleanup as it's saved: ValueError naming the first thing wrong with it.""" + if not isinstance(value, dict): + raise ValueError("transcribe.cleanup is {noise, gain, high_pass, echo, trim_silence, min_silence_ms}") + extra = sorted(set(value) - set(KEYS)) + if extra: + raise ValueError(f"transcribe.cleanup has no setting {extra[0]}") + out = {**DEFAULTS, **value} + if out["noise"] not in NOISE: + raise ValueError(f"transcribe.cleanup.noise is one of: {', '.join(NOISE)}") + if out["echo"] not in ECHO: + raise ValueError(f"transcribe.cleanup.echo is one of: {', '.join(ECHO)}") + for k in ("gain", "high_pass", "trim_silence"): + if not isinstance(out[k], bool): + raise ValueError(f"transcribe.cleanup.{k} is true or false") + ms = out["min_silence_ms"] + if not (isinstance(ms, int) and not isinstance(ms, bool) and 500 <= ms <= 60000): + raise ValueError("transcribe.cleanup.min_silence_ms is a whole number of milliseconds from 500 to 60000") + return out + + +def options(cfg): + """The audio cleanup settings, with defaults for what isn't set (a bad saved value falls back to its default).""" + mine = dict(((cfg.get("transcribe") or {}).get("cleanup")) or {}) + out = dict(DEFAULTS) + for k in KEYS: + if k in mine: + try: + out = check({**out, k: mine[k]}) + except ValueError: + pass + return out + + +def does_anything(o): + return o["noise"] != "off" or o["gain"] or o["high_pass"] or o["echo"] != "off" or o["trim_silence"] + + +def ready(): + return components.importable("pywebrtc_audio") + + +def ensure(cfg, say=None): + """pywebrtc-audio, fetched now if this worker hasn't got it (it's small); RuntimeError when it can't be had.""" + if ready(): + return + try: + components.pip_install(cfg, "webrtc", say) + except (components.Unavailable, RuntimeError, OSError) as e: + raise RuntimeError(f"{PACKAGE} isn't installed and couldn't be fetched ({e})") from None + if not ready(): + raise RuntimeError(f"{PACKAGE} was fetched but doesn't load on this machine") + + +def version(): + try: + return importlib.metadata.version(PACKAGE) + except importlib.metadata.PackageNotFoundError: + return None + + +def process(near, o, far=None): + """Samples (16 kHz mono float) through WebRTC's processing with these options: (the cleaned samples, each 10 ms + frame's speech probability, the gain applied at the end in dB or None). With `far` (what the speakers played, + the same length), the echo of it is taken out of `near`.""" + from pywebrtc_audio import AudioProcessor + + level = NOISE[o["noise"]] + ap = AudioProcessor( + sample_rate=SR, + noise_suppression=level is not None, + ns_level=level or 0, + auto_gain_control=bool(o["gain"]), + high_pass_filter=bool(o["high_pass"]), + echo_cancellation=far is not None, + ) + x = np.ascontiguousarray(near, dtype=np.float32) + f = None if far is None else np.ascontiguousarray(far, dtype=np.float32) + out = np.empty_like(x) + probs = np.empty((len(x) + FRAME - 1) // FRAME, dtype=np.float32) + for k, i in enumerate(range(0, len(x), FRAME)): + out[i : i + FRAME] = ap.process(x[i : i + FRAME]) if f is None else ap.process(x[i : i + FRAME], f[i : i + FRAME]) + probs[k] = ap.speech_probability + gain = round(float(ap.gain_db), 1) if o["gain"] and len(x) else None + return out, probs, gain + + +def _loud(probs): + """Whether each tenth of a second is speech.""" + k = -(-len(probs) // WINDOW) + padded = np.pad(probs, (0, k * WINDOW - len(probs)), constant_values=0) + return padded.reshape(k, WINDOW).mean(axis=1) >= THRESHOLD + + +def speech_ms(probs): + return int(_loud(probs).sum()) * WINDOW * 10 if len(probs) else 0 + + +def speech_spans(probs, min_silence_ms, total_ms): + """Where there's speech, from the frames' speech probabilities: [(start, end)] in ms, each padded by PAD_MS, with + pauses shorter than min_silence_ms kept in.""" + if not len(probs): + return [] + loud = _loud(probs) + k = len(loud) + step = WINDOW * 10 # ms per window + out = [] + i = 0 + while i < k: + if not loud[i]: + i += 1 + continue + j = i + while j < k and loud[j]: + j += 1 + a, b = max(0, i * step - PAD_MS), min(total_ms, j * step + PAD_MS) + if out and a - out[-1][1] < min_silence_ms: + out[-1] = (out[-1][0], b) + else: + out.append((a, b)) + i = j + if out and out[0][0] < min_silence_ms: # a short silence at either end isn't worth a join + out[0] = (0, out[0][1]) + if out and total_ms - out[-1][1] < min_silence_ms: + out[-1] = (out[-1][0], total_ms) + return out + + +def cut(audio, spans): + """Only the samples in these spans (ms), one after another: (samples, joins), a join being (where it starts in + what's kept, where it started in the recording, how long it is), all in samples.""" + parts, joins, at = [], [], 0 + for a, b in spans: + s, e = a * SR // 1000, min(len(audio), b * SR // 1000) + if e <= s: + continue + parts.append(audio[s:e]) + joins.append((at, s, e - s)) + at += e - s + return (np.concatenate(parts) if parts else audio[:0]), joins + + +def to_recording(ms, joins, end=False): + """A time in what was kept (ms) on the recording's own timeline. An end that falls on a join stays with what came + before it.""" + t = ms * SR / 1000 + pick = joins[0] + for j in joins: + if j[0] < t or (j[0] == t and not end): + pick = j + else: + break + at, start, n = pick + return int(round((start + min(max(t - at, 0), n)) * 1000 / SR)) + + +def remap(segs, joins): + """Segments transcribed from what was kept, on the recording's timeline (their words too).""" + import json + + for s in segs: + s["t0"], s["t1"] = to_recording(s["t0"], joins), to_recording(s["t1"], joins, end=True) + if s.get("words"): + words = json.loads(s["words"]) + s["words"] = json.dumps([[w, to_recording(a, joins), to_recording(b, joins, end=True)] for w, a, b in words]) + return segs + + +def _channels(path, o, channels): + """(microphone, speakers) for echo cancellation, or None when it's off or the recording has one channel.""" + if o["echo"] == "off" or (channels or 1) < 2: + return None + st = ingest.decode(path, channels=2) + far = 0 if o["echo"] == "left" else 1 + return np.ascontiguousarray(st[:, 1 - far]), np.ascontiguousarray(st[:, far]) + + +def clean(o, path, mono, channels=1): + """A recording's audio cleaned up with options o: (samples, frame speech probabilities, gain in dB, how echo was + handled). `mono` is the recording decoded as it is; a two-channel one is decoded again for echo cancellation.""" + pair = _channels(path, o, channels) + if pair is None: + out, probs, gain = process(mono, o) + echo = "no second channel" if o["echo"] != "off" else None + else: + near, far = pair + out, probs, gain = process(near, o, far) + out = np.clip((out + far) / 2, -1, 1).astype(np.float32) # both sides of the call, without the echo + echo = o["echo"] + return out, probs, gain, echo + + +def hear(cfg, rec, path, mono, log=print): + """What transcribing a recording hears: its audio cleaned up as its enhance step set, silence left out when it + asked. Returns (samples, joins or None); the original when this worker can't run the cleanup.""" + o = {**DEFAULTS, **{k: v for k, v in (rec.get("audio_cleanup") or {}).items() if k in KEYS}} + try: + ensure(cfg, log) + except RuntimeError as e: + log(f" transcribing the original audio: {e}") + return mono, None + out, probs, _, _ = clean(o, path, mono, rec.get("channels")) + if not o["trim_silence"]: + return out, None + total = len(out) * 1000 // SR + spans = speech_spans(probs, o["min_silence_ms"], total) + if not spans: + return out, None # it heard no speech: the engine has the last word + if spans == [(0, total)]: + return out, None + kept, joins = cut(out, spans) + log(f" {round((total - len(kept) * 1000 / SR) / 1000)} s of silence left out of what's transcribed") + return kept, joins + + +def step(db, cfg, rid, say): + """The enhance step: measure the recording through the cleanup and keep its options on it, or clear them when + every option is off. Returns what it kept, or None.""" + o = options(cfg) + r = R("recording", rid) + if not does_anything(o): + db.q("UPDATE $r SET audio_cleanup = NONE", r=r) + return None + rec = db.one("SELECT path, remote, space, channels, web FROM $r", r=r) or {} + if (rec.get("web") or {}).get("media"): # a video or episode given by its address: its media downloaded first + from . import mediaurl + + mediaurl.ensure(db, cfg, rid, rec, say) + rec = db.one("SELECT path, remote, space, channels FROM $r", r=r) or {} + ensure(cfg, say) + path = ingest.audio_path(db, cfg, rec) + mono = ingest.decode(path) + out, probs, gain, echo = clean(o, path, mono, rec.get("channels")) + del mono + total = len(out) * 1000 // SR + spans = speech_spans(probs, o["min_silence_ms"], total) + speech = speech_ms(probs) + kept = sum(b - a for a, b in spans) if o["trim_silence"] and spans else total + saved = store.clean( + { + **o, + "duration_ms": total, + "speech_ms": speech, + "kept_ms": kept, + "gain_db": gain, + "echo_note": echo if echo and echo not in ECHO else None, + "engine": f"{PACKAGE} {version() or ''}".strip(), + "at": store.now(), + } + ) + db.q("UPDATE $r SET audio_cleanup = $a", r=r, a=saved) + return saved + + +def summary(saved): + """One line for the step's log: what it found and what transcribing will hear.""" + if not saved: + return "every audio cleanup option is off: it's transcribed as it is" + total = max(saved["duration_ms"], 1) + parts = [f"speech in {round(100 * saved['speech_ms'] / total)}% of {round(total / 60000, 1)} min"] + if saved["noise"] != "off": + parts.append(f"noise suppressed ({saved['noise'].replace('_', ' ')})") + if saved.get("gain_db") is not None: + parts.append(f"gain {saved['gain_db']:+.1f} dB") + if saved.get("echo_note"): + parts.append(f"echo not cancelled: {saved['echo_note']}") + elif saved["echo"] != "off": + parts.append(f"echo of the {saved['echo']} channel cancelled") + if saved["trim_silence"] and saved["kept_ms"] < saved["duration_ms"]: + parts.append(f"{round((saved['duration_ms'] - saved['kept_ms']) / 60000, 1)} min of silence left out of transcription") + return "; ".join(parts) diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index 4e7239211..0d680c9df 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -735,6 +735,19 @@ def fetch(self, cfg, m, say): photos.fetch_model(cfg, say) +class WebrtcAudio(Component): + """pywebrtc-audio, for the enhance step (audio_cleanup.py): fetched the first time a job waits for that step.""" + + def needed(self, cfg, m): + return True # it waits for first use: an archive whose pipelines never clean up audio fetches nothing + + def present(self, cfg): + return importable("pywebrtc_audio") + + def fetch(self, cfg, m, say): + pip_install(cfg, "webrtc", say) + + class Msg(Component): def needed(self, cfg, m): return False # only when asked for (components.also): it's GPL-3.0 @@ -979,6 +992,14 @@ def detail(self, cfg): ), OllamaModel("llm-model", "Chat model", "the LLM provider's model, pulled on Ollama", section="llm"), OllamaModel("embedding-model", "Embedding model", "search by meaning's model, pulled on Ollama", section="embeddings", steps={"embed"}), + WebrtcAudio( + "webrtc-audio", + "WebRTC audio processing", + "cleans up a recording's sound before it's transcribed: noise, gain, echo and silence (the enhance step; pywebrtc-audio)", + steps={"enhance"}, + size_mb=1, + license="Apache-2.0", + ), Smb("smb", "SMB server", "shares the file tree over SMB, read-only (smb.enabled; impacket)", size_mb=15, license="Apache-1.1"), SandboxClient( "sandbox", diff --git a/fastapi_backend/app/domain/ingest.py b/fastapi_backend/app/domain/ingest.py index 2eb1bf7c5..7666ca0f9 100644 --- a/fastapi_backend/app/domain/ingest.py +++ b/fastapi_backend/app/domain/ingest.py @@ -498,12 +498,19 @@ def add_envelope(db, cfg, rid): def transcribe_one(db, cfg, rid, log=print, engine=None): - r = db.one("SELECT record::id(id) AS id, space, path, title, remote FROM $r", r=store.R("recording", rid)) + r = db.one("SELECT record::id(id) AS id, space, path, title, remote, channels, audio_cleanup FROM $r", r=store.R("recording", rid)) engine, t = engine or get_engine(cfg, log), time.time() - audio = decode(audio_path(db, cfg, r)) + path = audio_path(db, cfg, r) + audio, joins = decode(path), None + env = envelope(audio) # the player's waveform is the recording's own + if r.get("audio_cleanup"): # its enhance step asked for it to be heard cleaned up (audio_cleanup.py) + from . import audio_cleanup + + audio, joins = audio_cleanup.hear(cfg, r, path, audio, log) segs = engine.transcribe(audio) - env = envelope(audio) del audio + if joins: + segs = audio_cleanup.remap(segs, joins) write_transcript( db, rid, diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 242bda891..85c865afd 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -26,7 +26,7 @@ "record::id(id) AS id, recording, space, batch, pipeline, steps, step_index, next_step, status, worker, error, attempts, " "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total, cost_usd, tokens, cost_estimate, hold" ) -MEDIA_STEPS = {"transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"} # they take longer the longer the recording +MEDIA_STEPS = {"enhance", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"} # slower on longer media FILED = ("audio", "document", "image") # sources with a file of their own for the steps to work on TIMINGS = 25 # recent timings kept per kind of step, for estimates @@ -71,6 +71,19 @@ def _transcribe(db, cfg, rid, say, spec=None): ingest.transcribe_one(db, cfg, rid, say) +def _enhance(db, cfg, rid, say, spec=None): + """A recording's sound cleaned up before it's transcribed (audio_cleanup.py), as Settings → Transcription → Audio + cleanup sets it.""" + from . import audio_cleanup + + source = _source(db, rid) + if source in ("document", "image"): + raise Skip(f"{'an image' if source == 'image' else 'a document'} has no sound to clean up") + if source != "audio": + raise Skip("imported text has no sound to clean up") + say(audio_cleanup.summary(audio_cleanup.step(db, cfg, rid, say))) + + def _diarize(db, cfg, rid, say, spec=None): rec = db.one("SELECT source, diarizer FROM $r", r=R("recording", rid)) or {} if rec.get("source") in ("document", "image"): @@ -207,6 +220,7 @@ def _describe(db, cfg, rid, say, spec=None): VIDEO_STEPS = {"shots", "ocr", "faces", "objects", "describe"} # added after workers.steps lists were first written STEPS = { + "enhance": _enhance, "transcribe": _transcribe, "read": _read, "diarize": _diarize, @@ -235,14 +249,14 @@ def for_source(steps, source): nothing to transcribe and its speakers came with it, so it skips both steps too (unless they're all that was asked for: those still run, to say why there's nothing to do).""" if source == "transcript": - return [s for s in steps if s.get("type") not in ("transcribe", "diarize")] or steps + return [s for s in steps if s.get("type") not in ("enhance", "transcribe", "diarize")] or steps if source not in ("document", "image"): return steps out = [] for s in steps: if s.get("type") == "transcribe": s = {**s, "type": "read"} - if s.get("type") == "diarize" or (s.get("type") == "read" and any(o.get("type") == "read" for o in out)): + if s.get("type") in ("enhance", "diarize") or (s.get("type") == "read" and any(o.get("type") == "read" for o in out)): continue out.append(s) return out @@ -937,6 +951,8 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None): self.can |= VIDEO_STEPS if self.can & {"analyze", "summarize"}: # embedding needs what they need; lists written before it existed run it too self.can.add("embed") + if not steps and "transcribe" in self.can: # cleaning up the sound needs no more than transcribing; older lists run it + self.can.add("enhance") if "analyze" in self.can: # cleaning up a transcript needs no more than analysing it; older lists run it too self.can.add("clean") if not steps and "objects" in self.can: # NSFW classification runs on ONNX Runtime too; older lists run it as well diff --git a/fastapi_backend/app/domain/pipelines.py b/fastapi_backend/app/domain/pipelines.py index 1b52c5198..04f958f48 100644 --- a/fastapi_backend/app/domain/pipelines.py +++ b/fastapi_backend/app/domain/pipelines.py @@ -1,5 +1,6 @@ """Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without -one, recordings get the standard pipeline. Steps: transcribe (speech to text), read (a document's or an +one, recordings get the standard pipeline. Steps: enhance (a recording's sound cleaned up before it's transcribed, +audio_cleanup.py), transcribe (speech to text), read (a document's or an image's pages and their text), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), describe, clean (a transcript tidied before analysis, cleanup.py), analyze, embed (passages for search by meaning, semantic.py), summarize, llm (a prompt template whose structured result is saved as a named output), spec (a technical spec from the @@ -28,7 +29,7 @@ "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", "report", ] # fmt: skip TYPES = { - "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", + "enhance", "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", "llm", "spec", "report", diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 732202223..1e829fd0b 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -439,6 +439,10 @@ def _check(section, key, value, default): ): raise ValueError("iiif.viewers is a list of {name, url} with an http(s) URL; the URL may use {manifest} and {content_state}") return value + if (section, key) == ("transcribe", "cleanup"): + from . import audio_cleanup + + return audio_cleanup.check(value) if (section, key) == ("server", "trusted_proxies"): return ipgroups.proxies(value) if (section, key) == ("llm", "vision_model"): diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 362812c12..188d298b1 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -137,6 +137,16 @@ def labels(): "sensevoice": {"model": "iic/SenseVoiceSmall", "hub": "ms", "vad_max_segment_ms": 30000, "batch_size": 16}, "whisper": {"model": "large-v3-turbo", "compute_type": "default"}, "mlx_whisper": {"model": "mlx-community/whisper-large-v3-turbo"}, + # what the enhance step does to a recording's sound before it's transcribed (audio_cleanup.py); a pipeline only + # runs it when the step is in it + "cleanup": { + "noise": "moderate", + "gain": True, + "high_pass": True, + "echo": "off", + "trim_silence": True, + "min_silence_ms": 2000, + }, }, "diarize": { "engine": "auto", @@ -274,6 +284,7 @@ def labels(): "stale_minutes": 15, "max_attempts": 3, "steps": [ + "enhance", "transcribe", "read", "diarize", diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index 81642fe82..bcc211951 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -55,6 +55,24 @@ class NsfwHit(ResponseModel): frame: str | None = Field(None, description="a signed link to the frame or page") +class AudioCleanup(ResponseModel): + """How the enhance step cleaned up a recording's sound for transcription (docs/audio-cleanup.md).""" + + noise: str = Field(description="noise suppression: off, low, moderate, high or very_high (6, 12, 18 or 21 dB)") + gain: bool = Field(description="automatic gain control") + high_pass: bool = Field(description="the rumble below speech taken out") + echo: str = Field(description="echo cancellation: off, or the channel (left or right) holding what the speakers played") + trim_silence: bool = Field(description="silences of min_silence_ms or longer left out of what's transcribed") + min_silence_ms: int = Field(description="the shortest pause left out when trimming silence") + duration_ms: int = Field(description="how long the recording is") + speech_ms: int = Field(description="how much of it WebRTC's voice detection heard as speech") + kept_ms: int = Field(description="how much of it is transcribed: all of it unless silence is trimmed") + gain_db: float | None = Field(None, description="the gain applied by the end of the recording, in dB") + echo_note: str | None = Field(None, description="why echo wasn't cancelled though it was asked for") + engine: str | None = Field(None, description="what cleaned it up, with its version") + at: str | None = Field(None, description="when the step ran") + + class NsfwState(ResponseModel): """What NSFW classification found (docs/nsfw.md), where it's on in the resource's namespace.""" @@ -285,6 +303,7 @@ class Recording(ResponseModel): web: WebPage | None = Field(None, description="a web page captured as a document") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") nsfw: NsfwState | None = Field(None, description="NSFW classification, where it's on in its namespace and it was classified") + audio_cleanup: AudioCleanup | None = Field(None, description="how its enhance step cleaned up its sound before transcription") class Page(ResponseModel): diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index a7675df8f..8ebbc771d 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -69,6 +69,9 @@ sandbox = ["openshell>=0.1.4,<0.2"] # the assistant's web search through DuckDuckGo (web.py): ddgs (MIT), with its Rust HTTP client (primp) and lxml; # fetched when DuckDuckGo is picked in Settings › AI assistant › Web search web = ["ddgs>=9.16,<10"] +# the enhance step (audio_cleanup.py): WebRTC's noise suppression, gain control, echo cancellation and voice +# detection through pywebrtc-audio (Apache-2.0; about 0.5 MB, numpy only); fetched the first time a pipeline runs it +webrtc = ["pywebrtc-audio>=0.2.0,<0.3"] # Outlook .msg emails (extract-msg is GPL-3.0; .eml needs nothing) msg = ["extract-msg"] @@ -100,6 +103,7 @@ dev = [ "impacket>=0.13", # the SMB share's server, and the SMB client its tests sign in with "openshell>=0.1.4,<0.2", # the code sandbox's client, tested against a fake OpenShell gateway "botocore>=1.43.113", + "pywebrtc-audio>=0.2.0,<0.3", # the enhance step's WebRTC audio processing, tested on synthetic audio ] [tool.hatch.build.targets.wheel] diff --git a/fastapi_backend/tests/domain/test_audio_cleanup.py b/fastapi_backend/tests/domain/test_audio_cleanup.py new file mode 100644 index 000000000..2fcb4bf19 --- /dev/null +++ b/fastapi_backend/tests/domain/test_audio_cleanup.py @@ -0,0 +1,247 @@ +"""The enhance step: a recording's sound cleaned up with WebRTC's audio processing (pywebrtc-audio) before it's +transcribed, with silence left out of what's transcribed and the transcript put back on the recording's timeline.""" + +from __future__ import annotations + +import json +import wave + +import numpy as np +import pytest + +from app.domain import audio_cleanup, ingest, jobs, pipelines, settings, store +from tests.helpers import drain, quiet + +R = store.R +SR = 16000 + + +def babble(seconds, seed): + """Something WebRTC's voice detection hears as speech: syllables of a voice with two formants, with gaps.""" + rng = np.random.default_rng(seed) + n = int(seconds * SR) + out, i = np.zeros(n, np.float32), 0 + while i < n: + size, f0 = int(rng.uniform(0.12, 0.25) * SR), rng.uniform(100, 220) + t = np.arange(size) / SR + phase = np.cumsum(2 * np.pi * f0 * (1 + 0.1 * np.sin(2 * np.pi * rng.uniform(2, 5) * t)) / SR) + f1, f2 = rng.uniform(300, 800), rng.uniform(900, 2200) + s = sum( + np.sin(k * phase) * (np.exp(-(((k * f0 - f1) / 200) ** 2)) + 0.6 * np.exp(-(((k * f0 - f2) / 300) ** 2))) for k in range(1, 30) + ) + out[i : i + size] = (0.3 * s * np.sin(np.pi * np.arange(size) / size))[: n - i] + i += size + int(rng.uniform(0.03, 0.12) * SR) + return out + + +def quiet_for(seconds): + return np.zeros(int(seconds * SR), np.float32) + + +def write(path, *channels): + x = np.stack(channels, axis=1) if len(channels) > 1 else channels[0][:, None] + with wave.open(str(path), "wb") as w: + w.setnchannels(x.shape[1]) + w.setsampwidth(2) + w.setframerate(SR) + w.writeframes((np.clip(x, -1, 1) * 32767).astype(np.int16).tobytes()) + + +def talk(): + """2 s of quiet, 4 s of speech, 10 s of quiet, 3 s of speech, 3 s of quiet, over a faint hiss.""" + x = np.concatenate([quiet_for(2), babble(4, 1), quiet_for(10), babble(3, 2), quiet_for(3)]) + return (x + np.random.default_rng(0).normal(0, 0.003, len(x))).astype(np.float32) + + +class Engine: + """Hears what it's given and says two lines with word times, at 1 s and at 7 s of what it heard.""" + + name = "sensevoice" + heard = [] + + def transcribe(self, audio): + Engine.heard.append(len(audio)) + return [ + {"t0": 1000, "t1": 2000, "text": "first words", "words": json.dumps([["first", 1000, 1500], ["words", 1500, 2000]])}, + {"t0": 7000, "t1": 8000, "text": "later on", "words": None}, + ] + + +@pytest.fixture +def recording(db, cfg, folder, monkeypatch): + def make(*channels): + wav, tr = folder / "talk.wav", folder / "talk.txt" + write(wav, *(channels or (talk(),))) + tr.write_text("[00:00] Ann: Hello.\n") + return ingest.import_transcript(db, cfg, "pods", tr, audio=wav, log=quiet) + + Engine.heard = [] + monkeypatch.setattr(ingest, "get_engine", lambda cfg, log=None: Engine()) + return make + + +def _run(db, cfg, rid, *steps): + jid = jobs.enqueue(db, rid, [*steps, {"type": "transcribe", "force": True}]) + drain(db, cfg) + job = db.one("SELECT status, log FROM $j", j=R("job", jid)) + assert job["status"] == "succeeded", job["log"] + return job["log"] + + +def _segments(db, rid): + return db.rows("SELECT idx, t0, t1, text, words FROM segment WHERE recording = $r ORDER BY idx", r=rid) + + +def test_speech_is_found_and_silence_left_out_of_transcription(db, cfg, recording): + rid = recording() + log = _run(db, cfg, rid, "enhance") + saved = db.one("SELECT audio_cleanup FROM $r", r=R("recording", rid))["audio_cleanup"] + assert saved["duration_ms"] == 22000 + assert 6000 <= saved["speech_ms"] <= 8000 # 7 s were spoken + assert saved["noise"] == "moderate" and saved["gain"] is True and saved["engine"].startswith("pywebrtc-audio") + assert any("speech in" in line and "of silence left out of transcription" in line for line in log) + # the 10 s pause and the 3 s at the end weren't transcribed; the 2 s at the start are too short to leave out + heard = Engine.heard[-1] / SR + assert 9 <= heard <= 11.5 + first, later = _segments(db, rid) + assert (first["t0"], first["t1"]) == (1000, 2000) # before the pause: where it was + assert json.loads(first["words"]) == [["first", 1000, 1500], ["words", 1500, 2000]] + assert 15000 <= later["t0"] <= 16500 # after it: on the recording's timeline again + assert later["t1"] - later["t0"] == 1000 + # the player's waveform is still the whole recording's + rec = db.one("SELECT envelope, path FROM $r", r=R("recording", rid)) + assert rec["envelope"] == ingest.envelope(ingest.decode(rec["path"])) + + +def test_without_the_step_or_with_every_option_off_it_is_transcribed_as_it_is(db, cfg, recording): + rid = recording() + _run(db, cfg, rid) + assert Engine.heard[-1] == 22 * SR + _run(db, cfg, rid, "enhance") + assert Engine.heard[-1] < 22 * SR + cfg["transcribe"]["cleanup"] = {"noise": "off", "gain": False, "high_pass": False, "echo": "off", "trim_silence": False} + log = _run(db, cfg, rid, "enhance") + assert db.one("SELECT audio_cleanup FROM $r", r=R("recording", rid)).get("audio_cleanup") is None + assert any("transcribed as it is" in line for line in log) + assert Engine.heard[-1] == 22 * SR + + +def test_cleaned_but_not_trimmed(db, cfg, recording): + cfg["transcribe"]["cleanup"] = {"trim_silence": False, "noise": "very_high"} + rid = recording() + _run(db, cfg, rid, "enhance") + assert Engine.heard[-1] == 22 * SR + assert [s["t0"] for s in _segments(db, rid)] == [1000, 7000] + + +def test_echo_of_the_far_side_is_taken_out(cfg): + far = np.concatenate([babble(20, 5), quiet_for(4)]) + lag = int(0.03 * SR) + echo = (np.concatenate([np.zeros(lag, np.float32), far[:-lag]]) * 0.5).astype(np.float32) + o = {**audio_cleanup.DEFAULTS, "noise": "off", "gain": False, "high_pass": False} + out, _, _ = audio_cleanup.process(echo, o, far) + + def rms(x): # once it has settled + return float(np.sqrt(np.mean(np.square(x[4 * SR : 20 * SR])))) + + assert rms(out) < rms(echo) / 20 + + +def test_echo_needs_a_second_channel(db, cfg, recording, folder): + cfg["transcribe"]["cleanup"] = {"echo": "right"} + rid = recording() # one channel + log = _run(db, cfg, rid, "enhance") + assert db.one("SELECT audio_cleanup FROM $r", r=R("recording", rid))["audio_cleanup"]["echo_note"] == "no second channel" + assert any("echo not cancelled: no second channel" in line for line in log) + + mic, speakers = talk(), np.concatenate([quiet_for(16), babble(3, 9), quiet_for(3)]) + rid = recording(mic, speakers) + log = _run(db, cfg, rid, "enhance") + assert any("echo of the right channel cancelled" in line for line in log) + assert Engine.heard[-1] < 22 * SR # both sides heard, the quiet left out + + +def test_documents_and_imported_text_have_no_sound(db, cfg): + rid = ingest.import_text(db, cfg, "pods", "Ann|N|Hello there.", "Note") + assert jobs.for_source([{"type": "enhance"}, {"type": "analyze"}], "transcript") == [{"type": "analyze"}] + assert jobs.for_source([{"type": "enhance"}, {"type": "transcribe"}], "document") == [{"type": "read"}] + jid = jobs.enqueue(db, rid, ["enhance"]) + drain(db, cfg) + assert any( + "enhance skipped: imported text has no sound to clean up" in line for line in db.one("SELECT log FROM $j", j=R("job", jid))["log"] + ) + + +def test_a_worker_without_the_library_transcribes_the_original(db, cfg, recording, monkeypatch): + rid = recording() + _run(db, cfg, rid, "enhance") + + def missing(cfg, say=None): + raise RuntimeError("pywebrtc-audio isn't installed and couldn't be fetched (no network)") + + monkeypatch.setattr(audio_cleanup, "ensure", missing) + _run(db, cfg, rid) + assert Engine.heard[-1] == 22 * SR + + +def test_spans_joins_and_times(): + probs = np.zeros(400, np.float32) # 4 s of 10 ms frames + probs[50:100] = 0.9 # 0.5–1.0 s + probs[120:150] = 0.9 # a 0.2 s pause, then 1.2–1.5 s + probs[340:360] = 0.9 # 3.4–3.6 s, after 1.9 s of quiet + assert audio_cleanup.speech_ms(probs) == 1000 + assert audio_cleanup.speech_spans(probs, 1000, 4000) == [(0, 1800), (3100, 4000)] # padded, short ends kept + assert audio_cleanup.speech_spans(probs, 100, 4000) == [(200, 1800), (3100, 3900)] + x = np.arange(4 * SR, dtype=np.float32) + kept, joins = audio_cleanup.cut(x, [(200, 1800), (3100, 3900)]) + assert len(kept) == int(2.4 * SR) and kept[0] == 0.2 * SR and kept[int(1.6 * SR)] == 3.1 * SR + to = audio_cleanup.to_recording + assert [to(0, joins), to(1600, joins), to(1600, joins, end=True), to(2400, joins, end=True)] == [200, 3100, 1800, 3900] + assert audio_cleanup.speech_spans(np.zeros(100, np.float32), 1000, 1000) == [] + + +def test_settings_are_checked(db, cfg): + settings.save(db, cfg, "transcribe", {"cleanup": {"noise": "high", "echo": "left", "min_silence_ms": 1500}}) + eff = settings.effective(db, cfg) + assert audio_cleanup.options(eff) == {**audio_cleanup.DEFAULTS, "noise": "high", "echo": "left", "min_silence_ms": 1500} + for bad, says in ( + ({"noise": "loud"}, "noise is one of"), + ({"echo": "both"}, "echo is one of"), + ({"gain": "yes"}, "gain is true or false"), + ({"min_silence_ms": 10}, "min_silence_ms"), + ({"denoise": True}, "has no setting denoise"), + ): + with pytest.raises(ValueError, match=says): + settings.save(db, cfg, "transcribe", {"cleanup": bad}) + + +def test_pipelines_and_workers_take_the_step(db, cfg): + assert pipelines.validate_steps(db, ["enhance", "transcribe"]) == [{"type": "enhance"}, {"type": "transcribe"}] + assert "enhance" not in pipelines.STANDARD # nothing changes until a pipeline has it + cfg["workers"]["steps"] = ["transcribe", "diarize"] # a list written before the step existed + assert "enhance" in jobs.Worker(db, lambda: cfg, name="older").can + from app.domain import components + + assert components.BY_ID["webrtc-audio"].steps == {"enhance"} # fetched when a job first waits for it + + +def test_an_agent_reads_what_the_step_did(client, db, cfg, recording): + from tests.helpers import login, make_user + + rid = recording() + _run(db, cfg, rid, "enhance") + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + h = login(client, "ed@x.io", "editor password 1") + got = client.get(f"/api/v1/recordings/{rid}", headers=h).json()["audio_cleanup"] + assert got["kept_ms"] < got["duration_ms"] == 22000 and got["trim_silence"] is True + pid = client.post( + "/api/v1/pipelines", headers=login(client, *_admin(db)), json={"name": "Clean then transcribe", "steps": ["enhance", "transcribe"]} + ) + assert pid.status_code in (200, 201), pid.text + + +def _admin(db): + from tests.helpers import make_user + + make_user(db, "ad@x.io", "admin password 1", admin=True) + return "ad@x.io", "admin password 1" diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index 9842acede..68e4aabf0 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -2022,6 +2022,9 @@ voices = [ web = [ { name = "ddgs" }, ] +webrtc = [ + { name = "pywebrtc-audio" }, +] whisper = [ { name = "faster-whisper" }, ] @@ -2046,6 +2049,7 @@ dev = [ { name = "pytest-mock" }, { name = "pytest-xdist" }, { name = "python-dotenv" }, + { name = "pywebrtc-audio" }, { name = "ruff" }, { name = "tokenizers" }, { name = "types-pyyaml" }, @@ -2087,6 +2091,7 @@ requires-dist = [ { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin' and extra == 'mac-ocr'" }, { name = "pypdf", specifier = ">=6.20.1" }, { name = "python-doctr", marker = "extra == 'doctr'", specifier = ">=1.0" }, + { name = "pywebrtc-audio", marker = "extra == 'webrtc'", specifier = ">=0.2.0,<0.3" }, { name = "pyyaml", specifier = ">=6" }, { name = "rapidocr-onnxruntime", marker = "extra == 'rapidocr'" }, { name = "rdflib", specifier = ">=7.1,<8" }, @@ -2108,7 +2113,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'punctuation'" }, { name = "webauthn", specifier = ">=3.0.1,<4" }, ] -provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "msg"] +provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "webrtc", "msg"] [package.metadata.requires-dev] dev = [ @@ -2131,6 +2136,7 @@ dev = [ { name = "pytest-mock", specifier = ">=3.14.0,<4" }, { name = "pytest-xdist", specifier = ">=3.6,<4" }, { name = "python-dotenv", specifier = ">=1.2.4,<2" }, + { name = "pywebrtc-audio", specifier = ">=0.2.0,<0.3" }, { name = "ruff", specifier = ">=0.17.0,<0.18" }, { name = "tokenizers", specifier = ">=0.23.3" }, { name = "types-pyyaml" }, @@ -4291,6 +4297,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/7d/73ef5052f57b7720cad00e16598db3592a5ef4826745ffca67a2f085d4dc/pytorch_metric_learning-2.9.0-py3-none-any.whl", hash = "sha256:d51646006dc87168f00cf954785db133a4c5aac81253877248737aa42ef6432a", size = 127801, upload-time = "2025-08-17T17:11:18.185Z" }, ] +[[package]] +name = "pywebrtc-audio" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/b0/8d9e93083ac412cc758003d5e216d44f69b50619cdd0cfc7897e576eb90c/pywebrtc_audio-0.2.0.tar.gz", hash = "sha256:cdc47c063439b033bdd6110d2e1119fd6e36c6e007ed824bc369c9a5e3cb307a", size = 1748548, upload-time = "2026-09-03T19:03:15.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/74/a6e3a35e39199039ae9cc88e7728da1f5329b820c092ce51ced49fb842d2/pywebrtc_audio-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2b3552930655219101c59c1295cdc29f2ab38b389c3c495d9f41773183880615", size = 403707, upload-time = "2026-09-03T19:02:44.771Z" }, + { url = "https://files.pythonhosted.org/packages/33/01/6b48bbf13c765420aadf96667b433207e705179f67bc0dee6b8f87f93022/pywebrtc_audio-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46134bf1a6d8d1fcdf90674e2787ec93435533a0f290f67b530040f8eabdb793", size = 367960, upload-time = "2026-09-03T19:02:46.215Z" }, + { url = "https://files.pythonhosted.org/packages/14/bc/17ddeb099bb7b5a4cf16cd2abaee97c27b57455c0711a2c7d58813cf4abd/pywebrtc_audio-0.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5ddbe2fb5533b5aff7315e5463dfabbeb4f9d89135f5a6076144de81334bf1b", size = 400609, upload-time = "2026-09-03T19:02:47.88Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f0/7cde753b5bebf1cabff32289370caae8b2e1ae300477dfd00d05ea1aa84d/pywebrtc_audio-0.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0cbe0afe02c7fa3b5bb7c47a64a145664a90141c504abac64bd55163b456c60", size = 442400, upload-time = "2026-09-03T19:02:49.319Z" }, + { url = "https://files.pythonhosted.org/packages/d1/17/498662ef1dd4fe9b649d1507c95e65b41186a083f0cc1ca52c96dbdd12f5/pywebrtc_audio-0.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e295bb687cce04552d2585a01346d219a2eab545a0aa63458116633359f0f3e0", size = 1395946, upload-time = "2026-09-03T19:02:51.054Z" }, + { url = "https://files.pythonhosted.org/packages/ed/0d/40da061f539e635beec444a7aa83ce8705861e42238a29a697ad4a2fcc50/pywebrtc_audio-0.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66c5be23e71be322872378dce4df89aad13522ec968bf6d6fa7386c2699d88a6", size = 1484743, upload-time = "2026-09-03T19:02:52.39Z" }, + { url = "https://files.pythonhosted.org/packages/33/3e/1b9dee1fa750e744a128289c9d8fa991fffc8a2ff89b3285b238b275eb3c/pywebrtc_audio-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:0dabbdadd5d7fd7dcb88323266e5e14d46aaa136c58fa0573c4b0323b683c80b", size = 513391, upload-time = "2026-09-03T19:02:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/1f/cb/c3a0197971d132d8115a7ee88017183719e526ae302670acc1c3c2b88359/pywebrtc_audio-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9edfe0dc08ed3e5f856613786235e3ca5ac74f6dc14a6937da67244d19d1091", size = 403723, upload-time = "2026-09-03T19:02:55.641Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cd/ca9a2b1782e24ed009f21cc996eb1b77fc480d34f9958c29e3bc2b209091/pywebrtc_audio-0.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:22b31c256be3b8c7401bbeed5c483ba0f326b15a3e1640dc791ba40c49bc7e0d", size = 367981, upload-time = "2026-09-03T19:02:57.101Z" }, + { url = "https://files.pythonhosted.org/packages/3f/72/78715f87dff9b2e3868d565f106e2209e0cdd57b070ee9593cc5be033e05/pywebrtc_audio-0.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c56938aa82346511a749d8c95fa522061b1d4da26878a5bfd38dc8fea4352d", size = 400529, upload-time = "2026-09-03T19:02:58.341Z" }, + { url = "https://files.pythonhosted.org/packages/4b/43/5df4cd966ce6c3e5fd2c556cbcbde4d91404433e27d1a470c41b46aed51e/pywebrtc_audio-0.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b02aa10312378a9c8bec5d6f25586b3464c29dd1cc6961716a7f18f4350828e", size = 442498, upload-time = "2026-09-03T19:02:59.712Z" }, + { url = "https://files.pythonhosted.org/packages/07/61/957d2acdd8f020c8beba65747b602501e6e679f1abf1109f24a52d5a3cc6/pywebrtc_audio-0.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:36d22c1486b836fad39b249943b522dd3f94d475606a00b663a929d3aaf8ad39", size = 1395943, upload-time = "2026-09-03T19:03:01.208Z" }, + { url = "https://files.pythonhosted.org/packages/64/05/eae32f43de19bc271f5192859537611487e9cdd2c25ab3f295628443d150/pywebrtc_audio-0.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:99098022711832bb2bf82ccfc524ad4ce84062d386742232f1c1839f235a3acf", size = 1484804, upload-time = "2026-09-03T19:03:02.754Z" }, + { url = "https://files.pythonhosted.org/packages/89/1e/ff2d90d0450bb2c117f314c946b4a780ff80e65d98d7e760bb4a8d627b22/pywebrtc_audio-0.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:346f57a3e06e164fa9e8298c7ecb91d19333618e29a0ff6d972ac4c46a970a9d", size = 513446, upload-time = "2026-09-03T19:03:04.437Z" }, +] + [[package]] name = "pywin32" version = "312" diff --git a/nextjs-frontend/__tests__/recording-jobs.test.ts b/nextjs-frontend/__tests__/recording-jobs.test.ts index 5c5bc9cf0..371e91cd7 100644 --- a/nextjs-frontend/__tests__/recording-jobs.test.ts +++ b/nextjs-frontend/__tests__/recording-jobs.test.ts @@ -274,6 +274,7 @@ describe("reprocess picker (R9)", () => { const transcript = reprocessOptions({ video: false, hasAudio: false }); it("offers the backend's steps; video steps only for videos", () => { expect(audio.map((o) => o.key)).toEqual([ + "enhance", "transcribe", "diarize", "clean", @@ -283,6 +284,7 @@ describe("reprocess picker (R9)", () => { "report", ]); expect(reprocessOptions({ video: true, hasAudio: true }).map((o) => o.key)).toEqual([ + "enhance", "transcribe", "diarize", "shots", @@ -312,8 +314,8 @@ describe("reprocess picker (R9)", () => { "report", ]); }); - it("disables Transcribe and Diarize for transcript-only recordings", () => { - expect(transcript.filter((o) => o.disabled).map((o) => o.key)).toEqual(["transcribe", "diarize"]); + it("disables Clean up audio, Transcribe and Diarize for transcript-only recordings", () => { + expect(transcript.filter((o) => o.disabled).map((o) => o.key)).toEqual(["enhance", "transcribe", "diarize"]); }); it("ticks dependents; unticking removes just that step", () => { let s = toggleStep(new Set(), "diarize", true, audio); @@ -331,6 +333,11 @@ describe("reprocess picker (R9)", () => { "summarize", "report", ]); + // cleaning up the sound only matters to what's transcribed after it + expect(orderedSteps(toggleStep(new Set(), "enhance", true, audio)).slice(0, 2)).toEqual([ + "enhance", + "transcribe", + ]); }); }); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 123eda165..c15c2c2e0 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -185,6 +185,7 @@ export type { AssistantMemoryUpdate, AttachedTo, Attachment, + AudioCleanup, AuditEntry, AuthStatus, Automation, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 51de46f24..ba4c46230 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -1376,6 +1376,93 @@ export type Attachment = { [key: string]: unknown; }; +/** + * AudioCleanup + * + * How the enhance step cleaned up a recording's sound for transcription (docs/audio-cleanup.md). + */ +export type AudioCleanup = { + /** + * Noise + * + * noise suppression: off, low, moderate, high or very_high (6, 12, 18 or 21 dB) + */ + noise: string; + /** + * Gain + * + * automatic gain control + */ + gain: boolean; + /** + * High Pass + * + * the rumble below speech taken out + */ + high_pass: boolean; + /** + * Echo + * + * echo cancellation: off, or the channel (left or right) holding what the speakers played + */ + echo: string; + /** + * Trim Silence + * + * silences of min_silence_ms or longer left out of what's transcribed + */ + trim_silence: boolean; + /** + * Min Silence Ms + * + * the shortest pause left out when trimming silence + */ + min_silence_ms: number; + /** + * Duration Ms + * + * how long the recording is + */ + duration_ms: number; + /** + * Speech Ms + * + * how much of it WebRTC's voice detection heard as speech + */ + speech_ms: number; + /** + * Kept Ms + * + * how much of it is transcribed: all of it unless silence is trimmed + */ + kept_ms: number; + /** + * Gain Db + * + * the gain applied by the end of the recording, in dB + */ + gain_db?: number | null; + /** + * Echo Note + * + * why echo wasn't cancelled though it was asked for + */ + echo_note?: string | null; + /** + * Engine + * + * what cleaned it up, with its version + */ + engine?: string | null; + /** + * At + * + * when the step ran + */ + at?: string | null; + [key: string]: unknown; +}; + /** * AuditEntry */ @@ -17583,6 +17670,10 @@ export type Recording = { * NSFW classification, where it's on in its namespace and it was classified */ nsfw?: NsfwState | null; + /** + * how its enhance step cleaned up its sound before transcription + */ + audio_cleanup?: AudioCleanup | null; [key: string]: unknown; }; diff --git a/nextjs-frontend/components/batches/run-on.tsx b/nextjs-frontend/components/batches/run-on.tsx index 4ed3e19f6..549044ad9 100644 --- a/nextjs-frontend/components/batches/run-on.tsx +++ b/nextjs-frontend/components/batches/run-on.tsx @@ -33,6 +33,7 @@ import { cn } from "@/lib/utils"; type Kind = "template" | "pipeline" | "steps"; const STEP_ORDER = [ + "enhance", "transcribe", "diarize", "shots", diff --git a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx index a7592ea30..c82946e50 100644 --- a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx @@ -3,6 +3,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { AudioLines, + AudioWaveform, Captions, Clapperboard, Eraser, @@ -54,6 +55,7 @@ import { useArchive } from "@/lib/hooks/session"; import { cn } from "@/lib/utils"; const ICON: Record = { + enhance: AudioWaveform, transcribe: Captions, read: ScanText, diarize: AudioLines, @@ -74,7 +76,7 @@ const ICON: Record = { workflow: Workflow, }; const TONE = { intent: "blue", red: "red", green: "green", gate: "gold", neutral: "neutral" } as const; -const ASSETS = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"]; +const ASSETS = ["enhance", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"]; const METADATA = ["clean", "analyze", "embed", "summarize", "llm", "spec", "report", "export"]; /** A pipeline drawn as a graph: asset steps, metadata steps and workflows, an edge saying what runs after what. */ diff --git a/nextjs-frontend/components/pipelines/pipeline-editor.tsx b/nextjs-frontend/components/pipelines/pipeline-editor.tsx index 4c0eca48a..ef18aa2e4 100644 --- a/nextjs-frontend/components/pipelines/pipeline-editor.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-editor.tsx @@ -3,6 +3,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { AudioLines, + AudioWaveform, Captions, ChevronDown, Clapperboard, @@ -61,6 +62,7 @@ import { useArchive } from "@/lib/hooks/session"; import { cn } from "@/lib/utils"; const ICON: Record = { + enhance: AudioWaveform, transcribe: Captions, read: ScanText, diarize: AudioLines, @@ -80,6 +82,7 @@ const ICON: Record = { workflow: Workflow, }; const LIB_ORDER = [ + "enhance", "transcribe", "diarize", "shots", diff --git a/nextjs-frontend/components/pipelines/pipeline-model.ts b/nextjs-frontend/components/pipelines/pipeline-model.ts index 31bdedc85..5176830c7 100644 --- a/nextjs-frontend/components/pipelines/pipeline-model.ts +++ b/nextjs-frontend/components/pipelines/pipeline-model.ts @@ -70,6 +70,7 @@ export function cleanSpec(s: StepSpec): StepSpec | string { /** Which earlier steps a step reads from, when they are in the same pipeline. */ export const NEEDS: Record = { + transcribe: ["enhance"], diarize: ["transcribe"], ocr: ["shots"], faces: ["shots"], @@ -89,6 +90,7 @@ export const NEEDS: Record = { /** What a step makes available to later steps and templates. */ export const PROVIDES: Record = { + enhance: "audio_cleanup (speech · silence left out) for transcribe", transcribe: "transcript · segments[] · language", read: "pages[] · segments[]", diarize: "speakers[] · turns", @@ -110,6 +112,8 @@ export const PROVIDES: Record = { }; export const DESCRIBE: Record = { + enhance: + "Cleans up the sound before it’s transcribed with WebRTC: noise, gain, rumble, echo, and silence left out (Settings → Transcription)", transcribe: "Speech to text with word timings (Settings → Transcription)", read: "Draws a document’s or an image’s pages and reads their text (Settings → Documents)", diarize: "Who spoke when (Settings → Diarization)", diff --git a/nextjs-frontend/components/recording/jobs.ts b/nextjs-frontend/components/recording/jobs.ts index e4053dd90..1c3f5ed29 100644 --- a/nextjs-frontend/components/recording/jobs.ts +++ b/nextjs-frontend/components/recording/jobs.ts @@ -287,6 +287,7 @@ export function readyBefore(j: JobInfo): string[] { // ---------- Reprocess: choose steps ---------- export const STEP_ORDER = [ + "enhance", "transcribe", "diarize", "shots", @@ -304,6 +305,7 @@ export const STEP_ORDER = [ export type StepKey = (typeof STEP_ORDER)[number]; export const STEP_HELP: Record = { + enhance: "Clean up the sound for transcription: noise, gain, echo and silence (Settings → Transcription)", transcribe: "Audio → text with timings", diarize: "Split speakers and match voice IDs", shots: "Scene cuts, keyframes and sampled frames", @@ -322,6 +324,7 @@ export const STEP_HELP: Record = { /** Ticking a step also ticks the later steps that use its output. */ export const DEPENDENTS: Record = { + enhance: ["transcribe", "diarize", "clean", "analyze", "embed", "summarize", "report"], transcribe: ["diarize", "clean", "analyze", "embed", "summarize", "report"], diarize: ["analyze", "summarize", "report"], shots: ["ocr", "faces", "objects", "nsfw", "describe"], @@ -341,17 +344,20 @@ export type StepOption = { key: StepKey; disabled: string | null }; /** * The steps offered for a recording: video steps only for videos (faces, objects, nsfw and describe also for a - * document's or an image's pages); transcribe/diarize need audio; clean is for speech, so not for pages. + * document's or an image's pages); enhance/transcribe/diarize need audio; enhance and clean are for speech, so not for + * pages. */ export function reprocessOptions(opts: { video: boolean; hasAudio: boolean; paged?: boolean }): StepOption[] { const left = opts.video ? [] : opts.paged - ? ["shots", "ocr", "clean"] + ? ["enhance", "shots", "ocr", "clean"] : ["shots", "ocr", "faces", "objects", "nsfw", "describe"]; return STEP_ORDER.filter((k) => !left.includes(k)).map((key) => { let disabled: string | null = null; - if (!opts.hasAudio && key === "transcribe") + if (!opts.hasAudio && key === "enhance") + disabled = "This asset has no audio: its text was imported, so there's no sound to clean up."; + else if (!opts.hasAudio && key === "transcribe") disabled = "This asset has no audio: its text was imported, so there's nothing to transcribe."; else if (!opts.hasAudio && key === "diarize") disabled = "Speakers come from the imported text's labels; diarizing needs the audio."; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 72cb3aee2..ca671b61e 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -145,6 +145,31 @@ const ENGINE_KEY: Record = { "mlx-whisper": ["mlx_whisper", "mlx-whisper"], }; +/** What the Clean up audio pipeline step does before transcription (audio_cleanup.py); it runs where a pipeline has it. */ +function AudioCleanup({ ctx }: { ctx: BodyCtx }) { + return ( +
+
+ Audio cleanup +

+ What the Clean up audio step does to the sound before it’s transcribed, with WebRTC. It runs in pipelines that + have it (Pipelines); the recording itself is never changed. +

+
+
+ + + + + + {ctx.form["transcribe.cleanup.trim_silence"] !== false && ( + + )} +
+
+ ); +} + export function SectionBody({ ctx }: { ctx: BodyCtx }) { const v = (id: string) => ctx.values[id]; const raw = (id: string) => ctx.form[id]; @@ -164,6 +189,7 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { in Speech providers. SenseVoice and Whisper keep their models for when you switch back.

+ ); const [key, name] = ENGINE_KEY[engine] ?? ENGINE_KEY.sensevoice; @@ -184,6 +210,7 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) {

Each engine remembers its own model: {models.join(", ")}. Auto uses the fastest device each worker has.

+ ); } diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 16b39a852..7ae66efa9 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -627,6 +627,60 @@ export const FIELDS: FieldSpec[] = [ kind: "text", mono: true, }, + // Audio cleanup: what the Clean up audio step (enhance) does before transcription, in a pipeline that has it + { + section: "transcribe", + key: "cleanup.noise", + label: "Noise suppression", + kind: "select", + options: [ + { value: "off", label: "Off" }, + { value: "low", label: "Low (6 dB)" }, + { value: "moderate", label: "Moderate (12 dB)" }, + { value: "high", label: "High (18 dB)" }, + { value: "very_high", label: "Very high (21 dB)" }, + ], + }, + { + section: "transcribe", + key: "cleanup.echo", + label: "Echo cancellation", + kind: "select", + options: [ + { value: "off", label: "Off" }, + { value: "left", label: "Speakers on the left channel" }, + { value: "right", label: "Speakers on the right channel" }, + ], + hint: "For call recordings with what the speakers played on one channel and the microphone on the other", + }, + { + section: "transcribe", + key: "cleanup.gain", + label: "Even out the volume", + kind: "switch", + }, + { + section: "transcribe", + key: "cleanup.high_pass", + label: "Take out low rumble", + kind: "switch", + }, + { + section: "transcribe", + key: "cleanup.trim_silence", + label: "Leave silence out of what’s transcribed", + kind: "switch", + hint: "Fewer minutes for a cloud engine to bill, and faster on this server; times stay the recording’s own", + }, + { + section: "transcribe", + key: "cleanup.min_silence_ms", + label: "Shortest silence left out", + kind: "int", + min: 500, + max: 60000, + unit: "ms", + }, // Speaker separation { section: "diarize", diff --git a/nextjs-frontend/components/ui/loop.tsx b/nextjs-frontend/components/ui/loop.tsx index 57a6db46b..c202d087d 100644 --- a/nextjs-frontend/components/ui/loop.tsx +++ b/nextjs-frontend/components/ui/loop.tsx @@ -9,6 +9,7 @@ import { cn } from "@/lib/utils"; * upcoming ones are pale tints. */ export const STEP_TONE: Record = { + enhance: "intent", transcribe: "intent", read: "intent", shots: "intent", @@ -30,6 +31,7 @@ export const STEP_TONE: Record = { }; export const STEP_LABEL: Record = { + enhance: "Clean up audio", transcribe: "Transcribe", read: "Read pages", diarize: "Diarize", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index dcfb349c5..d07f9306a 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -34809,6 +34809,118 @@ ], "title": "Attachment" }, + "AudioCleanup": { + "properties": { + "noise": { + "type": "string", + "title": "Noise", + "description": "noise suppression: off, low, moderate, high or very_high (6, 12, 18 or 21 dB)" + }, + "gain": { + "type": "boolean", + "title": "Gain", + "description": "automatic gain control" + }, + "high_pass": { + "type": "boolean", + "title": "High Pass", + "description": "the rumble below speech taken out" + }, + "echo": { + "type": "string", + "title": "Echo", + "description": "echo cancellation: off, or the channel (left or right) holding what the speakers played" + }, + "trim_silence": { + "type": "boolean", + "title": "Trim Silence", + "description": "silences of min_silence_ms or longer left out of what's transcribed" + }, + "min_silence_ms": { + "type": "integer", + "title": "Min Silence Ms", + "description": "the shortest pause left out when trimming silence" + }, + "duration_ms": { + "type": "integer", + "title": "Duration Ms", + "description": "how long the recording is" + }, + "speech_ms": { + "type": "integer", + "title": "Speech Ms", + "description": "how much of it WebRTC's voice detection heard as speech" + }, + "kept_ms": { + "type": "integer", + "title": "Kept Ms", + "description": "how much of it is transcribed: all of it unless silence is trimmed" + }, + "gain_db": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Gain Db", + "description": "the gain applied by the end of the recording, in dB" + }, + "echo_note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Echo Note", + "description": "why echo wasn't cancelled though it was asked for" + }, + "engine": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Engine", + "description": "what cleaned it up, with its version" + }, + "at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "At", + "description": "when the step ran" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "noise", + "gain", + "high_pass", + "echo", + "trim_silence", + "min_silence_ms", + "duration_ms", + "speech_ms", + "kept_ms" + ], + "title": "AudioCleanup", + "description": "How the enhance step cleaned up a recording's sound for transcription (docs/audio-cleanup.md)." + }, "AuditEntry": { "properties": { "at": { @@ -62896,6 +63008,17 @@ } ], "description": "NSFW classification, where it's on in its namespace and it was classified" + }, + "audio_cleanup": { + "anyOf": [ + { + "$ref": "#/components/schemas/AudioCleanup" + }, + { + "type": "null" + } + ], + "description": "how its enhance step cleaned up its sound before transcription" } }, "additionalProperties": true, From c9df4180653e908d309c69237da4d7f8ad20a0bd Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:44:29 +0000 Subject: [PATCH 2/3] refactor(pipelines): name the step clean_audio, apart from the image enhance step in #479 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014WcBxU5ARBoLWMYZU46St6 --- CHANGELOG.md | 2 +- docs/components.md | 2 +- docs/processing.md | 8 ++-- fastapi_backend/app/domain/audio_cleanup.py | 8 ++-- fastapi_backend/app/domain/components.py | 6 +-- fastapi_backend/app/domain/ingest.py | 2 +- fastapi_backend/app/domain/jobs.py | 12 +++--- fastapi_backend/app/domain/pipelines.py | 4 +- fastapi_backend/app/domain/store.py | 4 +- fastapi_backend/app/schemas/recordings.py | 4 +- fastapi_backend/pyproject.toml | 4 +- .../tests/domain/test_audio_cleanup.py | 39 ++++++++++--------- .../__tests__/recording-jobs.test.ts | 10 ++--- .../app/openapi-client/types.gen.ts | 4 +- nextjs-frontend/components/batches/run-on.tsx | 2 +- .../components/pipelines/pipeline-canvas.tsx | 4 +- .../components/pipelines/pipeline-editor.tsx | 4 +- .../components/pipelines/pipeline-model.ts | 6 +-- nextjs-frontend/components/recording/jobs.ts | 12 +++--- nextjs-frontend/components/settings/model.ts | 2 +- nextjs-frontend/components/ui/loop.tsx | 4 +- nextjs-frontend/openapi.json | 4 +- 22 files changed, 75 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37f52edaf..8c8f38d0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased -- **Clean up a recording's sound before it's transcribed.** A new pipeline step, Clean up audio (`enhance`), runs a +- **Clean up a recording's sound before it's transcribed.** A new pipeline step, Clean up audio (`clean_audio`), runs a recording through WebRTC's audio processing (pywebrtc-audio, Apache-2.0, about 0.5 MB, fetched the first time it runs): noise suppression, automatic gain, a high-pass filter, echo cancellation for two-channel call recordings, and pauses left out of what's transcribed, with the transcript's times put back on the recording's timeline. Settings → diff --git a/docs/components.md b/docs/components.md index 07cd2d43d..736110b2a 100644 --- a/docs/components.md +++ b/docs/components.md @@ -25,7 +25,7 @@ never fetch a speech or voice model. | Prompt-injection classifier | `ai.injection_check` is `classifier` | ProtectAI's deberta-v3-base-prompt-injection-v2 (Apache-2.0, about 740 MB), the tokenizers library and ONNX Runtime ([third-party content](assistant.md#third-party-content-and-prompt-injection)) | | Punctuation model | `analysis.cleanup` is on and `analysis.cleanup_punctuation` isn't `off` ([Cleaning up transcripts](processing.md#cleaning-up-transcripts)) | kredor/punctuate-all at a pinned revision (MIT, about 1.1 GB), with Transformers and PyTorch's CPU build | | spaCy | `analysis.entities` is `spacy` ([Entities](processing.md#entities-and-the-graph-explorer)) | spaCy (MIT) and the model in `analysis.spacy_model`: `en_core_web_sm`, `de_core_news_sm` or `xx_ent_wiki_sm` (MIT, 11 to 15 MB each), about 60 MB in all | -| WebRTC audio processing | a job first reaches a pipeline's `enhance` step ([Cleaning up audio](processing.md#cleaning-up-audio)) | pywebrtc-audio (Apache-2.0, about 0.5 MB, numpy only): WebRTC's noise suppression, gain control, echo cancellation and voice detection | +| WebRTC audio processing | a job first reaches a pipeline's `clean_audio` step ([Cleaning up audio](processing.md#cleaning-up-audio)) | pywebrtc-audio (Apache-2.0, about 0.5 MB, numpy only): WebRTC's noise suppression, gain control, echo cancellation and voice detection | | DuckDuckGo search | `web.search` is `duckduckgo` | ddgs (MIT, about 30 MB with lxml and primp), the client the assistant's [web search](assistant.md#web-search-and-reading-pages) reads DuckDuckGo's results with | | yt-dlp | `documents.video_urls` is on | the yt-dlp program, release 2026.08.19 (Unlicense), to download videos and podcasts imported by address | | Chat model, embedding model | the LLM provider or embeddings server is Ollama | pulled on that server (`/api/pull`) | diff --git a/docs/processing.md b/docs/processing.md index ee230b9ca..73bda01df 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -7,7 +7,7 @@ How assets move through Lens: where they come from, what each step does, and how - `scan` finds audio under each namespace's paths and fingerprints it: moved files keep their history, duplicates are skipped, and so are files whose asset someone deleted or moved to another namespace (at the same path, or a copy of the same file). -- `enhance` (only in a pipeline that has it) cleans up the sound before it's transcribed: noise, volume, rumble, +- `clean_audio` (only in a pipeline that has it) cleans up the sound before it's transcribed: noise, volume, rumble, echo, and silence left out of what's transcribed ([Cleaning up audio](#cleaning-up-audio)). - `transcribe` uses SenseVoice, faster-whisper or mlx-whisper. When the configured engine isn't installed on a worker (or doesn't import there), it uses the next one that is and says so in the job log; with none at all the step fails, @@ -28,7 +28,7 @@ Every step takes `--ns`, `--limit` and `--force`; `run` does them all, and a loc ### Cleaning up audio A phone on a table, a fan, a quiet voice across the room or a call with the other side coming back through the -speakers: speech-to-text hears all of it. The `enhance` step (Clean up audio in the pipeline editor and Reprocess) +speakers: speech-to-text hears all of it. The `clean_audio` step (Clean up audio in the pipeline editor and Reprocess) runs a recording through WebRTC's audio processing, the code a browser runs on a call, with [pywebrtc-audio](https://github.com/strands-labs/pywebrtc-audio) (Apache-2.0, about 0.5 MB; fetched the first time the step runs, [Components](components.md)). No pipeline has it until you add it before Transcribe, so nothing changes @@ -53,7 +53,7 @@ stored: cleaning takes a fraction of a second per minute of audio, so it's done voice IDs still listen to the original. Run the step with every option off to have the asset transcribed as it is again. A worker that can't fetch pywebrtc-audio transcribes the original and says so in the job log. Run it on one asset with Reprocess → Clean up audio, or `POST /api/v1/recordings/{id}/reprocess` with -`{"steps": ["enhance", "transcribe"]}`; workers whose `workers.steps` list has `transcribe` run it too. +`{"steps": ["clean_audio", "transcribe"]}`; workers whose `workers.steps` list has `transcribe` run it too. ### Cleaning up transcripts @@ -695,7 +695,7 @@ A pipeline is a named, versioned list of steps; each namespace can choose its de analyze, summarize, report). Imports, watched folders and reprocessing use the namespace's pipeline; steps that need audio skip themselves for imported text. Steps: -- `enhance` ([Cleaning up audio](#cleaning-up-audio)), `transcribe`, `diarize`, `clean` +- `clean_audio` ([Cleaning up audio](#cleaning-up-audio)), `transcribe`, `diarize`, `clean` ([Cleaning up transcripts](#cleaning-up-transcripts)), `analyze`, `summarize`, and `report` (the built-in report). - `llm`: renders a prompt template, sends it to the configured model, checks the reply against the template's JSON Schema and saves it as a named output (e.g. `meeting_notes`). diff --git a/fastapi_backend/app/domain/audio_cleanup.py b/fastapi_backend/app/domain/audio_cleanup.py index d68e4e336..023fc1f5e 100644 --- a/fastapi_backend/app/domain/audio_cleanup.py +++ b/fastapi_backend/app/domain/audio_cleanup.py @@ -1,7 +1,7 @@ -"""Cleaning up a recording's sound before it's transcribed: the enhance step (Settings → Transcription → Audio cleanup). +"""Cleaning up a recording's sound before it's transcribed: the clean_audio step (Settings → Transcription → Audio cleanup). A phone on a table, a fan, a quiet speaker across the room, a call recorded with the other side's voice coming back -through the speakers: speech-to-text hears all of it. The enhance step runs the recording through WebRTC's audio +through the speakers: speech-to-text hears all of it. The clean_audio step runs the recording through WebRTC's audio processing, the same code a browser runs on a call, through pywebrtc-audio (Apache-2.0, about 0.5 MB, numpy only, fetched the first time the step runs). Every option is in the settings: @@ -236,7 +236,7 @@ def clean(o, path, mono, channels=1): def hear(cfg, rec, path, mono, log=print): - """What transcribing a recording hears: its audio cleaned up as its enhance step set, silence left out when it + """What transcribing a recording hears: its audio cleaned up as its clean_audio step set, silence left out when it asked. Returns (samples, joins or None); the original when this worker can't run the cleanup.""" o = {**DEFAULTS, **{k: v for k, v in (rec.get("audio_cleanup") or {}).items() if k in KEYS}} try: @@ -259,7 +259,7 @@ def hear(cfg, rec, path, mono, log=print): def step(db, cfg, rid, say): - """The enhance step: measure the recording through the cleanup and keep its options on it, or clear them when + """The clean_audio step: measure the recording through the cleanup and keep its options on it, or clear them when every option is off. Returns what it kept, or None.""" o = options(cfg) r = R("recording", rid) diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index 0d680c9df..70a20b095 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -736,7 +736,7 @@ def fetch(self, cfg, m, say): class WebrtcAudio(Component): - """pywebrtc-audio, for the enhance step (audio_cleanup.py): fetched the first time a job waits for that step.""" + """pywebrtc-audio, for the clean_audio step (audio_cleanup.py): fetched the first time a job waits for that step.""" def needed(self, cfg, m): return True # it waits for first use: an archive whose pipelines never clean up audio fetches nothing @@ -995,8 +995,8 @@ def detail(self, cfg): WebrtcAudio( "webrtc-audio", "WebRTC audio processing", - "cleans up a recording's sound before it's transcribed: noise, gain, echo and silence (the enhance step; pywebrtc-audio)", - steps={"enhance"}, + "cleans up a recording's sound before it's transcribed: noise, gain, echo and silence (the clean_audio step; pywebrtc-audio)", + steps={"clean_audio"}, size_mb=1, license="Apache-2.0", ), diff --git a/fastapi_backend/app/domain/ingest.py b/fastapi_backend/app/domain/ingest.py index 7666ca0f9..7fa223d6d 100644 --- a/fastapi_backend/app/domain/ingest.py +++ b/fastapi_backend/app/domain/ingest.py @@ -503,7 +503,7 @@ def transcribe_one(db, cfg, rid, log=print, engine=None): path = audio_path(db, cfg, r) audio, joins = decode(path), None env = envelope(audio) # the player's waveform is the recording's own - if r.get("audio_cleanup"): # its enhance step asked for it to be heard cleaned up (audio_cleanup.py) + if r.get("audio_cleanup"): # its clean_audio step asked for it to be heard cleaned up (audio_cleanup.py) from . import audio_cleanup audio, joins = audio_cleanup.hear(cfg, r, path, audio, log) diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index 194f64e93..13f910f96 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -26,7 +26,7 @@ "record::id(id) AS id, recording, space, batch, pipeline, steps, step_index, next_step, status, worker, error, attempts, " "created_by, created_at, started_at, finished_at, updated_at, cancel_requested, log_total, cost_usd, tokens, cost_estimate, hold" ) -MEDIA_STEPS = {"enhance", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"} # slower on longer media +MEDIA_STEPS = {"clean_audio", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"} # slower on longer media FILED = ("audio", "document", "image") # sources with a file of their own for the steps to work on TIMINGS = 25 # recent timings kept per kind of step, for estimates @@ -71,7 +71,7 @@ def _transcribe(db, cfg, rid, say, spec=None): ingest.transcribe_one(db, cfg, rid, say) -def _enhance(db, cfg, rid, say, spec=None): +def _clean_audio(db, cfg, rid, say, spec=None): """A recording's sound cleaned up before it's transcribed (audio_cleanup.py), as Settings → Transcription → Audio cleanup sets it.""" from . import audio_cleanup @@ -220,7 +220,7 @@ def _describe(db, cfg, rid, say, spec=None): VIDEO_STEPS = {"shots", "ocr", "faces", "objects", "describe"} # added after workers.steps lists were first written STEPS = { - "enhance": _enhance, + "clean_audio": _clean_audio, "transcribe": _transcribe, "read": _read, "diarize": _diarize, @@ -249,14 +249,14 @@ def for_source(steps, source): nothing to transcribe and its speakers came with it, so it skips both steps too (unless they're all that was asked for: those still run, to say why there's nothing to do).""" if source == "transcript": - return [s for s in steps if s.get("type") not in ("enhance", "transcribe", "diarize")] or steps + return [s for s in steps if s.get("type") not in ("clean_audio", "transcribe", "diarize")] or steps if source not in ("document", "image"): return steps out = [] for s in steps: if s.get("type") == "transcribe": s = {**s, "type": "read"} - if s.get("type") in ("enhance", "diarize") or (s.get("type") == "read" and any(o.get("type") == "read" for o in out)): + if s.get("type") in ("clean_audio", "diarize") or (s.get("type") == "read" and any(o.get("type") == "read" for o in out)): continue out.append(s) return out @@ -954,7 +954,7 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None): if self.can & {"analyze", "summarize"}: # embedding needs what they need; lists written before it existed run it too self.can.add("embed") if not steps and "transcribe" in self.can: # cleaning up the sound needs no more than transcribing; older lists run it - self.can.add("enhance") + self.can.add("clean_audio") if "analyze" in self.can: # cleaning up a transcript needs no more than analysing it; older lists run it too self.can.add("clean") if not steps and "objects" in self.can: # NSFW classification runs on ONNX Runtime too; older lists run it as well diff --git a/fastapi_backend/app/domain/pipelines.py b/fastapi_backend/app/domain/pipelines.py index 04f958f48..6d2ee3e69 100644 --- a/fastapi_backend/app/domain/pipelines.py +++ b/fastapi_backend/app/domain/pipelines.py @@ -1,5 +1,5 @@ """Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without -one, recordings get the standard pipeline. Steps: enhance (a recording's sound cleaned up before it's transcribed, +one, recordings get the standard pipeline. Steps: clean_audio (a recording's sound cleaned up before it's transcribed, audio_cleanup.py), transcribe (speech to text), read (a document's or an image's pages and their text), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), describe, clean (a transcript tidied before analysis, cleanup.py), analyze, embed (passages for search by meaning, semantic.py), @@ -29,7 +29,7 @@ "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", "report", ] # fmt: skip TYPES = { - "enhance", "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", + "clean_audio", "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", "llm", "spec", "report", diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 18f03a6dc..06293cdb5 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -137,7 +137,7 @@ def labels(): "sensevoice": {"model": "iic/SenseVoiceSmall", "hub": "ms", "vad_max_segment_ms": 30000, "batch_size": 16}, "whisper": {"model": "large-v3-turbo", "compute_type": "default"}, "mlx_whisper": {"model": "mlx-community/whisper-large-v3-turbo"}, - # what the enhance step does to a recording's sound before it's transcribed (audio_cleanup.py); a pipeline only + # what the clean_audio step does to a recording's sound before it's transcribed (audio_cleanup.py); a pipeline only # runs it when the step is in it "cleanup": { "noise": "moderate", @@ -284,7 +284,7 @@ def labels(): "stale_minutes": 15, "max_attempts": 3, "steps": [ - "enhance", + "clean_audio", "transcribe", "read", "diarize", diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index bcc211951..e0ddde776 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -56,7 +56,7 @@ class NsfwHit(ResponseModel): class AudioCleanup(ResponseModel): - """How the enhance step cleaned up a recording's sound for transcription (docs/audio-cleanup.md).""" + """How the clean_audio step cleaned up a recording's sound for transcription (docs/processing.md, Cleaning up audio).""" noise: str = Field(description="noise suppression: off, low, moderate, high or very_high (6, 12, 18 or 21 dB)") gain: bool = Field(description="automatic gain control") @@ -303,7 +303,7 @@ class Recording(ResponseModel): web: WebPage | None = Field(None, description="a web page captured as a document") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") nsfw: NsfwState | None = Field(None, description="NSFW classification, where it's on in its namespace and it was classified") - audio_cleanup: AudioCleanup | None = Field(None, description="how its enhance step cleaned up its sound before transcription") + audio_cleanup: AudioCleanup | None = Field(None, description="how its clean_audio step cleaned up its sound before transcription") class Page(ResponseModel): diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index 8ebbc771d..f4b6ae83f 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -69,7 +69,7 @@ sandbox = ["openshell>=0.1.4,<0.2"] # the assistant's web search through DuckDuckGo (web.py): ddgs (MIT), with its Rust HTTP client (primp) and lxml; # fetched when DuckDuckGo is picked in Settings › AI assistant › Web search web = ["ddgs>=9.16,<10"] -# the enhance step (audio_cleanup.py): WebRTC's noise suppression, gain control, echo cancellation and voice +# the clean_audio step (audio_cleanup.py): WebRTC's noise suppression, gain control, echo cancellation and voice # detection through pywebrtc-audio (Apache-2.0; about 0.5 MB, numpy only); fetched the first time a pipeline runs it webrtc = ["pywebrtc-audio>=0.2.0,<0.3"] # Outlook .msg emails (extract-msg is GPL-3.0; .eml needs nothing) @@ -103,7 +103,7 @@ dev = [ "impacket>=0.13", # the SMB share's server, and the SMB client its tests sign in with "openshell>=0.1.4,<0.2", # the code sandbox's client, tested against a fake OpenShell gateway "botocore>=1.43.113", - "pywebrtc-audio>=0.2.0,<0.3", # the enhance step's WebRTC audio processing, tested on synthetic audio + "pywebrtc-audio>=0.2.0,<0.3", # the clean_audio step's WebRTC audio processing, tested on synthetic audio ] [tool.hatch.build.targets.wheel] diff --git a/fastapi_backend/tests/domain/test_audio_cleanup.py b/fastapi_backend/tests/domain/test_audio_cleanup.py index 2fcb4bf19..1ec19e0ac 100644 --- a/fastapi_backend/tests/domain/test_audio_cleanup.py +++ b/fastapi_backend/tests/domain/test_audio_cleanup.py @@ -1,4 +1,4 @@ -"""The enhance step: a recording's sound cleaned up with WebRTC's audio processing (pywebrtc-audio) before it's +"""The clean_audio step: a recording's sound cleaned up with WebRTC's audio processing (pywebrtc-audio) before it's transcribed, with silence left out of what's transcribed and the transcript put back on the recording's timeline.""" from __future__ import annotations @@ -94,7 +94,7 @@ def _segments(db, rid): def test_speech_is_found_and_silence_left_out_of_transcription(db, cfg, recording): rid = recording() - log = _run(db, cfg, rid, "enhance") + log = _run(db, cfg, rid, "clean_audio") saved = db.one("SELECT audio_cleanup FROM $r", r=R("recording", rid))["audio_cleanup"] assert saved["duration_ms"] == 22000 assert 6000 <= saved["speech_ms"] <= 8000 # 7 s were spoken @@ -117,10 +117,10 @@ def test_without_the_step_or_with_every_option_off_it_is_transcribed_as_it_is(db rid = recording() _run(db, cfg, rid) assert Engine.heard[-1] == 22 * SR - _run(db, cfg, rid, "enhance") + _run(db, cfg, rid, "clean_audio") assert Engine.heard[-1] < 22 * SR cfg["transcribe"]["cleanup"] = {"noise": "off", "gain": False, "high_pass": False, "echo": "off", "trim_silence": False} - log = _run(db, cfg, rid, "enhance") + log = _run(db, cfg, rid, "clean_audio") assert db.one("SELECT audio_cleanup FROM $r", r=R("recording", rid)).get("audio_cleanup") is None assert any("transcribed as it is" in line for line in log) assert Engine.heard[-1] == 22 * SR @@ -129,7 +129,7 @@ def test_without_the_step_or_with_every_option_off_it_is_transcribed_as_it_is(db def test_cleaned_but_not_trimmed(db, cfg, recording): cfg["transcribe"]["cleanup"] = {"trim_silence": False, "noise": "very_high"} rid = recording() - _run(db, cfg, rid, "enhance") + _run(db, cfg, rid, "clean_audio") assert Engine.heard[-1] == 22 * SR assert [s["t0"] for s in _segments(db, rid)] == [1000, 7000] @@ -150,31 +150,32 @@ def rms(x): # once it has settled def test_echo_needs_a_second_channel(db, cfg, recording, folder): cfg["transcribe"]["cleanup"] = {"echo": "right"} rid = recording() # one channel - log = _run(db, cfg, rid, "enhance") + log = _run(db, cfg, rid, "clean_audio") assert db.one("SELECT audio_cleanup FROM $r", r=R("recording", rid))["audio_cleanup"]["echo_note"] == "no second channel" assert any("echo not cancelled: no second channel" in line for line in log) mic, speakers = talk(), np.concatenate([quiet_for(16), babble(3, 9), quiet_for(3)]) rid = recording(mic, speakers) - log = _run(db, cfg, rid, "enhance") + log = _run(db, cfg, rid, "clean_audio") assert any("echo of the right channel cancelled" in line for line in log) assert Engine.heard[-1] < 22 * SR # both sides heard, the quiet left out def test_documents_and_imported_text_have_no_sound(db, cfg): rid = ingest.import_text(db, cfg, "pods", "Ann|N|Hello there.", "Note") - assert jobs.for_source([{"type": "enhance"}, {"type": "analyze"}], "transcript") == [{"type": "analyze"}] - assert jobs.for_source([{"type": "enhance"}, {"type": "transcribe"}], "document") == [{"type": "read"}] - jid = jobs.enqueue(db, rid, ["enhance"]) + assert jobs.for_source([{"type": "clean_audio"}, {"type": "analyze"}], "transcript") == [{"type": "analyze"}] + assert jobs.for_source([{"type": "clean_audio"}, {"type": "transcribe"}], "document") == [{"type": "read"}] + jid = jobs.enqueue(db, rid, ["clean_audio"]) drain(db, cfg) assert any( - "enhance skipped: imported text has no sound to clean up" in line for line in db.one("SELECT log FROM $j", j=R("job", jid))["log"] + "clean_audio skipped: imported text has no sound to clean up" in line + for line in db.one("SELECT log FROM $j", j=R("job", jid))["log"] ) def test_a_worker_without_the_library_transcribes_the_original(db, cfg, recording, monkeypatch): rid = recording() - _run(db, cfg, rid, "enhance") + _run(db, cfg, rid, "clean_audio") def missing(cfg, say=None): raise RuntimeError("pywebrtc-audio isn't installed and couldn't be fetched (no network)") @@ -216,26 +217,28 @@ def test_settings_are_checked(db, cfg): def test_pipelines_and_workers_take_the_step(db, cfg): - assert pipelines.validate_steps(db, ["enhance", "transcribe"]) == [{"type": "enhance"}, {"type": "transcribe"}] - assert "enhance" not in pipelines.STANDARD # nothing changes until a pipeline has it + assert pipelines.validate_steps(db, ["clean_audio", "transcribe"]) == [{"type": "clean_audio"}, {"type": "transcribe"}] + assert "clean_audio" not in pipelines.STANDARD # nothing changes until a pipeline has it cfg["workers"]["steps"] = ["transcribe", "diarize"] # a list written before the step existed - assert "enhance" in jobs.Worker(db, lambda: cfg, name="older").can + assert "clean_audio" in jobs.Worker(db, lambda: cfg, name="older").can from app.domain import components - assert components.BY_ID["webrtc-audio"].steps == {"enhance"} # fetched when a job first waits for it + assert components.BY_ID["webrtc-audio"].steps == {"clean_audio"} # fetched when a job first waits for it def test_an_agent_reads_what_the_step_did(client, db, cfg, recording): from tests.helpers import login, make_user rid = recording() - _run(db, cfg, rid, "enhance") + _run(db, cfg, rid, "clean_audio") make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) h = login(client, "ed@x.io", "editor password 1") got = client.get(f"/api/v1/recordings/{rid}", headers=h).json()["audio_cleanup"] assert got["kept_ms"] < got["duration_ms"] == 22000 and got["trim_silence"] is True pid = client.post( - "/api/v1/pipelines", headers=login(client, *_admin(db)), json={"name": "Clean then transcribe", "steps": ["enhance", "transcribe"]} + "/api/v1/pipelines", + headers=login(client, *_admin(db)), + json={"name": "Clean then transcribe", "steps": ["clean_audio", "transcribe"]}, ) assert pid.status_code in (200, 201), pid.text diff --git a/nextjs-frontend/__tests__/recording-jobs.test.ts b/nextjs-frontend/__tests__/recording-jobs.test.ts index 371e91cd7..167c3ef46 100644 --- a/nextjs-frontend/__tests__/recording-jobs.test.ts +++ b/nextjs-frontend/__tests__/recording-jobs.test.ts @@ -274,7 +274,7 @@ describe("reprocess picker (R9)", () => { const transcript = reprocessOptions({ video: false, hasAudio: false }); it("offers the backend's steps; video steps only for videos", () => { expect(audio.map((o) => o.key)).toEqual([ - "enhance", + "clean_audio", "transcribe", "diarize", "clean", @@ -284,7 +284,7 @@ describe("reprocess picker (R9)", () => { "report", ]); expect(reprocessOptions({ video: true, hasAudio: true }).map((o) => o.key)).toEqual([ - "enhance", + "clean_audio", "transcribe", "diarize", "shots", @@ -315,7 +315,7 @@ describe("reprocess picker (R9)", () => { ]); }); it("disables Clean up audio, Transcribe and Diarize for transcript-only recordings", () => { - expect(transcript.filter((o) => o.disabled).map((o) => o.key)).toEqual(["enhance", "transcribe", "diarize"]); + expect(transcript.filter((o) => o.disabled).map((o) => o.key)).toEqual(["clean_audio", "transcribe", "diarize"]); }); it("ticks dependents; unticking removes just that step", () => { let s = toggleStep(new Set(), "diarize", true, audio); @@ -334,8 +334,8 @@ describe("reprocess picker (R9)", () => { "report", ]); // cleaning up the sound only matters to what's transcribed after it - expect(orderedSteps(toggleStep(new Set(), "enhance", true, audio)).slice(0, 2)).toEqual([ - "enhance", + expect(orderedSteps(toggleStep(new Set(), "clean_audio", true, audio)).slice(0, 2)).toEqual([ + "clean_audio", "transcribe", ]); }); diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 80a9f0cab..b0d363550 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -1379,7 +1379,7 @@ export type Attachment = { /** * AudioCleanup * - * How the enhance step cleaned up a recording's sound for transcription (docs/audio-cleanup.md). + * How the clean_audio step cleaned up a recording's sound for transcription (docs/processing.md, Cleaning up audio). */ export type AudioCleanup = { /** @@ -18150,7 +18150,7 @@ export type Recording = { */ nsfw?: NsfwState | null; /** - * how its enhance step cleaned up its sound before transcription + * how its clean_audio step cleaned up its sound before transcription */ audio_cleanup?: AudioCleanup | null; [key: string]: unknown; diff --git a/nextjs-frontend/components/batches/run-on.tsx b/nextjs-frontend/components/batches/run-on.tsx index 549044ad9..f669a18e9 100644 --- a/nextjs-frontend/components/batches/run-on.tsx +++ b/nextjs-frontend/components/batches/run-on.tsx @@ -33,7 +33,7 @@ import { cn } from "@/lib/utils"; type Kind = "template" | "pipeline" | "steps"; const STEP_ORDER = [ - "enhance", + "clean_audio", "transcribe", "diarize", "shots", diff --git a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx index c82946e50..1b84fb0e0 100644 --- a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx @@ -55,7 +55,7 @@ import { useArchive } from "@/lib/hooks/session"; import { cn } from "@/lib/utils"; const ICON: Record = { - enhance: AudioWaveform, + clean_audio: AudioWaveform, transcribe: Captions, read: ScanText, diarize: AudioLines, @@ -76,7 +76,7 @@ const ICON: Record = { workflow: Workflow, }; const TONE = { intent: "blue", red: "red", green: "green", gate: "gold", neutral: "neutral" } as const; -const ASSETS = ["enhance", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"]; +const ASSETS = ["clean_audio", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"]; const METADATA = ["clean", "analyze", "embed", "summarize", "llm", "spec", "report", "export"]; /** A pipeline drawn as a graph: asset steps, metadata steps and workflows, an edge saying what runs after what. */ diff --git a/nextjs-frontend/components/pipelines/pipeline-editor.tsx b/nextjs-frontend/components/pipelines/pipeline-editor.tsx index ef18aa2e4..4ef750e54 100644 --- a/nextjs-frontend/components/pipelines/pipeline-editor.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-editor.tsx @@ -62,7 +62,7 @@ import { useArchive } from "@/lib/hooks/session"; import { cn } from "@/lib/utils"; const ICON: Record = { - enhance: AudioWaveform, + clean_audio: AudioWaveform, transcribe: Captions, read: ScanText, diarize: AudioLines, @@ -82,7 +82,7 @@ const ICON: Record = { workflow: Workflow, }; const LIB_ORDER = [ - "enhance", + "clean_audio", "transcribe", "diarize", "shots", diff --git a/nextjs-frontend/components/pipelines/pipeline-model.ts b/nextjs-frontend/components/pipelines/pipeline-model.ts index 5176830c7..fc411cec1 100644 --- a/nextjs-frontend/components/pipelines/pipeline-model.ts +++ b/nextjs-frontend/components/pipelines/pipeline-model.ts @@ -70,7 +70,7 @@ export function cleanSpec(s: StepSpec): StepSpec | string { /** Which earlier steps a step reads from, when they are in the same pipeline. */ export const NEEDS: Record = { - transcribe: ["enhance"], + transcribe: ["clean_audio"], diarize: ["transcribe"], ocr: ["shots"], faces: ["shots"], @@ -90,7 +90,7 @@ export const NEEDS: Record = { /** What a step makes available to later steps and templates. */ export const PROVIDES: Record = { - enhance: "audio_cleanup (speech · silence left out) for transcribe", + clean_audio: "audio_cleanup (speech · silence left out) for transcribe", transcribe: "transcript · segments[] · language", read: "pages[] · segments[]", diarize: "speakers[] · turns", @@ -112,7 +112,7 @@ export const PROVIDES: Record = { }; export const DESCRIBE: Record = { - enhance: + clean_audio: "Cleans up the sound before it’s transcribed with WebRTC: noise, gain, rumble, echo, and silence left out (Settings → Transcription)", transcribe: "Speech to text with word timings (Settings → Transcription)", read: "Draws a document’s or an image’s pages and reads their text (Settings → Documents)", diff --git a/nextjs-frontend/components/recording/jobs.ts b/nextjs-frontend/components/recording/jobs.ts index 1c3f5ed29..75c580d8f 100644 --- a/nextjs-frontend/components/recording/jobs.ts +++ b/nextjs-frontend/components/recording/jobs.ts @@ -287,7 +287,7 @@ export function readyBefore(j: JobInfo): string[] { // ---------- Reprocess: choose steps ---------- export const STEP_ORDER = [ - "enhance", + "clean_audio", "transcribe", "diarize", "shots", @@ -305,7 +305,7 @@ export const STEP_ORDER = [ export type StepKey = (typeof STEP_ORDER)[number]; export const STEP_HELP: Record = { - enhance: "Clean up the sound for transcription: noise, gain, echo and silence (Settings → Transcription)", + clean_audio: "Clean up the sound for transcription: noise, gain, echo and silence (Settings → Transcription)", transcribe: "Audio → text with timings", diarize: "Split speakers and match voice IDs", shots: "Scene cuts, keyframes and sampled frames", @@ -324,7 +324,7 @@ export const STEP_HELP: Record = { /** Ticking a step also ticks the later steps that use its output. */ export const DEPENDENTS: Record = { - enhance: ["transcribe", "diarize", "clean", "analyze", "embed", "summarize", "report"], + clean_audio: ["transcribe", "diarize", "clean", "analyze", "embed", "summarize", "report"], transcribe: ["diarize", "clean", "analyze", "embed", "summarize", "report"], diarize: ["analyze", "summarize", "report"], shots: ["ocr", "faces", "objects", "nsfw", "describe"], @@ -344,18 +344,18 @@ export type StepOption = { key: StepKey; disabled: string | null }; /** * The steps offered for a recording: video steps only for videos (faces, objects, nsfw and describe also for a - * document's or an image's pages); enhance/transcribe/diarize need audio; enhance and clean are for speech, so not for + * document's or an image's pages); clean_audio/transcribe/diarize need audio; clean_audio and clean are for speech, so not for * pages. */ export function reprocessOptions(opts: { video: boolean; hasAudio: boolean; paged?: boolean }): StepOption[] { const left = opts.video ? [] : opts.paged - ? ["enhance", "shots", "ocr", "clean"] + ? ["clean_audio", "shots", "ocr", "clean"] : ["shots", "ocr", "faces", "objects", "nsfw", "describe"]; return STEP_ORDER.filter((k) => !left.includes(k)).map((key) => { let disabled: string | null = null; - if (!opts.hasAudio && key === "enhance") + if (!opts.hasAudio && key === "clean_audio") disabled = "This asset has no audio: its text was imported, so there's no sound to clean up."; else if (!opts.hasAudio && key === "transcribe") disabled = "This asset has no audio: its text was imported, so there's nothing to transcribe."; diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index c5071e7a0..04ef67951 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -627,7 +627,7 @@ export const FIELDS: FieldSpec[] = [ kind: "text", mono: true, }, - // Audio cleanup: what the Clean up audio step (enhance) does before transcription, in a pipeline that has it + // Audio cleanup: what the Clean up audio step (clean_audio) does before transcription, in a pipeline that has it { section: "transcribe", key: "cleanup.noise", diff --git a/nextjs-frontend/components/ui/loop.tsx b/nextjs-frontend/components/ui/loop.tsx index c202d087d..74ba726e7 100644 --- a/nextjs-frontend/components/ui/loop.tsx +++ b/nextjs-frontend/components/ui/loop.tsx @@ -9,7 +9,7 @@ import { cn } from "@/lib/utils"; * upcoming ones are pale tints. */ export const STEP_TONE: Record = { - enhance: "intent", + clean_audio: "intent", transcribe: "intent", read: "intent", shots: "intent", @@ -31,7 +31,7 @@ export const STEP_TONE: Record = { }; export const STEP_LABEL: Record = { - enhance: "Clean up audio", + clean_audio: "Clean up audio", transcribe: "Transcribe", read: "Read pages", diarize: "Diarize", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 996fd118b..b98862704 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -35361,7 +35361,7 @@ "kept_ms" ], "title": "AudioCleanup", - "description": "How the enhance step cleaned up a recording's sound for transcription (docs/audio-cleanup.md)." + "description": "How the clean_audio step cleaned up a recording's sound for transcription (docs/processing.md, Cleaning up audio)." }, "AuditEntry": { "properties": { @@ -64198,7 +64198,7 @@ "type": "null" } ], - "description": "how its enhance step cleaned up its sound before transcription" + "description": "how its clean_audio step cleaned up its sound before transcription" } }, "additionalProperties": true, From 2d61a81e85b1b58ade50933ed36929cb33d0d08e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 08:06:04 +0000 Subject: [PATCH 3/3] style(pipelines): format the canvas's asset steps --- .../components/pipelines/pipeline-canvas.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx index 9f959c4be..1aefc92e5 100644 --- a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx @@ -79,7 +79,18 @@ const ICON: Record = { workflow: Workflow, }; const TONE = { intent: "blue", red: "red", green: "green", gate: "gold", neutral: "neutral" } as const; -const ASSETS = ["clean_audio", "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "convert"]; +const ASSETS = [ + "clean_audio", + "transcribe", + "diarize", + "shots", + "ocr", + "faces", + "objects", + "nsfw", + "describe", + "convert", +]; const METADATA = ["clean", "analyze", "embed", "summarize", "llm", "spec", "report", "export"]; /** A pipeline drawn as a graph: asset steps, metadata steps and workflows, an edge saying what runs after what. */