From 5b7b2d5a91603480fe98af64e67609b8cdffd48b Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 12:29:38 +0000 Subject: [PATCH] feat(enhance): clean up low quality sound before it's transcribed Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012m72U9mPLCbuH8rSAZGSuZ --- README.md | 8 +- changelog.d/enhance-audio.md | 7 + docs/api.md | 22 +- docs/automations.md | 2 +- docs/components.md | 1 + docs/configuration.md | 2 +- docs/enhance.md | 106 +++++-- docs/mcp.md | 5 +- docs/requirements.md | 2 +- fastapi_backend/app/api/mcp_tools.py | 18 +- .../app/api/v1/routes/recordings.py | 7 +- fastapi_backend/app/api/v1/routes/video.py | 16 +- fastapi_backend/app/domain/actions_run.py | 17 +- fastapi_backend/app/domain/components.py | 25 ++ fastapi_backend/app/domain/deletion.py | 2 +- fastapi_backend/app/domain/enhance.py | 128 ++++++-- fastapi_backend/app/domain/enhance_audio.py | 297 +++++++++++++++++ fastapi_backend/app/domain/ingest.py | 10 +- fastapi_backend/app/domain/jobs.py | 9 +- fastapi_backend/app/domain/keyring.py | 13 +- fastapi_backend/app/domain/pipelines.py | 9 +- fastapi_backend/app/domain/settings.py | 3 +- fastapi_backend/app/domain/store.py | 4 + fastapi_backend/app/schemas/recordings.py | 39 ++- fastapi_backend/pyproject.toml | 3 + fastapi_backend/tests/api/test_enhance.py | 6 +- .../tests/api/test_enhance_audio.py | 300 ++++++++++++++++++ fastapi_backend/tests/api/test_jobs.py | 5 +- fastapi_backend/uv.lock | 16 +- mkdocs.yml | 2 +- nextjs-frontend/__tests__/enhance.test.tsx | 44 ++- nextjs-frontend/app/openapi-client/index.ts | 1 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 13 +- .../app/openapi-client/types.gen.ts | 82 ++++- .../components/pipelines/pipeline-model.ts | 5 +- .../components/recording/details-tab.tsx | 9 +- .../components/recording/enhance-model.ts | 30 +- .../components/recording/enhance.tsx | 72 +++-- .../components/settings/bodies.tsx | 15 +- nextjs-frontend/components/settings/model.ts | 38 ++- nextjs-frontend/openapi.json | 113 ++++++- threat-model.md | 1 + threat-model.yaml | 11 +- 43 files changed, 1313 insertions(+), 205 deletions(-) create mode 100644 changelog.d/enhance-audio.md create mode 100644 fastapi_backend/app/domain/enhance_audio.py create mode 100644 fastapi_backend/tests/api/test_enhance_audio.py diff --git a/README.md b/README.md index 72df7e174..1a543e1d5 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p [the assistant in your browser](docs/browser-assistant.md); [checks](docs/requirements.md#clip-from-the-browser)) - **Understand it.** Transcription, which can expect the names your namespace knows (off by default), people recognised by voice (long recordings go to a speech provider in pieces that fit its limits), text on screen and in scans, faces (off by default), NSFW pictures flagged by a model on your own - server (off by default, per namespace), low quality images enhanced before they're read (off by default), entities + server (off by default, per namespace), low quality images and sound enhanced before they're read or transcribed (off by default), entities and topics in a knowledge graph you can explore, query with Cypher or SPARQL, and see as it was at any version. Every entity has its own page at a wiki address (`/wiki//`), and entities, topics and people are edited and merged from their lists, the graph or an @@ -147,7 +147,7 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p ([Graph](docs/graph.md), [entity pages](docs/notes.md#entity-pages), [similar content](docs/processing.md#similar-content), [graph history](docs/graph-history.md), [topics](docs/topics.md), [video](docs/video.md), [NSFW classification](docs/nsfw.md), - [enhancing images](docs/enhance.md), [long recordings](docs/speech-providers.md#long-recordings), + [enhancing images and sound](docs/enhance.md), [long recordings](docs/speech-providers.md#long-recordings), [your own names](docs/processing.md#your-own-names-in-transcription), [cleaning up transcripts](docs/processing.md#cleaning-up-transcripts), [dates](docs/processing.md#dates); [checks](docs/requirements.md#understand-it)) @@ -318,8 +318,8 @@ The guides are on the [docs site](https://adeelahmad.github.io/lense/) and in th [the assistant by email](docs/assistant-email.md), [podcasts](docs/podcasts.md), [notes](docs/notes.md), [web clipper](docs/web-clipper.md), [the assistant in your browser](docs/browser-assistant.md), [sensors](docs/sensors.md), [processing](docs/processing.md), [video](docs/video.md), - [NSFW classification](docs/nsfw.md), [enhancing images](docs/enhance.md), [notifications](docs/notifications.md), - [automations](docs/automations.md), [timeline](docs/timeline.md) + [NSFW classification](docs/nsfw.md), [enhancing images and sound](docs/enhance.md), + [notifications](docs/notifications.md), [automations](docs/automations.md), [timeline](docs/timeline.md) - **Knowledge:** [graph and Cypher](docs/graph.md), [graph history](docs/graph-history.md), [topics](docs/topics.md), [the graph as files](docs/file-tree.md), [NFS share](docs/file-shares.md), [SMB share](docs/smb.md), [linked data (RDF)](docs/rdf.md), [IIIF](docs/iiif.md), [Fedora](docs/fedora.md) diff --git a/changelog.d/enhance-audio.md b/changelog.d/enhance-audio.md new file mode 100644 index 000000000..c923c66ec --- /dev/null +++ b/changelog.d/enhance-audio.md @@ -0,0 +1,7 @@ +- **Clean up low quality sound before it's transcribed.** The `enhance` step now listens to audio and video too: + a noisy, quiet or clipped recording gets a cleaned-up copy (declipped, rumble and steady noise taken out, its level + evened) that transcription hears, in step with the original sample for sample, so words land where they were said; + the original, its waveform and voice matching stay as they were. Off by default: Settings → Enhance. ffmpeg's + filters are the default; WebRTC's noise suppression and gain (webrtc-noise-gain, Apache-2.0) is fetched the first + time it's chosen. Enhance and transcribe again from a recording's Details tab, the API, MCP or an automation + ([Enhancing images and sound](docs/enhance.md)). diff --git a/docs/api.md b/docs/api.md index 697f24986..668c18776 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1399,16 +1399,18 @@ Audited as `recording.nsfw`. `GET /resources/nsfw` (`ns` for one namespace) says you read, how many resources are `flagged`, and how many of those the Library leaves out (`hidden`). Search hits and photo search results carry `nsfw: true` for the ones the web app blurs. -### Enhancing images - -The `enhance` step comes first in the standard pipeline: where [enhancing images](enhance.md) is on (off by -default), it measures an image and, when it's small, blurry, noisy, dull or has small faces, keeps an enhanced copy -that `read` and the steps after it work from; the original stays as it is. Other resources skip it. A resource -carries `enhanced`: `{kind, low, measured, enhanced, engine, scale, width, height, forced, at, download}` (`download` -a signed link to the copy), or `null` until the step looked at it. `POST /resources/{rid}/enhance` (the process -permission) enhances an image whatever the settings and its quality and runs its pipeline's later steps again, -answering the `job`; 400 for anything but an image. Audited as `recording.enhance`. `GET /resources/{rid}/enhanced` -is the copy, for members of its namespace. +### Enhancing images and sound + +The `enhance` step comes first in the standard pipeline: where [enhancing](enhance.md) is on (off by default), it +measures an image and, when it's small, blurry, noisy, dull or has small faces, keeps an enhanced copy that `read` +and the steps after it work from; it listens to audio and video and, when the sound is noisy, quiet or clipped, +keeps a cleaned-up copy (FLAC, 16 kHz mono, in step with the original) that `transcribe` hears. The original stays +as it is. Documents and transcripts skip it. A resource carries `enhanced`: `{kind, low, measured, enhanced, engine, +forced, at, download}`, with `scale`, `width` and `height` for an image and `filters` or `speech` for sound +(`download` a signed link to the copy), or `null` until the step looked at it. `POST /resources/{rid}/enhance` (the +process permission) enhances an image or a recording whatever the settings and its quality and runs its pipeline's +later steps again, answering the `job`; 400 for a document or a transcript. Audited as `recording.enhance`. +`GET /resources/{rid}/enhanced` is the copy, for members of its namespace. ### Descriptions diff --git a/docs/automations.md b/docs/automations.md index 2caf7cab5..bb813f272 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -658,7 +658,7 @@ or removed does nothing. What it writes is by the automation itself, `automation | `http_request` | sends a `method` (GET, POST or PUT) to a `url`, with `headers` and a `body` (text, or an object sent as JSON) | out | `{status, ok, body, json, content_type, truncated, method, url, headers}`: the answer, as outside text | | `run_workflow` | runs a recording `workflow` (its published `version`, or the one named) on an `asset` as a job | lens | `{job, workflow, version, asset}` | | `run_pipeline` | processes an `asset` again with a `pipeline` (or the one its namespace uses), keeping its outputs | lens | `{job, pipeline, asset}` | -| `enhance` | [enhances](enhance.md) an image `asset` whatever the settings, keeping the original, and reads it again from the copy | lens | `{job, asset}` | +| `enhance` | [enhances](enhance.md) an image, audio or video `asset` whatever the settings, keeping the original, and reads or transcribes it again from the copy | lens | `{job, asset}` | | `run_routine` | asks the scheduler to run a `routine` now (`propose_only` to only propose its changes) | lens, so it asks | `{routine, requested, propose_only}` | | `set_field` | sets one of an `asset`'s custom fields (`field`: its id or label) to `value` (empty clears it) | lens | `{asset, field, label, value, changed}` | | `add_to_collection` | moves an `asset` into a `collection` of its namespace | lens | `{asset, collection, name, moved, newly_see}` | diff --git a/docs/components.md b/docs/components.md index d45592995..396c9a771 100644 --- a/docs/components.md +++ b/docs/components.md @@ -23,6 +23,7 @@ never fetch a speech or voice model. | Florence-2 | `llm.describe_engine` is `florence-2` | Florence-2-base-ft or -large-ft (MIT, `llm.florence_model`; about 280 or 830 MB), their 8-bit ONNX exports, the tokenizers library and ONNX Runtime, to [describe pictures](configuration.md#descriptions) on this machine | | Moondream | `llm.describe_engine` is `moondream`, and this machine can run the chosen one | Moondream 3 Preview (BSL 1.1, about 19 GB, an NVIDIA GPU or a Mac with 32 GB) or Moondream 2 (Apache-2.0, about 3.7 GB), `llm.moondream_model`, at a pinned revision, with its tokenizer, PyTorch (the CPU build without an NVIDIA GPU) and transformers, to [describe pictures](configuration.md#descriptions) on this machine | | Real-ESRGAN | [enhancing images](enhance.md) is on and `enhance.image_engine` is `realesrgan` (off by default) | Real-ESRGAN general x4v3 as an ONNX model (BSD-3-Clause, 5 MB), and ONNX Runtime | +| WebRTC audio processing | [enhancing sound](enhance.md) is on and `enhance.audio_engine` is `webrtc` (off by default), on Linux on x86-64 or ARM64 | webrtc-noise-gain (Apache-2.0, about 1 MB): WebRTC's noise suppression, automatic gain and voice detector | | Prompt-injection classifier | `ai.injection_check` is `classifier` | ProtectAI's deberta-v3-base-prompt-injection-v2 (Apache-2.0, about 740 MB), the tokenizers library and ONNX Runtime ([third-party content](assistant.md#third-party-content-and-prompt-injection)) | | Punctuation model | `analysis.cleanup` is on and `analysis.cleanup_punctuation` isn't `off` ([Cleaning up transcripts](processing.md#cleaning-up-transcripts)) | kredor/punctuate-all at a pinned revision (MIT, about 1.1 GB), with Transformers and PyTorch's CPU build | | spaCy | `analysis.entities` is `spacy` ([Entities](processing.md#entities-and-the-graph-explorer)) | spaCy (MIT) and the model in `analysis.spacy_model`: `en_core_web_sm`, `de_core_news_sm` or `xx_ent_wiki_sm` (MIT, 11 to 15 MB each), about 60 MB in all | diff --git a/docs/configuration.md b/docs/configuration.md index 05f6089d7..142ba7a67 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,7 +113,7 @@ Admins can change, in Settings: transcription, speaker separation, speech provid provider and key, a local model, the AI assistant, the [code sandbox](assistant.md#code-sandbox), search (word search and search by meaning), reports and graph, video, OCR, faces and objects, [NSFW classification](nsfw.md), [enhancing -images](enhance.md), workers, sign-in, components (and the edition, full or +images and sound](enhance.md), workers, sign-in, components (and the edition, full or lean), access and embedding (whether other devices on the network reach Lens, embed frame ancestors, text and transcript upload limit, allowed hosts, trusted proxies, session length), remote access, email, chat rooms, notifications, automations, telemetry, the Fedora repository, sensors, file shares, uploads, storage, documents, API keys, diff --git a/docs/enhance.md b/docs/enhance.md index 4bf66a247..88063647f 100644 --- a/docs/enhance.md +++ b/docs/enhance.md @@ -1,24 +1,26 @@ -# Enhancing images +# Enhancing images and sound Lens can look at each image before it's read and, when it's low quality, keep an enhanced copy that reading (OCR), -faces, objects, NSFW classification and descriptions work from instead. The original stays as it is. It's **off** -until an admin turns it on, and it runs on your own server: pictures aren't sent anywhere, and no model provider is -called. - -Audio isn't enhanced yet. +faces, objects, NSFW classification and descriptions work from instead. It can listen to each audio or video file the +same way before it's transcribed, and keep a cleaned-up copy of its sound for transcription to hear. The original +stays as it is. Both are **off** until an admin turns them on, and they run on your own server: pictures and sound +aren't sent anywhere, and no model provider is called. ## Turning it on -Settings → Enhance, **Enhance low quality images** (`enhance.images`). The **enhance** step comes first in the -standard pipeline, so it looks at images added from then on. A pipeline of your own runs it when you add the -**enhance** step to it; a step with **force** set enhances every image it runs on, whatever the settings and its -quality. +Settings → Enhance, **Enhance low quality images** (`enhance.images`) and **Enhance low quality sound** +(`enhance.audio`). The **enhance** step comes first in the standard pipeline, so it looks at images and recordings +added from then on. A pipeline of your own runs it when you add the **enhance** step to it; a step with **force** set +enhances everything it runs on, whatever the settings and its quality. Documents and imported transcripts skip it. + +Any image or recording can also be enhanced on request, even with it off: **Enhance** in an image's Details tab, or +**Enhance and transcribe again** in a recording's (editors), `POST /api/v1/resources/{rid}/enhance`, the `enhance` +tool for agents, or the `enhance` automation action. That makes the copy and runs the rest of its pipeline again on +it. Transcribing again makes a new transcript, so the Details tab says how many corrected lines it would replace. -Any image can also be enhanced on request, even with it off: **Enhance** in its Details tab (editors), -`POST /api/v1/resources/{rid}/enhance`, the `enhance` tool for agents, or the `enhance` automation action. That makes -the copy and runs the rest of its pipeline again on it. +## Images -## What counts as low quality +### What counts as low quality The step measures the image and keeps why it looked low quality: @@ -36,7 +38,7 @@ Settings → Video. With **Only images with faces** (`enhance.only_faces`), an i An image with none of these is left as it is, and its job log says so: `good enough as it is (900×760, sharpness 312, noise 1.8, contrast 61)`. -## How it's enhanced +### How it's enhanced A small image, or one with small faces, is made bigger: enough to bring its shorter side to `enhance.min_side` and its smallest face to `enhance.min_face_px`, at most `enhance.max_scale` times and never past 4000 pixels a side. The @@ -51,15 +53,55 @@ Real-ESRGAN and ONNX Runtime are fetched the first time the step runs with it ch until then nothing is downloaded, so a Raspberry Pi that keeps `classic` stays small. An image over 1.5 megapixels is enhanced by `classic` instead, so a small machine isn't swamped, and the job log says why. +## Sound + +### What counts as low quality + +The step listens to the whole recording (a video's soundtrack too) at 16 kHz mono, as transcription does, in 30 ms +frames, and keeps why it sounded low quality: + +| Reason | When | +|---|---| +| noisy | its loudest frames are less than `enhance.min_snr_db` above its quietest (its noise floor) | +| quiet | the level of its loud half is under `enhance.min_level_db` dBFS | +| clipped | more than 0.1% of its samples are at full scale | + +A recording with none of these is left as it is (`good enough as it is (signal to noise 31 dB, level -21 dBFS, 0% +clipped)`), and a silent one is skipped. So is one whose transcript came with it, unless it's asked for, since nothing +would hear the copy. + +### How it's enhanced + +The engine (`enhance.audio_engine`): + +| Engine | What it does | +|---|---| +| `ffmpeg` (default) | ffmpeg's own filters, nothing to fetch: `adeclip` repairs clipping, a high-pass filter takes out rumble and hum below 70 Hz, `afftdn` takes out steady noise down from the floor it measured, and a quiet recording is turned up towards -20 dBFS (as far as its peak allows) and evened out with `dynaudnorm` | +| `webrtc` | WebRTC's audio processing through [webrtc-noise-gain](https://github.com/OHF-voice/webrtc-noise-gain) (Apache-2.0; WebRTC is BSD-3-Clause): its noise suppression, high-pass filter and, for a quiet recording, automatic gain, 10 ms at a time. Its voice detector says how much of it is speech | + +Asked for with nothing wrong, it gets a gentle clean-up of everything: the high-pass filter, light noise reduction +and `dynaudnorm`. + +The copy keeps the original's times sample for sample, so the transcript's words land where they were said in the +original: a filter that puts its output late (`afftdn` does) is measured once with a probe of clicks and +bursts, and the copy is moved back by that much. Speakers are still told apart from the original, so voiceprints stay +comparable between recordings, and the player's waveform is the original's. + +WebRTC's processing is fetched the first time the step runs with it chosen, on Linux on x86-64 or ARM64; until then, +and elsewhere, ffmpeg does it and the job log says why. Echo cancellation needs the sound of the far end of a call, +which a recording doesn't have, so neither engine does it. + ## The enhanced copy -The copy is a JPEG kept beside the original (`data_dir/renditions/-enhanced.jpg`), encrypted where the -namespace's files are, re-keyed with them, and deleted with the resource. Its Details tab says what was found and -made (`Small, blurry · 240×180 to 960×720 by the classic engine`) with **Save the copy**. Only members of the -namespace can download it; visitors to a shared resource see its pages as before. +The copy is kept beside the original, a JPEG for an image and FLAC (16 kHz mono) for sound +(`data_dir/renditions/-enhanced.jpg` or `.flac`), encrypted where the namespace's files are, re-keyed with them, +and deleted with the resource. Its Details tab says what was found and made (`Small, blurry · 240×180 to 960×720 by +the classic engine`, `Noisy · signal to noise 9.5 dB, level -24.1 dBFS · by ffmpeg's filters`) with **Save the +copy**. Only members of the namespace can download it; visitors to a shared resource see its pages or hear its media +as before. -Turning it off leaves copies alone until an image is processed again: then a copy the step made by itself is removed, -and one made on request is kept. +Turning it off leaves copies alone until an image or a recording is processed again: then a copy the step made by +itself is removed, and one made on request is kept. ## Settings @@ -72,21 +114,29 @@ and one made on request is kept. | `enhance.min_sharpness` | `100` | blurry below this sharpness (0 to 10000; 0 never counts one as blurry) | | `enhance.min_face_px` | `64` | faces smaller than this many pixels tall are too small (0 to 1024; 0 turns it off) | | `enhance.max_scale` | `4` | enlarge at most this many times (1 to 4) | +| `enhance.audio` | `false` | enhance low quality sound | +| `enhance.audio_engine` | `ffmpeg` | `ffmpeg` or `webrtc` | +| `enhance.min_snr_db` | `15` | noisy below this many dB of signal over noise (0 to 60) | +| `enhance.min_level_db` | `-35` | quiet below this level, dBFS (-70 to 0) | ## API and agents -- `POST /api/v1/resources/{rid}/enhance` (needs the process permission on the resource): enhance an image now and run - its pipeline's later steps again; answers the `job`. 400 for anything but an image. Audited as `recording.enhance`. -- A resource carries `enhanced`: `{kind, low, measured, enhanced, engine, scale, width, height, forced, at, download}`, - where `measured` is `{width, height, sharpness, noise, contrast, brightness, faces, smallest_face}` and `download` a - signed link to the copy; `null` until the step has looked at it. +- `POST /api/v1/resources/{rid}/enhance` (needs the process permission on the resource): enhance an image or a + recording now and run its pipeline's later steps again; answers the `job`. 400 for a document or a transcript. + Audited as `recording.enhance`. +- A resource carries `enhanced`: `{kind, low, measured, enhanced, engine, forced, at, download}`, with `scale`, `width` + and `height` for an image and `filters` (ffmpeg) or `speech` (WebRTC) for sound. `measured` is `{width, height, + sharpness, noise, contrast, brightness, faces, smallest_face}` for an image and `{duration, snr_db, level_db, + floor_db, peak_db, clipped}` for sound, and `download` a signed link to the copy; `null` until the step has looked + at it. - `GET /api/v1/resources/{rid}/enhanced`: the copy (members of its namespace). - [MCP](mcp.md): `enhance` (a write-scope token); `get_recording` says what the step found. - [Automations](automations.md): the `enhance` action, on an asset. ## Refine later -- Enhance audio before it's transcribed: noise, level and clipping. - Enhance a video's keyframes and a document's scanned pages, not only images. - A face restorer for small faces, with a permissive license. -- Show the original and the copy side by side in the asset page. +- A speech enhancement model for heavy, changing noise (babble, music), with a permissive license. +- Enhance a video or episode imported by its address before it's downloaded (now: on request after). +- Play the enhanced sound in the player, and show the original and the copy side by side. diff --git a/docs/mcp.md b/docs/mcp.md index 817f98eab..15225f4b8 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -114,7 +114,7 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `nsfw_settings` | how [NSFW classification](nsfw.md) works in a namespace: on or off, the threshold, how flagged resources show, and how many are classified, flagged and not classified yet | | `set_nsfw` | turn NSFW classification on or off, set its threshold and display, for a namespace (owners) or the archive (admins), and optionally classify what's there. Needs a write-scope token | | `mark_nsfw` | mark a resource NSFW or not by hand, or hand it back to the model's score. Needs a write-scope token and editor access | -| `enhance` | [enhance an image](enhance.md) whatever the settings, keeping the original, and read it again from the enhanced copy. Needs a write-scope token and the process permission | +| `enhance` | [enhance an image or a recording's sound](enhance.md) whatever the settings, keeping the original, and read or transcribe it again from the enhanced copy. Needs a write-scope token and the process permission | | `namespace_integrations` | whether a namespace has [its own MCP server](#one-namespaces-mcp-server) and [chat completions](api.md#chat-completions-for-any-app) on, their addresses, and the tools offered there | | `set_namespace_integrations` | turn a namespace's own MCP server or chat completions on or off, and pick the read tools its chat completions may use. Owners, with a write-scope token | | `list_files` | a folder of [the graph as files](file-tree.md): your namespaces, then their collections, topics (nested by narrower topics) and entities, each holding its assets' files | @@ -176,7 +176,8 @@ which records an undoable change (proposed by default), `undo_graph_change`, `ed app (merges and deletes can be undone), `write_note`, which adds a note, `create_podcast`, which queues an episode, `create_feed` and `publish_to_feed`, which make and fill a [published feed](feeds.md), `write_spec`, which queues a spec, `set_nsfw` and `mark_nsfw`, which change [NSFW classification](nsfw.md) (owners or admins, and -editors), `enhance`, which enhances an image and reads it again (the process permission), +editors), `enhance`, which enhances an image or a recording's sound and reads or transcribes it again (the process +permission), `set_namespace_integrations`, which turns a namespace's own MCP server or chat completions on or off (owners), `set_picture_descriptions`, which changes what describes pictures (admins), `set_file_storage`, which changes where new files are kept (admins), `set_secret_reference`, which points a secret field at a password manager's item (admins), `add_mcp_server`, which adds an MCP server (admins), `subscribe_feed` and `unsubscribe_feed`, which follow and stop following news feeds (admins), and `set_monitoring`, `set_service`, `set_sandbox_gateway` (the [code sandbox](assistant.md#code-sandbox)'s gateway) and `set_install_settings`, which have the machine change how Lens runs (admins). diff --git a/docs/requirements.md b/docs/requirements.md index 21a9b1d8e..cad992602 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -164,7 +164,7 @@ request, to another check that proves the same thing; loosening a requirement or | Text on screen and in scans | [test_video.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video.py) `test_shots_text_search_chat_and_iiif`; [test_documents.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_documents.py) `test_scans_and_images_are_read_by_ocr`; [test_ocr.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_ocr.py) `test_a_video_read_by_doctr_and_without_it` | | Faces, off by default | [test_video.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video.py) `test_people_on_screen`, `test_shots_text_search_chat_and_iiif`; [test_video.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_video.py) `test_face_modes` | | NSFW pictures flagged by a model on your own server, off by default, per namespace | [test_nsfw.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_nsfw.py) `test_off_by_default_nothing_is_fetched_or_classified`, `test_a_namespace_classifies_flags_blurs_and_hides`; [nsfw.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/nsfw.test.tsx) | -| Low quality images enhanced before they're read, off by default | [test_enhance.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_enhance.py) `test_off_by_default_the_original_is_read`, `test_a_low_quality_image_is_read_from_its_enhanced_copy`, `test_a_good_image_is_left_as_it_is`, `test_agents_and_automations_enhance_too`, `test_the_copy_is_encrypted_where_the_files_are`; [enhance.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/enhance.test.tsx) | +| Low quality images and sound enhanced before they're read or transcribed, off by default | [test_enhance.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_enhance.py) `test_off_by_default_the_original_is_read`, `test_a_low_quality_image_is_read_from_its_enhanced_copy`, `test_a_good_image_is_left_as_it_is`, `test_agents_and_automations_enhance_too`, `test_the_copy_is_encrypted_where_the_files_are`; [test_enhance_audio.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_enhance_audio.py) `test_off_by_default_the_original_is_heard`, `test_noisy_sound_is_transcribed_from_a_copy_in_step_with_it`, `test_quiet_and_clipped_sound`, `test_good_or_silent_sound_is_left_as_it_is`, `test_asked_for_it_is_enhanced_and_transcribed_again`, `test_webrtc_where_its_chosen`; [enhance.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/enhance.test.tsx) | | Entities and topics in a knowledge graph to explore and query with Cypher or SPARQL | [test_graph.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_graph.py) `test_query`; [test_rdf.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_rdf.py) `test_sparql`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `graph-find` | | The graph as it was at any version | [test_graph_history.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_graph_history.py) `test_the_explorer_and_cypher_as_of_a_version`, `test_the_canvas_as_of_a_version`; [test_graph_history.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_graph_history.py) `test_every_curation_is_an_event_and_every_version_replays` | | Every entity's own page at `/wiki//` | [test_entities.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_entities.py) `test_wiki_addresses`; [wiki-page.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/wiki-page.test.tsx); [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `entity-find` | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index bf8aecc59..722e676da 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -2456,20 +2456,22 @@ def mark_nsfw(ctx: Context, recording_id: int, verdict: str) -> dict[str, Any]: return {"recording_id": recording_id, "nsfw": got, "url": ctx.link(recording_id)} -# ---------- enhancing low quality images ---------- +# ---------- enhancing low quality images and sound ---------- @tool( "enhance", - "Enhance an image", - "Make an enhanced copy of an image asset (enlarged, sharpened, cleaned of noise, its contrast stretched), whatever " - "the settings and its quality, and read it again from that copy: its text, faces and objects. The original stays " - "as it is. Low quality images are enhanced on their own where an admin turned that on (enhance.images); " - "get_recording says what was found and done (enhanced). Editors, with a token that has the write scope.", - Arg("recording_id", "integer", "the image", required=True), + "Enhance an image or a recording", + "Make an enhanced copy of an image asset (enlarged, sharpened, cleaned of noise, its contrast stretched) and read " + "it again from that copy (its text, faces and objects), or of an audio or video asset's sound (noise taken out, its " + "level evened, clipping repaired) and transcribe it again from that copy, whatever the settings and its quality. " + "The original stays as it is; transcribing again replaces corrections made to the transcript. Low quality ones are " + "enhanced on their own where an admin turned that on (enhance.images, enhance.audio); get_recording says what was " + "found and done (enhanced). Editors, with a token that has the write scope.", + Arg("recording_id", "integer", "the image, audio or video", required=True), writes=True, ) def enhance_image(ctx: Context, recording_id: int) -> dict[str, Any]: if not ctx.user.can_write: - raise ToolError("this token is read-only: ask for a token with the write scope to enhance images") + raise ToolError("this token is read-only: ask for a token with the write scope to enhance assets") ctx.acl.recording(recording_id, "process") job = enhance.queue(ctx.db, recording_id, ctx.user.email) auth.audit(ctx.db, ctx.user.as_audit(), "recording.enhance", f"recording:{recording_id}", {"job": job, "via": "mcp"}) diff --git a/fastapi_backend/app/api/v1/routes/recordings.py b/fastapi_backend/app/api/v1/routes/recordings.py index d462df078..8da995d2d 100644 --- a/fastapi_backend/app/api/v1/routes/recordings.py +++ b/fastapi_backend/app/api/v1/routes/recordings.py @@ -637,9 +637,10 @@ def reprocess_recording(rid: int, acl: Acl, user: Writer, db: Db, body: Reproces @router.post("/{rid}/enhance") def enhance_recording(rid: int, acl: Acl, user: Writer, db: Db) -> JobQueued: - """Enhance an image whatever the settings and its quality (docs/enhance.md): an enhanced copy is made, and its - pipeline's steps after enhance (reading its text, faces, objects, …) run again on that copy. The original stays as - it is. 400 for anything but an image. Audited as `recording.enhance`.""" + """Enhance an image, or a recording's sound, whatever the settings and its quality (docs/enhance.md): an enhanced + copy is made, and its pipeline's steps after enhance (reading an image's text, faces, objects, …; transcribing a + recording and what follows) run again on that copy. The original stays as it is. 400 for a document or a + transcript. Audited as `recording.enhance`.""" acl.recording(rid, "process") with domain_errors(): job = enhance.queue(db, rid, user.email) diff --git a/fastapi_backend/app/api/v1/routes/video.py b/fastapi_backend/app/api/v1/routes/video.py index d346c63ec..81f94322e 100644 --- a/fastapi_backend/app/api/v1/routes/video.py +++ b/fastapi_backend/app/api/v1/routes/video.py @@ -81,19 +81,21 @@ def get_pdf(rid: int, request: Request, acl: Acl, db: Db, cfg: Cfg, s: str = "") return stored_file(db, cfg, path, request, "application/pdf", f"{name}.pdf", files.HEADERS) -@router.get("/recordings/{rid}/enhanced", response_class=FileResponse, responses={200: {"content": {"image/jpeg": {}}}}) +@router.get("/recordings/{rid}/enhanced", response_class=FileResponse, responses={200: {"content": {"image/jpeg": {}, "audio/flac": {}}}}) def get_enhanced(rid: int, request: Request, acl: Acl, db: Db, cfg: Cfg, s: str = "") -> Response: - """The enhanced copy the enhance step made of a low quality image (docs/enhance.md), to save. Members of its - namespace only (a bearer token, or a signed link the API gave one): visitors get its pages, with faces pixelated - where the namespace does that.""" + """The enhanced copy the enhance step made of a low quality image (a JPEG) or a recording's sound (FLAC, 16 kHz + mono) (docs/enhance.md), to save. Members of its namespace only (a bearer token, or a signed link the API gave + one): visitors get its pages or its media, with faces pixelated where the namespace does that.""" rec = acl.recording(rid, share=s) if not acl.member(rec): raise HTTPException(404, "not found") - path = enhance.enhanced_path(cfg, rid) + kind = "audio" if rec.get("source") == "audio" else "image" + path = enhance.enhanced_path(cfg, rid, kind) if not path.is_file(): raise HTTPException(404, "not found") - name = pathlib.PurePosixPath(str((rec.get("remote") or {}).get("path") or rec.get("path") or "image")).stem or "image" - return stored_file(db, cfg, path, request, "image/jpeg", f"{name}-enhanced.jpg", files.HEADERS) + name = pathlib.PurePosixPath(str((rec.get("remote") or {}).get("path") or rec.get("path") or kind)).stem or kind + ext = enhance.KINDS[kind] + return stored_file(db, cfg, path, request, "audio/flac" if kind == "audio" else "image/jpeg", f"{name}-enhanced{ext}", files.HEADERS) @router.get("/recordings/{rid}/frames/{name}", response_class=FileResponse, responses={200: {"content": {"image/jpeg": {}}}}) diff --git a/fastapi_backend/app/domain/actions_run.py b/fastapi_backend/app/domain/actions_run.py index e0a959185..1b91d3bf0 100644 --- a/fastapi_backend/app/domain/actions_run.py +++ b/fastapi_backend/app/domain/actions_run.py @@ -67,11 +67,13 @@ def _enhance(db, cfg, intent, user, dry): p = intent["params"] acc = actions.person(db, intent, user) rec = actions.asset_for(db, intent, acc, actions.asset_id(p.get("asset")), "process") - if (db.one("SELECT source FROM $r", r=store.R("recording", rec["id"])) or {}).get("source") != "image": - raise ValueError(f"{rec.get('title') or rec['id']} isn't an image: only images are enhanced so far") + source = (db.one("SELECT source FROM $r", r=store.R("recording", rec["id"])) or {}).get("source") + if source not in jobs.ENHANCED: + raise ValueError(f"{rec.get('title') or rec['id']} isn't an image, audio or video: only those are enhanced") out = {"asset": rec["id"]} if dry: - return {**out, "job": None, "would": f"enhance {rec.get('title') or rec['id']} and read it again"} + again = "read it again" if source == "image" else "transcribe it again" + return {**out, "job": None, "would": f"enhance {rec.get('title') or rec['id']} and {again}"} return {**out, "job": enhance.queue(db, rec["id"], actions.actor(db, intent)["email"])} @@ -161,11 +163,12 @@ def _run_routine(db, cfg, intent, user, dry): actions.register( actions.ActionSpec( name="enhance", - label="Enhance an image", + label="Enhance an image or a recording", group="Run", - description="Makes an enhanced copy of an image (bigger, sharper, less noisy) and reads it again from that copy; " - "the original stays.", - params={"type": "object", "properties": {"asset": actions.asset_param("Image")}}, + description="Makes an enhanced copy of an image (bigger, sharper, less noisy) and reads it again from that copy, " + "or of a recording's sound (less noise, an even level, clipping repaired) and transcribes it again; the original " + "stays.", + params={"type": "object", "properties": {"asset": actions.asset_param("Image, audio or video")}}, effect="lens", subject=actions.asset_subject, output={"type": "object", "properties": {"job": {"type": "integer"}, "asset": {"type": "integer"}}}, diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index f5183aac7..22fadeb7c 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -562,6 +562,23 @@ def fetch(self, cfg, m, say): fetch_file(cfg, name, say) +class EnhanceSound(Component): + """WebRTC's audio processing (webrtc-noise-gain; enhance_audio.py), when Settings → Enhance chooses it for sound. + Its wheels are for Linux on x86-64 and ARM64; elsewhere ffmpeg's filters do it.""" + + def needed(self, cfg, m): + from . import enhance + + o = enhance.options(cfg) + return bool(o["audio"] and o["audio_engine"] == "webrtc" and m["os"] == "linux" and m["arch"] in ("x86_64", "arm64")) + + def present(self, cfg): + return importable("webrtc_noise_gain") + + def fetch(self, cfg, m, say): + pip_install(cfg, "webrtc", say) + + class Florence(Component): """Florence-2 and ONNX Runtime (florence.py), when Settings → LLM provider has it describe pictures.""" @@ -1000,6 +1017,14 @@ def detail(self, cfg): size_mb=6, license="BSD-3-Clause", ), + EnhanceSound( + "enhance-sound", + "WebRTC audio processing", + "cleans up noisy or quiet sound before it's transcribed (enhance.audio_engine webrtc)", + steps={"enhance"}, + size_mb=1, + license="Apache-2.0", + ), Florence( "florence", "Florence-2", diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py index f943d3dd7..ef62b529b 100644 --- a/fastapi_backend/app/domain/deletion.py +++ b/fastapi_backend/app/domain/deletion.py @@ -94,7 +94,7 @@ def _files(db, cfg, rec, rid, ns): reports / f"{name}.html", *reports.glob(f"{name}--*.html"), data / "renditions" / f"{int(rid)}.pdf", - enhance.enhanced_path(cfg, rid), + *(enhance.enhanced_path(cfg, rid, kind) for kind in enhance.KINDS), ] if rec.get("converted_from") and rec.get("path") and keyring.owned(cfg, rec["path"]): out.append(pathlib.Path(rec["path"])) diff --git a/fastapi_backend/app/domain/enhance.py b/fastapi_backend/app/domain/enhance.py index f6aa02c1c..e08c44334 100644 --- a/fastapi_backend/app/domain/enhance.py +++ b/fastapi_backend/app/domain/enhance.py @@ -1,11 +1,12 @@ -"""Enhancing low quality pictures (docs/enhance.md): the enhance step looks at an image before it's read and, when it's -low quality, makes an enhanced copy that the read step (OCR), faces, objects, NSFW and descriptions then work from. The -original stays as it is: the copy is a rendition, data_dir/renditions/-enhanced.jpg (encrypted where the -namespace's files are), which the asset page offers to save. +"""Enhancing low quality pictures and sound (docs/enhance.md): the enhance step looks at an image before it's read and, +when it's low quality, makes an enhanced copy that the read step (OCR), faces, objects, NSFW and descriptions then work +from. The original stays as it is: the copy is a rendition, data_dir/renditions/-enhanced.jpg (encrypted +where the namespace's files are), which the asset page offers to save. Audio and video are listened to the same way +before they're transcribed (enhance_audio.py), with a copy of their sound, -enhanced.flac. -Off by default: Settings → Enhance, `enhance.images`. Run with `force` (an agent's `enhance` tool, POST -/resources/{rid}/enhance, the `enhance` automation action, or a pipeline step that says so), it enhances an image -whatever the settings and its quality. +Off by default: Settings → Enhance, `enhance.images` and `enhance.audio`. Run with `force` (an agent's `enhance` tool, +POST /resources/{rid}/enhance, the `enhance` automation action, or a pipeline step that says so), it enhances an image +or a recording whatever the settings and its quality. What counts as low quality (each a reason kept in recording.enhanced.low): - `small`: its shorter side is under enhance.min_side pixels. @@ -26,9 +27,10 @@ the result is brought to the size wanted. components.py fetches it and ONNX Runtime the first time the step runs with it chosen. A picture over REAL_MAX_PIXELS goes to classic instead, so a small machine isn't swamped. -What a resource keeps (recording.enhanced): `kind`, `low` (the reasons), `measured` (width, height, sharpness, noise, -contrast, brightness, and faces and smallest_face where they were looked for), `enhanced` (whether a copy was made), -and for a copy its `engine`, `scale`, `width`, `height` and `forced`; `at`. +What a resource keeps (recording.enhanced): `kind` (image or audio), `low` (the reasons), `measured` (for an image its +width, height, sharpness, noise, contrast, brightness, and faces and smallest_face where they were looked for; for +sound what enhance_audio.measure says), `enhanced` (whether a copy was made), and for a copy its `engine`, `forced`, an +image's `scale`, `width` and `height`, and the sound's `filters` or `speech`; `at`. """ from __future__ import annotations @@ -40,7 +42,7 @@ import numpy as np -from . import convert, ingest, jobs, keyring, store +from . import convert, enhance_audio, ingest, jobs, keyring, store R = store.R ENGINES = ("classic", "realesrgan") @@ -55,8 +57,23 @@ DULL = 30.0 # a standard deviation of tones under this is dull MIN_SCALE = 1.1 # enlarging less than this isn't worth it QUALITY = 92 # JPEG quality of the enhanced copy -LIMITS = {"min_side": (64, 4000), "min_sharpness": (0, 10000), "min_face_px": (0, 1024), "max_scale": (1, 4)} -REASONS = {"small": "small", "blurry": "blurry", "noisy": "noisy", "dull": "low contrast", "small_faces": "faces too small"} +LIMITS = { + "min_side": (64, 4000), + "min_sharpness": (0, 10000), + "min_face_px": (0, 1024), + "max_scale": (1, 4), + "min_snr_db": (0, 60), + "min_level_db": (-70, 0), +} +REASONS = { + "small": "small", + "blurry": "blurry", + "noisy": "noisy", + "dull": "low contrast", + "small_faces": "faces too small", + **enhance_audio.REASONS, +} +KINDS = {"image": ".jpg", "audio": ".flac"} # what's enhanced, and its copy's extension _SESSIONS: dict[str, object] = {} _LOCK = threading.Lock() @@ -67,7 +84,7 @@ def options(cfg): def check(key, value): """A setting's value, checked: ValueError naming the problem.""" - if key in ("images", "only_faces"): + if key in ("images", "only_faces", "audio"): if not isinstance(value, bool): raise ValueError(f"enhance.{key} is true or false") return value @@ -75,6 +92,10 @@ def check(key, value): if value not in ENGINES: raise ValueError(f"enhance.image_engine is one of {', '.join(ENGINES)}") return value + if key == "audio_engine": + if value not in enhance_audio.ENGINES: + raise ValueError(f"enhance.audio_engine is one of {', '.join(enhance_audio.ENGINES)}") + return value if key in LIMITS: lo, hi = LIMITS[key] if not (isinstance(value, (int, float)) and not isinstance(value, bool) and lo <= value <= hi): @@ -90,14 +111,13 @@ def wants_faces(cfg): # ---------- the copy ---------- -def enhanced_path(cfg, rid): - return pathlib.Path(cfg["data_dir"]) / "renditions" / f"{int(rid)}-enhanced.jpg" +def enhanced_path(cfg, rid, kind="image"): + return pathlib.Path(cfg["data_dir"]) / "renditions" / f"{int(rid)}-enhanced{KINDS[kind]}" def files(cfg, rid): """The enhanced copies a resource has, as paths.""" - p = enhanced_path(cfg, rid) - return [p] if p.is_file() else [] + return [p for p in (enhanced_path(cfg, rid, k) for k in KINDS) if p.is_file()] def picture(db, cfg, rid, path): @@ -106,6 +126,12 @@ def picture(db, cfg, rid, path): return keyring.working_copy(db, cfg, str(p)) if p.is_file() else path +def sound(db, cfg, rid, path): + """The file a recording is transcribed from: its enhanced sound when the enhance step made it, else its own file.""" + p = enhanced_path(cfg, rid, "audio") + return keyring.working_copy(db, cfg, str(p)) if p.is_file() else path + + def clear(db, cfg, rid): """Forget a resource's enhanced copy: the original is read again.""" for p in files(cfg, rid): @@ -314,11 +340,14 @@ def _said(m): def step(db, cfg, rid, say, spec=None): - """The enhance step: an image that looks low quality gets an enhanced copy, which it's read from.""" - rec = db.one("SELECT space, path, remote, source, enhanced FROM $r", r=R("recording", rid)) or {} + """The enhance step: an image that looks low quality gets an enhanced copy, which it's read from; a recording that + sounds low quality, a copy of its sound, which it's transcribed from.""" + rec = db.one("SELECT space, path, remote, source, enhanced, engine, status, web FROM $r", r=R("recording", rid)) or {} force = bool((spec or {}).get("force")) + if rec.get("source") == "audio": + return _sound_step(db, cfg, rid, rec, say, force) if rec.get("source") != "image": - raise jobs.Skip("only images are enhanced so far") + raise jobs.Skip("only images, audio and video are enhanced") opts = options(cfg) if not (opts["images"] or force): if (rec.get("enhanced") or {}).get("forced"): # someone asked for this one: kept @@ -379,6 +408,55 @@ def step(db, cfg, rid, say, spec=None): say(f"enhanced ({why}): {m['width']}×{m['height']} to {out.width}×{out.height} by {engine}") +def _sound_step(db, cfg, rid, rec, say, force): + """The enhance step on audio or a video's soundtrack (enhance_audio.py).""" + opts = options(cfg) + if not (opts["audio"] or force): + if (rec.get("enhanced") or {}).get("forced"): + raise jobs.Skip("enhancing sound is off (Settings → Enhance); kept the copy made on request") + clear(db, cfg, rid) + raise jobs.Skip("enhancing sound is off (Settings → Enhance)") + if str(rec.get("engine") or "").startswith("import:") and rec.get("status") != "new" and not force: + raise jobs.Skip("the transcript came with the media, so there's nothing to transcribe from a copy") + path = None if (rec.get("web") or {}).get("media") and not rec.get("path") else ingest.audio_path(db, cfg, rec) + if not path: + raise jobs.Skip("its media isn't downloaded yet (the transcribe step downloads it): enhance it on request after") + if not os.path.exists(path): + raise FileNotFoundError(f"its file isn't there ({rec.get('path')})") + m = enhance_audio.measure(path) + now = store.now() + if enhance_audio.silent(m): + clear(db, cfg, rid) + raise jobs.Skip(f"it's silent (its loud half is at {m['level_db']:g} dBFS): nothing to enhance") + low = enhance_audio.assess(m, opts) + if not low and not force: + for p in files(cfg, rid): + p.unlink(missing_ok=True) + _keep(db, rid, {"kind": "audio", "low": [], "measured": m, "enhanced": False, "at": now}) + return say(f"good enough as it is ({enhance_audio.said(m)})") + e = {"kind": "audio", "low": low, "measured": m, "enhanced": True, "forced": force or None, "at": now} + engine = opts["audio_engine"] + if engine == "webrtc" and not enhance_audio.webrtc_installed(): + say("WebRTC's audio processing isn't here yet (it's fetched the first time the step runs with it chosen): enhanced by ffmpeg") + engine = "ffmpeg" + with keyring.scratch(cfg) as d: + out = d / "enhanced.flac" + if engine == "webrtc": + e["speech"] = enhance_audio.run_webrtc(path, out, low, m) + else: + e["filters"] = enhance_audio.run_ffmpeg(path, out, low, m) + keyring.keep(db, cfg, rec["space"], enhanced_path(cfg, rid, "audio"), _chunks(out)) + _keep(db, rid, {**e, "engine": engine}) + why = ", ".join(REASONS[x] for x in low) if low else "on request" + done = f"{', '.join(e['filters'])}" if "filters" in e else f"WebRTC ({round(e['speech'] * 100)}% of it speech)" + say(f"enhanced its sound ({why}; {enhance_audio.said(m)}) by {done}") + + +def _chunks(path, size=1 << 20): + with open(path, "rb") as f: + yield from iter(lambda: f.read(size), b"") + + def _open(path): """An image file as one flat RGB picture (documents.py's way: its EXIF orientation, transparency on white), or None when it has several frames.""" @@ -401,15 +479,15 @@ def _open(path): def queue(db, rid, by=None): - """Enhance an image whatever the settings and its quality, then run what reads it again: the steps of its pipeline - after enhance (the read step, faces, objects, …). The job's id.""" + """Enhance an image or a recording whatever the settings and its quality, then run what reads it again: the steps + of its pipeline after enhance (the read step, faces, objects, …; or transcribe and what follows). The job's id.""" from . import content_types, pipelines rec = db.one("SELECT space, source FROM $r", r=R("recording", rid)) if not rec: raise KeyError(rid) - if rec.get("source") != "image": - raise ValueError("only images are enhanced so far") + if rec.get("source") not in jobs.ENHANCED: + raise ValueError("only images, audio and video are enhanced") steps, _ = pipelines.resolve(db, rec["space"], None, content_types.of_recording(db, rid)[0]) rest = [s for s in steps if (s if isinstance(s, str) else s.get("type")) != "enhance"] return jobs.add_steps(db, rid, [{"type": "enhance", "force": True}, *rest], by) diff --git a/fastapi_backend/app/domain/enhance_audio.py b/fastapi_backend/app/domain/enhance_audio.py new file mode 100644 index 000000000..295a735fc --- /dev/null +++ b/fastapi_backend/app/domain/enhance_audio.py @@ -0,0 +1,297 @@ +"""Enhancing low quality sound before it's transcribed (docs/enhance.md): the enhance step listens to a recording's +audio (or a video's soundtrack) and, when it's noisy, quiet or clipped, makes a cleaned-up copy that the transcribe +step hears instead. The original stays as it is, and so do its times: every filter keeps the copy sample for sample in +step with it, so the words land where they were said. The copy is a rendition, data_dir/renditions/- +enhanced.flac, 16 kHz mono like what transcription reads (encrypted where the namespace's files are). + +Off by default: Settings → Enhance, `enhance.audio`. enhance.py has the settings, the step and what's kept. + +What counts as low quality (each a reason kept in recording.enhanced.low), measured over 30 ms frames: +- `noisy`: speech is less than enhance.min_snr_db above the noise floor (the loudest frames, the 95th percentile, + against the quietest, the 10th). +- `quiet`: the loud half of it is under enhance.min_level_db (dBFS). +- `clipped`: more than CLIPPED of its samples are at full scale. + +The engine (`enhance.audio_engine`): +- `ffmpeg` (the default): ffmpeg's own filters, nothing to fetch. adeclip repairs clipping, a high-pass filter takes + out rumble and hum below 70 Hz, afftdn takes out steady noise (from the floor measured), a quiet one is turned up + towards LEVEL (as far as its peak allows), and dynaudnorm evens out the level, quiet parts more. +- `webrtc`: WebRTC's audio processing (webrtc-noise-gain, Apache-2.0; WebRTC itself is BSD-3-Clause): its noise + suppression, high-pass filter and automatic gain, 10 ms at a time, and its voice detector, which says how much of + it is speech. components.py fetches it the first time it's chosen (Linux on x86-64 and ARM64). + +Both are calibrated once per chain: a probe of clicks and bursts goes through it, and however late it comes out is cut +from the copy's start and added back at its end. Echo cancellation needs what the far end of a call said, which a +recording doesn't have, so neither does that. +""" + +from __future__ import annotations + +import functools +import subprocess +import tempfile + +import numpy as np + +from . import ingest + +SR = ingest.SR # what transcription reads: 16 kHz mono +FRAME = SR * 30 // 1000 # 30 ms +CLIPPED = 0.001 # more than this share of samples at full scale is clipped +SILENT = -70.0 # dBFS: the loudest frames under this are silence, with nothing to enhance +LEVEL = -20.0 # dBFS: what a quiet recording's loud half is brought up to, as far as its peak lets it +HIGHPASS = 70 # Hz +NR = 12 # afftdn's noise reduction, dB; less when there's no noise to speak of +ENGINES = ("ffmpeg", "webrtc") +REASONS = {"noisy": "noisy", "quiet": "quiet", "clipped": "clipped"} +CHUNK = SR * 4 * 10 # read 10 s of float samples at a time +TIMEOUT = ingest.DECODE_SECONDS +_FFMPEG = ["ffmpeg", "-nostdin", "-v", "error"] + + +def _decoder(path, fmt="f32le", af=None): + """ffmpeg reading a file's sound as 16 kHz mono samples on its stdout (errors to a file, so it never stalls).""" + err = tempfile.TemporaryFile() + cmd = [*_FFMPEG, "-format_whitelist", ingest.MEDIA_DEMUXERS, "-i", str(path), "-vn", "-sn", "-dn"] + cmd += ["-af", af] if af else [] + cmd += ["-f", fmt, "-ac", "1", "-ar", str(SR), "-"] + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=err), err + + +def _finish(proc, err): + try: + proc.wait(timeout=TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + raise RuntimeError(f"ffmpeg took longer than {TIMEOUT} s") from None + if proc.returncode: + err.seek(0) + raise RuntimeError("ffmpeg: " + err.read().decode(errors="replace")[-300:]) + err.close() + + +# ---------- how good it sounds ---------- +def measure(path): + """How good a file's sound is: its `duration` (s), `snr_db` (the loudest frames over the quietest), `level_db` (the + loud half, dBFS), `floor_db` (the quietest frames), `peak_db` and `clipped` (the share of samples at full scale). + Read 10 s at a time, so a long recording needs little memory.""" + proc, err = _decoder(path) + levels, rest, n, clipped, peak = [], np.empty(0, np.float32), 0, 0, 0.0 + assert proc.stdout + for chunk in iter(lambda: proc.stdout.read(CHUNK), b""): + x = np.frombuffer(chunk[: len(chunk) // 4 * 4], np.float32) + n += len(x) + a = np.abs(x) + clipped += int(np.count_nonzero(a >= 0.999)) + peak = max(peak, float(a.max(initial=0))) + x = np.concatenate([rest, x]) + k = len(x) // FRAME + if k: + f = x[: k * FRAME].reshape(k, FRAME).astype(np.float64) + levels.append(np.sqrt(np.mean(f * f, axis=1))) + rest = x[k * FRAME :] + _finish(proc, err) + return summarize(np.concatenate(levels) if levels else np.empty(0), n, clipped, peak) + + +def _db(v): + return 20 * np.log10(np.maximum(v, 1e-5)) + + +def summarize(rms, n, clipped, peak): + """measure()'s numbers from each frame's RMS level, how many samples, how many at full scale, and the peak.""" + if not len(rms): + return {"duration": round(n / SR, 2), "snr_db": 0.0, "level_db": -100.0, "floor_db": -100.0, "peak_db": -100.0, "clipped": 0.0} + db = _db(rms) + floor, top = float(np.percentile(db, 10)), float(np.percentile(db, 95)) + loud = rms[db >= floor + (top - floor) / 2] + return { + "duration": round(n / SR, 2), + "snr_db": round(top - floor, 1), + "level_db": round(float(_db(np.sqrt(np.mean(loud * loud)))), 1), + "floor_db": round(floor, 1), + "peak_db": round(float(_db(peak)), 1), + "clipped": round(clipped / max(1, n), 5), + } + + +def silent(m): + return m["level_db"] < SILENT + + +def assess(m, opts): + """The reasons a recording's sound is low quality.""" + low = [] + if m["snr_db"] < opts["min_snr_db"]: + low.append("noisy") + if m["level_db"] < opts["min_level_db"]: + low.append("quiet") + if m["clipped"] > CLIPPED: + low.append("clipped") + return low + + +def said(m): + return f"signal to noise {m['snr_db']:g} dB, level {m['level_db']:g} dBFS, {m['clipped'] * 100:g}% clipped" + + +# ---------- engines ---------- +def filters(low, m): + """ffmpeg's filters for these reasons (none: on request, a gentle clean-up of everything).""" + f = ["adeclip"] if "clipped" in low else [] + f.append(f"highpass=f={HIGHPASS}") + if "noisy" in low or not low: + f.append(f"afftdn=nr={NR if 'noisy' in low else NR // 2}:nf={int(min(-20, max(-80, round(m['floor_db']))))}") + if "quiet" in low: + f.append(f"volume={max(0.0, min(LEVEL - m['level_db'], -1 - m['peak_db'])):.1f}dB") + if "quiet" in low or not low: + f.append("dynaudnorm=f=250:g=31:m=10:p=0.9") + return f + + +def _probe(): + """Two seconds of 16 kHz sound that any of these chains lets through: noise bursts of several levels and a few + clicks, over a faint hiss.""" + rng = np.random.default_rng(7) + x = rng.normal(0, 0.003, 2 * SR).astype(np.float32) + for i, at in enumerate((0.15, 0.4, 0.7, 1.05, 1.3, 1.6)): + a = int(at * SR) + x[a : a + SR // 20] += rng.normal(0, 0.05 * (1 + i % 3), SR // 20).astype(np.float32) + for at in (0.55, 1.2, 1.85): + x[int(at * SR)] = 0.8 + return x + + +def lag(x, y, most=SR // 4): + """How many samples late `y` comes out against `x` (from -most to most), by cross-correlation.""" + n = min(len(x), len(y)) + a, b = x[:n] - x[:n].mean(), y[:n] - y[:n].mean() + c = np.fft.irfft(np.fft.rfft(b, 2 * n) * np.conj(np.fft.rfft(a, 2 * n))) + c = np.concatenate([c[-most:], c[: most + 1]]) + return int(np.argmax(c)) - most + + +def _wav(x): + import io + import wave + + out = io.BytesIO() + with wave.open(out, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(SR) + w.writeframes((np.clip(x, -1, 1) * 32767).astype(np.int16).tobytes()) + return out.getvalue() + + +@functools.lru_cache(maxsize=32) +def ffmpeg_delay(chain): + """How many samples late this chain of ffmpeg filters puts its output (afftdn's analysis window, say).""" + x = _probe() + out = subprocess.run( + [*_FFMPEG, "-f", "wav", "-i", "-", "-af", chain, "-f", "f32le", "-ac", "1", "-ar", str(SR), "-"], + input=_wav(x), + capture_output=True, + timeout=60, + ) + if out.returncode: + raise RuntimeError("ffmpeg: " + out.stderr.decode(errors="replace")[-300:]) + return max(0, lag(x, np.frombuffer(out.stdout, np.float32))) + + +def run_ffmpeg(src, dst, low, m): + """Write the copy with ffmpeg's filters: FLAC, 16 kHz mono, in step with the original. The filters it ran.""" + chain = filters(low, m) + af = ",".join(chain) + late = ffmpeg_delay(af) + full = f"aresample={SR},aformat=channel_layouts=mono,{af}" + (f",atrim=start_sample={late},apad=pad_len={late}" if late else "") + out = subprocess.run( + [*_FFMPEG, "-y", "-format_whitelist", ingest.MEDIA_DEMUXERS, "-i", str(src), "-vn", "-sn", "-dn", "-af", full] + + ["-ac", "1", "-ar", str(SR), "-c:a", "flac", "-sample_fmt", "s16", "-f", "flac", str(dst)], + capture_output=True, + timeout=TIMEOUT, + ) + if out.returncode: + raise RuntimeError("ffmpeg: " + out.stderr.decode(errors="replace")[-300:]) + return chain + + +def webrtc_installed(): + try: + import webrtc_noise_gain # noqa: F401 + except ImportError: + return False + return True + + +def _webrtc_settings(low): + """(automatic gain's target, dB under full scale, or 0 for none; noise suppression from 0 to 4).""" + return (15 if "quiet" in low else 0), (3 if "noisy" in low or not low else 1) + + +def _processor(low): + from webrtc_noise_gain import AudioProcessor + + return AudioProcessor(*_webrtc_settings(low)) + + +def _through(proc, pcm): + """16-bit samples through a WebRTC processor 10 ms at a time (the last part padded with silence, then cut back): + (what came out, how many 10 ms frames were speech, how many frames).""" + out, speech, frames, size = bytearray(), 0, 0, SR // 100 * 2 + for i in range(0, len(pcm), size): + part = pcm[i : i + size] + got = proc.Process10ms(part + bytes(size - len(part))) + out += got.audio[: len(part)] + speech += bool(got.is_speech) + frames += 1 + return bytes(out), speech, frames + + +@functools.lru_cache(maxsize=8) +def webrtc_delay(gain, ns): + from webrtc_noise_gain import AudioProcessor + + x = _probe() + pcm = (np.clip(x, -1, 1) * 32767).astype(np.int16).tobytes() + y, _, _ = _through(AudioProcessor(gain, ns), pcm) + return max(0, lag(x, np.frombuffer(y, np.int16).astype(np.float32) / 32768)) + + +def run_webrtc(src, dst, low, m): + """Write the copy through WebRTC's audio processing (adeclip first, where it's clipped), in step with the original. + The share of it that's speech.""" + proc, err = _decoder(src, fmt="s16le", af="adeclip" if "clipped" in low else None) + gain, ns = _webrtc_settings(low) + late = webrtc_delay(gain, ns) * 2 # bytes + enc = subprocess.Popen( + [*_FFMPEG, "-y", "-f", "s16le", "-ar", str(SR), "-ac", "1", "-i", "-", "-c:a", "flac", "-f", "flac", str(dst)], + stdin=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + assert proc.stdout and enc.stdin + wr, speech, frames, skip, carry = _processor(low), 0, 0, late, b"" + try: + for chunk in iter(lambda: proc.stdout.read(CHUNK // 2), b""): + chunk = carry + chunk + whole = len(chunk) // (SR // 100 * 2) * (SR // 100 * 2) + chunk, carry = chunk[:whole], chunk[whole:] + y, s, f = _through(wr, chunk) + speech, frames = speech + s, frames + f + cut = min(skip, len(y)) + skip -= cut + enc.stdin.write(y[cut:]) + if carry: + y, s, f = _through(wr, carry) + speech, frames = speech + s, frames + f + enc.stdin.write(y[min(skip, len(y)) :]) + enc.stdin.write(bytes(late - skip)) # what was cut from the start, as silence at the end + enc.stdin.close() + except BaseException: + enc.kill() + proc.kill() + raise + _finish(proc, err) + if enc.wait(timeout=TIMEOUT): + raise RuntimeError("ffmpeg couldn't write the enhanced copy") + return round(speech / max(1, frames), 3) diff --git a/fastapi_backend/app/domain/ingest.py b/fastapi_backend/app/domain/ingest.py index 303257ef2..669aa4181 100644 --- a/fastapi_backend/app/domain/ingest.py +++ b/fastapi_backend/app/domain/ingest.py @@ -501,10 +501,18 @@ def add_envelope(db, cfg, rid): def transcribe_one(db, cfg, rid, log=print, engine=None): r = db.one("SELECT record::id(id) AS id, space, path, title, remote FROM $r", r=store.R("recording", rid)) + from . import enhance + engine, t = engine or get_engine(cfg, log), time.time() - audio = decode(audio_path(db, cfg, r)) + own = audio_path(db, cfg, r) + heard = enhance.sound(db, cfg, rid, own) # its enhanced sound, where the enhance step made one + audio = decode(heard) words = vocabulary.terms(db, cfg, r["space"]) segs = engine.transcribe(audio, terms=words) if words else engine.transcribe(audio) # an engine of a plugin may take no hints + if heard != own: + log("transcribed from its enhanced sound") + del audio + audio = decode(own) # the waveform is the original's: what the player plays env = envelope(audio) del audio write_transcript( diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index c0016071f..21554272d 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -56,7 +56,8 @@ def _spec(s): def _enhance(db, cfg, rid, say, spec=None): - """A low quality image's enhanced copy, which the steps after it read (enhance.py).""" + """A low quality image's enhanced copy, which the steps after it read, or a recording's enhanced sound, which it's + transcribed from (enhance.py).""" from . import enhance enhance.step(db, cfg, rid, say, spec) @@ -258,15 +259,15 @@ def _describe(db, cfg, rid, say, spec=None): # The sources the enhance step works on (docs/enhance.md). -ENHANCED = {"image"} +ENHANCED = {"image", "audio"} def for_source(steps, source): """A pipeline's steps as they run on a resource of this source: a document or an image is read, not transcribed, and has no voices to tell apart, so a speech or voice model is never fetched for it. An imported transcript has nothing to transcribe and its speakers came with it, so it skips both steps too (unless they're all that was asked - for: those still run, to say why there's nothing to do). Only images are enhanced so far, so the others skip enhance - the same way.""" + for: those still run, to say why there's nothing to do). Only images and media (audio and video) are enhanced, so + the others skip enhance the same way.""" if source not in ENHANCED: steps = [s for s in steps if s.get("type") != "enhance"] or steps if source == "transcript": diff --git a/fastapi_backend/app/domain/keyring.py b/fastapi_backend/app/domain/keyring.py index c3ee69328..b8e3db1d3 100644 --- a/fastapi_backend/app/domain/keyring.py +++ b/fastapi_backend/app/domain/keyring.py @@ -33,6 +33,7 @@ import tempfile import threading import time +from collections.abc import Iterable from . import settings, store @@ -601,20 +602,22 @@ def protect(db, cfg, sid, path) -> bool: return True -def keep(db, cfg, sid, path, data: bytes | str) -> pathlib.Path: +def keep(db, cfg, sid, path, data: bytes | str | Iterable[bytes]) -> pathlib.Path: """Write a file Lens makes (a rendition, a report, an export) in one go: encrypted when wanted(), never plain on - disk on the way; else as it is. Either way it replaces `path` atomically.""" + disk on the way; else as it is. Either way it replaces `path` atomically. `data` may come in chunks.""" p = pathlib.Path(path) - raw = data.encode("utf-8") if isinstance(data, str) else data + chunks = [data.encode("utf-8") if isinstance(data, str) else data] if isinstance(data, (str, bytes)) else data p.parent.mkdir(parents=True, exist_ok=True) if wanted(db, cfg, sid, p): with Writer(db, cfg, sid, p) as w: - w.write(raw) + for c in chunks: + w.write(c) return p fd, tmp = tempfile.mkstemp(dir=p.parent, prefix=".keep-") try: with os.fdopen(fd, "wb") as f: - f.write(raw) + for c in chunks: + f.write(c) os.replace(tmp, p) except BaseException: with contextlib.suppress(OSError): diff --git a/fastapi_backend/app/domain/pipelines.py b/fastapi_backend/app/domain/pipelines.py index cd8a9cb8a..3ee495913 100644 --- a/fastapi_backend/app/domain/pipelines.py +++ b/fastapi_backend/app/domain/pipelines.py @@ -1,8 +1,9 @@ """Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without -one, recordings get the standard pipeline. Steps: enhance (an enhanced copy of a low quality image, enhance.py, which -the steps after it read), transcribe (speech to text), read (a document's or an -image's pages and their text), convert (new assets made from one: a video's soundtrack as audio, a document's pages -as images, conversions.py), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), describe, +one, recordings get the standard pipeline. Steps: enhance (an enhanced copy of a low quality image or of a +recording's sound, enhance.py, which the steps after it read or transcribe), transcribe (speech to text), read (a +document's or an image's pages and their text), convert (new assets made from one: a video's soundtrack as audio, a +document's pages as images, conversions.py), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), +describe, clean (a transcript tidied before analysis, cleanup.py), analyze, embed (passages for search by meaning, semantic.py), summarize, llm (a prompt template whose structured result is saved as a named output), spec (a technical spec from the transcript, written to a note linked to the recording, specs.py), report (built in, or from a report template) and diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 9214f69de..b862a3531 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -178,6 +178,7 @@ ("video", "face_engine"): {"opencv", "insightface", "anytopdf", "none"}, ("video", "object_engine"): {"yolox", "ultralytics", "anytopdf", "off"}, ("enhance", "image_engine"): {"classic", "realesrgan"}, + ("enhance", "audio_engine"): {"ffmpeg", "webrtc"}, ("decisions", "engine"): {"auto", "jev", "laya", "llm", "off"}, ("voice", "input"): {"auto", "server", "browser"}, ("voice", "stt"): {"same", "sensevoice", "whisper", "mlx-whisper", "openai", "elevenlabs", "assemblyai", "deepgram"}, @@ -581,7 +582,7 @@ def _check(section, key, value, default): return _component_setting(key, value) if section == "nsfw": return _nsfw_setting(key, value) - if section == "enhance" and key != "image_engine": + if section == "enhance" and key not in ("image_engine", "audio_engine"): from . import enhance return enhance.check(key, value) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 783df8996..72429e290 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -561,6 +561,10 @@ def labels(): "min_sharpness": 100, "min_face_px": 64, "max_scale": 4, + "audio": False, + "audio_engine": "ffmpeg", + "min_snr_db": 15, + "min_level_db": -35, }, # rclone and local_roots are bootstrap-only on purpose: the web app must not be able to pick an executable # or open up arbitrary folders on the server. Local folders can only be watched inside local_roots. diff --git a/fastapi_backend/app/schemas/recordings.py b/fastapi_backend/app/schemas/recordings.py index b3934a246..38fa38fbd 100644 --- a/fastapi_backend/app/schemas/recordings.py +++ b/fastapi_backend/app/schemas/recordings.py @@ -202,20 +202,35 @@ class EnhanceMeasured(ResponseModel): smallest_face: int | None = Field(None, description="the height of its smallest face, in pixels") +class EnhanceSoundMeasured(ResponseModel): + """How good a recording sounded to the enhance step (docs/enhance.md), over 30 ms frames of it at 16 kHz mono.""" + + duration: float = Field(description="seconds") + snr_db: float = Field(description="signal to noise: its loudest frames over its quietest, in dB") + level_db: float = Field(description="the level of its loud half, dBFS") + floor_db: float = Field(description="its noise floor (its quietest frames), dBFS") + peak_db: float = Field(description="its peak, dBFS") + clipped: float = Field(description="the share of its samples at full scale (0 to 1)") + + class Enhanced(ResponseModel): - """What the enhance step found and did (docs/enhance.md): whether the picture looked low quality and why, and the - enhanced copy it's read from, if one was made. The original stays as it is.""" + """What the enhance step found and did (docs/enhance.md): whether the picture or the sound was low quality and why, + and the enhanced copy it's read or transcribed from, if one was made. The original stays as it is.""" - kind: Literal["image"] - low: list[Literal["small", "blurry", "noisy", "dull", "small_faces"]] = Field( - default_factory=list, description="why it looked low quality; empty when it looked good enough" + kind: Literal["image", "audio"] + low: list[Literal["small", "blurry", "noisy", "dull", "small_faces", "quiet", "clipped"]] = Field( + default_factory=list, description="why it looked or sounded low quality; empty when it was good enough" ) - measured: EnhanceMeasured - enhanced: bool = Field(description="an enhanced copy was made, and the steps after it read that") - engine: str | None = Field(None, description="what enhanced it: classic, or realesr-general-x4v3 (Real-ESRGAN)") - scale: float | None = Field(None, description="how many times bigger the copy is") - width: int | None = Field(None, description="the copy's width in pixels") + measured: EnhanceMeasured | EnhanceSoundMeasured + enhanced: bool = Field(description="an enhanced copy was made, and the steps after it read or transcribe that") + engine: str | None = Field( + None, description="what enhanced it: classic or realesr-general-x4v3 (Real-ESRGAN) for an image, ffmpeg or webrtc for sound" + ) + scale: float | None = Field(None, description="how many times bigger an image's copy is") + width: int | None = Field(None, description="an image's copy's width in pixels") height: int | None = None + filters: list[str] | None = Field(None, description="the ffmpeg filters that made a sound's copy") + speech: float | None = Field(None, description="the share of the sound WebRTC's voice detector heard as speech (0 to 1)") forced: bool | None = Field(None, description="enhanced on request, whatever the settings and its quality") at: str | None = None download: str | None = Field(None, description="a signed link to save the enhanced copy") @@ -327,7 +342,9 @@ class Recording(ResponseModel): web: WebPage | None = Field(None, description="a web page captured as a document") enrichment: Enrichment | None = Field(None, description="the places and dates anytopdf found (analysis.anytopdf)") nsfw: NsfwState | None = Field(None, description="NSFW classification, where it's on in its namespace and it was classified") - enhanced: Enhanced | None = Field(None, description="an image the enhance step looked at: what it found, and its enhanced copy") + enhanced: Enhanced | None = Field( + None, description="an image or a recording the enhance step looked at: what it found, and its enhanced copy" + ) class Page(ResponseModel): diff --git a/fastapi_backend/pyproject.toml b/fastapi_backend/pyproject.toml index c3840eef0..93c23cbdc 100644 --- a/fastapi_backend/pyproject.toml +++ b/fastapi_backend/pyproject.toml @@ -72,6 +72,9 @@ web = ["ddgs>=9.16,<10"] # auto scaling onto AWS EC2 (scaling_aws.py): botocore (Apache-2.0); fetched when the first AWS provider is saved in # Settings › Auto scaling aws = ["botocore>=1.43.113"] +# WebRTC's noise suppression and automatic gain for enhancing sound (enhance.audio_engine "webrtc"): webrtc-noise-gain +# (Apache-2.0), wheels for Linux on x86-64 and ARM64 only +webrtc = ["webrtc-noise-gain>=1.3,<2; sys_platform == 'linux'"] # Outlook .msg emails (extract-msg is GPL-3.0; .eml needs nothing) msg = ["extract-msg"] # secrets read from a Vaultwarden or Bitwarden vault (vaultwarden_secrets.py): python-vaultwarden (Apache-2.0); fetched diff --git a/fastapi_backend/tests/api/test_enhance.py b/fastapi_backend/tests/api/test_enhance.py index 592e29b8b..22b1bb863 100644 --- a/fastapi_backend/tests/api/test_enhance.py +++ b/fastapi_backend/tests/api/test_enhance.py @@ -68,6 +68,10 @@ def test_off_by_default_the_original_is_read(client, env, db, cfg): "min_sharpness": 100, "min_face_px": 64, "max_scale": 4, + "audio": False, + "audio_engine": "ffmpeg", + "min_snr_db": 15, + "min_level_db": -35, } assert store.DEFAULTS["workers"]["steps"][0] == "enhance" and jobs.PIPELINE[0] == "enhance" rid = _photo(client, env["he"], _scene(160, 120).filter(ImageFilter.GaussianBlur(3))) @@ -137,7 +141,7 @@ def test_asked_for_it_is_enhanced_whatever_the_settings_and_kept(client, env, db drain(db, cfg) assert "kept the copy made on request" in _log(db, rid) and enhance.files(cfg, rid) - # documents and recordings aren't enhanced + # documents aren't enhanced doc = _upload(client, he, b"%PDF-1.4\n", "x.pdf")["recording"] r = client.post(f"/api/v1/recordings/{doc}/enhance", headers=he) assert r.status_code == 400 and "only images" in r.json()["detail"] diff --git a/fastapi_backend/tests/api/test_enhance_audio.py b/fastapi_backend/tests/api/test_enhance_audio.py new file mode 100644 index 000000000..8af4135f8 --- /dev/null +++ b/fastapi_backend/tests/api/test_enhance_audio.py @@ -0,0 +1,300 @@ +"""Enhancing low quality sound before it's transcribed (docs/enhance.md): off by default; turned on, the enhance step +listens to a recording and, when it's noisy, quiet or clipped, makes a cleaned-up copy (FLAC, 16 kHz mono) that the +transcribe step hears instead, keeping the original and its times: every sample of the copy stays where it was, so +words land where they were said. Asked for, it enhances whatever the settings. WebRTC's processing, where it's chosen +and installed, runs 10 ms at a time and is put back in step the same way.""" + +from __future__ import annotations + +import io +import subprocess +import sys +import types +import wave + +import numpy as np +import pytest + +from app.api import mcp_tools +from app.domain import components, enhance, enhance_audio, ingest, keyring, machine, settings, store +from tests.api.test_documents import _upload +from tests.api.test_mcp import tool +from tests.helpers import drain, login, make_user + +R = store.R +SR = enhance_audio.SR + + +@pytest.fixture +def env(client, db): + make_user(db, "root@x.io", "root password 1", admin=True) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"}) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + return { + "ha": login(client, "root@x.io", "root password 1"), + "he": login(client, "ed@x.io", "editor password 1"), + "hv": login(client, "vi@x.io", "viewer password 1"), + } + + +class Heard: + """A transcription engine that keeps what it was given.""" + + name = "sensevoice" + + def __init__(self): + self.got = [] + + def transcribe(self, audio): + self.got.append(np.array(audio)) + return [{"t0": 0, "t1": 1000, "text": "hello there", "lang": "en"}] + + +@pytest.fixture +def heard(monkeypatch): + e = Heard() + monkeypatch.setattr(ingest, "get_engine", lambda cfg, log=None: e) + return e + + +def on(cfg, **more): + return {**cfg, "enhance": {**cfg["enhance"], "audio": True, **more}, "diarize": {**cfg["diarize"], "engine": "none"}} + + +def _speech(seconds, level_db=-20.0, seed=1): + """Something like speech: a voice's harmonics in syllables four times a second, with pauses.""" + t = np.arange(int(seconds * SR)) / SR + phase = 2 * np.pi * np.cumsum(120 + 20 * np.sin(2 * np.pi * 0.5 * t)) / SR + x = sum(np.sin(k * phase) / k for k in range(1, 25)) + env = np.clip(np.sin(2 * np.pi * 4 * t), 0, 1) ** 2 * (np.sin(2 * np.pi * 0.25 * t) > -0.3) + x = x * env + return (x / np.sqrt(np.mean(x[env > 0.1] ** 2)) * 10 ** (level_db / 20)).astype(np.float32) + + +def _noise(n, level_db, seed=2): + return (np.random.default_rng(seed).normal(0, 1, n) * 10 ** (level_db / 20)).astype(np.float32) + + +def _wav(x, rate=SR): + out = io.BytesIO() + with wave.open(out, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes((np.clip(x, -1, 1) * 32767).astype(np.int16).tobytes()) + return out.getvalue() + + +def _clicked(x, *at): + x = x.copy() + for i, t in enumerate(at): + x[int(t * SR)] = 0.9 if i % 2 == 0 else -0.9 + return x + + +def _log(db, rid): + return " ".join(x for j in db.rows("SELECT log FROM job WHERE recording = $r", r=rid) for x in j["log"]) + + +def _decode(data): + out = subprocess.run( + ["ffmpeg", "-v", "error", "-i", "-", "-f", "f32le", "-ac", "1", "-ar", str(SR), "-"], input=data, capture_output=True, check=True + ) + return np.frombuffer(out.stdout, np.float32) + + +def _record(client, h, x, name="call.wav"): + return _upload(client, h, _wav(x), name)["recording"] + + +def test_off_by_default_the_original_is_heard(client, env, db, cfg, heard): + noisy = _speech(6) + _noise(6 * SR, -24) + rid = _record(client, env["he"], noisy) + drain(db, {**cfg, "diarize": {**cfg["diarize"], "engine": "none"}}) + assert "enhance skipped: enhancing sound is off (Settings → Enhance)" in _log(db, rid) + assert len(heard.got) == 1 and not enhance.files(cfg, rid) + assert client.get(f"/api/v1/recordings/{rid}", headers=env["hv"]).json()["enhanced"] is None + assert not components.BY_ID["enhance-sound"].wanted_by(db, cfg, machine.probe(cfg["data_dir"])) + + +def test_noisy_sound_is_transcribed_from_a_copy_in_step_with_it(client, env, db, cfg, heard): + x = _clicked(_speech(12) + _noise(12 * SR, -30), 3.3, 9.1) + rid = _record(client, env["he"], x) + drain(db, on(cfg)) + log = _log(db, rid) + assert "enhanced its sound (noisy; signal to noise" in log and "afftdn=nr=12" in log + assert "transcribed from its enhanced sound" in log + e = db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"] + assert (e["kind"], e["low"], e["engine"], e["enhanced"]) == ("audio", ["noisy"], "ffmpeg", True) + assert e["measured"]["snr_db"] < 15 and e["measured"]["duration"] == 12.0 + assert e["filters"][0] == "highpass=f=70" and e["filters"][1].startswith("afftdn") + + # what was transcribed is the copy: as long as the original, less noisy, and its clicks where they were + got = heard.got[-1] + assert len(got) == len(x) + before, after = enhance_audio.summarize(*_levels(x)), enhance_audio.summarize(*_levels(got)) + assert after["floor_db"] < before["floor_db"] - 6 and after["snr_db"] > before["snr_db"] + 6 + assert enhance_audio.lag(x, got) == 0 + + # the waveform is still the original's, and the original is kept as it was + env_ = db.one("SELECT envelope FROM $r", r=R("recording", rid))["envelope"] + assert env_ == ingest.envelope(ingest.decode(ingest.audio_path(db, cfg, db.one("SELECT * FROM $r", r=R("recording", rid))))) + + api = client.get(f"/api/v1/recordings/{rid}", headers=env["hv"]).json()["enhanced"] + assert api["kind"] == "audio" and api["low"] == ["noisy"] and api["filters"] == e["filters"] + r = client.get(api["download"]) + assert r.status_code == 200 and r.headers["content-type"] == "audio/flac" + assert 'filename="call-enhanced.flac"' in r.headers["content-disposition"] + assert len(_decode(r.content)) == len(x) + assert client.get(f"/api/v1/recordings/{rid}/enhanced", headers=env["hv"]).status_code == 200 + + +def _levels(x): + k = len(x) // enhance_audio.FRAME + f = x[: k * enhance_audio.FRAME].reshape(k, -1).astype(np.float64) + return np.sqrt(np.mean(f * f, axis=1)), len(x), int(np.count_nonzero(np.abs(x) >= 0.999)), float(np.abs(x).max()) + + +def test_quiet_and_clipped_sound(client, env, db, cfg, heard): + quiet = _speech(8, level_db=-48) + _noise(8 * SR, -90) + rid = _record(client, env["he"], quiet) + drain(db, on(cfg)) + e = db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"] + assert e["low"] == ["quiet"] and e["filters"][1].startswith("volume=") and e["filters"][2].startswith("dynaudnorm") + assert enhance_audio.summarize(*_levels(heard.got[-1]))["level_db"] > e["measured"]["level_db"] + 10 + + clipped = np.clip(_speech(8, level_db=-6) * 3, -1, 1) + _noise(8 * SR, -60) + rid = _record(client, env["he"], clipped, "loud.wav") + drain(db, on(cfg)) + e = db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"] + assert "clipped" in e["low"] and e["filters"][0] == "adeclip" and e["measured"]["clipped"] > enhance_audio.CLIPPED + + +def test_good_or_silent_sound_is_left_as_it_is(client, env, db, cfg, heard): + rid = _record(client, env["he"], _speech(6) + _noise(6 * SR, -70)) + drain(db, on(cfg)) + assert "good enough as it is (signal to noise" in _log(db, rid) + e = db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"] + assert e["enhanced"] is False and e["low"] == [] and not enhance.files(cfg, rid) + assert "transcribed from its enhanced sound" not in _log(db, rid) + + rid = _record(client, env["he"], np.zeros(3 * SR, np.float32), "hush.wav") + drain(db, on(cfg)) + assert "it's silent" in _log(db, rid) + + +def test_asked_for_it_is_enhanced_and_transcribed_again(client, env, db, cfg, heard, app): + he, hv = env["he"], env["hv"] + rid = _record(client, he, _speech(6) + _noise(6 * SR, -70)) + drain(db, {**cfg, "diarize": {**cfg["diarize"], "engine": "none"}}) + assert client.post(f"/api/v1/recordings/{rid}/enhance", headers=hv).status_code == 403 + job = client.post(f"/api/v1/recordings/{rid}/enhance", headers=he).json()["job"] + steps = [s["type"] for s in db.one("SELECT steps FROM $r", r=R("job", job))["steps"]] + assert steps[:2] == ["enhance", "transcribe"] and "enhance" not in steps[1:] + drain(db, {**cfg, "diarize": {**cfg["diarize"], "engine": "none"}}) + e = db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"] + assert e["forced"] is True and e["enhanced"] is True and e["low"] == [] + assert "enhanced its sound (on request" in _log(db, rid) and "transcribed from its enhanced sound" in _log(db, rid) + # on request with nothing wrong: a gentle clean-up of everything + assert [f.split("=")[0] for f in e["filters"]] == ["highpass", "afftdn", "dynaudnorm"] and "nr=6" in e["filters"][1] + + # agents too + assert "audio or video" in mcp_tools.TOOLS["enhance"].description + got = tool(client, he, "enhance", recording_id=rid) + assert got["job"] + # a transcript has no sound to enhance + t = client.post("/api/v1/import", headers=he, json={"namespace": "pods", "text": "[00:01] Ann: Hello."}).json() + r = client.post(f"/api/v1/recordings/{t['id']}/enhance", headers=he) + assert r.status_code == 400 and "only images, audio and video" in r.json()["detail"] + + +def test_the_copy_is_encrypted_where_the_files_are(client, env, db, cfg, heard): + eff = settings.effective(db, cfg) + x = _speech(5) + _noise(5 * SR, -28) + rid = _record(client, env["he"], x) + drain(db, on(eff)) + path = enhance.enhanced_path(cfg, rid, "audio") + assert keyring.is_encrypted(path) and str(path) in list(keyring.recording_files(db, cfg, rid)) + r = client.get(f"/api/v1/recordings/{rid}/enhanced", headers=env["he"]) + assert r.status_code == 200 and len(_decode(r.content)) == len(x) + assert client.delete(f"/api/v1/recordings/{rid}", headers=env["ha"]).status_code == 200 + assert not path.exists() + + +def test_measuring_and_deciding(): + opts = {"min_snr_db": 15, "min_level_db": -35} + m = enhance_audio.summarize(*_levels(_speech(4) + _noise(4 * SR, -26))) + assert enhance_audio.assess(m, opts) == ["noisy"] and 3 < m["snr_db"] < 15 + m = enhance_audio.summarize(*_levels(_speech(4) + _noise(4 * SR, -70))) + assert enhance_audio.assess(m, opts) == [] and m["snr_db"] > 30 and -24 < m["level_db"] < -16 + assert enhance_audio.summarize(np.empty(0), 0, 0, 0.0)["level_db"] == -100.0 + assert enhance_audio.filters(["clipped", "noisy", "quiet"], {"floor_db": -95.0, "level_db": -50.0, "peak_db": -12.0}) == [ + "adeclip", + "highpass=f=70", + "afftdn=nr=12:nf=-80", + "volume=11.0dB", # as far as its peak allows + "dynaudnorm=f=250:g=31:m=10:p=0.9", + ] + x = _noise(SR, -20) + assert enhance_audio.lag(x, np.concatenate([np.zeros(37, np.float32), x[:-37]])) == 37 + + +class FakeWebrtc: + """webrtc_noise_gain's AudioProcessor, as far as Lens uses it: 10 ms in, 10 ms out, a frame late (as WebRTC's own + noise suppression is), at half the level; speech where it's loud.""" + + made: list = [] + + def __init__(self, gain, ns): + FakeWebrtc.made.append((gain, ns)) + self.held = bytes(320) + + def Process10ms(self, chunk): + assert len(chunk) == 320 + out, self.held = self.held, chunk + y = (np.frombuffer(out, np.int16) // 2).astype(np.int16).tobytes() + return types.SimpleNamespace(audio=y, is_speech=int(np.abs(np.frombuffer(chunk, np.int16)).max() > 2000)) + + +def test_webrtc_where_its_chosen(client, env, db, cfg, heard, monkeypatch): + m = machine.probe(cfg["data_dir"]) + c = components.BY_ID["enhance-sound"] + assert c.needed(on(cfg, audio_engine="webrtc"), {**m, "os": "linux", "arch": "arm64"}) + assert not c.needed(on(cfg, audio_engine="webrtc"), {**m, "os": "darwin", "arch": "arm64"}) + assert not c.needed(on(cfg), {**m, "os": "linux", "arch": "x86_64"}) + + x = _clicked(_speech(6.03) + _noise(int(6.03 * SR), -30), 1.5, 4.25) # not a whole number of 10 ms frames + # not installed yet: ffmpeg does it, saying why + monkeypatch.setattr(enhance_audio, "webrtc_installed", lambda: False) + rid = _record(client, env["he"], x) + drain(db, on(cfg, audio_engine="webrtc")) + assert "WebRTC's audio processing isn't here yet" in _log(db, rid) + assert db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"]["engine"] == "ffmpeg" + + monkeypatch.setattr(enhance_audio, "webrtc_installed", lambda: True) + monkeypatch.setitem(sys.modules, "webrtc_noise_gain", types.SimpleNamespace(AudioProcessor=FakeWebrtc)) + enhance_audio.webrtc_delay.cache_clear() + job = client.post(f"/api/v1/recordings/{rid}/enhance", headers=env["he"]).json()["job"] + assert job + drain(db, on(cfg, audio_engine="webrtc")) + e = db.one("SELECT enhanced FROM $r", r=R("recording", rid))["enhanced"] + assert e["engine"] == "webrtc" and 0 < e["speech"] < 1 and "filters" not in e + assert FakeWebrtc.made[-1] == (0, 3) and enhance_audio.webrtc_delay(0, 3) == 160 + got = heard.got[-1] + assert len(got) == len(x) and enhance_audio.lag(x, got) == 0 # put back in step + assert "by WebRTC (" in _log(db, rid) + enhance_audio.webrtc_delay.cache_clear() + + +def test_sound_settings_are_checked(db, cfg): + settings.save(db, cfg, "enhance", {"audio": True, "audio_engine": "webrtc", "min_snr_db": 20, "min_level_db": -40}) + eff = settings.effective(db, cfg)["enhance"] + assert (eff["audio"], eff["audio_engine"], eff["min_snr_db"], eff["min_level_db"]) == (True, "webrtc", 20, -40) + for bad, said in ( + ({"audio_engine": "rnnoise"}, "enhance.audio_engine must be one of"), + ({"min_snr_db": 90}, "enhance.min_snr_db is a number from 0 to 60"), + ({"min_level_db": 3}, "enhance.min_level_db is a number from -70 to 0"), + ({"audio": 1}, "enhance.audio is true or false"), + ): + with pytest.raises(ValueError, match=said): + settings.save(db, cfg, "enhance", bad) diff --git a/fastapi_backend/tests/api/test_jobs.py b/fastapi_backend/tests/api/test_jobs.py index 528b5ef85..b0ebb6a99 100644 --- a/fastapi_backend/tests/api/test_jobs.py +++ b/fastapi_backend/tests/api/test_jobs.py @@ -398,8 +398,9 @@ def test_a_document_is_read_not_transcribed(db, cfg, folder): {"type": "read"}, {"type": "report"}, ] - # only images are enhanced, so the others drop the step unless it's all that was asked for - assert jobs.for_source([{"type": "enhance"}, {"type": "transcribe"}], "audio") == [{"type": "transcribe"}] + # only images and media are enhanced, so the others drop the step unless it's all that was asked for + assert jobs.for_source([{"type": "enhance"}, {"type": "transcribe"}], "audio") == [{"type": "enhance"}, {"type": "transcribe"}] + assert jobs.for_source([{"type": "enhance"}, {"type": "analyze"}], "transcript") == [{"type": "analyze"}] assert jobs.for_source([{"type": "enhance"}, {"type": "transcribe"}], "image") == [{"type": "enhance"}, {"type": "read"}] assert jobs.for_source([{"type": "enhance"}], "document") == [{"type": "enhance"}] # a worker whose steps list predates the read step runs it too diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index 3933de366..47c32cc91 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -2068,6 +2068,9 @@ voices = [ web = [ { name = "ddgs" }, ] +webrtc = [ + { name = "webrtc-noise-gain", marker = "sys_platform == 'linux'" }, +] whisper = [ { name = "faster-whisper" }, ] @@ -2156,8 +2159,9 @@ requires-dist = [ { name = "transformers", marker = "extra == 'moondream'" }, { name = "transformers", marker = "extra == 'punctuation'" }, { name = "webauthn", specifier = ">=3.0.1,<4" }, + { name = "webrtc-noise-gain", marker = "sys_platform == 'linux' and extra == 'webrtc'", specifier = ">=1.3,<2" }, ] -provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "aws", "msg", "vaultwarden"] +provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "moondream", "validate", "smb", "sandbox", "web", "aws", "webrtc", "msg", "vaultwarden"] [package.metadata.requires-dev] dev = [ @@ -5845,6 +5849,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/c6/040cbc72480d789a5f40d63fb484d3106554c4dfa2d2b70ad5022057750f/webencodings-0.6.1-py3-none-any.whl", hash = "sha256:7fab6269c8bf237c657876b52058ccb182e861518d1c695c1a9aaa8c1c105d5b", size = 8745, upload-time = "2026-08-15T14:22:56.31Z" }, ] +[[package]] +name = "webrtc-noise-gain" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/70/675fc3b28b7212f32959dd222bce6e753f235a9fcccc924826e135302e66/webrtc_noise_gain-1.3.0.tar.gz", hash = "sha256:67c96337db3b074a7a5dce0936c98a93886f22a0724e4387198ae86f31914643", size = 3073970, upload-time = "2026-05-28T18:24:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e2/711f829859a869aeda2dd58bc9eed32261418c5d1942b2d7929056fe74a0/webrtc_noise_gain-1.3.0-cp39-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7df808439bae994459be7e916802e384ffa4f0e750035b5284f7fe4ff466dc", size = 8803028, upload-time = "2026-05-28T18:24:52.933Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0f/b475e7c58de013d634df63da60e3d9a81601d8416c212739ab0eeb21e478/webrtc_noise_gain-1.3.0-cp39-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d052501c2cbe4948101e011450e93bff22b800f89014f192dfb861e319522a3", size = 8949702, upload-time = "2026-05-28T18:24:55.519Z" }, +] + [[package]] name = "websockets" version = "17.1" diff --git a/mkdocs.yml b/mkdocs.yml index 6568a3315..3556c3ab5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -90,7 +90,7 @@ nav: - Processing: processing.md - Video: video.md - NSFW classification: nsfw.md - - Enhancing images: enhance.md + - Enhancing images and sound: enhance.md - Notifications: notifications.md - Automations: automations.md - Schedules: schedules.md diff --git a/nextjs-frontend/__tests__/enhance.test.tsx b/nextjs-frontend/__tests__/enhance.test.tsx index b4fc78dd3..f74b754fa 100644 --- a/nextjs-frontend/__tests__/enhance.test.tsx +++ b/nextjs-frontend/__tests__/enhance.test.tsx @@ -12,7 +12,7 @@ import { FIELDS, SECTIONS, WORKER_STEPS } from "@/components/settings/model"; import { SETTINGS_GROUPS } from "@/components/settings/find"; import { STEP_LABEL } from "@/components/ui/loop"; -jest.mock("@/app/openapi-client", () => ({ Resources: { enhanceRecording: jest.fn() } })); +jest.mock("@/app/openapi-client", () => ({ Resources: { enhanceRecording: jest.fn(), listSegmentEdits: jest.fn() } })); jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); const push = jest.fn(); jest.mock("next/navigation", () => ({ useRouter: () => ({ push }) })); @@ -37,16 +37,26 @@ const MADE: Enhanced = { const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); const ctx = (enhanced: Enhanced | null, canEdit = true) => ({ id: 9, ns: "pods", rec: { id: 9, title: "Scan", enhanced }, paged: true, canEdit }) as unknown as RecordingCtx; -const wrap = (value: RecordingCtx) => { +const wrap = (value: RecordingCtx, sound = false) => { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( - + , ); }; +const SOUND: Enhanced = { + kind: "audio", + low: ["noisy", "clipped"], + measured: { duration: 12, snr_db: 9.5, level_db: -24.1, floor_db: -33.6, peak_db: 0, clipped: 0.004 }, + enhanced: true, + engine: "ffmpeg", + filters: ["adeclip", "highpass=f=70", "afftdn=nr=12:nf=-34"], + at: "2026-10-17T01:00:00Z", + download: "/api/v1/recordings/9/enhanced?sig=abc", +}; describe("enhance row text", () => { it("says what was found and what was made", () => { @@ -61,6 +71,14 @@ describe("enhance row text", () => { ); expect(engineName(null)).toBeNull(); }); + + it("says what was heard in a recording", () => { + expect(enhanceText(SOUND)).toBe("Noisy, clipped · signal to noise 9.5 dB, level -24.1 dBFS · by ffmpeg's filters"); + expect(enhanceText({ ...SOUND, low: [], engine: "webrtc", forced: true })).toMatch(/^On request · .* by WebRTC$/); + expect(enhanceText({ ...SOUND, low: [], enhanced: false })).toBe( + "good enough as it is (signal to noise 9.5 dB, level -24.1 dBFS)", + ); + }); }); describe("EnhanceDetail", () => { @@ -77,6 +95,19 @@ describe("EnhanceDetail", () => { await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Enhancing" }))); }); + it("on a recording, enhancing transcribes it again, and says what that replaces", async () => { + (Resources.listSegmentEdits as jest.Mock).mockReturnValue( + ok([{ after: { text: "fixed" } }, { after: { speaker: 2 } }, { after: { text: "again" } }]), + ); + (Resources.enhanceRecording as jest.Mock).mockReturnValue(ok({ ok: true, job: 32 })); + wrap(ctx(SOUND), true); + expect(await screen.findByText("Transcribing again replaces your 2 corrected lines.")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Enhance and transcribe again" })); + await waitFor(() => + expect(toast).toHaveBeenCalledWith(expect.objectContaining({ body: expect.stringMatching(/transcribed again/) })), + ); + }); + it("offers nothing to save before a copy exists, and no button to viewers", () => { wrap(ctx(null, false)); expect(screen.getByText("not run")).toBeInTheDocument(); @@ -96,16 +127,21 @@ describe("enhance in the web app", () => { "min_sharpness", "min_face_px", "max_scale", + "audio", + "audio_engine", + "min_snr_db", + "min_level_db", ]); expect(SECTIONS.some((s) => s.id === "enhance")).toBe(true); expect(SETTINGS_GROUPS.some((g) => g.sections.includes("enhance"))).toBe(true); }); - it("is a pipeline step that runs before read", () => { + it("is a pipeline step that runs before read and transcribe", () => { expect(WORKER_STEPS[0]).toBe("enhance"); expect(STEP_LABEL.enhance).toBe("Enhance"); expect(DESCRIBE.enhance).toMatch(/Settings → Enhance/); expect(NEEDS.read).toContain("enhance"); + expect(NEEDS.transcribe).toContain("enhance"); const steps = [{ type: "enhance" }, { type: "read" }]; expect(moveStep(steps, 1, 0).error).toBeTruthy(); }); diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index d9fda920e..543687b37 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1030,6 +1030,7 @@ export type { EnhanceRecordingErrors, EnhanceRecordingResponse, EnhanceRecordingResponses, + EnhanceSoundMeasured, EnrichedDate, EnrichedPlace, Enrichment, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 8e8f158ed..e4cec0d21 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -6002,9 +6002,10 @@ export class Resources { /** * Enhance Recording * - * Enhance an image whatever the settings and its quality (docs/enhance.md): an enhanced copy is made, and its - * pipeline's steps after enhance (reading its text, faces, objects, …) run again on that copy. The original stays as - * it is. 400 for anything but an image. Audited as `recording.enhance`. + * Enhance an image, or a recording's sound, whatever the settings and its quality (docs/enhance.md): an enhanced + * copy is made, and its pipeline's steps after enhance (reading an image's text, faces, objects, …; transcribing a + * recording and what follows) run again on that copy. The original stays as it is. 400 for a document or a + * transcript. Audited as `recording.enhance`. */ public static enhanceRecording( options: Options, @@ -8951,9 +8952,9 @@ export class Video { /** * Get Enhanced * - * The enhanced copy the enhance step made of a low quality image (docs/enhance.md), to save. Members of its - * namespace only (a bearer token, or a signed link the API gave one): visitors get its pages, with faces pixelated - * where the namespace does that. + * The enhanced copy the enhance step made of a low quality image (a JPEG) or a recording's sound (FLAC, 16 kHz + * mono) (docs/enhance.md), to save. Members of its namespace only (a bearer token, or a signed link the API gave + * one): visitors get its pages or its media, with faces pixelated where the namespace does that. */ public static getEnhanced( options: Options, diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index d34b7fe75..6bf1491d6 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -6263,52 +6263,112 @@ export type EnhanceMeasured = { [key: string]: unknown; }; +/** + * EnhanceSoundMeasured + * + * How good a recording sounded to the enhance step (docs/enhance.md), over 30 ms frames of it at 16 kHz mono. + */ +export type EnhanceSoundMeasured = { + /** + * Duration + * + * seconds + */ + duration: number; + /** + * Snr Db + * + * signal to noise: its loudest frames over its quietest, in dB + */ + snr_db: number; + /** + * Level Db + * + * the level of its loud half, dBFS + */ + level_db: number; + /** + * Floor Db + * + * its noise floor (its quietest frames), dBFS + */ + floor_db: number; + /** + * Peak Db + * + * its peak, dBFS + */ + peak_db: number; + /** + * Clipped + * + * the share of its samples at full scale (0 to 1) + */ + clipped: number; + [key: string]: unknown; +}; + /** * Enhanced * - * What the enhance step found and did (docs/enhance.md): whether the picture looked low quality and why, and the - * enhanced copy it's read from, if one was made. The original stays as it is. + * What the enhance step found and did (docs/enhance.md): whether the picture or the sound was low quality and why, + * and the enhanced copy it's read or transcribed from, if one was made. The original stays as it is. */ export type Enhanced = { /** * Kind */ - kind: "image"; + kind: "image" | "audio"; /** * Low * - * why it looked low quality; empty when it looked good enough + * why it looked or sounded low quality; empty when it was good enough */ - low?: Array<"small" | "blurry" | "noisy" | "dull" | "small_faces">; - measured: EnhanceMeasured; + low?: Array<"small" | "blurry" | "noisy" | "dull" | "small_faces" | "quiet" | "clipped">; + /** + * Measured + */ + measured: EnhanceMeasured | EnhanceSoundMeasured; /** * Enhanced * - * an enhanced copy was made, and the steps after it read that + * an enhanced copy was made, and the steps after it read or transcribe that */ enhanced: boolean; /** * Engine * - * what enhanced it: classic, or realesr-general-x4v3 (Real-ESRGAN) + * what enhanced it: classic or realesr-general-x4v3 (Real-ESRGAN) for an image, ffmpeg or webrtc for sound */ engine?: string | null; /** * Scale * - * how many times bigger the copy is + * how many times bigger an image's copy is */ scale?: number | null; /** * Width * - * the copy's width in pixels + * an image's copy's width in pixels */ width?: number | null; /** * Height */ height?: number | null; + /** + * Filters + * + * the ffmpeg filters that made a sound's copy + */ + filters?: Array | null; + /** + * Speech + * + * the share of the sound WebRTC's voice detector heard as speech (0 to 1) + */ + speech?: number | null; /** * Forced * @@ -19464,7 +19524,7 @@ export type Recording = { */ nsfw?: NsfwState | null; /** - * an image the enhance step looked at: what it found, and its enhanced copy + * an image or a recording the enhance step looked at: what it found, and its enhanced copy */ enhanced?: Enhanced | null; [key: string]: unknown; diff --git a/nextjs-frontend/components/pipelines/pipeline-model.ts b/nextjs-frontend/components/pipelines/pipeline-model.ts index 164cde979..77eb97fe2 100644 --- a/nextjs-frontend/components/pipelines/pipeline-model.ts +++ b/nextjs-frontend/components/pipelines/pipeline-model.ts @@ -91,6 +91,7 @@ export function cleanSpec(s: StepSpec): StepSpec | string { /** Which earlier steps a step reads from, when they are in the same pipeline. */ export const NEEDS: Record = { + transcribe: ["enhance"], read: ["enhance"], diarize: ["transcribe"], ocr: ["shots"], @@ -112,7 +113,7 @@ export const NEEDS: Record = { /** What a step makes available to later steps and templates. */ export const PROVIDES: Record = { - enhance: "an enhanced copy that read uses", + enhance: "an enhanced copy that read or transcribe uses", transcribe: "transcript · segments[] · language", read: "pages[] · segments[]", diarize: "speakers[] · turns", @@ -136,7 +137,7 @@ export const PROVIDES: Record = { export const DESCRIBE: Record = { enhance: - "Keeps a sharper copy of small, blurry, noisy or dull images for read to use; the original stays (Settings → Enhance)", + "Keeps a sharper copy of small, blurry, noisy or dull images for read, and a cleaned-up copy of noisy, quiet or clipped sound for transcribe; the original stays (Settings → Enhance)", transcribe: "Speech to text with word timings (Settings → Transcription)", read: "Draws a document’s or an image’s pages and reads their text (Settings → Documents)", diarize: "Who spoke when (Settings → Diarization)", diff --git a/nextjs-frontend/components/recording/details-tab.tsx b/nextjs-frontend/components/recording/details-tab.tsx index 0587f337c..44d2dce47 100644 --- a/nextjs-frontend/components/recording/details-tab.tsx +++ b/nextjs-frontend/components/recording/details-tab.tsx @@ -105,7 +105,14 @@ export function DetailsTab() { ["Fingerprint", rec.fingerprint ?? null, true], ]; const processing: Row[] = [ - ["Enhance", model.media.kind === "image" ? : null], + [ + "Enhance", + model.media.kind === "image" ? ( + + ) : !paged && model.audio ? ( + + ) : null, + ], [ "Transcribe", rec.transcribed_at diff --git a/nextjs-frontend/components/recording/enhance-model.ts b/nextjs-frontend/components/recording/enhance-model.ts index 1b38445c9..600c0cfc3 100644 --- a/nextjs-frontend/components/recording/enhance-model.ts +++ b/nextjs-frontend/components/recording/enhance-model.ts @@ -1,30 +1,42 @@ -import type { Enhanced } from "@/app/openapi-client/types.gen"; +import type { Enhanced, EnhanceMeasured, EnhanceSoundMeasured } from "@/app/openapi-client/types.gen"; -/** Why the enhance step thought an image looked low quality (docs/enhance.md). */ +/** Why the enhance step thought an image looked, or a recording sounded, low quality (docs/enhance.md). */ export const ENHANCE_REASONS: Record = { small: "small", blurry: "blurry", noisy: "noisy", dull: "low contrast", small_faces: "faces too small", + quiet: "quiet", + clipped: "clipped", }; /** The engine that made the copy, as people know it. */ export function engineName(engine: string | null | undefined): string | null { if (!engine) return null; - return engine === "classic" ? "the classic engine" : engine.startsWith("realesr") ? "Real-ESRGAN" : engine; + if (engine === "classic") return "the classic engine"; + if (engine.startsWith("realesr")) return "Real-ESRGAN"; + if (engine === "ffmpeg") return "ffmpeg's filters"; + if (engine === "webrtc") return "WebRTC"; + return engine; } +const sentence = (s: string) => (s ? `${s[0].toUpperCase()}${s.slice(1)}` : s); + /** One line for the Details tab: what the enhance step found and did ("not run" before it ran). */ export function enhanceText(e: Enhanced | null | undefined): string { if (!e) return "not run"; - const m = e.measured; + const why = (e.low ?? []).map((r) => ENHANCE_REASONS[r] ?? r).join(", "); + const by = engineName(e.engine); + if (e.kind === "audio") { + const m = e.measured as EnhanceSoundMeasured; + const heard = `signal to noise ${m.snr_db} dB, level ${m.level_db} dBFS`; + if (!e.enhanced) return `good enough as it is (${heard})`; + return [sentence(why) || "On request", heard, by && `by ${by}`].filter(Boolean).join(" · "); + } + const m = e.measured as EnhanceMeasured; const was = `${m.width}×${m.height}`; if (!e.enhanced) return `good enough as it is (${was})`; - const why = (e.low ?? []).map((r) => ENHANCE_REASONS[r] ?? r).join(", "); const made = e.width && e.height ? `${was} to ${e.width}×${e.height}` : was; - return [ - why ? `${why[0].toUpperCase()}${why.slice(1)}` : "On request", - [made, engineName(e.engine) && `by ${engineName(e.engine)}`].filter(Boolean).join(" "), - ].join(" · "); + return [sentence(why) || "On request", [made, by && `by ${by}`].filter(Boolean).join(" ")].join(" · "); } diff --git a/nextjs-frontend/components/recording/enhance.tsx b/nextjs-frontend/components/recording/enhance.tsx index 79b065ffa..e4ad84ba3 100644 --- a/nextjs-frontend/components/recording/enhance.tsx +++ b/nextjs-frontend/components/recording/enhance.tsx @@ -7,17 +7,17 @@ import { useRouter } from "next/navigation"; import { Resources } from "@/app/openapi-client"; import { useRec } from "@/components/recording/context"; import { enhanceText } from "@/components/recording/enhance-model"; -import { rk } from "@/components/recording/hooks"; +import { rk, useEdits } from "@/components/recording/hooks"; import { Button } from "@/components/ui/button"; import { useToast } from "@/components/ui/toast"; import { data, useApiClient } from "@/lib/api/browser"; import { absolute } from "@/lib/format"; /** - * An image's Enhance row (docs/enhance.md): what the step found, a link to save the enhanced copy, and, for editors, - * a button that enhances it now whatever the settings, then reads it again. + * An image's or a recording's Enhance row (docs/enhance.md): what the step found, a link to save the enhanced copy, + * and, for editors, a button that enhances it now whatever the settings, then reads or transcribes it again. */ -export function EnhanceDetail() { +export function EnhanceDetail({ sound = false }: { sound?: boolean }) { const { id, rec, canEdit } = useRec(); const client = useApiClient(); const qc = useQueryClient(); @@ -30,7 +30,9 @@ export function EnhanceDetail() { void qc.invalidateQueries({ queryKey: rk.jobs(id) }); toast({ title: "Enhancing", - body: "Its text is read again from the enhanced copy; the original stays as it is.", + body: sound + ? "It's transcribed again from the enhanced sound; the original stays as it is." + : "Its text is read again from the enhanced copy; the original stays as it is.", tone: "intent", action: { label: "View job", onClick: () => router.push(`/activity/${r.job}`) }, }); @@ -38,26 +40,48 @@ export function EnhanceDetail() { onError: (err) => toast({ title: "Couldn’t enhance it", body: (err as Error).message, tone: "red" }), }); return ( - - - {enhanceText(e)} - {e?.at ? ` · ${absolute(e.at)}` : ""} + + + + {enhanceText(e)} + {e?.at ? ` · ${absolute(e.at)}` : ""} + + {e?.enhanced && e.download && ( + + + Save the copy + + )} + {canEdit && ( + + )} - {e?.enhanced && e.download && ( - - - Save the copy - - )} - {canEdit && ( - - )} + {canEdit && sound && } + + ); +} + +/** Transcribing again makes a new transcript: say so where there are corrections it would replace. */ +function EditsReplaced() { + const { id } = useRec(); + const edits = useEdits(id); + const n = (edits.data ?? []).filter((x) => x.after && "text" in x.after).length; + if (!n) return null; + return ( + + Transcribing again replaces your {n} corrected {n === 1 ? "line" : "lines"}. ); } diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index a7a3317c6..dc1717c19 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -433,11 +433,18 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { + +
+ + + +

- The enhanced copy is kept beside the original, encrypted like it, and is what OCR and descriptions read. - Real-ESRGAN general x4v3 (BSD-3-Clause) runs with ONNX Runtime on this server: pictures aren’t sent - anywhere. Any image can be enhanced on request from its Details tab, even with this off. Audio isn’t - enhanced yet. + The enhanced copy is kept beside the original, encrypted like it, and is what OCR and descriptions read, or + what transcription hears; its times stay in step with the original, and voices are still told apart from the + original. Real-ESRGAN general x4v3 (BSD-3-Clause) runs with ONNX Runtime, and WebRTC’s audio processing + (Apache-2.0) in Lens, on this server: nothing is sent anywhere. Any image or recording can be enhanced on + request from its Details tab, even with this off.

); diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index c95c9cc84..7c0e1e2fe 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -213,7 +213,7 @@ export const SECTIONS: SectionSpec[] = [ label: "Enhance", backend: ["enhance"], description: - "Find small, blurry, noisy or dull images and keep a sharper copy that Lens reads instead. The original stays as it is. Off until you turn it on.", + "Find small, blurry, noisy or dull images, and noisy, quiet or clipped sound, and keep a better copy that Lens reads or transcribes instead. The original stays as it is. Off until you turn it on.", }, { id: "workers", @@ -2674,6 +2674,42 @@ export const FIELDS: FieldSpec[] = [ max: 4, hint: "Times the original size, and never past 4000 pixels a side", }, + { + section: "enhance", + key: "audio", + label: "Enhance low quality sound", + kind: "switch", + hint: "Keeps a cleaned-up copy of noisy, quiet or clipped audio and video sound and transcribes that instead; the original stays", + }, + { + section: "enhance", + key: "audio_engine", + label: "Sound engine", + kind: "select", + options: [ + { value: "ffmpeg", label: "ffmpeg: denoise, high-pass, declip, level (light)" }, + { value: "webrtc", label: "WebRTC noise suppression and gain (about 1 MB, Linux)" }, + ], + hint: "WebRTC is fetched the first time and runs on this server; elsewhere ffmpeg does it", + }, + { + section: "enhance", + key: "min_snr_db", + label: "Noisy below", + kind: "number", + min: 0, + max: 60, + hint: "dB of speech over the noise floor", + }, + { + section: "enhance", + key: "min_level_db", + label: "Quiet below", + kind: "number", + min: -70, + max: 0, + hint: "dBFS: the level of its loud half", + }, // Workers { section: "workers", diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 0e82ccfaa..f1e9c5166 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -11125,7 +11125,7 @@ "resources" ], "summary": "Enhance Recording", - "description": "Enhance an image whatever the settings and its quality (docs/enhance.md): an enhanced copy is made, and its\npipeline's steps after enhance (reading its text, faces, objects, \u2026) run again on that copy. The original stays as\nit is. 400 for anything but an image. Audited as `recording.enhance`.", + "description": "Enhance an image, or a recording's sound, whatever the settings and its quality (docs/enhance.md): an enhanced\ncopy is made, and its pipeline's steps after enhance (reading an image's text, faces, objects, \u2026; transcribing a\nrecording and what follows) run again on that copy. The original stays as it is. 400 for a document or a\ntranscript. Audited as `recording.enhance`.", "operationId": "enhance_recording", "parameters": [ { @@ -21106,7 +21106,7 @@ "video" ], "summary": "Get Enhanced", - "description": "The enhanced copy the enhance step made of a low quality image (docs/enhance.md), to save. Members of its\nnamespace only (a bearer token, or a signed link the API gave one): visitors get its pages, with faces pixelated\nwhere the namespace does that.", + "description": "The enhanced copy the enhance step made of a low quality image (a JPEG) or a recording's sound (FLAC, 16 kHz\nmono) (docs/enhance.md), to save. Members of its namespace only (a bearer token, or a signed link the API gave\none): visitors get its pages or its media, with faces pixelated where the namespace does that.", "operationId": "get_enhanced", "parameters": [ { @@ -21133,7 +21133,8 @@ "200": { "description": "Successful Response", "content": { - "image/jpeg": {} + "image/jpeg": {}, + "audio/flac": {} } }, "422": { @@ -45751,11 +45752,60 @@ "title": "EnhanceMeasured", "description": "How good a picture looked to the enhance step (docs/enhance.md)." }, + "EnhanceSoundMeasured": { + "properties": { + "duration": { + "type": "number", + "title": "Duration", + "description": "seconds" + }, + "snr_db": { + "type": "number", + "title": "Snr Db", + "description": "signal to noise: its loudest frames over its quietest, in dB" + }, + "level_db": { + "type": "number", + "title": "Level Db", + "description": "the level of its loud half, dBFS" + }, + "floor_db": { + "type": "number", + "title": "Floor Db", + "description": "its noise floor (its quietest frames), dBFS" + }, + "peak_db": { + "type": "number", + "title": "Peak Db", + "description": "its peak, dBFS" + }, + "clipped": { + "type": "number", + "title": "Clipped", + "description": "the share of its samples at full scale (0 to 1)" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "duration", + "snr_db", + "level_db", + "floor_db", + "peak_db", + "clipped" + ], + "title": "EnhanceSoundMeasured", + "description": "How good a recording sounded to the enhance step (docs/enhance.md), over 30 ms frames of it at 16 kHz mono." + }, "Enhanced": { "properties": { "kind": { "type": "string", - "const": "image", + "enum": [ + "image", + "audio" + ], "title": "Kind" }, "low": { @@ -45766,20 +45816,30 @@ "blurry", "noisy", "dull", - "small_faces" + "small_faces", + "quiet", + "clipped" ] }, "type": "array", "title": "Low", - "description": "why it looked low quality; empty when it looked good enough" + "description": "why it looked or sounded low quality; empty when it was good enough" }, "measured": { - "$ref": "#/components/schemas/EnhanceMeasured" + "anyOf": [ + { + "$ref": "#/components/schemas/EnhanceMeasured" + }, + { + "$ref": "#/components/schemas/EnhanceSoundMeasured" + } + ], + "title": "Measured" }, "enhanced": { "type": "boolean", "title": "Enhanced", - "description": "an enhanced copy was made, and the steps after it read that" + "description": "an enhanced copy was made, and the steps after it read or transcribe that" }, "engine": { "anyOf": [ @@ -45791,7 +45851,7 @@ } ], "title": "Engine", - "description": "what enhanced it: classic, or realesr-general-x4v3 (Real-ESRGAN)" + "description": "what enhanced it: classic or realesr-general-x4v3 (Real-ESRGAN) for an image, ffmpeg or webrtc for sound" }, "scale": { "anyOf": [ @@ -45803,7 +45863,7 @@ } ], "title": "Scale", - "description": "how many times bigger the copy is" + "description": "how many times bigger an image's copy is" }, "width": { "anyOf": [ @@ -45815,7 +45875,7 @@ } ], "title": "Width", - "description": "the copy's width in pixels" + "description": "an image's copy's width in pixels" }, "height": { "anyOf": [ @@ -45828,6 +45888,33 @@ ], "title": "Height" }, + "filters": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Filters", + "description": "the ffmpeg filters that made a sound's copy" + }, + "speech": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Speech", + "description": "the share of the sound WebRTC's voice detector heard as speech (0 to 1)" + }, "forced": { "anyOf": [ { @@ -45872,7 +45959,7 @@ "enhanced" ], "title": "Enhanced", - "description": "What the enhance step found and did (docs/enhance.md): whether the picture looked low quality and why, and the\nenhanced copy it's read from, if one was made. The original stays as it is." + "description": "What the enhance step found and did (docs/enhance.md): whether the picture or the sound was low quality and why,\nand the enhanced copy it's read or transcribed from, if one was made. The original stays as it is." }, "EnrichedDate": { "properties": { @@ -68431,7 +68518,7 @@ "type": "null" } ], - "description": "an image the enhance step looked at: what it found, and its enhanced copy" + "description": "an image or a recording the enhance step looked at: what it found, and its enhanced copy" } }, "additionalProperties": true, diff --git a/threat-model.md b/threat-model.md index 1019d7795..bba86fd8d 100644 --- a/threat-model.md +++ b/threat-model.md @@ -495,6 +495,7 @@ third-party source; dependencies come from `uv.lock`, the pnpm lockfile and cont | spaCy and its small models (`spacy` extra, opt-in: `analysis.entities` `spacy`) | finding names in text; the MIT model wheels (`en_core_web_sm`, `de_core_news_sm`, `xx_ent_wiki_sm` 3.8.0) come from spaCy's GitHub releases, are checked against their SHA-256 and installed without dependencies; a model is Python code that runs when it loads, so the checksum is what keeps a swapped file out | upstream for the model's judgement (a missed or wrong name); a model file that installs unchecked is in model | *(documented, docs/components.md "checked against their SHA-256")* | | Moondream (Moondream 3 Preview, BSL 1.1, or Moondream 2, Apache-2.0; `moondream` extra, opt-in: `llm.describe_engine` `moondream`) | describing the archive's pictures on the worker, so none is sent to a provider; the weights and the model's own Python code (transformers' `trust_remote_code`) are fetched at a pinned revision, so the code that runs in the worker can't change under Lens; its tokenizer (`moondream/starmie-v1`, data, not code) is fetched by name | upstream for what the model says (text a picture shows reaches the assistant as archive text, N-PROMPT) and for the files' integrity beyond HTTPS and the Hub; the pinned code runs with the worker's access, an admin's choice like any extra | *(documented, docs/configuration.md "the weights and the model's own code pinned to a revision")* | | Real-ESRGAN general x4v3 (`enhance.image_engine` `realesrgan`, opt-in) | enlarging and cleaning up low quality pictures before they're read; the ONNX model is fetched at a pinned revision and checked against its SHA-256, and runs in ONNX Runtime on the CPU in fixed tiles of pictures Lens has already opened and size-checked | upstream for ONNX Runtime and the model's output; a model file that passes unchecked is in model | *(documented, docs/enhance.md "fetched the first time the step runs with it chosen")* | +| webrtc-noise-gain (WebRTC audio processing, `webrtc` extra, `enhance.audio_engine` `webrtc`, opt-in, Linux x86-64 and ARM64) | noise suppression, gain and voice detection on sound Lens has already decoded to 16 kHz mono with its own ffmpeg, 10 ms at a time; installed from PyPI at the version uv.lock pins the first time the step runs with it chosen | upstream for the native library's processing; the package download as for other pip packages (N-DOWNLOADS) | *(documented, docs/enhance.md "fetched the first time the step runs with it chosen")* | | Optional ML extras (whisper, pyannote, speechbrain, insightface, doctr) | model inference; some fetch weights on their own | upstream | *(inferred, Q19)* | | StyleTTS2 service (`styletts2/`, opt-in Compose profile) and macOS `say` | text to speech; StyleTTS2 fetches its weights from Hugging Face and GitHub on first use and listens only on the Compose network; `say` gets the text on stdin, never as arguments | upstream for the model and synthesis; the argument handling in model | *(maintainer, 2026-10)* | | MCP runner (`mcp-runner/`, opt-in Compose profile) and the stdio MCP servers admins add (usually `npx` or `uvx` packages) | running those servers away from Lens: a container with none of Lens's data, keys or database, a non-root user, every capability dropped, `no-new-privileges`, a read-only image, memory and process limits, a network shared only with the backend (plus the internet to fetch packages), an empty home and only its extension's environment per server, and a bearer token on a volume only the two share | upstream for the servers and the packages they fetch (an admin's choice, trusted input, §1.3); the runner's isolation and token check in model | *(documented, docs/assistant.md "The MCP runner")* | diff --git a/threat-model.yaml b/threat-model.yaml index 1cfd9744d..aa643164b 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -8,7 +8,7 @@ project: lens prose_version: "threat-model.md@same-commit" model_status: under-review triage_policy: strict -confidence: {documented: 211, maintainer: 46, inferred: 91, assumption: 8} +confidence: {documented: 212, maintainer: 46, inferred: 91, assumption: 8} generation: model: "Claude Opus 5.5 (claude-opus-5-5)" @@ -1193,6 +1193,15 @@ dependencies: covered_here: false violation_disposition: "OUT-OF-MODEL: dependency-contract" provenance: {kind: documented, source: 'docs/enhance.md "fetched the first time the step runs with it chosen"'} + - name: "webrtc-noise-gain (WebRTC audio processing, webrtc extra, enhance.audio_engine webrtc, opt-in)" + relied_on_for: "noise suppression, gain and voice detection on sound Lens has already decoded to 16 kHz mono with its own ffmpeg, 10 ms at a time; installed from PyPI at the version uv.lock pins the first time the step runs with it chosen; the package download as for other pip packages (N-DOWNLOADS)" + relied_on_properties: [model-inference] + caller_obligations_acknowledged: [] + adversary_capabilities_forwarded: [supply-parsed-bytes] + outputs_consumed: [] + covered_here: false + violation_disposition: "OUT-OF-MODEL: dependency-contract" + provenance: {kind: documented, source: 'docs/enhance.md "fetched the first time the step runs with it chosen"'} - name: "Optional ML extras (whisper, pyannote, speechbrain, insightface, doctr)" relied_on_for: "model inference; some fetch weights on their own" relied_on_properties: [model-inference]