From 8eae3297da5f556d34d8105a11f1738ab17353be Mon Sep 17 00:00:00 2001
From: Adeel Ahmad <230291+adeelahmad@users.noreply.github.com>
Date: Sat, 7 Nov 2026 05:45:05 +1100
Subject: [PATCH 1/4] feat(voice): add audio workspace and podcast output modes
---
README.md | 5 +
changelog.d/voice-workspace.md | 1 +
docs/api.md | 8 +
docs/podcasts.md | 8 +
docs/requirements.md | 9 +
docs/speech-providers.md | 3 +
docs/voice-service.md | 90 ++++++++++
fastapi_backend/app/api/v1/routes/podcasts.py | 8 +-
fastapi_backend/app/api/v1/routes/voice.py | 87 +++++++---
fastapi_backend/app/domain/podcasts.py | 140 ++++++++++++++--
fastapi_backend/app/domain/voice.py | 96 ++++++++++-
fastapi_backend/app/schemas/podcasts.py | 11 +-
fastapi_backend/app/schemas/voice.py | 3 +-
fastapi_backend/tests/api/test_podcasts.py | 23 +++
fastapi_backend/tests/api/test_voice_raw.py | 134 +++++++++++++++
fastapi_backend/tests/domain/test_podcasts.py | 97 ++++++++++-
.../tests/domain/test_voice_raw.py | 103 ++++++++++++
nextjs-frontend/app/(app)/voice/page.tsx | 9 +
nextjs-frontend/components/app-shell/nav.tsx | 42 ++++-
.../podcasts/__tests__/create-dialog.test.tsx | 92 ++++++++++
.../podcasts/__tests__/create-model.test.ts | 18 ++
.../components/podcasts/create-dialog.tsx | 157 ++++++++++++++----
.../components/podcasts/create-model.ts | 15 ++
.../components/settings/bodies.tsx | 4 +-
nextjs-frontend/components/settings/model.ts | 8 +
.../components/voice/__tests__/model.test.ts | 24 +++
.../voice/__tests__/service.test.ts | 45 +++++
.../voice/__tests__/voice-workspace.test.tsx | 79 +++++++++
nextjs-frontend/components/voice/model.ts | 26 +++
nextjs-frontend/components/voice/service.ts | 28 ++++
.../components/voice/voice-workspace.tsx | 140 ++++++++++++++++
.../lib/api/__tests__/raw-voice-proxy.test.ts | 32 ++++
threat-model.json | 14 ++
threat-model.md | 8 +
threat-model.yaml | 25 +++
35 files changed, 1500 insertions(+), 92 deletions(-)
create mode 100644 changelog.d/voice-workspace.md
create mode 100644 docs/voice-service.md
create mode 100644 fastapi_backend/tests/api/test_voice_raw.py
create mode 100644 fastapi_backend/tests/domain/test_voice_raw.py
create mode 100644 nextjs-frontend/app/(app)/voice/page.tsx
create mode 100644 nextjs-frontend/components/podcasts/__tests__/create-dialog.test.tsx
create mode 100644 nextjs-frontend/components/podcasts/__tests__/create-model.test.ts
create mode 100644 nextjs-frontend/components/podcasts/create-model.ts
create mode 100644 nextjs-frontend/components/voice/__tests__/model.test.ts
create mode 100644 nextjs-frontend/components/voice/__tests__/service.test.ts
create mode 100644 nextjs-frontend/components/voice/__tests__/voice-workspace.test.tsx
create mode 100644 nextjs-frontend/components/voice/model.ts
create mode 100644 nextjs-frontend/components/voice/service.ts
create mode 100644 nextjs-frontend/components/voice/voice-workspace.tsx
create mode 100644 nextjs-frontend/lib/api/__tests__/raw-voice-proxy.test.ts
diff --git a/README.md b/README.md
index cdea051bb..56b5abc10 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,10 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p
server](docs/mcp.md#one-namespaces-mcp-server), [chat completions](docs/api.md#chat-completions-for-any-app),
[API](docs/api.md), [the graph as files](docs/file-tree.md), [authentication](docs/authentication.md);
[checks](docs/requirements.md#connect-your-agents))
+- **Use speech directly.** The Voice workspace turns raw text into downloadable audio and short media into text,
+ using the configured speech engines. Its API works at the same local or remote Lens address; anonymous raw
+ access is off until an admin enables it, with bounded admission and clip sizes.
+ ([Voice service](docs/voice-service.md); [checks](docs/requirements.md#use-speech-directly))
- **Ask, and get cited answers.** A chat assistant on every page, and by voice, searches your archive by words (with
"phrases", OR, and -word to leave lines out) and by meaning, once turned on, reads it, cites it and acts on it with
tools: changes wait for your approval. A search result opens the asset with Find showing the words. Ask it about Lens itself ("what's the MCP address?", "is Telegram on?") and it
@@ -431,3 +435,4 @@ licenses (for example `extract-msg`, GPL-3.0, for Outlook `.msg` files); none is
Built on the [Next.js FastAPI Template](https://github.com/vintasoftware/nextjs-fastapi-template) (MIT).
+
diff --git a/changelog.d/voice-workspace.md b/changelog.d/voice-workspace.md
new file mode 100644
index 000000000..7bdcdfcab
--- /dev/null
+++ b/changelog.d/voice-workspace.md
@@ -0,0 +1 @@
+Add a Voice workspace inside Lens for raw text-to-audio and media-to-text using existing speech engines. Raw requests reject oversized text instead of silently truncating it, bound media decoding and concurrent work, and provide explicit server-only errors. Administrators can enable anonymous raw speech on demand, off by default and without opening archive or podcast access. Existing deployments are not changed automatically.
diff --git a/docs/api.md b/docs/api.md
index 858d2e93c..ac30c20cd 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -2055,3 +2055,11 @@ many more there are. For someone signed in without permission,
it is open to everyone); asking again replaces the request. `GET /access-requests` lists the requests waiting for an
answer in the namespaces the caller owns. Assets the caller may not see at all (restricted ones
to visitors who aren't signed in, private ones to anyone without permission) answer 404, as missing ones do.
+
+
+## Raw voice service
+
+The [Voice service](voice-service.md) reuses `/api/v1/voice/speak` and `/api/v1/voice/transcribe` for
+direct audio and text, locally or at the existing remote origin. Authentication remains on by default;
+`voice.public_enabled` opens only the stateless raw service on demand. Use `server_only:true` for explicit
+synthesis errors instead of the assistant's browser fallback.
diff --git a/docs/podcasts.md b/docs/podcasts.md
index ea7a0b7ac..dd88f5f71 100644
--- a/docs/podcasts.md
+++ b/docs/podcasts.md
@@ -1,5 +1,12 @@
# Podcasts
+The create dialog supports four output modes: **Original** reads the complete selected source text without rewriting,
+**Summary** and **Explanation** generate a cited single-narrator episode, and **Dialogue** uses two hosts. Choose the
+narrator or host voices from the configured speech provider, or retain the podcast defaults. Import historical text,
+documents or URLs through the existing Library import flow, then select those assets to make episodes. Original mode
+rejects text above the configured context limit rather than silently shortening it. Audio still requires a working
+speech provider; a script-only episode is not playable audio.
+
**Status: phase 1 built.** Lens writes a cited, fact-checked script, records it in two voices, and serves it over the
API, MCP and the web app. Picking search hits, topics, notes and graph concepts comes next (see the
[dev log](https://github.com/adeelahmad/lense/blob/main/DEVLOG.md)).
@@ -97,3 +104,4 @@ the job when it's over.
- Choose the hosts' voices in the dialog; the API takes them already.
- Write long episodes in sections, for small local models.
- Check claims in batches when a script has many.
+
diff --git a/docs/requirements.md b/docs/requirements.md
index 7e4e437dd..77b310ca0 100644
--- a/docs/requirements.md
+++ b/docs/requirements.md
@@ -451,3 +451,12 @@ request, to another check that proves the same thing; loosening a requirement or
| The web app works on a phone: every journey is surveyed on one, and none may take more than its baseline | [clicks.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/clicks.yml) `Survey on a phone`; [baseline-phone.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline-phone.json); [phone-layout.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/phone-layout.test.tsx) |
| Heavy steps kept to their hours and to so many at a time, off by default | [test_quiet.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_quiet.py) `test_the_window_and_its_checks`, `test_heavy_steps_wait_for_their_hours_and_their_turn` |
| The default install runs on a Raspberry Pi (arm64) with a cloud model | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) |
+
+
+## Use speech directly
+
+| Requirement | Checks |
+|---|---|
+| Raw speech is authenticated by default; anonymous opt-in is narrowly scoped, bounded and fails explicitly | [raw voice API tests](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice_raw.py) |
+| Decoding bounds media duration and wall time before inference | [raw voice domain tests](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_voice_raw.py) |
+| The workspace preserves input, supports downloads and ignores cancelled results | [workspace tests](https://github.com/adeelahmad/lense/tree/main/nextjs-frontend/components/voice/__tests__) |
diff --git a/docs/speech-providers.md b/docs/speech-providers.md
index f4c839591..833f237a6 100644
--- a/docs/speech-providers.md
+++ b/docs/speech-providers.md
@@ -7,6 +7,8 @@ instead, which helps on a small machine such as a Raspberry Pi, or when you want
Status: built (transcription, speakers, language, sentiment, audio events, text to speech, prices per minute and per
character in the activity ledger). Planned: provider summaries, chapters and entities as optional imports.
+For direct text-to-audio and media-to-text work, use the [Voice workspace and API](voice-service.md).
+
## Providers
| Provider | Speech to text | Speakers | Language | Emotion | Sounds | Text to speech |
@@ -155,3 +157,4 @@ has no Opus), in pieces when it is longer or larger than the provider takes at o
to speech sends the answer's text.
Code: `fastapi_backend/app/domain/speech.py`.
+
diff --git a/docs/voice-service.md b/docs/voice-service.md
new file mode 100644
index 000000000..93e161715
--- /dev/null
+++ b/docs/voice-service.md
@@ -0,0 +1,90 @@
+# Voice service
+
+The **Voice** workspace in Lens turns raw text into downloadable audio and an audio or video clip into text.
+It uses the same speech providers as the assistant and podcasts. There is no second service to install, no new
+queue, and no model download until an administrator selects a local engine. Open **Voice** beside **Chat**, or use
+the link in **Settings → AI assistant → Voice**.
+
+## Text and media
+
+- Paste up to 4,000 characters, choose one of the configured provider's voices, and generate audio. The text is sent
+ exactly as entered, including punctuation and Markdown. The workspace plays and downloads the returned audio.
+ A missing provider or failed synthesis is an error, not a silent browser-speech substitute or truncated recording.
+- Upload an audio or video clip, up to 25 MiB and five minutes, to get a transcript you can copy or download. This
+ does not create a Library asset. Use normal Library import for longer recordings and background transcription.
+- Cancel stops the browser waiting and discards late results. A model call already running on the server can finish;
+ cancellation does not promise to stop provider billing.
+
+Readiness, transcription engine, and provider voice-list errors are shown in the workspace. Configure the engines in
+**Settings → AI assistant → Voice** and keys in **Settings → Speech providers**. See [speech providers](speech-providers.md)
+for real capability and platform requirements. Kokoro and StyleTTS2 are supported existing integrations. Piper is
+not a native Lens backend: an independently operated server that actually implements the OpenAI audio API can be
+configured as OpenAI-compatible, but Lens does not install or validate a Piper deployment here. macOS `say` cannot
+run in a Linux Proxmox container.
+
+## The same API locally and remotely
+
+Use your existing Lens origin, including its existing HTTPS/Cloudflare Tunnel address, with `/api/v1`:
+
+```sh
+# An ordinary Lens API token, kept in the shell environment, not in the URL.
+curl --fail-with-body "$LENS_URL/api/v1/voice/speak" \
+ -H "Authorization: Bearer $LENS_TOKEN" -H 'Content-Type: application/json' \
+ --data '{"text":"Read this exact text.","server_only":true}' --output speech.audio
+
+curl --fail-with-body "$LENS_URL/api/v1/voice/transcribe" \
+ -H "Authorization: Bearer $LENS_TOKEN" -H 'Content-Type: application/octet-stream' \
+ --data-binary @clip.wav
+```
+
+The speech response's `Content-Type` identifies its format; do not assume every provider returns MP3.
+Transcription returns `text`, `language`, `engine` and `seconds`. `GET /voice` reports capability, and authenticated
+`GET /voice/voices` lists selectable voices. A namespace-only key deliberately cannot use these global endpoints.
+The workspace asks for `server_only:true`. Existing assistant clients can still omit it and receive 204 for browser
+fallback. Blank or over-limit text returns 422; malformed media 400, oversized/long media 413, unavailable speech 503,
+and an unconfigured transcription engine 409. Busy/rate-limited requests return 429 with `Retry-After`.
+
+This feature does not create a tunnel, change firewall rules, install models on a live host, or upgrade an existing
+v0.20.0 instance. Those remain operator actions after the change is reviewed and released.
+
+## Anonymous access, only when needed
+
+Authentication stays required by default. An administrator can enable **Allow anonymous raw voice API** in the
+Voice settings (`voice.public_enabled`) and switch it off again afterward. This opens only capability, raw speech
+and raw transcription. It does not open the Voice web page, provider voice lists, settings, the archive, podcasts,
+or private feeds. Anonymous callers omit the Authorization header and cannot override the configured voice.
+Invalid or namespace-scoped credentials do not downgrade to anonymous access.
+
+Enabling this on an internet-reachable instance lets anyone who can reach it consume its configured speech service,
+including paid providers. The raw service has no waiting queue: at most two expensive requests run per API process,
+and all anonymous callers share ten provider requests per minute per process. A worker keeps its slot until it
+finishes, even when the HTTP request is cancelled. At most two media uploads are admitted concurrently.
+Forwarding headers do not create a fresh quota.
+Uploads time out after 30 seconds; decoding is bounded to 301 seconds of output, rejects anything over 300 seconds,
+and has a 30-second wall-clock timeout. Decoding accepts only media formats and local file/pipe protocols, not URL
+playlists. Successful audio and transcript responses use `Cache-Control: no-store`.
+
+These are process-local safeguards, not a global internet abuse firewall. Multiple API workers multiply the limits.
+Keep anonymous mode off when not in use, configure global budgets and provider spending limits, and use an edge
+rate limit for a publicly exposed deployment. Raw requests are not archive assets, but the activity ledger still
+records speech use; a configured cloud provider receives the supplied text/audio under its own retention policy.
+
+## Sources, podcasts and Apple Podcasts
+
+Import a web URL through Lens's guarded web importer, paste text through text import, or upload a document through
+the Library. Wait for its text to be extracted, then select the asset and choose **Podcast**. This preserves the
+existing URL network guard, source citations, namespace/vault permissions and podcast jobs rather than making an
+anonymous URL-fetching service. Do not send URLs to the raw transcription endpoint: it accepts media bytes.
+
+Episodes use Lens's existing [podcast flow](podcasts.md). Follow a saved collection or a published private podcast
+[feed](feeds.md) in Apple Podcasts using **Follow a Show by URL**. An episode must have audio and be in that feed's
+selection. A script without a working TTS provider is not an audio episode. Feed URLs are bearer secrets: anyone who
+has the URL can read that feed until its token is rotated. Do not publish a private feed to a podcast directory.
+
+## Alexa
+
+Alexa playback is not implemented by this change. A realistic follow-up is a private Alexa skill using Amazon's
+AudioPlayer interface and an HTTPS audio URL reachable by Amazon, with suitable authentication/linking and verified
+audio requirements. The Lens API must not be treated as an Alexa Skill endpoint, and a private Apple Podcasts feed
+does not automatically become available on Alexa. Building or publishing such a skill, and exposing its media,
+requires separate configuration and verification.
diff --git a/fastapi_backend/app/api/v1/routes/podcasts.py b/fastapi_backend/app/api/v1/routes/podcasts.py
index e71db24d6..7221299ea 100644
--- a/fastapi_backend/app/api/v1/routes/podcasts.py
+++ b/fastapi_backend/app/api/v1/routes/podcasts.py
@@ -1,4 +1,4 @@
-"""Podcasts: two-host learning episodes made from picked resources and passages, every claim cited
+"""Podcasts: original readings and cited learning episodes made from picked resources and passages
(app/domain/podcasts.py, docs/podcasts.md). An episode is a recording, so its audio, transcript, sharing and deletion
also go through /recordings; these routes start, show and remake episodes."""
@@ -41,6 +41,7 @@ def start(acl: Access, user: Principal, db: DB, cfg: dict, body: PodcastCreate)
style=body.style,
title=body.title,
voices=body.voices.model_dump(exclude_none=True) if body.voices else None,
+ mode=body.mode,
)
except podcasts.Problem as e:
raise HTTPException(400, str(e)) from None
@@ -54,7 +55,10 @@ def start(acl: Access, user: Principal, db: DB, cfg: dict, body: PodcastCreate)
def create_podcast(body: PodcastCreate, acl: Acl, user: Writer, db: Db, cfg: Cfg) -> PodcastStarted:
"""Make an episode about these resources and lines, in the background (watch its job on /events). It goes in the
podcasts namespace when everyone there can read every source, else in the sources' own namespace (400 when they
- span several); you need editor access where it goes. Audited as `podcast.create`."""
+ span several); you need editor access where it goes. Original reads the complete selected text without a model,
+ refusing text over podcasts.context_chars; summary and explanation narrate in voice a; dialogue uses a and b.
+ Import text, documents and URLs through the existing import routes before selecting their asset ids here.
+ Audited as `podcast.create`."""
return start(acl, user, db, cfg, body)
diff --git a/fastapi_backend/app/api/v1/routes/voice.py b/fastapi_backend/app/api/v1/routes/voice.py
index 417b18915..b6baaba36 100644
--- a/fastapi_backend/app/api/v1/routes/voice.py
+++ b/fastapi_backend/app/api/v1/routes/voice.py
@@ -2,10 +2,13 @@
from __future__ import annotations
-from fastapi import APIRouter, HTTPException, Query, Request, Response
+import asyncio
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
from starlette.concurrency import run_in_threadpool
-from app.api.deps import Cfg, CurrentUser
+from app.api.deps import Cfg, CurrentUser, OptionalUser, Principal, signed_out
from app.domain import speech, voice
from app.schemas.voice import Heard, SpeakRequest, VoiceInfo, VoiceList
@@ -13,42 +16,84 @@
CLIP = {"requestBody": {"required": True, "content": {"application/octet-stream": {"schema": {"type": "string", "format": "binary"}}}}}
+def raw_user(request: Request, user: OptionalUser, cfg: Cfg) -> Principal | None:
+ # Invalid/scoped credentials never silently downgrade to anonymous access.
+ if user is None and (request.headers.get("authorization") or (cfg.get("voice") or {}).get("public_enabled") is not True):
+ raise signed_out(request)
+ return user
+
+
+RawUser = Annotated[Principal | None, Depends(raw_user)]
+
+
@router.get("")
-def voice_info(user: CurrentUser, cfg: Cfg) -> VoiceInfo:
+def voice_info(user: RawUser, cfg: Cfg) -> VoiceInfo:
"""Whether the server hears and speaks itself; the web app uses the browser's speech for what it doesn't. Starts
loading the speech-to-text engine, so the first thing said isn't kept waiting."""
engine = voice.can_transcribe(cfg)
- voice.warm(cfg)
+ if user is not None:
+ voice.warm(cfg)
return VoiceInfo(transcribe=engine is not None, engine=engine, speak=voice.can_speak(cfg))
-@router.post("/transcribe", openapi_extra=CLIP)
-async def transcribe_clip(request: Request, user: CurrentUser, cfg: Cfg) -> Heard:
- """What was said in a short clip, sent as the raw request body (webm, ogg, mp4 or wav, up to 25 MB), by the
- server's speech-to-text engine. Nothing is kept. 409 when the server has no engine (use the browser's)."""
+@router.post("/transcribe", openapi_extra=CLIP, responses={413: {"description": "Clip exceeds 25 MiB or 300 seconds"}, 429: {"description": "Voice service busy"}, 503: {"description": "Transcription failed"}})
+async def transcribe_clip(request: Request, response: Response, user: RawUser, cfg: Cfg) -> Heard:
+ """Raw media to text, without archiving. At most 25 MiB and 300 seconds. 409 when no engine is configured.
+ Anonymous access is opt-in via voice.public_enabled and uses only the configured engine."""
if not voice.can_transcribe(cfg):
raise HTTPException(409, "this server has no speech-to-text engine yet")
- data = bytearray()
- async for piece in request.stream():
- data += piece
- if len(data) > voice.MAX_BYTES:
- raise HTTPException(413, "that clip is too long; say it in shorter parts")
try:
- return Heard(**await run_in_threadpool(voice.transcribe, cfg, bytes(data)))
- except RuntimeError as e:
- raise HTTPException(400, f"couldn't read that clip: {e}") from None
+ with voice.upload_admission():
+ data = bytearray()
+ async with asyncio.timeout(30):
+ async for piece in request.stream():
+ if len(data) + len(piece) > voice.MAX_BYTES:
+ raise HTTPException(413, "clips must be no larger than 25 MiB")
+ data += piece
+ if not data:
+ raise HTTPException(400, "send an audio or video clip")
+ response.headers["Cache-Control"] = "no-store"
+ # Raw clips must not inherit archive transcription's 30-minute provider timeout.
+ speech_cfg = cfg.get("speech") or {}
+ raw_cfg = {**cfg, "speech": {**speech_cfg, "timeout": min(float(speech_cfg.get("timeout") or 120), 120)}}
+ return Heard(**await run_in_threadpool(voice.run_admitted, voice.transcribe, raw_cfg, bytes(data), anonymous=user is None))
+ except voice.Busy as e:
+ raise HTTPException(429, str(e), headers={"Retry-After": "60"}) from None
+ except TimeoutError:
+ raise HTTPException(408, "clip upload timed out") from None
+ except voice.ClipTooLong as e:
+ raise HTTPException(413, str(e)) from None
+ except voice.InvalidClip as e:
+ raise HTTPException(400, str(e)) from None
+ except RuntimeError:
+ # Provider errors may contain URLs, credentials or response bodies; don't return those to callers.
+ raise HTTPException(503, "speech-to-text service is unavailable") from None
@router.post(
"/speak",
response_class=Response,
- responses={200: {"content": {"audio/mpeg": {}}, "description": "the text read aloud"}, 204: {"description": "the browser reads it"}},
+ responses={200: {"content": {"audio/mpeg": {}, "audio/wav": {}}, "description": "the text read aloud"},
+ 204: {"description": "the browser reads it"}, 429: {"description": "Voice service busy"},
+ 503: {"description": "Server speech unavailable (server_only=true)"}},
)
-async def speak_text(body: SpeakRequest, user: CurrentUser, cfg: Cfg) -> Response:
- """The text read aloud by the text-to-speech model (Settings → AI assistant → Voice). 204 when there's none, or it
- failed: the browser reads it instead."""
- out = await run_in_threadpool(voice.speak, cfg, body.text, body.voice)
+async def speak_text(body: SpeakRequest, user: RawUser, cfg: Cfg) -> Response:
+ """Raw text to audio using configured providers. Set server_only for a real error instead of browser fallback.
+ Anonymous callers cannot select private account voices; they use the administrator's configured default."""
+ if not body.text.strip():
+ raise HTTPException(422, "text must not be blank")
+ if user is None and body.voice is not None:
+ raise HTTPException(403, "sign in to select a voice")
+ try:
+ out = await run_in_threadpool(voice.run_admitted, voice.speak, cfg, body.text, body.voice, anonymous=user is None)
+ except voice.Busy as e:
+ raise HTTPException(429, str(e), headers={"Retry-After": "60"}) from None
+ except RuntimeError:
+ # Includes activity.OverBudget. Never expose provider or budget internals.
+ out = None
if out is None:
+ if body.server_only or user is None:
+ raise HTTPException(503, "text-to-speech service is unavailable")
return Response(status_code=204)
audio, media = out
return Response(content=audio, media_type=media, headers={"Cache-Control": "no-store"})
diff --git a/fastapi_backend/app/domain/podcasts.py b/fastapi_backend/app/domain/podcasts.py
index daeb8da93..c7f41d178 100644
--- a/fastapi_backend/app/domain/podcasts.py
+++ b/fastapi_backend/app/domain/podcasts.py
@@ -52,6 +52,20 @@
CITED = ("claim", "recap") # lines that state facts, so need a source
RUN_CHARS = 700 # an excerpt is a run of neighbouring lines up to about this long (a document's stays on one page)
LINE_CHARS = 600 # longest script line kept
+MODES = ("original", "summary", "explanation", "dialogue")
+MODE_GUIDES = {
+ "summary": (
+ "Make a concise summary, spoken by one narrator, speaker a only. Keep the source's essential points and "
+ "qualifications; leave out banter, invented dialogue and unnecessary detail. Every line must be a cited claim "
+ "or recap. These single-narrator instructions override any two-host instructions in the template."
+ ),
+ "explanation": (
+ "Explain the sources clearly, step by step, for a learner, spoken by one narrator, speaker a only. Define "
+ "unfamiliar terms using the sources and make their connections clear, without adding unsupported facts, "
+ "banter or invented dialogue. Every line must be a cited claim or recap. These single-narrator instructions "
+ "override any two-host instructions in the template."
+ ),
+}
class Problem(ValueError):
@@ -78,10 +92,13 @@ def selected_recordings(sel):
# ---------- options ----------
-def options(cfg, length=None, style=None, prompt=None, voices=None):
+def options(cfg, length=None, style=None, prompt=None, voices=None, mode=None):
p = cfg.get("podcasts") or {}
+ mode = "dialogue" if mode is None else mode
+ if mode not in MODES:
+ raise Problem(f"mode is one of {', '.join(MODES)}")
length = 10 if length is None else int(length)
- if not 1 <= length <= int(p.get("max_minutes") or 30):
+ if mode != "original" and not 1 <= length <= int(p.get("max_minutes") or 30):
raise Problem(f"length is 1 to {p.get('max_minutes') or 30} minutes")
style = style or "deep-dive"
if style not in STYLES:
@@ -95,7 +112,9 @@ def options(cfg, length=None, style=None, prompt=None, voices=None):
raise Problem("voices are {a, b}: a voice's name or id for each host")
if v.strip():
picked[k] = v.strip()
- return store.clean({"length": length, "style": style, "prompt": prompt or None, "voices": picked or None})
+ if mode != "dialogue":
+ picked.pop("b", None)
+ return store.clean({"length": length, "style": style, "prompt": prompt or None, "voices": picked or None, "mode": mode})
def hosts(cfg):
@@ -167,6 +186,7 @@ def create(
title=None,
voices=None,
space=None,
+ mode=None,
):
"""A new episode about what was picked, queued to be made. Returns {episode, job, namespace, placed}.
@@ -175,7 +195,7 @@ def create(
`space` is where it goes when the caller has decided (a routine's podcast episode node): it must fit (`fits`),
Problem otherwise."""
sel = clean_selection(selection)
- opts = options(cfg, length, style, prompt, voices)
+ opts = options(cfg, length, style, prompt, voices, mode)
rids = selected_recordings(sel)
rows = db.rows("SELECT record::id(id) AS id, space, title FROM recording WHERE id IN $ids", ids=[R("recording", r) for r in rids])
if len(rows) != len(rids):
@@ -183,6 +203,9 @@ def create(
also = sorted({int(x) for x in also or ()})
if readable is not None and any(r["space"] not in readable and r["id"] not in also for r in rows):
raise KeyError("recording")
+ if opts["mode"] == "original":
+ # Refuse unavailable or oversized text before making an episode or its namespace. The job checks again.
+ resolve_original(db, cfg, sel, readable=readable, also=also)
if space is not None:
if not fits(db, int(space), {r["space"] for r in rows}):
raise Problem("an episode about these sources can't go in that namespace: people there couldn't read them all")
@@ -229,12 +252,14 @@ def create(
}
),
)
- jid = jobs.enqueue(db, rid, steps(), by=by)
+ jid = jobs.enqueue(db, rid, steps(opts["mode"]), by=by)
return {"episode": rid, "job": jid, "namespace": sid, "placed": placed}
-def steps():
- return [{"type": "podcast", "stage": s, "name": STAGE_NAMES[s]} for s in STAGES] + AFTER
+def steps(mode="dialogue"):
+ stages = ("gather", "audio", "publish") if mode == "original" else STAGES
+ # An original reading makes no LLM calls, including the usual post-publication analysis and summary.
+ return [{"type": "podcast", "stage": s, "name": STAGE_NAMES[s]} for s in stages] + ([] if mode == "original" else AFTER)
def _collection(db, sid):
@@ -248,9 +273,9 @@ def _collection(db, sid):
def regenerate(db, rid, by=None):
"""Make an episode again from what was asked for (the sources are read afresh). Returns the job."""
- get_row(db, rid)
+ p = get_row(db, rid)
_save(db, rid, status="queued", error=None)
- return jobs.enqueue(db, rid, steps(), by=by)
+ return jobs.enqueue(db, rid, steps(p["request"].get("mode", "dialogue")), by=by)
# ---------- reading ----------
@@ -273,6 +298,8 @@ def step(db, cfg, rid, say, spec=None):
raise ValueError(f"a podcast step's stage is one of {', '.join(STAGES)}")
if not db.one("SELECT id FROM $r", r=R("podcast", rid)):
raise jobs.Skip("this resource isn't a podcast episode")
+ if stage in ("plan", "write", "check") and get_row(db, rid)["request"].get("mode") == "original":
+ raise jobs.Skip("an original reading keeps the source text without a model")
if stage in ("plan", "write", "check") and not llm.configured(cfg):
raise Problem("no LLM is configured (Settings → Models)")
_save(db, rid, status=STATUS[stage], error=None)
@@ -398,12 +425,81 @@ def resolve(db, cfg, sel, prompt=None, readable=None, budget=None, also=()):
return [_excerpt(n, rid, run, recs[rid], names, space_names) for n, (rid, run) in enumerate(chosen, 1)]
+def resolve_original(db, cfg, sel, readable=None, also=()):
+ """Every selected segment, unchanged and in reading order. Exact line selections add no neighbouring text.
+
+ Unlike the generation context, this is never sampled or shortened. Missing sources, missing selected lines,
+ unprocessed sources and text larger than podcasts.context_chars fail explicitly.
+ """
+ limit = int((cfg.get("podcasts") or {}).get("context_chars") or 24000)
+ rids = selected_recordings(sel)
+ allowed = set(also or ())
+ recs = {
+ r["id"]: r
+ for r in db.rows("SELECT record::id(id) AS id, title, space FROM recording WHERE id IN $ids", ids=[R("recording", i) for i in rids])
+ if readable is None or r["space"] in readable or r["id"] in allowed
+ }
+ if len(recs) != len(rids):
+ raise Problem("not all selected sources can be read any more; select the sources again")
+ whole = set(sel.get("recordings") or ())
+ picked = {}
+ for x in sel.get("excerpts") or ():
+ picked.setdefault(x["recording"], set()).add(x["idx"])
+ sources, count = [], 0
+ space_names = store.space_names(db)
+ for rid in rids:
+ if rid in whole:
+ segs = db.rows("SELECT idx, t0, t1, text, page FROM segment WHERE recording = $r ORDER BY idx", r=rid)
+ else:
+ segs = db.rows(
+ "SELECT idx, t0, t1, text, page FROM segment WHERE recording = $r AND idx IN $i ORDER BY idx",
+ r=rid,
+ i=sorted(picked[rid]),
+ )
+ if {s["idx"] for s in segs} != picked[rid]:
+ raise Problem("a selected line is no longer available; select the source text again")
+ if not segs or not any((s.get("text") or "").strip() for s in segs):
+ raise Problem("a selected source has no text yet; finish importing or processing it first")
+ for s in segs:
+ text = s.get("text") or ""
+ count += len(text)
+ if count > limit:
+ raise Problem(
+ f"original reading exceeds {limit} characters (podcasts.context_chars); select fewer sources "
+ "or passages, or raise the limit in Settings. Nothing was shortened."
+ )
+ source = _excerpt(len(sources) + 1, rid, [{**s, "text": text}], recs[rid], {}, space_names)
+ sources.append(source)
+ return sources
+
+
+def original_lines(sources):
+ """Verbatim source text in provider-sized pieces, never through tidy() or a model. Joining each source's pieces
+ restores it exactly, including whitespace; citation numbers remain metadata rather than spoken text."""
+ lines = []
+ for source in sources:
+ rest = source["text"]
+ while rest:
+ end = len(rest)
+ if end > LINE_CHARS:
+ end = rest.rfind(" ", 0, LINE_CHARS) + 1
+ if end < LINE_CHARS // 2:
+ end = LINE_CHARS
+ lines.append({"idx": len(lines), "speaker": "a", "kind": "original", "text": rest[:end], "citations": [source["n"]]})
+ rest = rest[end:]
+ return lines
+
+
def gather(db, cfg, rid, say):
p = get_row(db, rid)
req = p["request"]
readable = set(p["readable"]) if p.get("readable") is not None else None
- sources = resolve(db, cfg, req["selection"], req.get("prompt"), readable, also=p.get("also") or ())
- _save(db, rid, sources=sources, outline=None, lines=None, checks=None, audio=None)
+ original = req.get("mode") == "original"
+ if original:
+ sources = resolve_original(db, cfg, req["selection"], readable, also=p.get("also") or ())
+ else:
+ sources = resolve(db, cfg, req["selection"], req.get("prompt"), readable, also=p.get("also") or ())
+ _save(db, rid, sources=sources, outline=None, lines=original_lines(sources) if original else None, checks=[], audio=None)
say(f"gathered {len(sources)} excerpt(s) from {len({s['ref']['recording'] for s in sources})} source(s)")
@@ -527,8 +623,13 @@ def template(db, stage):
def _ask(db, cfg, stage, ctx):
t = template(db, stage)
prompt = templates.render_body(t["body"], ctx, "prompt")
+ system = t.get("system") or SYSTEM
+ mode = ctx["request"].get("mode", "dialogue")
+ if mode in MODE_GUIDES:
+ # Also covers existing installations' saved two-host templates without overwriting their custom versions.
+ system += "\n\n" + MODE_GUIDES[mode]
model = (cfg.get("podcasts") or {}).get("model") or None
- value = llm.json_out(cfg, t.get("system") or SYSTEM, prompt, t.get("schema") or {"type": "object"}, model)
+ value = llm.json_out(cfg, system, prompt, t.get("schema") or {"type": "object"}, model)
return value, {"template": t["id"], "version": t["version"]}
@@ -537,6 +638,7 @@ def _context(cfg, p, **more):
wpm = int((cfg.get("podcasts") or {}).get("words_per_minute") or 150)
return {
"request": {
+ "mode": req.get("mode", "dialogue"),
"prompt": req.get("prompt"),
"style": req["style"],
"style_guide": STYLES[req["style"]],
@@ -581,7 +683,7 @@ def plan(db, cfg, rid, say):
# ---------- write ----------
-def tidy(raw, known):
+def tidy(raw, known, mode="dialogue"):
"""Script lines as the model wrote them, made safe to keep: known speakers and kinds, text trimmed, citations only to
excerpts it was given. Returns (lines, problems): each problem says what was dropped."""
lines, problems = [], []
@@ -599,6 +701,9 @@ def tidy(raw, known):
if len(kept) < len([c for c in cites if c is not None]):
problems.append(f"line {k}: dropped citations to excerpts it wasn't given")
kind = l.get("kind") if l.get("kind") in KINDS else ("claim" if kept else "banter")
+ if mode in MODE_GUIDES:
+ speaker = "a"
+ kind = "recap" if kind == "recap" else "claim" # every narration line goes through the citation check
lines.append({"speaker": speaker, "kind": kind, "text": text[:LINE_CHARS], "citations": kept})
for i, l in enumerate(lines):
l["idx"] = i
@@ -610,8 +715,9 @@ def write(db, cfg, rid, say):
if not p.get("outline"):
raise Problem("there's no outline to write from")
out, used = _ask(db, cfg, "write", _context(cfg, p, outline=p["outline"]))
- lines, problems = tidy(out.get("lines"), {s["n"] for s in p["sources"]})
- if len(lines) < 2:
+ mode = p["request"].get("mode", "dialogue")
+ lines, problems = tidy(out.get("lines"), {s["n"] for s in p["sources"]}, mode)
+ if len(lines) < (2 if mode == "dialogue" else 1):
raise Problem("the model wrote no usable script")
_save(db, rid, lines=lines, checks=None, templates={**(p.get("templates") or {}), "write": used})
for x in problems[:20]:
@@ -711,7 +817,7 @@ def check(db, cfg, rid, say):
verdicts, used = _verdicts(db, cfg, p, todo) if todo else ({}, None)
second = lambda new: _verdicts(db, cfg, p, list(new.values()))[0] # noqa: E731
kept, log = apply_checks(lines, verdicts, known, second)
- if len(kept) < 2:
+ if len(kept) < (2 if p["request"].get("mode", "dialogue") == "dialogue" else 1):
raise Problem("nothing in the script held up against its sources")
for i, l in enumerate(kept):
l["idx"] = i
@@ -744,6 +850,8 @@ def voices(cfg, req=None):
first = speech.tts_voice(cfg) # None for say: the Mac's own voice
asked = (req or {}).get("voices") or {}
a = asked.get("a") or p.get("voice_a") or first
+ if (req or {}).get("mode", "dialogue") != "dialogue":
+ return {"a": a}
second = kokoro.partner(a) if provider == "kokoro" else SECOND_VOICE.get(provider, first)
return {"a": a, "b": asked.get("b") or p.get("voice_b") or second}
diff --git a/fastapi_backend/app/domain/voice.py b/fastapi_backend/app/domain/voice.py
index 27d20780e..ff21ebf33 100644
--- a/fastapi_backend/app/domain/voice.py
+++ b/fastapi_backend/app/domain/voice.py
@@ -13,6 +13,11 @@
import logging
import os
import tempfile
+import subprocess
+from collections import deque
+from contextlib import contextmanager
+
+import numpy as np
import threading
import time
@@ -21,6 +26,88 @@
log = logging.getLogger(__name__)
IDLE_SECONDS = 600
MAX_BYTES = 25 * 1024 * 1024
+MAX_SECONDS = 300
+DECODE_SECONDS = 30
+# Deliberately per process. Deployments with multiple workers need an edge/global budget too.
+_JOBS = threading.BoundedSemaphore(2)
+_UPLOADS = threading.BoundedSemaphore(2)
+_RATE_LOCK = threading.Lock()
+_PUBLIC_JOBS = deque()
+PUBLIC_PER_MINUTE = 10
+
+
+class Busy(RuntimeError):
+ pass
+
+
+class ClipTooLong(ValueError):
+ pass
+
+
+class InvalidClip(ValueError):
+ pass
+
+
+@contextmanager
+def admission(anonymous=False):
+ """No waiting queue; anonymous callers share a budget, never trust forwarding headers."""
+ if not _JOBS.acquire(blocking=False):
+ raise Busy("voice service is busy; try again shortly")
+ try:
+ if anonymous:
+ with _RATE_LOCK:
+ now = time.monotonic()
+ while _PUBLIC_JOBS and _PUBLIC_JOBS[0] <= now - 60:
+ _PUBLIC_JOBS.popleft()
+ if len(_PUBLIC_JOBS) >= PUBLIC_PER_MINUTE:
+ raise Busy("public voice request limit reached; try again in a minute")
+ _PUBLIC_JOBS.append(now)
+ yield
+ finally:
+ _JOBS.release()
+
+
+@contextmanager
+def upload_admission():
+ """Bound in-memory request bodies separately from running provider jobs."""
+ if not _UPLOADS.acquire(blocking=False):
+ raise Busy("voice service is busy; try again shortly")
+ try:
+ yield
+ finally:
+ _UPLOADS.release()
+
+
+def run_admitted(function, *args, anonymous=False):
+ """The worker owns its slot until it finishes, even if its HTTP waiter is cancelled."""
+ with admission(anonymous=anonymous):
+ return function(*args)
+
+
+def decode_clip(path):
+ """Decode at most 301 seconds, not an entire potentially hours-long compressed upload.
+
+ Only an uploaded local media file is accepted; playlists and network protocols cannot fetch other resources.
+ """
+ try:
+ out = subprocess.run(
+ ["ffmpeg", "-nostdin", "-v", "error", "-protocol_whitelist", "file,pipe",
+ "-format_whitelist", ingest.MEDIA_DEMUXERS, "-i", str(path),
+ "-t", str(MAX_SECONDS + 1), "-f", "f32le", "-acodec", "pcm_f32le",
+ "-ac", "1", "-ar", str(ingest.SR), "-"],
+ capture_output=True, timeout=DECODE_SECONDS,
+ )
+ except subprocess.TimeoutExpired:
+ raise InvalidClip("audio decoding timed out") from None
+ except OSError:
+ raise RuntimeError("audio decoder is unavailable") from None
+ if out.returncode or len(out.stdout) % 4:
+ raise InvalidClip("could not decode audio")
+ audio = np.frombuffer(out.stdout, dtype=np.float32)
+ if len(audio) > MAX_SECONDS * ingest.SR:
+ raise ClipTooLong("clips must be no longer than 300 seconds")
+ return audio
+
_LOCK = threading.Lock()
_ENGINE = {"key": None, "engine": None, "used": 0.0}
@@ -95,18 +182,21 @@ def transcribe(cfg, data):
if not data:
return {"text": "", "language": None, "engine": None, "seconds": 0.0}
if len(data) > MAX_BYTES:
- raise ValueError("that clip is too long; say it in shorter parts")
+ raise ClipTooLong("clips must be no larger than 25 MiB")
fd, path = tempfile.mkstemp(prefix="lens-voice-", suffix=".clip")
try:
with os.fdopen(fd, "wb") as f:
f.write(data)
- audio = ingest.decode(path)
+ audio = decode_clip(path)
finally:
os.unlink(path)
seconds = len(audio) / ingest.SR
if seconds < 0.3:
return {"text": "", "language": None, "engine": None, "seconds": round(seconds, 2)}
- e = engine(cfg)
+ try:
+ e = engine(cfg)
+ except ingest.EngineMissing:
+ raise RuntimeError("speech-to-text engine is unavailable") from None
t = threading.Timer(IDLE_SECONDS + 5, let_go)
t.daemon = True
t.start()
diff --git a/fastapi_backend/app/schemas/podcasts.py b/fastapi_backend/app/schemas/podcasts.py
index 6eae6ab99..19ed50ec5 100644
--- a/fastapi_backend/app/schemas/podcasts.py
+++ b/fastapi_backend/app/schemas/podcasts.py
@@ -9,6 +9,7 @@
from app.schemas.common import RequestModel, ResponseModel
Style = Literal["deep-dive", "recap", "debate", "beginner"]
+Mode = Literal["original", "summary", "explanation", "dialogue"]
class PickedLine(RequestModel):
@@ -22,14 +23,18 @@ class PodcastSelection(RequestModel):
class PodcastVoices(RequestModel):
- a: str | None = Field(None, max_length=100, description="the explaining host's voice")
- b: str | None = Field(None, max_length=100, description="the asking host's voice")
+ a: str | None = Field(None, max_length=100, description="the narrator or explaining host's voice id from GET /voice/voices")
+ b: str | None = Field(None, max_length=100, description="the second host's voice id; only used by dialogue mode")
class PodcastCreate(RequestModel):
selection: PodcastSelection
+ mode: Mode = Field(
+ "dialogue",
+ description="original reads the selected text unchanged; summary and explanation use one narrator; dialogue uses two hosts",
+ )
prompt: str | None = Field(None, max_length=2000, description='an angle, e.g. "explain it for a beginner"')
- length: int = Field(10, ge=1, le=120, description="minutes (up to podcasts.max_minutes)")
+ length: int = Field(10, ge=1, le=120, description="target minutes (up to podcasts.max_minutes); original follows the source's length")
style: Style = "deep-dive"
title: str | None = Field(None, max_length=200)
voices: PodcastVoices | None = None
diff --git a/fastapi_backend/app/schemas/voice.py b/fastapi_backend/app/schemas/voice.py
index eb9c75dde..f919e72c2 100644
--- a/fastapi_backend/app/schemas/voice.py
+++ b/fastapi_backend/app/schemas/voice.py
@@ -19,7 +19,8 @@ class Heard(ResponseModel):
class SpeakRequest(RequestModel):
- text: str = Field(max_length=8000)
+ text: str = Field(min_length=1, max_length=4000)
+ server_only: bool = Field(False, description="Return 503 rather than browser fallback when server speech is unavailable")
voice: str | None = Field(None, max_length=200, description="read in this voice (an id from GET /voice/voices); else the settings'")
diff --git a/fastapi_backend/tests/api/test_podcasts.py b/fastapi_backend/tests/api/test_podcasts.py
index 2de6f3abd..9837d0712 100644
--- a/fastapi_backend/tests/api/test_podcasts.py
+++ b/fastapi_backend/tests/api/test_podcasts.py
@@ -106,3 +106,26 @@ def test_agents_make_episodes(client, db, cfg, env):
assert got["changed_by_fact_check"] and got["sources"][0]["n"] == 1
assert "not found" in tool_error(client, env["ca"], "get_podcast", episode_id=out["episode"])
assert "not found" in tool_error(client, env["vi"], "get_podcast", episode_id=a)
+
+
+@pytest.mark.parametrize("mode", ["original", "summary", "explanation", "dialogue"])
+def test_modes_preserve_source_access_checks(client, env, mode):
+ a, _b, c = env["ids"]
+ body = {"selection": {"recordings": [c]}, "mode": mode}
+ assert client.post("/api/v1/podcasts", headers=env["ed"], json=body).status_code == 404
+ assert client.post("/api/v1/podcasts", json={**body, "selection": {"recordings": [a]}}).status_code == 401
+ assert client.post("/api/v1/podcasts", headers=env["vi"], json={**body, "selection": {"recordings": [a]}}).status_code == 400
+
+
+def test_original_mode_api_and_oversize_error(client, db, cfg, env):
+ a, _b, _c = env["ids"]
+ body = {"selection": {"excerpts": [{"recording": a, "idx": 0}]}, "mode": "original", "voices": {"a": "echo"}}
+ r = client.post("/api/v1/podcasts", headers=env["ed"], json=body)
+ assert r.status_code == 202, r.text
+ got = client.get(f"/api/v1/podcasts/{r.json()['episode']}", headers=env["vi"]).json()
+ assert got["request"]["mode"] == "original" and got["request"]["voices"] == {"a": "echo"}
+ assert client.post("/api/v1/podcasts", headers=env["ed"], json={**body, "mode": "song"}).status_code == 422
+ db.q("UPDATE segment SET text = $t WHERE recording = $r AND idx = 0", t="x" * (cfg["podcasts"]["context_chars"] + 1), r=a)
+ too_big = client.post("/api/v1/podcasts", headers=env["ed"], json=body)
+ assert too_big.status_code == 400 and "Nothing was shortened" in too_big.json()["detail"]
+
diff --git a/fastapi_backend/tests/api/test_voice_raw.py b/fastapi_backend/tests/api/test_voice_raw.py
new file mode 100644
index 000000000..5640125ee
--- /dev/null
+++ b/fastapi_backend/tests/api/test_voice_raw.py
@@ -0,0 +1,134 @@
+"""The standalone voice API has a narrow, opt-in anonymous boundary."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+
+from app.api.deps import current_user, get_cfg, optional_user
+from app.api.v1.routes.voice import router
+from app.domain import voice
+
+
+@pytest.fixture
+def raw(monkeypatch):
+ app = FastAPI()
+ app.include_router(router, prefix="/api/v1")
+ cfg = {"voice": {"public_enabled": False}}
+ app.dependency_overrides[get_cfg] = lambda: cfg
+ app.dependency_overrides[optional_user] = lambda: None
+ monkeypatch.setattr(voice, "can_transcribe", lambda cfg: "test")
+ monkeypatch.setattr(voice, "can_speak", lambda cfg: True)
+ monkeypatch.setattr(voice, "warm", lambda cfg: pytest.fail("anonymous GET must not warm models"))
+ voice._PUBLIC_JOBS.clear()
+ with TestClient(app) as client:
+ yield client, app, cfg
+ voice._PUBLIC_JOBS.clear()
+
+
+def test_anonymous_off_by_default_and_narrow_when_enabled(raw):
+ client, app, cfg = raw
+ for method, path, kw in [("get", "", {}), ("post", "/speak", {"json": {"text": "hi"}}),
+ ("post", "/transcribe", {"content": b"clip"})]:
+ assert getattr(client, method)("/api/v1/voice" + path, **kw).status_code == 401
+ cfg["voice"]["public_enabled"] = True
+ assert client.get("/api/v1/voice").status_code == 200
+ # Still delegates voice lists to the existing signed-in dependency, not raw_user.
+ def private():
+ raise HTTPException(401)
+
+ app.dependency_overrides[current_user] = private
+ assert client.get("/api/v1/voice/voices").status_code == 401
+ assert client.get("/api/v1/voice", headers={"Authorization": "Bearer invalid"}).status_code == 401
+ cfg["voice"]["public_enabled"] = False
+ assert client.get("/api/v1/voice").status_code == 401
+
+
+def test_raw_speech_errors_and_legacy_fallback(raw, monkeypatch):
+ client, app, cfg = raw
+ cfg["voice"]["public_enabled"] = True
+ monkeypatch.setattr(voice, "speak", lambda *a: None)
+ assert client.post("/api/v1/voice/speak", json={"text": "hi"}).status_code == 503
+ assert client.post("/api/v1/voice/speak", json={"text": "hi", "voice": "private"}).status_code == 403
+ app.dependency_overrides[optional_user] = lambda: SimpleNamespace(id=1)
+ assert client.post("/api/v1/voice/speak", json={"text": "hi"}).status_code == 204
+ assert client.post("/api/v1/voice/speak", json={"text": "hi", "server_only": True}).status_code == 503
+ for text in [" ", "x" * 4001]:
+ assert client.post("/api/v1/voice/speak", json={"text": text}).status_code == 422
+ monkeypatch.setattr(voice, "speak", lambda *a: (b"RIFF", "audio/wav"))
+ result = client.post("/api/v1/voice/speak", json={"text": "x" * 4000, "server_only": True})
+ assert result.content == b"RIFF" and result.headers["cache-control"] == "no-store"
+
+
+def test_raw_transcription_limits_and_errors(raw, monkeypatch):
+ client, _, cfg = raw
+ cfg["voice"]["public_enabled"] = True
+ monkeypatch.setattr(voice, "MAX_BYTES", 4)
+ assert client.post("/api/v1/voice/transcribe", content=b"12345").status_code == 413
+ assert client.post("/api/v1/voice/transcribe", content=b"").status_code == 400
+ for error, status in [(voice.ClipTooLong("long"), 413), (voice.InvalidClip("bad"), 400), (RuntimeError("secret"), 503)]:
+ def fail(*args, error=error):
+ raise error
+ monkeypatch.setattr(voice, "transcribe", fail)
+ result = client.post("/api/v1/voice/transcribe", content=b"123")
+ assert result.status_code == status
+ assert "secret" not in result.text
+
+
+def test_busy_and_global_anonymous_budget_ignore_forwarding_headers(raw, monkeypatch):
+ client, _, cfg = raw
+ cfg["voice"]["public_enabled"] = True
+ monkeypatch.setattr(voice, "speak", lambda *a: (b"audio", "audio/mpeg"))
+ with voice.admission(), voice.admission():
+ result = client.post("/api/v1/voice/speak", json={"text": "hi"})
+ assert result.status_code == 429 and result.headers["retry-after"] == "60"
+ for n in range(voice.PUBLIC_PER_MINUTE):
+ assert client.post("/api/v1/voice/speak", json={"text": "hi"}, headers={"X-Forwarded-For": str(n)}).status_code == 200
+ assert client.post("/api/v1/voice/speak", json={"text": "hi"}, headers={"X-Forwarded-For": "new"}).status_code == 429
+
+
+def test_only_admin_can_enable_public_voice(client, cfg, db):
+ from tests.helpers import login, make_user
+
+ make_user(db, "voice-admin@x.io", "voice password 1", admin=True)
+ admin = login(client, "voice-admin@x.io", "voice password 1")
+ make_user(db, "voice-viewer@x.io", "voice password 2", roles={"pods": "viewer"})
+ viewer = login(client, "voice-viewer@x.io", "voice password 2")
+ path = "/api/v1/settings/voice"
+ assert cfg["voice"]["public_enabled"] is False
+ assert client.put(path, json={"public_enabled": True}).status_code == 401
+ assert client.put(path, headers=viewer, json={"public_enabled": True}).status_code == 403
+ assert client.put(path, headers=admin, json={"public_enabled": "true"}).status_code == 400
+ assert client.put(path, headers=admin, json={"public_enabled": True}).status_code == 200
+ assert client.get("/api/v1/voice").status_code == 200
+ assert client.get("/api/v1/voice/voices").status_code == 401
+ assert client.get("/api/v1/settings").status_code == 401
+ assert client.put(path, headers=admin, json={"public_enabled": False}).status_code == 200
+ assert client.get("/api/v1/voice").status_code == 401
+
+
+def test_raw_provider_timeout_and_budget_errors(raw, monkeypatch):
+ client, app, cfg = raw
+ cfg["voice"]["public_enabled"] = True
+ cfg["speech"] = {"timeout": 1800}
+
+ def transcribe(raw_cfg, data):
+ assert raw_cfg["speech"]["timeout"] == 120
+ return {"text": "hello"}
+
+ monkeypatch.setattr(voice, "transcribe", transcribe)
+ result = client.post("/api/v1/voice/transcribe", content=b"audio")
+ assert result.status_code == 200 and result.headers["cache-control"] == "no-store"
+ assert cfg["speech"]["timeout"] == 1800 # archive settings are untouched
+
+ def over_budget(*args):
+ raise RuntimeError("private budget details")
+
+ monkeypatch.setattr(voice, "speak", over_budget)
+ result = client.post("/api/v1/voice/speak", json={"text": "hi", "server_only": True})
+ assert result.status_code == 503 and "private" not in result.text
+ app.dependency_overrides[optional_user] = lambda: SimpleNamespace(id=1)
+ assert client.post("/api/v1/voice/speak", json={"text": "hi"}).status_code == 204
diff --git a/fastapi_backend/tests/domain/test_podcasts.py b/fastapi_backend/tests/domain/test_podcasts.py
index 0761a6bee..8c544a6cf 100644
--- a/fastapi_backend/tests/domain/test_podcasts.py
+++ b/fastapi_backend/tests/domain/test_podcasts.py
@@ -47,7 +47,7 @@ def test_selection_is_cleaned():
def test_options_are_checked(cfg):
- assert podcasts.options(cfg) == {"length": 10, "style": "deep-dive"}
+ assert podcasts.options(cfg) == {"length": 10, "style": "deep-dive", "mode": "dialogue"}
assert podcasts.options(cfg, voices={"a": " af_heart ", "b": ""})["voices"] == {"a": "af_heart"}
assert podcasts.options(cfg, 5, "beginner", " explain it simply ")["prompt"] == "explain it simply"
for bad in ({"length": 0}, {"length": 31}, {"style": "opera"}, {"prompt": "x" * 2001}, {"voices": {"c": "x"}}, {"voices": {"a": 3}}):
@@ -55,6 +55,17 @@ def test_options_are_checked(cfg):
podcasts.options(cfg, **bad)
+def test_modes_keep_dialogue_defaults_and_one_narrator(cfg):
+ assert podcasts.options(cfg, mode="summary", voices={"a": "alloy", "b": "onyx"})["voices"] == {"a": "alloy"}
+ with pytest.raises(podcasts.Problem, match="mode is one of"):
+ podcasts.options(cfg, mode="song")
+ # Original duration follows its source, not a hidden generated-episode target.
+ cfg["podcasts"]["max_minutes"] = 5
+ assert podcasts.options(cfg, mode="original", length=20)["length"] == 20
+ assert [s["stage"] for s in podcasts.steps("original")] == ["gather", "audio", "publish"]
+ assert podcasts.voices(cfg, {"mode": "explanation", "voices": {"a": "echo", "b": "onyx"}}) == {"a": "echo"}
+
+
def test_runs_and_picking_within_a_budget():
segs = [{"idx": i, "t0": i * 1000, "text": ("gpu " if i == 7 else "") + "word " * 20, "page": None} for i in range(10)]
segs[5]["page"] = 1 # a page break starts a new run
@@ -90,6 +101,42 @@ def test_resolve_numbers_excerpts_and_respects_access(db, cfg, folder):
podcasts.resolve(db, cfg, podcasts.clean_selection({"recordings": [c]}), readable={pods})
+def test_original_reads_exact_picked_lines_without_neighbours_or_speaker_prefixes(db, cfg, folder):
+ a, _b, c = seed(db, cfg, folder)
+ pods = store.ns_id(db, "pods")
+ sel = podcasts.clean_selection({"excerpts": [{"recording": a, "idx": 1}]})
+ original = " Original words, unchanged.\n" + "long word " * 200 + "END. "
+ db.q("UPDATE segment SET text = $t WHERE recording = $r AND idx = 1", t=original, r=a)
+ out = podcasts.resolve_original(db, cfg, sel, readable={pods})
+ assert len(out) == 1 and out[0]["text"] == original
+ assert out[0]["ref"] == {"kind": "segment", "recording": a, "idx0": 1, "idx1": 1}
+ lines = podcasts.original_lines(out)
+ assert "".join(line["text"] for line in lines) == original
+ assert all(len(line["text"]) <= podcasts.LINE_CHARS and line["speaker"] == "a" and line["citations"] == [1] for line in lines)
+ with pytest.raises(podcasts.Problem, match="not all selected sources"):
+ podcasts.resolve_original(db, cfg, {"recordings": [a, c]}, readable={pods})
+ with pytest.raises(podcasts.Problem, match="selected line is no longer available"):
+ podcasts.resolve_original(db, cfg, {"excerpts": [{"recording": a, "idx": 999}]}, readable={pods})
+
+
+def test_original_refuses_oversized_sources_before_creating_an_episode(db, cfg, folder):
+ a, _b, _c = seed(db, cfg, folder)
+ cfg["podcasts"]["context_chars"] = 2000
+ db.q("UPDATE segment SET text = $t WHERE recording = $r AND idx = 0", t="x" * 2001, r=a)
+ with pytest.raises(podcasts.Problem, match="Nothing was shortened"):
+ podcasts.create(db, cfg, {"recordings": [a]}, mode="original", admin=True)
+ assert podcasts.list_episodes(db) == []
+ assert "podcasts" not in store.space_names(db).values()
+
+
+def test_original_stays_in_its_sources_vault(db, cfg, folder):
+ a, _b, _c = seed(db, cfg, folder)
+ pods = store.ns_id(db, "pods")
+ db.q("UPSERT $r MERGE {vault: true}", r=R("data_key", pods))
+ out = podcasts.create(db, cfg, {"recordings": [a]}, mode="original", admin=True)
+ assert (out["namespace"], out["placed"]) == (pods, "sources")
+
+
# ---------- script ----------
def test_tidy_keeps_only_what_is_safe():
lines, problems = podcasts.tidy(
@@ -294,6 +341,42 @@ def test_no_llm_fails_with_a_reason(db, cfg, folder):
assert "no LLM is configured" in jobs.get(db, out["job"])["error"]
+def test_original_job_never_calls_a_model_and_regenerates_in_original_mode(db, cfg, folder, monkeypatch):
+ a, _b, _c = seed(db, cfg, folder)
+ cfg["llm"].update(base_url=None, model=None)
+
+ def no_model(*args, **kwargs):
+ pytest.fail("an original reading must not call a language model")
+
+ monkeypatch.setattr(podcasts.llm, "json_out", no_model)
+ expected = db.rows("SELECT idx, text FROM segment WHERE recording = $r ORDER BY idx", r=a)
+ out = podcasts.create(db, cfg, {"recordings": [a]}, mode="original", admin=True)
+ drain(db, cfg)
+ assert jobs.get(db, out["job"])["status"] == "succeeded"
+ ep = podcasts.view(db, out["episode"])
+ assert ep["status"] == "script_only" and not ep["checks"] and not ep.get("templates")
+ assert [s["text"] for s in ep["sources"]] == [s["text"] for s in expected]
+ for source in ep["sources"]:
+ assert "".join(line["text"] for line in ep["lines"] if source["n"] in line["citations"]) == source["text"]
+ assert {line["speaker"] for line in ep["lines"]} == {"a"}
+ jid = podcasts.regenerate(db, out["episode"])
+ assert [s["stage"] for s in jobs.get(db, jid)["steps"]] == ["gather", "audio", "publish"]
+ drain(db, cfg)
+ assert jobs.get(db, jid)["status"] == "succeeded"
+
+
+@pytest.mark.parametrize("mode", ["summary", "explanation"])
+def test_narration_modes_are_cited_checked_and_single_speaker(db, cfg, folder, llm, mode):
+ a, _b, _c = seed(db, cfg, folder)
+ out = podcasts.create(db, cfg, {"recordings": [a]}, mode=mode, admin=True)
+ drain(db, cfg)
+ assert jobs.get(db, out["job"])["status"] == "succeeded"
+ ep = podcasts.view(db, out["episode"])
+ assert ep["request"]["mode"] == mode and ep["checks"]
+ assert all(line["speaker"] == "a" and line["kind"] in podcasts.CITED and line["citations"] for line in ep["lines"])
+ assert any(podcasts.MODE_GUIDES[mode] in m["messages"][0]["content"] for m in llm.seen)
+
+
def test_podcast_step_skips_other_recordings(db, cfg, folder):
a, _b, _c = seed(db, cfg, folder)
said = []
@@ -418,3 +501,15 @@ def test_tts_failure_fails_the_job(db, cfg, folder, llm):
drain(db, cfg)
assert "text to speech failed on line 1" in jobs.get(db, out["job"])["error"]
assert podcasts.view(db, out["episode"])["error"].startswith("recording the voices: ")
+
+
+def test_original_audio_uses_only_the_selected_narrator(db, cfg, folder, tts):
+ a, _b, _c = seed(db, cfg, folder)
+ cfg["llm"].update(base_url=None, model=None)
+ out = podcasts.create(db, cfg, {"excerpts": [{"recording": a, "idx": 0}]}, mode="original", voices={"a": "echo"}, admin=True)
+ drain(db, cfg)
+ assert jobs.get(db, out["job"])["status"] == "succeeded"
+ ep = podcasts.view(db, out["episode"])
+ assert ep["status"] == "ready" and ep["voicing"]["voices"] == {"a": "echo"}
+ assert [(b["input"], b["voice"]) for b in tts.seen] == [("Welcome back.", "echo")]
+
diff --git a/fastapi_backend/tests/domain/test_voice_raw.py b/fastapi_backend/tests/domain/test_voice_raw.py
new file mode 100644
index 000000000..a11347051
--- /dev/null
+++ b/fastapi_backend/tests/domain/test_voice_raw.py
@@ -0,0 +1,103 @@
+"""Bound decoding before invoking a model, even for tiny compressed long recordings."""
+
+import asyncio
+import subprocess
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+from starlette.concurrency import run_in_threadpool
+
+from app.domain import ingest, voice
+
+
+def test_decoder_bounds_duration_and_wall_time(monkeypatch):
+ def run(args, **kwargs):
+ assert args[args.index("-t") + 1] == "301"
+ assert args[args.index("-protocol_whitelist") + 1] == "file,pipe"
+ assert kwargs["timeout"] == 30
+ return SimpleNamespace(returncode=0, stdout=b"\0" * (301 * ingest.SR * 4))
+
+ monkeypatch.setattr(voice.subprocess, "run", run)
+ with pytest.raises(voice.ClipTooLong):
+ voice.decode_clip("upload.clip")
+
+
+def test_decoder_timeout_is_safe_error(monkeypatch):
+ def run(*args, **kwargs):
+ raise subprocess.TimeoutExpired("ffmpeg", 30)
+
+ monkeypatch.setattr(voice.subprocess, "run", run)
+ with pytest.raises(voice.InvalidClip, match="timed out"):
+ voice.decode_clip("upload.clip")
+
+
+def test_invalid_clip_and_missing_decoder(monkeypatch):
+ monkeypatch.setattr(voice.subprocess, "run", lambda *a, **kw: SimpleNamespace(returncode=1, stdout=b"", stderr=b"secret"))
+ with pytest.raises(voice.InvalidClip, match="could not decode"):
+ voice.decode_clip("upload.clip")
+
+ def missing(*args, **kwargs):
+ raise FileNotFoundError("ffmpeg")
+
+ monkeypatch.setattr(voice.subprocess, "run", missing)
+ with pytest.raises(RuntimeError, match="unavailable"):
+ voice.decode_clip("upload.clip")
+
+
+def test_failed_admission_releases_slot():
+ voice._PUBLIC_JOBS.clear()
+ try:
+ for _ in range(voice.PUBLIC_PER_MINUTE):
+ with voice.admission(anonymous=True):
+ pass
+ with pytest.raises(voice.Busy), voice.admission(anonymous=True):
+ pass
+ with voice.admission(), voice.admission():
+ with pytest.raises(voice.Busy), voice.admission():
+ pass
+ finally:
+ voice._PUBLIC_JOBS.clear()
+
+
+def test_cancelled_waiters_do_not_release_running_worker_slots():
+ async def scenario():
+ started = [threading.Event(), threading.Event()]
+ release = threading.Event()
+ finished = [threading.Event(), threading.Event()]
+
+ def work(index):
+ started[index].set()
+ assert release.wait(5), "test did not release provider work"
+
+ def admitted_work(index):
+ try:
+ voice.run_admitted(work, index)
+ finally:
+ finished[index].set()
+
+ tasks = [asyncio.create_task(run_in_threadpool(admitted_work, index)) for index in range(2)]
+ try:
+ for _ in range(100):
+ if all(event.is_set() for event in started):
+ break
+ await asyncio.sleep(0.01)
+ assert all(event.is_set() for event in started)
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ with pytest.raises(voice.Busy), voice.admission():
+ pass
+ finally:
+ release.set()
+ for _ in range(100):
+ if all(event.is_set() for event in finished):
+ break
+ await asyncio.sleep(0.01)
+ await asyncio.gather(*tasks, return_exceptions=True)
+ assert all(event.is_set() for event in finished)
+ with voice.admission(), voice.admission():
+ pass
+
+ asyncio.run(scenario())
diff --git a/nextjs-frontend/app/(app)/voice/page.tsx b/nextjs-frontend/app/(app)/voice/page.tsx
new file mode 100644
index 000000000..6c6eea957
--- /dev/null
+++ b/nextjs-frontend/app/(app)/voice/page.tsx
@@ -0,0 +1,9 @@
+import type { Metadata } from "next";
+
+import { VoiceWorkspace } from "@/components/voice/voice-workspace";
+
+export const metadata: Metadata = { title: "Voice" };
+
+export default function VoicePage() {
+ return