From 05339b7baea46598cffa8dda2a427ac9d6c8c700 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:55:03 +0000 Subject: [PATCH 1/2] feat(voice): read aloud with Gemini TTS or macOS say, and list each provider's voices Gemini (its 30 prebuilt voices, PCM wrapped as WAV) and macOS say (the Mac's own voices, text on stdin) join OpenAI-compatible, ElevenLabs and Deepgram as text-to-speech providers. GET /voice/voices lists the voices a provider reads in; POST /voice/speak takes a voice to try. Podcast hosts use the same voices. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01WkPdbtd7zvYMRWaRkLN2F6 --- DEVLOG.md | 16 ++ docs/assistant.md | 9 +- docs/speech-providers.md | 19 +- fastapi_backend/app/api/v1/routes/admin.py | 2 +- fastapi_backend/app/api/v1/routes/voice.py | 19 +- fastapi_backend/app/domain/podcasts.py | 13 +- fastapi_backend/app/domain/settings.py | 4 +- fastapi_backend/app/domain/speech.py | 254 ++++++++++++++++-- fastapi_backend/app/domain/store.py | 6 +- fastapi_backend/app/domain/voice.py | 9 +- fastapi_backend/app/schemas/voice.py | 14 + fastapi_backend/tests/api/test_voice.py | 8 + fastapi_backend/tests/domain/test_speech.py | 110 ++++++++ nextjs-frontend/app/openapi-client/index.ts | 7 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 18 ++ .../app/openapi-client/types.gen.ts | 86 ++++++ nextjs-frontend/openapi.json | 144 +++++++++- 17 files changed, 686 insertions(+), 52 deletions(-) diff --git a/DEVLOG.md b/DEVLOG.md index e0479cb62..0d2105828 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,22 @@ Plans and progress for work in flight. Newest first. +## 2026-10-10 · More voices for spoken answers + +Goal (Adeel): read answers and podcasts aloud with ElevenLabs, macOS `say`, Google Gemini TTS or StyleTTS2, with +the voice picked in Settings. ElevenLabs and Deepgram were already there (#116); the providers stay shared with +podcasts through `speech.speak()`. + +Todo: + +- [x] Gemini TTS and macOS `say` as text-to-speech providers; `GET /voice/voices` lists each provider's voices; + `POST /voice/speak` takes a voice to try +- [ ] StyleTTS2 as an opt-in local service (off by default; heavy) +- [ ] Settings → AI assistant → Voice: pick a voice from the list and hear it + +Refine later: pick a second `say` voice for podcast hosts from what the Mac has; Gemini's multi-speaker mode for a +whole podcast in one call; ElevenLabs voice settings (stability, style); cache the voice lists. + ## 2026-10-10 · Podcasts from picked knowledge Goal (Adeel): pick documents, search results, topics, notes or graph concepts and get a two-host, Recast-style diff --git a/docs/assistant.md b/docs/assistant.md index 1ca8ebef2..c1638c316 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -71,12 +71,13 @@ Voice goes through one hook, `useVoice()` in `nextjs-frontend/lib/voice.ts` (`li listening. Without a server engine, or with `voice.input: browser`, the browser's own recognition listens (Chrome, Edge and Safari; Chrome sends the audio to its own speech service). - **Speaking.** Answers are read by a speech model when one is set (`voice.tts_model`: any OpenAI-compatible - `/audio/speech`, such as Kokoro-FastAPI, LocalAI or OpenAI's `tts-1`, on `voice.tts_base_url` or the LLM provider); - else by the browser. + `/audio/speech`, such as Kokoro-FastAPI, LocalAI or OpenAI's `tts-1`, on `voice.tts_base_url` or the LLM provider), + or by ElevenLabs, Deepgram, Google Gemini or macOS `say` ([Speech providers](speech-providers.md#voices)); else by + the browser. Settings → AI assistant → Voice holds these. The API: `GET /voice` (what the server does; it also starts loading the -engine), `POST /voice/transcribe` (a clip as the raw body; nothing is kept), `POST /voice/speak` (`{text}`; 204 when -the browser should read it). +engine), `POST /voice/transcribe` (a clip as the raw body; nothing is kept), `POST /voice/speak` (`{text, voice?}`; +204 when the browser should read it), `GET /voice/voices` (the voices a provider reads in). ## The assistant's tools diff --git a/docs/speech-providers.md b/docs/speech-providers.md index b2b0a54e6..7f136e5f4 100644 --- a/docs/speech-providers.md +++ b/docs/speech-providers.md @@ -15,6 +15,8 @@ summaries, chapters and entities as optional imports, per-minute prices in the a | ElevenLabs | Scribe | yes | yes | no | laughter, applause, … | yes | | AssemblyAI | Universal, Slam-1 | yes | detected | from sentiment | no | no | | Deepgram | Nova | yes | detected | from sentiment (English) | no | Aura | +| Google Gemini | no | no | no | no | no | `gemini-2.5-flash-preview-tts`, 30 prebuilt voices | +| macOS `say` | no | no | no | no | no | the Mac's own voices; no key, nothing leaves the Mac | Sentiment becomes emotion as positive → Happy, negative → Sad, neutral → Neutral (turn it off with `speech.sentiment`). SenseVoice's own emotion tags stay richer; a provider only reports sentiment. @@ -35,7 +37,20 @@ Then pick where each task goes: - **Speaker separation → Method**: *Speech provider* uses the speakers the provider found; *Auto* uses them when a transcript has some, and otherwise works as before. Voice IDs across recordings still come from local voiceprints. - **AI assistant → Voice**: *Engine that hears it* (default: the transcription engine) and *Spoken answers by* (an - OpenAI-compatible speech server, ElevenLabs or Deepgram Aura). + OpenAI-compatible speech server, ElevenLabs, Deepgram Aura, Google Gemini or macOS `say`) and *Voice*. + +## Voices + +`GET /voice/voices?provider=…` lists the voices a provider reads in, as `{id, name, detail}`: ElevenLabs lists the +voices in your account (premade, cloned and from the library), Deepgram its Aura models, `say` the voices installed on +the Mac (System Settings → Accessibility → Spoken Content adds more), and an OpenAI-compatible server its own when it +has `/audio/voices` (Kokoro-FastAPI does); otherwise OpenAI's eleven. Gemini's are a fixed list. The voice picked is +`voice.tts_voice`; podcast hosts use it too (`podcasts.voice_a`, `voice_b`). `POST /voice/speak` takes `{text, +voice}` to try a voice before keeping it. + +`say` only works when Lens runs on the Mac itself (`make start-backend`, not Docker): a container has no `say`. It +writes WAV, and both podcast hosts get the same voice unless `podcasts.voice_b` names another, since no voice is on +every Mac. | Setting | Default | | --- | --- | @@ -43,10 +58,12 @@ Then pick where each task goes: | `speech.elevenlabs_base_url`, `elevenlabs_model` | `https://api.elevenlabs.io`, `scribe_v1` | | `speech.assemblyai_base_url`, `assemblyai_model` | `https://api.assemblyai.com`, `universal` | | `speech.deepgram_base_url`, `deepgram_model` | `https://api.deepgram.com`, `nova-3` | +| `speech.gemini_base_url`, `gemini_model` | `https://generativelanguage.googleapis.com`, `gemini-2.5-flash-preview-tts` | | `speech.sentiment` | on | | `speech.timeout` | 1800 seconds | | `voice.stt` | `same` | | `voice.tts_provider` | `openai` | +| `voice.tts_voice` | the provider's default (alloy, a premade ElevenLabs voice, aura-2-thalia-en, Kore, the Mac's) | ## What is sent diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py index 5249bfb4a..b4fb20474 100644 --- a/fastapi_backend/app/api/v1/routes/admin.py +++ b/fastapi_backend/app/api/v1/routes/admin.py @@ -238,7 +238,7 @@ def test_embeddings(user: AdminWriter, cfg: Cfg, db: Db) -> EmbedTestResult: @router.post("/settings/speech/test") def test_speech( - user: AdminWriter, cfg: Cfg, provider: str = Query(..., pattern="^(openai|elevenlabs|assemblyai|deepgram)$") + user: AdminWriter, cfg: Cfg, provider: str = Query(..., pattern="^(openai|elevenlabs|assemblyai|deepgram|gemini)$") ) -> SpeechTestResult: """Check a speech provider's address and key with the saved settings (lists its models; nothing is billed).""" t0 = time.time() diff --git a/fastapi_backend/app/api/v1/routes/voice.py b/fastapi_backend/app/api/v1/routes/voice.py index 444c62b91..417b18915 100644 --- a/fastapi_backend/app/api/v1/routes/voice.py +++ b/fastapi_backend/app/api/v1/routes/voice.py @@ -2,12 +2,12 @@ from __future__ import annotations -from fastapi import APIRouter, HTTPException, Request, Response +from fastapi import APIRouter, HTTPException, Query, Request, Response from starlette.concurrency import run_in_threadpool from app.api.deps import Cfg, CurrentUser -from app.domain import voice -from app.schemas.voice import Heard, SpeakRequest, VoiceInfo +from app.domain import speech, voice +from app.schemas.voice import Heard, SpeakRequest, VoiceInfo, VoiceList router = APIRouter(prefix="/voice", tags=["voice"]) CLIP = {"requestBody": {"required": True, "content": {"application/octet-stream": {"schema": {"type": "string", "format": "binary"}}}}} @@ -47,8 +47,19 @@ async def transcribe_clip(request: Request, user: CurrentUser, cfg: Cfg) -> Hear async def speak_text(body: SpeakRequest, user: CurrentUser, cfg: Cfg) -> Response: """The text read aloud by the text-to-speech model (Settings → AI assistant → Voice). 204 when there's none, or it failed: the browser reads it instead.""" - out = await run_in_threadpool(voice.speak, cfg, body.text) + out = await run_in_threadpool(voice.speak, cfg, body.text, body.voice) if out is None: return Response(status_code=204) audio, media = out return Response(content=audio, media_type=media, headers={"Cache-Control": "no-store"}) + + +@router.get("/voices") +async def list_voices( + user: CurrentUser, cfg: Cfg, provider: str | None = Query(None, pattern="^(" + "|".join(speech.TTS_PROVIDERS) + ")$") +) -> VoiceList: + """The voices a text-to-speech provider (else the one in the voice settings) reads in, from the provider when it + lists them (ElevenLabs, Deepgram, macOS say, OpenAI-compatible servers that do), else the ones it's known to have.""" + p = provider or speech.tts_provider(cfg) + found, error = await run_in_threadpool(speech.voices, cfg, p) + return VoiceList(provider=p, current=speech.tts_voice(cfg) if p == speech.tts_provider(cfg) else None, voices=found, error=error) diff --git a/fastapi_backend/app/domain/podcasts.py b/fastapi_backend/app/domain/podcasts.py index 006834778..866ea9a07 100644 --- a/fastapi_backend/app/domain/podcasts.py +++ b/fastapi_backend/app/domain/podcasts.py @@ -39,7 +39,7 @@ AFTER = ["analyze", "embed", "summarize"] # what any new transcript gets STATUS = {"gather": "gathering", "plan": "planning", "write": "writing", "check": "checking", "audio": "rendering", "publish": "publishing"} # the second host's voice when podcasts.voice_b isn't set, per text-to-speech provider (the first's is voice.tts_voice) -SECOND_VOICE = {"openai": "onyx", "elevenlabs": "EXAVITQu4vr4xnSDxMKL", "deepgram": "aura-2-orion-en"} +SECOND_VOICE = {"openai": "onyx", "elevenlabs": "EXAVITQu4vr4xnSDxMKL", "deepgram": "aura-2-orion-en", "gemini": "Puck"} PAUSE_MS = {"turn": 450, "same": 250} # silence before a line: when the other host speaks, and when the same one goes on LENGTHS = (5, 10, 20) # minutes offered; anything from 1 to podcasts.max_minutes goes STYLES = { @@ -718,14 +718,13 @@ def timed(lines, wpm=150, pause_ms=400): def voices(cfg, req=None): """{a, b}: the hosts' voices for the text-to-speech provider: what the episode asked for, else podcasts.voice_a / - voice_b, else voice.tts_voice and a second voice of the provider's.""" + voice_b, else voice.tts_voice and a second voice of the provider's (say has no voice every Mac is sure to have, so + both hosts get the same one unless podcasts.voice_b names another).""" from . import speech - v, p = cfg.get("voice") or {}, cfg.get("podcasts") or {} - provider = v.get("tts_provider") or "openai" - first = {"openai": v.get("tts_voice") or "alloy", "elevenlabs": v.get("tts_voice") or speech.ELEVENLABS_VOICE}.get( - provider, v.get("tts_model") or speech.DEEPGRAM_TTS_MODEL - ) + p = cfg.get("podcasts") or {} + provider = speech.tts_provider(cfg) + first = speech.tts_voice(cfg) # None for say: the Mac's own voice asked = (req or {}).get("voices") or {} return {"a": asked.get("a") or p.get("voice_a") or first, "b": asked.get("b") or p.get("voice_b") or SECOND_VOICE.get(provider, first)} diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index bc16ae640..6d4e704d1 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -110,7 +110,7 @@ "embeddings": ("api_key",), "decisions": ("api_key",), "voice": ("tts_api_key",), - "speech": ("openai_api_key", "elevenlabs_api_key", "assemblyai_api_key", "deepgram_api_key"), + "speech": ("openai_api_key", "elevenlabs_api_key", "assemblyai_api_key", "deepgram_api_key", "gemini_api_key"), "mail": ("password",), "bridge": ("token",), "matterbridge": ("slack_token", "discord_token", "telegram_token", "matrix_password"), @@ -135,7 +135,7 @@ ("decisions", "engine"): {"auto", "jev", "laya", "llm", "off"}, ("voice", "input"): {"auto", "server", "browser"}, ("voice", "stt"): {"same", "sensevoice", "whisper", "mlx-whisper", "openai", "elevenlabs", "assemblyai", "deepgram"}, - ("voice", "tts_provider"): {"openai", "elevenlabs", "deepgram"}, + ("voice", "tts_provider"): {"openai", "elevenlabs", "deepgram", "gemini", "say"}, ("mail", "security"): {"starttls", "ssl", "none"}, ("bridge", "answer"): {"mention", "all"}, ("bridge", "approve"): {"off", "low_risk"}, diff --git a/fastapi_backend/app/domain/speech.py b/fastapi_backend/app/domain/speech.py index 36623f33a..4889f10a9 100644 --- a/fastapi_backend/app/domain/speech.py +++ b/fastapi_backend/app/domain/speech.py @@ -9,6 +9,10 @@ * elevenlabs: Scribe speech to text (speakers, audio events) and text to speech. * assemblyai: transcripts with speaker labels, language detection and sentiment. * deepgram: Nova speech to text (speakers, language, sentiment) and Aura text to speech. +* gemini: Google Gemini text to speech (its prebuilt voices). + +Text to speech can also come from macOS `say` when Lens runs on a Mac itself (not in Docker): nothing to set up, the +Mac's own voices. voices() lists what each one can read in, for Settings and podcast hosts. A provider's transcript is the same list of segments a local engine returns, plus `provider_speaker`: the speaker separation step uses those labels (diarize.engine provider, or auto), and voice IDs still come from local voiceprints. @@ -16,16 +20,23 @@ from __future__ import annotations +import base64 import contextlib +import io import json import logging +import pathlib import re +import shutil import subprocess +import sys +import tempfile import time import urllib.error import urllib.parse import urllib.request import uuid +import wave import numpy as np @@ -34,8 +45,16 @@ log = logging.getLogger("lens") SR = 16000 PROVIDERS = ("openai", "elevenlabs", "assemblyai", "deepgram") -LABELS = {"openai": "OpenAI-compatible", "elevenlabs": "ElevenLabs", "assemblyai": "AssemblyAI", "deepgram": "Deepgram"} -TTS_PROVIDERS = ("openai", "elevenlabs", "deepgram") +SERVICES = (*PROVIDERS, "gemini") # with an address, model and key in the speech settings; gemini only reads aloud +LABELS = { + "openai": "OpenAI-compatible", + "elevenlabs": "ElevenLabs", + "assemblyai": "AssemblyAI", + "deepgram": "Deepgram", + "gemini": "Google Gemini", + "say": "macOS say", +} +TTS_PROVIDERS = ("openai", "elevenlabs", "deepgram", "gemini", "say") CHUNK_SECONDS = 600 # OpenAI-compatible servers take up to 25 MB a request; ten minutes of Opus is a few MB POLL_SECONDS = 3 MAX_WORDS = 40 @@ -47,6 +66,20 @@ ELEVENLABS_VOICE = "JBFqnCBsd6RMkjVDRZzb" # a premade voice every account has ELEVENLABS_TTS_MODEL = "eleven_multilingual_v2" DEEPGRAM_TTS_MODEL = "aura-2-thalia-en" +GEMINI_VOICE = "Kore" +# what each provider reads in when nothing is picked (say: the Mac's own voice) +DEFAULT_VOICE = {"openai": "alloy", "elevenlabs": ELEVENLABS_VOICE, "deepgram": DEEPGRAM_TTS_MODEL, "gemini": GEMINI_VOICE, "say": None} +OPENAI_VOICES = ("alloy", "ash", "ballad", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer", "verse") +# Gemini's prebuilt voices and how Google describes each +GEMINI_VOICES = { + "Zephyr": "Bright", "Puck": "Upbeat", "Charon": "Informative", "Kore": "Firm", "Fenrir": "Excitable", + "Leda": "Youthful", "Orus": "Firm", "Aoede": "Breezy", "Callirrhoe": "Easy-going", "Autonoe": "Bright", + "Enceladus": "Breathy", "Iapetus": "Clear", "Umbriel": "Easy-going", "Algieba": "Smooth", "Despina": "Smooth", + "Erinome": "Clear", "Algenib": "Gravelly", "Rasalgethi": "Informative", "Laomedeia": "Upbeat", "Achernar": "Soft", + "Alnilam": "Firm", "Schedar": "Even", "Gacrux": "Mature", "Pulcherrima": "Forward", "Achird": "Friendly", + "Zubenelgenubi": "Casual", "Vindemiatrix": "Gentle", "Sadachbia": "Lively", "Sadaltager": "Knowledgeable", + "Sulafat": "Warm", +} # fmt: skip class ProviderError(RuntimeError): @@ -73,7 +106,7 @@ def model(cfg, p): def ready(cfg, p): """Whether a provider can be called: it has a key, or (OpenAI-compatible) a server of its own that may need none.""" - if p not in PROVIDERS: + if p not in SERVICES: return False if _key(cfg, p): return True @@ -442,46 +475,209 @@ def transcribe(self, audio): # ---------- text to speech ---------- +def tts_provider(cfg): + return (cfg.get("voice") or {}).get("tts_provider") or "openai" + + +def say_available(): + """macOS `say` is there: Lens runs on a Mac itself, not in a container.""" + return sys.platform == "darwin" and shutil.which("say") is not None + + def tts_ready(cfg): v = cfg.get("voice") or {} - p = v.get("tts_provider") or "openai" + p = tts_provider(cfg) if p == "openai": return bool(v.get("tts_model") and (v.get("tts_base_url") or (cfg.get("llm") or {}).get("base_url"))) + if p == "say": + return say_available() return p in TTS_PROVIDERS and bool(_key(cfg, p)) -def speak(cfg, text, voice=None): - """The text read aloud as (bytes, media type), by the voice settings' provider, in `voice` (else voice.tts_voice; - for Deepgram the voice is its model). Raises ProviderError.""" +def tts_voice(cfg): + """The voice answers are read in: voice.tts_voice, else the provider's default (for Deepgram, voice.tts_model too, + where its voice was set before it had a voice of its own).""" v = cfg.get("voice") or {} - p = v.get("tts_provider") or "openai" + p = tts_provider(cfg) + if p == "deepgram": + return v.get("tts_voice") or v.get("tts_model") or DEEPGRAM_TTS_MODEL + return v.get("tts_voice") or DEFAULT_VOICE.get(p) + + +def wav(pcm, rate, channels=1): + """16-bit PCM as a WAV file.""" + out = io.BytesIO() + with wave.open(out, "wb") as w: + w.setnchannels(channels) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes(pcm) + return out.getvalue() + + +def _speak_elevenlabs(cfg, text, voice): + m = (cfg.get("voice") or {}).get("tts_model") or ELEVENLABS_TTS_MODEL + url = f"{base_url(cfg, 'elevenlabs')}/v1/text-to-speech/{urllib.parse.quote(voice, safe='')}?output_format=mp3_44100_128" + headers = {"Content-Type": "application/json", "xi-api-key": _key(cfg, "elevenlabs"), "Accept": "audio/mpeg"} + return m, lambda: _request(url, data=json.dumps({"text": text, "model_id": m}).encode(), headers=headers, timeout=60) + + +def _speak_deepgram(cfg, text, voice): + url = f"{base_url(cfg, 'deepgram')}/v1/speak?" + urllib.parse.urlencode({"model": voice, "encoding": "mp3"}) + headers = {"Content-Type": "application/json", "Authorization": f"Token {_key(cfg, 'deepgram')}"} + return voice, lambda: _request(url, data=json.dumps({"text": text}).encode(), headers=headers, timeout=60) + + +def _speak_gemini(cfg, text, voice): + m = model(cfg, "gemini") + url = f"{base_url(cfg, 'gemini')}/v1beta/models/{urllib.parse.quote(m, safe='')}:generateContent" + headers = {"Content-Type": "application/json", "x-goog-api-key": _key(cfg, "gemini")} + body = { + "contents": [{"parts": [{"text": text}]}], + "generationConfig": { + "responseModalities": ["AUDIO"], + "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}}, + }, + } + + def call(): + j = _json(url, data=json.dumps(body).encode(), headers=headers, timeout=120) + parts = (((j.get("candidates") or [{}])[0].get("content") or {}).get("parts")) or [] + blob = next((p["inlineData"] for p in parts if isinstance(p, dict) and p.get("inlineData")), None) + if not blob or not blob.get("data"): + raise ProviderError("Gemini answered without audio: " + json.dumps(j)[:200]) + rate = re.search(r"rate=(\d+)", blob.get("mimeType") or "") # raw 16-bit PCM, audio/L16;codec=pcm;rate=24000 + return wav(base64.b64decode(blob["data"]), int(rate.group(1)) if rate else 24000), "audio/wav" + + return m, call + + +def _speak_say(cfg, text, voice): + if not say_available(): + raise ProviderError("macOS say reads aloud only when Lens runs on a Mac itself, not in Docker") + if voice and (voice.startswith("-") or len(voice) > 100): + raise ProviderError(f"{voice!r} isn't a voice of say's") + + def call(): + with tempfile.TemporaryDirectory(prefix="lens-say-") as d: + out = pathlib.Path(d) / "speech.wav" + cmd = ["say", "-o", str(out), "--file-format=WAVE", "--data-format=LEI16@22050", *(["-v", voice] if voice else [])] + try: # the text goes in on stdin, so nothing in it is read as an option + r = subprocess.run(cmd, input=text.encode(), capture_output=True, timeout=120) + except subprocess.TimeoutExpired: + raise ProviderError("say took longer than 120 s") from None + if r.returncode or not out.is_file(): + raise ProviderError("say couldn't read it: " + (r.stderr.decode(errors="replace").strip()[-200:] or "no audio")) + return out.read_bytes(), "audio/wav" + + return voice or "default", call + + +def _speak_openai(cfg, text, voice): + v = cfg.get("voice") or {} + m = v["tts_model"] + base = (v.get("tts_base_url") or cfg["llm"]["base_url"]).rstrip("/") + key = v.get("tts_api_key") or (cfg["llm"].get("api_key") if not v.get("tts_base_url") else None) + headers = {"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})} + body = {"model": m, "input": text, "voice": voice, "response_format": "mp3"} + return m, lambda: _request(base + "/audio/speech", data=json.dumps(body).encode(), headers=headers, timeout=60) + + +SPEAK = {"openai": _speak_openai, "elevenlabs": _speak_elevenlabs, "deepgram": _speak_deepgram, "gemini": _speak_gemini, "say": _speak_say} + + +def speak(cfg, text, voice=None): + """The text read aloud as (bytes, media type), by the voice settings' provider, in `voice` (else tts_voice()). + Raises ProviderError.""" + p = tts_provider(cfg) + if p not in SPEAK: + raise ProviderError(f"unknown text-to-speech provider {p!r}") text = text[:4000] - if p == "elevenlabs": - m = v.get("tts_model") or ELEVENLABS_TTS_MODEL - url = f"{base_url(cfg, 'elevenlabs')}/v1/text-to-speech/{urllib.parse.quote(voice or v.get('tts_voice') or ELEVENLABS_VOICE, safe='')}?output_format=mp3_44100_128" - headers = {"Content-Type": "application/json", "xi-api-key": _key(cfg, "elevenlabs"), "Accept": "audio/mpeg"} - body = {"text": text, "model_id": m} - elif p == "deepgram": - m = voice or v.get("tts_model") or DEEPGRAM_TTS_MODEL - url = f"{base_url(cfg, 'deepgram')}/v1/speak?" + urllib.parse.urlencode({"model": m, "encoding": "mp3"}) - headers = {"Content-Type": "application/json", "Authorization": f"Token {_key(cfg, 'deepgram')}"} - body = {"text": text} - else: - m = v["tts_model"] - base = (v.get("tts_base_url") or cfg["llm"]["base_url"]).rstrip("/") - key = v.get("tts_api_key") or (cfg["llm"].get("api_key") if not v.get("tts_base_url") else None) - url = base + "/audio/speech" - headers = {"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})} - body = {"model": m, "input": text, "voice": voice or v.get("tts_voice") or "alloy", "response_format": "mp3"} + m, call = SPEAK[p](cfg, text, voice or tts_voice(cfg)) with _ledger("model.speech", cfg, f"{p}:{m}", {"chars": len(text)}): - data, ctype = _request(url, data=json.dumps(body).encode(), headers=headers, timeout=60) + data, ctype = call() return data, ctype if ctype and ctype.startswith("audio/") else "audio/mpeg" +# ---------- the voices a provider reads in ---------- +def _voice(id, name=None, detail=None): + return {"id": id, "name": name or id, "detail": detail or None} + + +def _say_voices(): + """`say -v ?` lines: "Name (maybe with spaces) en_US # A sample sentence".""" + out = subprocess.run(["say", "-v", "?"], capture_output=True, timeout=30).stdout.decode(errors="replace") + found = [] + for line in out.splitlines(): + m = re.match(r"^(.+?)\s+([a-z]{2,3}[_-][A-Za-z0-9]+)\s+#", line) + if m: + found.append(_voice(m.group(1).strip(), detail=m.group(2))) + return found + + +def _openai_voices(cfg): + v = cfg.get("voice") or {} + base = (v.get("tts_base_url") or (cfg.get("llm") or {}).get("base_url") or "").rstrip("/") + key = v.get("tts_api_key") or ((cfg.get("llm") or {}).get("api_key") if not v.get("tts_base_url") else None) + if base and "api.openai.com" not in base: # Kokoro-FastAPI, speaches and others list their own + with contextlib.suppress(ProviderError, AttributeError, TypeError): + j = _json(base + "/audio/voices", headers={"Authorization": f"Bearer {key}"} if key else {}, timeout=15) + got = j.get("voices") if isinstance(j, dict) else j + names = [x if isinstance(x, str) else (x.get("id") or x.get("name")) for x in got or []] + if any(names): + return [_voice(n) for n in names if n] + return [_voice(n) for n in OPENAI_VOICES] + + +def _elevenlabs_voices(cfg): + j = _json(f"{base_url(cfg, 'elevenlabs')}/v1/voices", headers={"xi-api-key": _key(cfg, "elevenlabs")}, timeout=20) + out = [] + for x in j.get("voices") or []: + labels = x.get("labels") or {} + detail = ", ".join(str(labels[k]) for k in ("gender", "accent", "age", "description", "use_case") if labels.get(k)) + out.append(_voice(x["voice_id"], x.get("name"), detail or x.get("category"))) + return out + + +def _deepgram_voices(cfg): + j = _json(f"{base_url(cfg, 'deepgram')}/v1/models", headers={"Authorization": f"Token {_key(cfg, 'deepgram')}"}, timeout=20) + out = [] + for x in j.get("tts") or []: + meta = x.get("metadata") or {} + detail = ", ".join(str(d) for d in (meta.get("accent"), *(x.get("languages") or [])[:1]) if d) + out.append(_voice(x.get("canonical_name") or x["name"], (x.get("name") or "").title() or None, detail)) + return out + + +def voices(cfg, provider=None): + """The voices a text-to-speech provider (else the voice settings') reads in: ([{id, name, detail}], error or None). + Providers that list them are asked; Gemini's and OpenAI's own are known.""" + p = provider or tts_provider(cfg) + if p not in TTS_PROVIDERS: + raise ValueError(f"unknown text-to-speech provider {p!r}") + if p == "gemini": + return [_voice(n, detail=d) for n, d in GEMINI_VOICES.items()], None + if p == "openai": + return _openai_voices(cfg), None + if p == "say": + if not say_available(): + return [], "macOS say reads aloud only when Lens runs on a Mac itself, not in Docker" + try: + return _say_voices(), None + except (OSError, subprocess.TimeoutExpired) as e: + return [], f"say didn't list its voices: {e}" + if not _key(cfg, p): + return [], f"{LABELS[p]} has no API key yet: add it in Settings → Speech providers" + try: + return {"elevenlabs": _elevenlabs_voices, "deepgram": _deepgram_voices}[p](cfg), None + except (ProviderError, KeyError, AttributeError, TypeError) as e: + return [], str(e) if isinstance(e, ProviderError) else f"{LABELS[p]} answered in a way Lens doesn't read" + + # ---------- checking a provider from Settings ---------- def check(cfg, provider): """Whether the provider answers with these settings: (ok, what it said). Lists models; nothing is billed.""" - if provider not in PROVIDERS: + if provider not in SERVICES: raise ValueError(f"unknown speech provider {provider!r}") if not ready(cfg, provider): return False, f"{LABELS[provider]} has no API key yet" @@ -494,6 +690,10 @@ def check(cfg, provider): if provider == "elevenlabs": j = _json(base + "/v1/models", headers={"xi-api-key": key}, timeout=15) return True, f"signed in; {len(j) if isinstance(j, list) else 0} model(s)" + if provider == "gemini": + j = _json(base + "/v1beta/models?pageSize=1000", headers={"x-goog-api-key": key}, timeout=15) + tts = [m for m in j.get("models") or [] if "tts" in (m.get("name") or "")] + return True, f"signed in; {len(tts)} speech model(s)" if provider == "assemblyai": _json(base + "/v2/transcript?limit=1", headers={"Authorization": key}, timeout=15) return True, "signed in" diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index de3e3f873..2f0acf63c 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -288,7 +288,8 @@ def labels(): # talking to Lens (voice.py): input auto uses the server's speech-to-text engine when it has one, else the # browser's; spoken answers come from tts_model (an OpenAI-compatible /audio/speech), else the browser reads them # stt: the engine that hears voice chat ("same": the transcription engine); tts_provider: who reads answers aloud - # (openai: the OpenAI-compatible speech server below; elevenlabs, deepgram: the keys in speech) + # (openai: the OpenAI-compatible speech server below; elevenlabs, deepgram, gemini: the keys in speech; say: macOS's + # own voices, when Lens runs on a Mac itself); tts_voice: the voice they read in (null: the provider's default) "voice": { "input": "auto", "stt": "same", @@ -341,6 +342,9 @@ def labels(): "deepgram_base_url": "https://api.deepgram.com", "deepgram_model": "nova-3", "deepgram_api_key": None, + "gemini_base_url": "https://generativelanguage.googleapis.com", + "gemini_model": "gemini-2.5-flash-preview-tts", + "gemini_api_key": None, "sentiment": True, "timeout": 1800, }, diff --git a/fastapi_backend/app/domain/voice.py b/fastapi_backend/app/domain/voice.py index 0283ae0c5..e2c8eacc5 100644 --- a/fastapi_backend/app/domain/voice.py +++ b/fastapi_backend/app/domain/voice.py @@ -123,13 +123,14 @@ def can_speak(cfg): return speech.tts_ready(cfg) -def speak(cfg, text): - """The text read aloud by the text-to-speech provider (an OpenAI-compatible /audio/speech, ElevenLabs or Deepgram), - as (bytes, media type); None when there's none or it failed, so the browser reads it instead.""" +def speak(cfg, text, voice=None): + """The text read aloud by the text-to-speech provider (an OpenAI-compatible /audio/speech, ElevenLabs, Deepgram, + Gemini or macOS say), in `voice` (else the settings'), as (bytes, media type); None when there's none or it failed, + so the browser reads it instead.""" if not can_speak(cfg) or not text.strip(): return None try: - return speech.speak(cfg, text) # it writes its own row in the activity ledger + return speech.speak(cfg, text, voice=voice) # it writes its own row in the activity ledger except speech.ProviderError as e: log.info("voice: text-to-speech failed, the browser reads it: %s", e) return None diff --git a/fastapi_backend/app/schemas/voice.py b/fastapi_backend/app/schemas/voice.py index 8cdf93ba7..eb9c75dde 100644 --- a/fastapi_backend/app/schemas/voice.py +++ b/fastapi_backend/app/schemas/voice.py @@ -20,3 +20,17 @@ class Heard(ResponseModel): class SpeakRequest(RequestModel): text: str = Field(max_length=8000) + voice: str | None = Field(None, max_length=200, description="read in this voice (an id from GET /voice/voices); else the settings'") + + +class VoiceOption(ResponseModel): + id: str = Field(description="what voice.tts_voice is set to") + name: str + detail: str | None = Field(None, description="what the provider says about it: accent, gender, style, language") + + +class VoiceList(ResponseModel): + provider: str + current: str | None = Field(None, description="the voice answers are read in now (null: the provider's default)") + voices: list[VoiceOption] + error: str | None = Field(None, description="why the provider's voices couldn't be listed") diff --git a/fastapi_backend/tests/api/test_voice.py b/fastapi_backend/tests/api/test_voice.py index a25ee7626..129995609 100644 --- a/fastapi_backend/tests/api/test_voice.py +++ b/fastapi_backend/tests/api/test_voice.py @@ -80,6 +80,14 @@ def test_answers_are_read_aloud_by_a_speech_model(client, signed_in, cfg, db): path, headers, body = Tts.seen[-1] assert path == "/v1/audio/speech" and headers["Authorization"] == "Bearer sk-1" assert body == {"model": "kokoro", "input": "The shipment leaves on Friday.", "voice": "af_heart", "response_format": "mp3"} + client.post("/api/v1/voice/speak", headers=h, json={"text": "Hi", "voice": "am_adam"}) # a voice tried in Settings + assert Tts.seen[-1][2]["voice"] == "am_adam" + r = client.get("/api/v1/voice/voices", headers=h).json() + assert r["provider"] == "openai" and r["current"] == "af_heart" and r["error"] is None + assert {"id": "alloy", "name": "alloy", "detail": None} in r["voices"] # this server doesn't list its own + r = client.get("/api/v1/voice/voices?provider=gemini", headers=h).json() + assert r["current"] is None and len(r["voices"]) == 30 + assert client.get("/api/v1/voice/voices?provider=nope", headers=h).status_code == 422 finally: srv.shutdown() settings.save(db, cfg, "llm", {"base_url": "http://127.0.0.1:9/v1"}, "t") # gone: the browser reads it diff --git a/fastapi_backend/tests/domain/test_speech.py b/fastapi_backend/tests/domain/test_speech.py index 72e6d4434..736225a6b 100644 --- a/fastapi_backend/tests/domain/test_speech.py +++ b/fastapi_backend/tests/domain/test_speech.py @@ -3,12 +3,16 @@ from __future__ import annotations +import base64 import http.server +import io import json import shutil +import subprocess import sys import threading import urllib.parse +import wave import numpy as np import pytest @@ -57,6 +61,23 @@ def do_GET(self): ], } ) + if self.path.startswith("/el/v1/voices"): + return self._send( + {"voices": [{"voice_id": "v42", "name": "Ada", "category": "premade", "labels": {"gender": "female", "accent": "british"}}]} + ) + if self.path == "/dg/v1/models": + return self._send( + { + "stt": [], + "tts": [ + {"name": "thalia", "canonical_name": "aura-2-thalia-en", "languages": ["en"], "metadata": {"accent": "American"}} + ], + } + ) + if self.path.startswith("/gm/v1beta/models"): + return self._send({"models": [{"name": "models/gemini-2.5-flash-preview-tts"}, {"name": "models/gemini-2.5-pro"}]}) + if self.path == "/oai/v1/audio/voices": + return self._send({"voices": ["af_heart", "am_adam"]}) if self.path.endswith("/models") or self.path.endswith("/v1/models"): return self._send({"data": [{"id": "whisper-1"}]} if "/oai/" in self.path else [{"model_id": "scribe_v1"}]) if "/v2/transcript" in self.path or self.path.endswith("/v1/projects"): @@ -117,6 +138,11 @@ def do_POST(self): } } ) + if p.startswith("/gm/v1beta/models/"): + pcm = base64.b64encode(b"\x00\x01" * 2400).decode() + return self._send( + {"candidates": [{"content": {"parts": [{"inlineData": {"mimeType": "audio/L16;codec=pcm;rate=24000", "data": pcm}}]}}]} + ) if p.startswith("/el/v1/text-to-speech/") or p.startswith("/dg/v1/speak"): return self._send(b"ID3", "audio/mpeg") self.send_error(404) @@ -145,6 +171,8 @@ def provided(db, cfg, base): "assemblyai_api_key": "aai-key", "deepgram_base_url": base + "/dg", "deepgram_api_key": "dg-key", + "gemini_base_url": base + "/gm", + "gemini_api_key": "gm-key", }, "t", ) @@ -282,6 +310,88 @@ def test_answers_are_read_by_elevenlabs_or_deepgram(db, cfg, fake): assert voice.speak(settings.effective(db, cfg), "Hello") is None # unreachable: the browser reads it +def test_answers_are_read_by_gemini_in_a_voice_picked(db, cfg, fake): + provided(db, cfg, fake) + settings.save(db, cfg, "voice", {"tts_provider": "gemini"}, "t") + eff = settings.effective(db, cfg) + assert voice.can_speak(eff) and speech.tts_voice(eff) == "Kore" + data, ctype = voice.speak(eff, "Hello", voice="Puck") + assert ctype == "audio/wav" and data[:4] == b"RIFF" + with wave.open(io.BytesIO(data)) as w: + assert (w.getframerate(), w.getnframes()) == (24000, 2400) + method, path, headers, body = Fake.seen[-1] + assert ( + path == "/gm/v1beta/models/gemini-2.5-flash-preview-tts:generateContent" + and {k.lower(): v for k, v in headers.items()}["x-goog-api-key"] == "gm-key" + ) + cfg_ = json.loads(body)["generationConfig"] + assert cfg_["responseModalities"] == ["AUDIO"] + assert cfg_["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Puck" + found, err = speech.voices(eff) + assert err is None and {"id": "Kore", "name": "Kore", "detail": "Firm"} in found and len(found) == 30 + assert speech.check(eff, "gemini") == (True, "signed in; 1 speech model(s)") + + +def test_each_provider_lists_its_voices(db, cfg, fake): + assert speech.voices(cfg, "elevenlabs") == ([], "ElevenLabs has no API key yet: add it in Settings → Speech providers") + assert [v["id"] for v in speech.voices(cfg, "openai")[0]][:2] == ["alloy", "ash"] # OpenAI's own + eff = provided(db, cfg, fake) + assert speech.voices(eff, "elevenlabs") == ([{"id": "v42", "name": "Ada", "detail": "female, british"}], None) + assert speech.voices(eff, "deepgram") == ([{"id": "aura-2-thalia-en", "name": "Thalia", "detail": "American, en"}], None) + assert Fake.seen[-1][2]["Authorization"] == "Token dg-key" + settings.save(db, cfg, "voice", {"tts_base_url": fake + "/oai/v1", "tts_model": "kokoro"}, "t") + found, _ = speech.voices(settings.effective(db, cfg), "openai") # a server that lists its own + assert [v["id"] for v in found] == ["af_heart", "am_adam"] + settings.save(db, cfg, "speech", {"elevenlabs_base_url": "http://127.0.0.1:9/el"}, "t") + found, err = speech.voices(settings.effective(db, cfg), "elevenlabs") + assert found == [] and err.startswith("can't reach 127.0.0.1:9") + with pytest.raises(ValueError): + speech.voices(eff, "nope") + + +SAY_VOICES = """Albert en_US # Hello! My name is Albert. +Eddy (English (UK)) en_GB # Hello! My name is Eddy. +Bad News en_US # Hello! My name is Bad News. +""" + + +def test_macos_say_reads_aloud_on_a_mac(db, cfg, monkeypatch): + settings.save(db, cfg, "voice", {"tts_provider": "say"}, "t") + eff = settings.effective(db, cfg) + monkeypatch.setattr(speech, "say_available", lambda: False) + assert not voice.can_speak(eff) + assert speech.voices(eff) == ([], "macOS say reads aloud only when Lens runs on a Mac itself, not in Docker") + with pytest.raises(speech.ProviderError, match="only when Lens runs on a Mac"): + speech.speak(eff, "Hi") + monkeypatch.setattr(speech, "say_available", lambda: True) + ran = [] + + def run(cmd, input=None, **kw): + ran.append((cmd, input)) + if cmd[1:] == ["-v", "?"]: + return subprocess.CompletedProcess(cmd, 0, SAY_VOICES.encode(), b"") + out = cmd[cmd.index("-o") + 1] + with open(out, "wb") as f: + f.write(speech.wav(b"\x00\x00" * 10, 22050)) + return subprocess.CompletedProcess(cmd, 0, b"", b"") + + monkeypatch.setattr(speech.subprocess, "run", run) + assert voice.can_speak(eff) and speech.tts_voice(eff) is None + data, ctype = voice.speak(eff, "-v Bad News? no, text", voice="Eddy (English (UK))") + assert ctype == "audio/wav" and data[:4] == b"RIFF" + cmd, sent = ran[-1] + assert cmd[-2:] == ["-v", "Eddy (English (UK))"] and sent == b"-v Bad News? no, text" # text on stdin, never an option + voice.speak(eff, "Hi") + assert "-v" not in ran[-1][0] # the Mac's own voice + assert speech.voices(eff)[0] == [ + {"id": "Albert", "name": "Albert", "detail": "en_US"}, + {"id": "Eddy (English (UK))", "name": "Eddy (English (UK))", "detail": "en_GB"}, + {"id": "Bad News", "name": "Bad News", "detail": "en_US"}, + ] + with pytest.raises(speech.ProviderError, match="isn't a voice"): + speech.speak(eff, "Hi", voice="--help") + + def test_voice_chat_can_use_its_own_engine(db, cfg, fake): eff = provided(db, cfg, fake) settings.save(db, cfg, "voice", {"stt": "elevenlabs"}, "t") diff --git a/nextjs-frontend/app/openapi-client/index.ts b/nextjs-frontend/app/openapi-client/index.ts index 17ad6f652..d3ff42cc4 100644 --- a/nextjs-frontend/app/openapi-client/index.ts +++ b/nextjs-frontend/app/openapi-client/index.ts @@ -1628,6 +1628,11 @@ export type { ListViewsData, ListViewsResponse, ListViewsResponses, + ListVoicesData, + ListVoicesError, + ListVoicesErrors, + ListVoicesResponse, + ListVoicesResponses, ListWatchesData, ListWatchesResponse, ListWatchesResponses, @@ -2851,6 +2856,8 @@ export type { VoiceInfoData, VoiceInfoResponse, VoiceInfoResponses, + VoiceList, + VoiceOption, Watch, WatchCreate, WatchPreview, diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 04852ba01..9690ee7dc 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -839,6 +839,9 @@ import type { ListUsersResponses, ListViewsData, ListViewsResponses, + ListVoicesData, + ListVoicesErrors, + ListVoicesResponses, ListWatchesData, ListWatchesResponses, ListWorkersData, @@ -8989,6 +8992,21 @@ export class Voice { }, }); } + + /** + * List Voices + * + * The voices a text-to-speech provider (else the one in the voice settings) reads in, from the provider when it + * lists them (ElevenLabs, Deepgram, macOS say, OpenAI-compatible servers that do), else the ones it's known to have. + */ + public static listVoices( + options?: Options, + ): RequestResult { + return (options?.client ?? client).get({ + url: "/api/v1/voice/voices", + ...options, + }); + } } export class Vaults { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index fe9bd67f7..35b61e218 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -14383,6 +14383,12 @@ export type SpeakRequest = { * Text */ text: string; + /** + * Voice + * + * read in this voice (an id from GET /voice/voices); else the settings' + */ + voice?: string | null; }; /** @@ -16498,6 +16504,56 @@ export type VoiceInfo = { [key: string]: unknown; }; +/** + * VoiceList + */ +export type VoiceList = { + /** + * Provider + */ + provider: string; + /** + * Current + * + * the voice answers are read in now (null: the provider's default) + */ + current?: string | null; + /** + * Voices + */ + voices: Array; + /** + * Error + * + * why the provider's voices couldn't be listed + */ + error?: string | null; + [key: string]: unknown; +}; + +/** + * VoiceOption + */ +export type VoiceOption = { + /** + * Id + * + * what voice.tts_voice is set to + */ + id: string; + /** + * Name + */ + name: string; + /** + * Detail + * + * what the provider says about it: accent, gender, style, language + */ + detail?: string | null; + [key: string]: unknown; +}; + /** * Watch */ @@ -31522,6 +31578,36 @@ export type SpeakTextResponses = { export type SpeakTextResponse = SpeakTextResponses[keyof SpeakTextResponses]; +export type ListVoicesData = { + body?: never; + path?: never; + query?: { + /** + * Provider + */ + provider?: string | null; + }; + url: "/api/v1/voice/voices"; +}; + +export type ListVoicesErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ListVoicesError = ListVoicesErrors[keyof ListVoicesErrors]; + +export type ListVoicesResponses = { + /** + * Successful Response + */ + 200: VoiceList; +}; + +export type ListVoicesResponse = ListVoicesResponses[keyof ListVoicesResponses]; + export type UnsealVaultData = { body?: never; path: { diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 1a7363de9..37e2150c6 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -2591,7 +2591,7 @@ "required": true, "schema": { "type": "string", - "pattern": "^(openai|elevenlabs|assemblyai|deepgram)$", + "pattern": "^(openai|elevenlabs|assemblyai|deepgram|gemini)$", "title": "Provider" } } @@ -23429,6 +23429,57 @@ } } }, + "/api/v1/voice/voices": { + "get": { + "tags": [ + "voice" + ], + "summary": "List Voices", + "description": "The voices a text-to-speech provider (else the one in the voice settings) reads in, from the provider when it\nlists them (ElevenLabs, Deepgram, macOS say, OpenAI-compatible servers that do), else the ones it's known to have.", + "operationId": "list_voices", + "parameters": [ + { + "name": "provider", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(openai|elevenlabs|deepgram|gemini|say)$" + }, + { + "type": "null" + } + ], + "title": "Provider" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceList" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/namespaces/{name}/vault": { "get": { "tags": [ @@ -49047,6 +49098,19 @@ "type": "string", "maxLength": 8000, "title": "Text" + }, + "voice": { + "anyOf": [ + { + "type": "string", + "maxLength": 200 + }, + { + "type": "null" + } + ], + "title": "Voice", + "description": "read in this voice (an id from GET /voice/voices); else the settings'" } }, "additionalProperties": false, @@ -52631,6 +52695,84 @@ ], "title": "VoiceInfo" }, + "VoiceList": { + "properties": { + "provider": { + "type": "string", + "title": "Provider" + }, + "current": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Current", + "description": "the voice answers are read in now (null: the provider's default)" + }, + "voices": { + "items": { + "$ref": "#/components/schemas/VoiceOption" + }, + "type": "array", + "title": "Voices" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "why the provider's voices couldn't be listed" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "provider", + "voices" + ], + "title": "VoiceList" + }, + "VoiceOption": { + "properties": { + "id": { + "type": "string", + "title": "Id", + "description": "what voice.tts_voice is set to" + }, + "name": { + "type": "string", + "title": "Name" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail", + "description": "what the provider says about it: accent, gender, style, language" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "id", + "name" + ], + "title": "VoiceOption" + }, "Watch": { "properties": { "id": { From fba3130f52d85535b8ffef5199097564d03fe6d7 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 19:58:47 +0000 Subject: [PATCH 2/2] feat(settings): pick the voice answers are read in from the provider's list, and hear it Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01WkPdbtd7zvYMRWaRkLN2F6 --- DEVLOG.md | 2 +- .../__tests__/settings-speech.test.tsx | 4 +- .../__tests__/settings-voice-picker.test.tsx | 75 +++++++++++ .../components/settings/bodies.tsx | 30 ++--- nextjs-frontend/components/settings/model.ts | 13 +- .../components/settings/voice-picker.tsx | 117 ++++++++++++++++++ 6 files changed, 222 insertions(+), 19 deletions(-) create mode 100644 nextjs-frontend/__tests__/settings-voice-picker.test.tsx create mode 100644 nextjs-frontend/components/settings/voice-picker.tsx diff --git a/DEVLOG.md b/DEVLOG.md index 0d2105828..a7620bb7d 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -13,7 +13,7 @@ Todo: - [x] Gemini TTS and macOS `say` as text-to-speech providers; `GET /voice/voices` lists each provider's voices; `POST /voice/speak` takes a voice to try - [ ] StyleTTS2 as an opt-in local service (off by default; heavy) -- [ ] Settings → AI assistant → Voice: pick a voice from the list and hear it +- [x] Settings → AI assistant → Voice: pick a voice from the list and hear it; Gemini in Speech providers Refine later: pick a second `say` voice for podcast hosts from what the Mac has; Gemini's multi-speaker mode for a whole podcast in one call; ElevenLabs voice settings (stability, style); cache the voice lists. diff --git a/nextjs-frontend/__tests__/settings-speech.test.tsx b/nextjs-frontend/__tests__/settings-speech.test.tsx index 6e6411826..a20eaae3d 100644 --- a/nextjs-frontend/__tests__/settings-speech.test.tsx +++ b/nextjs-frontend/__tests__/settings-speech.test.tsx @@ -57,9 +57,9 @@ describe("speech providers", () => { it("shows each provider with its own address, and tests one", async () => { show(); - for (const name of ["OpenAI-compatible", "ElevenLabs", "AssemblyAI", "Deepgram"]) + for (const name of ["OpenAI-compatible", "ElevenLabs", "AssemblyAI", "Deepgram", "Google Gemini"]) expect(screen.getByRole("heading", { name })).toBeInTheDocument(); - expect(screen.getAllByLabelText("Address")).toHaveLength(4); + expect(screen.getAllByLabelText("Address")).toHaveLength(5); fireEvent.click(screen.getAllByRole("button", { name: "Test" })[3]); await waitFor(() => expect(Admin.testSpeech).toHaveBeenCalledTimes(1)); expect((Admin.testSpeech as jest.Mock).mock.calls[0][0]).toMatchObject({ query: { provider: "deepgram" } }); diff --git a/nextjs-frontend/__tests__/settings-voice-picker.test.tsx b/nextjs-frontend/__tests__/settings-voice-picker.test.tsx new file mode 100644 index 000000000..0e3572e57 --- /dev/null +++ b/nextjs-frontend/__tests__/settings-voice-picker.test.tsx @@ -0,0 +1,75 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Voice } from "@/app/openapi-client"; +import { SAMPLE, VoicePicker } from "@/components/settings/voice-picker"; + +jest.mock("@/app/openapi-client", () => ({ Voice: { listVoices: jest.fn(), speakText: jest.fn() } })); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); + +const ok = (data: unknown, status = 200) => Promise.resolve({ data, response: { ok: true, status } }); + +function show(provider: string, value: string | null, onChange = jest.fn(), savedProvider = provider) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + render( + + + , + ); + return onChange; +} + +describe("voice picker", () => { + beforeEach(() => { + (Voice.listVoices as jest.Mock).mockReset(); + (Voice.speakText as jest.Mock).mockReset(); + }); + + it("offers the provider's voices and keeps the id picked", async () => { + (Voice.listVoices as jest.Mock).mockImplementation(() => + ok({ + provider: "elevenlabs", + voices: [ + { id: "v42", name: "Ada", detail: "female, british" }, + { id: "v7", name: "Bo", detail: null }, + ], + }), + ); + const onChange = show("elevenlabs", "old-id"); + expect(await screen.findByRole("option", { name: "Ada · female, british" })).toBeInTheDocument(); + const select = screen.getByLabelText("Voice"); + expect((Voice.listVoices as jest.Mock).mock.calls[0][0]).toMatchObject({ query: { provider: "elevenlabs" } }); + expect(screen.getByRole("option", { name: "old-id (not in the list)" })).toBeInTheDocument(); + fireEvent.change(select, { target: { value: "v42" } }); + expect(onChange).toHaveBeenCalledWith("v42"); + fireEvent.change(select, { target: { value: "" } }); + expect(onChange).toHaveBeenLastCalledWith(null); + }); + + it("falls back to typing an id when the list can't be had, and plays a voice", async () => { + (Voice.listVoices as jest.Mock).mockImplementation(() => + ok({ provider: "deepgram", voices: [], error: "Deepgram has no API key yet" }), + ); + (Voice.speakText as jest.Mock).mockImplementation(() => ok(new Blob(["ID3"]), 204)); + show("deepgram", "aura-2-orion-en"); + expect(await screen.findByText(/Deepgram has no API key yet\. Type a voice id instead\./)).toBeInTheDocument(); + expect(screen.getByLabelText("Voice")).toHaveValue("aura-2-orion-en"); + fireEvent.click(screen.getByRole("button", { name: "Hear this voice" })); + await waitFor(() => expect(Voice.speakText).toHaveBeenCalledTimes(1)); + expect((Voice.speakText as jest.Mock).mock.calls[0][0]).toMatchObject({ + body: { text: SAMPLE, voice: "aura-2-orion-en" }, + }); + expect(await screen.findByText(/It couldn’t read it/)).toBeInTheDocument(); + }); + + it("asks to save before hearing a provider not saved yet", async () => { + (Voice.listVoices as jest.Mock).mockImplementation(() => + ok({ provider: "gemini", voices: [{ id: "Kore", name: "Kore", detail: "Firm" }] }), + ); + show("gemini", null, jest.fn(), "openai"); + expect(await screen.findByRole("option", { name: "Default (Kore)" })).toBeInTheDocument(); + expect(screen.getByText("Save to hear a voice from this provider.")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Hear this voice" })).toBeDisabled(); + }); +}); diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index d7d16bf49..07af5a36c 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -14,6 +14,7 @@ import { RIGHTS } from "@/components/iiif/rights"; import { hubText } from "@/components/sensors/sensor-model"; import { SecretSetting, SettingField, ZoneBar, type FieldState } from "@/components/settings/fields"; import { SignInProviders } from "@/components/settings/sign-in-providers"; +import { VoicePicker } from "@/components/settings/voice-picker"; import { AddConnection } from "@/components/storage/add-connection"; import { NOT_STORAGE, fileStoreTry } from "@/components/storage/file-store"; import { @@ -1284,27 +1285,26 @@ function AiBody({ ctx }: { ctx: BodyCtx }) { - - {tts !== "deepgram" && ( + {(tts === "openai" || tts === "elevenlabs") && ( )} + {tts === "openai" && } - {tts === "openai" ? ( + {tts === "say" ? ( +

+ Uses the voices installed on the Mac Lens runs on (System Settings → Accessibility → Spoken Content adds + more). Not available when Lens runs in Docker. +

+ ) : tts === "openai" ? ( = { + openai: "alloy", + elevenlabs: "a premade voice", + deepgram: "aura-2-thalia-en", + gemini: "Kore", + say: "the Mac’s own voice", +}; + +/** voice.tts_voice: a voice from the list the provider gives (its id is kept), with a button to hear it. */ +export function VoicePicker({ + provider, + savedProvider, + state, +}: { + provider: string; + savedProvider: string; + state: FieldState; +}) { + const client = useApiClient(); + const id = useId(); + const msg = `${id}-msg`; + const audio = useRef(null); + const value = String(state.value ?? ""); + const list = useQuery({ + queryKey: ["voices", provider], + queryFn: () => data(Voice.listVoices({ client, query: { provider } })), + staleTime: 5 * 60_000, + }); + const hear = useMutation({ + mutationFn: async () => { + const r = await Voice.speakText({ client, body: { text: SAMPLE, voice: value || null }, parseAs: "blob" }); + if (r.response?.status !== 200 || !(r.data instanceof Blob) || !r.data.size) + throw new Error("It couldn’t read it: check the provider’s key and address, then try again."); + audio.current?.pause(); + const url = URL.createObjectURL(r.data); + const a = new Audio(url); + audio.current = a; + a.onended = () => URL.revokeObjectURL(url); + await a.play(); + }, + }); + + const voices = list.data?.voices ?? []; + const known = voices.some((v) => v.id === value); + const options = [ + { value: "", label: `Default (${DEFAULT_LABEL[provider] ?? "the provider’s"})` }, + ...(value && !known ? [{ value, label: `${value} (not in the list)` }] : []), + ...voices.map((v) => ({ value: v.id, label: v.detail ? `${v.name} · ${v.detail}` : v.name })), + ]; + const unsaved = provider !== savedProvider; + const note = state.error + ? state.error + : hear.error + ? hear.error.message + : list.data?.error + ? `${list.data.error}. Type a voice id instead.` + : unsaved + ? "Save to hear a voice from this provider." + : null; + + return ( +
+ +
+ {voices.length ? ( + state.onChange(e.target.value || null)} + /> + )} + +
+ {note && ( + + {note} + + )} +
+ ); +}