From 9db0215f94be631d3eb1c533f428851e32e7f859 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:02:41 +0000 Subject: [PATCH] feat(voice): read aloud with StyleTTS2 on this machine, as an opt-in service Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01WkPdbtd7zvYMRWaRkLN2F6 --- .github/workflows/styletts2.yml | 56 +++++++ DEVLOG.md | 5 +- docker-compose.prod.yml | 16 ++ docker-compose.yml | 15 ++ docs/speech-providers.md | 23 ++- fastapi_backend/app/domain/podcasts.py | 4 +- fastapi_backend/app/domain/settings.py | 6 +- fastapi_backend/app/domain/speech.py | 44 +++++- fastapi_backend/app/domain/store.py | 4 +- fastapi_backend/tests/domain/test_speech.py | 37 +++++ .../components/settings/bodies.tsx | 10 +- nextjs-frontend/components/settings/model.ts | 9 ++ .../components/settings/voice-picker.tsx | 1 + nextjs-frontend/openapi.json | 2 +- styletts2/Dockerfile | 18 +++ styletts2/server.py | 146 ++++++++++++++++++ threat-model.md | 1 + threat-model.yaml | 2 +- 18 files changed, 385 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/styletts2.yml create mode 100644 styletts2/Dockerfile create mode 100644 styletts2/server.py diff --git a/.github/workflows/styletts2.yml b/.github/workflows/styletts2.yml new file mode 100644 index 000000000..c6e468769 --- /dev/null +++ b/.github/workflows/styletts2.yml @@ -0,0 +1,56 @@ +name: StyleTTS2 + +# The StyleTTS2 service Lens can read aloud with (the optional styletts2 service, styletts2/): it builds, lists its +# voices, and reads a sentence aloud as WAV (fetching the model on the way). +on: + pull_request: + paths: + - styletts2/** + - .github/workflows/styletts2.yml + push: + branches: [main] + paths: + - styletts2/** + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + styletts2: + name: Build and read aloud + runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }} + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - name: Build + run: docker build -t lens-styletts2:ci styletts2 + + - name: Start + run: | + docker run -d --name st -p 127.0.0.1:8880:8880 lens-styletts2:ci + for _ in $(seq 1 30); do curl -fsS http://127.0.0.1:8880/health && break; sleep 2; done + curl -fsS http://127.0.0.1:8880/v1/audio/voices | tee /dev/stderr | grep -q '"default"' + + - name: Read a sentence aloud + run: | + code=$(curl -sS -o speech.wav -w '%{http_code}' --max-time 1500 \ + -H 'Content-Type: application/json' \ + -d '{"model":"styletts2","input":"Lens reads this sentence aloud.","voice":"default"}' \ + http://127.0.0.1:8880/v1/audio/speech) + echo "HTTP $code, $(stat -c %s speech.wav) bytes" + test "$code" = 200 + head -c 4 speech.wav | grep -q RIFF + test "$(stat -c %s speech.wav)" -gt 20000 + code=$(curl -sS -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' \ + -d '{"input":"Hi","voice":"nobody"}' http://127.0.0.1:8880/v1/audio/speech) + test "$code" = 404 + + - name: Logs + if: always() + run: docker logs st 2>&1 | tail -50 diff --git a/DEVLOG.md b/DEVLOG.md index a7620bb7d..1e41fc086 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -12,11 +12,12 @@ Todo: - [x] Gemini TTS and macOS `say` as text-to-speech providers; `GET /voice/voices` lists each provider's voices; `POST /voice/speak` takes a voice to try -- [ ] StyleTTS2 as an opt-in local service (off by default; heavy) +- [x] StyleTTS2 as an opt-in local service (`styletts2/`, Compose profile `styletts2`; voices are clips in a volume) - [x] Settings → AI assistant → Voice: pick a voice from the list and hear it; Gemini in Speech providers Refine later: pick a second `say` voice for podcast hosts from what the Mac has; Gemini's multi-speaker mode for a -whole podcast in one call; ElevenLabs voice settings (stability, style); cache the voice lists. +whole podcast in one call; ElevenLabs voice settings (stability, style); cache the voice lists; upload StyleTTS2 voice +clips from Settings (or use a speaker's own voiceprint clips); start the StyleTTS2 service from Settings; GPU build. ## 2026-10-10 · Podcasts from picked knowledge diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c5b41f574..fcea0984e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -139,6 +139,20 @@ services: restart: unless-stopped networks: [lens] + # StyleTTS2 (docs/speech-providers.md#styletts2): spoken answers and podcasts read on this machine, in a voice like a + # short clip you put in its voices volume. Heavy (PyTorch and a 1 GB model, fetched on first use), so opt-in: + # `docker compose --profile styletts2 up -d`, then Settings → AI assistant → Voice → StyleTTS2. + styletts2: + security_opt: ["no-new-privileges:true"] + image: lens-styletts2:${LENS_VERSION:-prod} + build: ./styletts2 + profiles: [styletts2] + volumes: + - styletts2-data:/data + - styletts2-voices:/voices + restart: unless-stopped + networks: [lens] + # Fedora 6 (docs/fedora.md), the repository Lens can keep a copy of the archive in. Opt-in, and not needed on small # machines: `docker compose --profile fedora up`, with LENS_FEDORA_URL=http://fedora:8080/fcrepo/rest (and # LENS_FEDORA_USER / LENS_FEDORA_PASSWORD) in .env, or set it in Settings → Fedora. @@ -172,6 +186,8 @@ services: volumes: surreal-data: matterbridge-data: + styletts2-data: + styletts2-voices: archive-data: fedora-data: opensearch-data: diff --git a/docker-compose.yml b/docker-compose.yml index 179d67cda..07bf37855 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,6 +131,19 @@ services: - matterbridge-data:/matterbridge networks: [lens] + # StyleTTS2 (docs/speech-providers.md#styletts2): spoken answers and podcasts read on this machine, in a voice like a + # short clip you put in its voices volume. Heavy (PyTorch and a 1 GB model, fetched on first use), so opt-in: + # `docker compose --profile styletts2 up -d`, then Settings → AI assistant → Voice → StyleTTS2. + styletts2: + security_opt: ["no-new-privileges:true"] + image: lens-styletts2:dev + build: ./styletts2 + profiles: [styletts2] + volumes: + - styletts2-data:/data + - styletts2-voices:/voices + networks: [lens] + # Fedora 6 (docs/fedora.md), the repository Lens can keep a copy of the archive in. Opt-in, and not needed on small # machines: `docker compose --profile fedora up`, with LENS_FEDORA_URL=http://fedora:8080/fcrepo/rest (and # LENS_FEDORA_USER / LENS_FEDORA_PASSWORD) in .env, or set it in Settings → Fedora. @@ -149,6 +162,8 @@ services: volumes: surreal-data: matterbridge-data: + styletts2-data: + styletts2-voices: archive-data: fedora-data: diff --git a/docs/speech-providers.md b/docs/speech-providers.md index 7f136e5f4..28252651a 100644 --- a/docs/speech-providers.md +++ b/docs/speech-providers.md @@ -17,6 +17,7 @@ summaries, chapters and entities as optional imports, per-minute prices in the a | Deepgram | Nova | yes | detected | from sentiment (English) | no | Aura | | Google Gemini | no | no | no | no | no | `gemini-2.5-flash-preview-tts`, 30 prebuilt voices | | macOS `say` | no | no | no | no | no | the Mac's own voices; no key, nothing leaves the Mac | +| StyleTTS2 | no | no | no | no | no | on this machine, in a voice like a clip you give it ([below](#styletts2)) | Sentiment becomes emotion as positive → Happy, negative → Sad, neutral → Neutral (turn it off with `speech.sentiment`). SenseVoice's own emotion tags stay richer; a provider only reports sentiment. @@ -63,7 +64,27 @@ every Mac. | `speech.timeout` | 1800 seconds | | `voice.stt` | `same` | | `voice.tts_provider` | `openai` | -| `voice.tts_voice` | the provider's default (alloy, a premade ElevenLabs voice, aura-2-thalia-en, Kore, the Mac's) | +| `voice.tts_voice` | the provider's default (alloy, a premade ElevenLabs voice, aura-2-thalia-en, Kore, the Mac's, `default`) | +| `voice.styletts2_url` | `http://styletts2:8880` | + +## StyleTTS2 + +[StyleTTS2](https://github.com/yl4579/StyleTTS2) reads aloud on your own machine: nothing leaves it, and nothing is +billed. It is heavy (PyTorch, a 2 GB image and a 1 GB model fetched from Hugging Face on first use; a sentence takes +a few seconds on a recent CPU), so it is off by default and runs as its own service, `styletts2/`: + +```sh +docker compose --profile styletts2 up -d +``` + +Then pick **StyleTTS2** in Settings → AI assistant → Voice. `voice.styletts2_url` is where Lens reaches it +(`http://styletts2:8880`, the Compose service). It speaks the OpenAI audio API (`POST /v1/audio/speech`, +`GET /v1/audio/voices`). + +Its voices are clips: put a few seconds of someone speaking (`.wav`, `.mp3` or `.flac`, named after the voice, e.g. +`grandpa.wav`) in the `styletts2-voices` volume and it reads in a voice like theirs. `default` is a public-domain +LibriVox reader. The model's authors ask that listeners are told the speech is synthesized, unless you have +permission to use the voice. ## What is sent diff --git a/fastapi_backend/app/domain/podcasts.py b/fastapi_backend/app/domain/podcasts.py index 866ea9a07..00f00c922 100644 --- a/fastapi_backend/app/domain/podcasts.py +++ b/fastapi_backend/app/domain/podcasts.py @@ -718,8 +718,8 @@ def timed(lines, wpm=150, pause_ms=400): def voices(cfg, req=None): """{a, b}: the hosts' voices for the text-to-speech provider: what the episode asked for, else podcasts.voice_a / - voice_b, else voice.tts_voice and a second voice of the provider's (say has no voice every Mac is sure to have, so - both hosts get the same one unless podcasts.voice_b names another).""" + voice_b, else voice.tts_voice and a second voice of the provider's (say and StyleTTS2 have no second voice they're + sure to have, so both hosts get the same one unless podcasts.voice_b names another).""" from . import speech p = cfg.get("podcasts") or {} diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 9f19ccfef..2363d42a6 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -137,7 +137,7 @@ ("decisions", "engine"): {"auto", "jev", "laya", "llm", "off"}, ("voice", "input"): {"auto", "server", "browser"}, ("voice", "stt"): {"same", "sensevoice", "whisper", "mlx-whisper", "openai", "elevenlabs", "assemblyai", "deepgram"}, - ("voice", "tts_provider"): {"openai", "elevenlabs", "deepgram", "gemini", "say"}, + ("voice", "tts_provider"): {"openai", "elevenlabs", "deepgram", "gemini", "say", "styletts2"}, ("mail", "security"): {"starttls", "ssl", "none"}, ("bridge", "answer"): {"mention", "all"}, ("bridge", "approve"): {"off", "low_risk"}, @@ -426,6 +426,10 @@ def _check(section, key, value, default): if not (isinstance(value, str) and VIEWER_URL.match(value.strip())): raise ValueError("voice.tts_base_url is the http(s) address of an OpenAI-compatible server with /audio/speech") return value.strip().rstrip("/") + if (section, key) == ("voice", "styletts2_url"): + if not (isinstance(value, str) and VIEWER_URL.match(value.strip())): + raise ValueError("voice.styletts2_url is the http(s) address of the styletts2 service") + return value.strip().rstrip("/") if section == "voice" and key in ("tts_model", "tts_voice", "tts_api_key"): if value in (None, ""): return None diff --git a/fastapi_backend/app/domain/speech.py b/fastapi_backend/app/domain/speech.py index 4889f10a9..da54867e6 100644 --- a/fastapi_backend/app/domain/speech.py +++ b/fastapi_backend/app/domain/speech.py @@ -12,7 +12,8 @@ * gemini: Google Gemini text to speech (its prebuilt voices). Text to speech can also come from macOS `say` when Lens runs on a Mac itself (not in Docker): nothing to set up, the -Mac's own voices. voices() lists what each one can read in, for Settings and podcast hosts. +Mac's own voices; or from StyleTTS2, the optional `styletts2` service (styletts2/), which reads in a voice like a +short clip of someone speaking. voices() lists what each one can read in, for Settings and podcast hosts. A provider's transcript is the same list of segments a local engine returns, plus `provider_speaker`: the speaker separation step uses those labels (diarize.engine provider, or auto), and voice IDs still come from local voiceprints. @@ -53,8 +54,9 @@ "deepgram": "Deepgram", "gemini": "Google Gemini", "say": "macOS say", + "styletts2": "StyleTTS2", } -TTS_PROVIDERS = ("openai", "elevenlabs", "deepgram", "gemini", "say") +TTS_PROVIDERS = ("openai", "elevenlabs", "deepgram", "gemini", "say", "styletts2") CHUNK_SECONDS = 600 # OpenAI-compatible servers take up to 25 MB a request; ten minutes of Opus is a few MB POLL_SECONDS = 3 MAX_WORDS = 40 @@ -68,7 +70,15 @@ DEEPGRAM_TTS_MODEL = "aura-2-thalia-en" GEMINI_VOICE = "Kore" # what each provider reads in when nothing is picked (say: the Mac's own voice) -DEFAULT_VOICE = {"openai": "alloy", "elevenlabs": ELEVENLABS_VOICE, "deepgram": DEEPGRAM_TTS_MODEL, "gemini": GEMINI_VOICE, "say": None} +DEFAULT_VOICE = { + "openai": "alloy", + "elevenlabs": ELEVENLABS_VOICE, + "deepgram": DEEPGRAM_TTS_MODEL, + "gemini": GEMINI_VOICE, + "say": None, + "styletts2": "default", +} +STYLETTS2_SECONDS = 600 # it reads on the CPU, a long answer slowly; the first call also loads the model OPENAI_VOICES = ("alloy", "ash", "ballad", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer", "verse") # Gemini's prebuilt voices and how Google describes each GEMINI_VOICES = { @@ -491,6 +501,8 @@ def tts_ready(cfg): return bool(v.get("tts_model") and (v.get("tts_base_url") or (cfg.get("llm") or {}).get("base_url"))) if p == "say": return say_available() + if p == "styletts2": + return bool(v.get("styletts2_url")) # its own service; when it isn't running the browser reads answers return p in TTS_PROVIDERS and bool(_key(cfg, p)) @@ -583,7 +595,25 @@ def _speak_openai(cfg, text, voice): return m, lambda: _request(base + "/audio/speech", data=json.dumps(body).encode(), headers=headers, timeout=60) -SPEAK = {"openai": _speak_openai, "elevenlabs": _speak_elevenlabs, "deepgram": _speak_deepgram, "gemini": _speak_gemini, "say": _speak_say} +def _styletts2_url(cfg): + return str((cfg.get("voice") or {}).get("styletts2_url") or "").rstrip("/") + + +def _speak_styletts2(cfg, text, voice): + body = {"model": "styletts2", "input": text, "voice": voice, "response_format": "wav"} + url = _styletts2_url(cfg) + "/v1/audio/speech" + headers = {"Content-Type": "application/json"} + return "styletts2", lambda: _request(url, data=json.dumps(body).encode(), headers=headers, timeout=STYLETTS2_SECONDS) + + +SPEAK = { + "styletts2": _speak_styletts2, + "openai": _speak_openai, + "elevenlabs": _speak_elevenlabs, + "deepgram": _speak_deepgram, + "gemini": _speak_gemini, + "say": _speak_say, +} def speak(cfg, text, voice=None): @@ -666,6 +696,12 @@ def voices(cfg, provider=None): return _say_voices(), None except (OSError, subprocess.TimeoutExpired) as e: return [], f"say didn't list its voices: {e}" + if p == "styletts2": + try: + names = _json(_styletts2_url(cfg) + "/v1/audio/voices", timeout=15).get("voices") or [] + except (ProviderError, AttributeError) as e: + return [], f"StyleTTS2 isn't running ({e}): docker compose --profile styletts2 up -d" + return [_voice(n, detail="its own reader" if n == "default" else "from your clip") for n in names if isinstance(n, str)], None if not _key(cfg, p): return [], f"{LABELS[p]} has no API key yet: add it in Settings → Speech providers" try: diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index bf6baa566..43785d105 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -292,7 +292,8 @@ def labels(): # browser's; spoken answers come from tts_model (an OpenAI-compatible /audio/speech), else the browser reads them # stt: the engine that hears voice chat ("same": the transcription engine); tts_provider: who reads answers aloud # (openai: the OpenAI-compatible speech server below; elevenlabs, deepgram, gemini: the keys in speech; say: macOS's - # own voices, when Lens runs on a Mac itself); tts_voice: the voice they read in (null: the provider's default) + # own voices, when Lens runs on a Mac itself; styletts2: the optional service at styletts2_url); tts_voice: the + # voice they read in (null: the provider's default) "voice": { "input": "auto", "stt": "same", @@ -301,6 +302,7 @@ def labels(): "tts_model": None, "tts_voice": None, "tts_api_key": None, + "styletts2_url": "http://styletts2:8880", }, # podcast episodes made from what someone picked (domain/podcasts.py): where they go, the hosts' names, how much # source text the model reads (characters), the longest episode (minutes), the speaking pace, and a model for diff --git a/fastapi_backend/tests/domain/test_speech.py b/fastapi_backend/tests/domain/test_speech.py index 736225a6b..9abce7123 100644 --- a/fastapi_backend/tests/domain/test_speech.py +++ b/fastapi_backend/tests/domain/test_speech.py @@ -76,6 +76,8 @@ def do_GET(self): ) if self.path.startswith("/gm/v1beta/models"): return self._send({"models": [{"name": "models/gemini-2.5-flash-preview-tts"}, {"name": "models/gemini-2.5-pro"}]}) + if self.path == "/st/v1/audio/voices": + return self._send({"voices": ["default", "grandpa"]}) if self.path == "/oai/v1/audio/voices": return self._send({"voices": ["af_heart", "am_adam"]}) if self.path.endswith("/models") or self.path.endswith("/v1/models"): @@ -138,6 +140,8 @@ def do_POST(self): } } ) + if p == "/st/v1/audio/speech": + return self._send(speech.wav(b"\x00\x00" * 24, 24000), "audio/wav") if p.startswith("/gm/v1beta/models/"): pcm = base64.b64encode(b"\x00\x01" * 2400).decode() return self._send( @@ -349,6 +353,39 @@ def test_each_provider_lists_its_voices(db, cfg, fake): speech.voices(eff, "nope") +def test_styletts2_reads_aloud_from_its_own_service(db, cfg, fake): + settings.save(db, cfg, "voice", {"tts_provider": "styletts2"}, "t") + eff = settings.effective(db, cfg) + assert eff["voice"]["styletts2_url"] == "http://styletts2:8880" and voice.can_speak(eff) # the compose service + settings.save(db, cfg, "voice", {"styletts2_url": fake + "/st/"}, "t") + eff = settings.effective(db, cfg) + data, ctype = voice.speak(eff, "Hello", voice="grandpa") + assert ctype == "audio/wav" and data[:4] == b"RIFF" + method, path, headers, body = Fake.seen[-1] + assert path == "/st/v1/audio/speech" and json.loads(body) == { + "model": "styletts2", + "input": "Hello", + "voice": "grandpa", + "response_format": "wav", + } + voice.speak(eff, "Hi") + assert json.loads(Fake.seen[-1][3])["voice"] == "default" + assert speech.voices(eff) == ( + [ + {"id": "default", "name": "default", "detail": "its own reader"}, + {"id": "grandpa", "name": "grandpa", "detail": "from your clip"}, + ], + None, + ) + settings.save(db, cfg, "voice", {"styletts2_url": "http://127.0.0.1:9"}, "t") + eff = settings.effective(db, cfg) + found, err = speech.voices(eff) + assert found == [] and err.startswith("StyleTTS2 isn't running") and "--profile styletts2" in err + assert voice.speak(eff, "Hello") is None # not running: the browser reads it + with pytest.raises(ValueError): + settings.save(db, cfg, "voice", {"styletts2_url": "styletts2:8880"}, "t") + + SAY_VOICES = """Albert en_US # Hello! My name is Albert. Eddy (English (UK)) en_GB # Hello! My name is Eddy. Bad News en_US # Hello! My name is Bad News. diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index cd2cf6bdc..ed1a1dcf5 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -1302,8 +1302,16 @@ function AiBody({ ctx }: { ctx: BodyCtx }) { state={ctx.state("voice.tts_voice")} /> {tts === "openai" && } + {tts === "styletts2" && } - {tts === "say" ? ( + {tts === "styletts2" ? ( +

+ Reads on this machine, in a voice like a short clip of someone speaking: put a few seconds of a .wav in the + service’s voices folder and pick it here. Heavy, so it runs only when started with{" "} + docker compose --profile styletts2 up -d. Tell listeners the + speech is synthesized unless you may use the voice. +

+ ) : tts === "say" ? (

Uses the voices installed on the Mac Lens runs on (System Settings → Accessibility → Spoken Content adds more). Not available when Lens runs in Docker. diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 80deb8049..3c84ee116 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -1063,6 +1063,7 @@ export const FIELDS: FieldSpec[] = [ { value: "deepgram", label: "Deepgram Aura (provider)" }, { value: "gemini", label: "Google Gemini (provider)" }, { value: "say", label: "macOS say (Lens running on a Mac)" }, + { value: "styletts2", label: "StyleTTS2 (on this machine)" }, ], }, { @@ -1094,6 +1095,14 @@ export const FIELDS: FieldSpec[] = [ placeholder: "the LLM provider’s", }, { section: "voice", key: "tts_api_key", label: "Speech server API key", kind: "secret" }, + { + section: "voice", + key: "styletts2_url", + label: "StyleTTS2 service", + kind: "text", + mono: true, + hint: "http://styletts2:8880 in Docker Compose", + }, // Local model { section: "local_llm", key: "enabled", label: "Run a model here", kind: "switch" }, { diff --git a/nextjs-frontend/components/settings/voice-picker.tsx b/nextjs-frontend/components/settings/voice-picker.tsx index 0578ed322..1891619b2 100644 --- a/nextjs-frontend/components/settings/voice-picker.tsx +++ b/nextjs-frontend/components/settings/voice-picker.tsx @@ -17,6 +17,7 @@ const DEFAULT_LABEL: Record = { deepgram: "aura-2-thalia-en", gemini: "Kore", say: "the Mac’s own voice", + styletts2: "its own reader", }; /** voice.tts_voice: a voice from the list the provider gives (its id is kept), with a button to hear it. */ diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 60a269183..fc8288f21 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -23661,7 +23661,7 @@ "anyOf": [ { "type": "string", - "pattern": "^(openai|elevenlabs|deepgram|gemini|say)$" + "pattern": "^(openai|elevenlabs|deepgram|gemini|say|styletts2)$" }, { "type": "null" diff --git a/styletts2/Dockerfile b/styletts2/Dockerfile new file mode 100644 index 000000000..796eb9cfb --- /dev/null +++ b/styletts2/Dockerfile @@ -0,0 +1,18 @@ +# StyleTTS2 (https://github.com/yl4579/StyleTTS2, MIT; packaged as `styletts2` on PyPI, MIT) for Lens's spoken answers +# and podcasts: the optional `styletts2` service in Docker Compose, picked in Settings → AI assistant → Voice +# (fastapi_backend/app/domain/speech.py). Heavy (PyTorch, about 2 GB of image and 1 GB of model), so it's opt-in: +# `docker compose --profile styletts2 up -d`. It runs on the CPU; the model is fetched into /data on first use. +FROM python:3.11-slim +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 HOME=/data +RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1 ffmpeg \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --index-url https://download.pytorch.org/whl/cpu torch==2.5.1 torchaudio==2.5.1 \ + && pip install "styletts2==0.1.6" "nltk==3.8.1" "numpy<2" \ + && python -m nltk.downloader -d /usr/local/share/nltk_data punkt +RUN useradd --uid 1000 --create-home --home-dir /data lens && mkdir -p /voices && chown lens /data /voices +COPY server.py /app/server.py +USER lens +VOLUME ["/data", "/voices"] +EXPOSE 8880 +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8880/health', timeout=4)" +CMD ["python", "/app/server.py"] diff --git a/styletts2/server.py b/styletts2/server.py new file mode 100644 index 000000000..c16e2f232 --- /dev/null +++ b/styletts2/server.py @@ -0,0 +1,146 @@ +"""StyleTTS2 behind the OpenAI audio API, for Lens (docs/speech-providers.md#styletts2). + +POST /v1/audio/speech {input, voice} the text read aloud, as 24 kHz WAV +GET /v1/audio/voices {"voices": ["default", ...]}: "default" and each clip in /voices +GET /health {"ok": true, "loaded": whether the model is in memory} + +A voice is a few seconds of someone speaking (a .wav, .mp3 or .flac in /voices, named after the voice): StyleTTS2 +reads in a voice like it. "default" is a public-domain LibriVox reader. The model loads on the first request and +stays loaded; requests are read one at a time. +""" + +from __future__ import annotations + +import io +import json +import os +import pathlib +import re +import threading +import wave +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +VOICES = pathlib.Path(os.environ.get("STYLETTS2_VOICES", "/voices")) +PORT = int(os.environ.get("PORT", "8880")) +MAX_CHARS = 4000 +AUDIO = (".wav", ".mp3", ".flac") +NAME = re.compile(r"^[\w .-]{1,80}$") +SR = 24000 + +_lock = threading.Lock() +_state: dict = {"model": None, "styles": {}} + + +def voices(): + found = sorted( + { + p.stem + for p in VOICES.glob("*") + if p.suffix.lower() in AUDIO and NAME.match(p.stem) + } + ) + return ["default", *[v for v in found if v != "default"]] + + +def _clip(name): + for ext in AUDIO: + p = VOICES / f"{name}{ext}" + if p.is_file(): + return p + return None + + +def read(text, voice): + """The text in the voice, as 16-bit WAV. Raises LookupError for a voice there's no clip of.""" + with _lock: + if _state["model"] is None: + from styletts2 import tts # slow: only when first asked + + _state["model"] = tts.StyleTTS2() + m = _state["model"] + if voice not in _state["styles"]: + if voice == "default": + from cached_path import cached_path + from styletts2.tts import DEFAULT_TARGET_VOICE_URL + + path = cached_path(DEFAULT_TARGET_VOICE_URL) + else: + path = _clip(voice) if NAME.match(voice) else None + if path is None: + raise LookupError(voice) + _state["styles"][voice] = m.compute_style(str(path)) + samples = m.inference(text, ref_s=_state["styles"][voice]) + import numpy as np + + pcm = ( + (np.clip(np.asarray(samples, dtype=np.float32), -1.0, 1.0) * 32767) + .astype(" 1 else ''}", + flush=True, + ) + + def _send(self, code, body, ctype="application/json"): + data = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _error(self, code, message): + self._send(code, {"error": {"message": message}}) + + def do_GET(self): + path = self.path.split("?")[0].rstrip("/") + if path == "/health": + return self._send(200, {"ok": True, "loaded": _state["model"] is not None}) + if path in ("/v1/audio/voices", "/audio/voices"): + return self._send(200, {"voices": voices()}) + self._error(404, "not found") + + def do_POST(self): + path = self.path.split("?")[0].rstrip("/") + if path not in ("/v1/audio/speech", "/audio/speech"): + return self._error(404, "not found") + try: + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(min(n, 1_000_000)) or b"{}") + except ValueError: + return self._error(400, "the body is JSON: {input, voice}") + text = body.get("input") if isinstance(body, dict) else None + if not isinstance(text, str) or not text.strip(): + return self._error(400, "input is the text to read") + voice = body.get("voice") or "default" + if not isinstance(voice, str): + return self._error(400, "voice is a voice's name") + try: + audio = read(text.strip()[:MAX_CHARS], voice) + except LookupError: + return self._error( + 404, + f"no voice {voice!r}: put a clip named {voice}.wav in the voices folder", + ) + except Exception as e: # noqa: BLE001 - the caller gets the reason; the server keeps going + return self._error(500, f"StyleTTS2 couldn't read it: {e}") + self._send(200, audio, "audio/wav") + + +if __name__ == "__main__": + print(f"styletts2: listening on :{PORT}; voices in {VOICES}", flush=True) + ThreadingHTTPServer( + ("0.0.0.0", PORT), Handler + ).serve_forever() # inside its container diff --git a/threat-model.md b/threat-model.md index 01f0272e9..36f617348 100644 --- a/threat-model.md +++ b/threat-model.md @@ -429,6 +429,7 @@ third-party source; dependencies come from `uv.lock`, the pnpm lockfile and cont | cloudflared, llama.cpp, `hf:` models, pip packages | integrity only as far as HTTPS and the publisher | upstream; tampered downloads are disclaimed (N-DOWNLOADS) | *(maintainer, 2026-10)* | | Catalog GGUF models | pinned revision and SHA-256 | in model (P-MODEL-CHECKSUM) | *(documented, docs/local-models.md "checked against their SHA-256")* | | Optional ML extras (whisper, pyannote, speechbrain, insightface, doctr) | model inference; some fetch weights on their own | upstream | *(inferred, Q19)* | +| StyleTTS2 service (`styletts2/`, opt-in Compose profile) and macOS `say` | text to speech; StyleTTS2 fetches its weights from Hugging Face and GitHub on first use and listens only on the Compose network; `say` gets the text on stdin, never as arguments | upstream for the model and synthesis; the argument handling in model | *(maintainer, 2026-10)* | | Build, lint and codegen toolchain (frontend dev dependencies, openapi-ts, linters) | building the images; in no running service | upstream; a report must show the code ships | *(inferred, Q45)* | | OIDC providers, SMTP, model and speech providers | honest responses over TLS | the admin chose them; trusted input | *(maintainer, 2026-10)* | diff --git a/threat-model.yaml b/threat-model.yaml index 872a1a204..76ecb5aa5 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:c7c945e760490e73dbb947e89beff868a218b89b839b4172f9d928d4853e992a" +prose_version: "threat-model.md@sha256:4525de4bf73248fe9511a6ad69c55ff7c021b11a9a3e46a285bf25fe5208ea99" model_status: under-review triage_policy: strict confidence: {documented: 201, maintainer: 46, inferred: 91, assumption: 8}