From 2db7aee62bf1e5df5c6c792b358b8420c70bf750 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 09:43:32 +0000 Subject: [PATCH 01/10] feat(voice): read answers and podcasts aloud with Kokoro on this machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kokoro v1.0 (hexgrad's Kokoro-82M, Apache-2.0) as a text-to-speech provider, voice.tts_provider "kokoro": spoken answers and podcast hosts are read on the server's own CPU through kokoro-onnx on ONNX Runtime, in any of its 54 voices across nine languages, with nothing sent anywhere and nothing billed. It's opt-in and fetched once it's picked: the `kokoro` extra (espeak-ng and phonemizer are GPL-3.0, so they stay out of the default install), and the model (voice.kokoro_model: fp32, fp16 or int8) and voices files from kokoro-onnx's release, checked against their SHA-256. Settings → AI assistant → Voice offers it with the model size and the voice picker; a podcast's second host gets a voice in the first host's language. kokoro.yml fetches and reads with each model on x64 and arm64. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- .github/workflows/kokoro.yml | 80 +++++++ CHANGELOG.md | 7 + DEVLOG.md | 14 ++ README.md | 6 +- docs/assistant.md | 4 +- docs/components.md | 5 +- docs/podcasts.md | 4 +- docs/requirements.md | 1 + docs/speech-providers.md | 37 +++- fastapi_backend/app/domain/components.py | 50 +++++ fastapi_backend/app/domain/kokoro.py | 148 +++++++++++++ fastapi_backend/app/domain/podcasts.py | 14 +- fastapi_backend/app/domain/settings.py | 3 +- fastapi_backend/app/domain/speech.py | 30 ++- fastapi_backend/app/domain/store.py | 6 +- fastapi_backend/pyproject.toml | 5 + fastapi_backend/tests/domain/test_kokoro.py | 206 ++++++++++++++++++ fastapi_backend/uv.lock | 60 ++++- .../__tests__/settings-kokoro.test.tsx | 119 ++++++++++ .../__tests__/settings-voice-picker.test.tsx | 20 ++ .../__tests__/settings-web-search.test.tsx | 1 + .../components/settings/bodies.tsx | 10 +- nextjs-frontend/components/settings/model.ts | 12 + .../components/settings/voice-picker.tsx | 5 +- nextjs-frontend/openapi.json | 2 +- threat-model.md | 1 + threat-model.yaml | 2 +- 27 files changed, 823 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/kokoro.yml create mode 100644 fastapi_backend/app/domain/kokoro.py create mode 100644 fastapi_backend/tests/domain/test_kokoro.py create mode 100644 nextjs-frontend/__tests__/settings-kokoro.test.tsx diff --git a/.github/workflows/kokoro.yml b/.github/workflows/kokoro.yml new file mode 100644 index 000000000..47702b01f --- /dev/null +++ b/.github/workflows/kokoro.yml @@ -0,0 +1,80 @@ +name: Kokoro + +# Kokoro reading aloud on this machine (docs/speech-providers.md#kokoro) on x64 and arm64 (a Raspberry Pi, Docker on a +# Mac): in the backend image, Lens fetches the kokoro extra at uv.lock's versions and the pinned model and voices files +# the way a worker does, and each of the three models reads a sentence aloud in a few languages. kokoro-onnx's wheels +# (espeak-ng's library is in one) and its release files are what could move, so this runs every week too. +on: + pull_request: + paths: + - fastapi_backend/Dockerfile + - fastapi_backend/pyproject.toml + - fastapi_backend/app/domain/kokoro.py + - fastapi_backend/app/domain/components.py + - .github/workflows/kokoro.yml + push: + branches: [main] + paths: + - fastapi_backend/Dockerfile + - fastapi_backend/pyproject.toml + - fastapi_backend/app/domain/kokoro.py + - fastapi_backend/app/domain/components.py + - .github/workflows/kokoro.yml + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + kokoro: + name: Kokoro in the backend image (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - { arch: x64, runner: ubuntu-latest } + - { arch: arm64, runner: ubuntu-24.04-arm } + steps: + - uses: actions/checkout@v7 + + - name: Build the backend image + run: docker build --target lean -t lens-backend:ci fastapi_backend + + - name: Lens fetches Kokoro and reads aloud with each model + run: | + docker run --rm -i lens-backend:ci python - <<'PY' + import time + + from app.domain import components, kokoro, machine + + cfg = {"data_dir": "/tmp/lens", "voice": {"tts_provider": "kokoro"}} + m = machine.probe(cfg["data_dir"]) + c = components.BY_ID["kokoro"] + assert c.needed(cfg, m) and not c.present(cfg), "nothing should be here before the fetch" + lines = ( + ("af_heart", "Hello from Lens. Kokoro reads this on this machine, and nothing is sent anywhere."), + ("bm_george", "Good morning. The shipment leaves on Friday."), + ("ef_dora", "Buenos días. El envío sale el viernes."), + ("jf_alpha", "おはようございます。"), + ) + for size in kokoro.MODELS: + cfg["voice"]["kokoro_model"] = size + c.fetch(cfg, m, print) # the extra the first time, then the model and the voices + assert c.present(cfg), f"{size} isn't ready after the fetch" + for voice, text in lines: + t = time.monotonic() + pcm, rate = kokoro.read(cfg, text, voice) + seconds = len(pcm) / 2 / rate + print(f"{size} {voice}: {seconds:.2f} s of audio in {time.monotonic() - t:.2f} s") + assert rate == 24000 and seconds > 0.5, f"{size} {voice} read {seconds:.2f} s" + kokoro._loaded.clear() + assert len(kokoro.voices()) == 54 + PY diff --git a/CHANGELOG.md b/CHANGELOG.md index aac866543..d59cc3855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Kokoro reads aloud on this machine.** Pick **Kokoro** as the voice in Settings → AI assistant → Voice, and spoken + answers and podcast hosts are read by Kokoro v1.0 on the server's own CPU, in any of its 54 voices across nine + languages, with nothing sent anywhere and nothing billed. It's fetched once picked (the `kokoro` extra with + kokoro-onnx and ONNX Runtime, the model in `voice.kokoro_model`, full, half or 8-bit, and the voices: about 450 MB, + checked against their SHA-256); podcast episodes wait for it, and the browser reads answers until it's here. A + podcast's second host gets a voice in the first host's language. Agents pick it through the settings API and list the + voices with `GET /api/v1/voice/voices?provider=kokoro` ([Kokoro](docs/speech-providers.md#kokoro)). - **Specs from transcripts.** Turn a voice session or a meeting into a technical spec, saved as a note linked to it: the model writes the spec, then lists every named decision, every "no" and "not that, this", every constraint and every gap the speaker left to decide later, each with its quote and a link to the moment it was said; a quote the diff --git a/DEVLOG.md b/DEVLOG.md index c8cab4ee5..20b6aa15c 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,20 @@ Plans and progress for work in flight. Newest first. +## 2026-10-17 · Kokoro reading aloud on this machine + +Adeel: Kokoro v1.0's model and voices for spoken answers and podcasts, so Lens reads aloud fully on its own machine. +`kokoro.py` runs it through kokoro-onnx on ONNX Runtime, in-process (no service to start, unlike StyleTTS2), as +`voice.tts_provider` `kokoro`; components.py fetches the `kokoro` extra and the pinned model and voices files once it's +picked, and podcast jobs wait for them. The voice list is fixed (54 voices), so the picker works before the fetch. +espeak-ng and phonemizer are GPL-3.0, so the extra is opt-in only (AGENTS.md 13). Measured here on a 4-core x64 +container, one 5-second sentence: fp32 2.8 s, fp16 3.0 s, int8 4.6 s, so fp32 is the default; arm64 not measured +beyond the kokoro.yml check. + +Refine later: misaki's G2P instead of espeak-ng for English (better names and numbers, and no GPL in the path); +speaking speed (`speed` 0.5 to 2); voice blends (`af_heart:60,am_michael:40`); stream long answers sentence by +sentence; pick the hosts' voices in the podcast dialog; Japanese and Mandarin through misaki's own G2P. + ## 2026-10-17 · Dates read as days Adeel: dates extraction. `dates.py` reads the dates each line gives (written out, numeric, and relative to the day) diff --git a/README.md b/README.md index 3532f9807..ffb4d4460 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p turns them on, it also searches the web and reads pages, cited like the archive, and runs Python and bash in a sandbox of each conversation's own. Regenerating an answer or editing a question makes a new version beside the old one, and nothing is lost. Each answer keeps a record of what produced it (the model, its tools, how long it took and what it - cost), opened from its header. It reads answers aloud with a speech provider, macOS `say` or StyleTTS2 on this machine - (opt-in), and `/clear` and `/compact` keep long conversations short. Each namespace can have its own assistant with + cost), opened from its header. It reads answers aloud with a speech provider, macOS `say`, or Kokoro (54 voices, fetched + once picked) or StyleTTS2 on this machine (opt-in), and `/clear` and `/compact` keep long conversations short. Each namespace can have its own assistant with its own name, instructions and memory, answering in Telegram and Slack (built in, with voice notes and files), in Discord, Matrix, WhatsApp and other chat rooms through Matterbridge, and by email at an address of its own that you email or forward things to. ([Chat](docs/assistant.md), [web search](docs/assistant.md#web-search-and-reading-pages), @@ -132,7 +132,7 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p [cleaning up transcripts](docs/processing.md#cleaning-up-transcripts), [dates](docs/processing.md#dates); [checks](docs/requirements.md#understand-it)) - **Learn from it.** Make a two-host podcast episode about what you pick, every claim cited and checked against its - source, read aloud in two voices. Turn a voice session or a meeting into a technical spec, in a note linked to it, + source, read aloud in two voices, by Kokoro on this machine or a speech provider. Turn a voice session or a meeting into a technical spec, in a note linked to it, that keeps every decision, every no and every constraint in the speaker's own words, linked to when they said them, flags what was left to decide later and marks any quote the transcript doesn't hold. Keep notes beside every asset, entity and topic, written by you or the assistant, in a tree where notes are renamed, moved, copied and pinned (pins diff --git a/docs/assistant.md b/docs/assistant.md index 4ab1e52b0..4c9d7d6a7 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -190,8 +190,8 @@ Voice goes through one hook, `useVoice()` in `nextjs-frontend/lib/voice.ts` (`li Edge and Safari; Chrome sends the audio to its own speech service). - **Speaking.** Answers are read by a speech model when one is set (`voice.tts_model`: any OpenAI-compatible `/audio/speech`, such as Kokoro-FastAPI, LocalAI or OpenAI's `tts-1`, on `voice.tts_base_url` or the LLM provider), - or by ElevenLabs, Deepgram, Google Gemini or macOS `say` ([Speech providers](speech-providers.md#voices)); else by - the browser. + or by ElevenLabs, Deepgram, Google Gemini, macOS `say`, or Kokoro on this machine + ([Speech providers](speech-providers.md#kokoro)); else by the browser. Settings → AI assistant → Voice holds these. The API: `GET /voice` (what the server does; it also starts loading the engine), `POST /voice/transcribe` (a clip as the raw body; nothing is kept), `POST /voice/speak` (`{text, voice?}`; diff --git a/docs/components.md b/docs/components.md index 074cf8efc..e85bf3b67 100644 --- a/docs/components.md +++ b/docs/components.md @@ -22,6 +22,7 @@ never fetch a speech or voice model. | NSFW model | [NSFW classification](nsfw.md) is on for the archive or a namespace (off by default) | the 8-bit ONNX export of AdamCodd/vit-base-nsfw-detector (Apache-2.0, 88 MB), and ONNX Runtime | | Florence-2 | `llm.describe_engine` is `florence-2` | Florence-2-base-ft or -large-ft (MIT, `llm.florence_model`; about 280 or 830 MB), their 8-bit ONNX exports, the tokenizers library and ONNX Runtime, to [describe pictures](configuration.md#descriptions) on this machine | | Prompt-injection classifier | `ai.injection_check` is `classifier` | ProtectAI's deberta-v3-base-prompt-injection-v2 (Apache-2.0, about 740 MB), the tokenizers library and ONNX Runtime ([third-party content](assistant.md#third-party-content-and-prompt-injection)) | +| Kokoro | `voice.tts_provider` is `kokoro`, as soon as it's saved (podcast jobs wait for it) | Kokoro v1.0 (Apache-2.0): the model in `voice.kokoro_model` (326, 164 or 114 MB) and its 54 voices (28 MB), and the `kokoro` extra: kokoro-onnx (MIT), ONNX Runtime, and espeak-ng with phonemizer (GPL-3.0, only in this extra), about 95 MB, to [read aloud](speech-providers.md#kokoro) on this machine | | Punctuation model | `analysis.cleanup` is on and `analysis.cleanup_punctuation` isn't `off` ([Cleaning up transcripts](processing.md#cleaning-up-transcripts)) | kredor/punctuate-all at a pinned revision (MIT, about 1.1 GB), with Transformers and PyTorch's CPU build | | spaCy | `analysis.entities` is `spacy` ([Entities](processing.md#entities-and-the-graph-explorer)) | spaCy (MIT) and the model in `analysis.spacy_model`: `en_core_web_sm`, `de_core_news_sm` or `xx_ent_wiki_sm` (MIT, 11 to 15 MB each), about 60 MB in all | | DuckDuckGo search | `web.search` is `duckduckgo` | ddgs (MIT, about 30 MB with lxml and primp), the client the assistant's [web search](assistant.md#web-search-and-reading-pages) reads DuckDuckGo's results with | @@ -33,8 +34,8 @@ never fetch a speech or voice model. - Python packages go into `data_dir/python`, at the versions in `uv.lock`. Packages the image already has win. - Models go into `data_dir/models`: the Hugging Face, ModelScope and PyTorch caches point there (unless `HF_HOME`, `MODELSCOPE_CACHE` or `TORCH_HOME` say otherwise). They outlast container rebuilds and upgrades. -- Model files fetched by URL (YOLOX, YuNet, SFace, the NSFW model, Florence-2, spaCy's models) are checked against their - SHA-256. +- Model files fetched by URL (YOLOX, YuNet, SFace, the NSFW model, Florence-2, Kokoro, spaCy's models) are checked + against their SHA-256. - A job step that needs something still being fetched waits for it instead of falling back or failing. If a fetch fails, the step runs as before (for transcription, falling back to another engine), and the fetch is tried again an hour later, when the settings change, or on **Check again now**. diff --git a/docs/podcasts.md b/docs/podcasts.md index 05af4fa70..76c44e985 100644 --- a/docs/podcasts.md +++ b/docs/podcasts.md @@ -41,7 +41,7 @@ One job on the episode's asset, one step per stage, visible in Activity and on t | planning | The model outlines the episode: key ideas, learner questions, 2 to 4 connections between excerpts and a recap, each tied to excerpt numbers. | | writing | The model writes the script as lines of `{speaker, kind, text, citations}`. Citations to excerpts it wasn't given are dropped. | | fact-checking | Every claim and recap line is checked against the excerpts it cites. A partly supported line takes the checker's correction; an unsupported one is rewritten once and checked again, or dropped, as is a claim with no citation. Every change is recorded. | -| recording the voices | Each line is read aloud in its host's voice by the [text-to-speech provider](speech-providers.md) (Settings › Voice: an OpenAI-compatible server such as a local Kokoro, ElevenLabs or Deepgram), the clips are joined with short pauses, and the loudness is evened out (-16 LUFS) into one MP3. Clips are kept, so making an episode again only reads changed lines. Without text to speech this stage is skipped and the episode stays a script. | +| recording the voices | Each line is read aloud in its host's voice by the [text-to-speech provider](speech-providers.md) (Settings › Voice: Kokoro on this machine, an OpenAI-compatible server, ElevenLabs or Deepgram), the clips are joined with short pauses, and the loudness is evened out (-16 LUFS) into one MP3. Clips are kept, so making an episode again only reads changed lines. Without text to speech this stage is skipped and the episode stays a script. | | publishing | The lines become the episode's transcript, timed where each line is in the audio (or at `podcasts.words_per_minute` without audio), and the audio becomes the episode's media. | Then the usual analyze, embed and summarize steps run. Workers run podcast steps when their `workers.steps` list has @@ -77,7 +77,7 @@ change what every later episode is asked; each episode records which versions ma | `podcasts.max_minutes` | 30 | The longest episode someone can ask for | | `podcasts.words_per_minute` | 150 | Speaking pace, for the script's length and its timing | | `podcasts.model` | none | A model for the podcast's LLM calls (none: `llm.model`) | -| `podcasts.voice_a`, `podcasts.voice_b` | none | The hosts' voices (none: `voice.tts_voice` for A, and onyx, ElevenLabs' Sarah or Deepgram's aura-2-orion-en for B) | +| `podcasts.voice_a`, `podcasts.voice_b` | none | The hosts' voices (none: `voice.tts_voice` for A, and onyx, ElevenLabs' Sarah, Deepgram's aura-2-orion-en or, with Kokoro, a voice in A's language such as am_michael for B) | ## Privacy and cost diff --git a/docs/requirements.md b/docs/requirements.md index b8552cd7e..0c9949351 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -83,6 +83,7 @@ request, to another check that proves the same thing; loosening a requirement or | Regenerating an answer or editing a question makes a new version beside the old one | [test_chat_versions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_versions.py) `test_regenerate_edit_switch_and_go_on`; [chat-versions.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-versions.test.tsx) `regenerating an answer`, `the version switch` | | Each answer keeps a record of what produced it: the model, its tools, how long it took and what it cost | [test_chat_records.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_records.py) `test_an_answer_records_what_produced_it`, `test_a_regenerated_answer_has_its_own_record`; [chat-details.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-details.test.tsx) `says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended` | | Answers read aloud by a speech provider, macOS `say` or StyleTTS2 (opt-in) | [test_voice.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice.py) `test_answers_are_read_aloud_by_a_speech_model`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_macos_say_reads_aloud_on_a_mac`, `test_styletts2_reads_aloud_from_its_own_service`; [styletts2.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/styletts2.yml) `Read a sentence aloud` | +| Answers and podcasts read aloud by Kokoro v1.0 on this machine, in 54 voices, fetched once it's picked and checked against its SHA-256 (opt-in) | [test_kokoro.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_kokoro.py) `test_kokoro_reads_answers_aloud_on_this_machine`, `test_the_model_size_is_a_setting`, `test_kokoro_is_fetched_once_it_is_picked`, `test_every_voice_is_listed_before_it_is_fetched`, `test_a_podcast_is_read_by_kokoro`; [settings-kokoro.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/settings-kokoro.test.tsx) `reads on this machine, with the model size to pick`; [kokoro.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/kokoro.yml) `Lens fetches Kokoro and reads aloud with each model` | | `/clear` and `/compact` | [test_compaction.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_compaction.py) `test_clear_starts_afresh_and_compact_folds_everything_now`, `test_commands_in_the_app` | | A namespace's own assistant, with its own name, instructions and memory | [test_ns_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_ns_assistant.py) `test_off_by_default_and_owners_turn_it_on`, `test_remembers_across_conversations_with_citations`; [ns-assistant.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/ns-assistant.test.tsx) `saving a name and forgetting a memory` | | Telegram and Slack built in, with voice notes and files | [test_telegram.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_telegram.py) `test_it_answers_private_chats_and_what_is_said_to_it_in_groups`, `test_voice_notes_are_heard`, `test_files_are_held_for_the_assistant_to_import`; [test_slack.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_slack.py) `test_it_answers_direct_messages_mentions_and_the_slash_command`, `test_voice_clips_are_heard_and_files_held` | diff --git a/docs/speech-providers.md b/docs/speech-providers.md index 6a0e039fb..8928fe457 100644 --- a/docs/speech-providers.md +++ b/docs/speech-providers.md @@ -17,6 +17,7 @@ summaries, chapters and entities as optional imports, per-minute prices in the a | Deepgram | Nova | yes | detected | from sentiment (English) | no | Aura | | Google Gemini | no | no | no | no | no | `gemini-2.5-flash-preview-tts`, 30 prebuilt voices | | macOS `say` | no | no | no | no | no | the Mac's own voices; no key, nothing leaves the Mac | +| Kokoro | no | no | no | no | no | on this machine, in 54 voices and nine languages; fetched once you pick it ([below](#kokoro)) | | StyleTTS2 | no | no | no | no | no | on this machine, in a voice like a clip you give it ([below](#styletts2)) | Sentiment becomes emotion as positive → Happy, negative → Sad, neutral → Neutral (turn it off with @@ -38,7 +39,8 @@ Then pick where each task goes: - **Speaker separation → Method**: *Speech provider* uses the speakers the provider found; *Auto* uses them when a transcript has some, and otherwise works as before. Voice IDs across recordings still come from local voiceprints. - **AI assistant → Voice**: *Engine that hears it* (default: the transcription engine) and *Spoken answers by* (an - OpenAI-compatible speech server, ElevenLabs, Deepgram Aura, Google Gemini or macOS `say`) and *Voice*. + OpenAI-compatible speech server, ElevenLabs, Deepgram Aura, Google Gemini, macOS `say`, Kokoro or StyleTTS2) and + *Voice*. ## Long recordings @@ -74,8 +76,8 @@ How a recording is cut: `GET /voice/voices?provider=…` lists the voices a provider reads in, as `{id, name, detail}`: ElevenLabs lists the voices in your account (premade, cloned and from the library), Deepgram its Aura models, `say` the voices installed on the Mac (System Settings → Accessibility → Spoken Content adds more), and an OpenAI-compatible server its own when it -has `/audio/voices` (Kokoro-FastAPI does); otherwise OpenAI's eleven. Gemini's are a fixed list. The voice picked is -`voice.tts_voice`; podcast hosts use it too (`podcasts.voice_a`, `voice_b`). `POST /voice/speak` takes `{text, +has `/audio/voices` (Kokoro-FastAPI does); otherwise OpenAI's eleven. Gemini's and Kokoro's are fixed lists. The voice +picked is `voice.tts_voice`; podcast hosts use it too (`podcasts.voice_a`, `voice_b`). `POST /voice/speak` takes `{text, voice}` to try a voice before keeping it. `say` only works when Lens runs on the Mac itself (`make start-backend`, not Docker): a container has no `say`. It @@ -93,9 +95,36 @@ every Mac. | `speech.timeout` | 1800 seconds | | `voice.stt` | `same` | | `voice.tts_provider` | `openai` | -| `voice.tts_voice` | the provider's default (alloy, a premade ElevenLabs voice, aura-2-thalia-en, Kore, the Mac's, `default`) | +| `voice.tts_voice` | the provider's default (alloy, a premade ElevenLabs voice, aura-2-thalia-en, Kore, the Mac's, af_heart, `default`) | +| `voice.kokoro_model` | `fp32` | | `voice.styletts2_url` | `http://styletts2:8880` | +## Kokoro + +[Kokoro](https://huggingface.co/hexgrad/Kokoro-82M) v1.0 (Apache-2.0) reads spoken answers and podcasts on your own +machine: nothing leaves it, nothing is billed, and there is no service to start. Pick **Kokoro** in Settings → AI +assistant → Voice and save; a worker then fetches it in the background (Settings → Components shows how far it got), +and until it's here the browser reads answers. Podcast episodes wait for it rather than coming out as scripts. + +What is fetched, once, into the data folder: + +- the `kokoro` extra: [kokoro-onnx](https://github.com/thewh1teagle/kokoro-onnx) (MIT) and ONNX Runtime, about 95 MB. + kokoro-onnx turns text into phonemes with espeak-ng through phonemizer, which are GPL-3.0, so they are in this + opt-in extra only, never in Lens's own install; +- the model, `voice.kokoro_model`: `fp32` (326 MB, the default: the best, and on most CPUs the fastest), `fp16` + (164 MB) or `int8` (114 MB, for a small disk); +- `voices-v1.0.bin` (28 MB): all 54 voices. + +The model and voices files are the ones kokoro-onnx publishes (its `model-files-v1.1` release), checked against their +SHA-256. The model is loaded the first time something is read aloud and let go after ten minutes of quiet, so a small +machine gets its memory back. Each read is a row in the activity ledger (`model.speech`, `kokoro:kokoro-v1.0`). + +A voice's first letter is its language, and its second whether it's a woman's (`f`) or a man's (`m`) voice: `af_heart` +(the default) is American English, female. The languages are American and British English (`a`, `b`), Spanish (`e`), +French (`f`), Hindi (`h`), Italian (`i`), Japanese (`j`), Brazilian Portuguese (`p`) and Mandarin (`z`); espeak-ng reads +the text as the voice's language, so pick a voice that speaks the language of what it reads. A podcast's second host +gets a voice in the first host's language (`am_michael` beside `af_heart`) unless `podcasts.voice_b` names one. + ## StyleTTS2 [StyleTTS2](https://github.com/yl4579/StyleTTS2) reads aloud on your own machine: nothing leaves it, and nothing is diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index 763f2b8ce..9b94a78ab 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -94,6 +94,16 @@ 89, ), } +# Kokoro v1.0 (hexgrad's Kokoro-82M, Apache-2.0) as kokoro-onnx exports it, from its model-files-v1.1 release +# (kokoro.py): the model at three precisions, and every voice: {file: (sha256, size in MB)} +KOKORO = { + "kokoro-v1.0.onnx": ("beb0d1848dee9a49da392cc3df26958d46cfa35d321edf434f52949153f0df3a", 326), + "kokoro-v1.0.fp16.onnx": ("f3a290d384fbb27966d462905c71a46cef9e5fd00516b40df32a0b4afe77ac96", 164), + "kokoro-v1.0.int8.onnx": ("ae315a79b623f244700e4afb9246c46a26066782e049ba174bf3ba433970ee9c", 114), + "voices-v1.0.bin": ("bca610b8308e8d99f32e6fe4197e7ec01679264efed0cac9140fe9c29f1fbf7d", 28), +} +for _name, (_sha, _mb) in KOKORO.items(): + FILES[_name] = (f"https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.1/{_name}", _sha, _mb) # Florence-2-base-ft and -large-ft (MIT), the 8-bit ONNX exports at onnx-community, at fixed revisions (florence.py): # {size: (revision, {file: (sha256, size in MB)})} FLORENCE = { @@ -564,6 +574,38 @@ def fetch(self, cfg, m, say): fetch_file(cfg, n, say) +class Kokoro(Component): + """kokoro-onnx, the Kokoro model and its voices (kokoro.py), when Settings → AI assistant → Voice has Kokoro read + answers and podcasts aloud. Fetched as soon as it's picked, since spoken answers aren't jobs; podcasts (its steps) + wait while it's being fetched.""" + + kind = "model" + + def needed(self, cfg, m): + return (cfg.get("voice") or {}).get("tts_provider") == "kokoro" + + def serves(self, cfg, m): + return set() + + def present(self, cfg): + from . import kokoro + + return kokoro.ready(cfg) + + def fetch(self, cfg, m, say): + from . import kokoro + + if not importable("kokoro_onnx"): + pip_install(cfg, "kokoro", say) + for n in kokoro.files(cfg): + fetch_file(cfg, n, say) + + def detail(self, cfg): + from . import kokoro + + return kokoro.name(cfg) + + SPACY_MODELS = {"en_core_web_sm": "en_core_web_sm-3.8.0-py3-none-any.whl", "de_core_news_sm": "de_core_news_sm-3.8.0-py3-none-any.whl", "xx_ent_wiki_sm": "xx_ent_wiki_sm-3.8.0-py3-none-any.whl"} # fmt: skip @@ -920,6 +962,14 @@ def detail(self, cfg): size_mb=280, license="MIT", ), + Kokoro( + "kokoro", + "Kokoro", + "reads answers and podcasts aloud on this machine, in 54 voices (voice.tts_provider kokoro; about 450 MB, 240 MB with the int8 model)", + steps={"podcast"}, + size_mb=450, + license="Apache-2.0 (model), MIT (kokoro-onnx), GPL-3.0 (espeak-ng, phonemizer)", + ), LAYA, Spacy( "spacy", diff --git a/fastapi_backend/app/domain/kokoro.py b/fastapi_backend/app/domain/kokoro.py new file mode 100644 index 000000000..3570e2c4a --- /dev/null +++ b/fastapi_backend/app/domain/kokoro.py @@ -0,0 +1,148 @@ +"""Kokoro reads aloud on this machine (docs/speech-providers.md#kokoro): with voice.tts_provider "kokoro", spoken +answers and podcast hosts are read by Kokoro v1.0 (hexgrad's Kokoro-82M, Apache-2.0) on ONNX Runtime. No text leaves +the machine, nothing is billed, and there's no service to start. + +It runs through kokoro-onnx (MIT): espeak-ng turns the text into phonemes, and the model reads them at 24 kHz in one of +its 54 voices. espeak-ng and phonemizer, which kokoro-onnx calls it through, are GPL-3.0, so all of it is the `kokoro` +extra: components.py fetches it, with the model and its voices, only once Kokoro is picked. The files are the ones +kokoro-onnx publishes (its model-files-v1.1 release), checked against their SHA-256: the model, voice.kokoro_model +(fp32, 326 MB, the best and on most CPUs the fastest; fp16, 164 MB; int8, 114 MB, for a small disk), and +voices-v1.0.bin (28 MB), every voice. + +A voice's first letter is its language and its second whether it's a woman's or a man's (af_heart: American English, +female); espeak-ng reads the text as that language. The model stays loaded while it's reading and is let go when +nothing has been read aloud for IDLE_SECONDS, so a small machine gets its memory back. +""" + +from __future__ import annotations + +import threading +import time + +from . import components + +MODELS = {"fp32": "kokoro-v1.0.onnx", "fp16": "kokoro-v1.0.fp16.onnx", "int8": "kokoro-v1.0.int8.onnx"} +VOICES_FILE = "voices-v1.0.bin" +DEFAULT_VOICE = "af_heart" +# the voice a podcast's second host gets, by language and sex, when there's more than one to pick from +PARTNERS = {"af": "af_heart", "am": "am_michael", "bf": "bf_emma", "bm": "bm_george"} +IDLE_SECONDS = 600 +# a voice's first letter: the language espeak-ng reads its text as, and what it's called +LANGS = { + "a": ("en-us", "American English"), + "b": ("en-gb", "British English"), + "e": ("es", "Spanish"), + "f": ("fr-fr", "French"), + "h": ("hi", "Hindi"), + "i": ("it", "Italian"), + "j": ("ja", "Japanese"), + "p": ("pt-br", "Brazilian Portuguese"), + "z": ("cmn", "Mandarin Chinese"), +} +# the voices in voices-v1.0.bin +VOICES = ( + "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", + "af_sarah", "af_sky", "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", "am_michael", "am_onyx", "am_puck", + "am_santa", "bf_alice", "bf_emma", "bf_isabella", "bf_lily", "bm_daniel", "bm_fable", "bm_george", "bm_lewis", + "ef_dora", "em_alex", "em_santa", "ff_siwis", "hf_alpha", "hf_beta", "hm_omega", "hm_psi", "if_sara", "im_nicola", + "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", "pf_dora", "pm_alex", "pm_santa", "zf_xiaobei", + "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", +) # fmt: skip + +_lock = threading.Lock() +_loaded: dict = {} + + +class Missing(RuntimeError): + """Kokoro isn't here yet: it's fetched once it's picked.""" + + +def size(cfg): + s = (cfg.get("voice") or {}).get("kokoro_model") or "fp32" + return s if s in MODELS else "fp32" + + +def name(cfg): + """The model, as the activity ledger names it.""" + s = size(cfg) + return "kokoro-v1.0" if s == "fp32" else f"kokoro-v1.0-{s}" + + +def files(cfg): + """Its files, as components.FILES names them: the model, then the voices.""" + return [MODELS[size(cfg)], VOICES_FILE] + + +def ready(cfg): + """Whether kokoro-onnx, the chosen model and the voices are here.""" + components.activate(cfg) + return components.importable("kokoro_onnx") and all(components.model_file(cfg, n) for n in files(cfg)) + + +def voices(): + """Every voice as (id, name, detail): af_heart is ("af_heart", "Heart", "American English, female").""" + return [(v, v[3:].title(), f"{LANGS[v[0]][1]}, {'female' if v[1] == 'f' else 'male'}") for v in VOICES] + + +def partner(voice): + """A voice for a podcast's second host: the same language, a man's for a woman's and a woman's for a man's when + there is one (am_michael for af_heart), else another of the same kind.""" + voice = voice if voice in VOICES else DEFAULT_VOICE + for kind in (voice[0] + ("m" if voice[1] == "f" else "f"), voice[:2]): + others = [v for v in VOICES if v[:2] == kind and v != voice] + if others: + return PARTNERS.get(kind) if PARTNERS.get(kind) in others else others[0] + return voice + + +def lang(voice): + """The language espeak-ng reads a voice's text as (American English for a voice it doesn't know).""" + return LANGS.get((voice or "")[:1], LANGS["a"])[0] + + +def _later(): + t = threading.Timer(IDLE_SECONDS + 5, let_go) + t.daemon = True + t.start() + return t + + +def let_go(): + """Free the model's memory when nothing has been read aloud for a while.""" + with _lock: + if not _loaded: + return + if time.time() - _loaded["used"] > IDLE_SECONDS: + _loaded.clear() + else: + _loaded["timer"] = _later() + + +def _load(cfg): + if not ready(cfg): + raise Missing("Kokoro isn't here yet: it's fetched once it's picked (Settings → Components shows how far it got)") + model, voices_path = (components.model_file(cfg, n) for n in files(cfg)) + with _lock: # one per process: loading the model takes longer than reading a sentence + if _loaded.get("model") != model: + import onnxruntime + from kokoro_onnx import Kokoro + + _loaded.clear() # another size's model goes: one is enough memory + opts = onnxruntime.SessionOptions() + opts.log_severity_level = 3 # the fp16 model warns about every node it can't fold + session = onnxruntime.InferenceSession(model, opts, providers=["CPUExecutionProvider"]) + _loaded.update(model=model, kokoro=Kokoro.from_session(session, voices_path)) + _loaded["used"] = time.time() + if not _loaded.get("timer"): + _loaded["timer"] = _later() + return _loaded["kokoro"] + + +def read(cfg, text, voice=None): + """The text read aloud in `voice` (else af_heart) as 16-bit PCM and its sample rate. Raises Missing when Kokoro + isn't here yet, ValueError for a voice it doesn't have or text it can't read.""" + import numpy as np + + voice = voice or DEFAULT_VOICE + samples, rate = _load(cfg).create(text, voice=voice, lang=lang(voice)) + return (np.clip(samples, -1.0, 1.0) * 32767).astype("=0.1.4,<0.2"] # the assistant's web search through DuckDuckGo (web.py): ddgs (MIT), with its Rust HTTP client (primp) and lxml; # fetched when DuckDuckGo is picked in Settings › AI assistant › Web search web = ["ddgs>=9.16,<10"] +# Kokoro v1.0 reading answers and podcasts aloud on this machine (kokoro.py, voice.tts_provider "kokoro"): kokoro-onnx +# (MIT) on ONNX Runtime. It turns text into phonemes with espeak-ng through phonemizer, both GPL-3.0, so it's fetched +# only once Kokoro is picked in Settings › AI assistant › Voice +kokoro = ["kokoro-onnx>=0.6.1,<0.7"] # Outlook .msg emails (extract-msg is GPL-3.0; .eml needs nothing) msg = ["extract-msg"] @@ -96,6 +100,7 @@ dev = [ "impacket>=0.13", # the SMB share's server, and the SMB client its tests sign in with "openshell>=0.1.4,<0.2", # the code sandbox's client, tested against a fake OpenShell gateway "botocore>=1.43.113", + "kokoro-onnx>=0.6.1,<0.7", # Kokoro's tests phonemize with espeak-ng and read with a tiny Kokoro-shaped model ] [tool.hatch.build.targets.wheel] diff --git a/fastapi_backend/tests/domain/test_kokoro.py b/fastapi_backend/tests/domain/test_kokoro.py new file mode 100644 index 000000000..933db74ae --- /dev/null +++ b/fastapi_backend/tests/domain/test_kokoro.py @@ -0,0 +1,206 @@ +"""Kokoro reading aloud on this machine (kokoro.py): espeak-ng turns the text into phonemes and a tiny model shaped like +Kokoro v1.0 (its token, style and speed inputs, a waveform out) reads them, so the tests run kokoro-onnx itself without +fetching the real model. The real one is read with on x64 and arm64 by .github/workflows/kokoro.yml.""" + +from __future__ import annotations + +import io +import re +import wave + +import numpy as np +import pytest + +from app.domain import activity, components, jobs, kokoro, machine, podcasts, settings, speech, store, voice +from tests import fake_llm +from tests.helpers import drain, make_user, seed + +pytest.importorskip("kokoro_onnx") +onnx = pytest.importorskip("onnx") + +R = store.R +SAMPLES_PER_TOKEN = 240 # 10 ms of audio for each phoneme + + +def tiny_model(path): + """A model with Kokoro v1.0's inputs: its waveform is the phonemes' token ids, 10 ms each, times the speed.""" + from onnx import TensorProto, helper + + graph = helper.make_graph( + [ + helper.make_node("Cast", ["input_ids"], ["ids"], to=TensorProto.FLOAT), + helper.make_node("Mul", ["ids", "scale"], ["level"]), + helper.make_node("ReduceSum", ["style"], ["styled"], keepdims=0), + helper.make_node("Mul", ["styled", "zero"], ["nothing"]), + helper.make_node("Add", ["level", "nothing"], ["heard"]), + helper.make_node("Tile", ["heard", "repeats"], ["long"]), + helper.make_node("Mul", ["long", "speed"], ["fast"]), + helper.make_node("Reshape", ["fast", "flat"], ["waveform"]), + ], + "tiny-kokoro", + [ + helper.make_tensor_value_info("input_ids", TensorProto.INT64, [1, None]), + helper.make_tensor_value_info("style", TensorProto.FLOAT, [1, 256]), + helper.make_tensor_value_info("speed", TensorProto.FLOAT, [1]), + ], + [helper.make_tensor_value_info("waveform", TensorProto.FLOAT, [None])], + [ + helper.make_tensor("scale", TensorProto.FLOAT, [], [0.004]), + helper.make_tensor("zero", TensorProto.FLOAT, [], [0.0]), + helper.make_tensor("repeats", TensorProto.INT64, [2], [1, SAMPLES_PER_TOKEN]), + helper.make_tensor("flat", TensorProto.INT64, [1], [-1]), + ], + ) + m = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + m.ir_version = 8 + onnx.save(m, str(path)) + + +@pytest.fixture +def here(cfg): + """Kokoro fetched: the tiny model at each precision, and a voices file with a few voices.""" + d = components.models_dir(cfg) + d.mkdir(parents=True, exist_ok=True) + for f in kokoro.MODELS.values(): + tiny_model(d / f) + rng = np.random.default_rng(7) + with open(d / kokoro.VOICES_FILE, "wb") as f: # an .npz, as voices-v1.0.bin is + np.savez(f, **{v: rng.random((510, 1, 256), dtype=np.float32) for v in ("af_heart", "am_michael", "bf_emma", "ef_dora")}) + kokoro._loaded.clear() + yield cfg + kokoro._loaded.clear() + + +def picked(db, cfg, **more): + settings.save(db, cfg, "voice", {"tts_provider": "kokoro", **more}, "t") + return settings.effective(db, cfg) + + +def test_kokoro_reads_answers_aloud_on_this_machine(db, here): + eff = picked(db, here) + assert voice.can_speak(eff) and speech.tts_voice(eff) == "af_heart" and eff["voice"]["kokoro_model"] == "fp32" + with activity.scope(db, "chat:1", cfg=eff): + data, ctype = voice.speak(eff, "The shipment leaves on Friday.") + assert ctype == "audio/wav" and data[:4] == b"RIFF" + with wave.open(io.BytesIO(data)) as w: + assert (w.getframerate(), w.getnchannels(), w.getsampwidth()) == (24000, 1, 2) + assert w.getnframes() > 10 * SAMPLES_PER_TOKEN # the phonemes espeak-ng found, read + # in the activity ledger, as the model it was read with + rows = [r for r in db.rows("SELECT * FROM activity") if r["action"] == "model.speech"] + assert [(r["model"], r["resources"]) for r in rows] == [("kokoro:kokoro-v1.0", ["chat:1"])] + # another voice; each reads in its own language + assert voice.speak(eff, "Good morning.", voice="bf_emma")[0][:4] == b"RIFF" + assert (kokoro.lang("bf_emma"), kokoro.lang("ef_dora"), kokoro.lang("zf_xiaobei"), kokoro.lang("nope")) == ( + "en-gb", + "es", + "cmn", + "en-us", + ) + # a voice it doesn't have: the browser reads it + with pytest.raises(speech.ProviderError, match="Kokoro couldn't read it: Voice am_nobody not found"): + speech.speak(eff, "Hello", voice="am_nobody") + assert voice.speak(eff, "Hello", voice="am_nobody") is None + + +def test_the_model_size_is_a_setting(db, here): + eff = picked(db, here, kokoro_model="int8") + assert kokoro.files(eff) == ["kokoro-v1.0.int8.onnx", "voices-v1.0.bin"] and kokoro.name(eff) == "kokoro-v1.0-int8" + with activity.scope(db, "chat:1", cfg=eff): + voice.speak(eff, "Hello") + assert kokoro._loaded["model"].endswith("kokoro-v1.0.int8.onnx") + rows = [r for r in db.rows("SELECT * FROM activity") if r["action"] == "model.speech"] + assert [r["model"] for r in rows] == ["kokoro:kokoro-v1.0-int8"] + voice.speak(picked(db, here, kokoro_model="fp16"), "Hello") # the other one is let go + assert kokoro._loaded["model"].endswith("kokoro-v1.0.fp16.onnx") + with pytest.raises(ValueError): + settings.save(db, here, "voice", {"kokoro_model": "fp64"}, "t") + + +def test_the_model_is_let_go_when_nothing_is_read(db, here, monkeypatch): + eff = picked(db, here) + voice.speak(eff, "Hello") + assert kokoro._loaded.get("kokoro") is not None + kokoro.let_go() # just used: kept, and looked at again later + assert kokoro._loaded.get("kokoro") is not None and kokoro._loaded["timer"].is_alive() + kokoro._loaded["timer"].cancel() + monkeypatch.setattr(kokoro, "IDLE_SECONDS", -1) + kokoro.let_go() + assert kokoro._loaded == {} + + +def test_every_voice_is_listed_before_it_is_fetched(db, cfg): + eff = picked(db, cfg) + assert not speech.tts_ready(eff) and voice.speak(eff, "Hello") is None # not fetched yet: the browser reads it + found, note = speech.voices(eff) + assert len(found) == 54 and note.startswith("Kokoro is fetched once") + assert found[0] == {"id": "af_alloy", "name": "Alloy", "detail": "American English, female"} + assert {"id": "bm_george", "name": "George", "detail": "British English, male"} in found + with pytest.raises(speech.ProviderError, match="Kokoro isn't here yet"): + speech.speak(eff, "Hello") + + +def test_kokoro_is_fetched_once_it_is_picked(db, cfg, monkeypatch): + c = components.BY_ID["kokoro"] + m = machine.probe(cfg["data_dir"]) + assert not c.needed(settings.effective(db, cfg), m) # nothing picked it: nothing fetched + eff = picked(db, cfg, kokoro_model="int8") + assert c.needed(eff, m) and not c.present(eff) + # fetched straight away, not when a job first needs it (spoken answers aren't jobs); podcasts wait for it + keeper = components.Keeper(db, lambda: eff, "w", {"podcast"}) + assert keeper._first_use(eff, c, m, set()) == set() and c.steps == {"podcast"} + got, extras = [], [] + monkeypatch.setattr(components, "fetch_file", lambda cfg_, n, say=None: got.append(n)) + monkeypatch.setattr(components, "pip_install", lambda cfg_, extra, say=None: extras.append(extra)) + monkeypatch.setattr(components, "importable", lambda *mods: False) + c.fetch(eff, m, None) + assert extras == ["kokoro"] and got == ["kokoro-v1.0.int8.onnx", "voices-v1.0.bin"] + # every file is pinned to kokoro-onnx's release and its SHA-256 + for n in (*kokoro.MODELS.values(), kokoro.VOICES_FILE): + url, sha, mb = components.FILES[n] + assert url == f"https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.1/{n}" + assert re.fullmatch(r"[0-9a-f]{64}", sha) and mb > 0 + row = next(r for r in components.catalog(eff, db) if r["id"] == "kokoro") + assert row["needed"] and "GPL-3.0" in row["license"] and "Apache-2.0" in row["license"] + + +def test_podcast_hosts_get_two_voices_in_one_language(cfg): + cfg["voice"]["tts_provider"] = "kokoro" + assert podcasts.voices(cfg) == {"a": "af_heart", "b": "am_michael"} + cfg["podcasts"]["voice_a"] = "ef_dora" + assert podcasts.voices(cfg) == {"a": "ef_dora", "b": "em_alex"} # Spanish hosts + assert podcasts.voices(cfg, {"voices": {"a": "bm_george"}}) == {"a": "bm_george", "b": "bf_emma"} + cfg["podcasts"]["voice_b"] = "jf_alpha" + assert podcasts.voices(cfg)["b"] == "jf_alpha" # named: kept + assert kokoro.partner("ff_siwis") == "ff_siwis" # the only French voice + assert kokoro.partner("hf_beta") == "hm_omega" and kokoro.partner("nope") == "am_michael" + + +@pytest.fixture +def llm(cfg): + srv, url = fake_llm.start() + cfg["llm"].update(base_url=url, model="fake") + fake_llm.Handler.podcast_script = None + yield fake_llm.Handler + srv.shutdown() + + +def test_a_podcast_is_read_by_kokoro(db, here, folder, llm): + from app.domain import auth + + cfg = here + cfg["voice"].update(tts_provider="kokoro") + a, _b, _c = seed(db, cfg, folder) + uid = make_user(db, "admin@x.io", "admin password 1", admin=True) + out = podcasts.create(db, cfg, {"recordings": [a]}, roles=auth.roles(db, {"id": uid, "admin": True}), admin=True) + drain(db, cfg) + assert jobs.get(db, out["job"])["status"] == "succeeded" + ep = podcasts.view(db, out["episode"]) + assert ep["status"] == "ready" and ep["voicing"]["voices"] == {"a": "af_heart", "b": "am_michael"} + rec = db.one("SELECT source, duration_ms FROM $r", r=R("recording", out["episode"])) + assert rec["source"] == "audio" and rec["duration_ms"] > 0 + # not fetched: the episode stays a script + for f in components.models_dir(cfg).iterdir(): + f.unlink() + podcasts.regenerate(db, out["episode"]) + drain(db, cfg) + assert podcasts.view(db, out["episode"])["status"] == "script_only" diff --git a/fastapi_backend/uv.lock b/fastapi_backend/uv.lock index acf40f4c1..4b289a275 100644 --- a/fastapi_backend/uv.lock +++ b/fastapi_backend/uv.lock @@ -951,6 +951,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, ] +[[package]] +name = "dlinfo" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/8e/8f2f94cd40af1b51e8e371a83b385d622170d42f98776441a6118f4dd682/dlinfo-2.0.0.tar.gz", hash = "sha256:88a2bc04f51d01bc604cdc9eb1c3cc0bde89057532ca6a3e71a41f6235433e17", size = 12727, upload-time = "2025-01-16T15:43:10.756Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/90/022c79d6e5e6f843268c10b84d4a021ee3afba0621d3c176d3ff2024bfc8/dlinfo-2.0.0-py3-none-any.whl", hash = "sha256:b32cc18e3ea67c0ca9ca409e5b41eed863bd1363dbc9dd3de90fedf11b61e7bc", size = 3654, upload-time = "2025-01-16T15:43:09.474Z" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -1000,6 +1009,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "espeakng-loader" +version = "0.2.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/92/f44ed7f531143c3c6c97d56e2b0f9be8728dc05e18b96d46eb539230ed46/espeakng_loader-0.2.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b77477ae2ddf62a748e04e49714eabb2f3a24f344166200b00539083bd669904", size = 9938387, upload-time = "2025-01-17T01:22:42.064Z" }, + { url = "https://files.pythonhosted.org/packages/a8/26/258c0cd43b9bc1043301c5f61767d6a6c3b679df82790c9cb43a3277b865/espeakng_loader-0.2.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d27cdca31112226e7299d8562e889d3e38a1e48055c9ee381b45d669072ee59f", size = 9892565, upload-time = "2025-01-17T01:22:40.365Z" }, + { url = "https://files.pythonhosted.org/packages/de/1e/25ec5ab07528c0fbb215a61800a38eca05c8a99445515a02d7fa5debcb32/espeakng_loader-0.2.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08721baf27d13d461f6be6eed9a65277e70d68234ff484fd8b9897b222cdcb6d", size = 10078484, upload-time = "2025-01-17T01:22:43.373Z" }, + { url = "https://files.pythonhosted.org/packages/d9/ad/1b768d8daffc2996e07bbcb6f534d8de3202cd75fce1f1c45eced1ce6465/espeakng_loader-0.2.4-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:d1e798141b46a050cdb75fcf3c17db969bb2c40394f3f4a48910655d547508b9", size = 10037736, upload-time = "2025-01-17T01:22:42.576Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ed/a3d872fbad4f3a3f3db0e8c31768ab14e77cd77306de16b8b20b1e1df7ea/espeakng_loader-0.2.4-py3-none-win_amd64.whl", hash = "sha256:41f1e08ac9deda2efd1ea9de0b81dab9f5ae3c4b24284f76533d0a7b1dd7abd7", size = 9437292, upload-time = "2025-01-17T01:23:27.463Z" }, + { url = "https://files.pythonhosted.org/packages/29/64/0b75bc50ec53b4e000bac913625511215aa96124adf5dba8c4baa17c02cd/espeakng_loader-0.2.4-py3-none-win_arm64.whl", hash = "sha256:d7a2928843eaeb2df82f99a370f44e8a630f59b02f9b0d1f168a03c4eeb76b89", size = 9426841, upload-time = "2025-01-17T01:23:21.766Z" }, +] + [[package]] name = "execnet" version = "2.1.2" @@ -1850,6 +1872,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/c1/69f00d627949580e43d57af0aa465df46868d7c29801c137a55374101294/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:a5a00665d1a0e26763a7338d7e911d4598fbc1d50dd0d6b7919b7dc6c5d6569f", size = 73362, upload-time = "2026-08-28T10:28:12.449Z" }, ] +[[package]] +name = "kokoro-onnx" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "espeakng-loader" }, + { name = "numpy" }, + { name = "onnxruntime" }, + { name = "phonemizer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/ef/b58dedba0a1417f16352352787fbcfcbaa0b907e284c2ea3ebaf5541109a/kokoro_onnx-0.6.1.tar.gz", hash = "sha256:7bbdb66dd53775f71088a99999a9aefdad240740daf54cb09410d8bb1e294e35", size = 69799, upload-time = "2026-08-19T00:40:37.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/e1/a27e5a70a525a5ee1fd5357596f07b724d02ff317f134e86cb6e3d9db968/kokoro_onnx-0.6.1-py3-none-any.whl", hash = "sha256:50c8de4950d601df41428ee5462a48c8a78bef441bf671f2492e070ef44d8a32", size = 26084, upload-time = "2026-08-19T00:40:35.746Z" }, +] + [[package]] name = "langdetect" version = "1.0.9" @@ -1966,6 +2003,9 @@ insightface = [ { name = "insightface" }, { name = "onnxruntime" }, ] +kokoro = [ + { name = "kokoro-onnx" }, +] laya = [ { name = "laya-mlx" }, ] @@ -2029,6 +2069,7 @@ dev = [ { name = "httpx" }, { name = "impacket" }, { name = "jsonschema" }, + { name = "kokoro-onnx" }, { name = "mcp" }, { name = "mkdocs-material", extra = ["imaging"] }, { name = "mypy" }, @@ -2061,6 +2102,7 @@ requires-dist = [ { name = "insightface", marker = "extra == 'insightface'" }, { name = "jinja2", specifier = ">=3.1" }, { name = "jsonschema", marker = "extra == 'validate'" }, + { name = "kokoro-onnx", marker = "extra == 'kokoro'", specifier = ">=0.6.1,<0.7" }, { name = "laya-mlx", marker = "extra == 'laya'", specifier = ">=0.4.0" }, { name = "markdown", specifier = ">=3.11.1" }, { name = "mlx-whisper", marker = "extra == 'mlx'" }, @@ -2100,7 +2142,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'punctuation'" }, { name = "webauthn", specifier = ">=3.0.1,<4" }, ] -provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "validate", "smb", "sandbox", "web", "msg"] +provides-extras = ["sensevoice", "whisper", "mlx", "laya", "voices", "pyannote", "faces", "mac-ocr", "rapidocr", "doctr", "insightface", "objects", "punctuation", "spacy", "injection", "florence", "validate", "smb", "sandbox", "web", "kokoro", "msg"] [package.metadata.requires-dev] dev = [ @@ -2110,6 +2152,7 @@ dev = [ { name = "httpx", specifier = ">=0.27" }, { name = "impacket", specifier = ">=0.13" }, { name = "jsonschema", specifier = ">=4.23" }, + { name = "kokoro-onnx", specifier = ">=0.6.1,<0.7" }, { name = "mcp", specifier = ">=2.3.0" }, { name = "mkdocs-material", specifier = ">=9.6.9" }, { name = "mkdocs-material", extras = ["imaging"], specifier = ">=9.7.7" }, @@ -3432,6 +3475,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/72/b380fb5c89d89c3afafac8cf02a71a45f4f4a4f35531ca949a34683962d1/pcodedmp-1.2.6-py2.py3-none-any.whl", hash = "sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278", size = 30939, upload-time = "2019-07-30T18:05:40.483Z" }, ] +[[package]] +name = "phonemizer" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "dlinfo" }, + { name = "joblib" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/7d/5a96ddb130552f6365a090b5fd12ace803a95a858e3f67258f2ad13dc51f/phonemizer-3.4.0.tar.gz", hash = "sha256:e13231980c50bc671ec0466379ba027260ad9d61929952d8ae9665b3d0f251eb", size = 93535, upload-time = "2026-07-31T15:53:17.329Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/41/09bafd0ccf4f4ddd1530c9f9d10e75a24056e10e120b9daa97f691fcabc3/phonemizer-3.4.0-py3-none-any.whl", hash = "sha256:5ff1215d0efa3606dd1b92449f6d71b5c1741efcc84c6d40c17bfaf64f6ded5f", size = 108529, upload-time = "2026-07-31T15:53:16.056Z" }, +] + [[package]] name = "pillow" version = "12.3.0" diff --git a/nextjs-frontend/__tests__/settings-kokoro.test.tsx b/nextjs-frontend/__tests__/settings-kokoro.test.tsx new file mode 100644 index 000000000..476a70bb1 --- /dev/null +++ b/nextjs-frontend/__tests__/settings-kokoro.test.tsx @@ -0,0 +1,119 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; + +import { Admin, Voice } from "@/app/openapi-client"; +import { SettingsSection } from "@/components/settings/section"; +import type { SettingsView } from "@/components/settings/model"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +jest.mock("@/app/openapi-client", () => ({ + Admin: { updateSettings: jest.fn(), decisionStatus: jest.fn(), testDecisions: jest.fn() }, + Voice: { listVoices: jest.fn(), speakText: jest.fn() }, + Metadata: {}, +})); +jest.mock("next-auth/react", () => ({ useSession: () => ({ data: { accessToken: "t" } }) })); +jest.mock("next/navigation", () => ({ useRouter: () => ({ push: jest.fn() }) })); +jest.mock("@/lib/hooks/session", () => ({ useArchive: () => ({ namespaces: [] }) })); +jest.mock("@/components/ui/toast", () => ({ useToast: () => jest.fn() })); + +const ok = (data: unknown) => Promise.resolve({ data, response: { ok: true, status: 200 } }); +const view = (voice: Record) => + ({ + // The server's defaults for the rest of the section. + ai: { + values: { + tools: true, + browser: true, + refine_notes: true, + organise_notes: true, + compact: true, + untrusted_marking: true, + untrusted_asks: true, + injection_check: "patterns", + disabled_tools: [], + max_steps: 6, + max_tool_calls: 50, + browser_steps: 25, + max_transcript_reads: 20, + confirm_over_recordings: 100, + confirm_over_cost: null, + price_in: null, + price_out: null, + compact_model: null, + context_tokens: null, + }, + }, + decisions: { + values: { + engine: "auto", + base_url: "https://api.typesafe.ai/v1", + model: "jev-latest", + api_key: { secret: true, set: false }, + act_above: 0.8, + timeout: 10, + price_per_call: null, + laya_model: "aac6fef/laya-mlx", + laya_url: null, + }, + }, + voice: { + values: { + input: "auto", + stt: "same", + tts_provider: "openai", + tts_base_url: null, + tts_model: null, + tts_voice: null, + tts_api_key: { secret: true, set: false }, + kokoro_model: "fp32", + styletts2_url: "http://styletts2:8880", + ...voice, + }, + }, + web: { + values: { + search: "off", + searxng_url: null, + api_key: { secret: true, set: false }, + results: 5, + read_chars: 20000, + render: false, + }, + }, + }) as unknown as SettingsView; + +function show(view: SettingsView) { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + return render( + + + undefined} /> + + , + ); +} + +describe("Kokoro in the voice settings", () => { + beforeEach(() => { + (Admin.updateSettings as jest.Mock).mockReset().mockImplementation(() => ok({ ok: true })); + (Admin.decisionStatus as jest.Mock).mockReset().mockImplementation(() => ok({})); + (Voice.listVoices as jest.Mock).mockReset().mockImplementation(() => ok({ provider: "kokoro", voices: [] })); + }); + + it("reads on this machine, with the model size to pick", async () => { + show(view({})); + expect(screen.queryByLabelText("Kokoro model")).not.toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Spoken answers by"), { target: { value: "kokoro" } }); + expect(screen.getByText(/nothing is sent anywhere and nothing is billed/)).toBeInTheDocument(); + const size = screen.getByLabelText("Kokoro model"); + expect(size).toHaveValue("fp32"); + fireEvent.change(size, { target: { value: "int8" } }); + fireEvent.click(screen.getByRole("button", { name: "Save" })); + await waitFor(() => expect(Admin.updateSettings).toHaveBeenCalledTimes(1)); + expect((Admin.updateSettings as jest.Mock).mock.calls[0][0]).toMatchObject({ + path: { section: "voice" }, + body: { tts_provider: "kokoro", kokoro_model: "int8" }, + }); + }); +}); diff --git a/nextjs-frontend/__tests__/settings-voice-picker.test.tsx b/nextjs-frontend/__tests__/settings-voice-picker.test.tsx index 0e3572e57..2706d0841 100644 --- a/nextjs-frontend/__tests__/settings-voice-picker.test.tsx +++ b/nextjs-frontend/__tests__/settings-voice-picker.test.tsx @@ -63,6 +63,26 @@ describe("voice picker", () => { expect(await screen.findByText(/It couldn’t read it/)).toBeInTheDocument(); }); + it("lists Kokoro's voices while it's still being fetched", async () => { + (Voice.listVoices as jest.Mock).mockImplementation(() => + ok({ + provider: "kokoro", + voices: [ + { id: "af_heart", name: "Heart", detail: "American English, female" }, + { id: "bm_george", name: "George", detail: "British English, male" }, + ], + error: "Kokoro is fetched once it’s saved as the voice (Settings → Components)", + }), + ); + show("kokoro", null); + expect(await screen.findByRole("option", { name: "George · British English, male" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Default (af_heart)" })).toBeInTheDocument(); + expect( + screen.getByText("Kokoro is fetched once it’s saved as the voice (Settings → Components)"), + ).toBeInTheDocument(); + expect(screen.queryByText(/Type a voice id instead/)).not.toBeInTheDocument(); + }); + it("asks to save before hearing a provider not saved yet", async () => { (Voice.listVoices as jest.Mock).mockImplementation(() => ok({ provider: "gemini", voices: [{ id: "Kore", name: "Kore", detail: "Firm" }] }), diff --git a/nextjs-frontend/__tests__/settings-web-search.test.tsx b/nextjs-frontend/__tests__/settings-web-search.test.tsx index 9e164a653..7393e4363 100644 --- a/nextjs-frontend/__tests__/settings-web-search.test.tsx +++ b/nextjs-frontend/__tests__/settings-web-search.test.tsx @@ -65,6 +65,7 @@ const web = (values: Record) => tts_model: null, tts_voice: null, tts_api_key: { secret: true, set: false }, + kokoro_model: "fp32", styletts2_url: "http://styletts2:8880", }, }, diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index eab5e9c80..b1ab102f6 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -2693,8 +2693,16 @@ function AiBody({ ctx }: { ctx: BodyCtx }) { /> {tts === "openai" && } {tts === "styletts2" && } + {tts === "kokoro" && } - {tts === "styletts2" ? ( + {tts === "kokoro" ? ( +

+ Reads answers and podcasts on this machine with Kokoro v1.0: nothing is sent anywhere and nothing is billed. + It’s fetched once you save it (see Components); until then the + browser reads answers. A voice’s first letter is its language (a: American English, b: British English, e: + Spanish, f: French, h: Hindi, i: Italian, j: Japanese, p: Portuguese, z: Mandarin). +

+ ) : tts === "styletts2" ? (

Reads on this machine, in a voice like a short clip of someone speaking: put a few seconds of a .wav in the service’s voices folder and pick it here. Heavy, so it runs only when started with{" "} diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index b9419a77b..8d0702845 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -1791,9 +1791,21 @@ export const FIELDS: FieldSpec[] = [ { value: "deepgram", label: "Deepgram Aura (provider)" }, { value: "gemini", label: "Google Gemini (provider)" }, { value: "say", label: "macOS say (Lens running on a Mac)" }, + { value: "kokoro", label: "Kokoro (on this machine, 54 voices)" }, { value: "styletts2", label: "StyleTTS2 (on this machine)" }, ], }, + { + section: "voice", + key: "kokoro_model", + label: "Kokoro model", + kind: "select", + options: [ + { value: "fp32", label: "Full (326 MB, the best, and fastest on most machines)" }, + { value: "fp16", label: "Half precision (164 MB)" }, + { value: "int8", label: "8-bit (114 MB, for a small disk)" }, + ], + }, { section: "voice", key: "tts_model", diff --git a/nextjs-frontend/components/settings/voice-picker.tsx b/nextjs-frontend/components/settings/voice-picker.tsx index 1891619b2..e94c34f23 100644 --- a/nextjs-frontend/components/settings/voice-picker.tsx +++ b/nextjs-frontend/components/settings/voice-picker.tsx @@ -17,6 +17,7 @@ const DEFAULT_LABEL: Record = { deepgram: "aura-2-thalia-en", gemini: "Kore", say: "the Mac’s own voice", + kokoro: "af_heart", styletts2: "its own reader", }; @@ -67,7 +68,9 @@ export function VoicePicker({ : hear.error ? hear.error.message : list.data?.error - ? `${list.data.error}. Type a voice id instead.` + ? voices.length + ? list.data.error + : `${list.data.error}. Type a voice id instead.` : unsaved ? "Save to hear a voice from this provider." : null; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 2d887e8f9..9e9dfd00e 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -30586,7 +30586,7 @@ "anyOf": [ { "type": "string", - "pattern": "^(openai|elevenlabs|deepgram|gemini|say|styletts2)$" + "pattern": "^(openai|elevenlabs|deepgram|gemini|say|kokoro|styletts2)$" }, { "type": "null" diff --git a/threat-model.md b/threat-model.md index e938f05aa..7d70f285b 100644 --- a/threat-model.md +++ b/threat-model.md @@ -469,6 +469,7 @@ third-party source; dependencies come from `uv.lock`, the pnpm lockfile and cont | Catalog GGUF models | pinned revision and SHA-256 | in model (P-MODEL-CHECKSUM) | *(documented, docs/local-models.md "checked against their SHA-256")* | | Prompt-injection classifier (ProtectAI deberta-v3-base-prompt-injection-v2, `injection` extra, opt-in) | a judgement on whether a tool's result reads like an injection; the model is fetched at a pinned revision and checked against its SHA-256 | upstream for the model's judgement: a miss or a false flag is N-PROMPT; a model file that passes unchecked is in model | *(documented, docs/assistant.md "fetched when you pick it")* | | Florence-2 (Florence-2-base-ft or -large-ft, their 8-bit ONNX exports, `florence` extra, opt-in: `llm.describe_engine` `florence-2`) | describing the archive's pictures on the worker's CPU, so none is sent to a provider; the model is fetched at a pinned revision and checked against its SHA-256 | upstream for what the model says: text a picture shows that ends up in its description reaches the assistant as archive text (N-PROMPT); a model file that passes unchecked is in model | *(documented, docs/configuration.md "pinned to a revision and checked against its SHA-256")* | +| Kokoro v1.0 (hexgrad's Kokoro-82M as kokoro-onnx exports it, `kokoro` extra, opt-in: `voice.tts_provider` `kokoro`) | reading answers and podcast lines aloud on the server's CPU, so no text goes to a speech provider; the model and voices files come from kokoro-onnx's GitHub release and are checked against their SHA-256; the voices file is a NumPy `.npz` that `np.load` reads with pickles off; espeak-ng (GPL-3.0, only in the extra) turns the text into phonemes in the process that reads it aloud | upstream for espeak-ng's handling of the text (an answer or a podcast script, which can quote the archive) and the synthesis; a model or voices file that passes unchecked is in model | *(documented, docs/speech-providers.md "checked against their SHA-256")* | | Punctuation model (kredor/punctuate-all, `punctuation` extra, opt-in: `analysis.cleanup_punctuation`) | full stops, commas and question marks for a transcript's lines; the model is fetched from Hugging Face at a pinned revision, and Transformers loads it there with no remote code | upstream for the model's judgement (a missed or wrong mark, which the edit history can revert); a model file from another revision is in model | *(documented, docs/components.md "at a pinned revision")* | | spaCy and its small models (`spacy` extra, opt-in: `analysis.entities` `spacy`) | finding names in text; the MIT model wheels (`en_core_web_sm`, `de_core_news_sm`, `xx_ent_wiki_sm` 3.8.0) come from spaCy's GitHub releases, are checked against their SHA-256 and installed without dependencies; a model is Python code that runs when it loads, so the checksum is what keeps a swapped file out | upstream for the model's judgement (a missed or wrong name); a model file that installs unchecked is in model | *(documented, docs/components.md "checked against their SHA-256")* | | Optional ML extras (whisper, pyannote, speechbrain, insightface, doctr) | model inference; some fetch weights on their own | upstream | *(inferred, Q19)* | diff --git a/threat-model.yaml b/threat-model.yaml index b31e307a0..f29628fd3 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -2,7 +2,7 @@ # Authority order: prose > yaml > json. Regenerate whenever the prose changes. schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:7914ab4b9c598440e8026ac0db1b968f42da60369831cac0884d9afc7ef42d60" +prose_version: "threat-model.md@sha256:33c7f51876ad52b2d696fa60d8752e58a6095e2dd2258431075ce5e4e6d406ee" model_status: under-review triage_policy: strict confidence: {documented: 205, maintainer: 46, inferred: 91, assumption: 8} From 51ac75c29c30e305a72ddf42a033ae29dbbaa0e7 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 01:12:27 +0000 Subject: [PATCH 02/10] test(settings): carry ai.auto_title in the Kokoro settings fixture main's conversation titles (#531) added ai.auto_title to the section the Kokoro settings test renders, so its fixture names it too; without it Save sent an ai update beside the voice one. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- nextjs-frontend/__tests__/settings-kokoro.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/nextjs-frontend/__tests__/settings-kokoro.test.tsx b/nextjs-frontend/__tests__/settings-kokoro.test.tsx index 476a70bb1..8a5a851e6 100644 --- a/nextjs-frontend/__tests__/settings-kokoro.test.tsx +++ b/nextjs-frontend/__tests__/settings-kokoro.test.tsx @@ -28,6 +28,7 @@ const view = (voice: Record) => refine_notes: true, organise_notes: true, compact: true, + auto_title: true, untrusted_marking: true, untrusted_asks: true, injection_check: "patterns", From d31bdea9190a5672a604fff8be7280b9f045e27f Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 02:55:01 +0000 Subject: [PATCH 03/10] docs(threat-model): count the Kokoro row in the JSON companion The threat model workflow's validate job wants a change to threat-model.md to change a companion too (#507): the Kokoro row is documented, so the JSON's documented count goes up by one, as #518 did for Moondream. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- threat-model.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threat-model.json b/threat-model.json index bbdc57df2..ba5c03554 100644 --- a/threat-model.json +++ b/threat-model.json @@ -6,7 +6,7 @@ "scope_subpath": null, "description": "Lens is a self-hosted archive for documents, recordings, photos and messages: it reads, transcribes, indexes and publishes them, and people search and chat over them. One operator (the instance admin) runs an instance for their own people: a FastAPI API, a Next.js web app that proxies the API and holds the sign-in session, and background workers, around one SurrealDB database, with content split into namespaces and per-namespace roles. Exposing an instance to the internet behind HTTPS (reverse proxy or Cloudflare tunnel) is supported. Only the instance admin (equivalent to host code execution) and the host operator are trusted; everyone else is trusted only as far as their role.", "confidence": { - "documented": 252, + "documented": 253, "inferred": 99 }, "components": [ From f8493ff45c172239a051436f76c8f0d3adefdc4e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 02:55:09 +0000 Subject: [PATCH 04/10] fix(voice): offer Kokoro in full precision and 8-bit only On GitHub's arm64 runner, ONNX Runtime's CPU provider returned an empty waveform for kokoro-onnx's fp16 export, and kokoro-onnx's pause insertion then failed on a zero-size array. On x64 it was no faster than fp32, so voice.kokoro_model is now fp32 (the default) or int8, and the fp16 file is no longer pinned. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- CHANGELOG.md | 2 +- DEVLOG.md | 6 ++++-- docs/speech-providers.md | 5 +++-- fastapi_backend/app/domain/components.py | 4 ++-- fastapi_backend/app/domain/kokoro.py | 9 +++++---- fastapi_backend/app/domain/settings.py | 2 +- fastapi_backend/app/domain/store.py | 2 +- fastapi_backend/tests/domain/test_kokoro.py | 8 ++++---- nextjs-frontend/components/settings/model.ts | 1 - 9 files changed, 21 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c646321d3..2d0b8fed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version - **Kokoro reads aloud on this machine.** Pick **Kokoro** as the voice in Settings → AI assistant → Voice, and spoken answers and podcast hosts are read by Kokoro v1.0 on the server's own CPU, in any of its 54 voices across nine languages, with nothing sent anywhere and nothing billed. It's fetched once picked (the `kokoro` extra with - kokoro-onnx and ONNX Runtime, the model in `voice.kokoro_model`, full, half or 8-bit, and the voices: about 450 MB, + kokoro-onnx and ONNX Runtime, the model in `voice.kokoro_model`, full or 8-bit, and the voices: about 450 MB, checked against their SHA-256); podcast episodes wait for it, and the browser reads answers until it's here. A podcast's second host gets a voice in the first host's language. Agents pick it through the settings API and list the voices with `GET /api/v1/voice/voices?provider=kokoro` ([Kokoro](docs/speech-providers.md#kokoro)). diff --git a/DEVLOG.md b/DEVLOG.md index 4d52f3ba1..93c1e146f 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -9,8 +9,10 @@ Adeel: Kokoro v1.0's model and voices for spoken answers and podcasts, so Lens r `voice.tts_provider` `kokoro`; components.py fetches the `kokoro` extra and the pinned model and voices files once it's picked, and podcast jobs wait for them. The voice list is fixed (54 voices), so the picker works before the fetch. espeak-ng and phonemizer are GPL-3.0, so the extra is opt-in only (AGENTS.md 13). Measured here on a 4-core x64 -container, one 5-second sentence: fp32 2.8 s, fp16 3.0 s, int8 4.6 s, so fp32 is the default; arm64 not measured -beyond the kokoro.yml check. +container, one 5-second sentence: fp32 2.8 s, fp16 3.0 s, int8 4.6 s, so fp32 is the default; on GitHub's arm64 +runner (kokoro.yml) fp32 read the same sentence in 3.2 s. kokoro-onnx's fp16 export isn't offered: on arm64 ONNX +Runtime's CPU provider returned an empty waveform for it (kokoro-onnx 0.6.1's pause insertion then fails on a zero-size +array), and on x64 it bought nothing over fp32; int8 is the small-disk option. Refine later: misaki's G2P instead of espeak-ng for English (better names and numbers, and no GPL in the path); speaking speed (`speed` 0.5 to 2); voice blends (`af_heart:60,am_michael:40`); stream long answers sentence by diff --git a/docs/speech-providers.md b/docs/speech-providers.md index 8928fe457..010bd8cce 100644 --- a/docs/speech-providers.md +++ b/docs/speech-providers.md @@ -111,8 +111,9 @@ What is fetched, once, into the data folder: - the `kokoro` extra: [kokoro-onnx](https://github.com/thewh1teagle/kokoro-onnx) (MIT) and ONNX Runtime, about 95 MB. kokoro-onnx turns text into phonemes with espeak-ng through phonemizer, which are GPL-3.0, so they are in this opt-in extra only, never in Lens's own install; -- the model, `voice.kokoro_model`: `fp32` (326 MB, the default: the best, and on most CPUs the fastest), `fp16` - (164 MB) or `int8` (114 MB, for a small disk); +- the model, `voice.kokoro_model`: `fp32` (326 MB, the default: the best, and on most CPUs the fastest) or `int8` + (114 MB, for a small disk). kokoro-onnx's half-precision export is not offered: on arm64, ONNX Runtime reads nothing + with it, and on x64 it is no faster than `fp32`; - `voices-v1.0.bin` (28 MB): all 54 voices. The model and voices files are the ones kokoro-onnx publishes (its `model-files-v1.1` release), checked against their diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py index a14c32095..380084694 100644 --- a/fastapi_backend/app/domain/components.py +++ b/fastapi_backend/app/domain/components.py @@ -95,10 +95,10 @@ ), } # Kokoro v1.0 (hexgrad's Kokoro-82M, Apache-2.0) as kokoro-onnx exports it, from its model-files-v1.1 release -# (kokoro.py): the model at three precisions, and every voice: {file: (sha256, size in MB)} +# (kokoro.py): the model in full precision and 8-bit (its fp16 export reads nothing on arm64, see kokoro.py), and every +# voice: {file: (sha256, size in MB)} KOKORO = { "kokoro-v1.0.onnx": ("beb0d1848dee9a49da392cc3df26958d46cfa35d321edf434f52949153f0df3a", 326), - "kokoro-v1.0.fp16.onnx": ("f3a290d384fbb27966d462905c71a46cef9e5fd00516b40df32a0b4afe77ac96", 164), "kokoro-v1.0.int8.onnx": ("ae315a79b623f244700e4afb9246c46a26066782e049ba174bf3ba433970ee9c", 114), "voices-v1.0.bin": ("bca610b8308e8d99f32e6fe4197e7ec01679264efed0cac9140fe9c29f1fbf7d", 28), } diff --git a/fastapi_backend/app/domain/kokoro.py b/fastapi_backend/app/domain/kokoro.py index 3570e2c4a..9b25f10bc 100644 --- a/fastapi_backend/app/domain/kokoro.py +++ b/fastapi_backend/app/domain/kokoro.py @@ -6,8 +6,9 @@ its 54 voices. espeak-ng and phonemizer, which kokoro-onnx calls it through, are GPL-3.0, so all of it is the `kokoro` extra: components.py fetches it, with the model and its voices, only once Kokoro is picked. The files are the ones kokoro-onnx publishes (its model-files-v1.1 release), checked against their SHA-256: the model, voice.kokoro_model -(fp32, 326 MB, the best and on most CPUs the fastest; fp16, 164 MB; int8, 114 MB, for a small disk), and -voices-v1.0.bin (28 MB), every voice. +(fp32, 326 MB, the best and on most CPUs the fastest; int8, 114 MB, for a small disk), and voices-v1.0.bin (28 MB), +every voice. kokoro-onnx's fp16 export isn't offered: ONNX Runtime's CPU provider on arm64 read nothing with it (an +empty waveform, which kokoro-onnx's pause insertion then trips over), and on x64 it was no faster than fp32. A voice's first letter is its language and its second whether it's a woman's or a man's (af_heart: American English, female); espeak-ng reads the text as that language. The model stays loaded while it's reading and is let go when @@ -21,7 +22,7 @@ from . import components -MODELS = {"fp32": "kokoro-v1.0.onnx", "fp16": "kokoro-v1.0.fp16.onnx", "int8": "kokoro-v1.0.int8.onnx"} +MODELS = {"fp32": "kokoro-v1.0.onnx", "int8": "kokoro-v1.0.int8.onnx"} VOICES_FILE = "voices-v1.0.bin" DEFAULT_VOICE = "af_heart" # the voice a podcast's second host gets, by language and sex, when there's more than one to pick from @@ -129,7 +130,7 @@ def _load(cfg): _loaded.clear() # another size's model goes: one is enough memory opts = onnxruntime.SessionOptions() - opts.log_severity_level = 3 # the fp16 model warns about every node it can't fold + opts.log_severity_level = 3 # errors only: its node-by-node warnings would fill the worker's log session = onnxruntime.InferenceSession(model, opts, providers=["CPUExecutionProvider"]) _loaded.update(model=model, kokoro=Kokoro.from_session(session, voices_path)) _loaded["used"] = time.time() diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index a05b6b1f8..833171982 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -174,7 +174,7 @@ ("voice", "input"): {"auto", "server", "browser"}, ("voice", "stt"): {"same", "sensevoice", "whisper", "mlx-whisper", "openai", "elevenlabs", "assemblyai", "deepgram"}, ("voice", "tts_provider"): {"openai", "elevenlabs", "deepgram", "gemini", "say", "kokoro", "styletts2"}, - ("voice", "kokoro_model"): {"fp32", "fp16", "int8"}, + ("voice", "kokoro_model"): {"fp32", "int8"}, ("mail", "security"): {"starttls", "ssl", "none"}, ("bridge", "answer"): {"mention", "all"}, ("bridge", "approve"): {"off", "low_risk"}, diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 9d9ec5383..63408578c 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -327,7 +327,7 @@ def labels(): # stt: the engine that hears voice chat ("same": the transcription engine); tts_provider: who reads answers aloud # (openai: the OpenAI-compatible speech server below; elevenlabs, deepgram, gemini: the keys in speech; say: macOS's # own voices, when Lens runs on a Mac itself; kokoro: Kokoro v1.0 on this machine, fetched once it's picked, its - # model kokoro_model (fp32, fp16 or int8); styletts2: the optional service at styletts2_url); tts_voice: the voice + # model kokoro_model (fp32 or int8); styletts2: the optional service at styletts2_url); tts_voice: the voice # they read in (null: the provider's default) "voice": { "input": "auto", diff --git a/fastapi_backend/tests/domain/test_kokoro.py b/fastapi_backend/tests/domain/test_kokoro.py index 933db74ae..42c909e06 100644 --- a/fastapi_backend/tests/domain/test_kokoro.py +++ b/fastapi_backend/tests/domain/test_kokoro.py @@ -110,10 +110,10 @@ def test_the_model_size_is_a_setting(db, here): assert kokoro._loaded["model"].endswith("kokoro-v1.0.int8.onnx") rows = [r for r in db.rows("SELECT * FROM activity") if r["action"] == "model.speech"] assert [r["model"] for r in rows] == ["kokoro:kokoro-v1.0-int8"] - voice.speak(picked(db, here, kokoro_model="fp16"), "Hello") # the other one is let go - assert kokoro._loaded["model"].endswith("kokoro-v1.0.fp16.onnx") - with pytest.raises(ValueError): - settings.save(db, here, "voice", {"kokoro_model": "fp64"}, "t") + voice.speak(picked(db, here, kokoro_model="fp32"), "Hello") # the other one is let go + assert kokoro._loaded["model"].endswith("kokoro-v1.0.onnx") + with pytest.raises(ValueError): # kokoro-onnx's fp16 export isn't offered: it reads nothing on arm64 + settings.save(db, here, "voice", {"kokoro_model": "fp16"}, "t") def test_the_model_is_let_go_when_nothing_is_read(db, here, monkeypatch): diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 7b41a1261..cd7db922e 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -1826,7 +1826,6 @@ export const FIELDS: FieldSpec[] = [ kind: "select", options: [ { value: "fp32", label: "Full (326 MB, the best, and fastest on most machines)" }, - { value: "fp16", label: "Half precision (164 MB)" }, { value: "int8", label: "8-bit (114 MB, for a small disk)" }, ], }, From 3ded53f5071b9840ec75fdb59d6705d203377312 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 02:59:15 +0000 Subject: [PATCH 05/10] test(settings): carry the research section in the Kokoro settings fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main's deep research (#517) added a research section to Settings › AI assistant, so the fixture carries its defaults and Save sends only the voice change. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- nextjs-frontend/__tests__/settings-kokoro.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/nextjs-frontend/__tests__/settings-kokoro.test.tsx b/nextjs-frontend/__tests__/settings-kokoro.test.tsx index 8a5a851e6..ec64f6ed8 100644 --- a/nextjs-frontend/__tests__/settings-kokoro.test.tsx +++ b/nextjs-frontend/__tests__/settings-kokoro.test.tsx @@ -58,6 +58,7 @@ const view = (voice: Record) => laya_url: null, }, }, + research: { values: { max_tool_calls: 40, max_minutes: 20, model: null, agents_skip_review: false } }, voice: { values: { input: "auto", From a02d70a01c3ee38f1a05d9eacc9e247027346743 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 06:09:17 +0000 Subject: [PATCH 06/10] test(settings): carry ai.compact_at in the Kokoro settings fixture Main added ai.compact_at (summarise long conversations early), so the fixture carries its default and Save sends only the voice change. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- nextjs-frontend/__tests__/settings-kokoro.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/nextjs-frontend/__tests__/settings-kokoro.test.tsx b/nextjs-frontend/__tests__/settings-kokoro.test.tsx index ec64f6ed8..5744340e3 100644 --- a/nextjs-frontend/__tests__/settings-kokoro.test.tsx +++ b/nextjs-frontend/__tests__/settings-kokoro.test.tsx @@ -43,6 +43,7 @@ const view = (voice: Record) => price_out: null, compact_model: null, context_tokens: null, + compact_at: 50, }, }, decisions: { From deee4ba242d505e040fee8cc05a0d7c8b46ff517 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 07:31:17 +0000 Subject: [PATCH 07/10] docs(readme): list answers about Lens itself as built, now that #400 is released Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- README.md | 13 +++++++------ docs/requirements.md | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3842575ac..6ccdaf6d2 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,11 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p [API](docs/api.md), [the graph as files](docs/file-tree.md), [authentication](docs/authentication.md); [checks](docs/requirements.md#connect-your-agents)) - **Ask, and get cited answers.** A chat assistant on every page, and by voice, searches your archive by words (and by - meaning, once turned on), reads it, cites it and acts on it with tools: changes wait for your approval. Once an admin - turns them on, it also searches the web and reads pages, cited like the archive, and runs Python and bash in a sandbox - of each conversation's own. An admin's assistant searches and reads the mail, calendars and files connected to Lens + meaning, once turned on), reads it, cites it and acts on it with tools: changes wait for your approval. Ask it about + Lens itself ("what's the MCP address?", "is Telegram on?") and it answers from Lens's docs and this server, citing + both. Once an admin turns them on, it also searches the web and reads pages, cited like the archive, and runs Python + and bash in a sandbox of each conversation's own. An admin's assistant searches and reads the mail, calendars and + files connected to Lens (Gmail, Google Calendar, Google Drive, S3 and the rest), cites messages and events, and brings them in once you approve; where an admin allows it, it sends mail, adds events and writes files, each only after you read all of it and say yes. Regenerating an answer or editing a question makes a new version beside the old one, and @@ -90,7 +92,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p assistant with its own name, instructions and memory, answering in Telegram and Slack (built in, with voice notes and files), in Discord, Matrix, WhatsApp and other chat rooms through Matterbridge, and by email at an address of its own that you - email or forward things to. ([Chat](docs/assistant.md), [web search](docs/assistant.md#web-search-and-reading-pages), + email or forward things to. ([Chat](docs/assistant.md), [questions about Lens](docs/assistant.md#questions-about-lens-itself), + [web search](docs/assistant.md#web-search-and-reading-pages), [mail, calendars and files](docs/assistant.md#mail-calendars-and-files), [code sandbox](docs/assistant.md#code-sandbox), [answer records](docs/assistant.md#what-each-answer-records), [chat rooms](docs/chat-rooms.md), [email](docs/assistant-email.md), [speech providers](docs/speech-providers.md); [checks](docs/requirements.md#ask-and-get-cited-answers)) @@ -235,8 +238,6 @@ still loading. ([Components](docs/components.md), These are open pull requests, not yet part of Lens: -- **The assistant** answering questions about Lens itself, from its docs and this server - ([#400](https://github.com/adeelahmad/lense/pull/400)). - **Chat:** a conversation exported as Markdown, PDF or JSON ([#474](https://github.com/adeelahmad/lense/pull/474)). - **Pictures:** low-quality images enhanced before they're read ([#479](https://github.com/adeelahmad/lense/pull/479)). - **Web clipper:** signing in, or using your API token, not only an import webhook's diff --git a/docs/requirements.md b/docs/requirements.md index 48d8a1bb6..83fe25134 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -78,6 +78,7 @@ request, to another check that proves the same thing; loosening a requirement or | It searches by words, and by meaning once search by meaning is set up | [test_chats.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chats.py) `test_tools_approvals_and_checking`; [test_semantic.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_semantic.py) `test_chat_retrieval_by_meaning`, `test_off_or_unconfigured` | | Answers cite their sources, and only what the person may read | [test_chats.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chats.py) `test_chat_cites_only_what_you_can_read`; [chat.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat.test.tsx) `splits [n], [n, m] and [n][m]` | | It acts with tools, and changes wait for approval | [test_chats.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chats.py) `test_tools_approvals_and_checking`; [test_approvals.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_approvals.py) `test_the_assistant_lists_and_declines_but_never_approves` | +| Questions about Lens itself answered from Lens's docs and this server's facts, both cited, for people and agents over MCP | [test_lens_help.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_lens_help.py) `test_the_docs_are_found_by_words`, `test_the_assistant_answers_about_lens_itself`, `test_agents_ask_lens_over_mcp`, `test_without_a_question_lens_help_still_gives_the_server` | | Web search and reading pages, cited like the archive, off until an admin turns it on | [test_web_search.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_search.py) `test_an_answer_cites_the_web_and_the_conversation_keeps_it`, `test_reading_pages_and_pdfs`; [chat-web-citations.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-web-citations.test.tsx) | | An admin's assistant searches and reads connected mail, calendars and files, cites messages and events, and brings them in only once approved | [test_connection_tools.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_connection_tools.py) `test_only_admins_get_them_and_only_for_what_is_connected`, `test_searching_and_reading_gmail`, `test_calendar_events_between_two_times`, `test_finding_and_reading_files_in_storage`, `test_bringing_things_in_waits_for_the_persons_yes`; [chat-web-citations.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-web-citations.test.tsx) `a message from a connected mailbox` | | Where an admin allows it, the assistant sends mail, adds events and writes files, each only after the person reads all of it and says yes | [test_connection_tools.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_connection_tools.py) `test_sending_mail_waits_for_the_persons_yes`, `test_adding_an_event_waits_for_the_persons_yes`, `test_writing_a_file_waits_for_the_persons_yes`; [approvals-page.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/approvals-page.test.tsx) `shows the assistant's whole message before it is sent`; [add-connection.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/add-connection.test.tsx) `connects Gmail the same way, and asks a service account whose mail it reads` | From 4f47f788b2b8994cf34a4ed9d8a1bbcef9a27023 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 11:05:36 +0000 Subject: [PATCH 08/10] docs(changelog): name the Kokoro fragment after its pull request Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- changelog.d/{kokoro.md => 538.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{kokoro.md => 538.md} (100%) diff --git a/changelog.d/kokoro.md b/changelog.d/538.md similarity index 100% rename from changelog.d/kokoro.md rename to changelog.d/538.md From e638a9da8ad1abf798053dde4595b4648d8f9964 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 12:15:44 +0000 Subject: [PATCH 09/10] docs(threat-model): count the Kokoro entries against main's newest model Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- threat-model.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threat-model.json b/threat-model.json index f702bca5f..c91137347 100644 --- a/threat-model.json +++ b/threat-model.json @@ -6,7 +6,7 @@ "scope_subpath": null, "description": "Lens is a self-hosted archive for documents, recordings, photos and messages: it reads, transcribes, indexes and publishes them, and people search and chat over them. One operator (the instance admin) runs an instance for their own people: a FastAPI API, a Next.js web app that proxies the API and holds the sign-in session, and background workers, around one SurrealDB database, with content split into namespaces and per-namespace roles. Exposing an instance to the internet behind HTTPS (reverse proxy or Cloudflare tunnel) is supported. Only the instance admin (equivalent to host code execution) and the host operator are trusted; everyone else is trusted only as far as their role.", "confidence": { - "documented": 258, + "documented": 259, "inferred": 99 }, "components": [ From 7e91636136e534d4083eefca638c4ed2980defd8 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 14:45:06 +0000 Subject: [PATCH 10/10] docs(readme): take Kokoro out of In progress, now that it's built Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dzm4YHdHn2qZy7aXer3KeH --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index aac0547d5..040a705f4 100644 --- a/README.md +++ b/README.md @@ -306,10 +306,7 @@ still loading. ([Components](docs/components.md), ## In progress -These are open pull requests, not yet part of Lens: - -- **Voice:** answers and podcasts read aloud with Kokoro on this machine - ([#538](https://github.com/adeelahmad/lense/pull/538)). +Open pull requests, not yet part of Lens, are listed here; none right now. ## Planned