From 22a08d62e1b0f760f7e5314740b1c8e23abf6f19 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Wed, 14 Oct 2026 15:44:04 +0000 Subject: [PATCH] feat(speech): send long recordings to a provider in pieces that fit its limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recording longer or larger than a speech provider takes in one request is cut at pauses into pieces within its limits, each piece is transcribed, and the transcripts are joined on the recording's timeline. Each provider starts with its published limits (speech.{provider}_max_minutes and _max_mb), editable in Settings → Speech providers. Replaces the ten-minute parts only OpenAI-compatible servers had. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01L9FY5GYBynyZQi49uouhUj --- CHANGELOG.md | 5 + DEVLOG.md | 14 ++ docs/speech-providers.md | 33 +++- fastapi_backend/app/domain/settings.py | 9 + fastapi_backend/app/domain/speech.py | 176 ++++++++++++------ fastapi_backend/app/domain/store.py | 10 + fastapi_backend/tests/domain/test_speech.py | 80 ++++++++ .../__tests__/settings-speech.test.tsx | 23 +++ .../components/settings/bodies.tsx | 6 + nextjs-frontend/components/settings/model.ts | 25 +++ 10 files changed, 322 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843bd1da4..777db7530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Long recordings go to a speech provider in pieces that fit it.** A recording longer or larger than a provider takes + in one request is cut at pauses into pieces within its limits, and their transcripts are joined back into one, where + only OpenAI-compatible servers had ten-minute parts before. Each provider starts with its published limits (10 hours + for ElevenLabs and AssemblyAI, 2 GB for Deepgram, 25 MB for OpenAI), editable in Settings → Speech providers + ([docs](docs/speech-providers.md#long-recordings)). - **Running the install line again keeps `.env` readable only by you.** Each update rewrote `.env` with the shell's default permissions, so after an update its secrets could be read by other users of the machine. It is now written for its owner alone, and an install an earlier update left open is closed again the next time the line runs. diff --git a/DEVLOG.md b/DEVLOG.md index 5719bcfd7..1b28864f9 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,20 @@ Plans and progress for work in flight. Newest first. +## 2026-10-14 · Long recordings in pieces a speech provider takes + +Adeel: large audio files are segmented automatically to each transcription provider's limits. `speech.Engine` cuts a +recording longer than `speech.{provider}_max_minutes` at the quietest tenth of a second in the half minute before the +limit, halves a piece whose encoded file is over `speech.{provider}_max_mb` (again at a pause), sends each piece and +joins the segments on the recording's timeline. Defaults follow each provider's published limits +(docs/speech-providers.md, Long recordings); they are in Settings → Speech providers. This replaces the ten-minute +parts only OpenAI-compatible servers had. + +Refine later: link a new voice heard in two pieces into one speaker by voiceprint (today it shows as two until merged); +decode and send a very long recording a piece at a time instead of holding all its samples in memory; send pieces in +parallel for providers that allow it; overlap pieces a little and drop the words heard twice, for a cut with no pause +near it. + ## 2026-10-14 · MCP servers that run as a program Admins add a stdio MCP server (`command`, `args`, `env`, as in Claude's `.mcp.json`) as an `mcp` extension, and a diff --git a/docs/speech-providers.md b/docs/speech-providers.md index 28252651a..6a0e039fb 100644 --- a/docs/speech-providers.md +++ b/docs/speech-providers.md @@ -40,6 +40,35 @@ Then pick where each task goes: - **AI assistant → Voice**: *Engine that hears it* (default: the transcription engine) and *Spoken answers by* (an OpenAI-compatible speech server, ElevenLabs, Deepgram Aura, Google Gemini or macOS `say`) and *Voice*. +## Long recordings + +Each provider takes so much audio in one request. A recording longer or larger than that is cut into pieces, each +piece is transcribed on its own, and the transcripts are joined back into one, timed from the start of the recording. +Nothing needs setting up: every provider starts with a limit from its published one. + +| Provider | Longest piece | Largest file | Where the default comes from | +| --- | --- | --- | --- | +| OpenAI-compatible | 10 minutes | 25 MB | [OpenAI](https://platform.openai.com/docs/guides/speech-to-text): 25 MB a file; GPT-4o transcription takes up to 25 minutes, and shorter pieces suit servers with smaller limits | +| ElevenLabs | 600 minutes | 3000 MB | [ElevenLabs](https://elevenlabs.io/docs/capabilities/speech-to-text): up to 10 hours and 3 GB | +| AssemblyAI | 600 minutes | 2200 MB | [AssemblyAI](https://www.assemblyai.com/docs/faq/are-there-any-limits-on-file-size-or-file-duration-for-files-submitted-to-the-api): up to 10 hours; 2.2 GB through its upload endpoint | +| Deepgram | 120 minutes | 2000 MB | [Deepgram](https://developers.deepgram.com/docs/pre-recorded-audio): 2 GB; it stops a request after 10 minutes of work, so two hours at a time | + +Change them in **Settings → Speech providers** (*Longest piece sent at once*, *Largest file sent at once*), or as +`speech.{provider}_max_minutes` and `speech.{provider}_max_mb` through the settings API or the assistant's +`change_settings` tool: lower them for a proxy or a compatible server that takes less (Groq's free tier, a small +self-hosted server), raise them for one that takes more. + +How a recording is cut: + +- A piece ends at the quietest tenth of a second in the half minute before the limit, so a cut falls in a pause rather + in the middle of a word. Pieces meet exactly: nothing is sent twice or left out. +- Each piece is encoded to send; one whose file is still larger than the limit is halved, again at a pause, until each + part fits. +- The provider numbers speakers afresh in every request, so a speaker label belongs to its piece (`c0:S1`, `c1:S1`). + Voice IDs still come from local voiceprints and match voices already known; a new voice heard in two pieces shows as + two speakers until you merge them. +- The activity ledger keeps one row for the recording, with its length in seconds and the number of pieces. + ## Voices `GET /voice/voices?provider=…` lists the voices a provider reads in, as `{id, name, detail}`: ElevenLabs lists the @@ -89,7 +118,7 @@ permission to use the voice. ## What is sent Only the audio of the recording (or the clip said into the mic), encoded as Ogg Opus at 32 kbit/s (FLAC when ffmpeg -has no Opus). Recordings longer than ten minutes go to OpenAI-compatible servers in ten-minute parts, since they take -at most 25 MB a request; speaker labels then belong to each part. Text to speech sends the answer's text. +has no Opus), in pieces when it is longer or larger than the provider takes at once ([above](#long-recordings)). Text +to speech sends the answer's text. Code: `fastapi_backend/app/domain/speech.py`. diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index 9ed4a99cd..412ea1f41 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -630,9 +630,18 @@ def _speech_setting(key, value): if not (isinstance(value, int) and not isinstance(value, bool) and 30 <= value <= 86400): raise ValueError("speech.timeout is a whole number of seconds from 30 to 86400") return value + for end, (lo, hi), what in (("_max_minutes", SPEECH_MINUTES, "minutes"), ("_max_mb", SPEECH_MB, "MB")): + if key.endswith(end): + if not (isinstance(value, int) and not isinstance(value, bool) and lo <= value <= hi): + raise ValueError(f"speech.{key} is a whole number of {what} from {lo} to {hi}") + return value raise ValueError(f"unknown setting speech.{key}") +SPEECH_MINUTES = (1, 1440) # the longest piece of a recording sent to a speech provider at once +SPEECH_MB = (1, 10_000) # its largest file + + PODCAST_RANGES = {"context_chars": (2000, 400_000), "max_minutes": (1, 120), "words_per_minute": (80, 250)} diff --git a/fastapi_backend/app/domain/speech.py b/fastapi_backend/app/domain/speech.py index da54867e6..b0646763a 100644 --- a/fastapi_backend/app/domain/speech.py +++ b/fastapi_backend/app/domain/speech.py @@ -17,6 +17,10 @@ A provider's transcript is the same list of segments a local engine returns, plus `provider_speaker`: the speaker separation step uses those labels (diarize.engine provider, or auto), and voice IDs still come from local voiceprints. + +Each provider takes so much audio in one request ({p}_max_minutes and {p}_max_mb in the speech settings, defaulting to +its published limits): a longer recording is cut into pieces at pauses, each piece is transcribed on its own, and their +transcripts are joined back on the recording's timeline. """ from __future__ import annotations @@ -57,7 +61,8 @@ "styletts2": "StyleTTS2", } TTS_PROVIDERS = ("openai", "elevenlabs", "deepgram", "gemini", "say", "styletts2") -CHUNK_SECONDS = 600 # OpenAI-compatible servers take up to 25 MB a request; ten minutes of Opus is a few MB +QUIET_SECONDS = 30 # a recording too long for one request is cut at the quietest moment in the half minute before a limit +QUIET_WINDOW = SR // 10 # ... measured in tenths of a second POLL_SECONDS = 3 MAX_WORDS = 40 GAP_MS = 1500 @@ -293,55 +298,44 @@ def _label(v): # ---------- providers ---------- -def _openai(cfg, audio, seconds): +def _openai(cfg, clip, seconds): base, m, key = base_url(cfg, "openai"), model(cfg, "openai"), _key(cfg, "openai") - diarize = "diarize" in m + data, name, ctype = clip lang = _language(cfg) - out, many = [], len(audio) > CHUNK_SECONDS * SR - for k, start in enumerate(range(0, max(len(audio), 1), CHUNK_SECONDS * SR)): - data, name, ctype = encode(audio[start : start + CHUNK_SECONDS * SR]) - if diarize: - fields = [("model", m), ("response_format", "diarized_json"), ("chunking_strategy", "auto")] - elif m.startswith("gpt-"): - fields = [("model", m), ("response_format", "json")] - else: - fields = [("model", m), ("response_format", "verbose_json"), ("timestamp_granularities[]", "segment"), - ("timestamp_granularities[]", "word")] # fmt: skip - if lang: - fields.append(("language", lang)) - body, ctype_ = multipart(fields, [("file", name, data, ctype)]) - headers = {"Content-Type": ctype_, **({"Authorization": f"Bearer {key}"} if key else {})} - j = _json(base + "/audio/transcriptions", data=body, headers=headers, timeout=section(cfg)["timeout"]) - off = start * 1000 // SR - got = j.get("language") or lang - segs = j.get("segments") or [] - words = j.get("words") or [] - if not segs: - end = min(len(audio), start + CHUNK_SECONDS * SR) - segs = [{"start": 0, "end": (end - start) / SR, "text": j.get("text") or ""}] - for s in segs: - t0, t1 = off + int(float(s.get("start") or 0) * 1000), off + int(float(s.get("end") or 0) * 1000) - ws = [ - [w["word"].strip(), off + int(w["start"] * 1000), off + int(w["end"] * 1000)] - for w in words - if t0 <= off + w["start"] * 1000 < t1 - ] - spk = _label(s.get("speaker")) - text = (s.get("text") or "").strip() - if any(ch.isalnum() for ch in text): - out.append( - { - "t0": t0, - "t1": max(t1, t0 + 1), - "text": text, - "raw_text": None, - "emotion": None, - "event": None, - "lang": _short(got), - "words": json.dumps(ws) if ws else None, - "provider_speaker": (f"c{k}:{spk}" if many else spk) if spk else None, - } # fmt: skip - ) + if "diarize" in m: + fields = [("model", m), ("response_format", "diarized_json"), ("chunking_strategy", "auto")] + elif m.startswith("gpt-"): + fields = [("model", m), ("response_format", "json")] + else: + fields = [("model", m), ("response_format", "verbose_json"), ("timestamp_granularities[]", "segment"), + ("timestamp_granularities[]", "word")] # fmt: skip + if lang: + fields.append(("language", lang)) + body, ctype_ = multipart(fields, [("file", name, data, ctype)]) + headers = {"Content-Type": ctype_, **({"Authorization": f"Bearer {key}"} if key else {})} + j = _json(base + "/audio/transcriptions", data=body, headers=headers, timeout=section(cfg)["timeout"]) + got = j.get("language") or lang + segs = j.get("segments") or [{"start": 0, "end": seconds, "text": j.get("text") or ""}] + words = j.get("words") or [] + out = [] + for s in segs: + t0, t1 = int(float(s.get("start") or 0) * 1000), int(float(s.get("end") or 0) * 1000) + ws = [[w["word"].strip(), int(w["start"] * 1000), int(w["end"] * 1000)] for w in words if t0 <= w["start"] * 1000 < t1] + text = (s.get("text") or "").strip() + if any(ch.isalnum() for ch in text): + out.append( + { + "t0": t0, + "t1": max(t1, t0 + 1), + "text": text, + "raw_text": None, + "emotion": None, + "event": None, + "lang": _short(got), + "words": json.dumps(ws) if ws else None, + "provider_speaker": _label(s.get("speaker")), + } # fmt: skip + ) return out @@ -356,8 +350,8 @@ def _short(lang): return names.get(lang) or lang.split("-")[0].split("_")[0][:3] -def _elevenlabs(cfg, audio, seconds): - data, name, ctype = encode(audio) +def _elevenlabs(cfg, clip, seconds): + data, name, ctype = clip fields = [("model_id", model(cfg, "elevenlabs")), ("diarize", "true"), ("tag_audio_events", "true"), ("timestamps_granularity", "word")] if _language(cfg): fields.append(("language_code", _language(cfg))) @@ -386,9 +380,9 @@ def _elevenlabs(cfg, audio, seconds): return group_words(words, _short(j.get("language_code"))) -def _assemblyai(cfg, audio, seconds): +def _assemblyai(cfg, clip, seconds): base, key, t = base_url(cfg, "assemblyai"), _key(cfg, "assemblyai"), section(cfg)["timeout"] - data, _, ctype = encode(audio) + data, _, ctype = clip up = _json(base + "/v2/upload", data=data, headers={"Authorization": key, "Content-Type": "application/octet-stream"}, timeout=t) ask = {"audio_url": up["upload_url"], "speaker_labels": True, "punctuate": True, "format_text": True} m = model(cfg, "assemblyai") @@ -426,8 +420,8 @@ def _assemblyai(cfg, audio, seconds): return add_sentiment(segs, spans) -def _deepgram(cfg, audio, seconds): - data, _, ctype = encode(audio) +def _deepgram(cfg, clip, seconds): + data, _, ctype = clip q = {"model": model(cfg, "deepgram"), "smart_format": "true", "punctuate": "true", "diarize": "true", "utterances": "true"} if _language(cfg): q["language"] = _language(cfg) @@ -465,9 +459,67 @@ def _deepgram(cfg, audio, seconds): return add_sentiment(segs, spans) +# each takes (cfg, an encoded clip as (bytes, file name, media type), its seconds) and returns its segments, timed +# from the clip's start TRANSCRIBE = {"openai": _openai, "elevenlabs": _elevenlabs, "assemblyai": _assemblyai, "deepgram": _deepgram} +# ---------- long recordings ---------- +def limits(cfg, p): + """The most a provider takes in one request: (samples, bytes) from {p}_max_minutes and {p}_max_mb.""" + s = section(cfg) + return int(s[f"{p}_max_minutes"]) * 60 * SR, int(s[f"{p}_max_mb"]) * 1_000_000 + + +def quiet_cut(audio, lo, hi): + """The quietest tenth of a second between samples lo and hi (the latest, of equally quiet ones), so a cut falls in + a pause rather than in a word; hi when there's too little to look at.""" + k = (hi - lo) // QUIET_WINDOW + if k < 2: + return hi + energy = np.square(audio[lo : lo + k * QUIET_WINDOW].reshape(k, QUIET_WINDOW)).mean(axis=1) + return lo + (k - 1 - int(np.argmin(energy[::-1]))) * QUIET_WINDOW + QUIET_WINDOW // 2 + + +def pieces(audio, most): + """Where to cut samples into pieces of at most `most` samples each, each cut at a pause: [(start, end)].""" + out, start = [], 0 + while len(audio) - start > most: + hi = start + most + cut = quiet_cut(audio, max(start + most // 2, hi - QUIET_SECONDS * SR), hi) + out.append((start, cut)) + start = cut + out.append((start, len(audio))) + return out + + +def clips(audio, start, end, most_bytes): + """A piece encoded to send, halved at a pause until each part's file fits the provider's limit: (start, end, file)s.""" + clip = encode(audio[start:end]) + if len(clip[0]) <= most_bytes: + yield start, end, clip + return + if end - start < 2 * SR: + mb = most_bytes // 1_000_000 + raise ProviderError(f"two seconds of audio already make a file larger than {mb} MB: raise the limit in Settings → Speech providers") + mid = (start + end) // 2 + mid = quiet_cut(audio, max(start + (end - start) // 4, mid - QUIET_SECONDS * SR), mid) + yield from clips(audio, start, mid, most_bytes) + yield from clips(audio, mid, end, most_bytes) + + +def _shift(segs, ms, piece): + """A piece's segments on the recording's timeline; with more than one piece, its speakers are its own (cN:label), + as a provider numbers the speakers of each request afresh.""" + for s in segs: + s["t0"], s["t1"] = s["t0"] + ms, s["t1"] + ms + if ms and s.get("words"): + s["words"] = json.dumps([[w, a + ms, b + ms] for w, a, b in json.loads(s["words"])]) + if piece is not None and s.get("provider_speaker"): + s["provider_speaker"] = f"c{piece}:{s['provider_speaker']}" + return segs + + class Engine: """A provider as a transcription engine: transcribe(16 kHz mono samples) -> segments, like SenseVoice's.""" @@ -478,9 +530,19 @@ def __init__(self, cfg, provider): self.cfg, self.provider, self.name = cfg, provider, provider def transcribe(self, audio): - seconds = round(len(audio) / SR, 2) - with _ledger("speech.transcribe", self.cfg, f"{self.provider}:{model(self.cfg, self.provider)}", {"seconds": seconds}): - segs = TRANSCRIBE[self.provider](self.cfg, audio, seconds) + """The recording's transcript: in one request when it fits the provider's limits, else piece by piece.""" + most, most_bytes = limits(self.cfg, self.provider) + detail = {"seconds": round(len(audio) / SR, 2)} # the ledger row reads it when the call ends + parts = [] + with _ledger("speech.transcribe", self.cfg, f"{self.provider}:{model(self.cfg, self.provider)}", detail): + for a, b in pieces(audio, most): + for start, end, clip in clips(audio, a, b, most_bytes): + parts.append((start, TRANSCRIBE[self.provider](self.cfg, clip, round((end - start) / SR, 2)))) + if len(parts) > 1: + detail["pieces"] = len(parts) + log.info("speech: %s s of audio went to %s in %d pieces", detail["seconds"], LABELS[self.provider], len(parts)) + many = len(parts) > 1 + segs = [s for k, (start, part) in enumerate(parts) for s in _shift(part, start * 1000 // SR, k if many else None)] return sorted(segs, key=lambda s: s["t0"]) diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 5cb28a45b..f2b0fa5cc 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -339,19 +339,29 @@ def labels(): }, # speech providers (domain/speech.py): transcription, speakers and text to speech by a service instead of this # server, picked in transcribe.engine, diarize.engine and voice. A base URL can be a proxy or a compatible server. + # {p}_max_minutes and {p}_max_mb are the most audio a provider takes in one request: a longer recording is cut into + # pieces at pauses and their transcripts joined (docs/speech-providers.md, Long recordings, says where each comes from) "speech": { "openai_base_url": "https://api.openai.com/v1", "openai_model": "whisper-1", "openai_api_key": None, + "openai_max_minutes": 10, # 25 MB a file; GPT-4o transcription takes up to 25 minutes + "openai_max_mb": 25, "elevenlabs_base_url": "https://api.elevenlabs.io", "elevenlabs_model": "scribe_v1", "elevenlabs_api_key": None, + "elevenlabs_max_minutes": 600, # up to 10 hours and 3 GB + "elevenlabs_max_mb": 3000, "assemblyai_base_url": "https://api.assemblyai.com", "assemblyai_model": "universal", "assemblyai_api_key": None, + "assemblyai_max_minutes": 600, # up to 10 hours; 2.2 GB through its upload endpoint + "assemblyai_max_mb": 2200, "deepgram_base_url": "https://api.deepgram.com", "deepgram_model": "nova-3", "deepgram_api_key": None, + "deepgram_max_minutes": 120, # 2 GB; it stops a request after 10 minutes of work, so two hours at a time + "deepgram_max_mb": 2000, "gemini_base_url": "https://generativelanguage.googleapis.com", "gemini_model": "gemini-2.5-flash-preview-tts", "gemini_api_key": None, diff --git a/fastapi_backend/tests/domain/test_speech.py b/fastapi_backend/tests/domain/test_speech.py index 9abce7123..19287df60 100644 --- a/fastapi_backend/tests/domain/test_speech.py +++ b/fastapi_backend/tests/domain/test_speech.py @@ -6,6 +6,7 @@ import base64 import http.server import io +import itertools import json import shutil import subprocess @@ -298,6 +299,85 @@ def test_speaker_separation_uses_the_providers_speakers(db, cfg, folder, fake): assert [(r["local_speaker"], r["provider_speaker"]) for r in rows] == [("S0", "S0"), ("S1", "S1")] +def talk(seconds, pauses): + """Noise like speech with a second of silence starting at each of `pauses` (seconds).""" + audio = clip(seconds) + for at in pauses: + audio[int(at * speech.SR) : int((at + 1) * speech.SR)] = 0 + return audio + + +def test_each_provider_has_limits_and_they_can_be_changed(db, cfg): + eff = settings.effective(db, cfg) + assert {p: speech.limits(eff, p) for p in speech.PROVIDERS} == { + "openai": (10 * 60 * speech.SR, 25_000_000), + "elevenlabs": (600 * 60 * speech.SR, 3_000_000_000), + "assemblyai": (600 * 60 * speech.SR, 2_200_000_000), + "deepgram": (120 * 60 * speech.SR, 2_000_000_000), + } + settings.save(db, cfg, "speech", {"openai_max_minutes": 20, "openai_max_mb": 100}, "t") + assert speech.limits(settings.effective(db, cfg), "openai") == (20 * 60 * speech.SR, 100_000_000) + for bad in ({"openai_max_minutes": 0}, {"deepgram_max_mb": 10_001}, {"elevenlabs_max_minutes": "60"}, {"gemini_max_mb": 5}): + with pytest.raises(ValueError): + settings.save(db, cfg, "speech", bad, "t") + + +def test_a_cut_falls_in_a_pause(): + sr = speech.SR + audio = talk(150, (48, 103)) + cuts = speech.pieces(audio, 60 * sr) + assert len(cuts) == 3 and cuts[0][0] == 0 and cuts[-1][1] == len(audio) + assert all(a == b for (_, a), (b, _) in itertools.pairwise(cuts)) # nothing lost or sent twice + assert all(e - s <= 60 * sr for s, e in cuts) + assert 48 * sr < cuts[0][1] < 49 * sr and 103 * sr < cuts[1][1] < 104 * sr + assert speech.pieces(audio, 150 * sr) == [(0, len(audio))] + flat = np.zeros(90 * sr, dtype=np.float32) # no pause stands out: cut as late as it may + assert speech.pieces(flat, 60 * sr)[0][1] > 59 * sr + + +def test_a_long_recording_goes_in_pieces_and_comes_back_whole(db, cfg, fake, monkeypatch): + rows = [] + monkeypatch.setattr(speech, "_ledger", lambda *a: _Ledger(rows, *a)) + settings.save(db, cfg, "speech", {"deepgram_max_minutes": 1}, "t") + eff = provided(db, cfg, fake) + segs = speech.Engine(eff, "deepgram").transcribe(talk(150, (48, 103))) + sent = [b for m, p, h, b in Fake.seen if p.startswith("/dg/v1/listen")] + assert len(sent) == 3 + starts = sorted({s["t0"] for s in segs if s["text"] == "Great work."}) + assert len(starts) == 3 and starts[0] == 0 and 48_000 < starts[1] < 49_000 and 103_000 < starts[2] < 104_000 + assert [s["provider_speaker"] for s in segs][:2] == ["c0:S0", "c0:S1"] + assert {s["provider_speaker"] for s in segs} == {f"c{k}:S{n}" for k in range(3) for n in range(2)} + second = next(s for s in segs if s["provider_speaker"] == "c1:S1") + assert json.loads(second["words"]) == [["Thanks.", starts[1] + 1000, starts[1] + 1500]] + assert [s["t0"] for s in segs] == sorted(s["t0"] for s in segs) + assert rows == [("speech.transcribe", "deepgram:nova-3", {"seconds": 150.0, "pieces": 3})] + + +def test_a_piece_too_large_to_send_is_halved(db, cfg, fake, monkeypatch): + monkeypatch.setattr(speech, "encode", lambda a: (b"\0" * (len(a) // speech.SR * 10_000), "audio.ogg", "audio/ogg")) + settings.save(db, cfg, "speech", {"openai_max_mb": 1}, "t") # 10 KB a second: 100 seconds a request + eff = provided(db, cfg, fake) + segs = speech.Engine(eff, "openai").transcribe(talk(250, (40, 110, 165))) # in half (at 110 s), then each half again + sent = [b for m, p, h, b in Fake.seen if p == "/oai/v1/audio/transcriptions"] + assert len(sent) == 4 and all(len(b) < 1_000_000 for b in sent) + assert [s["text"] for s in segs] == ["Book a call for Friday."] * 4 + assert [s["t0"] // 1000 for s in segs] == [0, 40, 110, 165] + monkeypatch.setattr(speech, "encode", lambda a: (b"\0" * 2_000_000, "audio.ogg", "audio/ogg")) + with pytest.raises(speech.ProviderError, match="raise the limit in Settings"): + speech.Engine(eff, "openai").transcribe(talk(10, ())) + + +class _Ledger: + def __init__(self, rows, action, cfg, model_, detail): + self.rows, self.row = rows, (action, model_, detail) + + def __enter__(self): + return self + + def __exit__(self, *a): + self.rows.append(self.row) + + def test_answers_are_read_by_elevenlabs_or_deepgram(db, cfg, fake): eff = provided(db, cfg, fake) settings.save(db, cfg, "voice", {"tts_provider": "elevenlabs", "tts_voice": "v42"}, "t") diff --git a/nextjs-frontend/__tests__/settings-speech.test.tsx b/nextjs-frontend/__tests__/settings-speech.test.tsx index a20eaae3d..710527b1d 100644 --- a/nextjs-frontend/__tests__/settings-speech.test.tsx +++ b/nextjs-frontend/__tests__/settings-speech.test.tsx @@ -23,15 +23,23 @@ const VIEW = { openai_base_url: "https://api.openai.com/v1", openai_model: "whisper-1", openai_api_key: { secret: true, set: false }, + openai_max_minutes: 10, + openai_max_mb: 25, elevenlabs_base_url: "https://api.elevenlabs.io", elevenlabs_model: "scribe_v1", elevenlabs_api_key: { secret: true, set: false }, + elevenlabs_max_minutes: 600, + elevenlabs_max_mb: 3000, assemblyai_base_url: "https://api.assemblyai.com", assemblyai_model: "universal", assemblyai_api_key: { secret: true, set: false }, + assemblyai_max_minutes: 600, + assemblyai_max_mb: 2200, deepgram_base_url: "https://api.deepgram.com", deepgram_model: "nova-3", deepgram_api_key: { secret: true, set: true }, + deepgram_max_minutes: 120, + deepgram_max_mb: 2000, sentiment: true, timeout: 1800, }, @@ -76,4 +84,19 @@ describe("speech providers", () => { body: { assemblyai_base_url: "https://api.eu.assemblyai.com" }, }); }); + + it("shows how much each transcribing provider takes at once, and saves a smaller limit", async () => { + show(); + const minutes = screen.getAllByLabelText("Longest piece sent at once"); + expect(minutes).toHaveLength(4); // Gemini only reads aloud + expect(minutes.map((x) => (x as HTMLInputElement).value)).toEqual(["10", "600", "600", "120"]); + expect(screen.getAllByLabelText("Largest file sent at once")).toHaveLength(4); + fireEvent.change(minutes[0], { target: { value: "5" } }); + fireEvent.click(screen.getByRole("button", { name: "Save" })); + await waitFor(() => expect(Admin.updateSettings).toHaveBeenCalledTimes(1)); + expect((Admin.updateSettings as jest.Mock).mock.calls[0][0]).toMatchObject({ + path: { section: "speech" }, + body: { openai_max_minutes: 5 }, + }); + }); }); diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 2bad54dfa..0412a9d51 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -2574,6 +2574,12 @@ function SpeechProvidersBody({ ctx }: { ctx: BodyCtx }) {
+ {!("readsOnly" in p) && ( + <> + + + + )}
!("readsOnly" in p)); + export const FIELDS: FieldSpec[] = [ // Transcription { @@ -1561,6 +1565,27 @@ export const FIELDS: FieldSpec[] = [ }, { section: "speech", key: `${p.id}_api_key`, label: `${p.label} API key`, kind: "secret" }, ]), + ...TRANSCRIBING_PROVIDERS.flatMap((p): FieldSpec[] => [ + { + section: "speech", + key: `${p.id}_max_minutes`, + label: "Longest piece sent at once", + kind: "int", + min: 1, + max: 1440, + unit: "minutes", + hint: "A longer recording is cut at pauses and the transcripts joined", + }, + { + section: "speech", + key: `${p.id}_max_mb`, + label: "Largest file sent at once", + kind: "int", + min: 1, + max: 10000, + unit: "MB", + }, + ]), { section: "speech", key: "sentiment",