From 88dcf3d64daff59465f7d1c8d5460797b467d383 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Mon, 19 Oct 2026 14:37:13 +0000 Subject: [PATCH] fix(speakers): never match a voice to someone it was said not to be match() ignored not_same: a voice split off from a person, or turned down as them, could be auto-assigned or suggested as that person again when it sat near their average voice, or when the person it sounded like was already taken in the same recording. Now, of two people said not to be the same, a voice goes to (or is suggested as) only the one it sounds more like, by their voice overall or in any one recording they're in. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 2 +- changelog.d/voice-not-same.md | 5 +++ docs/processing.md | 5 ++- docs/requirements.md | 2 +- fastapi_backend/app/domain/speakers.py | 36 +++++++++++++++++++-- fastapi_backend/tests/domain/test_voices.py | 28 ++++++++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 changelog.d/voice-not-same.md diff --git a/README.md b/README.md index 24635e483..30b91b732 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p SPARQL, and see as it was at any version, where every source, sensor, person and agent is a node linked to what it brought in. Every entity has its own page at a wiki address (`/wiki//`), and entities, topics and people are edited and merged from their lists, the graph or an - asset, with Undo instead of a confirm (and a voice matched to the wrong person split off). Names get their real types (people, places, organisations, events, works) by + asset, with Undo instead of a confirm (and a voice matched to the wrong person split off, and not matched to them again). Names get their real types (people, places, organisations, events, works) by rules, without a model, and a decision model types the ones the rules can't. Find what is most like a passage, an asset, a topic or an entity, in a Similar tab on every asset or by selecting text anywhere, and, with anytopdf, photos by what they show. Search filters moments by person, emotion or the sound heard in them (laughter, applause, diff --git a/changelog.d/voice-not-same.md b/changelog.d/voice-not-same.md new file mode 100644 index 000000000..a31b67e59 --- /dev/null +++ b/changelog.d/voice-not-same.md @@ -0,0 +1,5 @@ +- **A voice isn't given back to someone it was said not to be.** Once a voice was split off from a person (Not them + here) or a suggested pair was marked not the same, a voice in a new or re-diarized asset goes to, or is suggested as, + only the one of the two it sounds more like, by their voice overall or in any one asset they're in. Before, a voice + nearer the first person's average could be matched or suggested as them again + ([People and namespaces](docs/processing.md#people-and-namespaces)). diff --git a/docs/processing.md b/docs/processing.md index e5e740607..f9696f892 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -235,7 +235,10 @@ as the same person (and unlink them), and with `speakers.cross_namespace: sugges as dashed edges and People lists them with how alike the voices are. Saying a suggested pair is "not the same", in one namespace or across two, removes the suggestion for good. When a voice was matched to the wrong person in one asset, **Not them here** on that person's card in the asset's Speakers tab gives its lines there to a new person and works out -the first person's voice again without it, with Undo; the two aren't suggested as the same again. +the first person's voice again without it, with Undo; the two aren't suggested as the same again. Of two people said +not to be the same, either way, a voice in a new asset (or one transcribed again) goes to, or is suggested as, only the +one it sounds more like, by their voice overall or in any one asset they're in. So a voice split off or turned down +isn't given back to the person it was taken from just because it sits near their average voice. **Voice recognition off.** Voiceprints are biometric data. An owner can turn off People › Recognise voices across recordings for their namespace (`PUT /api/v1/namespaces//voices/mode {"mode": "separate"}`; `GET /speakers` says diff --git a/docs/requirements.md b/docs/requirements.md index db158a06b..ddd7b6187 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -199,7 +199,7 @@ request, to another check that proves the same thing; loosening a requirement or | Transcription | [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_an_openai_compatible_server_transcribes`, `test_deepgram_transcribes_with_speakers_and_sentiment` | | Transcription that expects the names the namespace knows (custom vocabulary, named people, defined entities, names people fixed), off by default | [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_the_namespaces_own_names_go_to_the_engine` | | Transcription by a local engine (SenseVoice, Whisper), run in CI | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) | -| One asset's voice split off from the person it was matched to, with Undo | [test_speakers.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_speakers.py) `test_detach_one_assets_voice` | +| One asset's voice split off from the person it was matched to, with Undo, and a voice never matched or suggested again as someone it was said not to be | [test_speakers.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_speakers.py) `test_detach_one_assets_voice`; [test_voices.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_voices.py) `test_a_voice_never_goes_to_someone_it_was_said_not_to_be` | | How sure transcription was of each word kept (Whisper, Deepgram, AssemblyAI, ElevenLabs), words from before still loading, the unsure ones marked in the transcript with how sure, a threshold in Settings (0 for none), and `unsure` words per line over MCP | [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_providers_keep_how_sure_they_were_of_each_word`, `test_whisper_keeps_how_sure_it_was_of_each_word`; [test_split_merge.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_split_merge.py) `test_words_keep_how_sure_transcription_was`, `test_words_from_before_how_sure_was_kept_still_load`; [test_mcp.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_mcp.py) `test_get_transcript_says_which_words_transcription_was_unsure_of`; [recording-split-words.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/recording-split-words.test.ts) `keeps how sure transcription was of each word, and words from before it was kept`, `marks the words below the threshold, and none when it's off`; [recording-unsure-words.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/recording-unsure-words.test.tsx) `are underlined with how sure, and a click still plays the line` | | People recognised by voice; long recordings sent to a speech provider in pieces that fit its limits | [test_voices.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_voices.py) `test_match_suggest_merge_undo_link`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_a_long_recording_goes_in_pieces_and_comes_back_whole`, `test_a_piece_too_large_to_send_is_halved` | | An owner turns voice recognition off for a namespace: voices told apart in each recording only, its voiceprints deleted | [test_voice_mode.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice_mode.py) `test_voices_can_be_told_apart_without_being_recognised` | diff --git a/fastapi_backend/app/domain/speakers.py b/fastapi_backend/app/domain/speakers.py index d0f3af03f..79b8aea6e 100644 --- a/fastapi_backend/app/domain/speakers.py +++ b/fastapi_backend/app/domain/speakers.py @@ -251,18 +251,50 @@ def _update_centroid(db, sid, emb, weight): db.q("UPDATE $r SET embedding = $e, n_obs = $n", r=R("speaker", sid), e=_vec(v / (np.linalg.norm(v) + 1e-9)), n=float(n + weight)) +def _not_them(db, prints, reg): + """{local_label: speakers that voice can't be}. Of two people someone said aren't the same, a voice goes to (or is + suggested as) only the one it sounds more like: their voice overall or in any one recording they're in, so a voice + split off from a person, or turned down as them, isn't given back to them because it sits near their average.""" + ids = {sid for sid, _ in reg} + no = {p for p in _dismissed(db, ids) if p[0] in ids and p[1] in ids} + if not no: + return {} + who = sorted({x for p in no for x in p}) + voices = {sid: [v] for sid, v in reg if sid in who} + for a in db.rows("SELECT speaker, embedding FROM appearance WHERE speaker IN $w AND embedding != NONE", w=who): + voices[a["speaker"]].append(np.asarray(a["embedding"], dtype=np.float64)) + out = {} + for l, (e, _) in prints.items(): + like = {sid: max((float(np.dot(e, v)) for v in vs if v.shape == e.shape), default=None) for sid, vs in voices.items()} + for a, b in no: + if like[a] is None or like[b] is None: + continue + out.setdefault(l, set()).update(x for x, y in ((a, b), (b, a)) if like[x] <= like[y]) + return out + + def match(db, cfg, nid, prints): """prints: {local_label: (embedding, seconds)} -> {local_label: (speaker_id, score, how)}. Greedy one-to-one assignment by similarity: two voices in one recording never map to the - same speaker. Close-but-unsure matches become new speakers with a suggested merge. + same speaker. Close-but-unsure matches become new speakers with a suggested merge. A voice is + never given to, or suggested as, someone a person said it isn't (_not_them). """ hi, lo = cfg["speakers"]["match_threshold"], cfg["speakers"]["review_threshold"] reg = [ (r["id"], np.asarray(r["embedding"], dtype=np.float64)) for r in db.rows("SELECT record::id(id) AS id, embedding FROM speaker WHERE space = $s AND embedding != NONE", s=nid) ] - pairs = sorted(((float(np.dot(e, v)), l, sid) for l, (e, _) in prints.items() for sid, v in reg if v.shape == e.shape), reverse=True) + nope = _not_them(db, prints, reg) + pairs = sorted( + ( + (float(np.dot(e, v)), l, sid) + for l, (e, _) in prints.items() + for sid, v in reg + if v.shape == e.shape and sid not in nope.get(l, ()) + ), + reverse=True, + ) out, used = {}, set() for sim, l, sid in pairs: if sim < hi: diff --git a/fastapi_backend/tests/domain/test_voices.py b/fastapi_backend/tests/domain/test_voices.py index fab74266f..d971b4112 100644 --- a/fastapi_backend/tests/domain/test_voices.py +++ b/fastapi_backend/tests/domain/test_voices.py @@ -55,3 +55,31 @@ def test_channels_and_clustering(): c = rng.normal(size=(3, 64)) E = np.stack([c[i] + 0.05 * rng.normal(size=64) for i in (0, 1, 0, 2, 1)]) assert list(speakers.cluster(E, 0.3)) == [0, 1, 0, 2, 1] + + +def _basis(n, seed=5): + q, _ = np.linalg.qr(np.random.default_rng(seed).normal(size=(192, n))) + return [q[:, i] for i in range(n)] + + +def test_a_voice_never_goes_to_someone_it_was_said_not_to_be(db, cfg): + e1, e2, e3, e4 = _basis(4) + pods = store.ns_id(db, "pods") + sa, sx = speakers.new_speaker(db, pods, emb=e1, weight=40.0), speakers.new_speaker(db, pods, emb=e2, weight=40.0) + # X's voice in one recording, split off from A (or turned down as A): not A's average, nor quite X's + db.q("CREATE appearance CONTENT $a", a={"recording": 1, "speaker": sx, "local_label": "S0", "space": pods, "embedding": list(e3)}) + sure = unit(0.7 * e1 + 0.714 * e3) # above match_threshold for A, nearer that recording than anything of A's + unsure = unit(0.45 * e1 + 0.40 * e2 + 0.6 * e3 + 0.527 * e4) # unsure of A and X both, A's average nearer + before = speakers.match(db, cfg, pods, {"S0": (sure, 1.0)}) + assert before["S0"][0] == sa # nobody said otherwise yet + db.q("UPDATE $r SET embedding = $e, n_obs = 40.0", r=store.R("speaker", sa), e=list(e1)) + speakers.not_same(db, sa, sx) + got = speakers.match(db, cfg, pods, {"S0": (sure, 1.0), "S1": (unsure, 1.0)}) + assert got["S0"][0] not in (sa, sx) and got["S0"][2] == "new" + assert got["S1"][2] == "new" + sug = {s["id"]: [g["id"] for g in s["suggestions"]] for s in speakers.list_speakers(db, pods)} + assert sug[got["S0"][0]] == [] and sug[got["S1"][0]] == [sx] # suggested as X, never as A + # a voice that sounds more like A still goes to A, and two voices nearest X don't spill over to A + assert speakers.match(db, cfg, pods, {"S0": (unit(0.9 * e1 + 0.1 * e3), 1.0)})["S0"][0] == sa + two = speakers.match(db, cfg, pods, {"S0": (e2, 1.0), "S1": (unit(0.8 * e2 + 0.6 * e1), 1.0)}) + assert two["S0"][0] == sx and two["S1"][0] != sa