From 6df54d482ad22cf035653f1531a959171ff8dcc1 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Mon, 19 Oct 2026 15:06:23 +0000 Subject: [PATCH] fix(speakers): carry "not the same" pairs over when merging people Merging B into A now rewrites each not_same pair naming B to name A, keyed and ordered as not_same() makes them, dropping a pair that would be A-A or that A already has; undoing the merge puts the original pairs back. A new upgrade step (speaker-not-same-merged) gives pairs left naming people merged away before this the person they were merged into; it keeps the old rows so undo still works, and is safe to run twice. Closes #687 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- changelog.d/speaker-merge-not-same.md | 4 ++ docs/api.md | 3 +- docs/database.md | 4 ++ docs/processing.md | 3 +- docs/requirements.md | 2 +- fastapi_backend/app/domain/migrations.py | 4 +- fastapi_backend/app/domain/speakers.py | 53 +++++++++++++++++++ .../tests/domain/test_migrations.py | 1 + fastapi_backend/tests/domain/test_voices.py | 41 ++++++++++++++ 9 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 changelog.d/speaker-merge-not-same.md diff --git a/changelog.d/speaker-merge-not-same.md b/changelog.d/speaker-merge-not-same.md new file mode 100644 index 00000000..6d479a71 --- /dev/null +++ b/changelog.d/speaker-merge-not-same.md @@ -0,0 +1,4 @@ +- **A "not the same" decision survives a merge.** Merging one person into another now carries every pair someone + said aren't the same person over to the person merged into, so a voice isn't suggested again for someone it was said + not to be; a pair that would name the person twice is dropped, and undoing the merge puts the pairs back. An upgrade + step (`speaker-not-same-merged`) does the same for merges made before ([Upgrades](docs/database.md#upgrades)). diff --git a/docs/api.md b/docs/api.md index a1ccc895..c154bc7a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1422,7 +1422,8 @@ People (everyone recognised by voice or face) live under `/speakers`. `GET /speakers?ns=` lists the namespace's people (each with its suggested merges), its last merges (with `by`, the `recordings` and `segments` that moved, and `undone_by`/`undone_at`), its links to other namespaces, and `cross`: voices in other shared namespaces you can read that are likely the same person (`score`, best first). `not-same -{with}` says two entries aren't the same person (one namespace or two): the suggestion goes and isn't made again. +{with}` says two entries aren't the same person (one namespace or two): the suggestion goes and isn't made again, +and a merge carries it over to the person merged into. `detach {recording}` says the voice heard as this person in one asset is someone else: its lines there go to a new person (`speaker` in the answer), this person's voice is worked out again from the assets they're still in, and the two are marked not the same; merging the new person back undoes it. It's a 400 when they aren't heard in that asset, or diff --git a/docs/database.md b/docs/database.md index 3a4d32dd..6ae1795b 100644 --- a/docs/database.md +++ b/docs/database.md @@ -127,6 +127,10 @@ The steps so far, in the order they run: a `model`, except a spec's note) get their words kept for search in `generated`, under their template's name ([Pipelines and templates](processing.md#pipelines-and-templates)). Outputs that already have them are left as they are, so it's safe to run twice. +* `speaker-not-same-merged`: pairs of people someone said aren't the same that name a person merged away before + merges carried them over (the `not_same` table) get a pair naming the person they were merged into, following merges + of merges; a pair that would name one person twice is dropped. The old rows stay, so undoing such a merge brings + them back, and a pair already there isn't made again, so it's safe to run twice. To add a step: diff --git a/docs/processing.md b/docs/processing.md index f9696f89..9eacbd56 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -233,7 +233,8 @@ You can rename and merge people in the web app or the CLI (`lens speakers …`), keep who made them and how many assets moved. People in different namespaces are never merged. You can link them as the same person (and unlink them), and with `speakers.cross_namespace: suggest` the graph shows likely voice matches as dashed edges and People lists them with how alike the voices are. Saying a suggested pair is "not the same", in one -namespace or across two, removes the suggestion for good. When a voice was matched to the wrong person in one asset, +namespace or across two, removes the suggestion for good, and it still holds after a merge: merging someone into +another person carries it over to that person (undoing the merge puts it back). When a voice was matched to the wrong person in one asset, **Not them here** on that person's card in the asset's Speakers tab gives its lines there to a new person and works out the first person's voice again without it, with Undo; the two aren't suggested as the same again. Of two people said not to be the same, either way, a voice in a new asset (or one transcribed again) goes to, or is suggested as, only the diff --git a/docs/requirements.md b/docs/requirements.md index ddd7b618..91beed0b 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -201,7 +201,7 @@ request, to another check that proves the same thing; loosening a requirement or | Transcription by a local engine (SenseVoice, Whisper), run in CI | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) | | One asset's voice split off from the person it was matched to, with Undo, and a voice never matched or suggested again as someone it was said not to be | [test_speakers.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_speakers.py) `test_detach_one_assets_voice`; [test_voices.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_voices.py) `test_a_voice_never_goes_to_someone_it_was_said_not_to_be` | | How sure transcription was of each word kept (Whisper, Deepgram, AssemblyAI, ElevenLabs), words from before still loading, the unsure ones marked in the transcript with how sure, a threshold in Settings (0 for none), and `unsure` words per line over MCP | [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_providers_keep_how_sure_they_were_of_each_word`, `test_whisper_keeps_how_sure_it_was_of_each_word`; [test_split_merge.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_split_merge.py) `test_words_keep_how_sure_transcription_was`, `test_words_from_before_how_sure_was_kept_still_load`; [test_mcp.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_mcp.py) `test_get_transcript_says_which_words_transcription_was_unsure_of`; [recording-split-words.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/recording-split-words.test.ts) `keeps how sure transcription was of each word, and words from before it was kept`, `marks the words below the threshold, and none when it's off`; [recording-unsure-words.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/recording-unsure-words.test.tsx) `are underlined with how sure, and a click still plays the line` | -| People recognised by voice; long recordings sent to a speech provider in pieces that fit its limits | [test_voices.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_voices.py) `test_match_suggest_merge_undo_link`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_a_long_recording_goes_in_pieces_and_comes_back_whole`, `test_a_piece_too_large_to_send_is_halved` | +| People recognised by voice; long recordings sent to a speech provider in pieces that fit its limits | [test_voices.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_voices.py) `test_match_suggest_merge_undo_link`, `test_merging_keeps_who_they_are_not`, `test_upgrade_gives_pairs_lost_to_merges_to_who_they_were_merged_into`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_a_long_recording_goes_in_pieces_and_comes_back_whole`, `test_a_piece_too_large_to_send_is_halved` | | An owner turns voice recognition off for a namespace: voices told apart in each recording only, its voiceprints deleted | [test_voice_mode.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice_mode.py) `test_voices_can_be_told_apart_without_being_recognised` | | Text on screen and in scans | [test_video.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video.py) `test_shots_text_search_chat_and_iiif`; [test_documents.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_documents.py) `test_scans_and_images_are_read_by_ocr`; [test_ocr.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_ocr.py) `test_a_video_read_by_doctr_and_without_it` | | Faces, off by default | [test_video.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_video.py) `test_people_on_screen`, `test_shots_text_search_chat_and_iiif`; [test_video.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_video.py) `test_face_modes` | diff --git a/fastapi_backend/app/domain/migrations.py b/fastapi_backend/app/domain/migrations.py index 9e90d005..2a4037ab 100644 --- a/fastapi_backend/app/domain/migrations.py +++ b/fastapi_backend/app/domain/migrations.py @@ -49,7 +49,7 @@ def steps(): """[(name, fn(db))] in the order they run. Append only.""" - from . import access, auth, chat, entities, hierarchy, jobs, mcp_servers, pipelines + from . import access, auth, chat, entities, hierarchy, jobs, mcp_servers, pipelines, speakers return [ # IIIF-only access levels (public, transcript, signed-in, private) become the access setting @@ -68,6 +68,8 @@ def steps(): ("entity-types", entities.migrate_types), # what a model step wrote about a recording before then is found by search too ("generated-search", pipelines.migrate_generated), + # what someone said a person merged away isn't now holds for the one they were merged into + ("speaker-not-same-merged", speakers.migrate_not_same), ] diff --git a/fastapi_backend/app/domain/speakers.py b/fastapi_backend/app/domain/speakers.py index 79b8aea6..a301e457 100644 --- a/fastapi_backend/app/domain/speakers.py +++ b/fastapi_backend/app/domain/speakers.py @@ -472,6 +472,11 @@ def merge(db, src, dst, by=None): patch["name"] = a["name"] moved = sorted({tuple(sorted((dst if x["a"] == src else x["a"], dst if x["b"] == src else x["b"]))) for x in links} - {(dst, dst)}) snap["moved_links"] = [list(x) for x in moved] + # what someone said `src` isn't now holds for `dst`: their pairs name `dst`, kept once each, never `dst` with itself + snap["not_same"] = db.rows("SELECT a, b, by, at FROM not_same WHERE a = $s OR b = $s", s=src) + have = {(r["a"], r["b"]) for r in db.rows("SELECT a, b FROM not_same WHERE a = $s OR b = $s", s=dst)} + kept = _repointed(snap["not_same"], {src: dst}, have) + snap["moved_not_same"] = [[r["a"], r["b"]] for r in kept] mid = db.next_id("merge") stmts = [ "UPDATE segment SET speaker = $dst WHERE speaker = $src", @@ -479,6 +484,7 @@ def merge(db, src, dst, by=None): "UPDATE mentions SET speaker = $dst WHERE speaker = $src", "DELETE suggestion WHERE speaker = $src OR candidate = $src", "DELETE same_as WHERE in = $srcr OR out = $srcr", + "DELETE not_same WHERE a = $src OR b = $src", "DELETE $srcr", "CREATE $mr CONTENT $m", ] @@ -511,6 +517,9 @@ def merge(db, src, dst, by=None): for k, (x, y) in enumerate(moved): stmts.append(f"RELATE $la{k}->same_as->$lb{k}") params.update({f"la{k}": R("speaker", x), f"lb{k}": R("speaker", y)}) + for k, r in enumerate(kept): + stmts.append(f"UPSERT $np{k} CONTENT $nd{k}") + params.update({f"np{k}": _pair(r["a"], r["b"]), f"nd{k}": store.clean(r)}) db.run(stmts, **params) return mid @@ -549,6 +558,12 @@ def undo(db, merge_id, by=None): for k, ln in enumerate(sn["links"]): stmts.append(f"RELATE $oa{k}->same_as->$ob{k}") params.update({f"oa{k}": R("speaker", ln["a"]), f"ob{k}": R("speaker", ln["b"])}) + for k, (x, y) in enumerate(sn.get("moved_not_same", [])): + stmts.append(f"DELETE $mn{k}") + params[f"mn{k}"] = _pair(x, y) + for k, r in enumerate(sn.get("not_same", [])): + stmts.append(f"UPSERT $on{k} CONTENT $od{k}") + params.update({f"on{k}": _pair(r["a"], r["b"]), f"od{k}": store.clean(r)}) if sn["suggestions"]: stmts.append("INSERT INTO suggestion $sugg") params["sugg"] = sn["suggestions"] @@ -628,6 +643,44 @@ def _pair(a, b): return R("not_same", f"{min(a, b)}-{max(a, b)}") +def _repointed(rows, into, have): + """The not-the-same pairs these rows (a, b, by, at) become once each speaker in `into` is the one it maps to: each + pair once with a < b, and none that names one person twice or is already in `have`.""" + out = {} + for r in rows: + x, y = into.get(r["a"], r["a"]), into.get(r["b"], r["b"]) + k = (min(x, y), max(x, y)) + if x != y and k not in have and k not in out: + out[k] = {**r, "a": k[0], "b": k[1]} + return list(out.values()) + + +def migrate_not_same(db): + """Upgrade step: pairs someone said aren't the same person that name a speaker merged away before merges carried + them over name the one they were merged into (through merges of merges), as merge() does now. The old rows stay, + so undoing that merge brings them back; safe to run twice, as a pair already there isn't made again.""" + into = {m["from_id"]: m["into_id"] for m in db.rows("SELECT from_id, into_id FROM merge WHERE undone = false")} + rows = db.rows("SELECT a, b, by, at FROM not_same") + ids = {i for r in rows for i in (r["a"], r["b"])} + alive = set(db.values("SELECT VALUE record::id(id) FROM $rs", rs=[R("speaker", i) for i in sorted(ids)])) if ids else set() + + def now_is(x): + seen = set() + while x not in alive and x in into and x not in seen: + seen.add(x) + x = into[x] + return x + + gone = [r for r in rows if not {r["a"], r["b"]} <= alive] + kept = [r for r in _repointed(gone, {i: now_is(i) for i in ids - alive}, {(r["a"], r["b"]) for r in rows}) if {r["a"], r["b"]} <= alive] + if kept: + db.run( + [f"UPSERT $p{k} CONTENT $d{k}" for k in range(len(kept))], + **{f"p{k}": _pair(r["a"], r["b"]) for k, r in enumerate(kept)}, + **{f"d{k}": store.clean(r) for k, r in enumerate(kept)}, + ) + + def not_same(db, a, b, by=None): """Someone says these two speakers aren't the same person (in one namespace or two): drop the suggestion and never make it again.""" diff --git a/fastapi_backend/tests/domain/test_migrations.py b/fastapi_backend/tests/domain/test_migrations.py index d236e49f..af5b0cb4 100644 --- a/fastapi_backend/tests/domain/test_migrations.py +++ b/fastapi_backend/tests/domain/test_migrations.py @@ -23,6 +23,7 @@ "mcp-sealed-secrets", "entity-types", "generated-search", + "speaker-not-same-merged", ] diff --git a/fastapi_backend/tests/domain/test_voices.py b/fastapi_backend/tests/domain/test_voices.py index d971b411..af4cd117 100644 --- a/fastapi_backend/tests/domain/test_voices.py +++ b/fastapi_backend/tests/domain/test_voices.py @@ -42,6 +42,47 @@ def test_match_suggest_merge_undo_link(db, cfg): speakers.undo(db, mid) # only once +def test_merging_keeps_who_they_are_not(db): + pods, calls = store.ns_id(db, "pods"), store.ns_id(db, "calls") + a, b, c, d = (speakers.new_speaker(db, pods) for _ in range(4)) + e = speakers.new_speaker(db, calls) + speakers.not_same(db, b, c, by="ann@example.com") # moves to a-c + speakers.not_same(db, b, a) # would be a-a: dropped + speakers.not_same(db, b, d) # a-d is already there: kept once + speakers.not_same(db, a, d) + speakers.not_same(db, e, b) # across namespaces too + before = db.rows("SELECT a, b, by FROM not_same ORDER BY a, b") + + def pairs(): + return sorted((r["a"], r["b"]) for r in db.rows("SELECT a, b FROM not_same")) + + mid = speakers.merge(db, b, a) + assert pairs() == sorted([(min(a, c), max(a, c)), (min(a, d), max(a, d)), (min(a, e), max(a, e))]) + assert db.one("SELECT by FROM $p", p=speakers._pair(a, c))["by"] == "ann@example.com" + ids = db.rows("SELECT record::id(id) AS k, a, b FROM not_same") + assert all(r["k"] == f"{r['a']}-{r['b']}" for r in ids) # keyed as not_same() keys them + speakers.undo(db, mid) + assert db.rows("SELECT a, b, by FROM not_same ORDER BY a, b") == before + + +def test_upgrade_gives_pairs_lost_to_merges_to_who_they_were_merged_into(db): + pods = store.ns_id(db, "pods") + a, b, c, x = (speakers.new_speaker(db, pods) for _ in range(4)) + speakers.not_same(db, b, c) + speakers.not_same(db, b, a) + speakers.not_same(db, x, c) + # as merges left it before they carried pairs over: b went into x, then x into a + for src, dst in ((b, x), (x, a)): + db.q("CREATE $r CONTENT $m", r=store.R("merge", db.next_id("merge")), m={"from_id": src, "into_id": dst, "undone": False}) + db.q("DELETE $r", r=store.R("speaker", src)) + speakers.migrate_not_same(db) + once = sorted((r["a"], r["b"]) for r in db.rows("SELECT a, b FROM not_same")) + assert (min(a, c), max(a, c)) in once and (min(a, b), max(a, b)) in once # the old rows stay, for undo + assert len(once) == 4 # b-c and x-c both became a-c, b-a became nothing + speakers.migrate_not_same(db) + assert sorted((r["a"], r["b"]) for r in db.rows("SELECT a, b FROM not_same")) == once + + def test_channels_and_clustering(): rng = np.random.default_rng(0) x = rng.normal(scale=0.01, size=(48000, 2)).astype(np.float32)