From 07170f032a9e0d27aa5f200db67b9538d9c903fe Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 08:17:12 +0000 Subject: [PATCH] feat(search): find and filter moments by sound events like laughter or applause Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QFUNzy1QZy5HhKAyJxTAYJ --- README.md | 13 ++-- changelog.d/sound-events.md | 5 ++ docs/api.md | 15 ++-- docs/mcp.md | 2 +- docs/processing.md | 6 +- docs/requirements.md | 1 + fastapi_backend/app/api/mcp_tools.py | 12 ++- fastapi_backend/app/api/v1/routes/search.py | 8 +- fastapi_backend/app/api/v1/routes/searches.py | 4 +- fastapi_backend/app/domain/search.py | 47 ++++++++---- fastapi_backend/app/domain/store.py | 9 +++ fastapi_backend/app/schemas/search.py | 7 +- fastapi_backend/app/schemas/views.py | 2 + fastapi_backend/tests/api/test_search.py | 28 +++++++ fastapi_backend/tests/api/test_semantic.py | 10 +++ nextjs-frontend/__tests__/search.test.ts | 25 +++++++ nextjs-frontend/app/openapi-client/sdk.gen.ts | 6 +- .../app/openapi-client/types.gen.ts | 35 ++++++++- .../components/search/facet-panel.tsx | 3 +- nextjs-frontend/components/search/facets.ts | 4 + .../components/search/no-results.tsx | 13 ++++ nextjs-frontend/components/search/query.ts | 32 ++++++-- .../components/search/save-search.tsx | 8 ++ .../components/search/search-page.tsx | 7 +- .../components/search/syntax-help.tsx | 1 + nextjs-frontend/openapi.json | 73 +++++++++++++++++-- 26 files changed, 316 insertions(+), 60 deletions(-) create mode 100644 changelog.d/sound-events.md diff --git a/README.md b/README.md index 6d2f0ba41..336aa9245 100644 --- a/README.md +++ b/README.md @@ -172,12 +172,13 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p asset, with Undo instead of a confirm (and a voice matched to the wrong person split off). Names get their real types (people, places, organisations, events, works) by rules, without a model, and a decision model types the ones the rules can't. Find what is most like a passage, an asset, a topic or an entity, in a Similar tab on every asset or by selecting text anywhere, and, with anytopdf, photos - by what they show. Pages and shots are described in words by the model provider's vision model or, off by default, by - Florence-2 or Moondream on your own server. A recording's sound can be cleaned up before it's transcribed (noise, - volume, echo, and silence left out, with the transcript's times kept), and transcripts cleaned up before analysis - (stutters, "um"s, spacing and, with a local model, punctuation put back, and grammar by a LanguageTool server you - run), entities found by spaCy instead of the rules, and the dates people say read as days on the calendar, each off - by default. + by what they show. Search filters moments by person, emotion or the sound heard in them (laughter, applause, + music), with a count for each. Pages and shots are described in words by the model provider's vision model or, off by + default, by Florence-2 or Moondream on your own server. A recording's sound can be cleaned up before it's transcribed + (noise, volume, echo, and silence left out, with the transcript's times kept), and transcripts cleaned up before + analysis (stutters, "um"s, spacing and, with a local model, punctuation put back, and grammar by a LanguageTool server + you run), entities found by spaCy instead of the rules, and the dates people say read as days on the calendar, each + off by default. ([Graph](docs/graph.md), [entity pages](docs/notes.md#entity-pages), [similar content](docs/processing.md#similar-content), [graph history](docs/graph-history.md), [topics](docs/topics.md), [video](docs/video.md), [NSFW classification](docs/nsfw.md), diff --git a/changelog.d/sound-events.md b/changelog.d/sound-events.md new file mode 100644 index 000000000..6118972ac --- /dev/null +++ b/changelog.d/sound-events.md @@ -0,0 +1,5 @@ +- **Search finds moments by the sounds in them.** Laughter, applause, music and the other sounds a speech provider + hears are a search filter like emotion: type `event:laughter` (any case; `music` is the BGM tag), pick one under + Sounds in the filters, or pass `event` to `GET /api/v1/search`, a saved search or the MCP `search` tool. Each hit + says its sound, and the facets count them (`events`), leaving plain speech out ([API](docs/api.md#search), + [search](docs/processing.md#search)). diff --git a/docs/api.md b/docs/api.md index fa519d447..50f5384a5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1252,11 +1252,12 @@ GET /api/v1/mentions supplementary transcripts, captions, translations and indexes, or in what a model says its shots and pages show) in the namespaces you can read, best first: every word (English stemming), "quoted phrases" as written, `OR` between alternatives, `NOT` or `-` before a word or phrase to leave out the lines -with it; `ns`, `speaker`, `emotion` and `recording` -narrow it, and `object` keeps to the assets a kind of object is seen in ([Objects](#objects)); +with it; `ns`, `speaker`, `emotion`, `event` (a sound heard, such as laughter, applause or music, typed any way; music +is stored as `BGM`) and `recording` narrow it, and `object` keeps to the assets a kind of object is seen in +([Objects](#objects)); `limit`/`offset` page through it. `total` counts the moments ranked so far (`capped` when there may be more). -With `facets=true` it also counts all the matching moments, whatever the page, by namespace, person, emotion and -asset (`facets`: up to 50 values each, most first, and `moments`), and lists the kinds of object seen in the +With `facets=true` it also counts all the matching moments, whatever the page, by namespace, person, emotion, sound +(`events`; plain speech isn't one) and asset (`facets`: up to 50 values each, most first, and `moments`), and lists the kinds of object seen in the assets they're in, with how many of those assets each is in (`objects`); past 20,000 moments the counts cover 20,000 of them (`partial`). @@ -1269,8 +1270,8 @@ image's text: its `page`, from 0, and its `box` on it), `object` (a kind of obje on a document's pages its `page`, with the `frame` and the `box` it's best seen in), `described` (what a shot or a page shows, [Descriptions](#descriptions): the shot's time, or the `page`, and its `frame`) or `file` (a line of a supplementary file: its `file`, `file_role` and `file_label`, and which `line`). A file's lines have no `t0` when the -file doesn't say when they are; the web app opens those in the resource's Files tab. A `speaker` or `emotion` filter -keeps to `said` hits. +file doesn't say when they are; the web app opens those in the resource's Files tab. A `speaker`, `emotion` or `event` +filter keeps to `said` hits, and each hit says its `emotion` and `event`. `mode` says how the words are matched ([search by meaning](processing.md#search-by-meaning)): `keyword` (BM25), `semantic` (passages an embedding model finds alike), `hybrid` (both, fused by rank) or `auto` (the default: hybrid @@ -1782,7 +1783,7 @@ DELETE /api/v1/searches/{sid} ``` Saved searches: `POST` with a `name` (unique among yours), the search's `q`, and any of `namespace`, `speaker`, -`emotion` and `recording`, as `GET /search` takes them. They follow the rules of saved views (above): yours, or +`emotion`, `event` and `recording`, as `GET /search` takes them. They follow the rules of saved views (above): yours, or `shared` with the namespace they search (editors there; a search of every namespace can't be shared); only their maker renames or shares them (`PATCH` with `name` or `shared`); their maker, or an owner of the namespace for a shared one, deletes them. `GET` lists yours, then the shared ones, with `speaker_name` and `recording_title` where you can read diff --git a/docs/mcp.md b/docs/mcp.md index 4cd642696..273bbcc09 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -69,7 +69,7 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | Tool | | |---|---| | `lens_help` | Lens itself rather than the archive: the sections of Lens's docs that match a question (or one page read whole), each with its link, and this server's own facts: version, addresses (web app, MCP, API), what is turned on and your access ([Questions about Lens itself](assistant.md#questions-about-lens-itself)) | -| `search` | moments where words appear (written, spoken or shown on screen), best first: the asset, person, time, the matching text (matches in **bold**) and a link to that moment. Words, `"phrases"`, `OR`; by namespace, person or asset | +| `search` | moments where words appear (written, spoken or shown on screen), best first: the asset, person, time, the matching text (matches in **bold**) and a link to that moment. Words, `"phrases"`, `OR`; by namespace, person, asset or sound heard (laughter, applause, music …) | | `find_similar` | the assets (recordings, documents and images) most like a text, an asset, a topic or an entity: why each is alike (meaning, key words, topics, entities in common) and the moments that show it, with links | | `find` | assets by any attribute: faces seen, people heard, entities, topics, kind, content type, dates (a day, month or year), tags, language, objects, emotions, length, collection and words in them; ids or names. Each says why it matched, with moments and links to cite ([Finding by any attribute](processing.md#finding-by-any-attribute)) | | `classify` | which of some labels fits an asset or a text, how sure, and every label ranked: labels you give, or Lens's content types or the namespace's topics; the decision model answers, else the language model. Read only: it changes nothing ([Classifying](processing.md#classifying)) | diff --git a/docs/processing.md b/docs/processing.md index 32539b822..999ee4185 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -375,7 +375,7 @@ about them come back as not found. All words must appear, matched after English stemming ("exploit" also finds exploits and exploiting); "quoted phrases" must appear as written; OR separates alternatives; NOT, or a `-` before a word or "phrase", leaves out the lines that have it (`budget -draft` finds budget without draft or drafts, `budget NOT "first draft"`), and a typed AND changes -nothing. Filter by namespace, person, emotion or asset. The words are +nothing. Filter by namespace, person, emotion, sound heard (`event:laughter`, applause, music …) or asset. The words are looked up in the index of `search.engine`: a SQLite FTS5 file by default, SurrealDB's own index, or OpenSearch ([configuration](configuration.md#word-search)). For archives that aren't in English set `search.stemming: none` and run `lens reindex`. There is no prefix search: a trailing `*` (`expl*`) is ignored and the word is searched as it is; @@ -399,8 +399,8 @@ A search is matched three ways (`mode` on `GET /search`, the Match switch in the * **keyword**: only the words, as above. * **semantic**: only by meaning. -A passage found by meaning is shown at its line that best fits (the person or emotion filtered on, else the one with -most of the query's words), marked Related, with how alike it is (`similarity`, cosine). One that holds a keyword hit +A passage found by meaning is shown at its line that best fits (the person, emotion or sound filtered on, else the one +with most of the query's words), marked Related, with how alike it is (`similarity`, cosine). One that holds a keyword hit adds to that hit's rank instead of showing twice. Only passages at least `embeddings.min_similarity` alike count, and none much further than the closest one, so an unrelated query finds nothing rather than whatever is least unlike it. Facets count the moments found by meaning too. The assistant's search tool and chat's retrieval use the same passages, diff --git a/docs/requirements.md b/docs/requirements.md index ff69c1720..f08ade24d 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -218,6 +218,7 @@ request, to another check that proves the same thing; loosening a requirement or | Names get their real types (people, places, organisations, events, works) by rules, without a model; older Terms are typed on upgrade | [test_name_types.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_name_types.py) `test_names_get_the_type_their_words_give`, `test_analysis_types_people_places_and_events`, `test_the_upgrade_types_the_old_terms_and_keeps_what_people_chose` | | A decision model (Jev or Laya) types the names the rules leave as Terms, in one call, keeping only types it is sure of; Settings › Analysis can leave them Terms | [test_name_types.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_name_types.py) `test_the_decision_model_types_what_the_rules_cant`, `test_a_type_the_model_isnt_sure_of_stays_a_term`, `test_by_default_the_language_model_isnt_asked`, `test_with_typing_rules_nothing_is_asked` | | What is most like a passage, an asset, a topic or an entity | [test_similar.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_similar.py) `test_like_a_text_by_its_key_words`, `test_like_a_resource_by_words_topics_and_entities`, `test_like_a_topic_and_an_entity`; [similar.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/similar.test.tsx) | +| Search finds and filters moments by the sound heard in them (laughter, applause, music), typed any way, and counts each | [test_search.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_search.py) `test_sound_events_filter_and_count`; [test_semantic.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_semantic.py) `test_search_by_meaning_keeps_to_a_sound`; [search.test.ts](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/search.test.ts) `reads sound events typed any way` | | With anytopdf, photos found by what they show | [test_photos.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_photos.py) `test_photos_found_by_what_they_show`, `test_off_by_default` | | Pages and shots described by the provider's vision model or, off by default, by Florence-2 on this server, fetched pinned and checked when picked | [test_florence.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_florence.py) `test_florence_describes_pictures_without_a_model_provider`, `test_florence_is_fetched_pinned_and_checked`, `test_admins_and_agents_choose_what_describes_pictures`, `test_a_picture_is_read_as_the_model_was_trained` | | Or by Moondream on this server, off by default, fetched at a pinned revision only where the chosen model can run, and saying why where it can't | [test_moondream.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_moondream.py) `test_moondream_describes_pictures_without_a_model_provider`, `test_where_each_moondream_can_run`, `test_on_a_cpu_moondream_3_is_never_fetched_and_the_step_says_why`, `test_moondream_is_fetched_pinned`, `test_admins_and_agents_choose_moondream` | diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 4e298e9d7..53e42ad4b 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -462,11 +462,19 @@ def _recording(ctx: Context, rid: int) -> dict[str, Any]: Arg("namespace", "string", "only this namespace (see list_namespaces)"), Arg("speaker_id", "integer", "only lines from this person (ids from list_speakers or get_recording)"), Arg("recording_id", "integer", "only within this asset"), + Arg("event", "string", "only lines with this sound: laughter, applause, music, cough …", max_length=40), Arg("limit", "integer", "how many moments", default=10, minimum=1, maximum=50), Arg("offset", "integer", "skip this many (for the next page)", default=0, minimum=0, maximum=10000), ) def search( - ctx: Context, query: str, namespace: str | None, speaker_id: int | None, recording_id: int | None, limit: int, offset: int + ctx: Context, + query: str, + namespace: str | None, + speaker_id: int | None, + recording_id: int | None, + event: str | None, + limit: int, + offset: int, ) -> dict[str, Any]: if recording_id is not None: _recording(ctx, recording_id) @@ -479,6 +487,7 @@ def search( ns=namespace, speaker=speaker_id, emotion=None, + event=event, recording=recording_id, object=None, limit=limit, @@ -497,6 +506,7 @@ def search( "recorded_at": h.recorded_at, "speaker": h.speaker, "speaker_id": h.speaker_id, + "event": h.event, "found": h.source, "at": store.tc(h.t0) if h.t0 is not None else None, "seconds": _seconds(h.t0), diff --git a/fastapi_backend/app/api/v1/routes/search.py b/fastapi_backend/app/api/v1/routes/search.py index e70179a7e..6d59181cd 100644 --- a/fastapi_backend/app/api/v1/routes/search.py +++ b/fastapi_backend/app/api/v1/routes/search.py @@ -28,13 +28,14 @@ def search_transcripts( ns: str | None = None, speaker: int | None = None, emotion: str | None = None, + event: str | None = Query(None, max_length=40, description="only lines with this sound (laughter, applause, music …), ignoring case"), recording: int | None = None, object: str | None = Query(None, max_length=60, description="only assets this kind of object is seen in (person, car …)"), limit: int = Query(50, ge=1, le=200), offset: int = Query(0, ge=0), facets: bool = Query( False, - description="also count all the matching moments by namespace, person, emotion and asset, and list their kinds of object", + description="also count all the matching moments by namespace, person, emotion, sound and asset, and list their kinds of object", ), mode: Literal["auto", "keyword", "semantic", "hybrid"] = Query( "auto", @@ -46,8 +47,8 @@ def search_transcripts( """Moments where the words appear (said in audio or video, shown on screen in a video, written in a document or in a resource's supplementary transcripts, captions, translations and indexes, or the kinds of object seen in videos, documents and images), - best first, in the namespaces you can read and the collections you were given a role on. A person or emotion - filter keeps to their lines. By meaning, a moment is a passage about what was asked, though its words may differ.""" + best first, in the namespaces you can read and the collections you were given a role on. A person, emotion or + sound filter keeps to their lines. By meaning, a moment is a passage about what was asked, though its words may differ.""" if ns: acl.scope(ns) # 404 unless they see some of it also = acl.partial_recordings() @@ -69,6 +70,7 @@ def search_transcripts( described=True, cfg=cfg, mode=mode, + event=event, ) blur = nsfw.blurred(db, cfg, [h.get("recording_id") for h in res.get("hits") or []]) for h in res.get("hits") or []: diff --git a/fastapi_backend/app/api/v1/routes/searches.py b/fastapi_backend/app/api/v1/routes/searches.py index 5d7ad28f3..57b69fd52 100644 --- a/fastapi_backend/app/api/v1/routes/searches.py +++ b/fastapi_backend/app/api/v1/routes/searches.py @@ -67,14 +67,14 @@ def list_searches(user: CurrentUser, acl: Acl, db: Db) -> list[SavedSearch]: @router.post("") def create_search(body: SearchCreate, user: Writer, acl: Acl, db: Db) -> SavedSearch: - """Save a search (its words, namespace, person, emotion, asset and object) under a name unique among yours. Sharing it + """Save a search (its words, namespace, person, emotion, sound, asset and object) under a name unique among yours. Sharing it with its namespace needs editor access there.""" sid = acl.namespace(body.namespace) if body.namespace else None if body.recording is not None: acl.recording(body.recording) if body.shared: check_share(acl, sid, "saved search") - state = body.model_dump(include={"q", "speaker", "emotion", "recording", "object"}) + state = body.model_dump(include={"q", "speaker", "emotion", "event", "recording", "object"}) with domain_errors(): vid = views.create(db, user.id, body.name, sid, state, body.shared, kind="search") if body.shared: diff --git a/fastapi_backend/app/domain/search.py b/fastapi_backend/app/domain/search.py index 3364e211d..5019f1bd2 100644 --- a/fastapi_backend/app/domain/search.py +++ b/fastapi_backend/app/domain/search.py @@ -109,14 +109,15 @@ def search( described=False, cfg=None, mode="keyword", + event=None, ): """Transcript lines (and, unless screen is false, text on screen in videos; with files, the lines of supplementary transcripts, captions, translations and indexes; with objects, the kinds of object seen in videos, documents and images; with described, what a model that can see said their pages and shots show) matching q. spaces limits the search to namespaces someone may read, and `also` adds recordings they may read beyond those (in collections they were given a role on); recordings limits it to a set of recordings (such as - the transcripts a visitor may read), and obj to those a kind of object is seen in. With facets, also how many of - all the matching moments (up to FACET_CAP) are in each namespace, speaker, emotion and recording, and which kinds - of object the recordings they're in have. + the transcripts a visitor may read), and obj to those a kind of object is seen in. event keeps to lines with that + sound (Laughter, Applause, BGM …). With facets, also how many of all the matching moments (up to FACET_CAP) are in + each namespace, speaker, emotion, sound and recording, and which kinds of object the recordings they're in have. `mode` is how the query is matched: "keyword" (its words, through the BM25 index), "semantic" (by meaning: the passages an embedding model finds most alike, semantic.py), "hybrid" (both, fused by reciprocal rank) or "auto" @@ -131,6 +132,7 @@ def search( recordings = having if recordings is None else set(recordings) & having if not groups or (recordings is not None and not recordings): return empty + event = store.norm_event(event) filt, params = [], {"m0": M0, "m1": M1} if ns: try: @@ -146,14 +148,19 @@ def search( if recordings is not None: filt.append("recording IN $recs") params["recs"] = sorted(recordings) - for cond, key, val in (("speaker = $spk", "spk", speaker), ("emotion = $emo", "emo", emotion), ("recording = $rec", "rec", recording)): + for cond, key, val in ( + ("speaker = $spk", "spk", speaker), + ("emotion = $emo", "emo", emotion), + ("event = $evt", "evt", event), + ("recording = $rec", "rec", recording), + ): if val not in (None, ""): filt.append(cond) params[key] = int(val) if key in ("spk", "rec") else val where_f = (" AND " + " AND ".join(filt)) if filt else "" base_params = dict(params) cap = min(1000, (offset + limit) * 3 + 50) - fields = "record::id(id) AS id, recording, idx, t0, t1, emotion, speaker, space, text, page, box" + fields = "record::id(id) AS id, recording, idx, t0, t1, emotion, event, speaker, space, text, page, box" rows = [] if used == "semantic" else textindex.rows(db, "segment", groups, fields, where_f, params, cap, (M0, M1)) if rows is None and db.ready_fulltext(): conds, sel = [], [] @@ -189,9 +196,9 @@ def search( hits.append(r) for h in hits: # a document's or an image's text is on its pages h["source"] = "said" if h.get("page") is None else "page" - alone = not speaker and not emotion + alone = not speaker and not emotion and not event if used != "semantic": - if screen and alone: # text shown on screen in videos has no speaker or emotion + if screen and alone: # text shown on screen in videos has no speaker, emotion or sound hits += _screen(db, groups, space_filter(ns, spaces, recording, params), cap, params) if files and alone: # nor do the lines of files (a speaker there is just a label) hits += _file_lines(db, groups, space_filter(ns, spaces, recording, params), cap, params) @@ -204,12 +211,14 @@ def search( if used != "keyword": kinds = {"said", "page"} | ({"described"} if described and alone else set()) try: - meant = _meaning(db, cfg, q, groups, kinds, offset + limit, base_params, ns, spaces, recording, speaker, emotion) + meant = _meaning(db, cfg, q, groups, kinds, offset + limit, base_params, ns, spaces, recording, speaker, emotion, event) except semantic.EmbedError as e: log.warning("search by meaning is unavailable: %s", e) note = "search by meaning is unavailable right now, so these match the words" if used == "semantic": # it was to be by meaning alone: by the words instead - args = dict(ns=ns, speaker=speaker, emotion=emotion, recording=recording, limit=limit, offset=offset, spaces=spaces) + args = dict( + ns=ns, speaker=speaker, emotion=emotion, event=event, recording=recording, limit=limit, offset=offset, spaces=spaces + ) more = dict(recordings=recordings, screen=screen, facets=facets, also=also, files=files, objects=objects, obj=obj) return search(db, q, **args, **more, described=described) | {"meaning": note} used = "keyword" @@ -257,6 +266,7 @@ def search( "t0": h.get("t0"), "t1": h.get("t1"), "emotion": h.get("emotion"), + "event": h.get("event"), "speaker_id": h.get("speaker"), "speaker": spk.get(h.get("speaker")), "title": recs.get(h["recording"], {}).get("title"), @@ -321,9 +331,10 @@ def _mode(cfg, mode, groups, can): return ("hybrid" if mode == "auto" else mode), None -def _meaning(db, cfg, q, groups, kinds, need, base, ns, spaces, recording, speaker, emotion): +def _meaning(db, cfg, q, groups, kinds, need, base, ns, spaces, recording, speaker, emotion, event=None): """The passages most like the query, as hits shaped like the keyword ones (best first), each at the line of its - passage that best matches: the speaker or emotion asked for, else the one with most of the query's words.""" + passage that best matches: the speaker, emotion or sound asked for, else the one with most of the query's words. + Passages don't record their sounds, so a sound is matched on their lines.""" params = {k: v for k, v in base.items() if k in ("sp", "allowed", "also", "rec", "recs", "spk", "emo")} where = space_filter(ns, spaces, recording, params) if "recs" in params: @@ -340,7 +351,7 @@ def _meaning(db, cfg, q, groups, kinds, need, base, ns, spaces, recording, speak { (r["recording"], r["idx"]): r for r in db.rows( - "SELECT record::id(id) AS id, recording, idx, t0, t1, emotion, speaker, space, text, page, box FROM segment WHERE id IN $ids", + "SELECT record::id(id) AS id, recording, idx, t0, t1, emotion, event, speaker, space, text, page, box FROM segment WHERE id IN $ids", ids=[store.R("segment", p["recording"] * store.SEG + i) for p in lined for i in range(p["idx0"], p["idx1"] + 1)], ) } @@ -366,9 +377,11 @@ def _meaning(db, cfg, q, groups, kinds, need, base, ns, spaces, recording, speak fits = [ s for s in lines - if (speaker in (None, "") or s.get("speaker") == int(speaker)) and (emotion in (None, "") or s.get("emotion") == emotion) + if (speaker in (None, "") or s.get("speaker") == int(speaker)) + and (emotion in (None, "") or s.get("emotion") == emotion) + and (event in (None, "") or s.get("event") == event) ] - if not fits: # none of its lines is by that speaker or in that mood (any more) + if not fits: # none of its lines is by that speaker, in that mood or with that sound (any more) continue best = max(fits, key=lambda s: len(rx.findall(s["text"].lower())) if rx else 0) out.append( @@ -483,10 +496,10 @@ def _matches(db, groups, table, fields, where_f, base): def _facets(db, groups, where_f, base, screen, ns, spaces, recording, files=False, objects=False, described=False, meant=(), words=True): - """How many matching moments are in each namespace, speaker, emotion and recording, most first; and the kinds of + """How many matching moments are in each namespace, speaker, emotion, sound and recording, most first; and the kinds of object seen in the recordings they're in, with how many of those recordings each is in. `meant`: the moments found by meaning alone, counted too; without `words`, only they are.""" - said = _matches(db, groups, "segment", "recording, space, speaker, emotion", where_f, base) if words else [] + said = _matches(db, groups, "segment", "recording, space, speaker, emotion, event", where_f, base) if words else [] said += [m for m in meant if m["source"] != "described"] seen = _matches(db, groups, "ocr_span", "recording, space", space_filter(ns, spaces, recording, base), base) if screen and words else [] filed = ( @@ -508,6 +521,7 @@ def _facets(db, groups, where_f, base, screen, ns, spaces, recording, files=Fals by_space, by_rec = Counter(r["space"] for r in rows), Counter(r["recording"] for r in rows) by_spk = Counter(r["speaker"] for r in rows if r.get("speaker")) by_emo = Counter(r["emotion"] for r in rows if r.get("emotion") and r["emotion"] != "Unknown") + by_evt = Counter(r["event"] for r in rows if r.get("event") and r["event"] != "Speech") # plain speech isn't a sound top = lambda c: c.most_common()[:FACET_VALUES] # noqa: E731 space_names = store.space_names(db) spk = top(by_spk) @@ -556,6 +570,7 @@ def _facets(db, groups, where_f, base, screen, ns, spaces, recording, files=Fals "namespaces": order([{"name": space_names.get(k) or str(k), "count": n} for k, n in top(by_space)], "name"), "speakers": order(speakers, "name"), "emotions": order([{"name": k, "count": n} for k, n in top(by_emo)], "name"), + "events": order([{"name": k, "count": n} for k, n in top(by_evt)], "name"), "recordings": order([{"id": k, "title": titles.get(k), "count": n} for k, n in recs], "id"), } diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index ba714f645..23bda0a1d 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -106,6 +106,15 @@ def norm_emotion(v): return c if c in EMOTIONS else ALIASES.get(c) +def norm_event(v): + """A sound as segments hold it (Laughter, BGM …), however it's typed; music is BGM. Unknown ones are kept as given.""" + s = str(v or "").strip() + if not s: + return None + known = {e.casefold(): e for e in SV_EVENTS} | {"music": "BGM"} + return known.get(s.casefold(), s) + + def labels(): return {"palette": PALETTE, "emoji": EMOJI, "events": EVENT_EMOJI, "speakers": SPEAKER_COLORS} diff --git a/fastapi_backend/app/schemas/search.py b/fastapi_backend/app/schemas/search.py index 8ed258647..fa8297753 100644 --- a/fastapi_backend/app/schemas/search.py +++ b/fastapi_backend/app/schemas/search.py @@ -14,6 +14,7 @@ class SearchHit(ResponseModel): t0: int | None = Field(description="ms; None for a line of a file that doesn't say when its lines are") t1: int | None emotion: str | None = None + event: str | None = Field(None, description="the sound heard with it (Laughter, Applause, BGM for music …), if any") speaker_id: int | None = None speaker: str | None = None title: str | None = None @@ -67,14 +68,16 @@ class ObjectFacet(ResponseModel): class SearchFacets(ResponseModel): - """How many of all the matching moments are in each namespace, person, emotion and asset (up to 50 of each, - most first), and the kinds of object seen in the assets they're in. Text on screen has no person or emotion.""" + """How many of all the matching moments are in each namespace, person, emotion, sound and asset (up to 50 of + each, most first), and the kinds of object seen in the assets they're in. Text on screen has no person, emotion or + sound.""" moments: int = Field(description="the matching moments counted: all of them, unless `partial`") partial: bool = Field(description="more than 20,000 moments match; the counts cover 20,000 of them") namespaces: list[FacetCount] = Field(default_factory=list) speakers: list[SpeakerFacet] = Field(default_factory=list) emotions: list[FacetCount] = Field(default_factory=list) + events: list[FacetCount] = Field(default_factory=list, description="sounds heard (Laughter, Applause, BGM …); plain speech isn't one") recordings: list[RecordingFacet] = Field(default_factory=list) objects: list[ObjectFacet] = Field(default_factory=list) diff --git a/fastapi_backend/app/schemas/views.py b/fastapi_backend/app/schemas/views.py index a0a6a5631..2c802dbf0 100644 --- a/fastapi_backend/app/schemas/views.py +++ b/fastapi_backend/app/schemas/views.py @@ -75,6 +75,7 @@ class SearchState(RequestModel): q: str = Field(min_length=1, max_length=500, description='words, "phrases", OR between alternatives, NOT or -word to leave out') speaker: int | None = None emotion: str | None = Field(None, max_length=40) + event: str | None = Field(None, max_length=40, description="a sound heard (laughter, applause, music …)") recording: int | None = None object: str | None = Field(None, max_length=60, description="a kind of object the assets have (person, car …)") @@ -88,6 +89,7 @@ class SavedSearch(ResponseModel): speaker: int | None = None speaker_name: str | None = Field(None, description="the person's name, when you can read their namespace") emotion: str | None = None + event: str | None = None recording: int | None = None recording_title: str | None = Field(None, description="the asset's title, when you can read it") object: str | None = Field(None, description="a kind of object the assets have") diff --git a/fastapi_backend/tests/api/test_search.py b/fastapi_backend/tests/api/test_search.py index adf5fd324..881cb5356 100644 --- a/fastapi_backend/tests/api/test_search.py +++ b/fastapi_backend/tests/api/test_search.py @@ -101,6 +101,34 @@ def test_facets_count_every_match(client, db, cfg, folder, monkeypatch): assert (capped["moments"], capped["partial"]) == (2, True) +def test_sound_events_filter_and_count(client, db, cfg, folder): + from tests.api.test_mcp import _bearer, _grant, tool + + seed(db, cfg, folder) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + hv = login(client, "vi@x.io", "viewer password 1") + pods = store.ns_id(db, "pods") + lines = db.rows("SELECT record::id(id) AS id, idx, recording, text FROM segment WHERE space = $s ORDER BY recording, idx", s=pods) + capsid = [store.R("segment", s["id"]) for s in lines if "capsid" in s["text"].lower()] + assert len(capsid) >= 3 + db.q("UPDATE $ids SET event = 'Laughter'", ids=capsid[:2]) + db.q("UPDATE $id SET event = 'Speech'", id=capsid[2]) # plain speech isn't a sound to count + + # typed any way; only the lines with that sound, each saying which + found = client.get("/api/v1/search", params={"q": "capsid", "event": "laughter", "facets": True}, headers=hv).json() + assert found["total"] == 2 and {h["event"] for h in found["hits"]} == {"Laughter"} + assert found["facets"]["events"] == [{"name": "Laughter", "count": 2}] + every = client.get("/api/v1/search", params={"q": "capsid", "facets": True}, headers=hv).json() + assert every["total"] > 2 and every["facets"]["events"] == [{"name": "Laughter", "count": 2}] + assert client.get("/api/v1/search", params={"q": "capsid", "event": "music"}, headers=hv).json()["total"] == 0 + # a saved search keeps it, and an agent can filter by it too + saved = client.post("/api/v1/searches", headers=hv, json={"name": "Laughs", "q": "capsid", "event": "Laughter"}) + assert saved.status_code == 200 and saved.json()["event"] == "Laughter" + _, tokens = _grant(client, hv, scope="read") + res = tool(client, _bearer(tokens), "search", query="capsid", event="Laughter")["results"] + assert len(res) == 2 and {r["event"] for r in res} == {"Laughter"} + + def test_word_suggestions_for_a_prefix(client, db, cfg, folder): seed(db, cfg, folder) make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) diff --git a/fastapi_backend/tests/api/test_semantic.py b/fastapi_backend/tests/api/test_semantic.py index 6aa49ef63..59c940ed6 100644 --- a/fastapi_backend/tests/api/test_semantic.py +++ b/fastapi_backend/tests/api/test_semantic.py @@ -87,6 +87,16 @@ def test_search_by_meaning_finds_other_words(client, db, cfg, folder, llm): assert search.search(db, "finances", cfg=cfg, mode="semantic", recording=ep1)["total"] == 0 +def test_search_by_meaning_keeps_to_a_sound(db, cfg, folder, llm): + *_, money = _archive(db, cfg, folder) + semantic.index_pending(db, cfg, log=quiet) + db.q("UPDATE segment SET event = 'Laughter' WHERE recording = $r AND idx = 1", r=money) + # passages don't record their sounds: the passage is shown at its line with that sound, or not at all + hits = search.search(db, "worried about our finances", cfg=cfg, mode="semantic", event="laughter")["hits"] + assert [(h["recording_id"], h["idx"], h["event"]) for h in hits] == [(money, 1, "Laughter")] + assert search.search(db, "worried about our finances", cfg=cfg, mode="semantic", event="Applause")["total"] == 0 + + def test_search_api_by_meaning_keeps_to_readable(client, db, cfg, folder, llm): _ep1, _ep2, call, money = _archive(db, cfg, folder) semantic.index_pending(db, cfg, log=quiet) diff --git a/nextjs-frontend/__tests__/search.test.ts b/nextjs-frontend/__tests__/search.test.ts index c162c10c0..094b16fd0 100644 --- a/nextjs-frontend/__tests__/search.test.ts +++ b/nextjs-frontend/__tests__/search.test.ts @@ -6,7 +6,9 @@ import { filterToken, fromParams, hasTerms, + eventName, normalizeEmotion, + normalizeEvent, parseQuery, phrases, prefixWords, @@ -33,6 +35,28 @@ describe("parseQuery", () => { }); }); + it("reads sound events typed any way", () => { + expect(parseQuery("event:Laughter capsid")).toEqual({ text: "capsid", typed: { event: "Laughter" } }); + expect(parseQuery('EVENT:"applause" ns:podcasts')).toEqual({ + text: "", + typed: { event: "applause", namespace: "podcasts" }, + }); + expect(normalizeEvent("laughter")).toBe("Laughter"); + expect(normalizeEvent(" APPLAUSE ")).toBe("Applause"); + expect(normalizeEvent("music")).toBe("BGM"); + expect(normalizeEvent("bgm")).toBe("BGM"); + expect(eventName("BGM")).toBe("Music"); + // in the URL, the facets and saved searches like the other filters + const qs = toParams("capsid", { event: "Laughter" }); + expect(qs).toBe("q=capsid&event=Laughter"); + expect(fromParams(new URLSearchParams(qs)).filters.event).toBe("Laughter"); + expect(activeFilterCount({ event: "BGM" })).toBe(1); + const f = fromServer({ moments: 2, partial: false, events: [{ name: "BGM", count: 2 }] }); + expect(f.events).toEqual([{ key: "BGM", label: "Music", count: 2 }]); + expect(savedSearchFilters({ q: "capsid", emotion: "Happy", event: "BGM" })).toBe("Happy · Music"); + expect(savedSearchHref({ q: "capsid", event: "Laughter" })).toBe("/search?q=capsid&event=Laughter"); + }); + it("leaves words with colons that aren't filters alone", () => { expect(parseQuery("time: 14:29 notes").text).toBe("time: 14:29 notes"); }); @@ -183,6 +207,7 @@ describe("facets and groups", () => { namespaces: [], speakers: [], emotions: [], + events: [], recordings: [], objects: [], }); diff --git a/nextjs-frontend/app/openapi-client/sdk.gen.ts b/nextjs-frontend/app/openapi-client/sdk.gen.ts index 7a318b4c4..78ef3b1cc 100644 --- a/nextjs-frontend/app/openapi-client/sdk.gen.ts +++ b/nextjs-frontend/app/openapi-client/sdk.gen.ts @@ -7768,8 +7768,8 @@ export class Search { * Moments where the words appear (said in audio or video, shown on screen in a video, written in a document or in * a resource's supplementary transcripts, captions, translations and indexes, or the kinds of object seen in videos, * documents and images), - * best first, in the namespaces you can read and the collections you were given a role on. A person or emotion - * filter keeps to their lines. By meaning, a moment is a passage about what was asked, though its words may differ. + * best first, in the namespaces you can read and the collections you were given a role on. A person, emotion or + * sound filter keeps to their lines. By meaning, a moment is a passage about what was asked, though its words may differ. */ public static searchTranscripts( options: Options, @@ -7920,7 +7920,7 @@ export class Searches { /** * Create Search * - * Save a search (its words, namespace, person, emotion, asset and object) under a name unique among yours. Sharing it + * Save a search (its words, namespace, person, emotion, sound, asset and object) under a name unique among yours. Sharing it * with its namespace needs editor access there. */ public static createSearch( diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index bb74608ce..54dfe4ab4 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -23317,6 +23317,10 @@ export type SavedSearch = { * Emotion */ emotion?: string | null; + /** + * Event + */ + event?: string | null; /** * Recording */ @@ -23648,6 +23652,12 @@ export type SearchCreate = { * Emotion */ emotion?: string | null; + /** + * Event + * + * a sound heard (laughter, applause, music …) + */ + event?: string | null; /** * Recording */ @@ -23679,8 +23689,9 @@ export type SearchCreate = { /** * SearchFacets * - * How many of all the matching moments are in each namespace, person, emotion and asset (up to 50 of each, - * most first), and the kinds of object seen in the assets they're in. Text on screen has no person or emotion. + * How many of all the matching moments are in each namespace, person, emotion, sound and asset (up to 50 of + * each, most first), and the kinds of object seen in the assets they're in. Text on screen has no person, emotion or + * sound. */ export type SearchFacets = { /** @@ -23707,6 +23718,12 @@ export type SearchFacets = { * Emotions */ emotions?: Array; + /** + * Events + * + * sounds heard (Laughter, Applause, BGM …); plain speech isn't one + */ + events?: Array; /** * Recordings */ @@ -23750,6 +23767,12 @@ export type SearchHit = { * Emotion */ emotion?: string | null; + /** + * Event + * + * the sound heard with it (Laughter, Applause, BGM for music …), if any + */ + event?: string | null; /** * Speaker Id */ @@ -40467,6 +40490,12 @@ export type SearchTranscriptsData = { * Emotion */ emotion?: string | null; + /** + * Event + * + * only lines with this sound (laughter, applause, music …), ignoring case + */ + event?: string | null; /** * Recording */ @@ -40488,7 +40517,7 @@ export type SearchTranscriptsData = { /** * Facets * - * also count all the matching moments by namespace, person, emotion and asset, and list their kinds of object + * also count all the matching moments by namespace, person, emotion, sound and asset, and list their kinds of object */ facets?: boolean; /** diff --git a/nextjs-frontend/components/search/facet-panel.tsx b/nextjs-frontend/components/search/facet-panel.tsx index abe7167c8..02c175898 100644 --- a/nextjs-frontend/components/search/facet-panel.tsx +++ b/nextjs-frontend/components/search/facet-panel.tsx @@ -121,7 +121,7 @@ function SavedRow({ s, onDelete, deleting }: { s: SavedSearch; onDelete: () => v } /** - * Facets for the current words: namespace, speaker, emotion and recording, with how many moments each has, and the + * Facets for the current words: namespace, speaker, emotion, sound and recording, with how many moments each has, and the * kinds of object seen in the recordings (with how many recordings). Counts come from the words alone, so they don't * vanish when you pick one; each facet is one filter, shared with its chip. */ @@ -157,6 +157,7 @@ export function FacetPanel({ { key: "namespace", title: "Namespace", values: facets.namespaces }, { key: "speaker", title: "Person", values: facets.speakers }, { key: "emotion", title: "Emotion", values: facets.emotions }, + { key: "event", title: "Sounds", values: facets.events }, { key: "recording", title: "Asset", values: facets.recordings }, { key: "object", title: "Objects seen", values: facets.objects }, ] diff --git a/nextjs-frontend/components/search/facets.ts b/nextjs-frontend/components/search/facets.ts index fee63be12..fed8ea822 100644 --- a/nextjs-frontend/components/search/facets.ts +++ b/nextjs-frontend/components/search/facets.ts @@ -1,5 +1,6 @@ import type { SearchFacets, SearchHit } from "@/app/openapi-client/types.gen"; import { objectName } from "@/components/recording/objects-model"; +import { eventName } from "@/components/search/query"; /** One value of a facet with how many moments have it. */ export type FacetValue = { @@ -14,6 +15,8 @@ export type Facets = { namespaces: FacetValue[]; speakers: FacetValue[]; emotions: FacetValue[]; + /** Sounds heard (laughter, applause, music …). */ + events: FacetValue[]; recordings: FacetValue[]; /** Kinds of object seen in the recordings, counted in recordings rather than moments. */ objects: FacetValue[]; @@ -36,6 +39,7 @@ export function fromServer(f: SearchFacets): Facets { sub: (names.get(s.name) ?? 0) > 1 ? (s.namespace ?? undefined) : undefined, })), emotions: (f.emotions ?? []).map((e) => ({ key: e.name, label: e.name, count: e.count })), + events: (f.events ?? []).map((e) => ({ key: e.name, label: eventName(e.name), count: e.count })), recordings: (f.recordings ?? []).map((r) => ({ key: String(r.id), id: r.id, diff --git a/nextjs-frontend/components/search/no-results.tsx b/nextjs-frontend/components/search/no-results.tsx index 7e5ff78ce..046ba932a 100644 --- a/nextjs-frontend/components/search/no-results.tsx +++ b/nextjs-frontend/components/search/no-results.tsx @@ -147,6 +147,19 @@ export function NoResults({ ), }); + if (filters.event) + hints.push({ + key: "evt", + node: ( + <> + The filter event: {labels.event ?? filters.event} keeps only segments where it’s heard.{" "} + + . + + ), + }); if (filters.recording != null) hints.push({ key: "rec", diff --git a/nextjs-frontend/components/search/query.ts b/nextjs-frontend/components/search/query.ts index 7faafd405..3d9b79d83 100644 --- a/nextjs-frontend/components/search/query.ts +++ b/nextjs-frontend/components/search/query.ts @@ -1,17 +1,19 @@ /** * The search box: free words plus typed filters (`speaker:"Host B"`, `namespace:podcasts`, `emotion:surprise`, - * `recording:"Episode 12"`, `object:car`). Typed filters become chips; the chip and the facet are one filter. The backend's rules: + * `event:laughter`, `recording:"Episode 12"`, `object:car`). Typed filters become chips; the chip and the facet are one filter. The backend's rules: * every word must appear (English stemming), "quoted phrases" as written, OR between alternatives, NOT or -word to * leave lines out, no prefix search. */ -export type FilterKey = "namespace" | "speaker" | "emotion" | "recording" | "object"; +export type FilterKey = "namespace" | "speaker" | "emotion" | "event" | "recording" | "object"; -/** Filters as the URL holds them: speaker and recording are ids, namespace a name, emotion a label. */ +/** Filters as the URL holds them: speaker and recording are ids, namespace a name, emotion and event labels. */ export type SearchFilters = { namespace?: string; speaker?: number; emotion?: string; + /** A sound heard (Laughter, Applause, BGM …). */ + event?: string; recording?: number; /** A kind of object the recordings have (person, car …). */ object?: string; @@ -25,10 +27,11 @@ const KEYS: Record = { ns: "namespace", speaker: "speaker", emotion: "emotion", + event: "event", recording: "recording", object: "object", }; -const FILTER_RX = /(?:^|\s)(namespace|ns|speaker|emotion|recording|object):(?:"([^"]*)"?|(\S+))/gi; +const FILTER_RX = /(?:^|\s)(namespace|ns|speaker|emotion|event|recording|object):(?:"([^"]*)"?|(\S+))/gi; /** Split what was typed into the words to search for and the typed filters. */ export function parseQuery(input: string): { @@ -58,6 +61,17 @@ export function normalizeEmotion(value: string): string { return v ? v[0].toUpperCase() + v.slice(1) : v; } +/** Sounds are stored capitalised (Laughter, Applause) and music as BGM; people type them any way. */ +export function normalizeEvent(value: string): string { + const v = normalizeEmotion(value); + return v === "Music" || v === "Bgm" ? "BGM" : v; +} + +/** How a sound reads: BGM is music. */ +export function eventName(event: string): string { + return event === "BGM" ? "Music" : event; +} + /** Words typed with a trailing * (prefix search isn't supported; the backend drops the *). */ export function prefixWords(text: string): string[] { return tokens(text) @@ -136,6 +150,7 @@ export function fromParams(p: URLSearchParams): { namespace: p.get("ns") || undefined, speaker: int(p.get("speaker")), emotion: p.get("emotion") || undefined, + event: p.get("event") || undefined, recording: int(p.get("recording")), object: p.get("object") || undefined, }, @@ -149,6 +164,7 @@ export function toParams(q: string, f: SearchFilters, mode: SearchMode = "auto") if (f.namespace) p.set("ns", f.namespace); if (f.speaker != null) p.set("speaker", String(f.speaker)); if (f.emotion) p.set("emotion", f.emotion); + if (f.event) p.set("event", f.event); if (f.recording != null) p.set("recording", String(f.recording)); if (f.object) p.set("object", f.object); if (mode !== "auto") p.set("by", mode === "keyword" ? "words" : "meaning"); @@ -156,7 +172,8 @@ export function toParams(q: string, f: SearchFilters, mode: SearchMode = "auto") } export function activeFilterCount(f: SearchFilters): number { - return [f.namespace, f.speaker, f.emotion, f.recording, f.object].filter((v) => v != null && v !== "").length; + return [f.namespace, f.speaker, f.emotion, f.event, f.recording, f.object].filter((v) => v != null && v !== "") + .length; } type Saved = { @@ -165,6 +182,7 @@ type Saved = { speaker?: number | null; speaker_name?: string | null; emotion?: string | null; + event?: string | null; recording?: number | null; recording_title?: string | null; object?: string | null; @@ -176,17 +194,19 @@ export function savedSearchHref(s: Saved): string { namespace: s.namespace ?? undefined, speaker: s.speaker ?? undefined, emotion: s.emotion ?? undefined, + event: s.event ?? undefined, recording: s.recording ?? undefined, object: s.object ?? undefined, })}`; } -/** "podcasts · Alice · Happy · Episode 12": a saved search's filters, by name where it has them. */ +/** "podcasts · Alice · Happy · Laughter · Episode 12": a saved search's filters, by name where it has them. */ export function savedSearchFilters(s: Saved): string { return [ s.namespace, s.speaker != null ? (s.speaker_name ?? `Person #${s.speaker}`) : null, s.emotion, + s.event ? eventName(s.event) : null, s.recording != null ? (s.recording_title ?? `Asset #${s.recording}`) : null, s.object ? `with ${s.object}` : null, ] diff --git a/nextjs-frontend/components/search/save-search.tsx b/nextjs-frontend/components/search/save-search.tsx index 793109e2a..f073c35d6 100644 --- a/nextjs-frontend/components/search/save-search.tsx +++ b/nextjs-frontend/components/search/save-search.tsx @@ -56,6 +56,7 @@ export function SaveSearchDialog({ namespace: ns, speaker: filters.speaker ?? null, emotion: filters.emotion ?? null, + event: filters.event ?? null, recording: filters.recording ?? null, object: filters.object ?? null, shared: shared && !noShare, @@ -98,6 +99,7 @@ export function SaveSearchDialog({ }); const dropped = [ filters.emotion ? `emotion: ${filters.emotion}` : null, + filters.event ? `event: ${labels.event ?? filters.event}` : null, filters.recording != null ? `recording: ${labels.recording ?? filters.recording}` : null, filters.object ? `object: ${filters.object}` : null, ].filter(Boolean); @@ -151,6 +153,12 @@ export function SaveSearchDialog({
{filters.emotion}
)} + {filters.event && ( + <> +
Sound
+
{labels.event ?? filters.event}
+ + )} {filters.recording != null && ( <>
Asset
diff --git a/nextjs-frontend/components/search/search-page.tsx b/nextjs-frontend/components/search/search-page.tsx index 9ca5e37cc..024e4e4f1 100644 --- a/nextjs-frontend/components/search/search-page.tsx +++ b/nextjs-frontend/components/search/search-page.tsx @@ -19,7 +19,9 @@ import { activeFilterCount, fromParams, hasTerms, + eventName, normalizeEmotion, + normalizeEvent, parseQuery, toParams, type SearchFilters, @@ -112,6 +114,7 @@ export function SearchPage() { ns: filters.namespace, speaker: filters.speaker, emotion: filters.emotion, + event: filters.event, recording: filters.recording, object: filters.object, mode, @@ -148,6 +151,7 @@ export function SearchPage() { const out: Partial> = {}; if (filters.namespace) out.namespace = filters.namespace; if (filters.emotion) out.emotion = filters.emotion; + if (filters.event) out.event = eventName(filters.event); if (filters.object) out.object = filters.object; if (filters.speaker != null) { const hit = [...(baseData?.hits ?? []), ...hits].find((h) => h.speaker_id === filters.speaker); @@ -202,6 +206,7 @@ export function SearchPage() { else issues.push(`You have no namespace called “${typed.namespace}”.`); } if (typed.emotion) next.emotion = normalizeEmotion(typed.emotion); + if (typed.event) next.event = normalizeEvent(typed.event); if (typed.object) next.object = typed.object.trim().toLowerCase(); if (typed.recording) { const want = typed.recording.toLowerCase(); @@ -228,7 +233,7 @@ export function SearchPage() { setDraft(text); }; - const chips: Chip[] = (["namespace", "speaker", "emotion", "recording", "object"] as const) + const chips: Chip[] = (["namespace", "speaker", "emotion", "event", "recording", "object"] as const) .filter((k) => filters[k] != null && filters[k] !== "") .map((k) => ({ key: k, diff --git a/nextjs-frontend/components/search/syntax-help.tsx b/nextjs-frontend/components/search/syntax-help.tsx index 648ca0389..305829281 100644 --- a/nextjs-frontend/components/search/syntax-help.tsx +++ b/nextjs-frontend/components/search/syntax-help.tsx @@ -9,6 +9,7 @@ export const SYNTAX: [string, string][] = [ ["budget -draft", "budget, leaving out lines with draft (or NOT draft)"], ['speaker:"Host B"', "only what Host B said"], ["emotion:surprise", "segments tagged Surprise"], + ["event:laughter", "segments where laughter is heard (or applause, music …)"], ["namespace:podcasts", "one namespace"], ['recording:"Episode 12"', "one asset"], ]; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 5cdde56f5..08377d421 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -16112,7 +16112,7 @@ "search" ], "summary": "Search Transcripts", - "description": "Moments where the words appear (said in audio or video, shown on screen in a video, written in a document or in\na resource's supplementary transcripts, captions, translations and indexes, or the kinds of object seen in videos,\ndocuments and images),\nbest first, in the namespaces you can read and the collections you were given a role on. A person or emotion\nfilter keeps to their lines. By meaning, a moment is a passage about what was asked, though its words may differ.", + "description": "Moments where the words appear (said in audio or video, shown on screen in a video, written in a document or in\na resource's supplementary transcripts, captions, translations and indexes, or the kinds of object seen in videos,\ndocuments and images),\nbest first, in the namespaces you can read and the collections you were given a role on. A person, emotion or\nsound filter keeps to their lines. By meaning, a moment is a passage about what was asked, though its words may differ.", "operationId": "search_transcripts", "parameters": [ { @@ -16174,6 +16174,25 @@ "title": "Emotion" } }, + { + "name": "event", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 40 + }, + { + "type": "null" + } + ], + "description": "only lines with this sound (laughter, applause, music \u2026), ignoring case", + "title": "Event" + }, + "description": "only lines with this sound (laughter, applause, music \u2026), ignoring case" + }, { "name": "recording", "in": "query", @@ -16238,11 +16257,11 @@ "required": false, "schema": { "type": "boolean", - "description": "also count all the matching moments by namespace, person, emotion and asset, and list their kinds of object", + "description": "also count all the matching moments by namespace, person, emotion, sound and asset, and list their kinds of object", "default": false, "title": "Facets" }, - "description": "also count all the matching moments by namespace, person, emotion and asset, and list their kinds of object" + "description": "also count all the matching moments by namespace, person, emotion, sound and asset, and list their kinds of object" }, { "name": "mode", @@ -16815,7 +16834,7 @@ "searches" ], "summary": "Create Search", - "description": "Save a search (its words, namespace, person, emotion, asset and object) under a name unique among yours. Sharing it\nwith its namespace needs editor access there.", + "description": "Save a search (its words, namespace, person, emotion, sound, asset and object) under a name unique among yours. Sharing it\nwith its namespace needs editor access there.", "operationId": "create_search", "requestBody": { "content": { @@ -77132,6 +77151,17 @@ ], "title": "Emotion" }, + "event": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Event" + }, "recording": { "anyOf": [ { @@ -77691,6 +77721,19 @@ ], "title": "Emotion" }, + "event": { + "anyOf": [ + { + "type": "string", + "maxLength": 40 + }, + { + "type": "null" + } + ], + "title": "Event", + "description": "a sound heard (laughter, applause, music \u2026)" + }, "recording": { "anyOf": [ { @@ -77781,6 +77824,14 @@ "type": "array", "title": "Emotions" }, + "events": { + "items": { + "$ref": "#/components/schemas/FacetCount" + }, + "type": "array", + "title": "Events", + "description": "sounds heard (Laughter, Applause, BGM \u2026); plain speech isn't one" + }, "recordings": { "items": { "$ref": "#/components/schemas/RecordingFacet" @@ -77803,7 +77854,7 @@ "partial" ], "title": "SearchFacets", - "description": "How many of all the matching moments are in each namespace, person, emotion and asset (up to 50 of each,\nmost first), and the kinds of object seen in the assets they're in. Text on screen has no person or emotion." + "description": "How many of all the matching moments are in each namespace, person, emotion, sound and asset (up to 50 of\neach, most first), and the kinds of object seen in the assets they're in. Text on screen has no person, emotion or\nsound." }, "SearchHit": { "properties": { @@ -77868,6 +77919,18 @@ ], "title": "Emotion" }, + "event": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Event", + "description": "the sound heard with it (Laughter, Applause, BGM for music \u2026), if any" + }, "speaker_id": { "anyOf": [ {