diff --git a/CHANGELOG.md b/CHANGELOG.md index e85d00782..119abbb98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **Clean up transcripts before analysis.** With Settings → Analysis → Clean up transcripts before analysis on (off + by default), a new Clean up step runs before Analyze: a word said three times in a row is kept once ("I I I looked" + → "I looked"), doubled pronouns and articles once, "um" and "uh" go, and spacing, "i" and sentence capitals are + fixed, by rules, on English lines. Every changed line is in Change history with Revert, and lines someone corrected + are left alone. Reprocess, pipelines, `POST /api/v1/recordings//reprocess` and `lens clean` run it + ([Cleaning up transcripts](docs/processing.md#cleaning-up-transcripts)). - **Automations run.** An automation saved over the API (`/api/v1/automations`) now reacts to new assets in its namespaces: when its If passes, it runs its workflow as the person who made it, within its limits and budgets, and each run shows what its actions did. Try one on an activity without keeping or sending anything, or run it now; diff --git a/DEVLOG.md b/DEVLOG.md index 1b28864f9..41bf8c20f 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,19 @@ Plans and progress for work in flight. Newest first. +## 2026-10-17 · Cleaning up transcripts before analysis + +Adeel: a text cleaning step before entity extraction and transcript processing, from a punctuation and grammar +script he pasted. The `clean` step (cleanup.py) runs before `analyze` in the standard pipeline, off until Settings → +Analysis turns it on: rules take out fillers and stutters and tidy spacing and capitals, each change kept in the edit +history with Revert. The script's alternating "Speaker 1 / Speaker 2" labels aren't carried over: Lens tells voices +apart by diarization, or keeps the labels a transcript came with. + +Next: punctuation restored by a local model (the script's deepmultilingualpunctuation, fetched on first use), grammar +from a LanguageTool server, spaCy fetched for named entities, and dates found and normalised. + +Refine later: revert a whole cleanup at once; cleanup rules for other languages' fillers; per-namespace switch. + ## 2026-10-14 · Long recordings in pieces a speech provider takes Adeel: large audio files are segmented automatically to each transcription provider's limits. `speech.Engine` cuts a diff --git a/docs/processing.md b/docs/processing.md index cd0cddf95..8d6c0bfeb 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -14,6 +14,8 @@ How assets move through Lens: where they come from, what each step does, and how ([Documents and images](configuration.md#documents-and-images)). - `diarize` splits genuinely two-channel files by channel, otherwise clusters voice embeddings (or uses pyannote), then matches voiceprints against the namespace's people. +- `clean` (off until Settings → Analysis → Clean up transcripts before analysis is on) tidies a transcript before + analysis reads it: [Cleaning up transcripts](#cleaning-up-transcripts). - `analyze` finds entities, chapters, keywords and talk statistics; `summarize` (optional) calls any OpenAI-compatible server for a summary, key points, action items (with who will do them), topics, people, tone and importance, each key point and action item with the time of the line it comes from; `report` writes static HTML per asset and @@ -21,6 +23,31 @@ How assets move through Lens: where they come from, what each step does, and how Every step takes `--ns`, `--limit` and `--force`; `run` does them all, and a lock stops two runs overlapping. +### Cleaning up transcripts + +Speech comes out of transcription as it was said, and the extractor, chapters, keywords and search all read it. With +`analysis.cleanup` on (Settings → Analysis → Clean up transcripts before analysis; off by default), the `clean` step, +which the standard pipeline runs just before `analyze`, tidies each line of a transcript with rules. It needs no model, +fetches nothing and sends nothing out: + +- a word said three times or more in a row is kept once ("I I I looked" → "I looked"), and a doubled pronoun, article + or conjunction ("the the", "I, I") once; "had had", "that that" and "very very" stay; +- "um", "uh", "erm" and "hmm" go, with the comma after them (`analysis.cleanup_fillers`, Take out "um" and "uh"; on); +- one space between words, none before a comma or a full stop, one after a comma; +- "i" becomes "I", and each sentence starts with a capital. A line that carries on the one before (same speaker, no + full stop) keeps its lower case. + +These are English rules ("um" is a word in German and Portuguese), so they apply to English lines: the line's or the +asset's language when transcription gave one, else when the text reads as English. In other languages only a word +said three times in a row is kept once, and sentences start with a capital. Documents and images keep their text as +it was read. + +Every line it changes is in the asset's Change history (by "text cleanup"), with Revert. A line someone corrected, +or reverted, is never cleaned again, and running the step again on cleaned text changes nothing. Turn it on for a +custom pipeline by adding the Clean up step before Analyze (with Force it runs even while the setting is off); run it +on an asset with Reprocess → Clean up, `POST /api/v1/recordings/{id}/reprocess` with `{"steps": ["clean", "analyze"]}`, +or `lens clean`. Workers whose `workers.steps` list has `analyze` run it too. + ## Importing text and transcripts {#importing-transcripts} Formats: .txt, .md, .markdown, .mdx, .docx, .doc, .pdf, .srt, .vtt, .json (lens, Whisper or a list of segments) and @@ -457,7 +484,8 @@ A pipeline is a named, versioned list of steps; each namespace can choose its de analyze, summarize, report). Imports, watched folders and reprocessing use the namespace's pipeline; steps that need audio skip themselves for imported text. Steps: -- `transcribe`, `diarize`, `analyze`, `summarize`, and `report` (the built-in report). +- `transcribe`, `diarize`, `clean` ([Cleaning up transcripts](#cleaning-up-transcripts)), `analyze`, `summarize`, and + `report` (the built-in report). - `llm`: renders a prompt template, sends it to the configured model, checks the reply against the template's JSON Schema and saves it as a named output (e.g. `meeting_notes`). - `report` with a template: renders an HTML page into the namespace's reports. These pages may not run scripts. diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py index ea587354a..08ccf1954 100644 --- a/fastapi_backend/app/cli.py +++ b/fastapi_backend/app/cli.py @@ -20,6 +20,14 @@ def run_steps(db, cfg, which, ns=None, limit=0, force=False, recording=None, aud log("transcribe:", ingest.transcribe_pending(db, cfg, ns, limit, force, log), "recording(s)") if "diarize" in which: log("diarize:", spk.diarize_pending(db, cfg, ns, limit, force, log), "recording(s)") + if "clean" in which: + from .domain import cleanup, settings + + ccfg = settings.effective(db, cfg) + if (ccfg.get("analysis") or {}).get("cleanup") or force: + log("clean:", cleanup.clean_pending(db, ccfg, ns, limit, force, log), "asset(s)") + else: + log("clean: skipped; cleaning up transcripts is off (analysis.cleanup, Settings → Analysis)") if "analyze" in which: log("analyze:", analyze.analyze_pending(db, cfg, ns, limit, force, log), "asset(s)") if "embed" in which: @@ -51,10 +59,11 @@ def _main_base(argv=None): ("scan", "register new files in each namespace's folders"), ("transcribe", "transcribe new recordings"), ("diarize", "split speakers and match voice IDs within each namespace"), + ("clean", "tidy transcripts before analysis: fillers, repeated words, spacing and capitals"), ("analyze", "named things, keywords, sections and speaking-time statistics"), ("embed", "index passages for search by meaning (with the configured embedding model)"), ("summarize", "optional LLM summaries"), - ("run", "scan, transcribe, diarize, analyze, embed, summarize and report, resuming where it stopped"), + ("run", "scan, transcribe, diarize, clean, analyze, embed, summarize and report, resuming where it stopped"), ): p = sub.add_parser(name, help=helptext) p.add_argument("--ns") @@ -136,8 +145,8 @@ def _main_base(argv=None): return conn = store.connect(cfg, upgrade=a.cmd != "migrations") # so a failed upgrade can still be looked at try: - if a.cmd in ("scan", "transcribe", "diarize", "analyze", "embed", "summarize", "run", "report"): - which = ["scan", "transcribe", "diarize", "analyze", "embed", "summarize", "report"] if a.cmd == "run" else [a.cmd] + if a.cmd in ("scan", "transcribe", "diarize", "clean", "analyze", "embed", "summarize", "run", "report"): + which = ["scan", "transcribe", "diarize", "clean", "analyze", "embed", "summarize", "report"] if a.cmd == "run" else [a.cmd] with store.lock(cfg, "pipeline"): run_steps( conn, diff --git a/fastapi_backend/app/domain/cleanup.py b/fastapi_backend/app/domain/cleanup.py new file mode 100644 index 000000000..1b4657e85 --- /dev/null +++ b/fastapi_backend/app/domain/cleanup.py @@ -0,0 +1,202 @@ +"""Cleaning up a transcript before analysis: the clean step (Settings → Analysis → Clean up transcripts). + +Speech comes out of transcription as it was said: "um" and "uh", stutters ("I I I looked"), a lower-case "i", stray +spaces before commas, sentences that start in lower case. The extractor, the chapters, the keywords and search all +read that text, so the clean step tidies each line first, with rules: no model, nothing to fetch, nothing sent out. + +- repeated words: a word said three times or more in a row is kept once, and a doubled pronoun, article or + conjunction ("the the", "I, I") once. "had had", "that that" and "very very" stay as they are. +- fillers (`analysis.cleanup_fillers`, on with cleanup): "um", "uh", "erm" and "hmm" go, with the comma after them. +- spacing: one space between words, none before a comma or a full stop, one after a comma. +- capitals: "i" is "I", and each sentence starts with a capital (a line that carries on the one before doesn't). + +The rules are English ones ("um" is a word in German and Portuguese), so they run on English lines: the line's or the +asset's language when transcription gave one, else when the text reads as English. Other languages only get a word +said three times or more in a row kept once, and their sentences capitalised. A document's or an image's text is kept +as it was read. + +Every line it changes is kept in the asset's edit history, by "text cleanup": Change history lists them and Revert +puts a line back. A line someone corrected is never cleaned, since people's corrections outrank the rules, and running +the rules again on what they made changes nothing. +""" + +from __future__ import annotations + +import re + +from . import store + +R = store.R +CLEANER = "text cleanup" # who the edit history says changed a line +SPEECH = ("audio", "transcript") # documents and images keep their text as read +FILLERS = {"um", "umm", "uh", "uhh", "uhm", "erm", "er", "hmm"} +# said twice in a row these are nearly always a stumble; had, that, is, very and so are often doubled on purpose +# ("what it is is", "very very good") +DOUBLED = set( + """i i'm i've i'll i'd a an the and but or to of in on at for with we you he she it they my our your their this + if it's we're you're they're there's""".split() +) +MARKERS = {"okay", "ok", "yeah", "yes", "no", "well", "so", "right", "oh", "alright", "anyway", "actually", "hey", "hi", "sorry"} +ENGLISH = set( + """the and to of a in is it you that i was for on are with as be this have not but they at we what so or if can + all there my your do just about he she one would like get will an by me from how know think yeah okay""".split() +) +ABBREV = {"mr", "mrs", "ms", "dr", "prof", "st", "vs", "etc", "e.g", "i.e", "approx", "no", "fig", "inc", "ltd", "jr", "sr"} +WORD = re.compile(r"[^\W\d_][\w'’]*") +SENTENCE = re.compile(r"(?= 3 and sum(w in ENGLISH for w in ws) / len(ws) >= 0.2 + + +def _repeats(tokens, doubled): + """A word three times or more in a row once; twice in a row once too when it's one of `doubled`.""" + out, k = list(tokens), 0 + while k < len(out): + w = _bare(out[k]) + j = k + while w and j + 1 < len(out) and _bare(out[j + 1]) == w: + j += 1 + if j - k >= 2 or (j - k == 1 and w in doubled): + last = out[j] + if out[k][:1].isupper() and not last[:1].isupper(): # "I i think", "The the" + last = last[:1].upper() + last[1:] + out[k : j + 1] = [last] + k += 1 + return out + + +def _is_filler(token): + w = _bare(token) + core = token.strip(",.;:!?-—–…") + return w in FILLERS and not (core.isupper() and len(core) > 1) and re.fullmatch(r"[\w'’]+", core) is not None + + +def _fillers(tokens): + """Without "um" and "uh": a sentence's end the filler carried moves to the word before, and a comma before it goes + with it unless it follows a word like "okay" or "well".""" + out, k = list(tokens), 0 + while k < len(out): + if not _is_filler(out[k]): + k += 1 + continue + t = out.pop(k) + if not k: + continue + prev, m = out[k - 1], OWN_END.search(t) + if m and prev[-1] not in ".!?": + out[k - 1] = prev.rstrip(",;:") + m.group(1) + elif t.endswith(",") and prev.endswith(",") and _bare(prev) not in MARKERS: # "was, uh, there"; "Okay, um, so" + out[k - 1] = prev[:-1] + return out + + +def _sentences(text): + """Each sentence starting with a capital, past abbreviations ("Dr. Smith", "e.g. this").""" + + def up(m): + before = text[: m.start()].rsplit(None, 1) + word = before[-1].lower().rstrip(".") if before else "" + if m.group(1) == "." and (word in ABBREV or len(word) == 1): + return m.group(0) + return m.group(1) + m.group(2) + m.group(3).upper() + + return SENTENCE.sub(up, text) + + +def clean_line(text, english_rules=True, fillers=True, starts_sentence=True): + """A transcript line as the rules leave it. `starts_sentence`: the line before ended a sentence (or there's none).""" + tokens = text.split() + if english_rules: + if fillers: + tokens = _fillers(tokens) + tokens = _repeats(tokens, DOUBLED) + else: + tokens = _repeats(tokens, ()) + out = " ".join(tokens) + if english_rules: + out = AFTER_COMMA.sub(r"\1 ", STRAY.sub(r"\1", out)) + out = I_ALONE.sub("I", out) + out = _sentences(out) + m = FIRST.match(out) # the line's first word, past an opening quote or bracket + if starts_sentence and m and m.group(1).islower() and not LINK.match(out[m.start(1) :]): + out = out[: m.start(1)] + m.group(1).upper() + out[m.end(1) :] + return out.strip() + + +def plan(db, cfg, rid): + """[(segment id, idx, text, cleaned)] for the lines the rules change; [] for a document or an image.""" + rec = db.one("SELECT source, language FROM $r", r=R("recording", rid)) or {} + if rec.get("source") not in SPEECH: + return [] + segs = db.rows("SELECT record::id(id) AS id, idx, text, lang, speaker FROM segment WHERE recording = $r ORDER BY idx", r=rid) + people = set(db.values("SELECT VALUE idx FROM segment_edit WHERE recording = $r AND by != $c AND after.text != NONE", r=rid, c=CLEANER)) + fillers = bool((cfg.get("analysis") or {}).get("cleanup_fillers", True)) + whole = english(rec.get("language"), [s.get("text") or "" for s in segs[:400]]) + out, prev = [], None + for s in segs: + text = s.get("text") or "" + ended = prev is None or prev["speaker"] != s.get("speaker") or re.search(r"[.!?…]['\"”’)]*$", prev["text"]) is not None + prev = {"speaker": s.get("speaker"), "text": text} + if s["idx"] in people or not text.strip(): + continue + lang = s.get("lang") + en = english(lang, [text]) if lang and lang not in ("auto", "nospeech") else whole + new = clean_line(text, en, fillers, ended) + if new and new != text: + out.append((s["id"], s["idx"], text, new)) + prev["text"] = new + return out + + +def clean_recording(db, cfg, rid): + """Clean an asset's transcript lines; how many changed. Each change goes into its edit history.""" + changes = plan(db, cfg, rid) + if not changes: + return 0 + last = int(db.values("UPSERT $r SET n += $k RETURN VALUE n", r=R("seq", "segment_edit"), k=len(changes))[0]) + t, first = store.now(), last - len(changes) + 1 + ups = [{"id": R("segment", sid), "text": new} for sid, _, _, new in changes] + edits = [ + {"recording": rid, "idx": idx, "before": {"text": old}, "after": {"text": new}, "by": CLEANER, "at": t, "n": first + k} + for k, (_, idx, old, new) in enumerate(changes) + ] + for a in range(0, len(ups), 500): + db.run( + ["FOR $u IN $ups { UPDATE $u.id SET text = $u.text; }", "INSERT INTO segment_edit $edits"], + ups=ups[a : a + 500], + edits=edits[a : a + 500], + ) + return len(changes) + + +def clean_pending(db, cfg, ns=None, limit=0, force=False, log=print): + """The command line's clean: transcripts not analysed yet (all of them with force).""" + where = "status IN ['transcribed', 'diarized', 'analyzed']" if force else "status IN ['transcribed', 'diarized']" + where += " AND source IN $src" + if ns: + where += " AND space = $s" + rows = db.rows( + f"SELECT record::id(id) AS id, title FROM recording WHERE {where} ORDER BY id", + src=list(SPEECH), + s=store.ns_id(db, ns, create=False) if ns else None, + ) + rows = rows[: limit or None] + for r in rows: + log(f" {r['title']}: {clean_recording(db, cfg, r['id'])} line(s) cleaned") + return len(rows) diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py index bf3b8e859..ccd74c61c 100644 --- a/fastapi_backend/app/domain/jobs.py +++ b/fastapi_backend/app/domain/jobs.py @@ -17,8 +17,10 @@ from . import activity, analyze, ingest, keyring, pipelines, render, speakers as spk, store, telemetry R = store.R -PIPELINE = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "analyze", "embed", "summarize", "report"] -AFTER_IMPORT = ["analyze", "embed", "summarize", "report"] +PIPELINE = [ + "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", "report", +] # fmt: skip +AFTER_IMPORT = ["clean", "analyze", "embed", "summarize", "report"] ACTIVE = ["queued", "running", "held"] # held: over budget, waiting for someone to pick (budgets.py) FIELDS = ( "record::id(id) AS id, recording, space, batch, pipeline, steps, step_index, next_step, status, worker, error, attempts, " @@ -78,6 +80,18 @@ def _diarize(db, cfg, rid, say, spec=None): spk.diarize_one(db, cfg, rid, say) +def _clean(db, cfg, rid, say, spec=None): + """A transcript's lines tidied before analysis (cleanup.py), where Settings → Analysis turns it on.""" + from . import cleanup + + if not ((cfg.get("analysis") or {}).get("cleanup") or (spec or {}).get("force")): + raise Skip("cleaning up transcripts is off (Settings → Analysis)") + if _source(db, rid) not in cleanup.SPEECH: + raise Skip("a document's or an image's text is kept as it was read") + n = cleanup.clean_recording(db, cfg, rid) + say(f"cleaned up {n} line(s); Change history lists them" if n else "nothing to clean up") + + def _analyze(db, cfg, rid, say, spec=None): analyze.analyze_recording(db, cfg, rid) say("analysed") @@ -192,6 +206,7 @@ def _describe(db, cfg, rid, say, spec=None): "objects": _objects, "nsfw": _nsfw, "describe": _describe, + "clean": _clean, "analyze": _analyze, "embed": _embed, "summarize": _summarize, @@ -911,6 +926,8 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None): self.can |= VIDEO_STEPS if self.can & {"analyze", "summarize"}: # embedding needs what they need; lists written before it existed run it too self.can.add("embed") + if "analyze" in self.can: # cleaning up a transcript needs no more than analysing it; older lists run it too + self.can.add("clean") if not steps and "objects" in self.can: # NSFW classification runs on ONNX Runtime too; older lists run it as well self.can.add("nsfw") if not steps and log and (missing := sorted(set(STEPS) - self.can)): diff --git a/fastapi_backend/app/domain/pipelines.py b/fastapi_backend/app/domain/pipelines.py index 1239ae528..5a044d62a 100644 --- a/fastapi_backend/app/domain/pipelines.py +++ b/fastapi_backend/app/domain/pipelines.py @@ -1,10 +1,10 @@ """Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without one, recordings get the standard pipeline. Steps: transcribe (speech to text), read (a document's or an image's pages and their text), diarize, shots, ocr, faces, objects, nsfw (NSFW classification, nsfw.py), describe, -analyze, embed (passages for search by meaning, semantic.py), summarize, llm (a prompt template whose structured -result is saved as a named output), report (built in, or from a report template) and export (a template rendered to -a file, optionally copied to a storage source) and workflow (a workflow graph, workflows.py, that turns what the steps -made into metadata). Any step can carry a condition. +clean (a transcript tidied before analysis, cleanup.py), analyze, embed (passages for search by meaning, semantic.py), +summarize, llm (a prompt template whose structured result is saved as a named output), report (built in, or from a +report template) and export (a template rendered to a file, optionally copied to a storage source) and workflow (a +workflow graph, workflows.py, that turns what the steps made into metadata). Any step can carry a condition. Which pipeline a recording gets also depends on its content type (content_types.py): the one chosen for the run, else the namespace's override for the recording's subtype, else the subtype's own pipeline, else the namespace's default, @@ -23,9 +23,12 @@ from . import keyring, llm, render, sources, store, templates R = store.R -STANDARD = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "analyze", "embed", "summarize", "report"] +STANDARD = [ + "transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", "report", +] # fmt: skip TYPES = { - "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "analyze", "embed", "summarize", "llm", + "transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe", "clean", "analyze", "embed", "summarize", + "llm", "report", "export", "workflow", diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index a08cead08..b3eb10441 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -153,8 +153,16 @@ def labels(): "sample_seconds": 60, "cross_namespace": "suggest", }, - # anytopdf: the places and dates anytopdf (run here) finds, as hints for the extractor (enrich.py) - "analysis": {"entities": "rules", "spacy_model": "en_core_web_sm", "gazetteer": [], "anytopdf": False}, + # anytopdf: the places and dates anytopdf (run here) finds, as hints for the extractor (enrich.py); cleanup: the + # clean step tidies transcripts before analysis (cleanup.py), cleanup_fillers taking out "um" and "uh" too + "analysis": { + "entities": "rules", + "spacy_model": "en_core_web_sm", + "gazetteer": [], + "anytopdf": False, + "cleanup": False, + "cleanup_fillers": True, + }, # chat_models: the models people may pick in Chat; empty: whatever the model server lists (docs/configuration.md) "llm": { "base_url": None, @@ -264,6 +272,7 @@ def labels(): "objects", "nsfw", "describe", + "clean", "analyze", "embed", "summarize", diff --git a/fastapi_backend/tests/domain/test_cleanup.py b/fastapi_backend/tests/domain/test_cleanup.py new file mode 100644 index 000000000..a1bec589e --- /dev/null +++ b/fastapi_backend/tests/domain/test_cleanup.py @@ -0,0 +1,135 @@ +"""The clean step: transcripts tidied by rules before analysis, each change in the edit history, people's corrections +left alone, and nothing done while it's off.""" + +from __future__ import annotations + +import pytest + +from app.domain import cleanup, ingest, jobs, store +from tests.helpers import drain, quiet + +R = store.R +TALK = """Ann|N|Yeah. Sorry, sorry about that. Okay, um, so now it's recording. +Ben|N|Because I I I looked into, you know, segmentation , like, uh, the the different methods. +Ben|N|and i think we should talk to Dyno Therapeutics about it +Ann|N|what it is is very very good, and we had had enough of it.""" + + +@pytest.mark.parametrize( + ("line", "clean"), + [ + ("Because I I I looked into it.", "Because I looked into it."), + ("um, i think the the thing is, uh, we should go.", "I think the thing is we should go."), + ("I was, uh, there.", "I was there."), + ("I, I, I think so", "I think so"), + ("So um... i'm not sure. Uh. what about it?", "So I'm not sure. What about it?"), + ("as well , like , this", "As well, like, this"), + ("what it is is very very good, we had had enough", "What it is is very very good, we had had enough"), + ("ask Dr. smith, e.g. later", "Ask Dr. smith, e.g. later"), + ("the ER was busy and UM won", "The ER was busy and UM won"), + ("5 mm wide", "5 mm wide"), + ("see www.example.com for more", "See www.example.com for more"), + ("www.example.com has it", "www.example.com has it"), + ], +) +def test_rules(line, clean): + assert cleanup.clean_line(line) == clean + assert cleanup.clean_line(clean) == clean # running them again changes nothing + + +def test_other_languages_keep_their_words(): + # "um" is German for "at", and "er" for "he": only a word said three times in a row goes, and capitals + assert cleanup.clean_line("ich ich ich komme um fünf. er kommt auch", english_rules=False) == "Ich komme um fünf. Er kommt auch" + assert cleanup.english(None, ["ich weiss nicht, das ist gut und er kommt um fünf"]) is False + assert cleanup.english(None, ["so I think we should go to the shop"]) is True + assert cleanup.english("de", ["so I think we should go to the shop"]) is False + + +def test_a_line_that_carries_on_keeps_its_lower_case(): + assert cleanup.clean_line("and then we went", starts_sentence=False) == "and then we went" + assert cleanup.clean_line("and then we went") == "And then we went" + + +def _talk(db, cfg): + return ingest.import_text(db, cfg, "pods", TALK, "Talk", "auto") + + +def _lines(db, rid): + return [s["text"] for s in db.rows("SELECT idx, text FROM segment WHERE recording = $r ORDER BY idx", r=rid)] + + +def test_off_by_default_changes_nothing(db, cfg): + rid = _talk(db, cfg) + before = _lines(db, rid) + jobs.enqueue(db, rid, ["clean", "analyze"]) + drain(db, cfg) + assert _lines(db, rid) == before + assert not db.rows("SELECT * FROM segment_edit WHERE recording = $r", r=rid) + + +def test_clean_step_tidies_lines_and_keeps_them_in_the_history(db, cfg, folder): + cfg["analysis"]["cleanup"] = True + rid = _talk(db, cfg) + before = _lines(db, rid) + jid = jobs.enqueue(db, rid, ["clean", "analyze"]) + drain(db, cfg) + assert _lines(db, rid) == [ + "Yeah. Sorry, sorry about that. Okay, so now it's recording.", + "Because I looked into, you know, segmentation, like the different methods.", + "And I think we should talk to Dyno Therapeutics about it", + "What it is is very very good, and we had had enough of it.", + ] + log = db.one("SELECT log FROM $j", j=R("job", jid))["log"] + assert any("cleaned up 4 line(s)" in line for line in log) + edits = db.rows("SELECT idx, before, after, by FROM segment_edit WHERE recording = $r ORDER BY idx", r=rid) + assert [e["by"] for e in edits] == [cleanup.CLEANER] * 4 + assert [e["before"]["text"] for e in edits] == before + # the extractor read the cleaned text + assert db.values("SELECT VALUE name FROM entity WHERE name = 'Dyno Therapeutics'") + + # running it again changes nothing + jobs.enqueue(db, rid, ["clean"]) + drain(db, cfg) + assert len(db.rows("SELECT * FROM segment_edit WHERE recording = $r", r=rid)) == 4 + + +def test_people_s_corrections_outrank_the_rules(db, cfg): + cfg["analysis"]["cleanup"] = True + rid = _talk(db, cfg) + sid = rid * store.SEG + 1 + kept = "Because I I I looked into it, you know." + db.q("UPDATE $s SET text = $t", s=R("segment", sid), t=kept) # a person's correction (or Revert) of line 2 + db.q( + "CREATE segment_edit CONTENT $d", + d={"recording": rid, "idx": 1, "before": {"text": "x"}, "after": {"text": kept}, "by": "ed@x.io", "at": store.now(), "n": 1}, + ) + jobs.enqueue(db, rid, ["clean"]) + drain(db, cfg) + assert _lines(db, rid)[1] == kept + assert _lines(db, rid)[0] == "Yeah. Sorry, sorry about that. Okay, so now it's recording." + + +def test_documents_and_fillers(db, cfg, folder): + cfg["analysis"]["cleanup"] = True + cfg["analysis"]["cleanup_fillers"] = False + rid = _talk(db, cfg) + jobs.enqueue(db, rid, ["clean"]) + drain(db, cfg) + assert _lines(db, rid)[0] == "Yeah. Sorry, sorry about that. Okay, um, so now it's recording." + db.q("UPDATE $r SET source = 'document'", r=R("recording", rid)) + assert cleanup.plan(db, cfg, rid) == [] + assert cleanup.clean_pending(db, cfg, log=quiet) == 0 # the command line's clean: transcripts only + + +def test_standard_pipeline_cleans_before_analysis(): + from app.domain import pipelines + + for steps in (pipelines.STANDARD, jobs.PIPELINE, jobs.AFTER_IMPORT): + assert steps.index("clean") < steps.index("analyze") + + +def test_workers_with_an_older_steps_list_run_it(db, cfg): + w = jobs.Worker(db, lambda: cfg, name="old", steps=None) + assert "clean" in w.can + cfg["workers"]["steps"] = ["analyze", "report"] + assert "clean" in jobs.Worker(db, lambda: cfg, name="older").can diff --git a/nextjs-frontend/__tests__/pipelines-templates.test.ts b/nextjs-frontend/__tests__/pipelines-templates.test.ts index 15b459d88..1f085cef2 100644 --- a/nextjs-frontend/__tests__/pipelines-templates.test.ts +++ b/nextjs-frontend/__tests__/pipelines-templates.test.ts @@ -41,6 +41,11 @@ describe("pipeline order", () => { expect(orderProblem([{ type: "analyze" }, { type: "summarize" }, { type: "report" }])).toBeNull(); expect(orderProblem([{ type: "summarize" }, { type: "analyze" }])).toMatch(/Summarize needs Analyze/); }); + it("keeps the transcript's cleanup before analysis", () => { + expect(orderProblem([{ type: "transcribe" }, { type: "clean" }, { type: "analyze" }])).toBeNull(); + expect(orderProblem([{ type: "analyze" }, { type: "clean" }])).toMatch(/Analyze needs Clean up’s output/); + expect(orderProblem([{ type: "clean" }, { type: "transcribe" }])).toMatch(/Clean up needs Transcribe/); + }); }); describe("navigation", () => { diff --git a/nextjs-frontend/__tests__/recording-jobs.test.ts b/nextjs-frontend/__tests__/recording-jobs.test.ts index e209adccd..5c5bc9cf0 100644 --- a/nextjs-frontend/__tests__/recording-jobs.test.ts +++ b/nextjs-frontend/__tests__/recording-jobs.test.ts @@ -273,7 +273,15 @@ describe("reprocess picker (R9)", () => { const audio = reprocessOptions({ video: false, hasAudio: true }); const transcript = reprocessOptions({ video: false, hasAudio: false }); it("offers the backend's steps; video steps only for videos", () => { - expect(audio.map((o) => o.key)).toEqual(["transcribe", "diarize", "analyze", "embed", "summarize", "report"]); + expect(audio.map((o) => o.key)).toEqual([ + "transcribe", + "diarize", + "clean", + "analyze", + "embed", + "summarize", + "report", + ]); expect(reprocessOptions({ video: true, hasAudio: true }).map((o) => o.key)).toEqual([ "transcribe", "diarize", @@ -283,12 +291,14 @@ describe("reprocess picker (R9)", () => { "objects", "nsfw", "describe", + "clean", "analyze", "embed", "summarize", "report", ]); - // a document's or an image's pages: faces, objects, NSFW and descriptions, not shots or text on screen + // a document's or an image's pages: faces, objects, NSFW and descriptions, not shots, text on screen or cleaning + // up speech expect(reprocessOptions({ video: false, hasAudio: false, paged: true }).map((o) => o.key)).toEqual([ "transcribe", "diarize", @@ -311,6 +321,16 @@ describe("reprocess picker (R9)", () => { s = toggleStep(s, "report", false, audio); expect(orderedSteps(s)).toEqual(["diarize", "analyze", "summarize"]); expect(orderedSteps(toggleStep(new Set(), "transcribe", true, transcript))).toEqual([]); + // a new transcript is cleaned up (where that's on) before it's analysed again + expect(orderedSteps(toggleStep(new Set(), "transcribe", true, audio))).toEqual([ + "transcribe", + "diarize", + "clean", + "analyze", + "embed", + "summarize", + "report", + ]); }); }); diff --git a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx index f126068b2..317eda30d 100644 --- a/nextjs-frontend/components/pipelines/pipeline-canvas.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-canvas.tsx @@ -5,6 +5,7 @@ import { AudioLines, Captions, Clapperboard, + Eraser, Eye, EyeOff, FileOutput, @@ -61,6 +62,7 @@ const ICON: Record = { objects: Shapes, nsfw: EyeOff, describe: Eye, + clean: Eraser, analyze: TextSearch, embed: ScanSearch, summarize: Sparkles, @@ -71,7 +73,7 @@ const ICON: Record = { }; const TONE = { intent: "blue", red: "red", green: "green", gate: "gold", neutral: "neutral" } as const; const ASSETS = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "nsfw", "describe"]; -const METADATA = ["analyze", "embed", "summarize", "llm", "report", "export"]; +const METADATA = ["clean", "analyze", "embed", "summarize", "llm", "report", "export"]; /** A pipeline drawn as a graph: asset steps, metadata steps and workflows, an edge saying what runs after what. */ export function PipelineCanvas({ id }: { id: number }) { diff --git a/nextjs-frontend/components/pipelines/pipeline-editor.tsx b/nextjs-frontend/components/pipelines/pipeline-editor.tsx index bc9688c11..68358c1ee 100644 --- a/nextjs-frontend/components/pipelines/pipeline-editor.tsx +++ b/nextjs-frontend/components/pipelines/pipeline-editor.tsx @@ -6,6 +6,7 @@ import { Captions, ChevronDown, Clapperboard, + Eraser, Eye, EyeOff, FileOutput, @@ -69,6 +70,7 @@ const ICON: Record = { objects: Shapes, nsfw: EyeOff, describe: Eye, + clean: Eraser, analyze: TextSearch, embed: ScanSearch, summarize: Sparkles, @@ -86,6 +88,7 @@ const LIB_ORDER = [ "objects", "nsfw", "describe", + "clean", "analyze", "embed", "summarize", diff --git a/nextjs-frontend/components/pipelines/pipeline-model.ts b/nextjs-frontend/components/pipelines/pipeline-model.ts index b4ff63817..791382395 100644 --- a/nextjs-frontend/components/pipelines/pipeline-model.ts +++ b/nextjs-frontend/components/pipelines/pipeline-model.ts @@ -76,13 +76,14 @@ export const NEEDS: Record = { objects: ["shots"], nsfw: ["read", "shots"], describe: ["shots"], - analyze: ["transcribe", "read", "diarize"], - embed: ["transcribe", "read", "describe"], + clean: ["transcribe", "diarize"], + analyze: ["transcribe", "read", "diarize", "clean"], + embed: ["transcribe", "read", "describe", "clean"], summarize: ["analyze"], - llm: ["transcribe", "read", "diarize", "analyze"], + llm: ["transcribe", "read", "diarize", "clean", "analyze"], report: ["analyze", "summarize", "llm"], export: ["analyze", "summarize", "llm"], - workflow: ["transcribe", "read", "diarize"], + workflow: ["transcribe", "read", "diarize", "clean"], }; /** What a step makes available to later steps and templates. */ @@ -96,6 +97,7 @@ export const PROVIDES: Record = { objects: "objects[]", nsfw: "nsfw (flagged · score)", describe: "descriptions[]", + clean: "tidied transcript lines", analyze: "sections[] · entities[] · keywords[] · stats", embed: "passages searchable by meaning", summarize: "summary.*", @@ -115,6 +117,8 @@ export const DESCRIBE: Record = { objects: "Finds objects (people, cars, animals …) in keyframes and pages (Settings → Video)", nsfw: "Flags NSFW pictures in keyframes, pages and images with a local model, where it’s on (Settings → NSFW)", describe: "Describes each shot and page with a model that can see images (Settings → LLM provider)", + clean: + "Tidies a transcript before analysis: fillers, repeated words, spacing and capitals, where it’s on (Settings → Analysis)", analyze: "Chapters, entities, keywords and talk-time stats", embed: "Embeds passages so search finds them by meaning, not only their words (Settings → Search)", summarize: "The built-in summary with the LLM in Settings", diff --git a/nextjs-frontend/components/pipelines/step-settings.tsx b/nextjs-frontend/components/pipelines/step-settings.tsx index 57617b04b..25f6d5b3b 100644 --- a/nextjs-frontend/components/pipelines/step-settings.tsx +++ b/nextjs-frontend/components/pipelines/step-settings.tsx @@ -282,12 +282,14 @@ export function StepSettings({ with a prompt template.

)} - {(step.type === "transcribe" || step.type === "diarize") && ( + {(step.type === "transcribe" || step.type === "diarize" || step.type === "clean") && ( set({ force: v || undefined })} diff --git a/nextjs-frontend/components/recording/dialogs.tsx b/nextjs-frontend/components/recording/dialogs.tsx index 66bea97c9..bca536623 100644 --- a/nextjs-frontend/components/recording/dialogs.tsx +++ b/nextjs-frontend/components/recording/dialogs.tsx @@ -15,6 +15,7 @@ import { PlaceDialog } from "@/components/library/collections-ui"; import { publicPath } from "@/components/public/model"; import { AttachAudioDialog } from "@/components/recording/attach-audio"; import { useRec } from "@/components/recording/context"; +import { CLEANER } from "@/components/recording/edits"; import { ShareEmbedDialog } from "@/components/sharing/share-dialog"; import { useEdits, useRecordingActions } from "@/components/recording/hooks"; import { orderedSteps, reprocessOptions, STEP_HELP, toggleStep, type StepKey } from "@/components/recording/jobs"; @@ -264,7 +265,7 @@ export function ReprocessDialog({ open, onOpenChange }: { open: boolean; onOpenC const router = useRouter(); const steps = orderedSteps(selected).filter((k) => options.some((o) => o.key === k && !o.disabled)); const speakerEdits = (edits.data ?? []).filter((e) => e.after && "speaker" in e.after).length; - const textEdits = (edits.data ?? []).filter((e) => e.after && "text" in e.after).length; + const textEdits = (edits.data ?? []).filter((e) => e.after && "text" in e.after && e.by !== CLEANER).length; const short = model.title.split(/\s+[—–-]\s+/)[0] || model.title; const run = () => diff --git a/nextjs-frontend/components/recording/edits.ts b/nextjs-frontend/components/recording/edits.ts index 563f66b96..cdd8a4ef0 100644 --- a/nextjs-frontend/components/recording/edits.ts +++ b/nextjs-frontend/components/recording/edits.ts @@ -10,6 +10,9 @@ export type EditRecord = { at?: string | null; }; +/** Who the edit history says changed a line when the clean step tidied it (cleanup.py). */ +export const CLEANER = "text cleanup"; + const clip = (s: string, n = 28) => (s.length > n ? `${s.slice(0, n - 1)}…` : s); /** The words that changed between two versions of a line: common leading and trailing words are dropped. */ diff --git a/nextjs-frontend/components/recording/jobs.ts b/nextjs-frontend/components/recording/jobs.ts index e27a839aa..e4053dd90 100644 --- a/nextjs-frontend/components/recording/jobs.ts +++ b/nextjs-frontend/components/recording/jobs.ts @@ -295,6 +295,7 @@ export const STEP_ORDER = [ "objects", "nsfw", "describe", + "clean", "analyze", "embed", "summarize", @@ -311,6 +312,8 @@ export const STEP_HELP: Record = { objects: "Find objects (people, cars, animals …) on the sampled frames or pages", nsfw: "Flag NSFW pictures on the sampled frames or pages (where the namespace has it on)", describe: "Describe each shot or page with a model that can see images (needs one in Settings)", + clean: + "Tidy the transcript first: fillers, repeated words, spacing and capitals (where it’s on in Settings → Analysis)", analyze: "Entities, chapters, keywords and stats (talk time for audio)", embed: "Index passages for search by meaning (needs an embedding model in Settings → Search)", summarize: "Summary, topics and action items (needs an LLM)", @@ -319,7 +322,7 @@ export const STEP_HELP: Record = { /** Ticking a step also ticks the later steps that use its output. */ export const DEPENDENTS: Record = { - transcribe: ["diarize", "analyze", "embed", "summarize", "report"], + transcribe: ["diarize", "clean", "analyze", "embed", "summarize", "report"], diarize: ["analyze", "summarize", "report"], shots: ["ocr", "faces", "objects", "nsfw", "describe"], ocr: [], @@ -327,6 +330,7 @@ export const DEPENDENTS: Record = { objects: [], nsfw: [], describe: ["embed"], + clean: ["analyze", "embed", "summarize", "report"], analyze: ["report"], embed: [], summarize: ["report"], @@ -337,13 +341,13 @@ export type StepOption = { key: StepKey; disabled: string | null }; /** * The steps offered for a recording: video steps only for videos (faces, objects, nsfw and describe also for a - * document's or an image's pages); transcribe/diarize need audio. + * document's or an image's pages); transcribe/diarize need audio; clean is for speech, so not for pages. */ export function reprocessOptions(opts: { video: boolean; hasAudio: boolean; paged?: boolean }): StepOption[] { const left = opts.video ? [] : opts.paged - ? ["shots", "ocr"] + ? ["shots", "ocr", "clean"] : ["shots", "ocr", "faces", "objects", "nsfw", "describe"]; return STEP_ORDER.filter((k) => !left.includes(k)).map((key) => { let disabled: string | null = null; diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index 0632f1f63..11423b33c 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -242,6 +242,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) { case "analysis": return ( <> + + {raw("analysis.cleanup") === true && } {raw("analysis.entities") === "spacy" && } diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index 8d84dbfa3..14f7f59a4 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -741,6 +741,19 @@ export const FIELDS: FieldSpec[] = [ kind: "switch", hint: "Offline: places a text names, dates it gives and where a photo was taken become hints for the entities found", }, + { + section: "analysis", + key: "cleanup", + label: "Clean up transcripts before analysis", + kind: "switch", + hint: "Repeated words, spacing and capitals tidied by rules before entities are found; every change is in Change history, with Revert", + }, + { + section: "analysis", + key: "cleanup_fillers", + label: "Take out “um” and “uh”", + kind: "switch", + }, { section: "analysis", key: "gazetteer", diff --git a/nextjs-frontend/components/ui/loop.tsx b/nextjs-frontend/components/ui/loop.tsx index b55f1da8c..9ae0485a0 100644 --- a/nextjs-frontend/components/ui/loop.tsx +++ b/nextjs-frontend/components/ui/loop.tsx @@ -18,6 +18,7 @@ export const STEP_TONE: Record = { objects: "red", nsfw: "red", describe: "gate", + clean: "green", analyze: "green", embed: "green", summarize: "gate", @@ -37,6 +38,7 @@ export const STEP_LABEL: Record = { objects: "Objects", nsfw: "NSFW", describe: "Describe", + clean: "Clean up", analyze: "Analyze", embed: "Index for meaning", summarize: "Summarize",