From 6c22e54f35e54f079f615a6c51353c1f48e6fcf3 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Thu, 15 Oct 2026 11:37:08 +0000 Subject: [PATCH] feat(classify): classify an asset or a text with the decision model POST /classify, the MCP tool classify and the assistant's classify say which of some labels fits an asset or a text, how sure, and every label ranked: labels given, Lens's content types or a namespace's topics. The decision model answers, else the language model; with apply, an editor's sure answer sets the content type, suggests the topic or adds the tag. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GKv2oVRvFRdiSf4USLsM1N --- CHANGELOG.md | 5 + docs/api.md | 11 + docs/assistant.md | 4 +- docs/mcp.md | 1 + docs/processing.md | 27 ++ fastapi_backend/app/api/mcp_tools.py | 44 +++ fastapi_backend/app/api/v1/router.py | 2 + fastapi_backend/app/api/v1/routes/classify.py | 47 +++ fastapi_backend/app/domain/ai_tools.py | 34 +++ fastapi_backend/app/domain/classify.py | 190 ++++++++++++ fastapi_backend/app/schemas/classify.py | 50 +++ fastapi_backend/tests/api/test_classify.py | 175 +++++++++++ nextjs-frontend/app/openapi-client/index.ts | 8 + nextjs-frontend/app/openapi-client/sdk.gen.ts | 25 ++ .../app/openapi-client/types.gen.ts | 172 +++++++++++ nextjs-frontend/openapi.json | 286 ++++++++++++++++++ 16 files changed, 1080 insertions(+), 1 deletion(-) create mode 100644 fastapi_backend/app/api/v1/routes/classify.py create mode 100644 fastapi_backend/app/domain/classify.py create mode 100644 fastapi_backend/app/schemas/classify.py create mode 100644 fastapi_backend/tests/api/test_classify.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ea5e7f1b9..3fb749320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,11 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version (`POST /find`, the MCP tool `find`, the assistant's `find_resources`; [Finding by any attribute](docs/processing.md#finding-by-any-attribute)). The Library list takes faces, entities, topics, content types and emotions as filters too. +- **Classify an asset or a text.** Agents and the API can ask which of some labels fits, how sure that is and every + label ranked: labels they give, Lens's content types or a namespace's topics. The decision model answers, else the + language model, logged in the activity ledger; with `apply`, an editor's sure answer sets the content type, suggests + the topic or adds the tag (`POST /classify`, the MCP and assistant tool `classify`; + [Classifying](docs/processing.md#classifying)). ## 0.13.0 October 16, 2026 {id="0.13.0"} diff --git a/docs/api.md b/docs/api.md index 0cc5aa293..da21ca395 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1111,6 +1111,7 @@ asset later leaves its uploaded file in place, like any other media file. GET /api/v1/search GET /api/v1/search/terms POST /api/v1/find +POST /api/v1/classify GET /api/v1/similar GET /api/v1/graph GET /api/v1/mentions @@ -1165,6 +1166,16 @@ per filter, and `narrower_topics`) and `text` (how the words were matched, as `G moments looked through were `capped`). A name that matches nothing in the namespaces you can read is a 400 that names the closest ones; an unknown field is a 422. +`POST /classify` says which label fits an asset or a text best ([classifying](processing.md#classifying)). The +body takes `recording` (an asset you can read) or `text` (at most 20,000 characters), and `labels` (a list, with +`label: what it means` items, or `{label: what it means}`) or `vocabulary` (`content_type` or `topic`; a text's topics +need `namespace`); `question` (at most 500 characters) asks it your way, and `apply` acts on a sure answer (an editor +of the asset). The reply has `choice`, `confidence` (0 to 1), `sure`, `ranked` (`[{label, p}]`, best first, with +the content type's `key` or the topic's `id`), `by` (`jev`, `laya`, `llm`, or `only option` when there is one), `vocabulary`, `question` and `applied` +(`content_type`, `topic_suggested`, `tag`, or null when nothing changed), plus the chosen content type's `key` or +topic's `id`. Bad labels or a missing choice are a 400, an asset you can't read a 404, `apply` without editing a 403, +and no model set up to answer a 409. + Search has no prefix search (`interp*` looks for the word "interp"). `GET /search/terms?prefix=interp` lists whole words found in the namespaces you can read (`ns` for one) that start with it, the most frequent first, with how often and in how many assets (`limit`, default 8, at most 20); the web app offers them as "Try …". diff --git a/docs/assistant.md b/docs/assistant.md index 66d9dcbda..b01c44ac2 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -143,7 +143,9 @@ When the configured model supports function calling, chat becomes an agent. [stream of activities](automations.md#timeline-reads); `unreviewed` for what the person hasn't marked reviewed), and assets by any attribute: faces seen, people heard, entities, topics, kind, content type, dates and the rest, each with the moments that show why it matched (`find_resources`, - [Finding by any attribute](processing.md#finding-by-any-attribute)). + [Finding by any attribute](processing.md#finding-by-any-attribute)). It can also ask which of some labels, content + types or topics fits an asset or a text, and how sure that is (`classify`, [Classifying](processing.md#classifying)); + it changes nothing. - **The person's review marks:** `mark_activities` marks activities of the timeline reviewed, important or hidden for the person ([Timeline](timeline.md#review-marks)). After it has read text from outside Lens, hiding or marking reviewed waits for their yes; marking important or bringing things back acts at once. diff --git a/docs/mcp.md b/docs/mcp.md index a426b5449..6f23f43f7 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -69,6 +69,7 @@ For an app that speaks OpenAI's API rather than MCP, the namespace can also answ | `search` | moments where words appear (written, spoken or shown on screen), best first: the asset, person, time, the matching text (matches in **bold**) and a link to that moment. Words, `"phrases"`, `OR`; by namespace, person or asset | | `find_similar` | the assets (recordings, documents and images) most like a text, an asset, a topic or an entity: why each is alike (meaning, key words, topics, entities in common) and the moments that show it, with links | | `find` | assets by any attribute: faces seen, people heard, entities, topics, kind, content type, dates (a day, month or year), tags, language, objects, emotions, length, collection and words in them; ids or names. Each says why it matched, with moments and links to cite ([Finding by any attribute](processing.md#finding-by-any-attribute)) | +| `classify` | which of some labels fits an asset or a text, how sure, and every label ranked: labels you give, or Lens's content types or the namespace's topics; the decision model answers, else the language model. Read only: it changes nothing ([Classifying](processing.md#classifying)) | | `list_namespaces` | the namespaces you can read, with their assets, hours, people and your role | | `list_recordings` | assets, newest first, by namespace, title words, dates, person, tag or kind, and whether they're flagged NSFW (`sensitive`) | | `list_speakers` | the people of your namespaces, the ones with the most talk time first | diff --git a/docs/processing.md b/docs/processing.md index df1ecee72..6262acb99 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -327,6 +327,33 @@ narrower topics), `content_type` and `emotion`, besides its other filters ([API] Planned, not built yet: these filters in the Library's filter bar and in saved collections, finding by what a photo shows (`GET /search/photos`) inside the same filter, and ranking by more than the best matching moment. +### Classifying + +The API (`POST /classify`), the assistant (`classify`) and the MCP server (`classify`) say which of some labels fits an +asset, or a piece of text, how sure that is, and every label ranked. The labels are: + +* given, as a list (`["science", "sport: games and teams"]`, what a label means after `: `) or as + `{label: what it means}`, two to 60 of them; +* or a vocabulary Lens keeps: `content_type`, the [content types](#content-types) of the asset's base type (all of + them for text), or `topic`, its namespace's topics with their definitions and other labels (a text names the + namespace), the most used first. + +The decision model answers when it is set up (Settings → AI assistant: Jev, typesafe.ai's System One, or Laya on +the machine), else the language model, asked for JSON; `by` says which, and the call is in the +[activity ledger](activity.md), counted for the asset and its namespace. The model reads the asset's kind, file +name, length, language, tags and accepted topics, and its title, summary, the entities it mentions most and the start +of its text, marked as data so instructions in it aren't followed +([prompt injection](assistant.md#third-party-content-and-prompt-injection)). `question` +asks something of your own ("Which desk should this go to?"). + +An answer is `sure` at `decisions.act_above` confidence or more (0.8 by default). Nothing changes unless `apply` is +set, by an editor, and the answer is sure: then the asset's content type is set (as choosing it in its Details tab +does), the topic is suggested for it, waiting for someone to accept it (one it already has, waits for or was dismissed +from stays as it is), or the label given is added as a tag (at most 40 characters). `applied` says which, and it is +in the audit log. The MCP and assistant tools never apply; the assistant can then suggest the topic (`suggest_topic`). + +Planned, not built yet: a classify step in workflows, classifying many assets at once, and labels with examples. + ## Background work Imports, pipeline runs and folder scans return at once and run as jobs stored in SurrealDB. The server runs diff --git a/fastapi_backend/app/api/mcp_tools.py b/fastapi_backend/app/api/mcp_tools.py index 0ff36e711..39bab5979 100644 --- a/fastapi_backend/app/api/mcp_tools.py +++ b/fastapi_backend/app/api/mcp_tools.py @@ -43,6 +43,7 @@ approvals, auth, blobs, + decide, events, feeds, filetree, @@ -61,6 +62,7 @@ updates, variables, ) +from app.domain import classify as classifymod from app.domain import entities as ents from app.domain import find as findmod from app.domain import speakers as spk @@ -750,6 +752,48 @@ def find( return out +@tool( + "classify", + "Classify an asset or a text", + "Which label fits an asset (or a text) best, how sure that is, and every label ranked: labels you give " + '("name" or "name: what it means"), Lens\'s content types (vocabulary content_type: podcast, interview, scan, ' + "email …, of the asset's kind) or its namespace's topics (vocabulary topic). A decision model answers when " + "one is set up, else the language model; `sure` says whether it is sure enough to act on without asking. " + "It changes nothing: to act, set the content type in the web app, suggest_topic, or tag it.", + Arg("recording_id", "integer", "the asset to classify"), + Arg("text", "string", "a text to classify instead", max_length=20000), + Arg("labels", "string[]", 'the labels to choose from (2 to 60): "name", or "name: what it means"'), + Arg("vocabulary", "string", "choose from Lens's content types or the namespace's topics instead", enum=("content_type", "topic")), + Arg("namespace", "string", "whose topics, when classifying a text by topic"), + Arg("question", "string", "what to ask (default: which label fits best)", max_length=500), +) +def classify( + ctx: Context, + recording_id: int | None, + text: str | None, + labels: list[str] | None, + vocabulary: str | None, + namespace: str | None, + question: str | None, +) -> dict[str, Any]: + space = None + if recording_id is not None: + _recording(ctx, recording_id) + elif namespace: + space = ctx.acl.namespace(namespace) + try: + res = classifymod.classify( + ctx.db, ctx.cfg, rid=recording_id, text=text, labels=labels, vocab=vocabulary, question=question, space=space + ) + except decide.Undecided as e: + raise ToolError(f"nothing could answer: {e}") from None + out = {k: res[k] for k in ("choice", "confidence", "sure", "ranked", "by")} + for k in ("key", "id"): + if res.get(k) is not None: + out[k] = res[k] + return out + + @tool( "list_speakers", "List people", diff --git a/fastapi_backend/app/api/v1/router.py b/fastapi_backend/app/api/v1/router.py index d98e61a9f..8243c6574 100644 --- a/fastapi_backend/app/api/v1/router.py +++ b/fastapi_backend/app/api/v1/router.py @@ -15,6 +15,7 @@ browser, budgets, chats, + classify, collections, comments, completions, @@ -109,6 +110,7 @@ uploads, search, find, + classify, searches, speakers, entities, diff --git a/fastapi_backend/app/api/v1/routes/classify.py b/fastapi_backend/app/api/v1/routes/classify.py new file mode 100644 index 000000000..a50533d07 --- /dev/null +++ b/fastapi_backend/app/api/v1/routes/classify.py @@ -0,0 +1,47 @@ +"""Classify an asset or a text with the decision model (domain/classify.py).""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.api.deps import Acl, Cfg, CurrentUser, Db, domain_errors +from app.domain import auth, decide +from app.domain import classify as classifymod +from app.schemas.classify import ClassifyRequest, ClassifyResult + +router = APIRouter(tags=["search"]) + + +@router.post("/classify") +def classify(body: ClassifyRequest, acl: Acl, user: CurrentUser, db: Db, cfg: Cfg) -> ClassifyResult: + """Which label fits an asset (or a text) best, how sure that is, and every label ranked: the labels given, Lens's + content types (of the asset's kind) or the namespace's topics. The decision model answers when it's set up + (Settings → AI assistant), else the language model; the call is in the activity ledger. With `apply` (editors), a + sure answer is acted on: the content type is set, the topic is suggested for someone to accept, or the label is + added as a tag; one that isn't sure changes nothing. 409 when no model is set up to answer.""" + space = None + if body.recording is not None: + acl.recording(body.recording, "edit" if body.apply else "viewer") + elif body.namespace: + space = acl.namespace(body.namespace) + with domain_errors(): + try: + res = classifymod.classify( + db, + cfg, + rid=body.recording, + text=body.text, + labels=body.labels, + vocab=body.vocabulary, + question=body.question, + apply=body.apply, + user=user.email, + space=space, + ) + except decide.Undecided as e: + raise HTTPException(409, f"nothing could answer: {e}") from None + if res["applied"]: + auth.audit( + db, user.as_audit(), "recording.classify", f"recording:{body.recording}", {"applied": res["applied"], "choice": res["choice"]} + ) + return ClassifyResult.model_validate(res) diff --git a/fastapi_backend/app/domain/ai_tools.py b/fastapi_backend/app/domain/ai_tools.py index 45d5c5304..04859908f 100644 --- a/fastapi_backend/app/domain/ai_tools.py +++ b/fastapi_backend/app/domain/ai_tools.py @@ -13,7 +13,9 @@ from . import ( auth, batches, + classify as classifymod, cypher, + decide, entities, event_marks, extensions, @@ -122,6 +124,23 @@ [], False, ), + ( + "classify", + "Classify an asset (or a text): which of some labels fits best, how sure, and every label ranked. Give " + 'labels (names, or "name: what it means"), or a vocabulary: content_type (Lens\'s content types, such ' + "as podcast, interview, scan, email) or topic (the namespace's topics). It changes nothing; sure says whether " + "the answer is sure enough to act on.", + { + "recording_id": _I, + "text": _S, + "labels": {"type": "array", "items": _S}, + "vocabulary": {"type": "string", "enum": list(classifymod.VOCABULARIES)}, + "namespace": _S, + "question": _S, + }, + [], + False, + ), ( "mark_activities", "Mark activities of the timeline (by their ids from timeline) reviewed, important or hidden, or clear those " @@ -1408,6 +1427,21 @@ def t_find_resources(self, namespace=None, limit=10, **spec): ) return {"total": res["total"], "resolved": res["resolved"] or None, "results": out}, f"Found {res['total']} asset(s)" + def t_classify(self, recording_id=None, text=None, labels=None, vocabulary=None, namespace=None, question=None): + space = None + if recording_id is not None: + self._ok(recording_id) + elif namespace: + space = self._space(namespace) + try: + res = classifymod.classify( + self.db, self.cfg, rid=recording_id, text=text, labels=labels, vocab=vocabulary, question=question, space=space + ) + except decide.Undecided as e: + raise ValueError(f"nothing could answer: {e}") from None + out = {k: res[k] for k in ("choice", "confidence", "sure", "ranked", "by", "key", "id") if res.get(k) is not None} + return out, f"Classified as {res['choice']} ({round(res['confidence'] * 100)}% sure)" + def t_read_transcript(self, recording_id, start_seconds=0, end_seconds=None, max_lines=60): cap = self.cfg["ai"].get("max_transcript_reads") or 20 if self.reads >= cap: diff --git a/fastapi_backend/app/domain/classify.py b/fastapi_backend/app/domain/classify.py new file mode 100644 index 000000000..e66382144 --- /dev/null +++ b/fastapi_backend/app/domain/classify.py @@ -0,0 +1,190 @@ +"""Classify an asset or a piece of text: which of some labels fits, how sure that is, and the other labels ranked, +the cheapest way that works. The decision model answers (decide.py: Jev, typesafe.ai's System One, or Laya on the +machine), else the language model asked for JSON; each call goes in the activity ledger, counted for the asset and +its namespace. + +The labels are given, as {label: what it means} or a list of labels, or come from a vocabulary Lens keeps: + +- `content_type`: the content types of the asset's base type (content_types.py), or all of them for text; +- `topic`: its namespace's topics (topics.py), with their definitions and other labels; the most used first when there + are more than MAX_LABELS. + +A resource is described to the model by its title, file name, kind, length, language, tags, summary, the entities it +mentions most, the topics it is about and the start of its text, the archive's words marked as data (untrusted.py). +Text given is marked the same way. + +`apply` acts on a sure answer (decide.sure: at least decisions.act_above, Settings → AI assistant): the asset's +content type is set (as choosing it on the resource does), the topic is suggested for it (it waits for someone to +accept it; one already accepted, suggested or dismissed stays as it is), or the label is added as a tag. An answer that isn't sure changes nothing; the ranked labels go back for +the person to pick. +""" + +from __future__ import annotations + +import pathlib + +from . import activity, content_types as ctypes, decide, library, render, store, topics as topicmod, untrusted + +R = store.R +VOCABULARIES = ("content_type", "topic") +MAX_LABELS = 60 # labels one question offers: decision models read about 32k tokens of state and criteria +LABEL_CHARS, MEANING_CHARS = 80, 300 +TEXT_CHARS = 6000 # of a resource's text, or of text given, that the model reads +QUESTION_CHARS = 500 + + +def _labels(labels): + """Given labels as {label: what it means}, checked. A list's items are labels, or "label: what it means".""" + if isinstance(labels, (list, tuple)): + labels = dict(tuple(x.strip() for x in str(item).partition(": ")[::2]) for item in labels) + if not isinstance(labels, dict): + raise ValueError("labels are a list of names, or {name: what it means}") + out = {} + for k, v in labels.items(): + name = " ".join(str(k).split()) + if not name: + continue + if len(name) > LABEL_CHARS: + raise ValueError(f"a label has at most {LABEL_CHARS} characters") + if v is not None and not isinstance(v, str): + raise ValueError(f"what {name} means is text") + out[name] = " ".join((v or "").split())[:MEANING_CHARS] or name + if len(out) < 2: + raise ValueError("give at least two labels to choose from") + if len(out) > MAX_LABELS: + raise ValueError(f"at most {MAX_LABELS} labels") + return out + + +def vocabulary(db, kind, rec=None, space=None): + """({label: what it means}, {label: what it stands for}) of a vocabulary Lens keeps, for a resource (`rec`) or a + namespace (`space`).""" + if kind == "content_type": + base = ctypes.base_of(rec) if rec else None + types = [t for t in ctypes.all_types(db) if base is None or t["base"] == base] + return ( + {t["label"]: f"{t['description'] or t['label']} ({t['base']})" for t in types}, + {t["label"]: {"key": t["key"]} for t in types}, + ) + if kind == "topic": + sid = rec["space"] if rec else space + if sid is None: + raise ValueError("topics are a namespace's: classify a resource, or name the namespace") + rows = db.rows(f"SELECT {topicmod.FIELDS} FROM topic WHERE space = $s", s=sid) + if len(rows) < 2: + raise ValueError("this namespace has fewer than two topics to choose from") + used = topicmod._counts(db, [sid]) + rows = sorted(rows, key=lambda t: (-used[t["id"]], t["label"].casefold()))[:MAX_LABELS] + meaning = { + t["label"]: "; ".join(x for x in (t.get("definition"), ", ".join(t.get("alt") or [])) if x)[:MEANING_CHARS] or t["label"] + for t in rows + } + return meaning, {t["label"]: {"id": t["id"]} for t in rows} + raise ValueError(f"vocabulary is one of {', '.join(VOCABULARIES)}") + + +def describe(db, cfg, rid): + """What the model reads about a resource.""" + rec = db.one( + "SELECT title, path, source, media, duration_ms, language, tags, summary, space FROM $r", + r=R("recording", int(rid)), + ) + if not rec: + raise KeyError(rid) + lines = db.rows("SELECT idx, text FROM segment WHERE recording = $r ORDER BY idx LIMIT 400", r=int(rid)) + text = "" + for line in lines: + if len(text) >= TEXT_CHARS: + break + text += (line.get("text") or "") + "\n" + ents = db.rows("SELECT entity, count() AS n FROM mentions WHERE recording = $r GROUP BY entity ORDER BY n DESC LIMIT 15", r=int(rid)) + names = { + e["id"]: e["name"] + for e in ( + db.rows("SELECT record::id(id) AS id, name FROM entity WHERE id IN $ids", ids=[R("entity", e["entity"]) for e in ents]) + if ents + else [] + ) + } + summary = rec.get("summary") if isinstance(rec.get("summary"), dict) else {} + about = [t["label"] for t in topicmod.of_recording(db, rid, status="accepted")] + words = { + "title": rec.get("title"), + "summary": summary.get("summary"), + "entities": [names[e["entity"]] for e in ents if e["entity"] in names], + "text": text[:TEXT_CHARS], + } + return store.clean( + { + "kind": render.kind(rec), + "file": pathlib.PurePosixPath(rec.get("path") or "").name or None, + "length": store.tc(rec["duration_ms"]) if rec.get("duration_ms") else None, + "language": rec.get("language"), + "tags": rec.get("tags") or None, + "topics": about or None, + "resource": untrusted.wrap(_lines(words), "the resource", cfg), + } + ) + + +def _lines(words): + out = [] + for k, v in words.items(): + if v: + out.append(f"{k}: {', '.join(v) if isinstance(v, list) else v}") + return "\n".join(out) + + +def classify(db, cfg, rid=None, text=None, labels=None, vocab=None, question=None, apply=False, user=None, space=None): + """Which label fits the resource `rid` (or `text`): {choice, confidence, sure, ranked, by, vocabulary, applied, + key?, id?}. Raises ValueError for a bad question and decide.Undecided when no model could answer.""" + if (rid is None) == (text is None or not str(text).strip()): + raise ValueError("classify an asset or a text: give one") + if (labels is None) == (vocab is None): + raise ValueError("give labels, or a vocabulary (content_type or topic): one") + if apply and rid is None: + raise ValueError("only a resource's classification can be applied") + rec = db.one(f"SELECT {ctypes.TYPE_FIELDS}, space FROM $r", r=R("recording", int(rid))) if rid is not None else None + if rid is not None and not rec: + raise KeyError(rid) + if vocab: + options, stands = vocabulary(db, vocab, rec, space) + else: + options, stands = _labels(labels), {} + if apply and any(len(k) > library.TAG_MAX for k in options): + raise ValueError(f"a label applied as a tag has at most {library.TAG_MAX} characters") + what = "resource" if rid is not None else "text" + asked = " ".join((question or "").split())[:QUESTION_CHARS] or { + "content_type": f"What kind of {ctypes.base_of(rec) if rec else 'content'} is this {what}?", + "topic": f"Which topic is this {what} about?", + }.get(vocab or "", f"Which label fits this {what} best?") + state = describe(db, cfg, rid) if rid is not None else {"text": untrusted.wrap(str(text)[:TEXT_CHARS], "the text", cfg)} + refs = (f"recording:{int(rid)}", f"space:{rec['space']}") if rec else ((f"space:{space}",) if space else ()) + with activity.scope(db, *refs, cfg=cfg): + d = decide.choose(cfg, asked, options, state) + sure = decide.sure(cfg, d) + out = { + "choice": d["choice"], + "confidence": round(float(d["confidence"]), 3), + "sure": sure, + "ranked": [{"label": x["option"], "p": x["p"], **stands.get(x["option"], {})} for x in d["ranked"]], + "by": d["by"], + "vocabulary": vocab, + "question": asked, + "applied": None, + **stands.get(d["choice"], {}), + } + if apply and sure: + out["applied"] = _apply(db, int(rid), vocab, d["choice"], stands, user) + return out + + +def _apply(db, rid, vocab, choice, stands, user): + """Act on a sure answer; what was done (None: nothing needed doing).""" + if vocab == "content_type": + ctypes.choose(db, rid, stands[choice]["key"]) + return "content_type" + if vocab == "topic": # what already holds, waits or was dismissed stays as it is + return "topic_suggested" if topicmod.propose(db, stands[choice]["id"], [rid], source="classify", user=user) else None + library.retag(db, [rid], add=[choice]) + return "tag" diff --git a/fastapi_backend/app/schemas/classify.py b/fastapi_backend/app/schemas/classify.py new file mode 100644 index 000000000..91d26a1ed --- /dev/null +++ b/fastapi_backend/app/schemas/classify.py @@ -0,0 +1,50 @@ +"""Classifying an asset or a text (domain/classify.py).""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field + +from app.schemas.common import RequestModel, ResponseModel + + +class ClassifyRequest(RequestModel): + """Give a `recording` or a `text`, and `labels` or a `vocabulary`.""" + + recording: int | None = Field(None, description="the asset to classify") + text: str | None = Field(None, max_length=20000, description="text to classify instead") + labels: dict[str, str] | list[str] | None = Field( + None, description="the labels to choose from (2 to 60): names, or {name: what it means}" + ) + vocabulary: Literal["content_type", "topic"] | None = Field( + None, description="choose from Lens's content types (of the asset's kind), or the namespace's topics" + ) + namespace: str | None = Field(None, description="whose topics, when classifying a text by topic") + question: str | None = Field(None, max_length=500, description="what to ask (default: which label fits best)") + apply: bool = Field( + False, + description="act on a sure answer: set the content type, suggest the topic, or add the label as a tag (editors)", + ) + + +class RankedLabel(ResponseModel): + label: str + p: float = Field(description="how likely, from 0 to 1") + key: str | None = Field(None, description="content_type: the content type's key") + id: int | None = Field(None, description="topic: the topic's id") + + +class ClassifyResult(ResponseModel): + choice: str = Field(description="the label that fits best") + confidence: float = Field(description="how sure, from 0 to 1") + sure: bool = Field(description="sure enough to act on without asking (decisions.act_above)") + ranked: list[RankedLabel] = Field(description="every label, the likeliest first") + by: str = Field(description="what answered: jev, laya, llm, or only option") + vocabulary: str | None = None + question: str + applied: Literal["content_type", "topic_suggested", "tag"] | None = Field( + None, description="what was done with a sure answer, when apply was asked" + ) + key: str | None = Field(None, description="content_type: the chosen content type's key") + id: int | None = Field(None, description="topic: the chosen topic's id") diff --git a/fastapi_backend/tests/api/test_classify.py b/fastapi_backend/tests/api/test_classify.py new file mode 100644 index 000000000..23fe170f1 --- /dev/null +++ b/fastapi_backend/tests/api/test_classify.py @@ -0,0 +1,175 @@ +"""Classifying a resource or a text (domain/classify.py): the decision model picks a label, the language model when +there's none, sure answers are acted on only when asked, and agents classify through MCP and the assistant.""" + +from __future__ import annotations + +import json + +import pytest + +from app.domain import activity, ai_tools, content_types, store, topics +from tests import fake_jev, fake_llm +from tests.api._assist import start_llm +from tests.api.test_mcp import rpc +from tests.api.test_oauth import _bearer, _grant +from tests.helpers import login, make_user, seed + +R = store.R + + +@pytest.fixture +def llm(cfg): + srv = start_llm(cfg) + yield fake_llm.Handler + srv.shutdown() + + +@pytest.fixture +def jev(cfg): + srv = fake_jev.start(cfg) + yield fake_jev.Handler + srv.shutdown() + + +@pytest.fixture +def app(cfg, db, llm, jev): + """Built after both models are set up: the app reads its settings once, until they change.""" + from app.main import create_app + + return create_app(cfg, db, background=False) + + +@pytest.fixture +def env(client, db, cfg, folder): + ep1, ep2, call = seed(db, cfg, folder) + pods = store.ns_id(db, "pods") + gene = topics.create(db, pods, "Gene therapy", definition="Treating disease by changing genes") + security = topics.create(db, pods, "Security", alt=["exploits"]) + make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor", "calls": "viewer"}) + make_user(db, "vi@x.io", "viewer password 1", roles={"pods": "viewer"}) + return { + "ep1": ep1, + "ep2": ep2, + "call": call, + "pods": pods, + "gene": gene, + "security": security, + "he": login(client, "ed@x.io", "editor password 1"), + "hv": login(client, "vi@x.io", "viewer password 1"), + } + + +def post(client, h, status=200, **body): + r = client.post("/api/v1/classify", json=body, headers=h) + assert r.status_code == status, r.text + return r.json() + + +def test_labels_given_and_what_the_model_reads(client, db, env, jev, llm): + jev.answer = {"type": "choice", "choice": "science", "confidence": 0.93, "probabilities": {"science": 0.93, "sport": 0.07}} + got = post(client, env["hv"], recording=env["ep1"], labels={"science": "research, biology, medicine", "sport": "games and teams"}) + assert (got["choice"], got["confidence"], got["sure"], got["by"]) == ("science", 0.93, True, "jev") + assert [x["label"] for x in got["ranked"]] == ["science", "sport"] and got["applied"] is None + _, body = jev.seen[-1] + q = body["questions"]["q"] + assert q["instructions"] == "Which label fits this resource best?" + assert q["criteria"] == {"science": "research, biology, medicine", "sport": "games and teams"} + state = json.loads(body["state"]) + assert state["kind"] == "transcript" and state["file"] == "ep1.txt" + assert "Dyno Therapeutics" in state["resource"] and state["resource"].startswith("( + options: Options, + ): RequestResult { + return (options.client ?? client).post({ + url: "/api/v1/classify", + ...options, + headers: { + "Content-Type": "application/json", + ...options.headers, + }, + }); + } } export class Searches { diff --git a/nextjs-frontend/app/openapi-client/types.gen.ts b/nextjs-frontend/app/openapi-client/types.gen.ts index 90338022b..296ca996d 100644 --- a/nextjs-frontend/app/openapi-client/types.gen.ts +++ b/nextjs-frontend/app/openapi-client/types.gen.ts @@ -3770,6 +3770,124 @@ export type ChatUpdate = { model?: string | null; }; +/** + * ClassifyRequest + * + * Give a `recording` or a `text`, and `labels` or a `vocabulary`. + */ +export type ClassifyRequest = { + /** + * Recording + * + * the asset to classify + */ + recording?: number | null; + /** + * Text + * + * text to classify instead + */ + text?: string | null; + /** + * Labels + * + * the labels to choose from (2 to 60): names, or {name: what it means} + */ + labels?: + | { + [key: string]: string; + } + | Array + | null; + /** + * Vocabulary + * + * choose from Lens's content types (of the asset's kind), or the namespace's topics + */ + vocabulary?: "content_type" | "topic" | null; + /** + * Namespace + * + * whose topics, when classifying a text by topic + */ + namespace?: string | null; + /** + * Question + * + * what to ask (default: which label fits best) + */ + question?: string | null; + /** + * Apply + * + * act on a sure answer: set the content type, suggest the topic, or add the label as a tag (editors) + */ + apply?: boolean; +}; + +/** + * ClassifyResult + */ +export type ClassifyResult = { + /** + * Choice + * + * the label that fits best + */ + choice: string; + /** + * Confidence + * + * how sure, from 0 to 1 + */ + confidence: number; + /** + * Sure + * + * sure enough to act on without asking (decisions.act_above) + */ + sure: boolean; + /** + * Ranked + * + * every label, the likeliest first + */ + ranked: Array; + /** + * By + * + * what answered: jev, laya, llm, or only option + */ + by: string; + /** + * Vocabulary + */ + vocabulary?: string | null; + /** + * Question + */ + question: string; + /** + * Applied + * + * what was done with a sure answer, when apply was asked + */ + applied?: "content_type" | "topic_suggested" | "tag" | null; + /** + * Key + * + * content_type: the chosen content type's key + */ + key?: string | null; + /** + * Id + * + * topic: the chosen topic's id + */ + id?: number | null; + [key: string]: unknown; +}; + /** * ClientRegistered */ @@ -16058,6 +16176,35 @@ export type Pushed = { [key: string]: unknown; }; +/** + * RankedLabel + */ +export type RankedLabel = { + /** + * Label + */ + label: string; + /** + * P + * + * how likely, from 0 to 1 + */ + p: number; + /** + * Key + * + * content_type: the content type's key + */ + key?: string | null; + /** + * Id + * + * topic: the topic's id + */ + id?: number | null; + [key: string]: unknown; +}; + /** * RcloneSignIn */ @@ -33471,6 +33618,31 @@ export type FindResourcesResponses = { export type FindResourcesResponse = FindResourcesResponses[keyof FindResourcesResponses]; +export type ClassifyData = { + body: ClassifyRequest; + path?: never; + query?: never; + url: "/api/v1/classify"; +}; + +export type ClassifyErrors = { + /** + * Validation Error + */ + 422: HttpValidationError; +}; + +export type ClassifyError = ClassifyErrors[keyof ClassifyErrors]; + +export type ClassifyResponses = { + /** + * Successful Response + */ + 200: ClassifyResult; +}; + +export type ClassifyResponse = ClassifyResponses[keyof ClassifyResponses]; + export type ListSearchesData = { body?: never; path?: never; diff --git a/nextjs-frontend/openapi.json b/nextjs-frontend/openapi.json index 2e81e58d3..11b819220 100644 --- a/nextjs-frontend/openapi.json +++ b/nextjs-frontend/openapi.json @@ -15457,6 +15457,48 @@ } } }, + "/api/v1/classify": { + "post": { + "tags": [ + "search" + ], + "summary": "Classify", + "description": "Which label fits an asset (or a text) best, how sure that is, and every label ranked: the labels given, Lens's\ncontent types (of the asset's kind) or the namespace's topics. The decision model answers when it's set up\n(Settings \u2192 AI assistant), else the language model; the call is in the activity ledger. With `apply` (editors), a\nsure answer is acted on: the content type is set, the topic is suggested for someone to accept, or the label is\nadded as a tag; one that isn't sure changes nothing. 409 when no model is set up to answer.", + "operationId": "classify", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClassifyRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClassifyResult" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/v1/searches": { "get": { "tags": [ @@ -37083,6 +37125,206 @@ "type": "object", "title": "ChatUpdate" }, + "ClassifyRequest": { + "properties": { + "recording": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Recording", + "description": "the asset to classify" + }, + "text": { + "anyOf": [ + { + "type": "string", + "maxLength": 20000 + }, + { + "type": "null" + } + ], + "title": "Text", + "description": "text to classify instead" + }, + "labels": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Labels", + "description": "the labels to choose from (2 to 60): names, or {name: what it means}" + }, + "vocabulary": { + "anyOf": [ + { + "type": "string", + "enum": [ + "content_type", + "topic" + ] + }, + { + "type": "null" + } + ], + "title": "Vocabulary", + "description": "choose from Lens's content types (of the asset's kind), or the namespace's topics" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace", + "description": "whose topics, when classifying a text by topic" + }, + "question": { + "anyOf": [ + { + "type": "string", + "maxLength": 500 + }, + { + "type": "null" + } + ], + "title": "Question", + "description": "what to ask (default: which label fits best)" + }, + "apply": { + "type": "boolean", + "title": "Apply", + "description": "act on a sure answer: set the content type, suggest the topic, or add the label as a tag (editors)", + "default": false + } + }, + "additionalProperties": false, + "type": "object", + "title": "ClassifyRequest", + "description": "Give a `recording` or a `text`, and `labels` or a `vocabulary`." + }, + "ClassifyResult": { + "properties": { + "choice": { + "type": "string", + "title": "Choice", + "description": "the label that fits best" + }, + "confidence": { + "type": "number", + "title": "Confidence", + "description": "how sure, from 0 to 1" + }, + "sure": { + "type": "boolean", + "title": "Sure", + "description": "sure enough to act on without asking (decisions.act_above)" + }, + "ranked": { + "items": { + "$ref": "#/components/schemas/RankedLabel" + }, + "type": "array", + "title": "Ranked", + "description": "every label, the likeliest first" + }, + "by": { + "type": "string", + "title": "By", + "description": "what answered: jev, laya, llm, or only option" + }, + "vocabulary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vocabulary" + }, + "question": { + "type": "string", + "title": "Question" + }, + "applied": { + "anyOf": [ + { + "type": "string", + "enum": [ + "content_type", + "topic_suggested", + "tag" + ] + }, + { + "type": "null" + } + ], + "title": "Applied", + "description": "what was done with a sure answer, when apply was asked" + }, + "key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key", + "description": "content_type: the chosen content type's key" + }, + "id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Id", + "description": "topic: the chosen topic's id" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "choice", + "confidence", + "sure", + "ranked", + "by", + "question" + ], + "title": "ClassifyResult" + }, "ClientRegistered": { "properties": { "client_id": { @@ -58300,6 +58542,50 @@ ], "title": "Pushed" }, + "RankedLabel": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "p": { + "type": "number", + "title": "P", + "description": "how likely, from 0 to 1" + }, + "key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key", + "description": "content_type: the content type's key" + }, + "id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Id", + "description": "topic: the topic's id" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "label", + "p" + ], + "title": "RankedLabel" + }, "RcloneSignIn": { "properties": { "id": {